myprecious 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be0d5a3297a77bea22a403968fa7c3a23f2fd7c2d4ce96b957c832788f29de91
-  data.tar.gz: d82b540dbb7be2ffb7601fe571cc07482d2fbe87563b8d78d937bc791bd22d3f
+  metadata.gz: d1d5adc30e0ccc2fb1c7a183402c7ac0015a9fbdde70e18267ad61d21bf4d790
+  data.tar.gz: '07394d3a1ff9a238220b9ae4d2be7328a9878ee08c47ec7179e9564af53e9e60'
 SHA512:
-  metadata.gz: 79e7317fa010abdc6ab9d7207d7be1229d85e2ba2a53cb2225e1a8bed7c95063d51294fab4917f1449e106210e1aa3ef5c52fa743bde9e6fb85f582a6942694a
-  data.tar.gz: 0fb7d8ec2c159fe58867b2f5cbbfa95a9a27c7f0e6f020ea2f69fb2ecf55b49b2ed385857c3a6162670c07a39871b54724f2005c71dd6897e691438701d33316
+  metadata.gz: 6f152193febe7482d2e9aeb1f3435b56c84f127a897cd0e3c3f8e35c5027a93d955bcb04b09c1e16f6d3a691bb92e85ff15047e31ba0b956daa5664ff6430287
+  data.tar.gz: ba2ea748b6a1c5f566f28375776639cb774449c885b11db306d02f709b7d6c86d7727acad771fd9eedaab610fa5be93914aa30b30607ada1d66691a9362c1f20

data/bin/myprecious

@@ -4,5 +4,5 @@ require 'rubygems'
 require 'myprecious'
 
 MyPrecious::Program.run(
-  on_error: (:exit_program! if ENV['TRACE_ERRORS'].to_s.empty?)
+  on_error: (:exit_program! unless MyPrecious.tracing_errors?)
 )

data/lib/myprecious.rb

@@ -13,6 +13,7 @@ module MyPrecious
   ONE_DAY = 60 * 60 * 24
 end
 require 'myprecious/data_caches'
+require 'myprecious/cves'
 
 module MyPrecious
   extend Rake::DSL
@@ -20,6 +21,10 @@ module MyPrecious
   Program = Rake::ToolkitProgram
   Program.title = "myprecious Dependecy Reporting Tool"
   
+  def self.tracing_errors?
+    !ENV['TRACE_ERRORS'].to_s.empty?
+  end
+  
   def self.common_program_args(parser, args)
     parser.on(
       '-o', '--out FILE',
@@ -34,11 +39,12 @@ module MyPrecious
       fpath = Pathname(fpath)
       parser.invalid_args!("#{fpath} does not exist.") unless fpath.exist?
       args.target = fpath
+      CVEs.config_dir = fpath
     end
     
     parser.on(
       '--[no-]cache',
-      "Control caching of gem information"
+      "Control caching of dependency information"
     ) {|v| MyPrecious.caching_disabled = v}
   end
   
@@ -273,13 +279,14 @@ module MyPrecious
     def common_col_title(attr)
       case attr
       when :current_version then 'Our Version'
-      when :age then 'Age (in days)'
+      when :age then 'Age (days)'
       when :latest_version then 'Latest Version'
       when :latest_released then 'Date Available'
       when :recommended_version then 'Recommended Version'
       when :license then 'License Type'
       when :changelog then 'Change Log'
       when :obsolescence then 'How Bad'
+      when :cves then 'CVEs'
       else
         warn("'#{attr}' column does not have a mapped name")
         attr
@@ -331,7 +338,7 @@ module MyPrecious
   # to some extent like frozen Array instances.
   #
   class ColumnOrder
-    DEFAULT = %i[name current_version age latest_version latest_released recommended_version license changelog].freeze
+    DEFAULT = %i[name current_version age latest_version latest_released recommended_version cves license changelog].freeze
     COLUMN_FROM_TEXT_NAME = {
       'gem' => :name,
       'package' => :name,
@@ -340,10 +347,11 @@ module MyPrecious
       'how bad' => :obsolescence,
       'latest version' => :latest_version,
       'date available' => :latest_released,
-      'age (in days)' => :age,
+      'age (days)' => :age,
       'license type' => :license,
       /change ?log/ => :changelog,
       'recommended version' => :recommended_version,
+      'cves' => :cves,
     }
     
     def initialize
@@ -422,80 +430,151 @@ module MyPrecious
   # and returns enhanced Markdown for selected columns (e.g. +name+).
   #
   class MarkdownAdapter
+    QS_VALUE_UNSAFE = /#{URI::UNSAFE}|[&=]/
+    
     def initialize(dep)
       super()
       @dependency = dep
     end
     attr_reader :dependency
     
+    def self.embellish(attr_name, &blk)
+      define_method(attr_name) do
+        value = begin
+          dependency.send(attr_name)
+        rescue NoMethodError
+          raise
+        rescue StandardError
+          return "(error)"
+        end
+        
+        begin
+          instance_exec(value, &blk)
+        rescue StandardError => ex
+          err_key = [ex.backtrace[0], ex.to_s]
+          unless (@errors ||= Set.new).include?(err_key)
+            @errors << err_key
+            if MyPrecious.tracing_errors?
+              $stderr.puts("Traceback (most recent call last):")
+              ex.backtrace[1..-1].reverse.each_with_index do |loc, i|
+                $stderr.puts("#{(i + 1).to_s.rjust(8)}: #{loc}")
+              end
+              $stderr.puts("#{ex.backtrace[0]}: #{ex} (#{ex.class} while computing #{attr_name})")
+            else
+              $stderr.puts("#{ex} (while computing #{attr_name})")
+            end
+          end
+          value
+        end
+      end
+    end
+    
     ##
     # Generate Markdown linking the +name+ to the homepage for the dependency
     #
-    def name
+    embellish(:name) do |base_val|
       cswatch = begin
         color_swatch + ' '
       rescue StandardError
         ''
       end
-      "#{cswatch}[#{dependency.name}](#{dependency.homepage_uri})"
-    rescue StandardError
-      dependency.name
+      "#{cswatch}[#{base_val}](#{dependency.homepage_uri})"
+    rescue Gems::NotFound
+      base_val
+    end
+    
+    ##
+    # Decorate the latest version as a link to the release history
+    #
+    embellish(:latest_version) do |base_val|
+      release_history_url = begin
+        dependency.release_history_url
+      rescue StandardError
+        nil
+      end
+      
+      if release_history_url
+        "[#{base_val}](#{release_history_url})"
+      else
+        base_val
+      end
     end
     
     ##
     # Include information about temporal difference between current and
     # recommended versions
     #
-    def recommended_version
-      recommended_version = dependency.recommended_version
-      if dependency.current_version < recommended_version
-        span_comment = begin
-          if days_newer = dependency.days_between_current_and_recommended
-            " -- #{days_newer} days newer"
-          else
-            ""
-          end
+    embellish(:recommended_version) do |recced_ver|
+      if recced_ver.kind_of?(String)
+        recced_ver = Gem::Version.new(recced_ver)
+      end
+      next recced_ver if dependency.current_version.nil? || dependency.current_version >= recced_ver
+      
+      span_comment = begin
+        if days_newer = dependency.days_between_current_and_recommended
+          " -- #{days_newer} days newer"
+        else
+          ""
         end
-        "**#{recommended_version}**#{span_comment}"
-      else
-        recommended_version
       end
-    rescue StandardError
-      recommended_version || "(error)"
+      
+      cve_url = URI('https://nvd.nist.gov/products/cpe/search/results')
+      cve_url.query = URI.encode_www_form(
+        keyword: "cpe:2.3:a:*:#{dependency.name.downcase}:#{recced_ver}",
+      )
+      
+      "**#{recced_ver}**#{span_comment} ([current CVEs](#{cve_url}))"
     end
     
     ##
     # Include update info in the license column
     #
-    def license
-      value = dependency.license
-      if value.update_info
-        "#{value}<br/>(#{value.update_info})"
+    embellish(:license) do |base_val|
+      begin
+        update_info = base_val.update_info
+      rescue NoMethodError
+        base_val
       else
-        value
+        update_info.to_s.empty? ? base_val : "#{base_val}<br/>(#{update_info})"
       end
-    rescue StandardError
-      "(error)"
     end
     
     ##
     # Render short links for http: or https: changelog URLs
     #
-    def changelog
-      base_val = begin
-        dependency.changelog
-      rescue StandardError
-        return "(error)"
-      end
-      
+    embellish(:changelog) do |base_val|
       begin
         uri = URI.parse(base_val)
         if ['http', 'https'].include?(uri.scheme)
-          return "[on #{uri.hostname}](#{base_val})"
+          next "[on #{uri.hostname}](#{base_val})"
         end
       rescue StandardError
       end
-      return base_val
+      
+      base_val
+    end
+    
+    ##
+    # Render links to NIST's NVD
+    #
+    NVD_CVE_URL_TEMPLATE = "https://nvd.nist.gov/vuln/detail/%s"
+    embellish(:cves) do |base_val|
+      base_val.map do |cve|
+        link_text_parts = [cve]
+        addnl_info_parts = []
+        begin
+          addnl_info_parts << cve.vendors.to_a.join(',') unless cve.vendors.empty?
+        rescue StandardError
+        end
+        begin
+          addnl_info_parts << cve.score.to_s if cve.score
+        rescue StandardError
+        end
+        unless addnl_info_parts.empty?
+          link_text_parts << "(#{addnl_info_parts.join(' - ')})"
+        end
+        "[#{link_text_parts.join(' ')}](#{NVD_CVE_URL_TEMPLATE % cve})"
+      end.join(', ')
     end
     
     def obsolescence
@@ -508,10 +587,16 @@ module MyPrecious
     # Get a CSS-style hex color code corresponding to the obsolescence of the dependency
     #
     def color
+      red = "fb0e0e"
+      
+      if (dependency.cves.map(&:score).compact.max || 0) >= 7
+        return red
+      end
+      
       case dependency.obsolescence
       when :mild then "dde418"
       when :moderate then "f9b733"
-      when :severe then "fb0e0e"
+      when :severe then red
       else "4dda1b"
       end
     end

data/lib/myprecious/cves.rb

@@ -0,0 +1,239 @@
+require 'date'
+require 'digest'
+require 'json'
+require 'myprecious/data_caches'
+require 'open3'
+require 'pathname'
+require 'rest-client'
+require 'set'
+
+module MyPrecious
+  module CVEs
+    extend DataCaching
+    
+    MIN_GAP_SECONDS = 5
+    CONFIG_FILE = '.myprecious-cves.rb'
+    
+    CVE_DATA_CACHE_DIR = MyPrecious.data_cache(DATA_DIR / "cve-data")
+    
+    class <<self
+      attr_reader :config_dir
+      
+      def config_dir=(val)
+        @config_dir = Pathname(val)
+      end
+    end
+    
+    def self.last_query_time
+      @last_query_time ||= DateTime.now - 1
+    end
+    
+    def self.queried!
+      @last_query_time = DateTime.now
+    end
+    
+    ##
+    # If you don't specify version, you get to match against the applicable
+    # configurations on your own to determine which CVEs returned apply to
+    # the versions of the named package in which you are interested
+    #
+    def self.get_for(package_name, version='*')
+      nvd_url = URI("https://services.nvd.nist.gov/rest/json/cves/1.0")
+      nvd_url.query = URI.encode_www_form(
+        cpeMatchString: "cpe:2.3:a:*:#{package_name.downcase}:#{version}:*:*:*:*:*:*:*",
+      )
+      
+      cache = CVE_DATA_CACHE_DIR / "#{Digest::SHA256.hexdigest(nvd_url.to_s)}.json"
+      cve_data = apply_cache(cache) do
+        # Use last_query_time to sleep if necessary
+        wait_time = MIN_GAP_SECONDS - (DateTime.now - last_query_time) * 24 * 3600
+        if wait_time > 0
+          sleep(wait_time)
+        end
+        
+        response = RestClient.get(nvd_url.to_s)
+        queried!
+        
+        JSON.parse(response.body)
+      end
+      
+      begin
+        return cve_data['result']['CVE_Items'].map do |e|
+          applicability = objectify_configurations(package_name, e['configurations'])
+          score = (((e['impact'] || {})['baseMetricV3'] || {})['cvssV3'] || {})['baseScore']
+          cve = CVERecord.new(
+            e['cve']['CVE_data_meta']['ID'],
+            applicability.respond_to?(:vendors) ? applicability.vendors : nil,
+            score
+          )
+          
+          [cve, applicability]
+        end.reject {|cve, a| a.respond_to?(:applies_to?) && !a.applies_to?(version)}
+      rescue StandardError => e
+        $stderr.puts "[WARN] #{e}\n\n#{JSON.dump(cve_data)}\n\n"
+        []
+      end
+    end
+    
+    def self.config
+      if !@config && config_dir
+        if (config_path = config_dir / CONFIG_FILE).exist?
+          @config = begin
+            config_prog_output, status = Open3.capture2(RbConfig.ruby, config_path.to_s)
+            if status.success?
+              JSON.parse(config_prog_output)
+            else
+              $stderr.puts "#{config_path} did not exit cleanly (code ${status.exitstatus})"
+              {}
+            end
+          rescue StandardError
+          end
+          
+          unless @config.kind_of?(Hash)
+            $stderr.puts "#{config_path} did not output a JSON configuration"
+            @config = {}
+          end
+        else
+          @config = {}
+        end
+      end
+      @config ||= {}
+    end
+    
+    def self.objectify_configurations(package_name, configs)
+      if configs.kind_of?(Hash) && configs['CVE_data_version'] == "4.0"
+        Applicability_V4_0.new(package_name, configs)
+      else
+        configs
+      end
+    end
+    
+    class CVERecord < String
+      def initialize(id, vendors, score)
+        super(id)
+        
+        @vendors = vendors
+        @score = score
+      end
+      
+      attr_accessor :vendors, :score
+    end
+    
+    class Applicability_V4_0 < Hash
+      def initialize(package, configs)
+        super()
+        self.update(configs)
+        @package = package.downcase
+      end
+      attr_reader :package
+      
+      def nodes
+        self['nodes']
+      end
+      
+      def applies_to?(version)
+        package_nodes(nodes).any? do |node|
+          version_matches_node?(version, node)
+        end
+      end
+      
+      def vendors
+        Set.new(each_vulnerable_cpe.map do |cpe|
+          cpe.split(':')[3]
+        end)
+      end
+      
+      def package_nodes(node_list)
+        node_list.select do |node|
+          node['children'] || node['cpe_match'].any? do |pattern|
+            pattern['cpe23Uri'] =~ package_cpe_regexp
+          end
+        end
+      end
+      
+      def each_vulnerable_cpe
+        return enum_for(:each_vulnerable_cpe) unless block_given?
+        
+        remaining = nodes.to_a.dup
+        while (node = remaining.shift)
+          if node['children']
+            remaining.insert(0, *node['children'])
+          else
+            node['cpe_match'].each do |pattern|
+              next unless pattern['vulnerable']
+              cpe = pattern['cpe23Uri']
+              if package_cpe_regexp =~ cpe
+                yield cpe
+              end
+            end
+          end
+        end
+      end
+      
+      def package_cpe_regexp
+        /^cpe:2.3:a:[^:]*:#{package}(:|$)/
+      end
+      
+      def version_matches_node?(version, node)
+        test = (node['operator'] == 'AND') ? :all? : :any?
+        if node['children']
+          return node['children'].send(test) {|child| version_matches_node?(version, child)}
+        end
+        
+        return node['cpe_match'].any? do |pattern|
+          cpe_entry_indicates_vulnerable_version?(version, pattern)
+        end
+      end
+      
+      def cpe_entry_indicates_vulnerable_version?(version, pattern)
+        return false unless pattern['vulnerable']
+        
+        cpe_vendor, cpe_product, cpe_version, cpe_update = pattern['cpe23Uri'].split(':')[3,4]
+        return false if (CVEs.config['blockedProducts'] ||= []).include?([cpe_vendor, cpe_product].join(':'))
+        return false if cpe_product != @package
+        if version == '*'
+          return true
+        end
+        return false unless [nil, '*', '-'].include?(cpe_update) # We'll ignore prerelease versions
+        if cpe_version != '*' && cpe_version == version
+          return true
+        end
+        
+        if (range_start = pattern['versionStartIncluding'])
+          range_test = :<=
+        elsif (range_start = pattern['versionStartExcluding'])
+          range_test = :<
+        else
+          range_test = nil
+        end
+        if range_test && !version_compare(range_start, version).send(range_test, 0)
+          return false
+        end
+        
+        if (range_end = pattern['versionEndIncluding'])
+          range_test = :<=
+        elsif (range_end = pattern['versionEndExcluding'])
+          range_test = :<
+        else
+          range_test = nil
+        end
+        if range_test && !version_compare(version, range_end).send(range_test, 0)
+          return false
+        end
+        
+        return range_start || range_end
+      end
+      
+      ##
+      # Return a <=> b for version strings a and b
+      #
+      def version_compare(a, b)
+        make_comparable(a) <=> make_comparable(b)
+      end
+      
+      def make_comparable(ver_str)
+        ver_str.split('.').map {|p| p.to_i}
+      end
+    end
+  end
+end

data/lib/myprecious/python_packages.rb

@@ -1,5 +1,7 @@
+require 'date'
 require 'json'
 require 'myprecious'
+require 'myprecious/cves'
 require 'myprecious/data_caches'
 require 'open-uri'
 require 'open3'
@@ -212,7 +214,7 @@ module MyPrecious
     end
     
     def latest_released
-      versions_with_release[0][1]
+      Date.parse(versions_with_release[0][1].to_s).to_s      
     end
     
     ##
@@ -260,12 +262,25 @@ module MyPrecious
       LicenseDescription.new(get_package_info['info']['license'])
     end
     
+    def cves
+      resolve_name!
+      resolve_version!
+      
+      CVEs.get_for(name, current_version.to_s).map do |cve, applicability|
+        cve
+      end
+    end
+    
     def changelog
       # This is wrong
       info = get_package_info['info']
       return info['project_url']
     end
     
+    def release_history_url
+      "https://pypi.org/project/#{name}/#history"
+    end
+    
     def days_between_current_and_recommended
       v, cv_rel = versions_with_release.find do |v, r|
         case 

data/lib/myprecious/ruby_gems.rb

@@ -1,4 +1,6 @@
+require 'date'
 require 'gems'
+require 'myprecious/cves'
 require 'myprecious/data_caches'
 require 'pathname'
 
@@ -33,11 +35,11 @@ module MyPrecious
     # used by the project.  Each gem name will appear in only one +:current+
     # iteration, but may occur in multiple +:reqs+ iterations.
     # 
-    def self.each_gem_used(fpath)
+    def self.each_gem_used(fpath, gemfile: 'Gemfile')
       return enum_for(:each_gem_used, fpath) unless block_given?
       
-      gemlock = Pathname(fpath).join('Gemfile.lock')
-      raise "No Gemfile.lock in #{fpath}" unless gemlock.exist?
+      gemlock = Pathname(fpath).join(gemfile + '.lock')
+      raise "No #{gemfile}.lock in #{fpath}" unless gemlock.exist?
       
       section = nil
       gemlock.each_line do |l|
@@ -64,9 +66,9 @@ module MyPrecious
     # #current_version values and meaningful information in #version_reqs,
     # as indicated in the Gemfile.lock for +fpath+.
     #
-    def self.accum_gem_lock_info(fpath)
+    def self.accum_gem_lock_info(fpath, **opts)
       {}.tap do |gems|
-        each_gem_used(fpath) do |entry_type, name, verreq|
+        each_gem_used(fpath, **opts) do |entry_type, name, verreq|
           g = (gems[name] ||= RubyGemInfo.new(name))
           
           case entry_type
@@ -156,7 +158,7 @@ module MyPrecious
     
     def latest_released
       return nil if versions_with_release.empty?
-      versions_with_release[0][1]
+      Date.parse(versions_with_release[0][1].to_s).to_s
     end
     
     ##
@@ -202,6 +204,12 @@ module MyPrecious
       end
     end
     
+    def cves
+      CVEs.get_for(name, current_version && current_version.to_s).map do |cve, appl|
+        cve
+      end
+    end
+    
     def changelogs
       gv_data = get_gems_versions.sort_by {|v| Gem::Version.new(v['number'])}.reverse
       if current_version
@@ -214,6 +222,16 @@ module MyPrecious
       changelogs[0]
     end
     
+    def release_history_url
+      "https://rubygems.org/gems/#{name}/versions" if (
+        begin
+          get_gems_versions
+        rescue StandardError
+          nil
+        end
+      )
+    end
+    
     def days_between_current_and_recommended
       v, cv_rel = versions_with_release.find {|v, r| v == current_version} || []
       v, rv_rel = versions_with_release.find {|v, r| v == recommended_version} || []

metadata

@@ -1,55 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: myprecious
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Balki Kodarapu
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-17 00:00:00.000000000 Z
+date: 2020-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gems
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: git
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.5'
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.5'
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: rake-toolkit_program
   requirement: !ruby/object:Gem::Requirement
@@ -68,22 +68,22 @@ dependencies:
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.0.2
     - - ">="
       - !ruby/object:Gem::Version
         version: '2.0'
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 2.0.2
     - - ">="
       - !ruby/object:Gem::Version
         version: '2.0'
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.2
 - !ruby/object:Gem::Dependency
   name: parslet
   requirement: !ruby/object:Gem::Requirement
@@ -178,6 +178,7 @@ extra_rdoc_files: []
 files:
 - bin/myprecious
 - lib/myprecious.rb
+- lib/myprecious/cves.rb
 - lib/myprecious/data_caches.rb
 - lib/myprecious/python_packages.rb
 - lib/myprecious/ruby_gems.rb
@@ -200,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Your precious dependencies!