spandx 0.10.1 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 694c16875566407ff41f6d791d7499de4a4686231e8f24b3c0393d14039d764c
-  data.tar.gz: 49873d42c52d18fe55e95a32f98b1f5b115a46aa12ebd8df0247a78173953c60
+  metadata.gz: da243ee84b54d710a15eeb5182f7b70084cbc69cb86beaa93c25c13dc650fe2a
+  data.tar.gz: 904eff3844f540311828386dde54ad70054f5c4fe90e456bb0af702efc9f7cc3
 SHA512:
-  metadata.gz: a0bce586e02b70dd48a48819970c6b51198048a8ee7b6310f5959e56b13772ada9925524f1718add10e392e0a52b64907a39eeeffedb6a8cd22486e15daf84c4
-  data.tar.gz: ddb03108be865151c7c39e78a57f7988c49333bd086ca333f4b54ecd13a55b6d8f82fe0ac0a64a869ea96c0b28bca910c3558bd783dc193ffdc650dd0649b0cb
+  metadata.gz: 67135d0cff701f454ce5d127a7f4ec7fd061d63bc832d5eb7df20bb26efa6e45bc90e28bc22eceaa2c135437858aa63337db8ff898245d870fe7acfcae14413d
+  data.tar.gz: a4086a92383a8c334c3488004b6be02a0ce80a79b9c7386eea5948d3d52d031d46e742d9e290afa45e306c4d3d1e4bbb7c2849cdd1d2d3dc98b5aba74fd13ed8

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-Version 0.10.1
+Version 0.11.0
 
 # Changelog
 
@@ -8,6 +8,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+
+## [0.11.0] - 2020-03-20
+### Added
+- Add `--format` option to scan command.
+- Read from offline `nuget` cache.
+
 ## [0.10.1] - 2020-03-16
 ### Fixed
 - Update location of `rubygems` index data

data/lib/spandx/cli/commands/index/build.rb

@@ -21,7 +21,8 @@ module Spandx
 
           def indexes
             [
-              Spandx::Dotnet::Index.new(directory: @options[:directory])
+              Spandx::Dotnet::Index.new(directory: @options[:directory]),
+              Spandx::Java::Index.new(directory: @options[:directory]),
             ]
           end
         end

data/lib/spandx/cli/commands/scan.rb

@@ -18,7 +18,7 @@ module Spandx
               report.add(dependency)
             end
           end
-          output.puts report.to_json
+          output.puts report.to(@options[:format])
         end
 
         private

data/lib/spandx/cli.rb

@@ -23,6 +23,7 @@ module Spandx
     method_option :recursive, aliases: '-r', type: :boolean, desc: 'Perform recursive scan', default: false
     method_option :airgap, aliases: '-a', type: :boolean, desc: 'Disable network connections', default: false
     method_option :logfile, aliases: '-l', type: :string, desc: 'Path to a logfile', default: '/dev/null'
+    method_option :format, aliases: '-f', type: :string, desc: 'Format of report', default: 'json'
     def scan(lockfile)
       Spandx.airgap = options[:airgap]
       Spandx.logger = Logger.new(options[:logfile])

data/lib/spandx/{rubygems/offline_index.rb → core/cache.rb}

@@ -1,16 +1,20 @@
 # frozen_string_literal: true
 
 module Spandx
-  module Rubygems
-    class OfflineIndex
-      attr_reader :db
+  module Core
+    class Cache
+      attr_reader :db, :package_manager
 
-      def initialize(package_manager)
-        @db = ::Spandx::Core::Database.new(url: "https://github.com/mokhan/spandx-#{package_manager}.git")
+      def initialize(package_manager, url:)
+        @package_manager = package_manager
+        @db = ::Spandx::Core::Database.new(url: url)
+        @cache = {}
+        @lines = {}
       end
 
       def licenses_for(name:, version:)
         found = search(name: name, version: version)
+        Spandx.logger.debug("Cache miss: #{name}-#{version}") if found.nil?
         found ? found[2].split('-|-') : []
       end
 
@@ -25,8 +29,9 @@ module Spandx
       end
 
       def search(name:, version:)
-        db.open(datafile_for(name)) do |io|
-          search_for("#{name}-#{version}", io, lines_in(io))
+        datafile = datafile_for(name)
+        db.open(datafile) do |io|
+          search_for("#{name}-#{version}", io, @lines.fetch(datafile) { |key| @lines[key] = lines_in(io) })
         end
       rescue Errno::ENOENT => error
         Spandx.logger.error(error)
@@ -34,7 +39,7 @@ module Spandx
       end
 
       def datafile_for(name)
-        ".index/#{key_for(name)}/rubygems"
+        ".index/#{key_for(name)}/#{package_manager}"
       end
 
       def lines_in(io)
@@ -49,10 +54,14 @@ module Spandx
 
       def search_for(term, io, lines)
         return if lines.empty?
+        return @cache[term] if @cache.key?(term)
 
         mid = lines.size == 1 ? 0 : lines.size / 2
         io.seek(lines[mid])
-        comparison = matches?(term, parse_row(io)) { |row| return row }
+        comparison = matches?(term, parse_row(io)) do |row|
+          return row
+        end
+
         search_for(term, io, partition(comparison, mid, lines))
       end
 
@@ -68,7 +77,9 @@ module Spandx
       end
 
       def parse_row(io)
-        CSV.parse(io.readline)[0]
+        row = CSV.parse(io.readline)[0]
+        @cache["#{row[0]}-#{row[1]}"] = row
+        row
       end
     end
   end

data/lib/spandx/{gateways → core}/http.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Spandx
-  module Gateways
+  module Core
     class Http
       attr_reader :driver
 

data/lib/spandx/core/report.rb

@@ -3,16 +3,29 @@
 module Spandx
   module Core
     class Report
-      def initialize(report: { version: '1.0', packages: [] })
-        @report = report
+      FORMATS = {
+        json: :to_json,
+        hash: :to_h,
+      }.freeze
+
+      def initialize
+        @dependencies = []
       end
 
       def add(dependency)
-        @report[:packages].push(dependency.to_h)
+        @dependencies.push(dependency)
+      end
+
+      def to(format)
+        public_send(FORMATS.fetch(format&.to_sym, :to_json))
       end
 
       def to_h
-        @report
+        { version: '1.0', dependencies: [] }.tap do |report|
+          @dependencies.each do |dependency|
+            report[:dependencies].push(dependency.to_h)
+          end
+        end
       end
 
       def to_json(*_args)

data/lib/spandx/dotnet/nuget_gateway.rb

@@ -15,6 +15,9 @@ module Spandx
       end
 
       def licenses_for(name, version)
+        found = cache.licenses_for(name: name, version: version)
+        return found if found.any?
+
         document = nuspec_for(name, version)
 
         extract_licenses_from(document) ||
@@ -33,6 +36,10 @@ module Spandx
 
       attr_reader :http, :guess
 
+      def cache
+        @cache ||= ::Spandx::Core::Cache.new(:nuget, url: 'https://github.com/mokhan/spandx-index.git')
+      end
+
       def each_page(start_page:)
         url = "https://#{host}/v3/catalog0/index.json"
         items_from(fetch_json(url))

data/lib/spandx/java/index.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Java
+    class Index
+      def initialize(directory:)
+        @directory = directory
+      end
+
+      def update!(catalogue:, output:); end
+    end
+  end
+end

data/lib/spandx/python/parsers/pipfile_lock.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Python
+    module Parsers
+      class PipfileLock < ::Spandx::Core::Parser
+        def self.matches?(filename)
+          filename.match?(/Pipfile.*\.lock/)
+        end
+
+        def parse(lockfile)
+          results = []
+          dependencies_from(lockfile) do |x|
+            results << ::Spandx::Core::Dependency.new(
+              name: x[:name],
+              version: x[:version],
+              licenses: x[:licenses]
+            )
+          end
+          results
+        end
+
+        private
+
+        def dependencies_from(lockfile)
+          json = JSON.parse(IO.read(lockfile))
+          each_dependency(pypi_for(json), json) do |name, version, definition|
+            yield({ name: name, version: version, licenses: [catalogue[definition['license']]] })
+          end
+        end
+
+        def each_dependency(pypi, json, groups: %w[default develop])
+          groups.each do |group|
+            json[group].each do |name, value|
+              version = canonicalize(value['version'])
+              yield name, version, pypi.definition_for(name, version)
+            end
+          end
+        end
+
+        def canonicalize(version)
+          version.gsub(/==/, '')
+        end
+
+        def pypi_for(json)
+          PyPI.new(sources: Source.sources_from(json))
+        end
+      end
+    end
+  end
+end

data/lib/spandx/python/pypi.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Python
+    class PyPI
+      def initialize(sources: [Source.default])
+        @sources = sources
+      end
+
+      def definition_for(name, version)
+        @sources.each do |source|
+          response = source.lookup(name, version)
+          return JSON.parse(response.body).fetch('info', {}) if response
+        end
+        {}
+      end
+    end
+  end
+end

data/lib/spandx/python/source.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Python
+    class Source
+      attr_reader :name, :uri, :verify_ssl
+
+      def initialize(source)
+        @name = source['name']
+        @uri = URI.parse(source['url'])
+        @verify_ssl = source['verify_ssl']
+      end
+
+      def host
+        @uri.host
+      end
+
+      def uri_for(name, version)
+        "https://#{host}/pypi/#{name}/#{version}/json"
+      end
+
+      def lookup(name, version, http: Spandx.http)
+        response = http.get(uri_for(name, version))
+        response if http.ok?(response)
+      end
+
+      class << self
+        def sources_from(json)
+          meta = json['_meta']
+          meta['sources'].map do |hash|
+            new(hash)
+          end
+        end
+
+        def default
+          new(
+            'name' => 'pypi',
+            'url' => 'https://pypi.org/simple',
+            'verify_ssl' => true
+          )
+        end
+      end
+    end
+  end
+end

data/lib/spandx/rubygems/gateway.rb

@@ -9,7 +9,7 @@ module Spandx
       end
 
       def licenses_for(name, version)
-        found = index.licenses_for(name: name, version: version)
+        found = cache.licenses_for(name: name, version: version)
         found.any? ? found : details_on(name, version)['licenses'] || []
       end
 
@@ -17,12 +17,11 @@ module Spandx
 
       attr_reader :http
 
-      def index
-        @index ||= OfflineIndex.new(:rubygems)
+      def cache
+        @cache ||= ::Spandx::Core::Cache.new(:rubygems, url: 'https://github.com/mokhan/spandx-rubygems.git')
       end
 
       def details_on(name, version)
-        Spandx.logger.debug("Cache miss: #{name}-#{version}")
         url = "https://rubygems.org/api/v2/rubygems/#{name}/versions/#{version}.json"
         response = http.get(url, default: {})
         http.ok?(response) ? parse(response.body) : {}

data/lib/spandx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spandx
-  VERSION = '0.10.1'
+  VERSION = '0.11.0'
 end

data/lib/spandx.rb

@@ -10,10 +10,12 @@ require 'net/hippie'
 require 'nokogiri'
 require 'pathname'
 
+require 'spandx/core/cache'
 require 'spandx/core/content'
 require 'spandx/core/database'
 require 'spandx/core/dependency'
 require 'spandx/core/guess'
+require 'spandx/core/http'
 require 'spandx/core/parser'
 require 'spandx/core/report'
 require 'spandx/core/score'
@@ -24,13 +26,13 @@ require 'spandx/dotnet/parsers/csproj'
 require 'spandx/dotnet/parsers/packages_config'
 require 'spandx/dotnet/parsers/sln'
 require 'spandx/dotnet/project_file'
-require 'spandx/gateways/http'
-require 'spandx/gateways/pypi'
+require 'spandx/java/index'
 require 'spandx/java/metadata'
 require 'spandx/java/parsers/maven'
-require 'spandx/parsers/pipfile_lock'
+require 'spandx/python/parsers/pipfile_lock'
+require 'spandx/python/pypi'
+require 'spandx/python/source'
 require 'spandx/rubygems/gateway'
-require 'spandx/rubygems/offline_index'
 require 'spandx/rubygems/parsers/gemfile_lock'
 require 'spandx/spdx/catalogue'
 require 'spandx/spdx/gateway'
@@ -52,7 +54,7 @@ module Spandx
     end
 
     def http
-      @http ||= Spandx::Gateways::Http.new
+      @http ||= Spandx::Core::Http.new
     end
 
     def logger

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spandx
 version: !ruby/object:Gem::Version
-  version: 0.10.1
+  version: 0.11.0
 platform: ruby
 authors:
 - mo khan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-16 00:00:00.000000000 Z
+date: 2020-03-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -260,10 +260,12 @@ files:
 - lib/spandx/cli/commands/index/build.rb
 - lib/spandx/cli/commands/index/update.rb
 - lib/spandx/cli/commands/scan.rb
+- lib/spandx/core/cache.rb
 - lib/spandx/core/content.rb
 - lib/spandx/core/database.rb
 - lib/spandx/core/dependency.rb
 - lib/spandx/core/guess.rb
+- lib/spandx/core/http.rb
 - lib/spandx/core/parser.rb
 - lib/spandx/core/report.rb
 - lib/spandx/core/score.rb
@@ -274,13 +276,13 @@ files:
 - lib/spandx/dotnet/parsers/packages_config.rb
 - lib/spandx/dotnet/parsers/sln.rb
 - lib/spandx/dotnet/project_file.rb
-- lib/spandx/gateways/http.rb
-- lib/spandx/gateways/pypi.rb
+- lib/spandx/java/index.rb
 - lib/spandx/java/metadata.rb
 - lib/spandx/java/parsers/maven.rb
-- lib/spandx/parsers/pipfile_lock.rb
+- lib/spandx/python/parsers/pipfile_lock.rb
+- lib/spandx/python/pypi.rb
+- lib/spandx/python/source.rb
 - lib/spandx/rubygems/gateway.rb
-- lib/spandx/rubygems/offline_index.rb
 - lib/spandx/rubygems/parsers/gemfile_lock.rb
 - lib/spandx/spdx/catalogue.rb
 - lib/spandx/spdx/gateway.rb

data/lib/spandx/gateways/pypi.rb

@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-module Spandx
-  module Gateways
-    class PyPI
-      class Source
-        attr_reader :name, :uri, :verify_ssl
-
-        def initialize(source)
-          @name = source['name']
-          @uri = URI.parse(source['url'])
-          @verify_ssl = source['verify_ssl']
-        end
-
-        def host
-          @uri.host
-        end
-
-        def uri_for(name, version)
-          "https://#{host}/pypi/#{name}/#{version}/json"
-        end
-
-        def lookup(name, version, http: Spandx.http)
-          response = http.get(uri_for(name, version))
-          response if http.ok?(response)
-        end
-
-        class << self
-          def sources_from(json)
-            meta = json['_meta']
-            meta['sources'].map do |hash|
-              Gateways::PyPI::Source.new(hash)
-            end
-          end
-
-          def default
-            new(
-              'name' => 'pypi',
-              'url' => 'https://pypi.org/simple',
-              'verify_ssl' => true
-            )
-          end
-        end
-      end
-
-      def initialize(sources: [Source.default])
-        @sources = sources
-      end
-
-      def definition_for(name, version)
-        @sources.each do |source|
-          response = source.lookup(name, version)
-          return JSON.parse(response.body).fetch('info', {}) if response
-        end
-        {}
-      end
-    end
-  end
-end

data/lib/spandx/parsers/pipfile_lock.rb

@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-module Spandx
-  module Parsers
-    class PipfileLock < ::Spandx::Core::Parser
-      def self.matches?(filename)
-        filename.match?(/Pipfile.*\.lock/)
-      end
-
-      def parse(lockfile)
-        results = []
-        dependencies_from(lockfile) do |x|
-          results << ::Spandx::Core::Dependency.new(
-            name: x[:name],
-            version: x[:version],
-            licenses: x[:licenses]
-          )
-        end
-        results
-      end
-
-      private
-
-      def dependencies_from(lockfile)
-        json = JSON.parse(IO.read(lockfile))
-        each_dependency(pypi_for(json), json) do |name, version, definition|
-          yield({ name: name, version: version, licenses: [catalogue[definition['license']]] })
-        end
-      end
-
-      def each_dependency(pypi, json, groups: %w[default develop])
-        groups.each do |group|
-          json[group].each do |name, value|
-            version = canonicalize(value['version'])
-            yield name, version, pypi.definition_for(name, version)
-          end
-        end
-      end
-
-      def canonicalize(version)
-        version.gsub(/==/, '')
-      end
-
-      def pypi_for(json)
-        Gateways::PyPI.new(
-          sources: Gateways::PyPI::Source.sources_from(json)
-        )
-      end
-    end
-  end
-end