spandx 0.12.3 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 03ab3629ac636e8be88e27061cbce6941672dfd0a99bce30a9a638a095ebddab
-  data.tar.gz: 3e0d7eb62b59887c2770ac9dfbef7b0037fcf3031e73f2f380a8ed1ba93d2cca
+  metadata.gz: a2cbe4ba53ef5f776dd0c52170e0c7b56b0bf2aa9893f221c45636fcc7080cb5
+  data.tar.gz: ee0936304c2322df4d568223aa1095dffe1b874a1902f6fc74504e28b63dffd4
 SHA512:
-  metadata.gz: c6962a430329a9d3ed3f274485a55a396cea66b9bcd8f664caf803d74c5768508aa9eee499a7ae99ab06bd840a6bedb67b85812cab193c1e5f6f725458c05fbd
-  data.tar.gz: a42dd5adeeb36e374a1f4a2582a881d5d4e28f2959d0044e653162edd0835895ade882261379bc5d47d3a7955b536c7f0ddc53b4a489ce8e68e6b6a3ad0c9731
+  metadata.gz: 19627c27dca8dc2609549f7a4245488689aa0dda9846dfc555e50b39735e6a2895686d2b1a672afaecc73080f02436070e34060b15b2cc250dbbe13ca46999eb
+  data.tar.gz: ee2b9816fb6e85e94c32ba05844540872244cf8cc4aac175ce3043c921429bbc4aaaf643c753b681c83a06eb6e7ae3bb044fd2944d9fa1e70508143e01bf18c1

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-Version 0.12.3
+Version 0.13.0
 
 # Changelog
 
@@ -9,6 +9,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.13.0] - 2020-05-12
+### Added
+- Add progress bar
+- Add SPDX expression parser.
+- Add index for each cache.
+- Update cache paths to point to Spandx organization.
+- Add optimized CSV parser.
+- Fetch dependency data concurrently.
+- Add profiling and benchmarking tools.
+
+### Changed
+- Update git pull command to fetch master branch with a depth of 1.
+- Update Nuget and PyPI cache builders to use same API for writing to cache.
+- Update license lookup to parse SPDX expressions.
+
+### Removed
+- Drop Ruby 2.4 support.
+- Drop Jaro Winkler algorithm.
+- Drop Levenshtein algorithm.
+
+### Fixed
+- Fix bug in spawning worker threads in thread pool.
+- Reset `http` global before each test to remove leakage between tests.
+
 ## [0.12.3] - 2020-04-19
 ### Fixed
 - Ignore nuget entries with missing `items`.
@@ -144,27 +168,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Provide ruby API to the latest SPDX catalogue.
 
-[Unreleased]: https://github.com/mokhan/spandx/compare/v0.12.3...HEAD
-[0.12.3]: https://github.com/mokhan/spandx/compare/v0.12.2...v0.12.3
-[0.12.2]: https://github.com/mokhan/spandx/compare/v0.12.1...v0.12.2
-[0.12.1]: https://github.com/mokhan/spandx/compare/v0.12.0...v0.12.1
-[0.12.0]: https://github.com/mokhan/spandx/compare/v0.11.0...v0.12.0
-[0.11.0]: https://github.com/mokhan/spandx/compare/v0.10.1...v0.11.0
-[0.10.1]: https://github.com/mokhan/spandx/compare/v0.10.0...v0.10.1
-[0.10.0]: https://github.com/mokhan/spandx/compare/v0.9.0...v0.10.0
-[0.9.0]: https://github.com/mokhan/spandx/compare/v0.8.0...v0.9.0
-[0.8.0]: https://github.com/mokhan/spandx/compare/v0.7.0...v0.8.0
-[0.7.0]: https://github.com/mokhan/spandx/compare/v0.6.0...v0.7.0
-[0.6.0]: https://github.com/mokhan/spandx/compare/v0.5.0...v0.6.0
-[0.5.0]: https://github.com/mokhan/spandx/compare/v0.4.1...v0.5.0
-[0.4.1]: https://github.com/mokhan/spandx/compare/v0.4.0...v0.4.1
-[0.4.0]: https://github.com/mokhan/spandx/compare/v0.3.0...v0.4.0
-[0.3.0]: https://github.com/mokhan/spandx/compare/v0.2.0...v0.3.0
-[0.2.0]: https://github.com/mokhan/spandx/compare/v0.1.7...v0.2.0
-[0.1.7]: https://github.com/mokhan/spandx/compare/v0.1.6...v0.1.7
-[0.1.6]: https://github.com/mokhan/spandx/compare/v0.1.5...v0.1.6
-[0.1.5]: https://github.com/mokhan/spandx/compare/v0.1.4...v0.1.5
-[0.1.4]: https://github.com/mokhan/spandx/compare/v0.1.3...v0.1.4
-[0.1.3]: https://github.com/mokhan/spandx/compare/v0.1.2...v0.1.3
-[0.1.2]: https://github.com/mokhan/spandx/compare/v0.1.1...v0.1.2
-[0.1.1]: https://github.com/mokhan/spandx/compare/v0.1.0...v0.1.1
+[Unreleased]: https://github.com/spandx/spandx/compare/v0.13.0...HEAD
+[0.13.0]: https://github.com/spandx/spandx/compare/v0.12.3...v0.13.0
+[0.12.3]: https://github.com/spandx/spandx/compare/v0.12.2...v0.12.3
+[0.12.2]: https://github.com/spandx/spandx/compare/v0.12.1...v0.12.2
+[0.12.1]: https://github.com/spandx/spandx/compare/v0.12.0...v0.12.1
+[0.12.0]: https://github.com/spandx/spandx/compare/v0.11.0...v0.12.0
+[0.11.0]: https://github.com/spandx/spandx/compare/v0.10.1...v0.11.0
+[0.10.1]: https://github.com/spandx/spandx/compare/v0.10.0...v0.10.1
+[0.10.0]: https://github.com/spandx/spandx/compare/v0.9.0...v0.10.0
+[0.9.0]: https://github.com/spandx/spandx/compare/v0.8.0...v0.9.0
+[0.8.0]: https://github.com/spandx/spandx/compare/v0.7.0...v0.8.0
+[0.7.0]: https://github.com/spandx/spandx/compare/v0.6.0...v0.7.0
+[0.6.0]: https://github.com/spandx/spandx/compare/v0.5.0...v0.6.0
+[0.5.0]: https://github.com/spandx/spandx/compare/v0.4.1...v0.5.0
+[0.4.1]: https://github.com/spandx/spandx/compare/v0.4.0...v0.4.1
+[0.4.0]: https://github.com/spandx/spandx/compare/v0.3.0...v0.4.0
+[0.3.0]: https://github.com/spandx/spandx/compare/v0.2.0...v0.3.0
+[0.2.0]: https://github.com/spandx/spandx/compare/v0.1.7...v0.2.0
+[0.1.7]: https://github.com/spandx/spandx/compare/v0.1.6...v0.1.7
+[0.1.6]: https://github.com/spandx/spandx/compare/v0.1.5...v0.1.6
+[0.1.5]: https://github.com/spandx/spandx/compare/v0.1.4...v0.1.5
+[0.1.4]: https://github.com/spandx/spandx/compare/v0.1.3...v0.1.4
+[0.1.3]: https://github.com/spandx/spandx/compare/v0.1.2...v0.1.3
+[0.1.2]: https://github.com/spandx/spandx/compare/v0.1.1...v0.1.2
+[0.1.1]: https://github.com/spandx/spandx/compare/v0.1.0...v0.1.1

data/README.md

@@ -1,10 +1,14 @@
-# Spandx ![badge](https://github.com/mokhan/spandx/workflows/ci/badge.svg)
+![Spandx](logo.gif)
+
+*Logo courtesy of [@speasley](https://github.com/speasley)*
+
+# Spandx ![badge](https://github.com/spandx/spandx/workflows/ci/badge.svg)
 
 A ruby API for interacting with the https://spdx.org software license catalogue.
 This gem includes a command line interface to scan a software project for the
 software licenses that are associated with each dependency in the project.
 `spandx` leverages an offline cache of software licenses for known dependencies.
-The offline cache allows spandx to perform a truly airgap friendly scan of software
+The offline cache allows Spandx to perform an air gap friendly scan of software
 projects.
 
 ### Supported project types
@@ -42,7 +46,7 @@ Or install it yourself as:
 
 ### Command line interface
 
-The command line interface supports operations to build and fetch the latest offline index.
+The command line interface supports operations to fetch the latest pre-built cache.
 See the help for each subcommand for more information on how to use the command.
 
 ```bash
@@ -62,19 +66,19 @@ To scan a specific project file use the `scan` command:
 モ spandx scan ruby/Gemfile.lock
 ```
 
-To activate airgap mode use the `--airgap` option:
+To activate air gap mode use the `--airgap` option:
 
 ```bash
 モ spandx scan dotnet/application.sln --airgap
 モ spandx scan ruby/Gemfile.lock --airgap
 ```
 
-Airgap mode assumes that an offline cache has been placed in `$HOME/.local/share/`.
+Air gap mode assumes that an offline cache has been placed in `$HOME/.local/share/`.
 
 To fetch the latest offline cache:
 
 ```bash
-モ spandx index update
+モ spandx pull
 ```
 
 ### Ruby API
@@ -106,7 +110,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/mokhan/spandx.
+Bug reports and pull requests are welcome on GitHub at https://github.com/spandx/spandx.
 
 ## License
 

data/exe/spandx

@@ -1,8 +1,7 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-lib_path = File.expand_path('../lib', __dir__)
-require "#{lib_path}/spandx"
+require 'spandx'
 
 Signal.trap('INT') do
   warn("\n#{caller.join("\n")}: interrupted")

data/ext/spandx/extconf.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require 'mkmf'
+
+create_makefile('spandx/spandx')

data/lib/spandx.rb

@@ -8,10 +8,13 @@ require 'json'
 require 'logger'
 require 'net/hippie'
 require 'nokogiri'
+require 'parslet'
 require 'pathname'
 require 'yaml'
 require 'zeitwerk'
 
+require 'spandx/spandx'
+
 loader = Zeitwerk::Loader.for_gem
 loader.setup # ready!
 
@@ -20,7 +23,7 @@ module Spandx
   Rubygems = Ruby
 
   class << self
-    attr_writer :airgap, :logger
+    attr_writer :airgap, :logger, :http, :git
 
     def root
       Pathname.new(File.dirname(__FILE__)).join('../..')
@@ -40,8 +43,8 @@ module Spandx
 
     def git
       @git ||= {
-        cache: ::Spandx::Core::Git.new(url: 'https://github.com/mokhan/spandx-index.git'),
-        rubygems: ::Spandx::Core::Git.new(url: 'https://github.com/mokhan/spandx-rubygems.git'),
+        cache: ::Spandx::Core::Git.new(url: 'https://github.com/spandx/cache.git'),
+        rubygems: ::Spandx::Core::Git.new(url: 'https://github.com/spandx/rubygems-cache.git'),
         spdx: ::Spandx::Core::Git.new(url: 'https://github.com/spdx/license-list-data.git'),
       }
     end

data/lib/spandx/cli.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'thor'
+require 'tty-progressbar'
 
 module Spandx
   module Cli

data/lib/spandx/cli/commands/build.rb

@@ -17,6 +17,7 @@ module Spandx
 
         def execute(output: $stdout)
           catalogue = Spandx::Spdx::Catalogue.from_git
+          build_buckets
           indexes.each do |index|
             output.puts index.name
             index.update!(catalogue: catalogue, output: output)
@@ -30,9 +31,19 @@ module Spandx
           index = INDEXES[@options[:index]&.to_sym]
 
           if index.nil?
-            INDEXES.values.uniq.map { |x| x.new(directory: @options[:directory]) }
+            INDEXES.values.uniq.map { |x| x.new(directory: directory) }
           else
-            [index.new(directory: @options[:directory])]
+            [index.new(directory: directory)]
+          end
+        end
+
+        def directory
+          @options.fetch(:directory, File.join(Dir.pwd, '.index'))
+        end
+
+        def build_buckets
+          (0x00..0xFF).map { |x| x.to_s(16).rjust(2, '0').downcase }.each do |hex|
+            FileUtils.mkdir_p(File.join(directory, hex))
           end
         end
       end

data/lib/spandx/cli/commands/scan.rb

@@ -4,6 +4,10 @@ module Spandx
   module Cli
     module Commands
       class Scan
+        NULL_BAR = Class.new do
+          def advance(*args); end
+        end.new
+
         attr_reader :scan_path
 
         def initialize(scan_path, options)
@@ -13,43 +17,37 @@ module Spandx
         end
 
         def execute(output: $stdout)
-          report = ::Spandx::Core::Report.new
-          each_file_in(scan_path) do |file|
-            each_dependency_from(file) do |dependency|
-              report.add(dependency)
+          Spandx::Core::ThreadPool.open do |pool|
+            report = ::Spandx::Core::Report.new
+            each_file do |file|
+              each_dependency_from(file, pool) do |dependency|
+                report.add(dependency)
+              end
             end
+            output.puts(format(report.to(@options[:format])))
           end
-          output.puts(format(report.to(@options[:format])))
         end
 
         private
 
-        def recursive?
-          @options['recursive']
+        def each_file
+          Spandx::Core::PathTraversal
+            .new(scan_path, recursive: @options['recursive'])
+            .each { |file| yield file }
         end
 
-        def each_file_in(dir, &block)
-          files = File.directory?(dir) ? Dir.glob(File.join(dir, '*')) : [dir]
-          files.each do |file|
-            if File.directory?(file)
-              each_file_in(file, &block) if recursive?
-            else
-              Spandx.logger.debug(file)
-              block.call(file)
+        def each_dependency_from(file, pool)
+          dependencies = ::Spandx::Core::Parser.for(file).parse(file)
+          with_progress(title_for(file), dependencies.size) do |bar|
+            ::Spandx::Core::Concurrent
+              .map(dependencies, pool: pool) { |dependency| enhance(dependency) }
+              .each do |dependency|
+              bar.advance(1)
+              yield dependency
             end
           end
         end
 
-        def each_dependency_from(file)
-          ::Spandx::Core::Parser
-            .for(file)
-            .parse(file)
-            .map { |dependency| enhance(dependency) }
-            .each { |dependency| yield dependency }
-        rescue StandardError => error
-          Spandx.logger.error(error)
-        end
-
         def format(output)
           Array(output).map(&:to_s)
         end
@@ -59,6 +57,14 @@ module Spandx
             .all
             .inject(dependency) { |memo, plugin| plugin.enhance(memo) }
         end
+
+        def title_for(file)
+          "#{file} [:bar, :elapsed] :percent"
+        end
+
+        def with_progress(title, total)
+          yield @options[:show_progress] ? TTY::ProgressBar.new(title, total: total) : NULL_BAR
+        end
       end
     end
   end

data/lib/spandx/cli/main.rb

@@ -11,6 +11,7 @@ module Spandx
       method_option :format, aliases: '-f', type: :string, desc: 'Format of report', default: 'table'
       method_option :pull, aliases: '-p', type: :boolean, desc: 'Pull the latest cache before the scan', default: false
       method_option :require, aliases: '-r', type: :string, desc: 'Causes spandx to load the library using require.', default: nil
+      method_option :show_progress, aliases: '-sp', type: :boolean, desc: 'Shows a progress bar', default: true
       def scan(lockfile)
         if options[:help]
           invoke :help, ['scan']

data/lib/spandx/core/cache.rb

@@ -3,83 +3,70 @@
 module Spandx
   module Core
     class Cache
-      attr_reader :db, :package_manager
+      include Enumerable
 
-      def initialize(package_manager, db: Spandx.git[:cache])
-        @package_manager = package_manager
-        @db = db
-        @cache = {}
-        @lines = {}
+      attr_reader :package_manager, :root
+
+      def initialize(package_manager, root:)
+        @package_manager = package_manager.to_s
+        @root = root
       end
 
       def licenses_for(name, version)
-        found = search(name: name, version: version)
-        Spandx.logger.debug("Cache miss: #{name}-#{version}") if found.nil?
+        return [] if name.nil? || name.empty?
+
+        found = datafile_for(name).search(name: name, version: version)
+        Spandx.logger.debug { "Cache miss: #{name}-#{version}" } unless found
         found ? found[2].split('-|-') : []
       end
 
-      private
-
-      def digest_for(components)
-        Digest::SHA1.hexdigest(Array(components).join('/'))
+      def each
+        datafiles.each do |_hex, datafile|
+          datafile.each do |item|
+            yield item
+          end
+        end
       end
 
-      def key_for(name)
-        digest_for(name)[0...2]
+      def insert(name, version, licenses)
+        return if name.nil? || name.empty?
+
+        datafile_for(name).insert(name, version, licenses)
       end
 
-      def search(name:, version:)
-        datafile = datafile_for(name)
-        db.open(datafile) do |io|
-          search_for("#{name}-#{version}", io, @lines.fetch(datafile) { |key| @lines[key] = lines_in(io) })
-        end
-      rescue Errno::ENOENT => error
-        Spandx.logger.error(error)
-        nil
+      def insert!(*args)
+        insert(*args)
+        rebuild_index
       end
 
       def datafile_for(name)
-        ".index/#{key_for(name)}/#{package_manager}"
+        datafiles.fetch(key_for(name))
       end
 
-      def lines_in(io)
-        lines = []
-        io.seek(0)
-        until io.eof?
-          lines << io.pos
-          io.gets
+      def rebuild_index
+        datafiles.each do |_hex, datafile|
+          datafile.index.update!
         end
-        lines
       end
 
-      def search_for(term, io, lines)
-        return if lines.empty?
-        return @cache[term] if @cache.key?(term)
-
-        mid = lines.size == 1 ? 0 : lines.size / 2
-        io.seek(lines[mid])
-        comparison = matches?(term, parse_row(io)) do |row|
-          return row
-        end
+      private
 
-        search_for(term, io, partition(comparison, mid, lines))
+      def digest_for(name)
+        Digest::SHA1.hexdigest(name)
       end
 
-      def matches?(term, row)
-        (term <=> "#{row[0]}-#{row[1]}").tap do |comparison|
-          yield row if comparison.zero?
-        end
+      def key_for(name)
+        digest_for(name)[0...2]
       end
 
-      def partition(comparison, mid, lines)
-        min, max = comparison.positive? ? [mid + 1, lines.length] : [0, mid]
-        lines.slice(min, max)
+      def datafiles
+        @datafiles ||= candidate_keys.each_with_object({}) do |key, memo|
+          memo[key] = DataFile.new(File.join(root, key, package_manager))
+        end
       end
 
-      def parse_row(io)
-        row = CSV.parse(io.readline)[0]
-        @cache["#{row[0]}-#{row[1]}"] = row
-        row
+      def candidate_keys
+        (0x00..0xFF).map { |x| x.to_s(16).upcase.rjust(2, '0').downcase }
       end
     end
   end