spandx 0.12.3 → 0.13.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 03ab3629ac636e8be88e27061cbce6941672dfd0a99bce30a9a638a095ebddab
-  data.tar.gz: 3e0d7eb62b59887c2770ac9dfbef7b0037fcf3031e73f2f380a8ed1ba93d2cca
+  metadata.gz: ef9e117562bb153d2bf7a7aa8561244f50f7dcbca8a81300e10279efec45c674
+  data.tar.gz: '069f96ae764417f3b005ebd8e7fee919c66eee38a63649918f7dea3209a9bc34'
 SHA512:
-  metadata.gz: c6962a430329a9d3ed3f274485a55a396cea66b9bcd8f664caf803d74c5768508aa9eee499a7ae99ab06bd840a6bedb67b85812cab193c1e5f6f725458c05fbd
-  data.tar.gz: a42dd5adeeb36e374a1f4a2582a881d5d4e28f2959d0044e653162edd0835895ade882261379bc5d47d3a7955b536c7f0ddc53b4a489ce8e68e6b6a3ad0c9731
+  metadata.gz: 67ec66d00236b0c4a98bc770e0b33698ecabb6e868b4e1b309c735a0d3cf5288a49393280f0cff2829d8ab5d1005920b16a877bdabdb2a0b63c88ac9f7af7df9
+  data.tar.gz: 9260aeb08a495fb4f8bd1ba4c2b17fed8e34c2e60e96d5c826c79f7c51d83a8686b4194e060fa77e3b898f1beb17404006dd5370b727d2a7ce98d87ddce41507

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-Version 0.12.3
+Version 0.13.4
 
 # Changelog
 
@@ -9,6 +9,51 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.13.4] - 2020-05-26
+### Added
+- Add detected file path to report output.
+
+### Changed
+- Use `Pathname` instead of `String` to represent file paths.
+- Scan current directory when a path is not specified.
+
+## [0.13.3] - 2020-05-19
+### Fixed
+- Ignore invalid URLs during scan.
+
+## [0.13.2] - 2020-05-17
+### Fixed
+- Detect licenses when provided as an array.
+- Skip empty lockfiles.
+
+## [0.13.1] - 2020-05-16
+### Fixed
+- Add `ext/**/*.c` and `ext/**/*.h` to list of files.
+
+## [0.13.0] - 2020-05-12
+### Added
+- Add progress bar
+- Add SPDX expression parser.
+- Add index for each cache.
+- Update cache paths to point to Spandx organization.
+- Add optimized CSV parser.
+- Fetch dependency data concurrently.
+- Add profiling and benchmarking tools.
+
+### Changed
+- Update git pull command to fetch master branch with a depth of 1.
+- Update Nuget and PyPI cache builders to use same API for writing to cache.
+- Update license lookup to parse SPDX expressions.
+
+### Removed
+- Drop Ruby 2.4 support.
+- Drop Jaro Winkler algorithm.
+- Drop Levenshtein algorithm.
+
+### Fixed
+- Fix bug in spawning worker threads in thread pool.
+- Reset `http` global before each test to remove leakage between tests.
+
 ## [0.12.3] - 2020-04-19
 ### Fixed
 - Ignore nuget entries with missing `items`.
@@ -144,27 +189,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Provide ruby API to the latest SPDX catalogue.
 
-[Unreleased]: https://github.com/mokhan/spandx/compare/v0.12.3...HEAD
-[0.12.3]: https://github.com/mokhan/spandx/compare/v0.12.2...v0.12.3
-[0.12.2]: https://github.com/mokhan/spandx/compare/v0.12.1...v0.12.2
-[0.12.1]: https://github.com/mokhan/spandx/compare/v0.12.0...v0.12.1
-[0.12.0]: https://github.com/mokhan/spandx/compare/v0.11.0...v0.12.0
-[0.11.0]: https://github.com/mokhan/spandx/compare/v0.10.1...v0.11.0
-[0.10.1]: https://github.com/mokhan/spandx/compare/v0.10.0...v0.10.1
-[0.10.0]: https://github.com/mokhan/spandx/compare/v0.9.0...v0.10.0
-[0.9.0]: https://github.com/mokhan/spandx/compare/v0.8.0...v0.9.0
-[0.8.0]: https://github.com/mokhan/spandx/compare/v0.7.0...v0.8.0
-[0.7.0]: https://github.com/mokhan/spandx/compare/v0.6.0...v0.7.0
-[0.6.0]: https://github.com/mokhan/spandx/compare/v0.5.0...v0.6.0
-[0.5.0]: https://github.com/mokhan/spandx/compare/v0.4.1...v0.5.0
-[0.4.1]: https://github.com/mokhan/spandx/compare/v0.4.0...v0.4.1
-[0.4.0]: https://github.com/mokhan/spandx/compare/v0.3.0...v0.4.0
-[0.3.0]: https://github.com/mokhan/spandx/compare/v0.2.0...v0.3.0
-[0.2.0]: https://github.com/mokhan/spandx/compare/v0.1.7...v0.2.0
-[0.1.7]: https://github.com/mokhan/spandx/compare/v0.1.6...v0.1.7
-[0.1.6]: https://github.com/mokhan/spandx/compare/v0.1.5...v0.1.6
-[0.1.5]: https://github.com/mokhan/spandx/compare/v0.1.4...v0.1.5
-[0.1.4]: https://github.com/mokhan/spandx/compare/v0.1.3...v0.1.4
-[0.1.3]: https://github.com/mokhan/spandx/compare/v0.1.2...v0.1.3
-[0.1.2]: https://github.com/mokhan/spandx/compare/v0.1.1...v0.1.2
-[0.1.1]: https://github.com/mokhan/spandx/compare/v0.1.0...v0.1.1
+[Unreleased]: https://github.com/spandx/spandx/compare/v0.13.3...HEAD
+[0.13.3]: https://github.com/spandx/spandx/compare/v0.13.2...v0.13.3
+[0.13.2]: https://github.com/spandx/spandx/compare/v0.13.1...v0.13.2
+[0.13.1]: https://github.com/spandx/spandx/compare/v0.13.0...v0.13.1
+[0.13.0]: https://github.com/spandx/spandx/compare/v0.12.3...v0.13.0
+[0.12.3]: https://github.com/spandx/spandx/compare/v0.12.2...v0.12.3
+[0.12.2]: https://github.com/spandx/spandx/compare/v0.12.1...v0.12.2
+[0.12.1]: https://github.com/spandx/spandx/compare/v0.12.0...v0.12.1
+[0.12.0]: https://github.com/spandx/spandx/compare/v0.11.0...v0.12.0
+[0.11.0]: https://github.com/spandx/spandx/compare/v0.10.1...v0.11.0
+[0.10.1]: https://github.com/spandx/spandx/compare/v0.10.0...v0.10.1
+[0.10.0]: https://github.com/spandx/spandx/compare/v0.9.0...v0.10.0
+[0.9.0]: https://github.com/spandx/spandx/compare/v0.8.0...v0.9.0
+[0.8.0]: https://github.com/spandx/spandx/compare/v0.7.0...v0.8.0
+[0.7.0]: https://github.com/spandx/spandx/compare/v0.6.0...v0.7.0
+[0.6.0]: https://github.com/spandx/spandx/compare/v0.5.0...v0.6.0
+[0.5.0]: https://github.com/spandx/spandx/compare/v0.4.1...v0.5.0
+[0.4.1]: https://github.com/spandx/spandx/compare/v0.4.0...v0.4.1
+[0.4.0]: https://github.com/spandx/spandx/compare/v0.3.0...v0.4.0
+[0.3.0]: https://github.com/spandx/spandx/compare/v0.2.0...v0.3.0
+[0.2.0]: https://github.com/spandx/spandx/compare/v0.1.7...v0.2.0
+[0.1.7]: https://github.com/spandx/spandx/compare/v0.1.6...v0.1.7
+[0.1.6]: https://github.com/spandx/spandx/compare/v0.1.5...v0.1.6
+[0.1.5]: https://github.com/spandx/spandx/compare/v0.1.4...v0.1.5
+[0.1.4]: https://github.com/spandx/spandx/compare/v0.1.3...v0.1.4
+[0.1.3]: https://github.com/spandx/spandx/compare/v0.1.2...v0.1.3
+[0.1.2]: https://github.com/spandx/spandx/compare/v0.1.1...v0.1.2
+[0.1.1]: https://github.com/spandx/spandx/compare/v0.1.0...v0.1.1

data/README.md

@@ -1,10 +1,14 @@
-# Spandx ![badge](https://github.com/mokhan/spandx/workflows/ci/badge.svg)
+![Spandx](logo.gif)
+
+*Logo courtesy of [@speasley](https://github.com/speasley)*
+
+# Spandx ![badge](https://github.com/spandx/spandx/workflows/ci/badge.svg)
 
 A ruby API for interacting with the https://spdx.org software license catalogue.
 This gem includes a command line interface to scan a software project for the
 software licenses that are associated with each dependency in the project.
 `spandx` leverages an offline cache of software licenses for known dependencies.
-The offline cache allows spandx to perform a truly airgap friendly scan of software
+The offline cache allows Spandx to perform an air gap friendly scan of software
 projects.
 
 ### Supported project types
@@ -42,7 +46,7 @@ Or install it yourself as:
 
 ### Command line interface
 
-The command line interface supports operations to build and fetch the latest offline index.
+The command line interface supports operations to fetch the latest pre-built cache.
 See the help for each subcommand for more information on how to use the command.
 
 ```bash
@@ -62,19 +66,19 @@ To scan a specific project file use the `scan` command:
 モ spandx scan ruby/Gemfile.lock
 ```
 
-To activate airgap mode use the `--airgap` option:
+To activate air gap mode use the `--airgap` option:
 
 ```bash
 モ spandx scan dotnet/application.sln --airgap
 モ spandx scan ruby/Gemfile.lock --airgap
 ```
 
-Airgap mode assumes that an offline cache has been placed in `$HOME/.local/share/`.
+Air gap mode assumes that an offline cache has been placed in `$HOME/.local/share/`.
 
 To fetch the latest offline cache:
 
 ```bash
-モ spandx index update
+モ spandx pull
 ```
 
 ### Ruby API
@@ -106,7 +110,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/mokhan/spandx.
+Bug reports and pull requests are welcome on GitHub at https://github.com/spandx/spandx.
 
 ## License
 

data/exe/spandx

@@ -1,8 +1,7 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-lib_path = File.expand_path('../lib', __dir__)
-require "#{lib_path}/spandx"
+require 'spandx'
 
 Signal.trap('INT') do
   warn("\n#{caller.join("\n")}: interrupted")

data/ext/spandx/extconf.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require 'mkmf'
+
+create_makefile('spandx/spandx')

data/ext/spandx/spandx.c

@@ -0,0 +1,55 @@
+#include "spandx.h"
+
+#define NEWLINE 10
+
+VALUE rb_mSpandx;
+VALUE rb_mCore;
+VALUE rb_mCsvParser;
+
+// "name","version","license"
+// "name","version","license"\n
+// "name","version","license"\r
+// "name","version","license"\r\n
+// "name","version",""\r\n
+VALUE parse(VALUE self, VALUE line)
+{
+  if (NIL_P(line)) return Qnil;
+
+  char *p;
+
+  p = RSTRING_PTR(line);
+  if (*p != '"') return Qnil;
+
+  const VALUE items = rb_ary_new2(3);
+  const char *s, *n;
+  const long len = RSTRING_LEN(line);
+  enum { open, closed } state = closed;
+
+  for (int i = 0; i < len && *p; i++) {
+    if (*p == '"') {
+      n = p;
+      if (i < (len - 1)) *n++;
+
+      if (state == closed) {
+        s = n;
+        state = open;
+      } else if (state == open) {
+        if (!*n || n == p || *n == ',' || *n == NEWLINE) {
+          rb_ary_push(items, rb_str_new(s, p - s));
+          state = closed;
+        }
+      }
+    }
+    *(p++);
+  }
+
+  return items;
+}
+
+void Init_spandx(void)
+{
+  rb_mSpandx = rb_define_module("Spandx");
+  rb_mCore = rb_define_module_under(rb_mSpandx, "Core");
+  rb_mCsvParser = rb_define_module_under(rb_mCore, "CsvParser");
+  rb_define_module_function(rb_mCsvParser, "parse", parse, 1);
+}

data/ext/spandx/spandx.h

@@ -0,0 +1,6 @@
+#ifndef SPANDX_H
+#define SPANDX_H 1
+
+#include "ruby.h"
+
+#endif /* SPANDX_H */

data/lib/spandx.rb

@@ -8,9 +8,12 @@ require 'json'
 require 'logger'
 require 'net/hippie'
 require 'nokogiri'
+require 'parslet'
 require 'pathname'
 require 'yaml'
 require 'zeitwerk'
+require 'terminal-table'
+require 'spandx/spandx'
 
 loader = Zeitwerk::Loader.for_gem
 loader.setup # ready!
@@ -20,7 +23,7 @@ module Spandx
   Rubygems = Ruby
 
   class << self
-    attr_writer :airgap, :logger
+    attr_writer :airgap, :logger, :http, :git
 
     def root
       Pathname.new(File.dirname(__FILE__)).join('../..')
@@ -40,8 +43,8 @@ module Spandx
 
     def git
       @git ||= {
-        cache: ::Spandx::Core::Git.new(url: 'https://github.com/mokhan/spandx-index.git'),
-        rubygems: ::Spandx::Core::Git.new(url: 'https://github.com/mokhan/spandx-rubygems.git'),
+        cache: ::Spandx::Core::Git.new(url: 'https://github.com/spandx/cache.git'),
+        rubygems: ::Spandx::Core::Git.new(url: 'https://github.com/spandx/rubygems-cache.git'),
         spdx: ::Spandx::Core::Git.new(url: 'https://github.com/spdx/license-list-data.git'),
       }
     end

data/lib/spandx/cli.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
+require 'nanospinner'
 require 'thor'
+require 'tty-screen'
 
 module Spandx
   module Cli

data/lib/spandx/cli/commands/build.rb

@@ -17,6 +17,7 @@ module Spandx
 
         def execute(output: $stdout)
           catalogue = Spandx::Spdx::Catalogue.from_git
+          build_buckets
           indexes.each do |index|
             output.puts index.name
             index.update!(catalogue: catalogue, output: output)
@@ -30,9 +31,19 @@ module Spandx
           index = INDEXES[@options[:index]&.to_sym]
 
           if index.nil?
-            INDEXES.values.uniq.map { |x| x.new(directory: @options[:directory]) }
+            INDEXES.values.uniq.map { |x| x.new(directory: directory) }
           else
-            [index.new(directory: @options[:directory])]
+            [index.new(directory: directory)]
+          end
+        end
+
+        def directory
+          @options.fetch(:directory, File.join(Dir.pwd, '.index'))
+        end
+
+        def build_buckets
+          (0x00..0xFF).map { |x| x.to_s(16).rjust(2, '0').downcase }.each do |hex|
+            FileUtils.mkdir_p(File.join(directory, hex))
           end
         end
       end

data/lib/spandx/cli/commands/scan.rb

@@ -4,50 +4,41 @@ module Spandx
   module Cli
     module Commands
       class Scan
-        attr_reader :scan_path
+        attr_reader :scan_path, :spinner
 
         def initialize(scan_path, options)
           @scan_path = ::Pathname.new(scan_path)
           @options = options
+          @spinner = options[:show_progress] ? ::Spandx::Core::Spinner.new : ::Spandx::Core::Spinner::NULL
           require(options[:require]) if options[:require]
         end
 
         def execute(output: $stdout)
           report = ::Spandx::Core::Report.new
-          each_file_in(scan_path) do |file|
+          each_file do |file|
+            spinner.spin(file)
             each_dependency_from(file) do |dependency|
+              spinner.spin(file)
               report.add(dependency)
             end
           end
+          spinner.stop
           output.puts(format(report.to(@options[:format])))
         end
 
         private
 
-        def recursive?
-          @options['recursive']
-        end
-
-        def each_file_in(dir, &block)
-          files = File.directory?(dir) ? Dir.glob(File.join(dir, '*')) : [dir]
-          files.each do |file|
-            if File.directory?(file)
-              each_file_in(file, &block) if recursive?
-            else
-              Spandx.logger.debug(file)
-              block.call(file)
-            end
-          end
+        def each_file
+          Spandx::Core::PathTraversal
+            .new(scan_path, recursive: @options[:recursive])
+            .each { |file| yield file }
         end
 
         def each_dependency_from(file)
           ::Spandx::Core::Parser
-            .for(file)
             .parse(file)
-            .map { |dependency| enhance(dependency) }
+            .map { |x| enhance(x) }
             .each { |dependency| yield dependency }
-        rescue StandardError => error
-          Spandx.logger.error(error)
         end
 
         def format(output)

data/lib/spandx/cli/main.rb

@@ -8,10 +8,11 @@ module Spandx
       method_option :recursive, aliases: '-R', type: :boolean, desc: 'Perform recursive scan', default: false
       method_option :airgap, aliases: '-a', type: :boolean, desc: 'Disable network connections', default: false
       method_option :logfile, aliases: '-l', type: :string, desc: 'Path to a logfile', default: '/dev/null'
-      method_option :format, aliases: '-f', type: :string, desc: 'Format of report', default: 'table'
+      method_option :format, aliases: '-f', type: :string, desc: 'Format of report. (table, csv, json, hash)', default: 'table'
       method_option :pull, aliases: '-p', type: :boolean, desc: 'Pull the latest cache before the scan', default: false
       method_option :require, aliases: '-r', type: :string, desc: 'Causes spandx to load the library using require.', default: nil
-      def scan(lockfile)
+      method_option :show_progress, aliases: '-sp', type: :boolean, desc: 'Shows a progress bar', default: true
+      def scan(lockfile = Pathname.pwd)
         if options[:help]
           invoke :help, ['scan']
         else

data/lib/spandx/core/cache.rb

@@ -3,83 +3,70 @@
 module Spandx
   module Core
     class Cache
-      attr_reader :db, :package_manager
+      include Enumerable
 
-      def initialize(package_manager, db: Spandx.git[:cache])
-        @package_manager = package_manager
-        @db = db
-        @cache = {}
-        @lines = {}
+      attr_reader :package_manager, :root
+
+      def initialize(package_manager, root:)
+        @package_manager = package_manager.to_s
+        @root = root
       end
 
       def licenses_for(name, version)
-        found = search(name: name, version: version)
-        Spandx.logger.debug("Cache miss: #{name}-#{version}") if found.nil?
+        return [] if name.nil? || name.empty?
+
+        found = datafile_for(name).search(name: name, version: version)
+        Spandx.logger.debug { "Cache miss: #{name}-#{version}" } unless found
         found ? found[2].split('-|-') : []
       end
 
-      private
-
-      def digest_for(components)
-        Digest::SHA1.hexdigest(Array(components).join('/'))
+      def each
+        datafiles.each do |_hex, datafile|
+          datafile.each do |item|
+            yield item
+          end
+        end
       end
 
-      def key_for(name)
-        digest_for(name)[0...2]
+      def insert(name, version, licenses)
+        return if name.nil? || name.empty?
+
+        datafile_for(name).insert(name, version, licenses)
       end
 
-      def search(name:, version:)
-        datafile = datafile_for(name)
-        db.open(datafile) do |io|
-          search_for("#{name}-#{version}", io, @lines.fetch(datafile) { |key| @lines[key] = lines_in(io) })
-        end
-      rescue Errno::ENOENT => error
-        Spandx.logger.error(error)
-        nil
+      def insert!(*args)
+        insert(*args)
+        rebuild_index
       end
 
       def datafile_for(name)
-        ".index/#{key_for(name)}/#{package_manager}"
+        datafiles.fetch(key_for(name))
       end
 
-      def lines_in(io)
-        lines = []
-        io.seek(0)
-        until io.eof?
-          lines << io.pos
-          io.gets
+      def rebuild_index
+        datafiles.each do |_hex, datafile|
+          datafile.index.update!
         end
-        lines
       end
 
-      def search_for(term, io, lines)
-        return if lines.empty?
-        return @cache[term] if @cache.key?(term)
-
-        mid = lines.size == 1 ? 0 : lines.size / 2
-        io.seek(lines[mid])
-        comparison = matches?(term, parse_row(io)) do |row|
-          return row
-        end
+      private
 
-        search_for(term, io, partition(comparison, mid, lines))
+      def digest_for(name)
+        Digest::SHA1.hexdigest(name)
       end
 
-      def matches?(term, row)
-        (term <=> "#{row[0]}-#{row[1]}").tap do |comparison|
-          yield row if comparison.zero?
-        end
+      def key_for(name)
+        digest_for(name)[0...2]
       end
 
-      def partition(comparison, mid, lines)
-        min, max = comparison.positive? ? [mid + 1, lines.length] : [0, mid]
-        lines.slice(min, max)
+      def datafiles
+        @datafiles ||= candidate_keys.each_with_object({}) do |key, memo|
+          memo[key] = DataFile.new(File.join(root, key, package_manager))
+        end
       end
 
-      def parse_row(io)
-        row = CSV.parse(io.readline)[0]
-        @cache["#{row[0]}-#{row[1]}"] = row
-        row
+      def candidate_keys
+        (0x00..0xFF).map { |x| x.to_s(16).upcase.rjust(2, '0').downcase }
       end
     end
   end