spandx 0.13.3 → 0.15.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b36f49bab527c52c6d3f6ddf1d70e022422f70c678bf865231921982460a4b4
-  data.tar.gz: 5d31efbe54079dd42a07c46f9d47bbe508d712ebed5d7294537685112d170079
+  metadata.gz: 120ae97e54f2a138b7d1053d9e71c8b702174cf70fbac0fc861b8971f4aec0bd
+  data.tar.gz: c89d3ffe76d00d0a6fb18e91b5cdd3ef1487a826047213dd264be3d59796d188
 SHA512:
-  metadata.gz: abe3e8e231a35f5861b3e85502a7b5c415684ce351756b0d64c77cfcf417bfd099e25587714c9e88b9a3ddb7309154921e0b57b4601bbe9aa74de7f057165773
-  data.tar.gz: 757dd76cebbd921d4034a69e794beffac0c9ff00ec846491b4037bf01a687d06aad5a8d258dccb59dc6cce6d94626dbb6df77eae53ccc8755982482a780bf65f
+  metadata.gz: ab85f497f8ce4fe46a03b31d25ffa07816975dcbcb91a1438cf09bcd38f857ed652a1003569e2a8cbb091c5fb5b5b88a52043f4a61eec71337eff58107c75737
+  data.tar.gz: e73a923228358065c7d67bdfa8aaea328657269758f2c2d664621b5e6dd6da449b1d585655ae351e0edd99dbd155ef2efa24f0e37c95cd24c75f85da517503af

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-Version 0.13.3
+Version 0.15.1
 
 # Changelog
 
@@ -9,6 +9,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.15.1] - 2020-11-18
+### Fixed
+- Rebuild index after pulling latest cache.
+
+## [0.15.0] - 2020-11-18
+### Added
+- Parse `/var/lib/dpkg/status` file.
+
+## [0.14.0] - 2020-11-14
+### Added
+- Parse `/lib/apk/db/installed` file.
+
+## [0.13.5] - 2020-05-26
+### Fixed
+- Process PyPI package urls with single digit versions.
+- Remove unsupported `hash` report from help text.
+
+### Changed
+- Stream output to output stream as soon as results are available.
+- Switch to `Oj` for JSON parsing.
+- Run spinner on background thread.
+
+## [0.13.4] - 2020-05-26
+### Added
+- Add detected file path to report output.
+
+### Changed
+- Use `Pathname` instead of `String` to represent file paths.
+- Scan current directory when a path is not specified.
+
 ## [0.13.3] - 2020-05-19
 ### Fixed
 - Ignore invalid URLs during scan.
@@ -181,7 +211,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Provide ruby API to the latest SPDX catalogue.
 
-[Unreleased]: https://github.com/spandx/spandx/compare/v0.13.3...HEAD
+[Unreleased]: https://github.com/spandx/spandx/compare/v0.15.1...HEAD
+[0.15.1]: https://github.com/spandx/spandx/compare/v0.15.0...v0.15.1
+[0.15.0]: https://github.com/spandx/spandx/compare/v0.14.0...v0.15.0
+[0.14.0]: https://github.com/spandx/spandx/compare/v0.13.5...v0.14.0
+[0.13.5]: https://github.com/spandx/spandx/compare/v0.13.4...v0.13.5
+[0.13.4]: https://github.com/spandx/spandx/compare/v0.13.3...v0.13.4
 [0.13.3]: https://github.com/spandx/spandx/compare/v0.13.2...v0.13.3
 [0.13.2]: https://github.com/spandx/spandx/compare/v0.13.1...v0.13.2
 [0.13.1]: https://github.com/spandx/spandx/compare/v0.13.0...v0.13.1

data/exe/spandx

@@ -4,7 +4,6 @@
 require 'spandx'
 
 Signal.trap('INT') do
-  warn("\n#{caller.join("\n")}: interrupted")
   exit(1)
 end
 

data/ext/spandx/spandx.c

@@ -1,5 +1,7 @@
 #include "spandx.h"
 
+#define NEWLINE 10
+
 VALUE rb_mSpandx;
 VALUE rb_mCore;
 VALUE rb_mCsvParser;
@@ -9,7 +11,7 @@ VALUE rb_mCsvParser;
 // "name","version","license"\r
 // "name","version","license"\r\n
 // "name","version",""\r\n
-static VALUE parse(VALUE self, VALUE line)
+VALUE parse(VALUE self, VALUE line)
 {
   if (NIL_P(line)) return Qnil;
 
@@ -32,13 +34,13 @@ static VALUE parse(VALUE self, VALUE line)
         s = n;
         state = open;
       } else if (state == open) {
-        if (!*n || n == p || *n == ',' || *n == 10) {
+        if (!*n || n == p || *n == ',' || *n == NEWLINE) {
           rb_ary_push(items, rb_str_new(s, p - s));
           state = closed;
         }
       }
     }
-    *p++;
+    *(p++);
   }
 
   return items;
@@ -50,4 +52,6 @@ void Init_spandx(void)
   rb_mCore = rb_define_module_under(rb_mSpandx, "Core");
   rb_mCsvParser = rb_define_module_under(rb_mCore, "CsvParser");
   rb_define_module_function(rb_mCsvParser, "parse", parse, 1);
+
+  rb_gc_register_mark_object(rb_mCsvParser);
 }

data/lib/spandx.rb

@@ -8,11 +8,11 @@ require 'json'
 require 'logger'
 require 'net/hippie'
 require 'nokogiri'
+require 'oj'
 require 'parslet'
 require 'pathname'
 require 'yaml'
 require 'zeitwerk'
-
 require 'spandx/spandx'
 
 loader = Zeitwerk::Loader.for_gem

data/lib/spandx/cli.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'nanospinner'
 require 'thor'
-require 'tty-screen'
+require 'tty-spinner'
+require 'terminal-table'
 
 module Spandx
   module Cli

data/lib/spandx/cli/commands/pull.rb

@@ -8,12 +8,41 @@ module Spandx
           @options = options
         end
 
-        def execute(output: $stdout)
+        def execute(output: $stderr)
+          sync(output)
+          build(output, ::Spandx::Core::Dependency::PACKAGE_MANAGERS.values.uniq)
+          output.puts 'OK'
+        end
+
+        private
+
+        def sync(output)
           Spandx.git.each_value do |db|
-            output.puts "Updating #{db.url}..."
-            db.update!
+            with_spinner("Updating #{db.url}...", output: output) do
+              db.update!
+            end
           end
-          output.puts 'OK'
+        end
+
+        def build(output, sources)
+          index_path = Spandx.git[:cache].root.join('.index')
+
+          with_spinner('Rebuilding index...', output: output) do
+            sources.each do |source|
+              Spandx::Core::Cache
+                .new(source, root: index_path)
+                .rebuild_index
+            end
+          end
+        end
+
+        def with_spinner(message, output:)
+          spinner = TTY::Spinner.new("[:spinner] #{message}", output: output)
+          spinner.auto_spin
+          yield
+          spinner.success('(done)')
+        ensure
+          spinner.stop
         end
       end
     end

data/lib/spandx/cli/commands/scan.rb

@@ -4,52 +4,49 @@ module Spandx
   module Cli
     module Commands
       class Scan
-        attr_reader :scan_path, :spinner
+        include Spandx::Core
+        attr_reader :scan_path
 
         def initialize(scan_path, options)
           @scan_path = ::Pathname.new(scan_path)
           @options = options
-          @spinner = options[:show_progress] ? ::Spandx::Core::Spinner.new : ::Spandx::Core::Spinner::NULL
           require(options[:require]) if options[:require]
         end
 
         def execute(output: $stdout)
-          report = ::Spandx::Core::Report.new
-          each_file do |file|
-            spinner.spin(file)
-            each_dependency_from(file) do |dependency|
-              spinner.spin(file)
-              report.add(dependency)
+          with_printer(output) do |printer|
+            each_dependency do |dependency|
+              printer.print_line(Plugin.enhance(dependency), output)
             end
           end
-          spinner.stop
-          output.puts(format(report.to(@options[:format])))
         end
 
         private
 
         def each_file
-          Spandx::Core::PathTraversal
-            .new(scan_path, recursive: @options['recursive'])
+          PathTraversal
+            .new(scan_path, recursive: @options[:recursive])
             .each { |file| yield file }
         end
 
-        def each_dependency_from(file)
-          ::Spandx::Core::Parser
-            .for(file)
-            .parse(file)
-            .map { |x| enhance(x) }
-            .each { |dependency| yield dependency }
+        def each_dependency
+          each_file do |file|
+            Parser.parse(file).each do |dependency|
+              yield dependency
+            end
+          end
         end
 
         def format(output)
           Array(output).map(&:to_s)
         end
 
-        def enhance(dependency)
-          ::Spandx::Core::Plugin
-            .all
-            .inject(dependency) { |memo, plugin| plugin.enhance(memo) }
+        def with_printer(output)
+          printer = ::Spandx::Cli::Printer.for(@options[:format])
+          printer.print_header(output)
+          yield printer
+        ensure
+          printer.print_footer(output)
         end
       end
     end

data/lib/spandx/cli/main.rb

@@ -8,14 +8,14 @@ module Spandx
       method_option :recursive, aliases: '-R', type: :boolean, desc: 'Perform recursive scan', default: false
       method_option :airgap, aliases: '-a', type: :boolean, desc: 'Disable network connections', default: false
       method_option :logfile, aliases: '-l', type: :string, desc: 'Path to a logfile', default: '/dev/null'
-      method_option :format, aliases: '-f', type: :string, desc: 'Format of report', default: 'table'
+      method_option :format, aliases: '-f', type: :string, desc: 'Format of report. (table, csv, json)', default: 'table'
       method_option :pull, aliases: '-p', type: :boolean, desc: 'Pull the latest cache before the scan', default: false
       method_option :require, aliases: '-r', type: :string, desc: 'Causes spandx to load the library using require.', default: nil
-      method_option :show_progress, aliases: '-sp', type: :boolean, desc: 'Shows a progress bar', default: true
-      def scan(lockfile)
+      def scan(lockfile = Pathname.pwd)
         if options[:help]
           invoke :help, ['scan']
         else
+          Oj.default_options = { mode: :strict }
           Spandx.airgap = options[:airgap]
           Spandx.logger = Logger.new(options[:logfile])
           pull if options[:pull]

data/lib/spandx/cli/printer.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Cli
+    class Printer
+      def match?(_format)
+        raise ::Spandx::Error, :match?
+      end
+
+      def print_header(io); end
+
+      def print_line(dependency, io)
+        io.puts(dependency.to_s)
+      end
+
+      def print_footer(io); end
+
+      class << self
+        include Core::Registerable
+
+        def for(format)
+          find { |x| x.match?(format) } || new
+        end
+      end
+    end
+  end
+end

data/lib/spandx/cli/printers/csv.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Cli
+    module Printers
+      class Csv < Printer
+        def match?(format)
+          format.to_sym == :csv
+        end
+
+        def print_line(dependency, io)
+          io.puts(CSV.generate_line(dependency.to_a))
+        end
+      end
+    end
+  end
+end

data/lib/spandx/cli/printers/json.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Cli
+    module Printers
+      class Json < Printer
+        def match?(format)
+          format.to_sym == :json
+        end
+
+        def print_line(dependency, io)
+          io.puts(Oj.dump(dependency.to_h))
+        end
+      end
+    end
+  end
+end

data/lib/spandx/cli/printers/table.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Cli
+    module Printers
+      class Table < Printer
+        HEADINGS = ['Name', 'Version', 'Licenses', 'Location'].freeze
+
+        def initialize(output: $stderr)
+          @spinner = TTY::Spinner.new('[:spinner] Scanning...', output: output, clear: true, format: :dots)
+          @spinner.auto_spin
+        end
+
+        def match?(format)
+          format.to_sym == :table
+        end
+
+        def print_header(_io)
+          @dependencies = SortedSet.new
+        end
+
+        def print_line(dependency, _io)
+          @dependencies << dependency
+        end
+
+        def print_footer(io)
+          @spinner.stop
+          @spinner.reset
+          io.puts(to_table(@dependencies.map(&:to_a)))
+        end
+
+        private
+
+        def to_table(rows)
+          Terminal::Table.new(headings: HEADINGS) do |table|
+            rows.each { |row| table.add_row(row) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spandx/core/dependency.rb

@@ -3,46 +3,81 @@
 module Spandx
   module Core
     class Dependency
-      attr_reader :package_manager, :name, :version, :licenses, :meta
+      PACKAGE_MANAGERS = {
+        Spandx::Dotnet::Parsers::Csproj => :nuget,
+        Spandx::Dotnet::Parsers::PackagesConfig => :nuget,
+        Spandx::Dotnet::Parsers::Sln => :nuget,
+        Spandx::Java::Parsers::Maven => :maven,
+        Spandx::Js::Parsers::Npm => :npm,
+        Spandx::Js::Parsers::Yarn => :yarn,
+        Spandx::Php::Parsers::Composer => :composer,
+        Spandx::Python::Parsers::PipfileLock => :pypi,
+        Spandx::Ruby::Parsers::GemfileLock => :rubygems,
+        Spandx::Os::Parsers::Apk => :apk,
+      }.freeze
+      attr_reader :path, :name, :version, :licenses, :meta
 
-      def initialize(package_manager:, name:, version:, licenses: [], meta: {})
-        @package_manager = package_manager
-        @name = name
-        @version = version
-        @licenses = licenses
+      def initialize(name:, version:, path:, meta: {})
+        @path = Pathname.new(path).realpath
+        @name = name || @path.basename.to_s
+        @version = version || @path.mtime.to_i.to_s
+        @licenses = []
         @meta = meta
       end
 
-      def managed_by?(value)
-        package_manager == value&.to_sym
+      def package_manager
+        PACKAGE_MANAGERS[Parser.for(path).class]
       end
 
       def <=>(other)
-        to_s <=> other.to_s
+        return 1 if other.nil?
+
+        score = (name <=> other.name)
+        score = score.zero? ? (version <=> other&.version) : score
+        score.zero? ? (path.to_s <=> other&.path.to_s) : score
       end
 
       def hash
         to_s.hash
       end
 
+      def ==(other)
+        eql?(other)
+      end
+
       def eql?(other)
         to_s == other.to_s
       end
 
       def to_s
-        @to_s ||= [name, version].compact.join(' ')
+        @to_s ||= [name, version, path].compact.join(' ')
       end
 
       def inspect
-        "#<Spandx::Core::Dependency name=#{name}, version=#{version}>"
+        "#<#{self.class} name=#{name} version=#{version} path=#{relative_path}>"
       end
 
       def to_a
-        [name, version, licenses.map(&:id)]
+        [name, version, license_expression, relative_path.to_s]
       end
 
       def to_h
-        { name: name, version: version, licenses: licenses.map(&:id) }
+        {
+          name: name,
+          version: version,
+          licenses: license_expression,
+          path: relative_path.to_s
+        }
+      end
+
+      private
+
+      def relative_path(from: Pathname.pwd)
+        path.relative_path_from(from)
+      end
+
+      def license_expression
+        licenses.map(&:id).join(' AND ')
       end
     end
   end