spandx 0.13.1 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 288989db458fbe8da46cf782ab6d0410dbcfe5f167a4fa258b2fb0bdd819b2c3
-  data.tar.gz: 3df236283d2d8149bb9c6c9633969540147cd31cc6dfdabb4c26b56d3a6d5b92
+  metadata.gz: 7e01f7023f4a164fb867c7d457769d1c9dd2eb2b480cee88c5d4d682c2d6dc4e
+  data.tar.gz: f202f85c254d11041b79e1305d12641eb66ea66cec9afea25a38e9724a5636d6
 SHA512:
-  metadata.gz: 7c6bb83298a7298814f484236df79234aac9710cf990eb728db53465e1ec82943085a9057bc7d29e8f49ce74e76a6229606f1392bb13835db13ebbbfa18f173d
-  data.tar.gz: a95189cec4d7b0d5f4a6e8007bb79b805fe5072bf5a502c995140cfa5b495fb774e9e36fc2c6dbc17190ce8641fc9eb4ca1485e8dd033c70fce7960382a5d346
+  metadata.gz: 926df592dfc76466a7e26bcdfd9fc581957b2c748c9272e30a22180a61f4d6498ebb14e04e8acbbccc813f4aae929ecb8ba82ee54064d642990e5874b05bb0b1
+  data.tar.gz: 5dede807761bf9d4fa91f6a0ea9df1bec9531d0a397fedea5c687b7c12860216e41df47c09fc7e94c8951e7dc0f88c99fc74395913002890b199409b700830f0

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-Version 0.13.1
+Version 0.14.0
 
 # Changelog
 
@@ -9,6 +9,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.14.0] - 2020-11-14
+### Added
+- Parse `/lib/apk/db/installed` file.
+
+## [0.13.5] - 2020-05-26
+### Fixed
+- Process PyPI package urls with single digit versions.
+- Remove unsupported `hash` report from help text.
+
+### Changed
+- Stream output to output stream as soon as results are available.
+- Switch to `Oj` for JSON parsing.
+- Run spinner on background thread.
+
+## [0.13.4] - 2020-05-26
+### Added
+- Add detected file path to report output.
+
+### Changed
+- Use `Pathname` instead of `String` to represent file paths.
+- Scan current directory when a path is not specified.
+
+## [0.13.3] - 2020-05-19
+### Fixed
+- Ignore invalid URLs during scan.
+
+## [0.13.2] - 2020-05-17
+### Fixed
+- Detect licenses when provided as an array.
+- Skip empty lockfiles.
+
 ## [0.13.1] - 2020-05-16
 ### Fixed
 - Add `ext/**/*.c` and `ext/**/*.h` to list of files.
@@ -172,7 +203,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Provide ruby API to the latest SPDX catalogue.
 
-[Unreleased]: https://github.com/spandx/spandx/compare/v0.13.0...HEAD
+[Unreleased]: https://github.com/spandx/spandx/compare/v0.14.0...HEAD
+[0.14.0]: https://github.com/spandx/spandx/compare/v0.13.5...v0.14.0
+[0.13.5]: https://github.com/spandx/spandx/compare/v0.13.4...v0.13.5
+[0.13.4]: https://github.com/spandx/spandx/compare/v0.13.3...v0.13.4
+[0.13.3]: https://github.com/spandx/spandx/compare/v0.13.2...v0.13.3
+[0.13.2]: https://github.com/spandx/spandx/compare/v0.13.1...v0.13.2
+[0.13.1]: https://github.com/spandx/spandx/compare/v0.13.0...v0.13.1
 [0.13.0]: https://github.com/spandx/spandx/compare/v0.12.3...v0.13.0
 [0.12.3]: https://github.com/spandx/spandx/compare/v0.12.2...v0.12.3
 [0.12.2]: https://github.com/spandx/spandx/compare/v0.12.1...v0.12.2

data/exe/spandx

@@ -4,7 +4,6 @@
 require 'spandx'
 
 Signal.trap('INT') do
-  warn("\n#{caller.join("\n")}: interrupted")
   exit(1)
 end
 

data/ext/spandx/spandx.c

@@ -1,5 +1,7 @@
 #include "spandx.h"
 
+#define NEWLINE 10
+
 VALUE rb_mSpandx;
 VALUE rb_mCore;
 VALUE rb_mCsvParser;
@@ -9,7 +11,7 @@ VALUE rb_mCsvParser;
 // "name","version","license"\r
 // "name","version","license"\r\n
 // "name","version",""\r\n
-static VALUE parse(VALUE self, VALUE line)
+VALUE parse(VALUE self, VALUE line)
 {
   if (NIL_P(line)) return Qnil;
 
@@ -20,7 +22,7 @@ static VALUE parse(VALUE self, VALUE line)
 
   const VALUE items = rb_ary_new2(3);
   const char *s, *n;
-  const int len = RSTRING_LEN(line);
+  const long len = RSTRING_LEN(line);
   enum { open, closed } state = closed;
 
   for (int i = 0; i < len && *p; i++) {
@@ -32,13 +34,13 @@ static VALUE parse(VALUE self, VALUE line)
         s = n;
         state = open;
       } else if (state == open) {
-        if (!*n || n == p || *n == ',' || *n == 10) {
+        if (!*n || n == p || *n == ',' || *n == NEWLINE) {
           rb_ary_push(items, rb_str_new(s, p - s));
           state = closed;
         }
       }
     }
-    *p++;
+    *(p++);
   }
 
   return items;

data/lib/spandx.rb

@@ -8,11 +8,11 @@ require 'json'
 require 'logger'
 require 'net/hippie'
 require 'nokogiri'
+require 'oj'
 require 'parslet'
 require 'pathname'
 require 'yaml'
 require 'zeitwerk'
-
 require 'spandx/spandx'
 
 loader = Zeitwerk::Loader.for_gem

data/lib/spandx/cli.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
 require 'thor'
-require 'tty-progressbar'
+require 'tty-spinner'
+require 'terminal-table'
 
 module Spandx
   module Cli

data/lib/spandx/cli/commands/scan.rb

@@ -4,10 +4,7 @@ module Spandx
   module Cli
     module Commands
       class Scan
-        NULL_BAR = Class.new do
-          def advance(*args); end
-        end.new
-
+        include Spandx::Core
         attr_reader :scan_path
 
         def initialize(scan_path, options)
@@ -17,32 +14,24 @@ module Spandx
         end
 
         def execute(output: $stdout)
-          Spandx::Core::ThreadPool.open do |pool|
-            report = ::Spandx::Core::Report.new
-            each_file do |file|
-              each_dependency_from(file, pool) do |dependency|
-                report.add(dependency)
-              end
+          with_printer(output) do |printer|
+            each_dependency do |dependency|
+              printer.print_line(Plugin.enhance(dependency), output)
             end
-            output.puts(format(report.to(@options[:format])))
           end
         end
 
         private
 
         def each_file
-          Spandx::Core::PathTraversal
-            .new(scan_path, recursive: @options['recursive'])
+          PathTraversal
+            .new(scan_path, recursive: @options[:recursive])
             .each { |file| yield file }
         end
 
-        def each_dependency_from(file, pool)
-          dependencies = ::Spandx::Core::Parser.for(file).parse(file)
-          with_progress(title_for(file), dependencies.size) do |bar|
-            ::Spandx::Core::Concurrent
-              .map(dependencies, pool: pool) { |dependency| enhance(dependency) }
-              .each do |dependency|
-              bar.advance(1)
+        def each_dependency
+          each_file do |file|
+            Parser.parse(file).each do |dependency|
               yield dependency
             end
           end
@@ -52,18 +41,12 @@ module Spandx
           Array(output).map(&:to_s)
         end
 
-        def enhance(dependency)
-          ::Spandx::Core::Plugin
-            .all
-            .inject(dependency) { |memo, plugin| plugin.enhance(memo) }
-        end
-
-        def title_for(file)
-          "#{file} [:bar, :elapsed] :percent"
-        end
-
-        def with_progress(title, total)
-          yield @options[:show_progress] ? TTY::ProgressBar.new(title, total: total) : NULL_BAR
+        def with_printer(output)
+          printer = ::Spandx::Cli::Printer.for(@options[:format])
+          printer.print_header(output)
+          yield printer
+        ensure
+          printer.print_footer(output)
         end
       end
     end

data/lib/spandx/cli/main.rb

@@ -8,14 +8,14 @@ module Spandx
       method_option :recursive, aliases: '-R', type: :boolean, desc: 'Perform recursive scan', default: false
       method_option :airgap, aliases: '-a', type: :boolean, desc: 'Disable network connections', default: false
       method_option :logfile, aliases: '-l', type: :string, desc: 'Path to a logfile', default: '/dev/null'
-      method_option :format, aliases: '-f', type: :string, desc: 'Format of report', default: 'table'
+      method_option :format, aliases: '-f', type: :string, desc: 'Format of report. (table, csv, json)', default: 'table'
       method_option :pull, aliases: '-p', type: :boolean, desc: 'Pull the latest cache before the scan', default: false
       method_option :require, aliases: '-r', type: :string, desc: 'Causes spandx to load the library using require.', default: nil
-      method_option :show_progress, aliases: '-sp', type: :boolean, desc: 'Shows a progress bar', default: true
-      def scan(lockfile)
+      def scan(lockfile = Pathname.pwd)
         if options[:help]
           invoke :help, ['scan']
         else
+          Oj.default_options = { mode: :strict }
           Spandx.airgap = options[:airgap]
           Spandx.logger = Logger.new(options[:logfile])
           pull if options[:pull]

data/lib/spandx/cli/printer.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Cli
+    class Printer
+      def match?(_format)
+        raise ::Spandx::Error, :match?
+      end
+
+      def print_header(io); end
+
+      def print_line(dependency, io)
+        io.puts(dependency.to_s)
+      end
+
+      def print_footer(io); end
+
+      class << self
+        include Core::Registerable
+
+        def for(format)
+          find { |x| x.match?(format) } || new
+        end
+      end
+    end
+  end
+end

data/lib/spandx/cli/printers/csv.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Cli
+    module Printers
+      class Csv < Printer
+        def match?(format)
+          format.to_sym == :csv
+        end
+
+        def print_line(dependency, io)
+          io.puts(CSV.generate_line(dependency.to_a))
+        end
+      end
+    end
+  end
+end

data/lib/spandx/cli/printers/json.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Cli
+    module Printers
+      class Json < Printer
+        def match?(format)
+          format.to_sym == :json
+        end
+
+        def print_line(dependency, io)
+          io.puts(Oj.dump(dependency.to_h))
+        end
+      end
+    end
+  end
+end

data/lib/spandx/cli/printers/table.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Cli
+    module Printers
+      class Table < Printer
+        HEADINGS = ['Name', 'Version', 'Licenses', 'Location'].freeze
+
+        def initialize
+          @spinner = TTY::Spinner.new(output: $stderr)
+        end
+
+        def match?(format)
+          format.to_sym == :table
+        end
+
+        def print_header(_io)
+          @dependencies = SortedSet.new
+          @spinner.auto_spin
+        end
+
+        def print_line(dependency, _io)
+          @dependencies << dependency
+        end
+
+        def print_footer(io)
+          @spinner.stop
+          io.puts(to_table(@dependencies.map(&:to_a)))
+        end
+
+        private
+
+        def to_table(rows)
+          Terminal::Table.new(headings: HEADINGS) do |table|
+            rows.each { |row| table.add_row(row) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spandx/core/dependency.rb

@@ -3,46 +3,81 @@
 module Spandx
   module Core
     class Dependency
-      attr_reader :package_manager, :name, :version, :licenses, :meta
+      PACKAGE_MANAGERS = {
+        Spandx::Dotnet::Parsers::Csproj => :nuget,
+        Spandx::Dotnet::Parsers::PackagesConfig => :nuget,
+        Spandx::Dotnet::Parsers::Sln => :nuget,
+        Spandx::Java::Parsers::Maven => :maven,
+        Spandx::Js::Parsers::Npm => :npm,
+        Spandx::Js::Parsers::Yarn => :yarn,
+        Spandx::Php::Parsers::Composer => :composer,
+        Spandx::Python::Parsers::PipfileLock => :pypi,
+        Spandx::Ruby::Parsers::GemfileLock => :rubygems,
+        Spandx::Os::Parsers::Apk => :apk,
+      }.freeze
+      attr_reader :path, :name, :version, :licenses, :meta
 
-      def initialize(package_manager:, name:, version:, licenses: [], meta: {})
-        @package_manager = package_manager
-        @name = name
-        @version = version
-        @licenses = licenses
+      def initialize(name:, version:, path:, meta: {})
+        @path = Pathname.new(path).realpath
+        @name = name || @path.basename.to_s
+        @version = version || @path.mtime.to_i.to_s
+        @licenses = []
         @meta = meta
       end
 
-      def managed_by?(value)
-        package_manager == value&.to_sym
+      def package_manager
+        PACKAGE_MANAGERS[Parser.for(path).class]
       end
 
       def <=>(other)
-        to_s <=> other.to_s
+        return 1 if other.nil?
+
+        score = (name <=> other.name)
+        score = score.zero? ? (version <=> other&.version) : score
+        score.zero? ? (path.to_s <=> other&.path.to_s) : score
       end
 
       def hash
         to_s.hash
       end
 
+      def ==(other)
+        eql?(other)
+      end
+
       def eql?(other)
         to_s == other.to_s
       end
 
       def to_s
-        @to_s ||= [name, version].compact.join(' ')
+        @to_s ||= [name, version, path].compact.join(' ')
       end
 
       def inspect
-        "#<Spandx::Core::Dependency name=#{name}, version=#{version}>"
+        "#<#{self.class} name=#{name} version=#{version} path=#{relative_path}>"
       end
 
       def to_a
-        [name, version, licenses.map(&:id)]
+        [name, version, license_expression, relative_path.to_s]
       end
 
       def to_h
-        { name: name, version: version, licenses: licenses.map(&:id) }
+        {
+          name: name,
+          version: version,
+          licenses: license_expression,
+          path: relative_path.to_s
+        }
+      end
+
+      private
+
+      def relative_path(from: Pathname.pwd)
+        path.relative_path_from(from)
+      end
+
+      def license_expression
+        licenses.map(&:id).join(' AND ')
       end
     end
   end

data/lib/spandx/core/git.rb

@@ -11,9 +11,8 @@ module Spandx
       end
 
       def read(path)
-        full_path = File.join(root, path)
-
-        IO.read(full_path) if File.exist?(full_path)
+        full_path = root.join(path)
+        full_path.read if full_path.exist?
       end
 
       def update!
@@ -25,15 +24,16 @@ module Spandx
       def path_for(url)
         uri = URI.parse(url)
         name = uri.path.gsub(/\.git$/, '')
-        File.expand_path(File.join(Dir.home, '.local', 'share', name))
+        Pathname(File.expand_path(File.join(Dir.home, '.local', 'share', name)))
       end
 
       def dotgit?
-        File.directory?(File.join(root, '.git'))
+        root.join('.git').directory?
       end
 
       def clone!
-        system('git', 'clone', '--quiet', '--depth=1', '--single-branch', '--branch', 'master', url, root)
+        system('rm', '-rf', root.to_s) if root.exist?
+        system('git', 'clone', '--quiet', '--depth=1', '--single-branch', '--branch', 'master', url, root.to_s)
       end
 
       def pull!
@@ -42,7 +42,5 @@ module Spandx
         end
       end
     end
-
-    Database = Git
   end
 end