spandx 0.13.5 → 0.16.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01abc42f6e315aee9f35bf60cdad7a4801ee95ae4a186ef3ee001f2617c9891e
-  data.tar.gz: 78248675cdddbcb197f347239c85016862254a113b17894e4d6ffe7ecd33cddd
+  metadata.gz: ba0b42d3a015296733d9666c1d4b3b4bf89a86622941dd608eca9eef258f4e7a
+  data.tar.gz: 2dc3760f3e265a751a1b5affabfb13abeceb77d9c5980203fabea0758cc64347
 SHA512:
-  metadata.gz: d6d4462c74dc412f9016ff576f55e67bdc9a6b341059d3b372b505f6e7ee730a92da53a4b5d0ab836df298b2cb527d0890c599fdc48f92a848b3d93c6c7d67ab
-  data.tar.gz: fdc618b97c619aa7d8a99b799dc5b1569d8396e9cfbbee5ee91cf7b994335e7fcbd9a5abac03b87a21615b23eef7913d5b59aabf277cacaa3eeac8497d795f38
+  metadata.gz: 3e41d8d13f883bf9def9c684ff633c6d78077a25e999be320d5841e885cd3a45df39486a12fbee0eef8f632e32236dd294a1303901a401214b66c0dc387fd531
+  data.tar.gz: 451efa448a0c702593e694e25ef0d62dbe49683b33d304951284642176340a94fd3e355978782ebac0d32a5cf224a962f2c80b3afbb489e1ce0b1204191cc9e4

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-Version 0.13.5
+Version 0.16.1
 
 # Changelog
 
@@ -8,6 +8,28 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+
+## [0.16.1] - 2020-11-19
+### Fixed
+- Start spinner for table printer only
+
+## [0.16.0] - 2020-11-19
+### Changed
+- Pull smaller license cache.
+- Print index files after building them.
+
+## [0.15.1] - 2020-11-18
+### Fixed
+- Rebuild index after pulling latest cache.
+
+## [0.15.0] - 2020-11-18
+### Added
+- Parse `/var/lib/dpkg/status` file.
+
+## [0.14.0] - 2020-11-14
+### Added
+- Parse `/lib/apk/db/installed` file.
+
 ## [0.13.5] - 2020-05-26
 ### Fixed
 - Process PyPI package urls with single digit versions.
@@ -198,7 +220,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Provide ruby API to the latest SPDX catalogue.
 
-[Unreleased]: https://github.com/spandx/spandx/compare/v0.13.5...HEAD
+[Unreleased]: https://github.com/spandx/spandx/compare/v0.16.1...HEAD
+[0.16.1]: https://github.com/spandx/spandx/compare/v0.16.0...v0.16.1
+[0.16.0]: https://github.com/spandx/spandx/compare/v0.15.1...v0.16.0
+[0.15.1]: https://github.com/spandx/spandx/compare/v0.15.0...v0.15.1
+[0.15.0]: https://github.com/spandx/spandx/compare/v0.14.0...v0.15.0
+[0.14.0]: https://github.com/spandx/spandx/compare/v0.13.5...v0.14.0
 [0.13.5]: https://github.com/spandx/spandx/compare/v0.13.4...v0.13.5
 [0.13.4]: https://github.com/spandx/spandx/compare/v0.13.3...v0.13.4
 [0.13.3]: https://github.com/spandx/spandx/compare/v0.13.2...v0.13.3

data/ext/spandx/spandx.c

@@ -52,4 +52,6 @@ void Init_spandx(void)
   rb_mCore = rb_define_module_under(rb_mSpandx, "Core");
   rb_mCsvParser = rb_define_module_under(rb_mCore, "CsvParser");
   rb_define_module_function(rb_mCsvParser, "parse", parse, 1);
+
+  rb_gc_register_mark_object(rb_mCsvParser);
 }

data/lib/spandx.rb

@@ -45,7 +45,7 @@ module Spandx
       @git ||= {
         cache: ::Spandx::Core::Git.new(url: 'https://github.com/spandx/cache.git'),
         rubygems: ::Spandx::Core::Git.new(url: 'https://github.com/spandx/rubygems-cache.git'),
-        spdx: ::Spandx::Core::Git.new(url: 'https://github.com/spdx/license-list-data.git'),
+        spdx: ::Spandx::Core::Git.new(url: 'https://github.com/spdx/license-list-data.git', default_branch: 'master'),
       }
     end
   end

data/lib/spandx/cli/commands/pull.rb

@@ -4,17 +4,56 @@ module Spandx
   module Cli
     module Commands
       class Pull
+        attr_reader :cache_dir, :rubygems_cache_dir
+
         def initialize(options)
           @options = options
+          @cache_dir = Spandx.git[:cache].root.join('.index')
+          @rubygems_cache_dir = Spandx.git[:rubygems].root.join('.index')
         end
 
-        def execute(output: $stdout)
-          Spandx.git.each_value do |db|
-            output.puts "Updating #{db.url}..."
-            db.update!
+        def execute(output: $stderr)
+          sync(output)
+          build(output, ::Spandx::Core::Dependency::PACKAGE_MANAGERS.values.uniq)
+          index_files_in(cache_dir, rubygems_cache_dir).each do |item|
+            output.puts item.to_s.gsub(Dir.home, '~')
           end
           output.puts 'OK'
         end
+
+        private
+
+        def sync(output)
+          Spandx.git.each_value do |db|
+            with_spinner("Updating #{db.url}...", output: output) do
+              db.update!
+            end
+          end
+        end
+
+        def build(output, sources)
+          with_spinner('Building index...', output: output) do
+            sources.each do |source|
+              Spandx::Core::Cache.new(source, root: cache_dir).rebuild_index
+            end
+            Spandx::Core::Cache.new(:rubygems, root: rubygems_cache_dir).rebuild_index
+          end
+        end
+
+        def with_spinner(message, output:)
+          spinner = TTY::Spinner.new("[:spinner] #{message}", output: output)
+          spinner.auto_spin
+          yield
+          spinner.success('(done)')
+        rescue StandardError => error
+          spinner.error("(#{error.message})")
+        ensure
+          spinner.stop
+        end
+
+        def index_files_in(*dirs)
+          dirs.map { |x| x.glob('**/*.idx') }.flatten.sort
+        end
       end
     end
   end

data/lib/spandx/cli/printers/table.rb

@@ -6,8 +6,8 @@ module Spandx
       class Table < Printer
         HEADINGS = ['Name', 'Version', 'Licenses', 'Location'].freeze
 
-        def initialize
-          @spinner = TTY::Spinner.new(output: $stderr)
+        def initialize(output: $stderr)
+          @spinner = TTY::Spinner.new('[:spinner] Scanning...', output: output, clear: true, format: :dots)
         end
 
         def match?(format)
@@ -15,8 +15,8 @@ module Spandx
         end
 
         def print_header(_io)
-          @dependencies = SortedSet.new
           @spinner.auto_spin
+          @dependencies = SortedSet.new
         end
 
         def print_line(dependency, _io)
@@ -25,6 +25,7 @@ module Spandx
 
         def print_footer(io)
           @spinner.stop
+          @spinner.reset
           io.puts(to_table(@dependencies.map(&:to_a)))
         end
 

data/lib/spandx/core/data_file.rb

@@ -56,6 +56,10 @@ module Spandx
         @index ||= IndexFile.new(self)
       end
 
+      def to_s
+        absolute_path.to_s
+      end
+
       private
 
       def to_csv(array)

data/lib/spandx/core/dependency.rb

@@ -13,6 +13,7 @@ module Spandx
         Spandx::Php::Parsers::Composer => :composer,
         Spandx::Python::Parsers::PipfileLock => :pypi,
         Spandx::Ruby::Parsers::GemfileLock => :rubygems,
+        Spandx::Os::Parsers::Apk => :apk,
       }.freeze
       attr_reader :path, :name, :version, :licenses, :meta
 

data/lib/spandx/core/git.rb

@@ -3,10 +3,11 @@
 module Spandx
   module Core
     class Git
-      attr_reader :root, :url
+      attr_reader :root, :url, :default_branch
 
-      def initialize(url:)
+      def initialize(url:, default_branch: 'main')
         @url = url
+        @default_branch = default_branch
         @root = path_for(url)
       end
 
@@ -31,14 +32,15 @@ module Spandx
         root.join('.git').directory?
       end
 
-      def clone!
+      def clone!(branch: default_branch)
         system('rm', '-rf', root.to_s) if root.exist?
-        system('git', 'clone', '--quiet', '--depth=1', '--single-branch', '--branch', 'master', url, root.to_s)
+        system('git', 'clone', '--quiet', '--depth=1', '--single-branch', '--branch', branch, url, root.to_s)
       end
 
-      def pull!
+      def pull!(remote: 'origin', branch: default_branch)
         Dir.chdir(root) do
-          system('git', 'pull', '--no-rebase', '--quiet', 'origin', 'master')
+          system('git', 'fetch', '--quiet', '--depth=1', '--prune', '--no-tags', remote)
+          system('git', 'checkout', '--quiet', branch)
         end
       end
     end

data/lib/spandx/core/http.rb

@@ -36,12 +36,12 @@ module Spandx
       end
 
       def self.default_driver
-        @default_driver ||= Net::Hippie::Client.new.tap do |client|
-          client.logger = Spandx.logger
-          client.open_timeout = 1
-          client.read_timeout = 5
-          client.follow_redirects = 3
-        end
+        @default_driver ||= Net::Hippie::Client.new(
+          follow_redirects: 3,
+          logger: Spandx.logger,
+          open_timeout: 1,
+          read_timeout: 5
+        )
       end
 
       private

data/lib/spandx/core/license_plugin.rb

@@ -33,7 +33,7 @@ module Spandx
       end
 
       def known?(package_manager)
-        %i[nuget maven rubygems npm yarn pypi composer].include?(package_manager)
+        %i[nuget maven rubygems npm yarn pypi composer apk].include?(package_manager)
       end
 
       def gateway_for(dependency)

data/lib/spandx/os/parsers/apk.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Os
+    module Parsers
+      class Apk < ::Spandx::Core::Parser
+        def match?(path)
+          path.basename.fnmatch?('installed')
+        end
+
+        def parse(lockfile)
+          path = lockfile.to_s
+
+          [].tap do |items|
+            lockfile.open(mode: 'r') do |io|
+              each_package(io) do |data|
+                items.push(map_from(data, path))
+              end
+            end
+          end
+        end
+
+        private
+
+        def each_package(io)
+          package = {}
+
+          until io.eof?
+            line = io.readline.chomp
+            if line.empty?
+              yield package
+
+              package = {}
+            else
+              line.split(':').tap { |(key, value)| package[key] = value }
+            end
+          end
+        end
+
+        def map_from(data, path)
+          ::Spandx::Core::Dependency.new(
+            path: path,
+            name: data['P'],
+            version: data['V'],
+            meta: data.merge('license' => [data['L']])
+          )
+        end
+      end
+    end
+  end
+end

data/lib/spandx/os/parsers/dpkg.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Os
+    module Parsers
+      class Dpkg < ::Spandx::Core::Parser
+        class LineReader
+          attr_reader :io
+
+          def initialize(io)
+            @io = io
+          end
+
+          def each
+            yield read_package(io, Hash.new(''), nil) until io.eof?
+          end
+
+          private
+
+          def read_package(io, package, prev_key)
+            return package if io.eof?
+
+            line = io.readline.chomp
+            return package if line.empty?
+
+            key, value = split(line, prev_key)
+            package[key] += value
+            read_package(io, package, key)
+          end
+
+          def split(line, prev_key)
+            if prev_key && line.start_with?(' ')
+              [prev_key, line]
+            else
+              key, *rest = line.split(':')
+              value = rest&.join(':')&.strip
+              [key, value]
+            end
+          end
+        end
+
+        def match?(path)
+          path.basename.fnmatch?('status')
+        end
+
+        def parse(lockfile)
+          [].tap do |items|
+            lockfile.open(mode: 'r') do |io|
+              LineReader.new(io).each do |data|
+                items.push(map_from(data, lockfile.to_s))
+              end
+            end
+          end
+        end
+
+        private
+
+        def map_from(data, path)
+          ::Spandx::Core::Dependency.new(
+            path: path,
+            name: data['Package'],
+            version: data['Version'],
+            meta: data
+          )
+        end
+      end
+    end
+  end
+end

data/lib/spandx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spandx
-  VERSION = '0.13.5'
+  VERSION = '0.16.1'
 end

data/spandx.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = 'https://github.com/spandx/spandx'
-  spec.metadata['changelog_uri'] = 'https://github.com/spandx/spandx/blob/master/CHANGELOG.md'
+  spec.metadata['changelog_uri'] = 'https://github.com/spandx/spandx/blob/main/CHANGELOG.md'
 
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
     Dir.glob('exe/*') +
@@ -34,7 +34,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'addressable', '~> 2.7'
   spec.add_dependency 'bundler', '>= 1.16', '< 3.0.0'
-  spec.add_dependency 'net-hippie', '~> 0.3'
+  spec.add_dependency 'net-hippie', '~> 1.0'
   spec.add_dependency 'nokogiri', '~> 1.10'
   spec.add_dependency 'oj', '~> 3.10'
   spec.add_dependency 'parslet', '~> 2.0'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: spandx
 version: !ruby/object:Gem::Version
-  version: 0.13.5
+  version: 0.16.1
 platform: ruby
 authors:
 - Can Eldem
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-06-23 00:00:00.000000000 Z
+date: 2020-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -51,14 +51,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -404,6 +404,8 @@ files:
 - lib/spandx/js/parsers/yarn.rb
 - lib/spandx/js/yarn_lock.rb
 - lib/spandx/js/yarn_pkg.rb
+- lib/spandx/os/parsers/apk.rb
+- lib/spandx/os/parsers/dpkg.rb
 - lib/spandx/php/packagist_gateway.rb
 - lib/spandx/php/parsers/composer.rb
 - lib/spandx/python/index.rb
@@ -425,7 +427,7 @@ licenses:
 metadata:
   homepage_uri: https://spandx.github.io/
   source_code_uri: https://github.com/spandx/spandx
-  changelog_uri: https://github.com/spandx/spandx/blob/master/CHANGELOG.md
+  changelog_uri: https://github.com/spandx/spandx/blob/main/CHANGELOG.md
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -441,7 +443,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: A ruby interface to the SPDX catalogue.