spandx 0.15.0 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee3b447710888b33bf24ecf467ce1bf41e3cf6f2c91d2c3c381f256d0fa1ea6f
-  data.tar.gz: cd802eabfd2f0e383ae198217992ce465c28ac7d53587264ca1a2f4746e9e9ce
+  metadata.gz: d25ff66ba00edb25cf87b9d9af71ed9433bf084a819a80f86676714754359b34
+  data.tar.gz: 59e6783d2cba9287e0a65836aeae6705434fc0c9ccdfb093763d83b23d673207
 SHA512:
-  metadata.gz: ed0fec87962da7ad1576131d6a463324deeaa9ea2b9a6b289dc9c639d1eb9dda98e2bd7aa02df2f9d40526fc4793cedafe3311c9c5fb6c6aedba5594b68fcc1f
-  data.tar.gz: f6bdc294db3b95093c8e3872bb5113879c0a12f9d45ec718557281ff9397f925d0b0a443ba41fad0c5a48e86c77babf95b855b7315d795ec8e869bf8c1dbd6c4
+  metadata.gz: c75d55cc57a24d00912a98d0a339b22f21e04fb0c84459ce9b769b900cce83f5877557cbe70e2cdff522d1869ba4d89688b9e9d67913719247cc53897216cb01
+  data.tar.gz: 770105bb92091a740afc3d886194712eb54908718af73e2769acfd512afe553d3f00de0e1ada81930b3d5c09a0a6053dfcc3cd8c9f48ecdb76673e1534ad9fff

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-Version 0.15.0
+Version 0.18.0
 
 # Changelog
 
@@ -9,6 +9,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.18.0] - 2021-05-10
+### Added
+- Add support for parsing `.terraform.lock.hcl` files.
+
+## [0.17.0] - 2020-12-28
+### Added
+- Allow indexing gems from index.rubygems.org.
+
+## [0.16.1] - 2020-11-19
+### Fixed
+- Start spinner for table printer only
+
+## [0.16.0] - 2020-11-19
+### Changed
+- Pull smaller license cache.
+- Print index files after building them.
+
+## [0.15.1] - 2020-11-18
+### Fixed
+- Rebuild index after pulling latest cache.
+
 ## [0.15.0] - 2020-11-18
 ### Added
 - Parse `/var/lib/dpkg/status` file.
@@ -207,7 +228,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Provide ruby API to the latest SPDX catalogue.
 
-[Unreleased]: https://github.com/spandx/spandx/compare/v0.15.0...HEAD
+[Unreleased]: https://github.com/spandx/spandx/compare/v0.18.0...HEAD
+[0.18.0]: https://github.com/spandx/spandx/compare/v0.17.0...v0.18.0
+[0.17.0]: https://github.com/spandx/spandx/compare/v0.16.1...v0.17.0
+[0.16.1]: https://github.com/spandx/spandx/compare/v0.16.0...v0.16.1
+[0.16.0]: https://github.com/spandx/spandx/compare/v0.15.1...v0.16.0
+[0.15.1]: https://github.com/spandx/spandx/compare/v0.15.0...v0.15.1
 [0.15.0]: https://github.com/spandx/spandx/compare/v0.14.0...v0.15.0
 [0.14.0]: https://github.com/spandx/spandx/compare/v0.13.5...v0.14.0
 [0.13.5]: https://github.com/spandx/spandx/compare/v0.13.4...v0.13.5

data/README.md

@@ -4,7 +4,7 @@
 
 # Spandx ![badge](https://github.com/spandx/spandx/workflows/ci/badge.svg)
 
-A ruby API for interacting with the https://spdx.org software license catalogue.
+A Ruby API for interacting with the https://spdx.org software license catalogue.
 This gem includes a command line interface to scan a software project for the
 software licenses that are associated with each dependency in the project.
 `spandx` leverages an offline cache of software licenses for known dependencies.
@@ -104,7 +104,7 @@ end
 
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/cibuild` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 

data/ext/spandx/spandx.c

@@ -52,4 +52,6 @@ void Init_spandx(void)
   rb_mCore = rb_define_module_under(rb_mSpandx, "Core");
   rb_mCsvParser = rb_define_module_under(rb_mCore, "CsvParser");
   rb_define_module_function(rb_mCsvParser, "parse", parse, 1);
+
+  rb_gc_register_mark_object(rb_mCsvParser);
 }

data/lib/spandx.rb

@@ -11,6 +11,7 @@ require 'nokogiri'
 require 'oj'
 require 'parslet'
 require 'pathname'
+require 'sorted_set'
 require 'yaml'
 require 'zeitwerk'
 require 'spandx/spandx'
@@ -45,7 +46,7 @@ module Spandx
       @git ||= {
         cache: ::Spandx::Core::Git.new(url: 'https://github.com/spandx/cache.git'),
         rubygems: ::Spandx::Core::Git.new(url: 'https://github.com/spandx/rubygems-cache.git'),
-        spdx: ::Spandx::Core::Git.new(url: 'https://github.com/spdx/license-list-data.git'),
+        spdx: ::Spandx::Core::Git.new(url: 'https://github.com/spdx/license-list-data.git', default_branch: 'master'),
       }
     end
   end

data/lib/spandx/cli/commands/build.rb

@@ -5,10 +5,11 @@ module Spandx
     module Commands
       class Build
         INDEXES = {
+          dotnet: Spandx::Dotnet::Index,
           maven: Spandx::Java::Index,
           nuget: Spandx::Dotnet::Index,
-          dotnet: Spandx::Dotnet::Index,
           pypi: Spandx::Python::Index,
+          rubygems: Spandx::Ruby::Index,
         }.freeze
 
         def initialize(options)

data/lib/spandx/cli/commands/pull.rb

@@ -4,17 +4,56 @@ module Spandx
   module Cli
     module Commands
       class Pull
+        attr_reader :cache_dir, :rubygems_cache_dir
+
         def initialize(options)
           @options = options
+          @cache_dir = Spandx.git[:cache].root.join('.index')
+          @rubygems_cache_dir = Spandx.git[:rubygems].root.join('.index')
         end
 
-        def execute(output: $stdout)
-          Spandx.git.each_value do |db|
-            output.puts "Updating #{db.url}..."
-            db.update!
+        def execute(output: $stderr)
+          sync(output)
+          build(output, ::Spandx::Core::Dependency::PACKAGE_MANAGERS.values.uniq)
+          index_files_in(cache_dir, rubygems_cache_dir).each do |item|
+            output.puts item.to_s.gsub(Dir.home, '~')
           end
           output.puts 'OK'
         end
+
+        private
+
+        def sync(output)
+          Spandx.git.each_value do |db|
+            with_spinner("Updating #{db.url}...", output: output) do
+              db.update!
+            end
+          end
+        end
+
+        def build(output, sources)
+          with_spinner('Building index...', output: output) do
+            sources.each do |source|
+              Spandx::Core::Cache.new(source, root: cache_dir).rebuild_index
+            end
+            Spandx::Core::Cache.new(:rubygems, root: rubygems_cache_dir).rebuild_index
+          end
+        end
+
+        def with_spinner(message, output:)
+          spinner = TTY::Spinner.new("[:spinner] #{message}", output: output)
+          spinner.auto_spin
+          yield
+          spinner.success('(done)')
+        rescue StandardError => error
+          spinner.error("(#{error.message})")
+        ensure
+          spinner.stop
+        end
+
+        def index_files_in(*dirs)
+          dirs.map { |x| x.glob('**/*.idx') }.flatten.sort
+        end
       end
     end
   end

data/lib/spandx/cli/main.rb

@@ -12,15 +12,11 @@ module Spandx
       method_option :pull, aliases: '-p', type: :boolean, desc: 'Pull the latest cache before the scan', default: false
       method_option :require, aliases: '-r', type: :string, desc: 'Causes spandx to load the library using require.', default: nil
       def scan(lockfile = Pathname.pwd)
-        if options[:help]
-          invoke :help, ['scan']
-        else
-          Oj.default_options = { mode: :strict }
-          Spandx.airgap = options[:airgap]
-          Spandx.logger = Logger.new(options[:logfile])
-          pull if options[:pull]
-          Spandx::Cli::Commands::Scan.new(lockfile, options).execute
-        end
+        return invoke :help, ['scan'] if options[:help]
+
+        prepare(options)
+        pull if options[:pull]
+        Spandx::Cli::Commands::Scan.new(lockfile, options).execute
       end
 
       desc 'pull', 'Pull the latest offline cache'
@@ -52,6 +48,14 @@ module Spandx
         puts "v#{Spandx::VERSION}"
       end
       map %w[--version -v] => :version
+
+      private
+
+      def prepare(options)
+        Oj.default_options = { mode: :strict }
+        Spandx.airgap = options[:airgap]
+        Spandx.logger = Logger.new(options[:logfile])
+      end
     end
   end
 end

data/lib/spandx/cli/printers/table.rb

@@ -6,8 +6,8 @@ module Spandx
       class Table < Printer
         HEADINGS = ['Name', 'Version', 'Licenses', 'Location'].freeze
 
-        def initialize
-          @spinner = TTY::Spinner.new(output: $stderr)
+        def initialize(output: $stderr)
+          @spinner = TTY::Spinner.new('[:spinner] Scanning...', output: output, clear: true, format: :dots)
         end
 
         def match?(format)
@@ -15,8 +15,8 @@ module Spandx
         end
 
         def print_header(_io)
-          @dependencies = SortedSet.new
           @spinner.auto_spin
+          @dependencies = SortedSet.new
         end
 
         def print_line(dependency, _io)
@@ -25,6 +25,7 @@ module Spandx
 
         def print_footer(io)
           @spinner.stop
+          @spinner.reset
           io.puts(to_table(@dependencies.map(&:to_a)))
         end
 

data/lib/spandx/core/data_file.rb

@@ -56,6 +56,10 @@ module Spandx
         @index ||= IndexFile.new(self)
       end
 
+      def to_s
+        absolute_path.to_s
+      end
+
       private
 
       def to_csv(array)

data/lib/spandx/core/git.rb

@@ -3,10 +3,11 @@
 module Spandx
   module Core
     class Git
-      attr_reader :root, :url
+      attr_reader :root, :url, :default_branch
 
-      def initialize(url:)
+      def initialize(url:, default_branch: 'main')
         @url = url
+        @default_branch = default_branch
         @root = path_for(url)
       end
 
@@ -31,14 +32,15 @@ module Spandx
         root.join('.git').directory?
       end
 
-      def clone!
+      def clone!(branch: default_branch)
         system('rm', '-rf', root.to_s) if root.exist?
-        system('git', 'clone', '--quiet', '--depth=1', '--single-branch', '--branch', 'master', url, root.to_s)
+        system('git', 'clone', '--quiet', '--depth=1', '--single-branch', '--branch', branch, url, root.to_s)
       end
 
-      def pull!
+      def pull!(remote: 'origin', branch: default_branch)
         Dir.chdir(root) do
-          system('git', 'pull', '--no-rebase', '--quiet', 'origin', 'master')
+          system('git', 'fetch', '--quiet', '--depth=1', '--prune', '--no-tags', remote)
+          system('git', 'checkout', '--quiet', branch)
         end
       end
     end

data/lib/spandx/ruby/gateway.rb

@@ -8,8 +8,21 @@ module Spandx
         @http = http
       end
 
+      def each
+        response = http.get('https://index.rubygems.org/versions')
+        return unless http.ok?(response)
+
+        parse_each_from(StringIO.new(response.body)) do |item|
+          yield item
+        end
+      end
+
       def licenses_for(dependency)
-        details_on(dependency.name, dependency.version)['licenses'] || []
+        licenses(dependency.name, dependency.version)
+      end
+
+      def licenses(name, version)
+        details_on(name, version)['licenses'] || []
       end
 
       def matches?(dependency)
@@ -20,6 +33,17 @@ module Spandx
 
       attr_reader :http
 
+      def parse_each_from(io)
+        _created_at = io.readline
+        _triple_dash = io.readline
+        until io.eof?
+          name, versions, _digest = io.readline.split(' ')
+          versions.split(',').each do |version|
+            yield({ name: name, version: version })
+          end
+        end
+      end
+
       def details_on(name, version)
         url = "https://rubygems.org/api/v2/rubygems/#{name}/versions/#{version}.json"
         response = http.get(url, default: {})

data/lib/spandx/ruby/index.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Ruby
+    class Index
+      include Enumerable
+
+      attr_reader :directory, :name, :rubygems
+
+      def initialize(directory:)
+        @directory = directory
+        @name = 'rubygems'
+        @cache = ::Spandx::Core::Cache.new(@name, root: directory)
+        @rubygems = ::Spandx::Ruby::Gateway.new
+      end
+
+      def update!(*)
+        queue = Queue.new
+        [fetch(queue), save(queue)].each(&:join)
+        cache.rebuild_index
+      end
+
+      private
+
+      attr_reader :cache
+
+      def fetch(queue)
+        Thread.new do
+          rubygems.each do |item|
+            queue.enq(
+              item.merge(
+                licenses: rubygems.licenses(item[:name], item[:version])
+              )
+            )
+          end
+          queue.enq(:stop)
+        end
+      end
+
+      def save(queue)
+        Thread.new do
+          loop do
+            item = queue.deq
+            break if item == :stop
+
+            cache.insert(item[:name], item[:version], item[:licenses])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spandx/terraform/parsers/hcl.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Terraform
+    module Parsers
+      class Hcl < Parslet::Parser
+        rule(:alpha) { match['a-zA-Z'] }
+        rule(:assign) { str('=') }
+        rule(:comma) { str(',') }
+        rule(:comment) { (str('#') | str('//')) >> ((str("\n") >> str("\r").maybe).absent? >> any).repeat >> eol }
+        rule(:crlf) { match('[\r\n]') }
+        rule(:digit) { match('\d') }
+        rule(:dot) { str('.') }
+        rule(:eol) { whitespace? >> crlf.repeat }
+        rule(:greater_than_or_equal_to) { str('>=') }
+        rule(:hyphen) { str('-') }
+        rule(:lbracket) { str('[') }
+        rule(:lcurly) { str('{') }
+        rule(:major) { number }
+        rule(:major_minor) { (number >> dot >> number) }
+        rule(:major_minor_patch) { number >> dot >> number >> dot >> number }
+        rule(:multiline_comment) { str('/*') >> (str('*/').absent? >> any).repeat >> str('*/') }
+        rule(:number) { digit.repeat }
+        rule(:pre_release) { hyphen >> (alpha | digit).repeat }
+        rule(:pre_release?) { pre_release.maybe }
+        rule(:quote) { str('"') }
+        rule(:rbracket) { str(']') }
+        rule(:rcurly) { str('}') }
+        rule(:space) { match('\s') }
+        rule(:tilda_wacka) { str('~>') }
+        rule(:version) { number >> dot >> number >> dot >> number >> pre_release? }
+        rule(:whitespace) { (multiline_comment | comment | space).repeat }
+        rule(:whitespace?) { whitespace.maybe }
+
+        rule(:pessimistic_version_constraint) do
+          tilda_wacka >> whitespace >> (
+            major_minor_patch |
+            major_minor |
+            major
+          )
+        end
+
+        rule(:greater_than_or_equal_to_version) do
+          greater_than_or_equal_to >> whitespace >> (
+            major_minor_patch |
+            major_minor |
+            major
+          )
+        end
+
+        rule(:version_constraint) do
+          pessimistic_version_constraint | greater_than_or_equal_to_version
+        end
+
+        rule :version_assignment do
+          str('version') >> whitespace >> assign >> whitespace >> quote >> version.as(:version) >> quote
+        end
+
+        rule :constraint_assignment do
+          str('constraints') >> whitespace >> assign >> whitespace >> quote >> version_constraint.as(:constraints) >> quote
+        end
+
+        rule :string do
+          quote >> match('[0-9A-Za-z.~> :=/]').repeat.as(:value) >> quote
+        end
+
+        rule :array_item do
+          whitespace >> string >> comma >> eol
+        end
+
+        rule :array do
+          lbracket >> eol >> array_item.repeat >> rbracket
+        end
+
+        rule :argument do
+          alpha.repeat.as(:name) >> whitespace >> assign >> whitespace >> (array.as(:values) | string)
+        end
+
+        rule :arguments do
+          (argument >> eol).repeat
+        end
+
+        rule :identifier do
+          whitespace >> quote >> ((alpha | match('[./]')).repeat).as(:name) >> quote >> whitespace
+        end
+
+        rule :block_body do
+          arguments.as(:arguments)
+        end
+
+        rule :block do
+          whitespace? >> (alpha.repeat).as(:type) >> identifier >> whitespace >> lcurly >> eol >> block_body >> rcurly >> eol
+        end
+
+        rule :blocks do
+          block.repeat.as(:blocks)
+        end
+
+        root(:blocks)
+      end
+    end
+  end
+end

data/lib/spandx/terraform/parsers/lock_file.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Terraform
+    module Parsers
+      class LockFile < ::Spandx::Core::Parser
+        def initialize
+          @parser = Spandx::Terraform::Parsers::Hcl.new
+        end
+
+        def match?(pathname)
+          basename = pathname.basename
+          basename.fnmatch?('.terraform.lock.hcl')
+        end
+
+        def parse(path)
+          tree = @parser.parse(path.read)
+          tree[:blocks].map do |block|
+            version_arg = version_arg_from(block)
+            ::Spandx::Core::Dependency.new(
+              name: block[:name].to_s,
+              version: version_arg[:value]&.to_s,
+              path: path
+            )
+          end
+        end
+
+        private
+
+        def version_arg_from(block)
+          block[:arguments].find do |x|
+            x[:name] == 'version'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spandx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spandx
-  VERSION = '0.15.0'
+  VERSION = '0.18.0'
 end

data/spandx.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = 'https://github.com/spandx/spandx'
-  spec.metadata['changelog_uri'] = 'https://github.com/spandx/spandx/blob/master/CHANGELOG.md'
+  spec.metadata['changelog_uri'] = 'https://github.com/spandx/spandx/blob/main/CHANGELOG.md'
 
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
     Dir.glob('exe/*') +
@@ -38,6 +38,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'nokogiri', '~> 1.10'
   spec.add_dependency 'oj', '~> 3.10'
   spec.add_dependency 'parslet', '~> 2.0'
+  spec.add_dependency 'sorted_set', '~> 1.0'
   spec.add_dependency 'terminal-table', '~> 1.8'
   spec.add_dependency 'thor'
   spec.add_dependency 'tty-spinner', '~> 0.9'

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: spandx
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.18.0
 platform: ruby
 authors:
 - Can Eldem
 - mo khan
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-11-18 00:00:00.000000000 Z
+date: 2021-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -101,6 +101,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: sorted_set
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
@@ -413,12 +427,15 @@ files:
 - lib/spandx/python/pypi.rb
 - lib/spandx/python/source.rb
 - lib/spandx/ruby/gateway.rb
+- lib/spandx/ruby/index.rb
 - lib/spandx/ruby/parsers/gemfile_lock.rb
 - lib/spandx/spdx/catalogue.rb
 - lib/spandx/spdx/composite_license.rb
 - lib/spandx/spdx/expression.rb
 - lib/spandx/spdx/gateway.rb
 - lib/spandx/spdx/license.rb
+- lib/spandx/terraform/parsers/hcl.rb
+- lib/spandx/terraform/parsers/lock_file.rb
 - lib/spandx/version.rb
 - spandx.gemspec
 homepage: https://spandx.github.io/
@@ -427,8 +444,8 @@ licenses:
 metadata:
   homepage_uri: https://spandx.github.io/
   source_code_uri: https://github.com/spandx/spandx
-  changelog_uri: https://github.com/spandx/spandx/blob/master/CHANGELOG.md
-post_install_message: 
+  changelog_uri: https://github.com/spandx/spandx/blob/main/CHANGELOG.md
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -443,8 +460,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.2.3
+signing_key:
 specification_version: 4
 summary: A ruby interface to the SPDX catalogue.
 test_files: []