spandx 0.16.1 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ba0b42d3a015296733d9666c1d4b3b4bf89a86622941dd608eca9eef258f4e7a
-  data.tar.gz: 2dc3760f3e265a751a1b5affabfb13abeceb77d9c5980203fabea0758cc64347
+  metadata.gz: d884c729b0d1cb14391435f68b78a77cfb2a73860b8757d4421bda9ff06276bd
+  data.tar.gz: 2fe44244d7285b7fc5bd9ddf8660b2bd4b435d8908b6c5e67d305181e6356658
 SHA512:
-  metadata.gz: 3e41d8d13f883bf9def9c684ff633c6d78077a25e999be320d5841e885cd3a45df39486a12fbee0eef8f632e32236dd294a1303901a401214b66c0dc387fd531
-  data.tar.gz: 451efa448a0c702593e694e25ef0d62dbe49683b33d304951284642176340a94fd3e355978782ebac0d32a5cf224a962f2c80b3afbb489e1ce0b1204191cc9e4
+  metadata.gz: a6f25542efd0f506dbe65f5dbd179d562fbb37fd9498cffc1cee2ac8a0bfdfabab614269920aa2799736a198529408300277c2d87c7303f3463bd37e0ff67ee8
+  data.tar.gz: ebe7266727203ec7a84e741e2dfa5fa76db8a9da30dbfbb258618adbda6e7463a04174fd0c706c23c65068857f06584eef2fcdd25e05fe1ad0ebf5dbd9489c06

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-Version 0.16.1
+Version 0.17.0
 
 # Changelog
 
@@ -8,6 +8,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+## [0.17.0] - 2020-12-28
+### Added
+- Allow indexing gems from index.rubygems.org.
 
 ## [0.16.1] - 2020-11-19
 ### Fixed
@@ -220,7 +223,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Provide ruby API to the latest SPDX catalogue.
 
-[Unreleased]: https://github.com/spandx/spandx/compare/v0.16.1...HEAD
+[Unreleased]: https://github.com/spandx/spandx/compare/v0.17.0...HEAD
+[0.17.0]: https://github.com/spandx/spandx/compare/v0.16.1...v0.17.0
 [0.16.1]: https://github.com/spandx/spandx/compare/v0.16.0...v0.16.1
 [0.16.0]: https://github.com/spandx/spandx/compare/v0.15.1...v0.16.0
 [0.15.1]: https://github.com/spandx/spandx/compare/v0.15.0...v0.15.1

data/lib/spandx.rb

@@ -11,6 +11,7 @@ require 'nokogiri'
 require 'oj'
 require 'parslet'
 require 'pathname'
+require 'sorted_set'
 require 'yaml'
 require 'zeitwerk'
 require 'spandx/spandx'

data/lib/spandx/cli/commands/build.rb

@@ -5,10 +5,11 @@ module Spandx
     module Commands
       class Build
         INDEXES = {
+          dotnet: Spandx::Dotnet::Index,
           maven: Spandx::Java::Index,
           nuget: Spandx::Dotnet::Index,
-          dotnet: Spandx::Dotnet::Index,
           pypi: Spandx::Python::Index,
+          rubygems: Spandx::Ruby::Index,
         }.freeze
 
         def initialize(options)

data/lib/spandx/cli/main.rb

@@ -12,15 +12,11 @@ module Spandx
       method_option :pull, aliases: '-p', type: :boolean, desc: 'Pull the latest cache before the scan', default: false
       method_option :require, aliases: '-r', type: :string, desc: 'Causes spandx to load the library using require.', default: nil
       def scan(lockfile = Pathname.pwd)
-        if options[:help]
-          invoke :help, ['scan']
-        else
-          Oj.default_options = { mode: :strict }
-          Spandx.airgap = options[:airgap]
-          Spandx.logger = Logger.new(options[:logfile])
-          pull if options[:pull]
-          Spandx::Cli::Commands::Scan.new(lockfile, options).execute
-        end
+        return invoke :help, ['scan'] if options[:help]
+
+        prepare(options)
+        pull if options[:pull]
+        Spandx::Cli::Commands::Scan.new(lockfile, options).execute
       end
 
       desc 'pull', 'Pull the latest offline cache'
@@ -52,6 +48,14 @@ module Spandx
         puts "v#{Spandx::VERSION}"
       end
       map %w[--version -v] => :version
+
+      private
+
+      def prepare(options)
+        Oj.default_options = { mode: :strict }
+        Spandx.airgap = options[:airgap]
+        Spandx.logger = Logger.new(options[:logfile])
+      end
     end
   end
 end

data/lib/spandx/ruby/gateway.rb

@@ -8,8 +8,21 @@ module Spandx
         @http = http
       end
 
+      def each
+        response = http.get('https://index.rubygems.org/versions')
+        return unless http.ok?(response)
+
+        parse_each_from(StringIO.new(response.body)) do |item|
+          yield item
+        end
+      end
+
       def licenses_for(dependency)
-        details_on(dependency.name, dependency.version)['licenses'] || []
+        licenses(dependency.name, dependency.version)
+      end
+
+      def licenses(name, version)
+        details_on(name, version)['licenses'] || []
       end
 
       def matches?(dependency)
@@ -20,6 +33,17 @@ module Spandx
 
       attr_reader :http
 
+      def parse_each_from(io)
+        _created_at = io.readline
+        _triple_dash = io.readline
+        until io.eof?
+          name, versions, _digest = io.readline.split(' ')
+          versions.split(',').each do |version|
+            yield({ name: name, version: version })
+          end
+        end
+      end
+
       def details_on(name, version)
         url = "https://rubygems.org/api/v2/rubygems/#{name}/versions/#{version}.json"
         response = http.get(url, default: {})

data/lib/spandx/ruby/index.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Ruby
+    class Index
+      include Enumerable
+
+      attr_reader :directory, :name, :rubygems
+
+      def initialize(directory:)
+        @directory = directory
+        @name = 'rubygems'
+        @cache = ::Spandx::Core::Cache.new(@name, root: directory)
+        @rubygems = ::Spandx::Ruby::Gateway.new
+      end
+
+      def update!(*)
+        queue = Queue.new
+        [fetch(queue), save(queue)].each(&:join)
+        cache.rebuild_index
+      end
+
+      private
+
+      attr_reader :cache
+
+      def fetch(queue)
+        Thread.new do
+          rubygems.each do |item|
+            queue.enq(
+              item.merge(
+                licenses: rubygems.licenses(item[:name], item[:version])
+              )
+            )
+          end
+          queue.enq(:stop)
+        end
+      end
+
+      def save(queue)
+        Thread.new do
+          loop do
+            item = queue.deq
+            break if item == :stop
+
+            cache.insert(item[:name], item[:version], item[:licenses])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/spandx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spandx
-  VERSION = '0.16.1'
+  VERSION = '0.17.0'
 end

data/spandx.gemspec

@@ -38,6 +38,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'nokogiri', '~> 1.10'
   spec.add_dependency 'oj', '~> 3.10'
   spec.add_dependency 'parslet', '~> 2.0'
+  spec.add_dependency 'sorted_set', '~> 1.0'
   spec.add_dependency 'terminal-table', '~> 1.8'
   spec.add_dependency 'thor'
   spec.add_dependency 'tty-spinner', '~> 0.9'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: spandx
 version: !ruby/object:Gem::Version
-  version: 0.16.1
+  version: 0.17.0
 platform: ruby
 authors:
 - Can Eldem
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-19 00:00:00.000000000 Z
+date: 2020-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -101,6 +101,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: sorted_set
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
@@ -413,6 +427,7 @@ files:
 - lib/spandx/python/pypi.rb
 - lib/spandx/python/source.rb
 - lib/spandx/ruby/gateway.rb
+- lib/spandx/ruby/index.rb
 - lib/spandx/ruby/parsers/gemfile_lock.rb
 - lib/spandx/spdx/catalogue.rb
 - lib/spandx/spdx/composite_license.rb
@@ -443,7 +458,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.3
 signing_key: 
 specification_version: 4
 summary: A ruby interface to the SPDX catalogue.