spandx 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 14b8433caf34da68102cdf2db650684c9dbccc2e32509a63a44021978a8d1f6e
-  data.tar.gz: c7c342040a9c5a666708096f30228823a815256587ddd99b9b77fcb7f6856259
+  metadata.gz: 60e89d9935bbf82bb28b848858df9b48a736613600808721b6feddf511ed2f8a
+  data.tar.gz: a3e0396a012747f80b97376c1afbccb646f4e8a7bc379998fb89fed35d1a188b
 SHA512:
-  metadata.gz: 9d916195c87c1162ec60ace64a7c36c5b693c51445361cf11242ca9ceb32c9a8b932d1a2bd65026757c56e556cdc677a738bb42f6dd266652c897737ce2da281
-  data.tar.gz: 92df5d8346276b1998f28bfad73ef26fd6e9bf300589997e5d6ad517e7044974b52b659082acae52e496ea51ff77e3f7bd8d4faa5f4d14d39c20c44a8fb529af
+  metadata.gz: c27a27d68f51fcecd579edbcfd2a1ce591f2d2ca9d647df94280c78aa78e0b5a4af7dfc2b08ca0cccb93d3f512a3989dfc6d11846540d69bd814974df2e55d44
+  data.tar.gz: b01f6403eff92f6438f707806b4b976e651c0765d3b9dae5beb46fd801aeab982e7ca1c3e7315102660006f023fb67b8a66e8984533c58678408613c8fd5c051

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-Version 0.3.0
+Version 0.4.0
 
 # Changelog
 
@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.4.0] - 2020-02-02
+### Added
+- Add command to build offline index of nuget packages and their licenses.
+
 ## [0.3.0] - 2020-01-29
 ### Added
 - Add `pom.xml` parser
@@ -57,7 +61,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Provide ruby API to the latest SPDX catalogue.
 
-[Unreleased]: https://github.com/mokhan/spandx/compare/v0.3.0...HEAD
+[Unreleased]: https://github.com/mokhan/spandx/compare/v0.4.0...HEAD
+[0.4.0]: https://github.com/mokhan/spandx/compare/v0.3.0...v0.4.0
 [0.3.0]: https://github.com/mokhan/spandx/compare/v0.2.0...v0.3.0
 [0.2.0]: https://github.com/mokhan/spandx/compare/v0.1.7...v0.2.0
 [0.1.7]: https://github.com/mokhan/spandx/compare/v0.1.6...v0.1.7

data/lib/spandx/cli.rb

@@ -4,6 +4,7 @@ require 'thor'
 
 require 'spandx'
 require 'spandx/command'
+require 'spandx/commands/build'
 require 'spandx/commands/scan'
 
 module Spandx
@@ -16,7 +17,21 @@ module Spandx
     end
     map %w[--version -v] => :version
 
-    desc 'scan LOCKFILE', 'Command description...'
+    desc 'build', 'Build a package index'
+    method_option :help, aliases: '-h', type: :boolean,
+                         desc: 'Display usage information'
+    method_option :directory, aliases: '-d', type: :string,
+                              desc: 'Directory to build index in'
+    def build(*)
+      if options[:help]
+        invoke :help, ['build']
+      else
+        require_relative 'commands/build'
+        Spandx::Commands::Build.new(options).execute
+      end
+    end
+
+    desc 'scan LOCKFILE', 'Scan a lockfile and list dependencies/licenses'
     method_option :help, aliases: '-h', type: :boolean,
                          desc: 'Display usage information'
     def scan(lockfile = nil)

data/lib/spandx/commands/build.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative '../command'
+
+module Spandx
+  module Commands
+    class Build < Spandx::Command
+      def initialize(options)
+        @options = options
+      end
+
+      def execute(output: $stdout)
+        index = Spandx::Index.new(directory: @options[:directory])
+        gateways.each do |gateway|
+          gateway.update!(index)
+        end
+        output.puts 'OK'
+      end
+
+      private
+
+      def catalogue
+        Spandx::Catalogue.from_git
+      end
+
+      def gateways
+        [
+          Spandx::Gateways::Nuget.new(catalogue: catalogue)
+        ]
+      end
+    end
+  end
+end

data/lib/spandx/gateways/http.rb

@@ -11,7 +11,7 @@ module Spandx
 
       def get(uri, default: nil)
         driver.with_retry do |client|
-          client.get(uri)
+          client.get(Addressable::URI.escape(uri))
         end
       rescue *Net::Hippie::CONNECTION_ERRORS
         default

data/lib/spandx/gateways/nuget.rb

@@ -6,10 +6,25 @@ module Spandx
     # https://api.nuget.org/v3-flatcontainer/#{package.name}/index.json
     # https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource
     class Nuget
+      attr_reader :host
+
       def initialize(http: Spandx.http, catalogue:)
         @http = http
         @catalogue = catalogue
         @guess = Guess.new(catalogue)
+        @host = 'api.nuget.org'
+      end
+
+      def update!(index, limit: nil)
+        counter = Concurrent::AtomicFixnum.new(0)
+        each do |spec|
+          upsert_into!(index, spec)
+
+          if limit
+            counter.increment
+            break if counter.value > limit
+          end
+        end
       end
 
       def licenses_for(name, version)
@@ -23,19 +38,34 @@ module Spandx
 
       attr_reader :http, :catalogue, :guess
 
+      def each
+        each_page do |page|
+          items_from(page).each do |item|
+            yield fetch_json(item['@id'])
+          end
+        end
+      end
+
+      def each_page
+        url = "https://#{host}/v3/catalog0/index.json"
+        items_from(fetch_json(url)).each do |page|
+          yield fetch_json(page['@id'])
+        end
+      end
+
       def nuspec_url_for(name, version)
-        "https://api.nuget.org/v3-flatcontainer/#{name}/#{version}/#{name}.nuspec"
+        "https://#{host}/v3-flatcontainer/#{name}/#{version}/#{name}.nuspec"
       end
 
       def nuspec_for(name, version)
-        response = http.get(nuspec_url_for(name, version))
-        from_xml(response.body) if http.ok?(response)
+        fetch_xml(nuspec_url_for(name, version))
       end
 
       def from_xml(xml)
         Nokogiri::XML(xml).tap(&:remove_namespaces!)
       end
 
+      # TODO: Fix parsing https://github.com/NuGet/Home/wiki/Packaging-License-within-the-nupkg#license
       def extract_licenses_from(document)
         licenses = document.search('//package/metadata/license')
         licenses.map(&:text) if licenses.any?
@@ -53,6 +83,32 @@ module Spandx
 
         guess.license_for(response.body) if http.ok?(response)
       end
+
+      def fetch_json(url)
+        response = http.get(url)
+        http.ok?(response) ? JSON.parse(response.body) : {}
+      end
+
+      def fetch_xml(url)
+        response = http.get(url)
+        http.ok?(response) ? from_xml(response.body) : from_xml('<empty />')
+      end
+
+      def items_from(page)
+        page['items']
+          .sort_by { |x| x['commitTimeStamp'] }
+          .reverse
+      end
+
+      def upsert_into!(index, spec)
+        key = [host, spec['id'], spec['version']]
+        return if index.indexed?(key)
+
+        if (license = spec['licenseExpression'])
+          index.write(key, [license])
+        end
+        puts [license, key].inspect
+      end
     end
   end
 end

data/lib/spandx/gateways/pypi.rb

@@ -17,7 +17,7 @@ module Spandx
         end
 
         def uri_for(name, version)
-          URI.parse("https://#{host}/pypi/#{name}/#{version}/json")
+          "https://#{host}/pypi/#{name}/#{version}/json"
         end
 
         def lookup(name, version, http: Spandx.http)

data/lib/spandx/index.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Spandx
+  class Index
+    DEFAULT_DIR = File.expand_path(File.join(Dir.home, '.local', 'share', 'spandx'))
+    attr_reader :directory
+
+    def initialize(directory: DEFAULT_DIR)
+      @directory = directory ? File.expand_path(directory) : DEFAULT_DIR
+    end
+
+    def indexed?(key)
+      File.exist?(data_file_for(digest_for(key)))
+    end
+
+    def read(key)
+      open_data(digest_for(key), mode: 'r', &:read)
+    end
+
+    def write(key, data)
+      return if data.nil? || data.empty?
+
+      open_data(digest_for(key)) do |x|
+        x.write(data)
+      end
+    end
+
+    private
+
+    def digest_for(components)
+      Digest::SHA1.hexdigest(Array(components).join('/'))
+    end
+
+    def open_data(key, mode: 'w')
+      FileUtils.mkdir_p(data_dir_for(key))
+      File.open(data_file_for(key), mode) do |file|
+        yield file
+      end
+    end
+
+    def data_dir_for(index_key)
+      File.join(directory, *index_key.scan(/../))
+    end
+
+    def data_file_for(key)
+      File.join(data_dir_for(key), 'data')
+    end
+  end
+end

data/lib/spandx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spandx
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

data/lib/spandx.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
+require 'addressable/uri'
 require 'bundler'
+require 'concurrent'
 require 'forwardable'
 require 'json'
 require 'net/hippie'
@@ -18,6 +20,7 @@ require 'spandx/gateways/pypi'
 require 'spandx/gateways/rubygems'
 require 'spandx/gateways/spdx'
 require 'spandx/guess'
+require 'spandx/index'
 require 'spandx/license'
 require 'spandx/parsers'
 require 'spandx/report'

data/spandx.gemspec

@@ -30,7 +30,9 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.required_ruby_version = '>= 2.4.0'
+  spec.add_dependency 'addressable', '~> 2.7'
   spec.add_dependency 'bundler', '>= 1.16', '< 3.0.0'
+  spec.add_dependency 'concurrent-ruby-ext', '~> 1.1'
   spec.add_dependency 'net-hippie', '~> 0.3'
   spec.add_dependency 'nokogiri', '~> 1.10'
   spec.add_dependency 'text', '~> 1.3'

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: spandx
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - mo khan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-29 00:00:00.000000000 Z
+date: 2020-02-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -30,6 +44,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby-ext
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: net-hippie
   requirement: !ruby/object:Gem::Requirement
@@ -214,6 +242,7 @@ files:
 - lib/spandx/catalogue.rb
 - lib/spandx/cli.rb
 - lib/spandx/command.rb
+- lib/spandx/commands/build.rb
 - lib/spandx/commands/scan.rb
 - lib/spandx/content.rb
 - lib/spandx/database.rb
@@ -224,6 +253,7 @@ files:
 - lib/spandx/gateways/rubygems.rb
 - lib/spandx/gateways/spdx.rb
 - lib/spandx/guess.rb
+- lib/spandx/index.rb
 - lib/spandx/license.rb
 - lib/spandx/parsers.rb
 - lib/spandx/parsers/base.rb