spandx 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b3a0275ae468012967376aa3bd813b3dc1efef7783eb11d0151d7babf4ac6a8
-  data.tar.gz: b86bd82707ad5afa0b83c59674e4f0d08c12733d5f1788038845f30079bb8ba6
+  metadata.gz: ddd706dad19138c25501144fed49ae2148e7b5a703cdf06b9c4cd4bed4a940aa
+  data.tar.gz: c4596fdfa833988f80f7e3b17bce65e6fda8b8a0c059e441ff5b32583687b95d
 SHA512:
-  metadata.gz: f71dd4bf8438ec8857f622076ea345f3c48c8dce3354949459b564ace2c0c002bcc95d09d9d906ba9063bdeb092ade90160f51bdcfb9900a0225114d1aa5aa80
-  data.tar.gz: e0bbbdd0156924ac21d9730aa322b332f4d3c1ef43b530f58c56f411c57bfb95c154574bda77c61dda3fbd80b9c8fc9e8a2fea4c09053fa6bfda2c5a6ac86dc5
+  metadata.gz: 8749463ff0bacbe4e125b9822c317d910dded8688e04122989777c42ac37b908f4c06a0419c824cde4698c7bab823934b751520fba786ca497b26ef9d266f9b7
+  data.tar.gz: 5229f971f6b36428ef9a09d3b91bab6f5b049f1312e12daee926bc27baf1f463e39204e30d69daa51566df3277712d901f8790871f3d272e0d353eea66dbaaa3

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-Version 0.6.0
+Version 0.7.0
 
 # Changelog
 
@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.7.0] - 2020-03-11
+### Changed
+- Improve how the `nuget` index is built.
+
 ## [0.6.0] - 2020-03-03
 ### Added
 - Add `spandx index update` command to fetch the latest `spandx-rubygems` index.

data/README.md

@@ -25,7 +25,7 @@ Or install it yourself as:
 To fetch the latest version of the catalogue data from [SPDX](https://spdx.org/licenses/licenses.json).
 
 ```ruby
-catalogue = Spandx::Catalogue.latest
+catalogue = Spandx::Spdx::Catalogue.latest
 catalogue.each do |license|
   puts license.inspect
 end
@@ -35,7 +35,7 @@ To load an offline copy of the data.
 
 ```ruby
 path = File.join(Dir.pwd, 'licenses.json')
-catalogue = Spandx::Catalogue.from_file(path)
+catalogue = Spandx::Spdx::Catalogue.from_file(path)
 catalogue.each do |license|
   puts license.inspect
 end

data/lib/spandx/cli/commands/index/build.rb

@@ -12,7 +12,7 @@ module Spandx
           def execute(output: $stdout)
             catalogue = Spandx::Spdx::Catalogue.from_git
             indexes.each do |index|
-              index.update!(catalogue: catalogue)
+              index.update!(catalogue: catalogue, output: output)
             end
             output.puts 'OK'
           end

data/lib/spandx/dotnet/index.rb

@@ -10,62 +10,87 @@ module Spandx
         @directory = directory ? File.expand_path(directory) : DEFAULT_DIR
       end
 
-      def update!(catalogue:, limit: nil)
-        counter = 0
-        gateway = Spandx::Dotnet::NugetGateway.new(catalogue: catalogue)
-        gateway.each do |spec|
-          next unless spec['licenseExpression']
-
-          write([gateway.host, spec['id'], spec['version']], spec['licenseExpression'])
-
-          if limit
-            counter += 1
-            break if counter > limit
-          end
+      def licenses_for(name:, version:)
+        search_key = [name, version].join
+        open_data(name, mode: 'r') do |io|
+          found = io.readlines.bsearch { |x| search_key <=> [x[0], x[1]].join }
+          found ? found[2].split('-|-') : []
         end
       end
 
-      def indexed?(key)
-        File.exist?(data_file_for(digest_for(key)))
-      end
-
-      def read(key)
-        open_data(digest_for(key), mode: 'r', &:read)
+      def update!(catalogue:, output: StringIO.new)
+        insert_latest(Spandx::Dotnet::NugetGateway.new(catalogue: catalogue)) do |page|
+          output.puts "Checkpoint #{page}"
+          checkpoint!(page)
+        end
+        sort_index!
       end
 
-      def write(key, data)
-        return if data.nil? || data.empty?
+      private
 
-        open_data(digest_for(key)) do |x|
-          x.write(data)
+      def files(pattern)
+        Dir.glob(pattern, base: directory).sort.each do |file|
+          fullpath = File.join(directory, file)
+          yield fullpath unless File.directory?(fullpath)
         end
       end
 
-      private
+      def sort_index!
+        files('**/*') do |path|
+          IO.write(path, IO.readlines(path).sort.join)
+        end
+      end
 
       def digest_for(components)
         Digest::SHA1.hexdigest(Array(components).join('/'))
       end
 
-      def open_data(key, mode: 'w')
-        FileUtils.mkdir_p(data_dir_for(key))
-        File.open(data_file_for(key), mode) do |file|
-          yield file
+      def open_data(name, mode: 'a')
+        data_dir = data_dir_for(name)
+        FileUtils.mkdir_p(data_dir)
+        CSV.open(data_file_for(name), mode, force_quotes: true) do |csv|
+          yield csv
         end
       end
 
-      def data_dir_for(index_key)
-        File.join(directory, *index_key.scan(/../)).downcase
+      def data_dir_for(name)
+        digest = digest_for(name)
+        File.join(directory, digest[0...2].downcase)
       end
 
-      def data_file_for(key)
-        File.join(data_dir_for(key), 'data')
+      def data_file_for(name)
+        File.join(data_dir_for(name), 'nuget')
       end
 
-      def upsert!(spec)
-        return unless spec['licenseExpression']
+      def checkpoints_filepath
+        @checkpoints_filepath ||= File.join(directory, 'nuget.checkpoints')
+      end
 
-        write([host, spec['id'], spec['version']], spec['licenseExpression'])
+      def checkpoints
+        @checkpoints ||= File.exist?(checkpoints_filepath) ? JSON.parse(IO.read(checkpoints_filepath)) : {}
+      end
+
+      def checkpoint!(page)
+        checkpoints[page.to_s] = Time.now.utc
+        IO.write(checkpoints_filepath, JSON.pretty_generate(checkpoints))
+      end
+
+      def insert(id, version, license)
+        open_data(id) do |io|
+          io << [id, version, license]
+        end
+      end
+
+      def insert_latest(gateway)
+        current_page = nil
+        gateway.each do |spec, page|
+          next unless spec['licenseExpression']
+          break if checkpoints[page.to_s]
+
+          yield current_page if current_page && page != current_page
+          current_page = page
+          insert(spec['id'], spec['version'], spec['licenseExpression'])
+        end
       end
     end
   end

data/lib/spandx/dotnet/nuget_gateway.rb

@@ -21,10 +21,10 @@ module Spandx
           guess_licenses_from(document)
       end
 
-      def each
-        each_page do |page|
-          items_from(page).each do |item|
-            yield fetch_json(item['@id'])
+      def each(page: Float::INFINITY)
+        each_page(start_page: page) do |page_json|
+          items_from(page_json).each do |item|
+            yield(fetch_json(item['@id']), page_number_from(page_json['@id']))
           end
         end
       end
@@ -33,11 +33,11 @@ module Spandx
 
       attr_reader :http, :guess
 
-      def each_page
+      def each_page(start_page:)
         url = "https://#{host}/v3/catalog0/index.json"
-        items_from(fetch_json(url)).each do |page|
-          yield fetch_json(page['@id'])
-        end
+        items_from(fetch_json(url))
+          .find_all { |page| page_number_from(page['@id']) <= start_page }
+          .each { |page| yield fetch_json(page['@id']) }
       end
 
       def nuspec_url_for(name, version)
@@ -86,6 +86,10 @@ module Spandx
           .sort_by { |x| x['commitTimeStamp'] }
           .reverse
       end
+
+      def page_number_from(url)
+        url.match(/page(?<page_number>\d+)\.json/)[:page_number].to_i
+      end
     end
   end
 end

data/lib/spandx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spandx
-  VERSION = '0.6.0'
+  VERSION = '0.7.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spandx
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - mo khan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-03 00:00:00.000000000 Z
+date: 2020-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable