spandx 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ddd706dad19138c25501144fed49ae2148e7b5a703cdf06b9c4cd4bed4a940aa
-  data.tar.gz: c4596fdfa833988f80f7e3b17bce65e6fda8b8a0c059e441ff5b32583687b95d
+  metadata.gz: c8d1e5a7326e983e0bfdacf9b2e30b65f91357bec5a0a6f6f1ea617e53b20e69
+  data.tar.gz: e1c386c30848bda74c9728fdae88898e430b3048005e39a0d6ba64bc8146fddb
 SHA512:
-  metadata.gz: 8749463ff0bacbe4e125b9822c317d910dded8688e04122989777c42ac37b908f4c06a0419c824cde4698c7bab823934b751520fba786ca497b26ef9d266f9b7
-  data.tar.gz: 5229f971f6b36428ef9a09d3b91bab6f5b049f1312e12daee926bc27baf1f463e39204e30d69daa51566df3277712d901f8790871f3d272e0d353eea66dbaaa3
+  metadata.gz: 2e65997919e6ea2e23cabc18b2bdaceebc469553472241a45e87e65e701d9435e594faa188c5f94e5ada7d959de4e51260b2bc68903f8f7bcbc95ec6ecf3d12f
+  data.tar.gz: a2754d35a2a863634ef67b153bb4d54c72151ab083f9fe608836ee33aa3ff80b76d018bb8438af55f907c67dad6218c55b821a94ec45a6782bf1e5ffb8339cf9

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-Version 0.7.0
+Version 0.8.0
 
 # Changelog
 
@@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.8.0] - 2020-03-11
+### Added
+- Allow scanning a directory.
+- Allow recursive scanning of a directory.
+
 ## [0.7.0] - 2020-03-11
 ### Changed
 - Improve how the `nuget` index is built.
@@ -88,7 +93,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Provide ruby API to the latest SPDX catalogue.
 
-[Unreleased]: https://github.com/mokhan/spandx/compare/v0.6.0...HEAD
+[Unreleased]: https://github.com/mokhan/spandx/compare/v0.8.0...HEAD
+[0.8.0]: https://github.com/mokhan/spandx/compare/v0.7.0...v0.8.0
+[0.7.0]: https://github.com/mokhan/spandx/compare/v0.6.0...v0.7.0
 [0.6.0]: https://github.com/mokhan/spandx/compare/v0.5.0...v0.6.0
 [0.5.0]: https://github.com/mokhan/spandx/compare/v0.4.1...v0.5.0
 [0.4.1]: https://github.com/mokhan/spandx/compare/v0.4.0...v0.4.1

data/lib/spandx/cli/commands/scan.rb

@@ -4,23 +4,45 @@ module Spandx
   module Cli
     module Commands
       class Scan < Spandx::Cli::Command
-        attr_reader :lockfile
+        attr_reader :scan_path
 
-        def initialize(lockfile, options)
-          @lockfile = lockfile ? ::Pathname.new(File.expand_path(lockfile)) : nil
+        def initialize(scan_path, options)
+          @scan_path = ::Pathname.new(scan_path)
           @options = options
         end
 
         def execute(output: $stdout)
-          if lockfile.nil?
-            output.puts 'OK'
-          else
-            report = ::Spandx::Core::Report.new
-            ::Spandx::Core::Parser.for(lockfile).parse(lockfile).each do |dependency|
+          report = ::Spandx::Core::Report.new
+          each_file_in(scan_path) do |file|
+            each_dependency_from(file) do |dependency|
               report.add(dependency)
             end
-            output.puts report.to_json
           end
+          output.puts report.to_json
+        end
+
+        private
+
+        def recursive?
+          @options['recursive']
+        end
+
+        def each_file_in(dir, &block)
+          files = File.directory?(dir) ? Dir.glob(File.join(dir, '*')) : [dir]
+          files.each do |file|
+            if File.directory?(file)
+              each_file_in(file, &block) if recursive?
+            else
+              block.call(file)
+            end
+          end
+        end
+
+        def each_dependency_from(file)
+          ::Spandx::Core::Parser
+            .for(file)
+            .parse(file)
+            .each { |dependency| yield dependency }
         end
       end
     end

data/lib/spandx/cli.rb

@@ -19,9 +19,9 @@ module Spandx
     register Spandx::Cli::Commands::Index, 'index', 'index [SUBCOMMAND]', 'Command description...'
 
     desc 'scan LOCKFILE', 'Scan a lockfile and list dependencies/licenses'
-    method_option :help, aliases: '-h', type: :boolean,
-                         desc: 'Display usage information'
-    def scan(lockfile = nil)
+    method_option :help, aliases: '-h', type: :boolean, desc: 'Display usage information'
+    method_option :recursive, aliases: '-r', type: :boolean, desc: 'Perform recursive scan', default: false
+    def scan(lockfile)
       if options[:help]
         invoke :help, ['scan']
       else

data/lib/spandx/core/parser.rb

@@ -33,6 +33,7 @@ module Spandx
         end
 
         def for(path, catalogue: Spandx::Spdx::Catalogue.from_git)
+          Spandx.logger.debug(path)
           result = ::Spandx::Core::Parser.find do |x|
             x.matches?(File.basename(path))
           end

data/lib/spandx/dotnet/index.rb

@@ -12,7 +12,7 @@ module Spandx
 
       def licenses_for(name:, version:)
         search_key = [name, version].join
-        open_data(name, mode: 'r') do |io|
+        CSV.open(data_file_for(name), 'r') do |io|
           found = io.readlines.bsearch { |x| search_key <=> [x[0], x[1]].join }
           found ? found[2].split('-|-') : []
         end
@@ -37,6 +37,8 @@ module Spandx
 
       def sort_index!
         files('**/*') do |path|
+          next if File.extname(path) == '.checkpoints'
+
           IO.write(path, IO.readlines(path).sort.join)
         end
       end
@@ -45,14 +47,6 @@ module Spandx
         Digest::SHA1.hexdigest(Array(components).join('/'))
       end
 
-      def open_data(name, mode: 'a')
-        data_dir = data_dir_for(name)
-        FileUtils.mkdir_p(data_dir)
-        CSV.open(data_file_for(name), mode, force_quotes: true) do |csv|
-          yield csv
-        end
-      end
-
       def data_dir_for(name)
         digest = digest_for(name)
         File.join(directory, digest[0...2].downcase)
@@ -75,16 +69,27 @@ module Spandx
         IO.write(checkpoints_filepath, JSON.pretty_generate(checkpoints))
       end
 
-      def insert(id, version, license)
-        open_data(id) do |io|
-          io << [id, version, license]
-        end
+      def insert(name, version, license)
+        path = license ? data_file_for(name) : dead_letter_path
+        FileUtils.mkdir_p(File.dirname(path))
+        IO.write(
+          path,
+          CSV.generate_line([name, version, license], force_quotes: true),
+          mode: 'a'
+        )
+      end
+
+      def completed_pages
+        checkpoints.keys.map(&:to_i)
+      end
+
+      def dead_letter_path
+        @dead_letter_path ||= File.join(directory, 'nuget.unknown')
       end
 
       def insert_latest(gateway)
-        current_page = nil
-        gateway.each do |spec, page|
-          next unless spec['licenseExpression']
+        current_page = completed_pages.max || 0
+        gateway.each(start_page: current_page) do |spec, page|
           break if checkpoints[page.to_s]
 
           yield current_page if current_page && page != current_page

data/lib/spandx/dotnet/nuget_gateway.rb

@@ -21,8 +21,8 @@ module Spandx
           guess_licenses_from(document)
       end
 
-      def each(page: Float::INFINITY)
-        each_page(start_page: page) do |page_json|
+      def each(start_page: 0)
+        each_page(start_page: start_page) do |page_json|
           items_from(page_json).each do |item|
             yield(fetch_json(item['@id']), page_number_from(page_json['@id']))
           end
@@ -36,7 +36,7 @@ module Spandx
       def each_page(start_page:)
         url = "https://#{host}/v3/catalog0/index.json"
         items_from(fetch_json(url))
-          .find_all { |page| page_number_from(page['@id']) <= start_page }
+          .find_all { |page| page_number_from(page['@id']) >= start_page }
           .each { |page| yield fetch_json(page['@id']) }
       end
 
@@ -82,9 +82,7 @@ module Spandx
       end
 
       def items_from(page)
-        page['items']
-          .sort_by { |x| x['commitTimeStamp'] }
-          .reverse
+        page['items'].sort_by { |x| x['commitTimeStamp'] }
       end
 
       def page_number_from(url)

data/lib/spandx/gateways/http.rb

@@ -23,7 +23,7 @@ module Spandx
 
       def self.default_driver
         @default_driver ||= Net::Hippie::Client.new.tap do |client|
-          client.logger = ::Logger.new('http.log')
+          client.logger = Spandx.logger
           client.follow_redirects = 3
         end
       end

data/lib/spandx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spandx
-  VERSION = '0.7.0'
+  VERSION = '0.8.0'
 end

data/lib/spandx.rb

@@ -5,6 +5,7 @@ require 'bundler'
 require 'csv'
 require 'forwardable'
 require 'json'
+require 'logger'
 require 'net/hippie'
 require 'nokogiri'
 require 'pathname'
@@ -40,6 +41,8 @@ module Spandx
   class Error < StandardError; end
 
   class << self
+    attr_writer :logger
+
     def root
       Pathname.new(File.dirname(__FILE__)).join('../..')
     end
@@ -48,6 +51,10 @@ module Spandx
       @http ||= Spandx::Gateways::Http.new
     end
 
+    def logger
+      @logger ||= Logger.new('/dev/null')
+    end
+
     def spdx_db
       @spdx_db ||= Spandx::Core::Database
         .new(url: 'https://github.com/spdx/license-list-data.git')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spandx
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - mo khan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-11 00:00:00.000000000 Z
+date: 2020-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable