spandx 0.11.0 → 0.12.0

data/lib/spandx/core/dependency.rb

@@ -3,21 +3,46 @@
 module Spandx
   module Core
     class Dependency
-      attr_reader :name, :meta, :version, :licenses
+      attr_reader :package_manager, :name, :version, :licenses, :meta
 
-      def initialize(name:, version:, licenses: [], meta: {})
+      def initialize(package_manager:, name:, version:, licenses: [], meta: {})
+        @package_manager = package_manager
         @name = name
         @version = version
         @licenses = licenses
         @meta = meta
       end
 
+      def managed_by?(value)
+        package_manager == value&.to_sym
+      end
+
+      def <=>(other)
+        to_s <=> other.to_s
+      end
+
+      def hash
+        to_s.hash
+      end
+
+      def eql?(other)
+        to_s == other.to_s
+      end
+
+      def to_s
+        @to_s ||= [name, version].compact.join(' ')
+      end
+
+      def inspect
+        "#<Spandx::Core::Dependency name=#{name}, version=#{version}>"
+      end
+
+      def to_a
+        [name, version, licenses.map(&:id)]
+      end
+
       def to_h
-        {
-          name: name,
-          version: version,
-          licenses: licenses.compact.map(&:id)
-        }
+        { name: name, version: version, licenses: licenses.map(&:id) }
       end
     end
   end

data/lib/spandx/core/gateway.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class Gateway
+      def matches?(_dependency)
+        raise ::Spandx::Error, :matches?
+      end
+
+      def licenses_for(_dependency)
+        raise ::Spandx::Error, :licenses_for
+      end
+
+      class << self
+        include Registerable
+      end
+    end
+  end
+end

data/lib/spandx/core/{database.rb → git.rb}

@@ -2,7 +2,7 @@
 
 module Spandx
   module Core
-    class Database
+    class Git
       attr_reader :path, :url
 
       def initialize(url:)
@@ -28,7 +28,10 @@ module Spandx
       def open(path, mode: 'r')
         update! unless dotgit?
 
-        File.open(expand_path(path), mode) do |io|
+        full_path = expand_path(path)
+        return unless File.exist?(full_path)
+
+        File.open(full_path, mode) do |io|
           yield io
         end
       end
@@ -61,5 +64,7 @@ module Spandx
         end
       end
     end
+
+    Database = Git
   end
 end

data/lib/spandx/core/guess.rb

@@ -9,8 +9,44 @@ module Spandx
         @catalogue = catalogue
       end
 
-      def license_for(raw_content, algorithm: :dice_coefficient)
-        content = Content.new(raw_content)
+      def license_for(raw, algorithm: :dice_coefficient)
+        raw.is_a?(Hash) ? from_hash(raw, algorithm) : from_string(raw, algorithm)
+      end
+
+      private
+
+      def from_hash(hash, algorithm)
+        from_string(hash[:name], algorithm) ||
+          from_url(hash[:url], algorithm) ||
+          unknown(hash[:name] || hash[:url])
+      end
+
+      def from_string(raw, algorithm)
+        content = Content.new(raw)
+
+        catalogue[raw] ||
+          match_name(content, algorithm) ||
+          match_body(content, algorithm) ||
+          unknown(raw)
+      end
+
+      def from_url(url, algorithm)
+        return if url.nil? || url.empty?
+
+        response = Spandx.http.get(url)
+        return unless Spandx.http.ok?(response)
+
+        license_for(response.body, algorithm: algorithm)
+      end
+
+      def match_name(content, _algorithm)
+        catalogue.find do |license|
+          score = content.similarity_score(::Spandx::Core::Content.new(license.name))
+          score > 85
+        end
+      end
+
+      def match_body(content, algorithm)
         score = Score.new(nil, nil)
         threshold = threshold_for(algorithm)
         direction = algorithm == :levenshtein ? method(:min) : method(:max)
@@ -18,10 +54,12 @@ module Spandx
         catalogue.each do |license|
           direction.call(content, license, score, threshold, algorithm) unless license.deprecated_license_id?
         end
-        score&.item&.id
+        score&.item
       end
 
-      private
+      def unknown(text)
+        ::Spandx::Spdx::License.unknown(text)
+      end
 
       def threshold_for(algorithm)
         {

data/lib/spandx/core/http.rb

@@ -3,17 +3,24 @@
 module Spandx
   module Core
     class Http
-      attr_reader :driver
+      attr_reader :driver, :retries
 
-      def initialize(driver: Http.default_driver)
+      def initialize(driver: Http.default_driver, retries: 3)
         @driver = driver
+        @retries = retries
+        @circuits = Hash.new { |hash, key| hash[key] = Circuit.new }
       end
 
-      def get(uri, default: nil)
+      def get(uri, default: nil, escape: true)
         return default if Spandx.airgap?
 
-        driver.with_retry do |client|
-          client.get(Addressable::URI.escape(uri))
+        circuit = circuit_for(uri)
+        return default if circuit.open?
+
+        circuit.attempt do
+          driver.with_retry(retries: retries) do |client|
+            client.get(escape ? Addressable::URI.escape(uri) : uri)
+          end
         end
       rescue *Net::Hippie::CONNECTION_ERRORS
         default
@@ -26,9 +33,27 @@ module Spandx
       def self.default_driver
         @default_driver ||= Net::Hippie::Client.new.tap do |client|
           client.logger = Spandx.logger
+          client.open_timeout = 1
+          client.read_timeout = 5
           client.follow_redirects = 3
         end
       end
+
+      private
+
+      def circuit_breaker_for(host, default)
+        return default unless @circuits[host]
+
+        @circuits[host] = false
+        result = yield
+        @circuits[host] = true
+        result
+      end
+
+      def circuit_for(url)
+        uri = URI.parse(url.to_s)
+        @circuits[uri.host]
+      end
     end
   end
 end

data/lib/spandx/core/license_plugin.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class LicensePlugin < Spandx::Core::Plugin
+      def initialize(catalogue: Spdx::Catalogue.from_git)
+        @guess = Guess.new(catalogue)
+      end
+
+      def enhance(dependency)
+        return dependency unless known?(dependency.package_manager)
+        return enhance_from_metadata(dependency) if available_in?(dependency.meta)
+
+        licenses_for(dependency).each do |text|
+          dependency.licenses << @guess.license_for(text)
+        end
+        dependency
+      end
+
+      private
+
+      def licenses_for(dependency)
+        results = cache_for(dependency).licenses_for(dependency.name, dependency.version)
+        results && !results.empty? ? results : gateway_for(dependency).licenses_for(dependency)
+      end
+
+      def cache_for(dependency, git: Spandx.git)
+        db = git[dependency.package_manager.to_sym] || git[:cache]
+        Spandx::Core::Cache.new(dependency.package_manager, db: db)
+      end
+
+      def known?(package_manager)
+        %i[nuget maven rubygems npm yarn pypi composer].include?(package_manager)
+      end
+
+      def gateway_for(dependency)
+        ::Spandx::Core::Gateway.find do |gateway|
+          gateway.matches?(dependency)
+        end
+      end
+
+      def available_in?(metadata)
+        metadata.respond_to?(:[]) && metadata['license']
+      end
+
+      def enhance_from_metadata(dependency)
+        dependency.meta['license'].each do |x|
+          dependency.licenses << @guess.license_for(x)
+        end
+        dependency
+      end
+    end
+  end
+end

data/lib/spandx/core/null_gateway.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class NullGateway
+      def licenses_for(*_args)
+        []
+      end
+    end
+  end
+end

data/lib/spandx/core/parser.rb

@@ -9,36 +9,19 @@ module Spandx
         end
       end
 
-      attr_reader :catalogue
+      def matches?(_filename)
+        raise ::Spandx::Error, :matches?
+      end
 
-      def initialize(catalogue:)
-        @catalogue = catalogue
+      def parse(_dependency)
+        raise ::Spandx::Error, :parse
       end
 
       class << self
-        include Enumerable
-
-        def each(&block)
-          registry.each do |x|
-            block.call(x)
-          end
-        end
-
-        def inherited(subclass)
-          registry.push(subclass)
-        end
-
-        def registry
-          @registry ||= []
-        end
-
-        def for(path, catalogue: Spandx::Spdx::Catalogue.from_git)
-          Spandx.logger.debug(path)
-          result = ::Spandx::Core::Parser.find do |x|
-            x.matches?(File.basename(path))
-          end
+        include Registerable
 
-          result&.new(catalogue: catalogue) || UNKNOWN
+        def for(path)
+          find { |x| x.matches?(File.basename(path)) } || UNKNOWN
         end
       end
     end

data/lib/spandx/core/plugin.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class Plugin
+      def enhance(_dependency)
+        raise ::Spandx::Error, :enhance
+      end
+
+      class << self
+        include Registerable
+      end
+    end
+  end
+end

data/lib/spandx/core/registerable.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    module Registerable
+      include Enumerable
+
+      def all
+        @all ||= registry.map(&:new)
+      end
+
+      def each(&block)
+        all.each do |x|
+          block.call(x)
+        end
+      end
+
+      def inherited(subclass)
+        registry.push(subclass)
+      end
+
+      def registry
+        @registry ||= []
+      end
+    end
+  end
+end

data/lib/spandx/core/report.rb

@@ -3,26 +3,44 @@
 module Spandx
   module Core
     class Report
+      include Enumerable
+
       FORMATS = {
-        json: :to_json,
+        csv: :to_csv,
         hash: :to_h,
+        json: :to_json,
+        table: :to_table,
       }.freeze
 
       def initialize
-        @dependencies = []
+        @dependencies = SortedSet.new
       end
 
       def add(dependency)
-        @dependencies.push(dependency)
+        @dependencies << dependency
       end
 
-      def to(format)
-        public_send(FORMATS.fetch(format&.to_sym, :to_json))
+      def each
+        @dependencies.each do |dependency|
+          yield dependency
+        end
+      end
+
+      def to(format, formats: FORMATS)
+        public_send(formats.fetch(format&.to_sym, :to_json))
+      end
+
+      def to_table
+        Table.new do |table|
+          map do |dependency|
+            table << dependency
+          end
+        end
       end
 
       def to_h
         { version: '1.0', dependencies: [] }.tap do |report|
-          @dependencies.each do |dependency|
+          each do |dependency|
             report[:dependencies].push(dependency.to_h)
           end
         end
@@ -31,6 +49,12 @@ module Spandx
       def to_json(*_args)
         JSON.pretty_generate(to_h)
       end
+
+      def to_csv
+        map do |dependency|
+          CSV.generate_line(dependency.to_a)
+        end
+      end
     end
   end
 end