spandx 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bd43f4534fd78176bb372673b73a3fcd135872f4b784ed84a6b9d518fbb7001d
-  data.tar.gz: a66deaed24189e97e130258647b4c10f00554514ae508510169ec5ff54dee30f
+  metadata.gz: 440933f7b4b8706a0a4da8ed5c84c5db80f93e40e7db258715bb73b11c5cbb31
+  data.tar.gz: 182e055dea23b17d3bb67fc16ce84ce245aa2dbfd247d9eead1b33162bf05472
 SHA512:
-  metadata.gz: e25407370b0ec723f948f128a80e7030e969da7744740f3d49d9293b473c68370a59615bf8638da168b1bf47f5ecaa682f74a4689c77e1f15bc30bd3dcfebfcc
-  data.tar.gz: 89151e50569afac8e15e8fb4e882e6241c083939d30a7ffd4b15151d5420b9dba3953c7aa774006d51d2fde1df8e4eb0da550c6c05584d9a8f7b841529cb912c
+  metadata.gz: 26f4bc7516adf9f96c3725b15acc6ead6a2bdad6deaaf718a145106dcbec9f7c95c0f2053163f3c913a7466d2a8c3af860d4903a78bcb6eec8329f05a0cdb17f
+  data.tar.gz: 441abbd1f3cd090514627386e6f175ff5e53ba2909a2722f7e34e0f9a5e514532c431ffdd54e23d501dd20c6626c382b0fba215939989b7e0d32ba477bf86aa9

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-Version 0.4.1
+Version 0.5.0
 
 # Changelog
 
@@ -9,6 +9,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.5.0] - 2020-02-13
+### Added
+- Add jaro winkler string similarity support.
+- Attempt to resolve rubygems dependencies via `spandx-rubygems` index.
+
+### Changed
+- Make `text` and `jaro_winkler` gems a soft dependency.
+
 ## [0.4.1] - 2020-02-02
 ### Fixed
 - Save license expression as string instead of array.
@@ -65,7 +73,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Provide ruby API to the latest SPDX catalogue.
 
-[Unreleased]: https://github.com/mokhan/spandx/compare/v0.4.1...HEAD
+[Unreleased]: https://github.com/mokhan/spandx/compare/v0.5.0...HEAD
+[0.5.0]: https://github.com/mokhan/spandx/compare/v0.4.1...v0.5.0
 [0.4.1]: https://github.com/mokhan/spandx/compare/v0.4.0...v0.4.1
 [0.4.0]: https://github.com/mokhan/spandx/compare/v0.3.0...v0.4.0
 [0.3.0]: https://github.com/mokhan/spandx/compare/v0.2.0...v0.3.0

data/lib/spandx/catalogue.rb

@@ -36,7 +36,7 @@ module Spandx
       end
 
       def from_git
-        from_json(Spandx.db.read('json/licenses.json'))
+        from_json(Spandx.spdx_db.read('json/licenses.json'))
       end
 
       def empty

data/lib/spandx/content.rb

@@ -2,10 +2,9 @@
 
 module Spandx
   class Content
-    attr_reader :raw, :threshold
+    attr_reader :raw
 
-    def initialize(raw, threshold: 89.0)
-      @threshold = threshold
+    def initialize(raw)
       @raw = raw
     end
 
@@ -16,9 +15,11 @@ module Spandx
     def similar?(other, algorithm: :dice_coefficient)
       case algorithm
       when :dice_coefficient
-        similarity_score(other) > threshold
+        similarity_score(other, algorithm: algorithm) > 89.0
       when :levenshtein
-        similarity_score(other) < threshold
+        similarity_score(other, algorithm: algorithm) < 3
+      when :jaro_winkler
+        similarity_score(other, algorithm: algorithm) > 89.0
       end
     end
 
@@ -27,7 +28,13 @@ module Spandx
       when :dice_coefficient
         dice_coefficient(other)
       when :levenshtein
+        require 'text'
+
         Text::Levenshtein.distance(raw, other.raw, 100)
+      when :jaro_winkler
+        require 'jaro_winkler'
+
+        JaroWinkler.distance(raw, other.raw) * 100.0
       end
     end
 

data/lib/spandx/gateways/nuget.rb

@@ -16,13 +16,13 @@ module Spandx
       end
 
       def update!(index, limit: nil)
-        counter = Concurrent::AtomicFixnum.new(0)
+        counter = 0
         each do |spec|
           upsert_into!(index, spec)
 
           if limit
-            counter.increment
-            break if counter.value > limit
+            counter += 1
+            break if counter > limit
           end
         end
       end
@@ -104,10 +104,9 @@ module Spandx
         key = [host, spec['id'], spec['version']]
         return if index.indexed?(key)
 
-        if (license = spec['licenseExpression'])
-          index.write(key, license)
-        end
-        puts [license, key].inspect
+        return unless spec['licenseExpression']
+
+        index.write(key, spec['licenseExpression'])
       end
     end
   end

data/lib/spandx/gateways/rubygems.rb

@@ -9,13 +9,18 @@ module Spandx
       end
 
       def licenses_for(name, version)
-        details_on(name, version)['licenses'] || []
+        found = index.licenses_for(name: name, version: version)
+        found.any? ? found : details_on(name, version)['licenses'] || []
       end
 
       private
 
       attr_reader :http
 
+      def index
+        @index ||= Spandx::Rubygems::Index.new
+      end
+
       def details_on(name, version)
         url = "https://rubygems.org/api/v2/rubygems/#{name}/versions/#{version}.json"
         response = http.get(url, default: {})

data/lib/spandx/guess.rb

@@ -8,10 +8,18 @@ module Spandx
       attr_reader :score, :item
 
       def initialize(score, item)
+        update(score || 0.0, item)
+      end
+
+      def update(score, item)
         @score = score
         @item = item
       end
 
+      def empty?
+        score.nil? || item.nil?
+      end
+
       def <=>(other)
         score <=> other.score
       end
@@ -29,33 +37,40 @@ module Spandx
 
     def license_for(raw_content, algorithm: :dice_coefficient)
       content = Content.new(raw_content)
-      score = nil
-      catalogue.each do |license|
-        next if license.deprecated_license_id?
+      score = Score.new(nil, nil)
+      threshold = threshold_for(algorithm)
+      direction = algorithm == :levenshtein ? method(:min) : method(:max)
 
-        score = algorithm == :levenshtein ? levenshtein(content, license, score) : dice(content, license, score)
+      catalogue.each do |license|
+        direction.call(content, license, score, threshold, algorithm) unless license.deprecated_license_id?
       end
       score&.item&.id
     end
 
     private
 
-    def levenshtein(target, other, score)
-      percentage = target.similarity_score(other.content, algorithm: :levenshtein)
-      if score.nil? || percentage < score.score
-        return Score.new(percentage, other)
-      end
+    def threshold_for(algorithm)
+      {
+        dice_coefficient: 89.0,
+        jaro_winkler: 80.0,
+        levenshtein: 80.0,
+      }[algorithm.to_sym]
+    end
 
-      score
+    def min(target, other, score, threshold, algorithm)
+      percentage = target.similarity_score(other.content, algorithm: algorithm)
+      return if percentage > threshold
+      return if score.score > 0.0 && score.score < percentage
+
+      score.update(percentage, other)
     end
 
-    def dice(target, other, score)
-      percentage = target.similarity_score(other.content, algorithm: :dice_coefficient)
-      if (percentage > 89.0) && (score.nil? || percentage > score.score)
-        return Score.new(percentage, other)
-      end
+    def max(target, other, score, threshold, algorithm)
+      percentage = target.similarity_score(other.content, algorithm: algorithm)
+      return if percentage < threshold
+      return if score.score >= percentage
 
-      score
+      score.update(percentage, other)
     end
   end
 end

data/lib/spandx/license.rb

@@ -65,7 +65,7 @@ module Spandx
     end
 
     def raw_content
-      @raw_content ||= (Spandx.db.read("text/#{id}.txt") || '')
+      @raw_content ||= (Spandx.spdx_db.read("text/#{id}.txt") || '')
     end
 
     def <=>(other)

data/lib/spandx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spandx
-  VERSION = '0.4.1'
+  VERSION = '0.5.0'
 end

data/lib/spandx.rb

@@ -2,13 +2,11 @@
 
 require 'addressable/uri'
 require 'bundler'
-require 'concurrent'
 require 'forwardable'
 require 'json'
 require 'net/hippie'
 require 'nokogiri'
 require 'pathname'
-require 'text'
 
 require 'spandx/catalogue'
 require 'spandx/content'
@@ -24,6 +22,7 @@ require 'spandx/index'
 require 'spandx/license'
 require 'spandx/parsers'
 require 'spandx/report'
+require 'spandx/rubygems'
 require 'spandx/version'
 
 module Spandx
@@ -38,8 +37,8 @@ module Spandx
       @http ||= Spandx::Gateways::Http.new
     end
 
-    def db
-      @db ||= Spandx::Database.new(url: 'https://github.com/spdx/license-list-data.git').tap(&:update!)
+    def spdx_db
+      @spdx_db ||= Spandx::Database.new(url: 'https://github.com/spdx/license-list-data.git').tap(&:update!)
     end
   end
 end

data/spandx.gemspec

@@ -14,6 +14,7 @@ Gem::Specification.new do |spec|
   spec.description   = 'A ruby interface to the SPDX catalogue.'
   spec.homepage      = 'https://github.com/mokhan/spandx'
   spec.license       = 'MIT'
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.4.0')
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = 'https://github.com/mokhan/spandx'
@@ -29,20 +30,21 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.required_ruby_version = '>= 2.4.0'
   spec.add_dependency 'addressable', '~> 2.7'
   spec.add_dependency 'bundler', '>= 1.16', '< 3.0.0'
-  spec.add_dependency 'concurrent-ruby-ext', '~> 1.1'
   spec.add_dependency 'net-hippie', '~> 0.3'
   spec.add_dependency 'nokogiri', '~> 1.10'
-  spec.add_dependency 'text', '~> 1.3'
-  spec.add_dependency 'thor', '~> 0.1'
+  spec.add_dependency 'spandx-rubygems', '~> 0.1'
+  spec.add_dependency 'thor'
+
   spec.add_development_dependency 'bundler-audit', '~> 0.6'
+  spec.add_development_dependency 'jaro_winkler', '~> 1.5'
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rspec-benchmark', '~> 0.5'
   spec.add_development_dependency 'rubocop', '~> 0.52'
   spec.add_development_dependency 'rubocop-rspec', '~> 1.22'
+  spec.add_development_dependency 'text', '~> 1.3'
   spec.add_development_dependency 'vcr', '~> 5.0'
   spec.add_development_dependency 'webmock', '~> 3.7'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spandx
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - mo khan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-02-02 00:00:00.000000000 Z
+date: 2020-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -44,20 +44,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
-- !ruby/object:Gem::Dependency
-  name: concurrent-ruby-ext
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: net-hippie
   requirement: !ruby/object:Gem::Requirement
@@ -87,33 +73,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.10'
 - !ruby/object:Gem::Dependency
-  name: text
+  name: spandx-rubygems
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler-audit
   requirement: !ruby/object:Gem::Requirement
@@ -128,6 +114,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: jaro_winkler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -198,6 +198,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.22'
+- !ruby/object:Gem::Dependency
+  name: text
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement