spandx 0.1.7 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9bc9c449db0cbce37e841c358d608953e9d9cde996aec5b8c4304457bce934a4
-  data.tar.gz: 28fe5ca88aa73cd47e1de6aa70bb49868a642896da10ddde3ecbcf6037fc612e
+  metadata.gz: a9c82d16a4a78e075ecebcd7308580b5e3ca44bcc274d3070ade1d546b9a38fa
+  data.tar.gz: 540a6a7f1d12f14c5d44feb15826b2f61ba8ee422d72e662328457b8df7260fc
 SHA512:
-  metadata.gz: 33d9b54d4efa74cb7a76032c13ece5be12e11ef52c2dc8d088bb2f78837a1b92c1f85c2e7b01df17d3dae0566dad4db24758d29dae438751e466d95fb06952cb
-  data.tar.gz: b9f35181f330a8782e314b1118b731f32f98844f51f01560b3fbf975daed4d63e13923082483fb8713fb38826e89abe0f3889e5f39d9695780a95536d132c4b9
+  metadata.gz: 1bb0a40e4723a2b3fc2b50a92d3df34df978753f5e14a76534758d9c946349475dfa18a123c1e0b3f1d89ac01fbbe9eecb520ca6842c122ba5af122d9466c639
+  data.tar.gz: faaedb6aaaf9ae486219ef396092c108ff359723a267291b5ef37b2bcf834195b5ace2578067598f97c71b68f78f4efb8ade261c3953184fd4a7be34eed9d7a9

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-Version 0.1.7
+Version 0.2.0
 
 # Changelog
 
@@ -9,7 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 ### Added
-- nil
+- Nil
+
+## [0.2.0] - 2020-01-28
+### Added
+- Parse .NET `sln` files
+- Add ability to choose Levenshtein algorithm
 
 ## [0.1.7] - 2020-01-28
 ### Added
@@ -47,7 +52,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Provide ruby API to the latest SPDX catalogue.
 
-[Unreleased]: https://github.com/mokhan/spandx/compare/v0.1.7...HEAD
+[Unreleased]: https://github.com/mokhan/spandx/compare/v0.2.0...HEAD
+[0.2.0]: https://github.com/mokhan/spandx/compare/v0.1.7...v0.2.0
 [0.1.7]: https://github.com/mokhan/spandx/compare/v0.1.6...v0.1.7
 [0.1.6]: https://github.com/mokhan/spandx/compare/v0.1.5...v0.1.6
 [0.1.5]: https://github.com/mokhan/spandx/compare/v0.1.4...v0.1.5

data/lib/spandx/content.rb

@@ -2,22 +2,33 @@
 
 module Spandx
   class Content
-    attr_reader :tokens, :threshold
+    attr_reader :raw, :threshold
 
-    def initialize(content, threshold: 89.0)
+    def initialize(raw, threshold: 89.0)
       @threshold = threshold
-      @tokens = tokenize(canonicalize(content)).to_set
+      @raw = raw
     end
 
-    def similar?(other)
-      similarity_score(other) > threshold
+    def tokens
+      @tokens ||= tokenize(canonicalize(raw)).to_set
     end
 
-    # https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Dice%27s_coefficient#Ruby
-    def similarity_score(other)
-      overlap = (tokens & other.tokens).size
-      total = tokens.size + other.tokens.size
-      100.0 * (overlap * 2.0 / total)
+    def similar?(other, algorithm: :dice_coefficient)
+      case algorithm
+      when :dice_coefficient
+        similarity_score(other) > threshold
+      when :levenshtein
+        similarity_score(other) < threshold
+      end
+    end
+
+    def similarity_score(other, algorithm: :dice_coefficient)
+      case algorithm
+      when :dice_coefficient
+        dice_coefficient(other)
+      when :levenshtein
+        Text::Levenshtein.distance(raw, other.raw, 100)
+      end
     end
 
     private
@@ -33,5 +44,12 @@ module Spandx
     def blank?(content)
       content.nil? || content.chomp.strip.empty?
     end
+
+    # https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Dice%27s_coefficient#Ruby
+    def dice_coefficient(other)
+      overlap = (tokens & other.tokens).size
+      total = tokens.size + other.tokens.size
+      100.0 * (overlap * 2.0 / total)
+    end
   end
 end

data/lib/spandx/database.rb

@@ -14,7 +14,8 @@ module Spandx
     end
 
     def read(file)
-      IO.read(File.join(path, file))
+      full_path = File.join(path, file)
+      IO.read(full_path) if File.exist?(full_path)
     end
 
     private

data/lib/spandx/gateways/nuget.rb

@@ -9,41 +9,49 @@ module Spandx
       def initialize(http: Spandx.http, catalogue:)
         @http = http
         @catalogue = catalogue
+        @guess = Guess.new(catalogue)
       end
 
       def licenses_for(name, version)
         document = nuspec_for(name, version)
 
-        exact_licenses_from(document) ||
+        extract_licenses_from(document) ||
           guess_licenses_from(document)
       end
 
       private
 
-      attr_reader :http, :catalogue
+      attr_reader :http, :catalogue, :guess
 
       def nuspec_url_for(name, version)
         "https://api.nuget.org/v3-flatcontainer/#{name}/#{version}/#{name}.nuspec"
       end
 
       def nuspec_for(name, version)
-        from_xml(http.get(nuspec_url_for(name, version)).body)
+        response = http.get(nuspec_url_for(name, version))
+        from_xml(response.body) if http.ok?(response)
       end
 
       def from_xml(xml)
         Nokogiri::XML(xml).tap(&:remove_namespaces!)
       end
 
-      def exact_licenses_from(document)
+      def extract_licenses_from(document)
         licenses = document.search('//package/metadata/license')
         licenses.map(&:text) if licenses.any?
       end
 
       def guess_licenses_from(document)
-        guess = Guess.new(catalogue)
         document
           .search('//package/metadata/licenseUrl')
-          .map { |node| guess.license_for(http.get(node.text).body) }
+          .map { |node| guess_license_for(node.text) }
+          .compact
+      end
+
+      def guess_license_for(url)
+        response = http.get(url)
+
+        guess.license_for(response.body) if http.ok?(response)
       end
     end
   end

data/lib/spandx/guess.rb

@@ -27,19 +27,35 @@ module Spandx
       @catalogue = catalogue
     end
 
-    def license_for(raw_content)
+    def license_for(raw_content, algorithm: :dice_coefficient)
       content = Content.new(raw_content)
-
-      max_score = nil
+      score = nil
       catalogue.each do |license|
         next if license.deprecated_license_id?
 
-        percentage = content.similarity_score(license.content)
-        if max_score.nil? || percentage > max_score.score
-          max_score = Score.new(percentage, license)
-        end
+        score = algorithm == :levenshtein ? levenshtein(content, license, score) : dice(content, license, score)
+      end
+      score&.item&.id
+    end
+
+    private
+
+    def levenshtein(target, other, score)
+      percentage = target.similarity_score(other.content, algorithm: :levenshtein)
+      if score.nil? || percentage < score.score
+        return Score.new(percentage, other)
       end
-      max_score.item.id
+
+      score
+    end
+
+    def dice(target, other, score)
+      percentage = target.similarity_score(other.content, algorithm: :dice_coefficient)
+      if (percentage > 89.0) && (score.nil? || percentage > score.score)
+        return Score.new(percentage, other)
+      end
+
+      score
     end
   end
 end

data/lib/spandx/license.rb

@@ -61,7 +61,11 @@ module Spandx
     end
 
     def content
-      @content ||= Content.new(Spandx.db.read("text/#{id}.txt"))
+      @content ||= Content.new(raw_content)
+    end
+
+    def raw_content
+      @raw_content ||= (Spandx.db.read("text/#{id}.txt") || '')
     end
 
     def <=>(other)

data/lib/spandx/parsers/sln.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Parsers
+    class Sln < Base
+      def self.matches?(filename)
+        filename.match?(/.*\.sln/)
+      end
+
+      def parse(file_path)
+        project_paths_from(file_path).map do |path|
+          Parsers
+            .for(path, catalogue: catalogue)
+            .parse(path)
+        end.flatten
+      end
+
+      private
+
+      def project_paths_from(file_path)
+        IO.readlines(file_path).map do |line|
+          next unless project_line?(line)
+
+          path = project_path_from(line)
+          next unless path
+
+          path = File.join(File.dirname(file_path), path)
+          Pathname.new(path).cleanpath.to_path
+        end.compact
+      end
+
+      def project_line?(line)
+        line.match?(/^\s*Project\(/)
+      end
+
+      def project_path_from(line)
+        path = line.split('"')[5]
+        return unless path
+
+        path = path.tr('\\', '/')
+        path.match?(/\.[a-z]{2}proj$/) ? path : nil
+      end
+    end
+  end
+end

data/lib/spandx/parsers.rb

@@ -5,6 +5,7 @@ require 'spandx/parsers/csproj'
 require 'spandx/parsers/gemfile_lock'
 require 'spandx/parsers/packages_config'
 require 'spandx/parsers/pipfile_lock'
+require 'spandx/parsers/sln'
 
 module Spandx
   module Parsers

data/lib/spandx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spandx
-  VERSION = '0.1.7'
+  VERSION = '0.2.0'
 end

data/lib/spandx.rb

@@ -6,6 +6,7 @@ require 'json'
 require 'net/hippie'
 require 'nokogiri'
 require 'pathname'
+require 'text'
 
 require 'spandx/catalogue'
 require 'spandx/content'

data/spandx.gemspec

@@ -33,10 +33,12 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'bundler', '>= 1.16', '< 3.0.0'
   spec.add_dependency 'net-hippie', '~> 0.3'
   spec.add_dependency 'nokogiri', '~> 1.10'
+  spec.add_dependency 'text', '~> 1.3'
   spec.add_dependency 'thor', '~> 0.1'
   spec.add_development_dependency 'bundler-audit', '~> 0.6'
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rspec-benchmark', '~> 0.5'
   spec.add_development_dependency 'rubocop', '~> 0.52'
   spec.add_development_dependency 'rubocop-rspec', '~> 1.22'
   spec.add_development_dependency 'vcr', '~> 5.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spandx
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.2.0
 platform: ruby
 authors:
 - mo khan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-28 00:00:00.000000000 Z
+date: 2020-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,6 +58,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: text
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -114,6 +128,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec-benchmark
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -205,6 +233,7 @@ files:
 - lib/spandx/parsers/gemfile_lock.rb
 - lib/spandx/parsers/packages_config.rb
 - lib/spandx/parsers/pipfile_lock.rb
+- lib/spandx/parsers/sln.rb
 - lib/spandx/report.rb
 - lib/spandx/version.rb
 - spandx.gemspec