spandx 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4da284f37427fcbe7d30317385ddf6e6eab0100046ccb8c0987bd4031072ad45
-  data.tar.gz: 9c14d614550b6f34ad01e85a47ee4445281c94455b1902dd340ac47ec995a529
+  metadata.gz: 4e7053e8bfb9564f28b5ebe1b4b2250966eaf94f0edc22503e76d6d2fca76427
+  data.tar.gz: fb273d38a70bd4fa58d674c1a4f9032e76c0a1967c6d1390e78aa7671b37c3ca
 SHA512:
-  metadata.gz: b12af7d885681938adc39cd0ae9d2b4987eccd87e95d200ced84c8d8715101f6d99e32498e62f567b7ad35035ac1570071418636e844b3bdfccfb35ddbeafb6f
-  data.tar.gz: 5bb060dc2fb56a1bc980b6bca8e65c9f7fd68f8b4aa414fd2edfe9ba974ea23b01721a7c24f6755b26fe3596798c2fe3ff917a23687a1359cda17ab3bbb8696d
+  metadata.gz: f71bcfd8fd4fc3e0ce4081db8f397f3051ff58a1e76e5c214c410487b913163b584547b5a179d5140a45fb81be061676d3b65d52ad31ca7ae09389c69710d40a
+  data.tar.gz: c56ea13c20421a44e2dc29a33420187c1dbe017b0c02d38c10aa3d65cd6ddd5764ccbca3f1f9e65662b41bb3bd02bb322aea60f560a85551f5d8116ac288f792

data/CHANGELOG.md

@@ -1,4 +1,4 @@
-Version 0.1.5
+Version 0.1.6
 
 # Changelog
 
@@ -8,8 +8,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
 - nil
 
+## [0.1.6] - 2020-01-27
+### Added
+- Scan csproj files that depend on other project files
+- Replace licensee dependency with simple tokenizer
+- Fetch license data from git clone of SPDX license list data
+
 ## [0.1.5] - 2020-01-23
 ### Added
 - Exclude `nil` licenses from report
@@ -36,7 +43,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Provide ruby API to the latest SPDX catalogue.
 
-[Unreleased]: https://github.com/mokhan/spandx/compare/v0.1.5...HEAD
+[Unreleased]: https://github.com/mokhan/spandx/compare/v0.1.6...HEAD
+[0.1.6]: https://github.com/mokhan/spandx/compare/v0.1.5...v0.1.6
 [0.1.5]: https://github.com/mokhan/spandx/compare/v0.1.4...v0.1.5
 [0.1.4]: https://github.com/mokhan/spandx/compare/v0.1.3...v0.1.4
 [0.1.3]: https://github.com/mokhan/spandx/compare/v0.1.2...v0.1.3

data/lib/spandx/catalogue.rb

@@ -18,17 +18,25 @@ module Spandx
 
     def each
       licenses.each do |license|
-        yield license if present?(license.id)
+        yield license
       end
     end
 
     class << self
       def latest(gateway: ::Spandx::Gateways::Spdx.new)
-        gateway.fetch
+        new(gateway.fetch)
+      end
+
+      def from_json(json)
+        new(JSON.parse(json, symbolize_names: true))
       end
 
       def from_file(path)
-        new(JSON.parse(IO.read(path), symbolize_names: true))
+        from_json(IO.read(path))
+      end
+
+      def from_git
+        from_json(Spandx.db.read('json/licenses.json'))
       end
 
       def empty
@@ -41,11 +49,7 @@ module Spandx
     attr_reader :catalogue
 
     def licenses
-      @licenses ||= identity_map.values
-    end
-
-    def map_from(license_hash)
-      License.new(license_hash)
+      @licenses ||= identity_map.values.sort
     end
 
     def present?(item)
@@ -55,7 +59,7 @@ module Spandx
     def identity_map
       @identity_map ||=
         catalogue.fetch(:licenses, []).each_with_object({}) do |hash, memo|
-          license = map_from(hash)
+          license = License.new(hash)
           memo[license.id] = license if present?(license.id)
         end
     end

data/lib/spandx/content.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Spandx
+  class Content
+    attr_reader :tokens, :threshold
+
+    def initialize(content, threshold: 89.0)
+      @threshold = threshold
+      @tokens = tokenize(canonicalize(content)).to_set
+    end
+
+    def similar?(other)
+      similarity_score(other) > threshold
+    end
+
+    # https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Dice%27s_coefficient#Ruby
+    def similarity_score(other)
+      overlap = (tokens & other.tokens).size
+      total = tokens.size + other.tokens.size
+      100.0 * (overlap * 2.0 / total)
+    end
+
+    private
+
+    def canonicalize(content)
+      content&.downcase
+    end
+
+    def tokenize(content)
+      content.to_s.scan(/[a-zA-Z]+/)
+    end
+
+    def blank?(content)
+      content.nil? || content.chomp.strip.empty?
+    end
+  end
+end

data/lib/spandx/database.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Spandx
+  class Database
+    attr_reader :path, :url
+
+    def initialize(url:)
+      @url = url
+      @path = path_for(url)
+    end
+
+    def update!
+      dotgit? ? pull! : clone!
+    end
+
+    def read(file)
+      IO.read(File.join(path, file))
+    end
+
+    private
+
+    def path_for(url)
+      uri = URI.parse(url)
+      name = uri.path.gsub(/\.git$/, '')
+      File.expand_path(File.join(Dir.home, '.local', 'share', name))
+    end
+
+    def dotgit?
+      File.directory?(File.join(path, '.git'))
+    end
+
+    def clone!
+      system('git', 'clone', '--quiet', url, path)
+    end
+
+    def pull!
+      within do
+        system('git', 'pull', '--no-rebase', '--quiet', 'origin', 'master')
+      end
+    end
+
+    def within
+      Dir.chdir(path) do
+        yield
+      end
+    end
+  end
+end

data/lib/spandx/gateways/nuget.rb

@@ -6,8 +6,9 @@ module Spandx
     # https://api.nuget.org/v3-flatcontainer/#{package.name}/index.json
     # https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource
     class Nuget
-      def initialize(http: Spandx.http)
+      def initialize(http: Spandx.http, catalogue:)
         @http = http
+        @catalogue = catalogue
       end
 
       def licenses_for(name, version)
@@ -19,7 +20,7 @@ module Spandx
 
       private
 
-      attr_reader :http
+      attr_reader :http, :catalogue
 
       def nuspec_url_for(name, version)
         "https://api.nuget.org/v3-flatcontainer/#{name}/#{version}/#{name}.nuspec"
@@ -29,26 +30,20 @@ module Spandx
         from_xml(http.get(nuspec_url_for(name, version)).body)
       end
 
-      def guess_license_in(content)
-        Licensee::ProjectFiles::LicenseFile.new(content).license.key.upcase
-      end
-
       def from_xml(xml)
         Nokogiri::XML(xml).tap(&:remove_namespaces!)
       end
 
       def exact_licenses_from(document)
-        if (licenses = document.search('//package/metadata/license')).any?
-          return licenses.map(&:text)
-        end
-
-        nil
+        licenses = document.search('//package/metadata/license')
+        licenses.map(&:text) if licenses.any?
       end
 
       def guess_licenses_from(document)
+        guess = Guess.new(catalogue)
         document
           .search('//package/metadata/licenseUrl')
-          .map { |node| guess_license_in(Spandx.http.get(node.text).body) }
+          .map { |node| guess.license_for(http.get(node.text).body) }
       end
     end
   end

data/lib/spandx/gateways/spdx.rb

@@ -5,7 +5,7 @@ module Spandx
     class Spdx
       URL = 'https://spdx.org/licenses/licenses.json'
 
-      def fetch(url: URL, http: Spandx.http, default: Catalogue.empty)
+      def fetch(url: URL, http: Spandx.http, default: {})
         response = http.get(url, default: default)
         http.ok?(response) ? parse(response.body) : default
       end
@@ -13,7 +13,7 @@ module Spandx
       private
 
       def parse(json)
-        Catalogue.new(JSON.parse(json, symbolize_names: true))
+        JSON.parse(json, symbolize_names: true)
       end
     end
   end

data/lib/spandx/guess.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Spandx
+  class Guess
+    class Score
+      include Comparable
+
+      attr_reader :score, :item
+
+      def initialize(score, item)
+        @score = score
+        @item = item
+      end
+
+      def <=>(other)
+        score <=> other.score
+      end
+
+      def to_s
+        "#{score}: #{item}"
+      end
+    end
+
+    attr_reader :catalogue
+
+    def initialize(catalogue)
+      @catalogue = catalogue
+    end
+
+    def license_for(raw_content)
+      content = Content.new(raw_content)
+
+      max_score = nil
+      catalogue.each do |license|
+        next if license.deprecated_license_id?
+
+        percentage = content.similarity_score(license.content)
+        if max_score.nil? || percentage > max_score.score
+          max_score = Score.new(percentage, license)
+        end
+      end
+      max_score.item.id
+    end
+  end
+end

data/lib/spandx/license.rb

@@ -59,5 +59,17 @@ module Spandx
     def reference_number=(value)
       attributes[:referenceNumber] = value
     end
+
+    def content
+      @content ||= Content.new(Spandx.db.read("text/#{id}.txt"))
+    end
+
+    def <=>(other)
+      id <=> other.id
+    end
+
+    def to_s
+      id
+    end
   end
 end

data/lib/spandx/parsers/csproj/package_reference.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Parsers
+    class Csproj
+      PackageReference = Struct.new(:name, :version, keyword_init: true)
+    end
+  end
+end

data/lib/spandx/parsers/csproj/project_file.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Parsers
+    class Csproj
+      class ProjectFile
+        attr_reader :catalogue, :document, :nuget
+
+        def initialize(path)
+          @path = path
+          @dir = File.dirname(path)
+          @document = Nokogiri::XML(IO.read(path))
+        end
+
+        def package_references
+          other = project_references.map(&:package_references).flatten
+          other + document.search('//PackageReference').map do |node|
+            PackageReference.new(
+              name: attribute_for('Include', node),
+              version: attribute_for('Version', node)
+            )
+          end
+        end
+
+        private
+
+        def project_references
+          document.search('//ProjectReference').map do |node|
+            relative_project_path = node.attribute('Include').value.strip.tr('\\', '/')
+            absolute_project_path = File.expand_path(File.join(@dir, relative_project_path))
+            self.class.new(absolute_project_path)
+          end
+        end
+
+        def attribute_for(key, node)
+          node.attribute(key)&.value&.strip ||
+            node.at_xpath("./#{key}")&.content&.strip
+        end
+      end
+    end
+  end
+end

data/lib/spandx/parsers/csproj.rb

@@ -8,32 +8,33 @@ module Spandx
       end
 
       def parse(lockfile)
-        document = from_xml(IO.read(lockfile))
-        document.search('//PackageReference').map do |node|
-          name = attribute_for('Include', node)
-          version = attribute_for('Version', node)
-          Dependency.new(
-            name: name,
-            version: version,
-            licenses: nuget.licenses_for(name, version).map { |x| catalogue[x] }
-          )
-        end
+        ProjectFile
+          .new(lockfile)
+          .package_references
+          .map { |x| map_from(x) }
       end
 
       private
 
-      def from_xml(xml)
-        Nokogiri::XML(xml)
+      def map_from(package_reference)
+        Dependency.new(
+          name: package_reference.name,
+          version: package_reference.version,
+          licenses: licenses_for(package_reference)
+        )
       end
 
-      def attribute_for(key, node)
-        node.attribute(key)&.value&.strip ||
-          node.at_xpath("./#{key}")&.content&.strip
+      def licenses_for(package_reference)
+        nuget
+          .licenses_for(package_reference.name, package_reference.version)
+          .map { |x| catalogue[x] }
       end
 
       def nuget
-        @nuget ||= Gateways::Nuget.new
+        @nuget ||= Gateways::Nuget.new(catalogue: catalogue)
       end
     end
   end
 end
+require 'spandx/parsers/csproj/package_reference'
+require 'spandx/parsers/csproj/project_file'

data/lib/spandx/parsers/packages_config.rb

@@ -30,7 +30,7 @@ module Spandx
       end
 
       def nuget
-        @nuget ||= Gateways::Nuget.new
+        @nuget ||= Gateways::Nuget.new(catalogue: catalogue)
       end
     end
   end

data/lib/spandx/parsers.rb

@@ -15,7 +15,7 @@ module Spandx
     end
 
     class << self
-      def for(path, catalogue: Spandx::Catalogue.latest)
+      def for(path, catalogue: Spandx::Catalogue.from_git)
         result = ::Spandx::Parsers::Base.find do |x|
           x.matches?(File.basename(path))
         end

data/lib/spandx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Spandx
-  VERSION = '0.1.5'
+  VERSION = '0.1.6'
 end

data/lib/spandx.rb

@@ -3,18 +3,20 @@
 require 'bundler'
 require 'forwardable'
 require 'json'
-require 'licensee'
 require 'net/hippie'
 require 'nokogiri'
 require 'pathname'
 
 require 'spandx/catalogue'
+require 'spandx/content'
+require 'spandx/database'
 require 'spandx/dependency'
 require 'spandx/gateways/http'
 require 'spandx/gateways/nuget'
 require 'spandx/gateways/pypi'
 require 'spandx/gateways/rubygems'
 require 'spandx/gateways/spdx'
+require 'spandx/guess'
 require 'spandx/license'
 require 'spandx/parsers'
 require 'spandx/report'
@@ -31,5 +33,9 @@ module Spandx
     def http
       @http ||= Spandx::Gateways::Http.new
     end
+
+    def db
+      @db ||= Spandx::Database.new(url: 'https://github.com/spdx/license-list-data.git').tap(&:update!)
+    end
   end
 end

data/spandx.gemspec

@@ -29,8 +29,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
+  spec.required_ruby_version = '>= 2.5.0'
   spec.add_dependency 'bundler', '>= 1.16', '< 3.0.0'
-  spec.add_dependency 'licensee', '~> 9.13'
   spec.add_dependency 'net-hippie', '~> 0.3'
   spec.add_dependency 'nokogiri', '~> 1.10'
   spec.add_dependency 'thor', '~> 0.1'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spandx
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - mo khan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-01-23 00:00:00.000000000 Z
+date: 2020-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -30,20 +30,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
-- !ruby/object:Gem::Dependency
-  name: licensee
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '9.13'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '9.13'
 - !ruby/object:Gem::Dependency
   name: net-hippie
   requirement: !ruby/object:Gem::Requirement
@@ -201,16 +187,21 @@ files:
 - lib/spandx/cli.rb
 - lib/spandx/command.rb
 - lib/spandx/commands/scan.rb
+- lib/spandx/content.rb
+- lib/spandx/database.rb
 - lib/spandx/dependency.rb
 - lib/spandx/gateways/http.rb
 - lib/spandx/gateways/nuget.rb
 - lib/spandx/gateways/pypi.rb
 - lib/spandx/gateways/rubygems.rb
 - lib/spandx/gateways/spdx.rb
+- lib/spandx/guess.rb
 - lib/spandx/license.rb
 - lib/spandx/parsers.rb
 - lib/spandx/parsers/base.rb
 - lib/spandx/parsers/csproj.rb
+- lib/spandx/parsers/csproj/package_reference.rb
+- lib/spandx/parsers/csproj/project_file.rb
 - lib/spandx/parsers/gemfile_lock.rb
 - lib/spandx/parsers/packages_config.rb
 - lib/spandx/parsers/pipfile_lock.rb
@@ -232,7 +223,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="