licensee 9.13.2 → 9.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d931f50190ecf7abb790530607bd57eb31c7190926a394bca7aa9ec0550cfba8
-  data.tar.gz: f119b575b2ff9538133a587ef3a23638756fac47e30c40b442ddad679bb62036
+  metadata.gz: 37e4eac01861b05d0484cc0e81e7c6865386004e2529c2c74e3be8d9e350c14b
+  data.tar.gz: c506aa44124763dd74b0fa5771fe0c0f750e43a39020dc90d03a716f97861633
 SHA512:
-  metadata.gz: 4b423e68fb6496eefc0f4259fac2539f34430a13e1eb6d3758a6876c604fc40e5a763a04836025070410c082c2a516b28988ceeb46ed2b2a06276b318b9d0fb6
-  data.tar.gz: f0e150efc09980729793f86bbfcff323617349f0cae92ff2d4ebace2e29dac96e980deacc224b6ab08a21e3224f77b95541bdfaab9fa016ba81a6f94c3fdcce7
+  metadata.gz: df862cae20c0deeaf1caa8c74456603aa91aae1d180103c415885343a0af3ccd6b4bd6d683c1af351b038e50225488d86e83ad031bb061ebf381da38b9ac1537
+  data.tar.gz: be9e4697ce9f1ca2f410cc7bcb3ef952435f75470d8f69e1236a2aa66f6104a4f6446c185f50117278f2a2c06e455bb68aa5436a9af494a761eb852030c2aa04

data/lib/licensee/content_helper.rb

@@ -112,32 +112,27 @@ module Licensee
       @wordset ||= content_normalized&.scan(%r{(?:[\w\/](?:'s|(?<=s)')?)+})&.to_set
     end
 
-    # Number of characteres in the normalized content
+    # Number of characters in the normalized content
     def length
       return 0 unless content_normalized
 
       content_normalized.length
     end
 
-    # Number of characters that could be added/removed to still be
-    # considered a potential match
-    def max_delta
-      @max_delta ||= fields_normalized.size * 10 +
-                     (length * Licensee.inverse_confidence_threshold).to_i
-    end
-
     # Given another license or project file, calculates the difference in length
     def length_delta(other)
       (length - other.length).abs
     end
 
     # Given another license or project file, calculates the similarity
-    # as a percentage of words in common
+    # as a percentage of words in common, minus a tiny penalty that
+    # increases with size difference between licenses so that false
+    # positives for long licnses are ruled out by this score alone.
     def similarity(other)
       overlap = (wordset_fieldless & other.wordset).size
       total = wordset_fieldless.size + other.wordset.size -
               fields_normalized_set.size
-      100.0 * (overlap * 2.0 / total)
+      (overlap * 200.0) / (total + length_delta(other) / 10)
     end
 
     # SHA1 of the normalized content

data/lib/licensee/matchers.rb

@@ -12,6 +12,7 @@ module Licensee
     autoload :Exact,     'licensee/matchers/exact'
     autoload :Gemspec,   'licensee/matchers/gemspec'
     autoload :NpmBower,  'licensee/matchers/npm_bower'
+    autoload :NuGet,     'licensee/matchers/nuget'
     autoload :Package,   'licensee/matchers/package'
     autoload :Reference, 'licensee/matchers/reference'
     autoload :Spdx,      'licensee/matchers/spdx'

data/lib/licensee/matchers/dice.rb

@@ -26,7 +26,7 @@ module Licensee
             if license.creative_commons? && file.potential_false_positive?
               false
             else
-              license.wordset && license.length_delta(file) <= license.max_delta
+              license.wordset
             end
           end
         end

data/lib/licensee/matchers/nuget.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Licensee
+  module Matchers
+    class NuGet < Licensee::Matchers::Package
+      # While we could parse the nuspec file, prefer a lenient regex for speed and security.
+      # Moar parsing moar problems.
+      LICENSE_REGEX = %r{
+        <license\s*type\s*=\s*["']expression["']\s*>([a-z\-0-9\. +()]+)<\/license\s*>
+      }ix.freeze
+
+      LICENSE_URL_REGEX = %r{<licenseUrl>\s*(.*)\s*<\/licenseUrl>}i.freeze
+
+      NUGET_REGEX = %r{https?:\/\/licenses.nuget.org\/(.*)}i.freeze
+      OPENSOURCE_REGEX = %r{https?:\/\/(?:www\.)?opensource.org\/licenses\/(.*)}i.freeze
+      SPDX_REGEX = %r{https?:\/\/(?:www\.)?spdx.org\/licenses\/(.*?)(?:\.html|\.txt)?$}i.freeze
+      APACHE_REGEX = %r{https?:\/\/(?:www\.)?apache.org\/licenses\/(.*?)(?:\.html|\.txt)?$}i.freeze
+
+      private
+
+      def license_from_first_capture(url, pattern)
+        match = url.match(pattern)
+        match[1].downcase if match && match[1]
+      end
+
+      def license_from_url(url)
+        license_from_first_capture(url, NUGET_REGEX) ||
+          license_from_first_capture(url, OPENSOURCE_REGEX) ||
+          license_from_first_capture(url, SPDX_REGEX) ||
+          license_from_first_capture(url, APACHE_REGEX)&.gsub('license', 'apache')
+      end
+
+      def license_property
+        # Prefer the explicit <license type="expression"> element
+        match = @file.content.match LICENSE_REGEX
+        return match[1].downcase if match && match[1]
+
+        url_match = @file.content.match LICENSE_URL_REGEX
+        license_from_url(url_match[1]) if url_match && url_match[1]
+      end
+    end
+  end
+end

data/lib/licensee/project_files/package_manager_file.rb

@@ -7,7 +7,8 @@ module Licensee
       MATCHERS_EXTENSIONS = {
         '.gemspec' => [Matchers::Gemspec],
         '.json'    => [Matchers::NpmBower],
-        '.cabal'   => [Matchers::Cabal]
+        '.cabal'   => [Matchers::Cabal],
+        '.nuspec'  => [Matchers::NuGet]
       }.freeze
 
       # Hash of Filename => [possible matchers]
@@ -33,7 +34,7 @@ module Licensee
       end
 
       def self.name_score(filename)
-        return 1.0 if ['.gemspec', '.cabal'].include?(File.extname(filename))
+        return 1.0 if ['.gemspec', '.cabal', '.nuspec'].include?(File.extname(filename))
 
         FILENAMES_SCORES[filename] || 0.0
       end

data/lib/licensee/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Licensee
-  VERSION = '9.13.2'
+  VERSION = '9.14.0'
 end

data/spec/licensee/content_helper_spec.rb

@@ -55,21 +55,13 @@ RSpec.describe Licensee::ContentHelper do
     expect(subject.length).to be(135)
   end
 
-  context 'a very long license' do
-    let(:content) { 'license' * 1000 }
-
-    it 'returns the max delta' do
-      expect(subject.max_delta).to be(140)
-    end
-  end
-
   it 'knows the length delta' do
     expect(subject.length_delta(mit)).to be(885)
     expect(subject.length_delta(subject)).to be(0)
   end
 
   it 'knows the similarity' do
-    expect(subject.similarity(mit)).to be_within(1).of(11)
+    expect(subject.similarity(mit)).to be_within(1).of(6)
     expect(subject.similarity(subject)).to be(100.0)
   end
 

data/spec/licensee/matchers/dice_matcher_spec.rb

@@ -6,6 +6,7 @@ RSpec.describe Licensee::Matchers::Dice do
   let(:mit) { Licensee::License.find('mit') }
   let(:gpl) { Licensee::License.find('gpl-3.0') }
   let(:agpl) { Licensee::License.find('agpl-3.0') }
+  let(:lgpl) { Licensee::License.find('lgpl-2.1') }
   let(:cc_by) { Licensee::License.find('cc-by-4.0') }
   let(:cc_by_sa) { Licensee::License.find('cc-by-sa-4.0') }
   let(:content) { sub_copyright_info(gpl) }
@@ -19,18 +20,10 @@ RSpec.describe Licensee::Matchers::Dice do
     expect(subject.match).to eql(gpl)
   end
 
-  it 'builds a list of potential licenses' do
-    expect(subject.potential_matches).to eql([agpl, gpl])
-  end
-
   it 'sorts licenses by similarity' do
     expect(subject.matches_by_similarity[0]).to eql([gpl, 100.0])
-    expect(subject.matches_by_similarity[1]).to eql([agpl, 95.6842105263158])
-  end
-
-  it 'returns a list of licenses above the confidence threshold' do
-    expect(subject.matches_by_similarity[0]).to eql([gpl, 100.0])
-    expect(subject.matches_by_similarity[1]).to eql([agpl, 95.6842105263158])
+    expect(subject.matches_by_similarity[1]).to eql([agpl, 95.28301886792453])
+    expect(subject.matches_by_similarity[2]).to eql([lgpl, 39.33253873659118])
   end
 
   it 'returns the match confidence' do

data/spec/licensee/matchers/nu_get_matcher_spec.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+RSpec.describe Licensee::Matchers::NuGet do
+  subject { described_class.new(file) }
+
+  let(:content) { '<license type="expression">mit</license>' }
+  let(:file) { Licensee::ProjectFiles::LicenseFile.new(content, 'foo.nuspec') }
+  let(:mit) { Licensee::License.find('mit') }
+  let(:apache2) { Licensee::License.find('apache-2.0') }
+  let(:other) { Licensee::License.find('other') }
+
+  it 'matches' do
+    expect(subject.match).to eql(mit)
+  end
+
+  it 'has a confidence' do
+    expect(subject.confidence).to be(90)
+  end
+
+  {
+    'double quotes'      => '<license type="expression">mit</license>',
+    'single quotes'      => "<license type='expression'>mit</license>",
+    'whitespace'         => '<license  type = "expression" >mit</license >',
+    'leading whitespace' => ' <license type="expression">mit</license>'
+  }.each do |description, license_declaration|
+    context "with a #{description} license element" do
+      let(:content) { license_declaration }
+
+      it 'matches' do
+        expect(subject.match).to eql(mit)
+      end
+    end
+  end
+
+  context 'no license field' do
+    let(:content) { '<file>wrongelement</file>' }
+
+    it 'returns nil' do
+      expect(subject.match).to be_nil
+    end
+  end
+
+  context 'an unknown license' do
+    let(:content) { '<license type="expression">foo</license>' }
+
+    it 'returns other' do
+      expect(subject.match).to eql(other)
+    end
+  end
+
+  context 'a license expression' do
+    let(:content) { '<license type="expression">BSD-2-Clause OR MIT</license>' }
+
+    it 'returns other' do
+      expect(subject.match).to eql(other)
+    end
+  end
+
+  {
+    'nuget'            => '<licenseUrl>https://licenses.nuget.org/Apache-2.0</licenseUrl>',
+    'nuget (http)'     => '<licenseUrl>http://licenses.nuget.org/Apache-2.0</licenseUrl>',
+    'opensource'       => '<licenseUrl>https://opensource.org/licenses/Apache-2.0</licenseUrl>',
+    'opensource (www)' => '<licenseUrl>http://www.opensource.org/licenses/Apache-2.0</licenseUrl>',
+    'spdx'             => '<licenseUrl>https://spdx.org/licenses/Apache-2.0</licenseUrl>',
+    'spdx (www)'       => '<licenseUrl>http://www.spdx.org/licenses/Apache-2.0</licenseUrl>',
+    'spdx (html)'      => '<licenseUrl>https://spdx.org/licenses/Apache-2.0.html</licenseUrl>',
+    'spdx (txt)'       => '<licenseUrl>https://spdx.org/licenses/Apache-2.0.txt</licenseUrl>'
+  }.each do |description, license_declaration|
+    context "with a #{description} licenseUrl element containing SPDX" do
+      let(:content) { license_declaration }
+
+      it 'matches' do
+        expect(subject.match).to eql(apache2)
+      end
+    end
+  end
+
+  {
+    '2.0 (https)'    => '<licenseUrl>https://apache.org/licenses/LICENSE-2.0</licenseUrl>',
+    '2.0 (http/www)' => '<licenseUrl>http://www.apache.org/licenses/LICENSE-2.0</licenseUrl>',
+    '2.0 (txt)'      => '<licenseUrl>https://apache.org/licenses/LICENSE-2.0.txt</licenseUrl>'
+  }.each do |description, license_declaration|
+    context "with an apache.org #{description} licenseUrl element" do
+      let(:content) { license_declaration }
+
+      it 'matches' do
+        expect(subject.match).to eql(apache2)
+      end
+    end
+  end
+end

data/spec/licensee/project_files/package_info_spec.rb

@@ -71,5 +71,13 @@ RSpec.describe Licensee::ProjectFiles::PackageManagerFile do
         expect(possible_matchers).to eql([Licensee::Matchers::Cran])
       end
     end
+
+    context 'with nuspec file' do
+      let(:filename) { 'foo.nuspec' }
+
+      it 'returns the NuGet matcher' do
+        expect(possible_matchers).to eql([Licensee::Matchers::NuGet])
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: licensee
 version: !ruby/object:Gem::Version
-  version: 9.13.2
+  version: 9.14.0
 platform: ruby
 authors:
 - Ben Balter
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-28 00:00:00.000000000 Z
+date: 2020-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dotenv
@@ -237,6 +237,7 @@ files:
 - lib/licensee/matchers/gemspec.rb
 - lib/licensee/matchers/matcher.rb
 - lib/licensee/matchers/npm_bower.rb
+- lib/licensee/matchers/nuget.rb
 - lib/licensee/matchers/package.rb
 - lib/licensee/matchers/reference.rb
 - lib/licensee/matchers/spdx.rb
@@ -323,6 +324,7 @@ files:
 - spec/licensee/matchers/gemspec_matcher_spec.rb
 - spec/licensee/matchers/matcher_spec.rb
 - spec/licensee/matchers/npm_bower_matcher_spec.rb
+- spec/licensee/matchers/nu_get_matcher_spec.rb
 - spec/licensee/matchers/package_matcher_spec.rb
 - spec/licensee/matchers/reference_matcher_spec.rb
 - spec/licensee/matchers/spdx_matcher_spec.rb
@@ -398,7 +400,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.3
 signing_key: 
 specification_version: 4
 summary: A Ruby Gem to detect open source project licenses