licensee 9.19.0 → 9.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4878becb1edfcb446503f89645e4daeeacc60715ca978a8d4391418b0493f255
-  data.tar.gz: 651e41f44d60ab16669584edeb693967e4aa1a6a6c911c7a944fc6b35bd05503
+  metadata.gz: 5c28e30b13570d0619bc9ec837c115c8184e3ecd3a46f17a5a3146b425c75262
+  data.tar.gz: 79fdbe8f8702fa484280083757db94ec93ac7106f3adb5b54aa3287ad7edbc45
 SHA512:
-  metadata.gz: c7e96fea4cd873710794c9e68f93edc8f016a620c55464f0f5c8d8ba82c90de8195376f858d8bc6125bb1ac170e980a98557fd135d68ab38ed98e2ba1d5e030e
-  data.tar.gz: caf4cc661f13ca7ce0ea847e6cd95ec45dd21472eb03cb6862d0cb39c52823818a70373b1133f55570492297543e7db6d4c7adeeefdbef6dd8c5795f67b19919
+  metadata.gz: f761cf9ba0303f36693af8d2b91acf2c64b35daf7a2409e7ba7c463a8bba3eea1c7f16a8657e7af47694bbcb640d46507bf341ce62198241d66066e1a0fc8991
+  data.tar.gz: 66599cd5fda2597a8d063c547383a259b16b1d8854e7c88057096a2b054193683d3ff2ffd7b47738f30d81d49a68371eee8025fd139187cf9fd8a2bb71b68bc6

data/lib/licensee/content_helper/constants.rb

@@ -24,6 +24,8 @@ module Licensee
         developed_by:        /#{START_REGEX}developed by:.*?\n\n/im,
         cc_dedication:       /The\s+text\s+of\s+the\s+Creative\s+Commons.*?Public\s+Domain\s+Dedication\./im,
         cc_wiki:             /wiki\.creativecommons\.org/i,
+        cc_preamble:         /creative\s+commons\s+corporation.*?(?=by\s+exercising\s+the\s+licensed\s+rights)/im,
+        cc_notice:           /creative\s+commons\s+is\s+not\s+a\s+party\s+to\s+its\s+public\s+licenses\..*\z/im,
         cc_legal_code:       /^\s*Creative Commons Legal Code\s*$/i,
         cc0_info:            /For more information, please see\s*\S+zero\S+/im,
         cc0_disclaimer:      /CREATIVE COMMONS CORPORATION.*?\n\n/im,
@@ -81,7 +83,7 @@ module Licensee
         'sub license'     => 'sublicense',
         'utilisation'     => 'utilization',
         'whilst'          => 'while',
-        'wilful'          => 'wilfull',
+        'wilful'          => 'willful',
         'non-commercial'  => 'noncommercial',
         'per cent'        => 'percent',
         'copyright owner' => 'copyright holder'

data/lib/licensee/content_helper/normalization_methods.rb

@@ -65,8 +65,13 @@ module Licensee
       end
 
       def strip_copyright
-        regex = Regexp.union(Matchers::Copyright::REGEX, ContentHelper::REGEXES[:all_rights_reserved])
-        strip(regex) while _content =~ regex
+        copyright_notice_regex = Matchers::Copyright::MAIN_LINE_REGEX
+        copyright_regex = Regexp.union(Matchers::Copyright::REGEX, ContentHelper::REGEXES[:all_rights_reserved])
+        # Strip opening paragraph only when "All rights reserved." is present — confirms attribution, not license text.
+        strip(/\A.*?(?=\n\n)/m) if (p = _content[/\A.*?(?=\n\n)/m]) &&
+                                   p =~ copyright_notice_regex && /all rights reserved/i.match?(p)
+        # Strip any remaining copyright lines (e.g. when no blank line is present)
+        strip(copyright_regex) while _content =~ copyright_regex
       end
 
       def strip_cc0_optional
@@ -82,6 +87,8 @@ module Licensee
 
         strip(ContentHelper::REGEXES[:cc_dedication])
         strip(ContentHelper::REGEXES[:cc_wiki])
+        strip(ContentHelper::REGEXES[:cc_preamble]) if _content.include? 'creative commons corporation'
+        strip(ContentHelper::REGEXES[:cc_notice]) if _content.include? 'creative commons is not a party'
       end
 
       def strip_unlicense_optional
@@ -125,7 +132,12 @@ module Licensee
       end
 
       def normalize_spelling
-        normalize(/\b#{Regexp.union(ContentHelper::VARIETAL_WORDS.keys)}\b/, ContentHelper::VARIETAL_WORDS)
+        # Use flexible whitespace between words so that line-wrapped content
+        # (e.g. "copyright\nowner") is still normalized correctly.
+        ContentHelper::VARIETAL_WORDS.each do |phrase, replacement|
+          pattern = phrase.split.map { |w| Regexp.escape(w) }.join('\s+')
+          @_content = _content.gsub(/\b#{pattern}\b/, replacement)
+        end
       end
 
       def normalize_bullets

data/lib/licensee/content_helper/similarity_methods.rb

@@ -7,12 +7,26 @@ module Licensee
       # Given another license or project file, calculates the similarity
       # as a percentage of words in common, minus a tiny penalty that
       # increases with size difference between licenses so that false
-      # positives for long licnses are ruled out by this score alone.
+      # positives for long licenses are ruled out by this score alone.
       def similarity(other)
         overlap = (wordset_fieldless & other.wordset).size
         (overlap * 200.0) / similarity_denominator(other)
       end
 
+      # Given another license or project file, calculates the Dice coefficient
+      # over bigrams (consecutive word pairs).  Unlike wordset similarity this
+      # is sensitive to word order, making it resistant to adversarial scrambling
+      # where all the correct words appear but in the wrong sequence.
+      def bigram_similarity(other)
+        my_bigrams = bigrams
+        other_bigrams = other.bigrams
+        total = my_bigrams.size + other_bigrams.size
+        return 0.0 if total.zero?
+
+        overlap = (my_bigrams & other_bigrams).size
+        (overlap * 200.0) / total
+      end
+
       private
 
       def wordset_fieldless

data/lib/licensee/content_helper.rb

@@ -14,7 +14,18 @@ module Licensee
 
     # A set of each word in the license, without duplicates
     def wordset
-      @wordset ||= content_normalized&.scan(%r{(?:[\w/-](?:'s|(?<=s)')?)+})&.to_set
+      @wordset ||= words&.to_set
+    end
+
+    # A set of consecutive word pairs (bigrams) in the license, without duplicates.
+    # Unlike wordset, bigrams are order-sensitive, making similarity scores
+    # robust against adversarial word scrambling (see GitHub issue #602).
+    def bigrams
+      @bigrams ||= if words.nil? || words.length < 2
+                     Set.new
+                   else
+                     words.each_cons(2).to_set { |a, b| "#{a} #{b}" }
+                   end
     end
 
     # Number of characters in the normalized content
@@ -72,7 +83,7 @@ module Licensee
 
     def self.title_regex
       @title_regex ||= begin
-        licenses = Licensee::License.all(hidden: true, psuedo: false)
+        licenses = Licensee::License.all(hidden: true, pseudo: false)
         titles = licenses.map(&:title_regex)
 
         # Title regex must include the version to support matching within
@@ -90,6 +101,12 @@ module Licensee
 
     private
 
+    # Ordered array of words extracted from the normalized content.
+    # Memoized so that both wordset and bigrams share the same scan result.
+    def words
+      @words ||= content_normalized&.scan(%r{(?:[\w/-](?:'s|(?<=s)')?)+})
+    end
+
     def _content
       @_content ||= content.to_s.dup.strip
     end

data/lib/licensee/matchers/cabal.rb

@@ -6,7 +6,8 @@ module Licensee
     class Cabal < Licensee::Matchers::Package
       # While we could parse the cabal file, prefer
       # a lenient regex for speed and security. Moar parsing moar problems.
-      LICENSE_REGEX = /^\s*license\s*:\s*([a-z\-0-9.]+)\s*$/ix
+      # The "+" suffix is the pre-SPDX Cabal notation for "or-later" (e.g. GPL-2+).
+      LICENSE_REGEX = /^\s*license\s*:\s*([a-z\-0-9.+]+)\s*$/ix
       LICENSE_CONVERSIONS = {
         'GPL-2'  => 'GPL-2.0',
         'GPL-3'  => 'GPL-3.0',
@@ -24,7 +25,9 @@ module Licensee
       end
 
       def spdx_name(cabal_name)
-        LICENSE_CONVERSIONS[cabal_name] || cabal_name
+        # Strip pre-SPDX "or-later" suffix (+) before looking up conversions
+        normalized = cabal_name.chomp('+')
+        LICENSE_CONVERSIONS[normalized] || normalized
       end
     end
   end

data/lib/licensee/matchers/dice.rb

@@ -43,8 +43,9 @@ module Licensee
       alias licenses_by_similarity matches_by_similarity
 
       def matches
-        @matches ||= matches_by_similarity.select do |_, similarity|
-          similarity >= minimum_confidence
+        @matches ||= matches_by_similarity.select do |license, similarity|
+          similarity >= minimum_confidence &&
+            license.bigram_similarity(file) >= minimum_bigram_confidence
         end
       end
 
@@ -58,6 +59,15 @@ module Licensee
       def minimum_confidence
         Licensee.confidence_threshold
       end
+
+      # A floor for bigram similarity, used to reject adversarially scrambled
+      # content that achieves high wordset similarity by including all the right
+      # words in the wrong order.  Set to half the wordset threshold so that any
+      # genuine license match (which typically scores 90%+ on bigrams) passes,
+      # while scrambled content (which scores near 0%) is rejected.
+      def minimum_bigram_confidence
+        Licensee.confidence_threshold / 2.0
+      end
     end
   end
 end

data/lib/licensee/matchers/gemspec.rb

@@ -13,10 +13,6 @@ module Licensee
       # non-value groups
       ARRAY_REGEX = /\s*\[#{VALUE_REGEX}(?:,#{VALUE_REGEX})*\]\s*/i
 
-      DECLARATION_REGEX = /
-        ^\s*[a-z0-9_]+\.([a-z0-9_]+)\s*=#{VALUE_REGEX}$
-      /ix
-
       LICENSE_REGEX = /
         ^\s*[a-z0-9_]+\.license\s*=#{VALUE_REGEX}$
       /ix
@@ -45,10 +41,6 @@ module Licensee
         match = @file.content.match LICENSE_ARRAY_REGEX
         match.captures.compact.map(&:downcase) if match
       end
-
-      def declarations
-        @declarations ||= @file.content.match DECLARATION_REGEX
-      end
     end
   end
 end

data/lib/licensee/matchers/matcher.rb

@@ -19,17 +19,17 @@ module Licensee
       end
 
       def match
-        raise 'Not implemented'
+        raise NotImplementedError, "#{self.class}#match is not implemented"
       end
 
       def confidence
-        raise 'Not implemented'
+        raise NotImplementedError, "#{self.class}#confidence is not implemented"
       end
 
       private
 
       def potential_matches
-        @potential_matches ||= Licensee.licenses(hidden: true, psuedo: false)
+        @potential_matches ||= Licensee.licenses(hidden: true, pseudo: false)
       end
     end
   end

data/lib/licensee/matchers/package.rb

@@ -25,7 +25,7 @@ module Licensee
       end
 
       def license_property
-        raise 'Not implemented'
+        raise NotImplementedError, "#{self.class}#license_property is not implemented"
       end
 
       private

data/lib/licensee/project_files/project_file.rb

@@ -66,7 +66,7 @@ module Licensee
       alias relative_path path_relative_to_root
 
       def possible_matchers
-        raise 'Not implemented'
+        raise NotImplementedError, "#{self.class}#possible_matchers is not implemented"
       end
 
       def matcher

data/lib/licensee/projects/project.rb

@@ -152,11 +152,11 @@ module Licensee
       end
 
       def files
-        raise 'Not implemented'
+        raise NotImplementedError, "#{self.class}#files is not implemented"
       end
 
       def load_file(_file)
-        raise 'Not implemented'
+        raise NotImplementedError, "#{self.class}#load_file is not implemented"
       end
     end
   end

data/lib/licensee/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Licensee
-  VERSION = '9.19.0'
+  VERSION = '9.20.0'
 end

data/spec/fixture_spec.rb

@@ -18,9 +18,9 @@ RSpec.describe Fixture do
         Licensee::License.find('none')
       end
 
-      it 'has an expected license in fixtures-licenses.yml' do
+      it 'has an expected license in fixtures.yml' do
         msg = +'Expected an entry in `'
-        msg << fixture_path('fixtures-licenses.yml')
+        msg << fixture_path('fixtures.yml')
         msg << "` for the `#{fixture}` fixture. Please run "
         msg << 'script/dump-fixture-licenses and confirm the output.'
         expect(fixture_licenses).to have_key(fixture), msg

data/spec/fixtures/bsd-3-linebreak-owner/LICENSE

@@ -0,0 +1,30 @@
+Copyright (c) 2023, Karl Pettersson
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+
+    * Neither the name of Karl Pettersson nor the names of other
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/spec/fixtures/bsd-3-multilinecopyright/LICENSE

@@ -0,0 +1,27 @@
+BSD 3-Clause License
+
+Copyright (c) 2016-2026 by University of Kassel and Fraunhofer Institute for Energy Economics
+and Energy System Technology (IEE) Kassel and individual contributors (see AUTHORS file for details).
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted
+provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions
+and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of
+conditions and the following disclaimer in the documentation and/or other materials provided
+with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to
+endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
+WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/spec/fixtures/detect.json

@@ -93,7 +93,7 @@
     },
     {
       "filename": "licensee.gemspec",
-      "content": "# frozen_string_literal: true\n\nrequire File.expand_path('lib/licensee/version', __dir__)\n\nGem::Specification.new do |gem|\n  gem.name    = 'licensee'\n  gem.version = Licensee::VERSION\n\n  gem.summary = 'A Ruby Gem to detect open source project licenses'\n  gem.description = <<-DESC\n    Licensee automates the process of reading LICENSE files and\n    compares their contents to known licenses using a fancy maths.\n  DESC\n\n  gem.authors  = ['Ben Balter']\n  gem.email    = 'ben.balter@github.com'\n  gem.homepage = 'https://github.com/benbalter/licensee'\n  gem.license  = 'MIT'\n  gem.metadata['rubygems_mfa_required'] = 'true'\n\n  gem.bindir = 'bin'\n  gem.executables << 'licensee'\n\n  gem.add_dependency('dotenv', '>= 2', '< 4')\n  gem.add_dependency('octokit', '>= 4.20', '< 11.0')\n  gem.add_dependency('reverse_markdown', '>= 1', '< 4')\n  gem.add_dependency('rugged', '>= 0.24', '<2.0')\n  gem.add_dependency('thor', '>= 0.19', '< 2.0')\n\n  gem.add_development_dependency('gem-release', '~> 2.0')\n  gem.add_development_dependency('mustache', '>= 0.9', '< 2.0')\n  gem.add_development_dependency('pry', '~> 0.9')\n  gem.add_development_dependency('rspec', '~> 3.5')\n  gem.add_development_dependency('rubocop', '~> 1.0')\n  gem.add_development_dependency('rubocop-performance', '~> 1.5')\n  gem.add_development_dependency('rubocop-rspec', '~> 3.0')\n  gem.add_development_dependency('simplecov', '~> 0.16')\n  gem.add_development_dependency('webmock', '~> 3.1')\n\n  gem.required_ruby_version = '>= 3.2'\n\n  # ensure the gem is built out of versioned files\n  gem.files = Dir[\n    '{bin,lib,man,test,vendor,spec}/**/*',\n    'README*', 'LICENSE*'\n  ] & `git ls-files -z`.split(\"\\0\")\nend\n",
+      "content": "# frozen_string_literal: true\n\nrequire File.expand_path('lib/licensee/version', __dir__)\n\nGem::Specification.new do |gem|\n  gem.name    = 'licensee'\n  gem.version = Licensee::VERSION\n\n  gem.summary = 'A Ruby Gem to detect open source project licenses'\n  gem.description = <<-DESC\n    Licensee automates the process of reading LICENSE files and\n    compares their contents to known licenses using a fancy maths.\n  DESC\n\n  gem.authors  = ['Ben Balter']\n  gem.email    = 'ben.balter@github.com'\n  gem.homepage = 'https://github.com/licensee/licensee'\n  gem.license  = 'MIT'\n  gem.metadata['rubygems_mfa_required'] = 'true'\n\n  gem.bindir = 'bin'\n  gem.executables << 'licensee'\n\n  gem.add_dependency('dotenv', '>= 2', '< 4')\n  gem.add_dependency('octokit', '>= 4.20', '< 11.0')\n  gem.add_dependency('reverse_markdown', '>= 1', '< 4')\n  gem.add_dependency('rugged', '>= 0.24', '<2.0')\n  gem.add_dependency('thor', '>= 0.19', '< 2.0')\n\n  gem.add_development_dependency('gem-release', '~> 2.0')\n  gem.add_development_dependency('mustache', '>= 0.9', '< 2.0')\n  gem.add_development_dependency('pry', '~> 0.9')\n  gem.add_development_dependency('rspec', '~> 3.5')\n  gem.add_development_dependency('rubocop', '~> 1.0')\n  gem.add_development_dependency('rubocop-performance', '~> 1.5')\n  gem.add_development_dependency('rubocop-rspec', '~> 3.0')\n  gem.add_development_dependency('simplecov', '~> 0.16')\n  gem.add_development_dependency('webmock', '~> 3.1')\n\n  gem.required_ruby_version = '>= 3.2'\n\n  # ensure the gem is built out of versioned files\n  gem.files = Dir[\n    '{bin,lib,man,test,vendor,spec}/**/*',\n    'README*', 'LICENSE*'\n  ] & `git ls-files -z`.split(\"\\0\")\nend\n",
       "content_hash": null,
       "content_normalized": null,
       "matcher": {

data/spec/fixtures/fixtures.yml

@@ -7,7 +7,7 @@ agpl-3.0_markdown:
 apache-2.0_markdown:
   key: apache-2.0
   matcher: dice
-  hash: 62937105bf0baec879cc6cc96d2fc2ce4922fc6d
+  hash: 0b6213bf4bf883f67e804a39c23f92cb63c70f1d
 apache-with-readme-notice:
   key: apache-2.0
   matcher: exact
@@ -28,10 +28,18 @@ bsd-3-authorowner:
   key: bsd-3-clause
   matcher: dice
   hash: 2e6f215833d1a3d10e6194d479dbb2b4be2f64d7
+bsd-3-multilinecopyright:
+  key: bsd-3-clause
+  matcher: exact
+  hash: a961b19cc6921d510e29a13b0ba1a826fcffe41c
 bsd-3-clause_markdown:
   key: bsd-3-clause
   matcher: dice
   hash: 2449fc8ece2fa342f2e82bbbf86f01d19329a531
+bsd-3-linebreak-owner:
+  key: bsd-3-clause
+  matcher: dice
+  hash: 9a1ab486a9182629581b5598b415df85b48eb008
 bsd-3-lists:
   key: bsd-3-clause
   matcher:
@@ -54,28 +62,28 @@ case-sensitive:
   hash: da39a3ee5e6b4b0d3255bfef95601890afd80709
 cc-by-4.0_markdown:
   key: cc-by-4.0
-  matcher: dice
-  hash: b72312bbcd6400de2c6103f2b9981e6239d89f36
+  matcher: exact
+  hash: f2a70fcab522bfb2fbcdefb47b94d2a928e22091
 cc-by-nc-sa:
   key: other
   matcher:
-  hash: c1c5bf7b6f130913568b20ed0441a6dbef5d6f11
+  hash: ec364756344b00a9f6a59cb239a16634f7b40770
 cc-by-nd:
   key: other
   matcher:
-  hash: a006cd136405570d81aafb4cfd19de78012bbbe4
+  hash: b3cd0e6254ff1116ad09860ec03ea871cff30f64
 cc-by-sa-4.0_markdown:
   key: cc-by-sa-4.0
-  matcher: dice
-  hash: 973d41ed4a3fca2a1205ace75835029d66c1e224
+  matcher: exact
+  hash: 33464647bebf0a285d9df642ccb07c8e26b3e268
 cc-by-sa-mdlinks:
   key: cc-by-sa-4.0
-  matcher: dice
-  hash: 4b0a634a0db5015914cbdde602672b2addfd66e9
+  matcher: exact
+  hash: a81c2a3a07b59f58b3c09387874724670122fc90
 cc-by-sa-nocclicensor:
   key: cc-by-sa-4.0
   matcher: dice
-  hash: 9cf238d2d4dd703a149d8b5ae7d3f5a04206f772
+  hash: 6fc6425689d3156be764c8e765ae2553a54e3589
 cc0-1.0_markdown:
   key: cc0-1.0
   matcher: dice
@@ -111,7 +119,7 @@ epl-1.0_markdown:
 eupl-cal2017:
   key: eupl-1.2
   matcher: exact
-  hash: 7425a276b011dea63591fe8876146f2451cfd777
+  hash: d8debddd73476c481fc6ceed37c75f0ae97a6e81
 fcpl-modified-mpl:
   key: other
   matcher:

data/spec/fixtures/license-hashes.json

@@ -11,8 +11,8 @@
   "bsd-3-clause-clear": "0fcdb12c4060ce8f406e17bc67787e50a9b36a61",
   "bsd-4-clause": "3b2917580b2b6f13efaaea37546b8b7a53716a30",
   "bsl-1.0": "27e28f20b57048cf04be07e1532b6fb501a0753b",
-  "cc-by-4.0": "7ff5344de1b567d0bb090ea7dd6988b7fa4cd351",
-  "cc-by-sa-4.0": "f8c9d796e80f6e19458f8b3bbe8bfadd615958a5",
+  "cc-by-4.0": "e8c6c40ba40ff2a44f19d74987731e98facc1451",
+  "cc-by-sa-4.0": "35168cd69d6ef5b9dd81f7793898fad53c1798ec",
   "cc0-1.0": "34dbb82be40b15f7c521d4f2d1a36ebe76246936",
   "cecill-2.1": "ea372810464d71db27e62ad499628991ea2818cf",
   "cern-ohl-p-2.0": "f10b4b8d75502ab65a7bdbe1d616e5eb8d157aed",
@@ -21,8 +21,8 @@
   "ecl-2.0": "296976ce9e84ba380866e4519b68a779c2059b3a",
   "epl-1.0": "5e3cb10996b4ba2821d04d5c99a912c924b3bdcb",
   "epl-2.0": "e2f3e266432478d9248422228a75a404cce1c43c",
-  "eupl-1.1": "b35810b4113910f5f85af75f24e2538ba64c8876",
-  "eupl-1.2": "2098182069695981c6dc71093888f6204c7bbdae",
+  "eupl-1.1": "2e384f67f0cb5adb7f63470c5dcea0280873f2b3",
+  "eupl-1.2": "169fa5fdd2118679d1453414d0a5d28b2a5fcdc4",
   "gfdl-1.3": "164a858691ea0a6fb0dd06c5ca00e5dd7620eef8",
   "gpl-2.0": "32108116603c30687d8d0d2f77f140fb6ecea082",
   "gpl-3.0": "7d4cdf499d39e2e1ce27b2878e22872f0f5a74dd",

data/spec/licensee/content_helper_spec.rb

@@ -13,10 +13,6 @@ class ContentHelperTestHelper
   def filename
     @data[:filename]
   end
-
-  def spdx_id
-    @data[:spdx_id]
-  end
 end
 
 RSpec.describe Licensee::ContentHelper do
@@ -57,10 +53,41 @@ RSpec.describe Licensee::ContentHelper do
     )
   end
 
+  def expected_bigrams
+    Set.new(
+      [
+        'the made', 'made up', 'up license', 'license this', 'this license',
+        'license provided', 'provided as', 'as is\'', 'is\' please', 'please respect',
+        'respect the', 'the contributors\'', 'contributors\' wishes', 'wishes when',
+        'when implementing', 'implementing the', 'the license\'s', 'license\'s software'
+      ]
+    )
+  end
+
   it 'creates the wordset' do
     expect(helper.wordset).to eql(expected_wordset)
   end
 
+  it 'creates bigrams' do
+    expect(helper.bigrams).to eql(expected_bigrams)
+  end
+
+  it 'returns empty set for content with fewer than two words' do
+    single_word = ContentHelperTestHelper.new('word', filename: 'LICENSE')
+    expect(single_word.bigrams).to eql(Set.new)
+  end
+
+  it 'calculates bigram_similarity for exact content' do
+    expect(mit.bigram_similarity(mit)).to eq(100.0)
+  end
+
+  it 'calculates bigram_similarity near zero for scrambled wordset' do
+    # All unique words from MIT sorted alphabetically: same wordset, different order.
+    sorted_words = mit.content_normalized.scan(%r{(?:[\w/-](?:'s|(?<=s)')?)+}).uniq.sort
+    scrambled = ContentHelperTestHelper.new(sorted_words.join(' '), filename: 'LICENSE')
+    expect(mit.bigram_similarity(scrambled)).to be < 5.0
+  end
+
   it 'knows the length' do
     expect(helper.length).to be(135)
   end
@@ -275,6 +302,14 @@ RSpec.describe Licensee::ContentHelper do
       end
     end
 
+    context 'when normalizing wilful to willful' do
+      let(:content) { 'wilful misconduct' }
+
+      it 'normalizes wilful to willful' do
+        expect(helper.content_normalized).to eql('willful misconduct')
+      end
+    end
+
     Licensee::License.all(hidden: true).each do |license|
       context "with the #{license.name} license" do
         let(:stripped_content) { helper.content_without_title_and_version }
@@ -334,6 +369,35 @@ RSpec.describe Licensee::ContentHelper do
         expect(normalized_content).to eql('foo')
       end
     end
+
+    context 'with a multi-line copyright holder name followed by All rights reserved' do
+      let(:content) do
+        "Copyright (c) 2020 by Corporation Name and\n" \
+          "its Subsidiaries (see AUTHORS).\n" \
+          "All rights reserved.\n\n" \
+          'Foo'
+      end
+
+      it 'strips the wrapped copyright holder continuation and all rights reserved' do
+        expect(normalized_content).to eql('foo')
+      end
+    end
+
+    context 'with a multi-line copyright holder name without All rights reserved' do
+      let(:content) do
+        "Copyright (c) 2020 by Corporation Name and\n" \
+          "its Subsidiaries (see AUTHORS).\n\n" \
+          'Foo'
+      end
+
+      it 'leaves the ambiguous continuation' do
+        expect(normalized_content).to include('subsidiaries')
+      end
+
+      it 'strips the copyright notice line' do
+        expect(normalized_content).not_to include('copyright (c) 2020 by corporation name and')
+      end
+    end
   end
 
   context 'when matching title regex' do

data/spec/licensee/hash_helper_spec.rb

@@ -21,13 +21,15 @@ class HashHelperSpecFixture
     Licensee::Rule.all
   end
 
-  def baz
-    'baz'
-  end
-
   def nil_value
     nil
   end
+
+  # Method not listed in HASH_METHODS; used to ensure HashHelper#to_h
+  # does not expose arbitrary instance methods.
+  def baz
+    'not included'
+  end
 end
 
 RSpec.describe Licensee::HashHelper do

data/spec/licensee/license_spec.rb

@@ -204,6 +204,10 @@ RSpec.describe Licensee::License do
     expect(mit.key).to eql('mit')
   end
 
+  it 'has a useful inspect string' do
+    expect(mit.inspect).to eql('#<Licensee::License key=mit>')
+  end
+
   it 'exposes the SPDX ID' do
     expect(gpl.spdx_id).to eql('GPL-3.0')
   end

data/spec/licensee/matchers/cabal_matcher_spec.rb

@@ -122,6 +122,42 @@ RSpec.describe Licensee::Matchers::Cabal do
     end
   end
 
+  context 'with pre-SPDX "or-later" (+) suffix' do
+    let(:content) { "license: #{cabal_license}" }
+
+    context 'with GPL-2+' do
+      let(:cabal_license) { 'GPL-2+' }
+
+      it 'returns GPL-2.0' do
+        expect(matcher.match).to eql(Licensee::License.find('GPL-2.0'))
+      end
+    end
+
+    context 'with GPL-3+' do
+      let(:cabal_license) { 'GPL-3+' }
+
+      it 'returns GPL-3.0' do
+        expect(matcher.match).to eql(Licensee::License.find('GPL-3.0'))
+      end
+    end
+
+    context 'with LGPL-3+' do
+      let(:cabal_license) { 'LGPL-3+' }
+
+      it 'returns LGPL-3.0' do
+        expect(matcher.match).to eql(Licensee::License.find('LGPL-3.0'))
+      end
+    end
+
+    context 'with AGPL-3+' do
+      let(:cabal_license) { 'AGPL-3+' }
+
+      it 'returns AGPL-3.0' do
+        expect(matcher.match).to eql(Licensee::License.find('AGPL-3.0'))
+      end
+    end
+  end
+
   context 'with no license field' do
     let(:content) { 'foo: bar' }
 

data/spec/licensee/matchers/copyright_matcher_spec.rb

@@ -57,4 +57,15 @@ RSpec.describe Licensee::Matchers::Copyright do
       expect(matcher.match).to be_nil
     end
   end
+
+  context 'with encoding-incompatible content' do
+    # A string with non-ASCII bytes in an encoding incompatible with the
+    # UTF-8 copyright regex triggers Encoding::CompatibilityError
+    let(:raw_content) { (+"\xC2\xA9 2015 Ben Balter").force_encoding('EUC-KR') }
+
+    it 'returns nil gracefully' do
+      allow(file).to receive(:content).and_return(raw_content)
+      expect(matcher.match).to be_nil
+    end
+  end
 end