licensee 8.3.1 → 8.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3b3f800aaf46c58f74814dc25a5f9121985b619c
-  data.tar.gz: fbe2299058f5b7108e1f05462ced09f24cd9c13e
+  metadata.gz: 0ea7d783ce6fc8e59946877a34fc32d84b01f449
+  data.tar.gz: 65142dc4101a66360d080b7eede89977d5ca10c6
 SHA512:
-  metadata.gz: 841d87d6848ec806dd1ecf8d557c67f2e1f7cfbf76f9dbe5175cf2363efed2e1b6f373db9831990c0884d19bdea9e2bd826b4e493169c4b71bf66a3c137a85f5
-  data.tar.gz: 624464f0ccbd8bac06df571a31bdc8be1996a426b041ac1064f88d2aee3b57b52b9d6eb567bcce244a68bfa5c08a5f157c40360545e26c0916dcf55e1bab1a5e
+  metadata.gz: 9b93a5da892c334a4e3bce03be2ef7342ffda08ae64b6db412216f691af53ee9d8a7d54596c9aadaff6cd6d02038c04da21f4d4e98af222283788fdb77a340cc
+  data.tar.gz: d066ee5ed370b2c840cd5282f19fccf525f26cde2469a6205039ab886b5b1f2ca5e63e9ac88fc2888fa1c4657cda2b50d2f57f4da79632ac22ad4f96779bc33a

data/Rakefile

@@ -3,7 +3,7 @@ require 'rake/testtask'
 
 Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_licensee*.rb'
+  test.pattern = 'test/**/test_*.rb'
 end
 
 desc 'Open console with Licensee loaded'

data/lib/licensee/content_helper.rb

@@ -5,22 +5,50 @@ module Licensee
   module ContentHelper
     DIGEST = Digest::SHA1
 
+    # A set of each word in the license, without duplicates
     def wordset
       @wordset ||= if content_normalized
         content_normalized.scan(/[\w']+/).to_set
       end
     end
 
+    # Number of characteres in the normalized content
+    def length
+      return 0 unless content_normalized
+      content_normalized.length
+    end
+
+    # Number of characters that could be added/removed to still be
+    # considered a potential match
+    def max_delta
+      (length * Licensee.inverse_confidence_threshold).to_i
+    end
+
+    # Given another license or project file, calculates the difference in length
+    def length_delta(other)
+      (length - other.length).abs
+    end
+
+    # Given another license or project file, calculates the similarity
+    # as a percentage of words in common
+    def similarity(other)
+      overlap = (wordset & other.wordset).size
+      total = wordset.size + other.wordset.size
+      100.0 * (overlap * 2.0 / total)
+    end
+
+    # SHA1 of the normalized content
     def hash
       @hash ||= DIGEST.hexdigest content_normalized
     end
 
+    # Content with copyright header and linebreaks removed
     def content_normalized
       return unless content
       @content_normalized ||= begin
         content_normalized = content.downcase.strip
         content_normalized.gsub!(/^#{Matchers::Copyright::REGEX}$/i, '')
-        content_normalized.tr("\n", ' ').squeeze(' ')
+        content_normalized.tr("\n", ' ').squeeze(' ').strip
       end
     end
   end

data/lib/licensee/license.rb

@@ -111,6 +111,10 @@ module Licensee
       meta['hidden']
     end
 
+    def gpl?
+      key == 'gpl-2.0' || key == 'gpl-3.0'
+    end
+
     # The license body (e.g., contents - frontmatter)
     def content
       @content ||= parts[2] if parts && parts[2]

data/lib/licensee/matchers/copyright_matcher.rb

@@ -2,7 +2,8 @@
 module Licensee
   module Matchers
     class Copyright
-      REGEX = /\s*Copyright (©|\(c\)|\xC2\xA9)? ?(\d{4}|\[year\])(.*)?\s*/i
+      # rubocop:disable Metrics/LineLength
+      REGEX = /\s*(Copyright|\(c\)) (©|\(c\)|\xC2\xA9)? ?(\d{4}|\[year\])(.*)?\s*/i
 
       def initialize(file)
         @file = file

data/lib/licensee/matchers/dice_matcher.rb

@@ -1,6 +1,8 @@
 module Licensee
   module Matchers
     class Dice
+      attr_reader :file
+
       def initialize(file)
         @file = file
       end
@@ -10,9 +12,8 @@ module Licensee
       def match
         return @match if defined? @match
         matches = potential_licenses.map do |license|
-          if (sim = similarity(license)) >= Licensee.confidence_threshold
-            [license, sim]
-          end
+          similarity = license.similarity(file)
+          [license, similarity] if similarity >= Licensee.confidence_threshold
         end
         matches.compact!
         @match = if matches.empty?
@@ -22,46 +23,20 @@ module Licensee
         end
       end
 
-      # Sort all licenses, in decending order, by difference in
-      # length to the file
-      # Difference in lengths cannot exceed the file's length *
-      # the confidence threshold / 100
+      # Licenses that may be a match for this file.
+      # To avoid false positives, the percentage change in file length
+      # may not exceed the inverse of the confidence threshold
       def potential_licenses
         @potential_licenses ||= begin
-          licenses = Licensee.licenses(hidden: true)
-          licenses = licenses.select do |license|
-            license.wordset && length_delta(license) <= max_delta
+          Licensee.licenses(hidden: true).select do |license|
+            license.wordset && license.length_delta(file) <= license.max_delta
           end
-          licenses.sort_by { |l| length_delta(l) }
         end
       end
 
-      # Calculate the difference between the file length and a given
-      # license's length
-      def length_delta(license)
-        (@file.wordset.size - license.wordset.size).abs
-      end
-
-      # Maximum possible difference between file length and license length
-      # for a license to be a potential license to be matched
-      def max_delta
-        @max_delta ||= (
-          @file.wordset.size * (Licensee.confidence_threshold / 100.0)
-        )
-      end
-
       # Confidence that the matched license is a match
       def confidence
-        @confidence ||= match ? similarity(match) : 0
-      end
-
-      private
-
-      # Calculate percent changed between file and potential license
-      def similarity(license)
-        overlap = (@file.wordset & license.wordset).size
-        total = @file.wordset.size + license.wordset.size
-        100.0 * (overlap * 2.0 / total)
+        @confidence ||= match ? file.similarity(match) : 0
       end
     end
   end

data/lib/licensee/matchers/exact_matcher.rb

@@ -6,7 +6,9 @@ module Licensee
       end
 
       def match
-        Licensee.licenses(hidden: true).find { |l| l.wordset == @file.wordset }
+        Licensee.licenses(hidden: true).find do |license|
+          license.length == @file.length && license.wordset == @file.wordset
+        end
       end
 
       def confidence

data/lib/licensee/project.rb

@@ -23,8 +23,16 @@ module Licensee
     def license_file
       return @license_file if defined? @license_file
       @license_file = begin
-        content, name = find_file { |n| LicenseFile.name_score(n) }
-        LicenseFile.new(content, name) if content && name
+        license_file = license_from_file { |n| LicenseFile.name_score(n) }
+        return license_file unless license_file && license_file.license
+
+        # Special case LGPL, which actuall lives in LICENSE.lesser, per the
+        # license instructions. See https://git.io/viwyK
+        lesser = if license_file.license.gpl?
+          license_from_file { |file| LicenseFile.lesser_gpl_score(file) }
+        end
+
+        lesser || license_file
       end
     end
 
@@ -46,5 +54,31 @@ module Licensee
         PackageInfo.new(content, name) if content && name
       end
     end
+
+    private
+
+    # Given a block, passes each filename to that block, and expects a numeric
+    # score in response. Returns an array of all files with a score > 0,
+    # sorted by file score descending
+    def find_files
+      return [] if files.empty? || files.nil?
+      found = files.each { |file| file[:score] = yield(file[:name]) }
+      found.select! { |file| file[:score] > 0 }
+      found.sort { |a, b| b[:score] <=> a[:score] }
+    end
+
+    # Given a block, passes each filename to that block, and expects a numeric
+    # score in response. Returns a hash representing the top scoring file
+    # or nil, if no file scored > 0
+    def find_file(&block)
+      return if files.empty? || files.nil?
+      file = find_files(&block).first
+      [load_file(file), file[:name]] if file
+    end
+
+    def license_from_file(&block)
+      content, name = find_file(&block)
+      LicenseFile.new(content, name) if content && name
+    end
   end
 end

data/lib/licensee/project_files/license_file.rb

@@ -20,6 +20,11 @@ module Licensee
         return 0.5 if filename =~ /licen[sc]e/i
         0.0
       end
+
+      # case-insensitive block to determine if the given file is LICENSE.lesser
+      def self.lesser_gpl_score(filename)
+        filename.casecmp('copying.lesser').zero? ? 1 : 0
+      end
     end
   end
 end

data/lib/licensee/projects/fs_project.rb

@@ -12,7 +12,9 @@ module Licensee
 
     private
 
-    def find_file
+    # Returns an array of hashes representing the project's files.
+    # Hashes will have the :name key, with the relative path to the file
+    def files
       files = []
 
       if ::File.file?(path)
@@ -24,17 +26,20 @@ module Licensee
 
       Dir.glob(::File.join(path, pattern)) do |file|
         next unless ::File.file?(file)
-        file = ::File.basename(file)
-        if (score = yield file) > 0
-          files.push(name: file, score: score)
-        end
+        files.push(name: ::File.basename(file))
       end
 
-      return if files.empty?
-      files.sort! { |a, b| b[:score] <=> a[:score] }
+      files
+    end
 
-      f = files.first
-      [::File.read(::File.join(path, f[:name])), f[:name]]
+    # Retrieve a file's content from disk
+    #
+    # file - the file hash, with the :name key as the file's relative path
+    # path - the base path to the project
+    #
+    # Returns the fiel contents as a string
+    def load_file(file)
+      ::File.read(::File.join(path, file[:name]))
     end
   end
 end

data/lib/licensee/projects/git_project.rb

@@ -36,24 +36,25 @@ module Licensee
 
     MAX_LICENSE_SIZE = 64 * 1024
 
-    def load_blob_data(oid)
-      data, = Rugged::Blob.to_buffer(repository, oid, MAX_LICENSE_SIZE)
+    # Retrieve a file's content from the Git database
+    #
+    # file - the file hash, including the file's OID
+    #
+    # Returns a string representing the file's contents
+    def load_file(file)
+      data, = Rugged::Blob.to_buffer(repository, file[:oid], MAX_LICENSE_SIZE)
       data
     end
 
-    def find_file
-      files = commit.tree.map do |entry|
+    # Returns an array of hashes representing the project's files.
+    # Hashes will have the the following keys:
+    #  :name - the file's path relative to the repo root
+    #  :oid  - the file's OID
+    def files
+      commit.tree.map do |entry|
         next unless entry[:type] == :blob
-        if (score = yield entry[:name]) > 0
-          { name: entry[:name], oid: entry[:oid], score: score }
-        end
+        { name: entry[:name], oid: entry[:oid] }
       end.compact
-
-      return if files.empty?
-      files.sort! { |a, b| b[:score] <=> a[:score] }
-
-      f = files.first
-      [load_blob_data(f[:oid]), f[:name]]
     end
   end
 end

data/lib/licensee/version.rb

@@ -1,3 +1,3 @@
 module Licensee
-  VERSION = '8.3.1'.freeze
+  VERSION = '8.4.0'.freeze
 end

data/lib/licensee.rb

@@ -23,7 +23,7 @@ require_relative 'licensee/matchers/npm_bower_matcher'
 
 module Licensee
   # Over which percent is a match considered a match by default
-  CONFIDENCE_THRESHOLD = 90
+  CONFIDENCE_THRESHOLD = 95
 
   # Base domain from which to build license URLs
   DOMAIN = 'http://choosealicense.com'.freeze
@@ -50,5 +50,11 @@ module Licensee
     def confidence_threshold
       @confidence_threshold ||= CONFIDENCE_THRESHOLD
     end
+
+    # Inverse of the confidence threshold, represented as a float
+    # By default this will be 0.1
+    def inverse_confidence_threshold
+      @inverse ||= (1 - Licensee.confidence_threshold / 100.0).round(2)
+    end
   end
 end

data/test/fixtures/lgpl.git/HEAD

@@ -0,0 +1 @@
+ref: refs/heads/master

data/test/fixtures/lgpl.git/config

@@ -0,0 +1,5 @@
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = true
+	ignorecase = true

data/test/fixtures/lgpl.git/objects/1a/cd060ebbbeac294200008657d9502130f93465

@@ -0,0 +1,2 @@
+x
��Qj�0D��S�b${�HPJ�MV�c��`+��>N�	�7o�
+

data/test/fixtures/lgpl.git/objects/4b/1f98c1e95b6d4dae0b0a160f554be97e5afece

@@ -0,0 +1,2 @@
+x
��[
+�0E��*f��C�B�Nf��

data/test/fixtures/lgpl.git/refs/heads/master

@@ -0,0 +1 @@
+1acd060ebbbeac294200008657d9502130f93465

data/test/functions.rb

@@ -19,9 +19,11 @@ def license_from_path(path)
   license
 end
 
-def chaos_monkey(string)
-  Random.rand(3).times do
-    string[Random.rand(string.length)] = SecureRandom.base64(Random.rand(3))
+# Add random words to the end of a license to test similarity tollerances
+def chaos_monkey(string, count: 5)
+  ipsum = %w[lorem ipsum dolor sit amet consectetur adipiscing elit]
+  Random.rand(count).times do
+    string << " #{ipsum[Random.rand(ipsum.length)]}"
   end
   string
 end
@@ -35,15 +37,20 @@ def verify_license_file(license, chaos = false, wrap = false)
 
   license_file = Licensee::Project::LicenseFile.new(text)
 
-  actual = license_file.license
-  msg = "No match for #{expected}."
-
-  assert actual, msg
+  detected = license_file.license
+  msg = "No match for #{expected}.\n"
+  matcher = Licensee::Matchers::Dice.new(license_file)
+  matcher.potential_licenses.each do |potential_license|
+    msg << "Potential license: #{potential_license.key}\n"
+    msg << "Potential similiarity: #{potential_license.similarity(license_file)}\n"
+  end
+  msg << "Text: #{license_file.content_normalized}"
+  assert detected, msg
 
-  msg = "Expeceted #{expected} but got #{actual.key} for .match. "
+  msg = "Expeceted #{expected} but got #{detected.key} for .match. "
   msg << "Confidence: #{license_file.confidence}. "
   msg << "Method: #{license_file.matcher.class}"
-  assert_equal expected, actual.key, msg
+  assert_equal expected, detected.key, msg
 end
 
 def wrap(text, line_width = 80)

data/test/{test_licensee_copyright_matcher.rb → licensee/matchers/test_copyright_matcher.rb}

@@ -26,6 +26,12 @@ class TestLicenseeCopyrightMatchers < Minitest::Test
     assert_equal 'no-license', Licensee::Matchers::Copyright.new(file).match.key
   end
 
+  should 'match (c) copyright notices' do
+    text = '(C) 2015 Ben Balter'
+    file = Licensee::Project::LicenseFile.new(text)
+    assert_equal 'no-license', Licensee::Matchers::Copyright.new(file).match.key
+  end
+
   should 'not false positive' do
     path = Licensee::License.find('mit').path
     text = File.read(path, encoding: 'utf-8').split('---').last

data/test/{test_licensee_dice_matcher.rb → licensee/matchers/test_dice_matcher.rb}

@@ -6,6 +6,12 @@ class TestLicenseeDiceMatchers < Minitest::Test
     @mit = Licensee::Project::LicenseFile.new(text)
   end
 
+  def concat_licenses(*args)
+    args.map do |license|
+      license_from_path(Licensee::License.find(license).path)
+    end.join("\n")
+  end
+
   should 'match the license' do
     assert_equal 'mit', Licensee::Matchers::Dice.new(@mit).match.key
   end
@@ -15,7 +21,10 @@ class TestLicenseeDiceMatchers < Minitest::Test
     assert matcher.confidence > 95, "#{matcher.confidence} < 95"
   end
 
-  should 'calculate max delta' do
-    assert_equal 83.7, Licensee::Matchers::Dice.new(@mit).max_delta
+  should 'know when two licenses have be concatenated' do
+    text = concat_licenses('mit', 'gpl-2.0')
+    license = Licensee::Project::LicenseFile.new(text)
+    matcher = Licensee::Matchers::Dice.new(license)
+    refute matcher.match
   end
 end

data/test/{test_licensee_exact_matcher.rb → licensee/matchers/test_exact_matcher.rb}

@@ -14,4 +14,12 @@ class TestLicenseeExactMatchers < Minitest::Test
   should 'know the match confidence' do
     assert_equal 100, Licensee::Matchers::Exact.new(@mit).confidence
   end
+
+  should 'require the file to be the same length' do
+    path = Licensee::License.find('mit').path
+    text = File.read(path, encoding: 'utf-8').split('---').last
+    text << ' MIT'
+    license = Licensee::Project::LicenseFile.new(text)
+    refute Licensee::Matchers::Exact.new(license).match
+  end
 end

data/test/{test_licensee_gemspec_matcher.rb → licensee/matchers/test_gemspec_matcher.rb}

@@ -2,7 +2,7 @@ require 'helper'
 
 class TestLicenseeGemspecMatchers < Minitest::Test
   should 'detect its own license' do
-    root = File.expand_path '../', File.dirname(__FILE__)
+    root = File.expand_path '../../../', File.dirname(__FILE__)
     project = Licensee::GitProject.new(root, detect_packages: true)
     matcher = Licensee::Matchers::Gemspec.new(project.package_file)
     assert_equal 'mit', matcher.send(:license_property)

data/test/licensee/matchers/test_package_matcher.rb

@@ -0,0 +1,27 @@
+require 'helper'
+
+class PackageMatcherTestHelper < Licensee::Matchers::Package
+  attr_accessor :license_property, :file
+end
+
+class TestLicenseePackageMatcher < Minitest::Test
+  def setup
+    pkg = File.read fixture_path('npm/package.json')
+    @file = Licensee::Project::PackageInfo.new(pkg)
+    @matcher = PackageMatcherTestHelper.new(@file)
+    @matcher.license_property = 'mit'
+  end
+
+  should 'store the file' do
+    assert_equal @matcher.file, @file
+  end
+
+  should 'match' do
+    assert @matcher.match
+    assert_equal 'mit', @matcher.match.key
+  end
+
+  should 'return the confidence' do
+    assert_equal 90, @matcher.confidence
+  end
+end

data/test/{test_licensee_license_file.rb → licensee/project_files/test_license_file.rb}

@@ -55,4 +55,19 @@ class TestLicenseeLicenseFile < Minitest::Test
       end
     end
   end
+
+  context 'LGPL scoring' do
+    {
+      'COPYING.lesser' => 1,
+      'copying.lesser' => 1,
+      'license.lesser' => 0,
+      'LICENSE.md'     => 0,
+      'FOO.md'         => 0
+    }.each do |filename, expected|
+      should "score a license named `#{filename}` as `#{expected}`" do
+        score = Licensee::Project::LicenseFile.lesser_gpl_score(filename)
+        assert_equal expected, score
+      end
+    end
+  end
 end

data/test/licensee/test_content_helper.rb

@@ -0,0 +1,56 @@
+require 'helper'
+
+class ContentHelperTestHelper
+  include Licensee::ContentHelper
+  attr_accessor :content
+
+  DEFAULT_CONTENT = <<-EOS.freeze
+Copyright 2016 Ben Balter
+
+The made
+up license.
+  EOS
+
+  def initialize(content = nil)
+    @content = content || DEFAULT_CONTENT
+  end
+end
+
+class TestLicenseeContentHelper < Minitest::Test
+  def setup
+    @helper = ContentHelperTestHelper.new
+  end
+
+  should 'normalize the content' do
+    assert_equal 'the made up license.', @helper.content_normalized
+  end
+
+  should 'generate the hash' do
+    assert_equal '3c59634b9fae4396a76a978f3f6aa718ed790a9a', @helper.hash
+  end
+
+  should 'calculate the length' do
+    assert_equal 20, @helper.length
+  end
+
+  should 'know the max delta' do
+    assert_equal 1, @helper.max_delta
+  end
+
+  should 'build the wordset' do
+    assert_equal %w(the made up license).to_set, @helper.wordset
+  end
+
+  should 'calculate the length delta' do
+    other = ContentHelperTestHelper.new 'asdf'
+    assert_equal 16, @helper.length_delta(other)
+  end
+
+  should 'calculate the similarity' do
+    other = ContentHelperTestHelper.new 'asdf'
+    assert_equal 0, @helper.similarity(other)
+
+    other = ContentHelperTestHelper.new 'Not really the made up license'
+    assert_equal 80.0, @helper.similarity(other)
+  end
+end

data/test/{test_licensee_license.rb → licensee/test_license.rb}

@@ -117,6 +117,17 @@ class TestLicenseeLicense < Minitest::Test
     assert_equal '750260c322080bab4c19fd55eb78bc73e1ae8f11', @license.hash
   end
 
+  should "recognize GPL'd licenses" do
+    license = Licensee::License.new 'gpl-2.0'
+    assert license.gpl?
+
+    license = Licensee::License.new 'gpl-3.0'
+    assert license.gpl?
+
+    license = Licensee::License.new 'mit'
+    refute license.gpl?
+  end
+
   describe 'name without version' do
     should 'strip the version from the license name' do
       expected = 'GNU Affero General Public License'

data/test/{test_licensee_project.rb → licensee/test_project.rb}

@@ -72,6 +72,11 @@ class TestLicenseeProject < Minitest::Test
         project = make_project 'no-license.git'
         assert_equal nil, project.license
       end
+
+      should 'detect an LGPL licensed project with LICENSE.lesser' do
+        project = make_project 'lgpl.git'
+        assert_equal 'lgpl-3.0', project.license.key
+      end
     end
   end
 

data/test/test_licensee.rb

@@ -24,13 +24,17 @@ class TestLicensee < Minitest::Test
 
   context 'confidence threshold' do
     should 'return the confidence threshold' do
-      assert_equal 90, Licensee.confidence_threshold
+      assert_equal 95, Licensee.confidence_threshold
     end
 
     should 'let the user override the confidence threshold' do
       Licensee.confidence_threshold = 50
       assert_equal 50, Licensee.confidence_threshold
-      Licensee.confidence_threshold = 90
+      Licensee.confidence_threshold = 95
+    end
+
+    should 'return the inverse of the confidence threshold' do
+      assert_equal 0.05, Licensee.inverse_confidence_threshold
     end
   end
 end