licensee 9.13.0 → 9.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7aeb33320898c7e64f31f6ea15708f4e4c23a6856e10c54e004054ee6997200a
-  data.tar.gz: dcbac4f8b08ff1d55e0ac7de2b603c67ce4f9a2b7d9486fbfccae3fa0dc6d304
+  metadata.gz: 63f3f7938e9e0c58907e71bcf9de72e69d2d494d2dbbba6c71e150b44190eda3
+  data.tar.gz: 51583db05dee9239fb50e373f3dc4c79dfa4515a5461facc2722dabe5ee0fb9a
 SHA512:
-  metadata.gz: d92bb1d7662b504e92ad6a2fa3f0c8fa23716fa3825715333f8483085a8662d94d61032ec2e4155ae93b9bc0e71627f9e841268e1c7b2229078f13da216d0394
-  data.tar.gz: 364fe3f78e38f703a1e7c9317453b598e4a4bc0b8db773efd4e6e701e7f3a730426698c3a4561e34ba09e839a35a24d4526d87595de31f687a7e83c932bf4ae2
+  metadata.gz: 87b8a85992cec7c2e7c30fab5b96e33a0fdd4271df34a99ca2b99584f1f94cdcb70c5663d9a584ccf1164fd10eca2b1e7b26f09bb7b43a81a8c3705dc6341666
+  data.tar.gz: b261c2cf6a2b40fd5b4b10bd4644d179dda3e35686b6f6361af4177dcda852eb45c22bf0dafc43291824a2c48b30f1297c2f933a7786f12dbefbb0bde6dfb83b

data/LICENSE.md

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2014-2017 Ben Balter
+Copyright (c) 2014-2021 Ben Balter and Licensee contributors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/bin/licensee

@@ -19,7 +19,7 @@ class LicenseeCLI < Thor
                 args.first || Dir.pwd
               else
                 "https://github.com/#{args.first}"
-    end
+              end
   end
 
   def project
@@ -33,6 +33,6 @@ class LicenseeCLI < Thor
 end
 
 commands_dir = File.expand_path '../lib/licensee/commands/', __dir__
-Dir["#{commands_dir}/*.rb"].each { |c| require(c) }
+Dir["#{commands_dir}/*.rb"].sort.each { |c| require(c) }
 
 LicenseeCLI.start(ARGV)

data/lib/licensee.rb

@@ -37,13 +37,13 @@ module Licensee
     end
 
     def project(path, **args)
-      if path =~ %r{\Ahttps://github.com}
-        Licensee::Projects::GitHubProject.new(path, args)
+      if %r{\Ahttps://github.com}.match?(path)
+        Licensee::Projects::GitHubProject.new(path, **args)
       else
-        Licensee::Projects::GitProject.new(path, args)
+        Licensee::Projects::GitProject.new(path, **args)
       end
     rescue Licensee::Projects::GitProject::InvalidRepository
-      Licensee::Projects::FSProject.new(path, args)
+      Licensee::Projects::FSProject.new(path, **args)
     end
 
     def confidence_threshold

data/lib/licensee/commands/detect.rb

@@ -28,11 +28,9 @@ class LicenseeCLI < Thor
               ['Licenses:', project.licenses.map(&:spdx_id)]
             else
               ['License:', set_color('None', :red)]
-    end
+            end
 
-    unless project.matched_files.empty?
-      rows << ['Matched files:', project.matched_files.map(&:filename).join(', ')]
-    end
+    rows << ['Matched files:', project.matched_files.map(&:filename).join(', ')] unless project.matched_files.empty?
 
     print_table rows
 
@@ -88,7 +86,7 @@ class LicenseeCLI < Thor
     when :confidence
       Licensee::ContentHelper.format_percent(value)
     when :method
-      value.to_s.tr('_', ' ').capitalize + ':'
+      "#{value.to_s.tr('_', ' ').capitalize}:"
     else
       value
     end

data/lib/licensee/commands/diff.rb

@@ -41,17 +41,15 @@ class LicenseeCLI < Thor
 
   def license_to_diff
     return options[:license_to_diff] if options[:license_to_diff]
-    return project.license_file if remote? || STDIN.tty? && project.license_file
+    return project.license_file if remote? || $stdin.tty? && project.license_file
 
     @license_to_diff ||= begin
-      Licensee::ProjectFiles::LicenseFile.new(STDIN.read, 'LICENSE')
+      Licensee::ProjectFiles::LicenseFile.new($stdin.read, 'LICENSE')
     end
   end
 
   def expected_license
-    if options[:license]
-      @expected_license ||= Licensee::License.find options[:license]
-    end
+    @expected_license ||= Licensee::License.find options[:license] if options[:license]
     return @expected_license if @expected_license
 
     if options[:license]

data/lib/licensee/content_helper.rb

@@ -7,23 +7,24 @@ module Licensee
   module ContentHelper
     DIGEST = Digest::SHA1
     START_REGEX = /\A\s*/.freeze
-    END_OF_TERMS_REGEX = /^[\s#*_]*end of terms and conditions\s*$/i.freeze
+    END_OF_TERMS_REGEX = /^[\s#*_]*end of terms and conditions[\s#*_]*$/i.freeze
     REGEXES = {
-      hrs:                 /^\s*[=\-\*]{3,}\s*$/,
+      bom:                 /#{START_REGEX}\xEF\xBB\xBF/,
+      hrs:                 /^\s*[=\-*]{3,}\s*$/,
       all_rights_reserved: /#{START_REGEX}all rights reserved\.?$/i,
       whitespace:          /\s+/,
-      markdown_headings:   /#{START_REGEX}#+/,
+      markdown_headings:   /^\s*#+/,
       version:             /#{START_REGEX}version.*$/i,
       span_markup:         /[_*~]+(.*?)[_*~]+/,
       link_markup:         /\[(.+?)\]\(.+?\)/,
       block_markup:        /^\s*>/,
-      border_markup:       /^[\*-](.*?)[\*-]$/,
-      comment_markup:      %r{^\s*?[/\*]{1,2}},
+      border_markup:       /^[*-](.*?)[*-]$/,
+      comment_markup:      %r{^\s*?[/*]{1,2}},
       url:                 %r{#{START_REGEX}https?://[^ ]+\n},
-      bullet:              /\n\n\s*(?:[*-]|\(?[\da-z]{1,2}[)\.])\s+/i,
+      bullet:              /\n\n\s*(?:[*-]|\(?[\da-z]{1,2}[).])\s+/i,
       developed_by:        /#{START_REGEX}developed by:.*?\n\n/im,
-      quote_begin:         /[`'"‘“]/,
-      quote_end:           /[`'"’”]/,
+      cc_dedication:       /The\s+text\s+of\s+the\s+Creative\s+Commons.*?Public\s+Domain\s+Dedication./im,
+      cc_wiki:             /wiki.creativecommons.org/i,
       cc_legal_code:       /^\s*Creative Commons Legal Code\s*$/i,
       cc0_info:            /For more information, please see\s*\S+zero\S+/im,
       cc0_disclaimer:      /CREATIVE COMMONS CORPORATION.*?\n\n/im,
@@ -35,10 +36,7 @@ module Licensee
       https:      { from: /http:/, to: 'https:' },
       ampersands: { from: '&', to: 'and' },
       dashes:     { from: /(?<!^)([—–-]+)(?!$)/, to: '-' },
-      quotes:     {
-        from: /#{REGEXES[:quote_begin]}+([\w -]*?\w)#{REGEXES[:quote_end]}+/,
-        to:   '"\1"'
-      }
+      quote:      { from: /[`'"‘“’”]/, to: "'" }
     }.freeze
 
     # Legally equivalent words that schould be ignored for comparison
@@ -88,10 +86,10 @@ module Licensee
       'owner'           => 'holder'
     }.freeze
     STRIP_METHODS = %i[
+      bom
+      cc_optional
       cc0_optional
       unlicense_optional
-      hrs
-      markdown_headings
       borders
       title
       version
@@ -99,9 +97,6 @@ module Licensee
       copyright
       title
       block_markup
-      span_markup
-      link_markup
-      all_rights_reserved
       developed_by
       end_of_terms
       whitespace
@@ -110,35 +105,30 @@ module Licensee
 
     # A set of each word in the license, without duplicates
     def wordset
-      @wordset ||= content_normalized&.scan(/(?:\w(?:'s|(?<=s)')?)+/)&.to_set
+      @wordset ||= content_normalized&.scan(%r{(?:[\w/-](?:'s|(?<=s)')?)+})&.to_set
     end
 
-    # Number of characteres in the normalized content
+    # Number of characters in the normalized content
     def length
       return 0 unless content_normalized
 
       content_normalized.length
     end
 
-    # Number of characters that could be added/removed to still be
-    # considered a potential match
-    def max_delta
-      @max_delta ||= fields_normalized.size * 10 +
-                     (length * Licensee.inverse_confidence_threshold).to_i
-    end
-
     # Given another license or project file, calculates the difference in length
     def length_delta(other)
       (length - other.length).abs
     end
 
     # Given another license or project file, calculates the similarity
-    # as a percentage of words in common
+    # as a percentage of words in common, minus a tiny penalty that
+    # increases with size difference between licenses so that false
+    # positives for long licnses are ruled out by this score alone.
     def similarity(other)
       overlap = (wordset_fieldless & other.wordset).size
       total = wordset_fieldless.size + other.wordset.size -
               fields_normalized_set.size
-      100.0 * (overlap * 2.0 / total)
+      (overlap * 200.0) / (total + variation_adjusted_length_delta(other) / 4)
     end
 
     # SHA1 of the normalized content
@@ -153,7 +143,7 @@ module Licensee
     def content_without_title_and_version
       @content_without_title_and_version ||= begin
         @_content = nil
-        ops = %i[html hrs comments markdown_headings title version]
+        ops = %i[html hrs comments markdown_headings link_markup title version]
         ops.each { |op| strip(op) }
         _content
       end
@@ -163,7 +153,7 @@ module Licensee
       @content_normalized ||= begin
         @_content = content_without_title_and_version.downcase
 
-        (NORMALIZATIONS.keys + %i[spelling bullets]).each { |op| normalize(op) }
+        (NORMALIZATIONS.keys + %i[spelling span_markup bullets]).each { |op| normalize(op) }
         STRIP_METHODS.each { |op| strip(op) }
 
         _content
@@ -191,12 +181,10 @@ module Licensee
       text.gsub!(/([^\n])\n([^\n])/, '\1 \2')
 
       text = text.split("\n").collect do |line|
-        if line =~ REGEXES[:hrs]
+        if line =~ REGEXES[:hrs] || line.length <= line_width
           line
-        elsif line.length > line_width
-          line.gsub(/(.{1,#{line_width}})(\s+|$)/, "\\1\n").strip
         else
-          line
+          line.gsub(/(.{1,#{line_width}})(\s+|$)/, "\\1\n").strip
         end
       end * "\n"
 
@@ -238,9 +226,7 @@ module Licensee
         meth = "strip_#{regex_or_sym}"
         return send(meth) if respond_to?(meth, true)
 
-        unless REGEXES[regex_or_sym]
-          raise ArgumentError, "#{regex_or_sym} is an invalid regex reference"
-        end
+        raise ArgumentError, "#{regex_or_sym} is an invalid regex reference" unless REGEXES[regex_or_sym]
 
         regex_or_sym = REGEXES[regex_or_sym]
       end
@@ -249,9 +235,7 @@ module Licensee
     end
 
     def strip_title
-      while _content =~ ContentHelper.title_regex
-        strip(ContentHelper.title_regex)
-      end
+      strip(ContentHelper.title_regex) while _content =~ ContentHelper.title_regex
     end
 
     def strip_borders
@@ -267,7 +251,7 @@ module Licensee
     end
 
     def strip_copyright
-      regex = Matchers::Copyright::REGEX
+      regex = Regexp.union(Matchers::Copyright::REGEX, REGEXES[:all_rights_reserved])
       strip(regex) while _content =~ regex
     end
 
@@ -279,6 +263,13 @@ module Licensee
       strip(REGEXES[:cc0_disclaimer])
     end
 
+    def strip_cc_optional
+      return unless _content.include? 'creative commons'
+
+      strip(REGEXES[:cc_dedication])
+      strip(REGEXES[:cc_wiki])
+    end
+
     def strip_unlicense_optional
       return unless _content.include? 'unlicense'
 
@@ -290,7 +281,7 @@ module Licensee
       @_content = body
     end
 
-    def strip_span_markup
+    def normalize_span_markup
       normalize(REGEXES[:span_markup], '\1')
     end
 
@@ -300,7 +291,7 @@ module Licensee
 
     def strip_html
       return unless respond_to?(:filename) && filename
-      return unless File.extname(filename) =~ /\.html?/i
+      return unless /\.html?/i.match?(File.extname(filename))
 
       require 'reverse_markdown'
       @_content = ReverseMarkdown.convert(_content, unknown_tags: :bypass)
@@ -324,7 +315,7 @@ module Licensee
     end
 
     def normalize_bullets
-      normalize(REGEXES[:bullet], "\n\n* ")
+      normalize(REGEXES[:bullet], "\n\n- ")
       normalize(/\)\s+\(/, ')(')
     end
 
@@ -341,5 +332,11 @@ module Licensee
     def fields_normalized_set
       @fields_normalized_set ||= fields_normalized.to_set
     end
+
+    def variation_adjusted_length_delta(other)
+      delta = length_delta(other)
+      adjusted_delta = delta - [fields_normalized.size, spdx_alt_segments].max * 4
+      adjusted_delta.positive? ? adjusted_delta : 0
+    end
   end
 end

data/lib/licensee/hash_helper.rb

@@ -13,7 +13,7 @@ module Licensee
                       value.to_h
                     else
                       value
-        end
+                    end
       end
 
       hash

data/lib/licensee/license.rb

@@ -4,6 +4,7 @@ require 'uri'
 
 module Licensee
   class InvalidLicense < ArgumentError; end
+
   class License
     @all = {}
     @keys_licenses = {}
@@ -26,9 +27,11 @@ module Licensee
           output.reject!(&:hidden?) unless options[:hidden]
           output.reject!(&:pseudo_license?) unless options[:pseudo]
           output.sort_by!(&:key)
-          return output if options[:featured].nil?
-
-          output.select { |l| l.featured? == options[:featured] }
+          if options[:featured].nil?
+            output
+          else
+            output.select { |l| l.featured? == options[:featured] }
+          end
         end
       end
 
@@ -53,14 +56,17 @@ module Licensee
       end
 
       def license_dir
-        dir = ::File.dirname(__FILE__)
-        ::File.expand_path '../../vendor/choosealicense.com/_licenses', dir
+        ::File.expand_path '../../vendor/choosealicense.com/_licenses', __dir__
       end
 
       def license_files
         @license_files ||= Dir.glob("#{license_dir}/*.txt")
       end
 
+      def spdx_dir
+        ::File.expand_path '../../vendor/license-list-XML/src', __dir__
+      end
+
       private
 
       def licenses
@@ -82,7 +88,7 @@ module Licensee
     # `other` - The project had a license, but we were not able to detect it
     # `no-license` - The project is not licensed (e.g., all rights reserved)
     #
-    # Note: A lack of detected license will be a nil license
+    # NOTE: A lack of detected license will be a nil license
     PSEUDO_LICENSES = %w[other no-license].freeze
 
     # Default options to use when retrieving licenses via #all
@@ -93,7 +99,7 @@ module Licensee
     }.freeze
 
     SOURCE_PREFIX = %r{https?://(?:www\.)?}i.freeze
-    SOURCE_SUFFIX = %r{(?:\.html?|\.txt|\/)(?:\?[^\s]*)?}i.freeze
+    SOURCE_SUFFIX = %r{(?:\.html?|\.txt|/)(?:\?[^\s]*)?}i.freeze
 
     HASH_METHODS = %i[
       key spdx_id meta url rules fields other? gpl? lgpl? cc?
@@ -154,9 +160,7 @@ module Licensee
       key_regex = Regexp.new string, 'i'
 
       parts = [title_regex, key_regex]
-      if meta.nickname
-        parts.push Regexp.new meta.nickname.sub(/\bGNU /i, '(?:GNU )?')
-      end
+      parts.push Regexp.new meta.nickname.sub(/\bGNU /i, '(?:GNU )?') if meta.nickname
 
       @title_regex = Regexp.union parts
     end
@@ -173,8 +177,8 @@ module Licensee
       return @source_regex if defined? @source_regex
       return unless meta.source
 
-      source = meta.source.dup.sub(/\A#{SOURCE_PREFIX}/, '')
-      source = source.sub(/#{SOURCE_SUFFIX}\z/, '')
+      source = meta.source.dup.sub(/\A#{SOURCE_PREFIX}/o, '')
+      source = source.sub(/#{SOURCE_SUFFIX}\z/o, '')
 
       escaped_source = Regexp.escape(source)
       @source_regex = /#{SOURCE_PREFIX}#{escaped_source}(?:#{SOURCE_SUFFIX})?/i
@@ -244,9 +248,7 @@ module Licensee
     # Raw content of license file, including YAML front matter
     def raw_content
       return if pseudo_license?
-      unless File.exist?(path)
-        raise Licensee::InvalidLicense, "'#{key}' is not a valid license key"
-      end
+      raise Licensee::InvalidLicense, "'#{key}' is not a valid license key" unless File.exist?(path)
 
       @raw_content ||= File.read(path, encoding: 'utf-8')
     end
@@ -260,5 +262,17 @@ module Licensee
     def yaml
       @yaml ||= parts[1] if parts
     end
+
+    def spdx_alt_segments
+      @spdx_alt_segments ||= begin
+        path = File.expand_path "#{spdx_id}.xml", Licensee::License.spdx_dir
+        raw_xml = File.read(path, encoding: 'utf-8')
+        text = raw_xml.match(%r{<text>(.*)</text>}m)[1]
+        text.gsub!(%r{<copyrightText>.*?</copyrightText>}m, '')
+        text.gsub!(%r{<titleText>.*?</titleText>}m, '')
+        text.gsub!(%r{<optional.*?>.*?</optional>}m, '')
+        text.scan(/<alt .*?>/m).size
+      end
+    end
   end
 end

data/lib/licensee/matchers.rb

@@ -12,6 +12,7 @@ module Licensee
     autoload :Exact,     'licensee/matchers/exact'
     autoload :Gemspec,   'licensee/matchers/gemspec'
     autoload :NpmBower,  'licensee/matchers/npm_bower'
+    autoload :NuGet,     'licensee/matchers/nuget'
     autoload :Package,   'licensee/matchers/package'
     autoload :Reference, 'licensee/matchers/reference'
     autoload :Spdx,      'licensee/matchers/spdx'