licensee 9.10.0 → 9.13.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa1cf2d146414880f019c339c2803443db7d63eb2568f7807079aac27a7aec98
-  data.tar.gz: a6243b9cc4b9068090023c70c89a5ea0253580575b534da0c00deaa799441bb5
+  metadata.gz: bd74e74f07c0dc4111b3cbbebf62ae3ab140a788ed511491b5995d45f2371de0
+  data.tar.gz: 90561b8b85ded55614a88e38b0469e80d110bd113e3a6ddfee9bdd840dec237a
 SHA512:
-  metadata.gz: cff2872a3d87b3c708ac2585dc41b64dd665008064303ec66a3501ec5160dbc87b5e78a4128921d5a95e3e7eb160ecc625d3d12f0fad36b05b59fa577088581b
-  data.tar.gz: 96466e4da40eb17a01625110f236c40cc90cb99b2e39c7057f2867721c77c565ba16cf08d4c4b02c799d1675a11c765e0dd313c761e273354b19fa8cb0fa1528
+  metadata.gz: bc64456f4f05411ab8152ce3ade984aa52851f39399d0bffe1e6c51fa4f9de6b00fcd84008bed0d0b71b0569a9799ba0b539ce6323fee55b4436c583ae4ce92c
+  data.tar.gz: d797f174a17e2f91eb8f4e7c9eff9b3d4631f58531d4d57cc9da98e75e062345a52354a9ca3a8e1f991cb70a92b91e7b600814dddca345c3528742011e92c8ed

data/bin/licensee

@@ -1,4 +1,5 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
 require 'dotenv/load'
 require 'thor'
@@ -15,9 +16,9 @@ class LicenseeCLI < Thor
 
   def path
     @path ||= if !options[:remote] || args.first =~ %r{^https://}
-      args.first || Dir.pwd
-    else
-      "https://github.com/#{args.first}"
+                args.first || Dir.pwd
+              else
+                "https://github.com/#{args.first}"
     end
   end
 
@@ -32,6 +33,6 @@ class LicenseeCLI < Thor
 end
 
 commands_dir = File.expand_path '../lib/licensee/commands/', __dir__
-Dir["#{commands_dir}/*.rb"].each { |c| require(c) }
+Dir["#{commands_dir}/*.rb"].sort.each { |c| require(c) }
 
 LicenseeCLI.start(ARGV)

data/lib/licensee.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative 'licensee/version'
 require 'forwardable'
 require 'pathname'
@@ -19,7 +21,7 @@ module Licensee
   CONFIDENCE_THRESHOLD = 98
 
   # Base domain from which to build license URLs
-  DOMAIN = 'http://choosealicense.com'.freeze
+  DOMAIN = 'http://choosealicense.com'
 
   class << self
     attr_writer :confidence_threshold
@@ -49,7 +51,7 @@ module Licensee
     end
 
     # Inverse of the confidence threshold, represented as a float
-    # By default this will be 0.05
+    # By default this will be 0.02
     def inverse_confidence_threshold
       @inverse_confidence_threshold ||=
         (1 - Licensee.confidence_threshold / 100.0).round(2)

data/lib/licensee/commands/detect.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class LicenseeCLI < Thor
   # Methods to call when displaying information about ProjectFiles
   MATCHED_FILE_METHODS = %i[
@@ -21,11 +23,11 @@ class LicenseeCLI < Thor
 
     rows = []
     rows << if project.license
-      ['License:', project.license.spdx_id]
-    elsif !project.licenses.empty?
-      ['Licenses:', project.licenses.map(&:spdx_id)]
-    else
-      ['License:', set_color('None', :red)]
+              ['License:', project.license.spdx_id]
+            elsif !project.licenses.empty?
+              ['Licenses:', project.licenses.map(&:spdx_id)]
+            else
+              ['License:', set_color('None', :red)]
     end
 
     unless project.matched_files.empty?
@@ -40,8 +42,10 @@ class LicenseeCLI < Thor
 
       MATCHED_FILE_METHODS.each do |method|
         next unless matched_file.respond_to? method
+
         value = matched_file.public_send method
         next if value.nil?
+
         rows << [humanize(method, :method), humanize(value, method)]
       end
       print_table rows, indent: 2
@@ -51,6 +55,7 @@ class LicenseeCLI < Thor
 
       licenses = licenses_by_similarity(matched_file)
       next if licenses.empty?
+
       say '  Closest non-matching licenses:'
       rows = licenses[0...3].map do |license, similarity|
         spdx_id = license.meta['spdx-id']

data/lib/licensee/commands/diff.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'tmpdir'
 
 class LicenseeCLI < Thor
@@ -39,26 +41,23 @@ class LicenseeCLI < Thor
 
   def license_to_diff
     return options[:license_to_diff] if options[:license_to_diff]
-    return project.license_file if remote?
+    return project.license_file if remote? || STDIN.tty? && project.license_file
 
     @license_to_diff ||= begin
-      if STDIN.tty?
-        error 'You must pipe license contents to the command via STDIN'
-        exit 1
-      end
-
       Licensee::ProjectFiles::LicenseFile.new(STDIN.read, 'LICENSE')
     end
   end
 
   def expected_license
-    @expected_license ||= Licensee::License.find options[:license] if options[:license]
+    if options[:license]
+      @expected_license ||= Licensee::License.find options[:license]
+    end
     return @expected_license if @expected_license
 
     if options[:license]
       error "#{options[:license]} is not a valid license"
     else
-      error 'You must provide an expected license'
+      error 'Usage: provide a license to diff against with --license (spdx name)'
     end
 
     error "Valid licenses: #{Licensee::License.all(hidden: true).map(&:key).join(', ')}"

data/lib/licensee/commands/license_path.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class LicenseeCLI < Thor
   desc 'license-path [PATH]', "Returns the path to the given project's license file"
   def license_path(_path)

data/lib/licensee/commands/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class LicenseeCLI < Thor
   desc 'version', 'Return the Licensee version'
   def version

data/lib/licensee/content_helper.rb

@@ -1,38 +1,129 @@
+# frozen_string_literal: true
+
 require 'set'
 require 'digest'
 
 module Licensee
   module ContentHelper
     DIGEST = Digest::SHA1
-    END_OF_TERMS_REGEX = /^[\s#*_]*end of terms and conditions\s*$/i
-    HR_REGEX = /[=\-\*][=\-\*\s]{3,}/
-    ALT_TITLE_REGEX = License::ALT_TITLE_REGEX
-    ALL_RIGHTS_RESERVED_REGEX = /\Aall rights reserved\.?$/i
-    WHITESPACE_REGEX = /\s+/
-    MARKDOWN_HEADING_REGEX = /\A\s*#+/
-    VERSION_REGEX = /\Aversion.*$/i
-    MARKUP_REGEX = /[#_*=~\[\]()`|>]+/
-    DEVELOPED_BY_REGEX = /\Adeveloped by:.*?\n\n/im
-    QUOTE_BEGIN_REGEX = /[`'"‘“]/
-    QUOTE_END_REGEX = /['"’”]/
+    START_REGEX = /\A\s*/.freeze
+    END_OF_TERMS_REGEX = /^[\s#*_]*end of terms and conditions\s*$/i.freeze
+    REGEXES = {
+      hrs:                 /^\s*[=\-\*]{3,}\s*$/,
+      all_rights_reserved: /#{START_REGEX}all rights reserved\.?$/i,
+      whitespace:          /\s+/,
+      markdown_headings:   /#{START_REGEX}#+/,
+      version:             /#{START_REGEX}version.*$/i,
+      span_markup:         /[_*~]+(.*?)[_*~]+/,
+      link_markup:         /\[(.+?)\]\(.+?\)/,
+      block_markup:        /^\s*>/,
+      border_markup:       /^[\*-](.*?)[\*-]$/,
+      comment_markup:      %r{^\s*?[/\*]{1,2}},
+      url:                 %r{#{START_REGEX}https?://[^ ]+\n},
+      bullet:              /\n\n\s*(?:[*-]|\(?[\da-z]{1,2}[)\.])\s+/i,
+      developed_by:        /#{START_REGEX}developed by:.*?\n\n/im,
+      quote_begin:         /[`'"‘“]/,
+      quote_end:           /[`'"’”]/,
+      cc_legal_code:       /^\s*Creative Commons Legal Code\s*$/i,
+      cc0_info:            /For more information, please see\s*\S+zero\S+/im,
+      cc0_disclaimer:      /CREATIVE COMMONS CORPORATION.*?\n\n/im,
+      unlicense_info:      /For more information, please.*\S+unlicense\S+/im,
+      mit_optional:        /\(including the next paragraph\)/i
+    }.freeze
+    NORMALIZATIONS = {
+      lists:      { from: /^\s*(?:\d\.|\*)\s+([^\n])/, to: '- \1' },
+      https:      { from: /http:/, to: 'https:' },
+      ampersands: { from: '&', to: 'and' },
+      dashes:     { from: /(?<!^)([—–-]+)(?!$)/, to: '-' },
+      quotes:     {
+        from: /#{REGEXES[:quote_begin]}+([\w -]*?\w)#{REGEXES[:quote_end]}+/,
+        to:   '"\1"'
+      }
+    }.freeze
+
+    # Legally equivalent words that schould be ignored for comparison
+    # See https://spdx.org/spdx-license-list/matching-guidelines
+    VARIETAL_WORDS = {
+      'acknowledgment'  => 'acknowledgement',
+      'analogue'        => 'analog',
+      'analyse'         => 'analyze',
+      'artefact'        => 'artifact',
+      'authorisation'   => 'authorization',
+      'authorised'      => 'authorized',
+      'calibre'         => 'caliber',
+      'cancelled'       => 'canceled',
+      'capitalisations' => 'capitalizations',
+      'catalogue'       => 'catalog',
+      'categorise'      => 'categorize',
+      'centre'          => 'center',
+      'emphasised'      => 'emphasized',
+      'favour'          => 'favor',
+      'favourite'       => 'favorite',
+      'fulfil'          => 'fulfill',
+      'fulfilment'      => 'fulfillment',
+      'initialise'      => 'initialize',
+      'judgment'        => 'judgement',
+      'labelling'       => 'labeling',
+      'labour'          => 'labor',
+      'licence'         => 'license',
+      'maximise'        => 'maximize',
+      'modelled'        => 'modeled',
+      'modelling'       => 'modeling',
+      'offence'         => 'offense',
+      'optimise'        => 'optimize',
+      'organisation'    => 'organization',
+      'organise'        => 'organize',
+      'practise'        => 'practice',
+      'programme'       => 'program',
+      'realise'         => 'realize',
+      'recognise'       => 'recognize',
+      'signalling'      => 'signaling',
+      'sub-license'     => 'sublicense',
+      'sub license'     => 'sublicense',
+      'utilisation'     => 'utilization',
+      'whilst'          => 'while',
+      'wilful'          => 'wilfull',
+      'non-commercial'  => 'noncommercial',
+      'cent'            => 'percent',
+      'owner'           => 'holder'
+    }.freeze
+    STRIP_METHODS = %i[
+      cc0_optional
+      unlicense_optional
+      hrs
+      markdown_headings
+      borders
+      title
+      version
+      url
+      copyright
+      title
+      block_markup
+      span_markup
+      link_markup
+      developed_by
+      end_of_terms
+      whitespace
+      mit_optional
+    ].freeze
 
     # A set of each word in the license, without duplicates
     def wordset
-      @wordset ||= if content_normalized
-        content_normalized.scan(/(?:\w(?:'s|(?<=s)')?)+/).to_set
-      end
+      @wordset ||= content_normalized&.scan(%r{(?:[\w\/](?:'s|(?<=s)')?)+})&.to_set
     end
 
     # Number of characteres in the normalized content
     def length
       return 0 unless content_normalized
+
       content_normalized.length
     end
 
     # Number of characters that could be added/removed to still be
     # considered a potential match
     def max_delta
-      @max_delta ||= (length * Licensee.inverse_confidence_threshold).to_i
+      @max_delta ||= fields_normalized.size * 10 +
+                     (length * Licensee.inverse_confidence_threshold).to_i
     end
 
     # Given another license or project file, calculates the difference in length
@@ -43,8 +134,9 @@ module Licensee
     # Given another license or project file, calculates the similarity
     # as a percentage of words in common
     def similarity(other)
-      overlap = (wordset & other.wordset).size
-      total = wordset.size + other.wordset.size
+      overlap = (wordset_fieldless & other.wordset).size
+      total = wordset_fieldless.size + other.wordset.size -
+              fields_normalized_set.size
       100.0 * (overlap * 2.0 / total)
     end
 
@@ -59,34 +151,21 @@ module Licensee
     # content with attribution first to detect attribuion in LicenseFile
     def content_without_title_and_version
       @content_without_title_and_version ||= begin
-        string = content.strip
-        string = strip_markdown_headings(string)
-        string = strip_hrs(string)
-        string = strip_title(string) while string =~ ContentHelper.title_regex
-        strip_version(string).strip
+        @_content = nil
+        ops = %i[html hrs comments markdown_headings title version]
+        ops.each { |op| strip(op) }
+        _content
       end
     end
 
-    # Content without title, version, copyright, whitespace, or insturctions
-    #
-    # wrap - Optional width to wrap the content
-    #
-    # Returns a string
     def content_normalized(wrap: nil)
-      return unless content
       @content_normalized ||= begin
-        string = content_without_title_and_version.downcase
-        while string =~ Matchers::Copyright::REGEX
-          string = strip_copyright(string)
-        end
-        string = strip_all_rights_reserved(string)
-        string = strip_developed_by(string)
-        string, _partition, _instructions = string.partition(END_OF_TERMS_REGEX)
-        string = normalize_lists(string)
-        string = normalize_quotes(string)
-        string = normalize_https(string)
-        string = strip_markup(string)
-        strip_whitespace(string)
+        @_content = content_without_title_and_version.downcase
+
+        (NORMALIZATIONS.keys + %i[spelling bullets]).each { |op| normalize(op) }
+        STRIP_METHODS.each { |op| strip(op) }
+
+        _content
       end
 
       if wrap.nil?
@@ -96,14 +175,24 @@ module Licensee
       end
     end
 
+    # Backwards compatibalize constants to avoid a breaking change
+    def self.const_missing(const)
+      key = const.to_s.downcase.gsub('_regex', '').to_sym
+      REGEXES[key] || super
+    end
+
     # Wrap text to the given line length
     def self.wrap(text, line_width = 80)
       return if text.nil?
+
       text = text.clone
+      text.gsub!(REGEXES[:bullet]) { |m| "\n#{m}\n" }
       text.gsub!(/([^\n])\n([^\n])/, '\1 \2')
 
       text = text.split("\n").collect do |line|
-        if line.length > line_width
+        if line =~ REGEXES[:hrs]
+          line
+        elsif line.length > line_width
           line.gsub(/(.{1,#{line_width}})(\s+|$)/, "\\1\n").strip
         else
           line
@@ -114,82 +203,142 @@ module Licensee
     end
 
     def self.format_percent(float)
-      "#{format('%.2f', float)}%"
+      "#{format('%<float>.2f', float: float)}%"
     end
 
     def self.title_regex
-      licenses = Licensee::License.all(hidden: true, psuedo: false)
-      titles = licenses.map(&:title_regex)
-
-      # Title regex must include the version to support matching within
-      # families, but for sake of normalization, we can be less strict
-      without_versions = licenses.map do |license|
-        next if license.title == license.name_without_version
-        Regexp.new Regexp.escape(license.name_without_version), 'i'
-      end
-      titles.concat(without_versions.compact)
+      @title_regex ||= begin
+        licenses = Licensee::License.all(hidden: true, psuedo: false)
+        titles = licenses.map(&:title_regex)
+
+        # Title regex must include the version to support matching within
+        # families, but for sake of normalization, we can be less strict
+        without_versions = licenses.map do |license|
+          next if license.title == license.name_without_version
+
+          Regexp.new Regexp.escape(license.name_without_version), 'i'
+        end
+        titles.concat(without_versions.compact)
 
-      /\A\s*\(?(the )?#{Regexp.union titles}.*$/i
+        /#{START_REGEX}\(?(?:the )?#{Regexp.union titles}.*?$/i
+      end
     end
 
     private
 
-    def strip_title(string)
-      strip(string, ContentHelper.title_regex)
+    def _content
+      @_content ||= content.to_s.dup.strip
+    end
+
+    def strip(regex_or_sym)
+      return unless _content
+
+      if regex_or_sym.is_a?(Symbol)
+        meth = "strip_#{regex_or_sym}"
+        return send(meth) if respond_to?(meth, true)
+
+        unless REGEXES[regex_or_sym]
+          raise ArgumentError, "#{regex_or_sym} is an invalid regex reference"
+        end
+
+        regex_or_sym = REGEXES[regex_or_sym]
+      end
+
+      @_content = _content.gsub(regex_or_sym, ' ').squeeze(' ').strip
+    end
+
+    def strip_title
+      while _content =~ ContentHelper.title_regex
+        strip(ContentHelper.title_regex)
+      end
+    end
+
+    def strip_borders
+      normalize(REGEXES[:border_markup], '\1')
+    end
+
+    def strip_comments
+      lines = _content.split("\n")
+      return if lines.count == 1
+      return unless lines.all? { |line| line =~ REGEXES[:comment_markup] }
+
+      strip(:comment_markup)
     end
 
-    def strip_version(string)
-      strip(string, VERSION_REGEX)
+    def strip_copyright
+      regex = Regexp.union(Matchers::Copyright::REGEX, REGEXES[:all_rights_reserved])
+      strip(regex) while _content =~ regex
+    end
+
+    def strip_cc0_optional
+      return unless _content.include? 'associating cc0'
+
+      strip(REGEXES[:cc_legal_code])
+      strip(REGEXES[:cc0_info])
+      strip(REGEXES[:cc0_disclaimer])
     end
 
-    def strip_copyright(string)
-      strip(string, Matchers::Copyright::REGEX)
+    def strip_unlicense_optional
+      return unless _content.include? 'unlicense'
+
+      strip(REGEXES[:unlicense_info])
     end
 
-    # Strip HRs from MPL
-    def strip_hrs(string)
-      strip(string, HR_REGEX)
+    def strip_end_of_terms
+      body, _partition, _instructions = _content.partition(END_OF_TERMS_REGEX)
+      @_content = body
     end
 
-    # Strip leading #s from the document
-    def strip_markdown_headings(string)
-      strip(string, MARKDOWN_HEADING_REGEX)
+    def strip_span_markup
+      normalize(REGEXES[:span_markup], '\1')
     end
 
-    def strip_whitespace(string)
-      strip(string, WHITESPACE_REGEX)
+    def strip_link_markup
+      normalize(REGEXES[:link_markup], '\1')
     end
 
-    def strip_all_rights_reserved(string)
-      strip(string, ALL_RIGHTS_RESERVED_REGEX)
+    def strip_html
+      return unless respond_to?(:filename) && filename
+      return unless File.extname(filename) =~ /\.html?/i
+
+      require 'reverse_markdown'
+      @_content = ReverseMarkdown.convert(_content, unknown_tags: :bypass)
     end
 
-    def strip_markup(string)
-      strip(string, MARKUP_REGEX)
+    def normalize(from_or_key, to = nil)
+      operation = { from: from_or_key, to: to } if to
+      operation ||= NORMALIZATIONS[from_or_key]
+
+      if operation
+        @_content = _content.gsub operation[:from], operation[:to]
+      elsif respond_to?("normalize_#{from_or_key}", true)
+        send("normalize_#{from_or_key}")
+      else
+        raise ArgumentError, "#{from_or_key} is an invalid normalization"
+      end
     end
 
-    def strip_developed_by(string)
-      strip(string, DEVELOPED_BY_REGEX)
+    def normalize_spelling
+      normalize(/\b#{Regexp.union(VARIETAL_WORDS.keys)}\b/, VARIETAL_WORDS)
     end
 
-    def strip(string, regex)
-      string.gsub(regex, ' ').squeeze(' ').strip
+    def normalize_bullets
+      normalize(REGEXES[:bullet], "\n\n* ")
+      normalize(/\)\s+\(/, ')(')
     end
 
-    # Replace all enclosing quotes with double quotes
-    # Single versus double quotes don't alter the meaning, and it's easier to
-    # strip double quotes if we still want to allow possessives
-    def normalize_quotes(string)
-      string.gsub(/#{QUOTE_BEGIN_REGEX}+([\w -]*?\w)#{QUOTE_END_REGEX}+/,
-                  '"\1"')
+    def wordset_fieldless
+      @wordset_fieldless ||= wordset - fields_normalized_set
     end
 
-    def normalize_https(string)
-      string.gsub(/http:/, 'https:')
+    # Returns an array of strings of substitutable fields in normalized content
+    def fields_normalized
+      @fields_normalized ||=
+        content_normalized.scan(LicenseField::FIELD_REGEX).flatten
     end
 
-    def normalize_lists(string)
-      string.gsub(/^\s*(\d\.|\*)/, '-')
+    def fields_normalized_set
+      @fields_normalized_set ||= fields_normalized.to_set
     end
   end
 end