licensee 9.10.1 → 9.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 354a3c318aa962a9c184c71850ba04574618b17f28582f92660a69e5405218ee
-  data.tar.gz: 1a96874eaaff65c949ca73261aced972cf20ab53d67da7b42d887ac360810eaf
+  metadata.gz: 613429f0350489c8dbdbdc80973d2557ad655efd2b89786096f290cf52632344
+  data.tar.gz: 87fca6ceddb6972dd946720355e6efb2661793d671ff92ce8934e281affa6ad5
 SHA512:
-  metadata.gz: 227b37abb076fd1d76d9fb296056bdf16b5a24b00ef2e95a0c8a883c7be6dee011bbea1df834b20d6ffe331815cc22f4726c94c1894400d04ae32944f01622dd
-  data.tar.gz: 41582f813fd9e57fa3c497c7edfd11fc447a131e7414ba7a1c3c25266239ec2efb6b867b84adffac704a5ba96e992b72d3f87215222e0aca32f40305e285cc2d
+  metadata.gz: 7bcb1c30bdebf4cad0a84ef7fd9dd4afe922ba8acac1a9124bdd96dc64d4ef736787f1b70c4ab0d01c3c05ba3547300caa3c882b2d99ea15117f72335f56b0a5
+  data.tar.gz: bb2f30c4f70a4610f55a530c36b65852270558b8b36342b1fe8ad1113afa4dca711214545b3abbeb19cdba8d7e2d05643b0e138c8964a5b3f51106ce8c1bf3db

data/lib/licensee/commands/diff.rb

@@ -39,14 +39,9 @@ class LicenseeCLI < Thor
 
   def license_to_diff
     return options[:license_to_diff] if options[:license_to_diff]
-    return project.license_file if remote?
+    return project.license_file if remote? || STDIN.tty? && project.license_file
 
     @license_to_diff ||= begin
-      if STDIN.tty?
-        error 'You must pipe license contents to the command via STDIN'
-        exit 1
-      end
-
       Licensee::ProjectFiles::LicenseFile.new(STDIN.read, 'LICENSE')
     end
   end
@@ -58,7 +53,7 @@ class LicenseeCLI < Thor
     if options[:license]
       error "#{options[:license]} is not a valid license"
     else
-      error 'You must provide an expected license'
+      error 'Usage: provide a license to diff against with --license (spdx name)'
     end
 
     error "Valid licenses: #{Licensee::License.all(hidden: true).map(&:key).join(', ')}"

data/lib/licensee/content_helper.rb

@@ -4,17 +4,88 @@ require 'digest'
 module Licensee
   module ContentHelper
     DIGEST = Digest::SHA1
-    END_OF_TERMS_REGEX = /^[\s#*_]*end of terms and conditions\s*$/i
-    HR_REGEX = /[=\-\*][=\-\*\s]{3,}/
+    START_REGEX = /\A\s*/.freeze
+    END_OF_TERMS_REGEX = /^[\s#*_]*end of terms and conditions\s*$/i.freeze
     ALT_TITLE_REGEX = License::ALT_TITLE_REGEX
-    ALL_RIGHTS_RESERVED_REGEX = /\Aall rights reserved\.?$/i
-    WHITESPACE_REGEX = /\s+/
-    MARKDOWN_HEADING_REGEX = /\A\s*#+/
-    VERSION_REGEX = /\Aversion.*$/i
-    MARKUP_REGEX = /[#_*=~\[\]()`|>]+/
-    DEVELOPED_BY_REGEX = /\Adeveloped by:.*?\n\n/im
-    QUOTE_BEGIN_REGEX = /[`'"‘“]/
-    QUOTE_END_REGEX = /['"’”]/
+    REGEXES = {
+      hrs:                 /^\s*[=\-\*]{3,}\s*$/,
+      all_rights_reserved: /#{START_REGEX}all rights reserved\.?$/i,
+      whitespace:          /\s+/,
+      markdown_headings:   /#{START_REGEX}#+/,
+      version:             /#{START_REGEX}version.*$/i,
+      span_markup:         /[_*~]+(.*?)[_*~]+/,
+      link_markup:         /\[(.+?)\]\(.+?\)/,
+      block_markup:        /^\s*>/,
+      border_markup:       /^[\*-](.*?)[\*-]$/,
+      comment_markup:      %r{^\s*?[/\*]{1,2}},
+      url:                 %r{#{START_REGEX}https?://[^ ]+\n},
+      bullet:              /\n\n\s*(?:[*-]|\(?[\da-z]{1,2}[)\.])\s+/i,
+      developed_by:        /#{START_REGEX}developed by:.*?\n\n/im,
+      quote_begin:         /[`'"‘“]/,
+      quote_end:           /[`'"’”]/
+    }.freeze
+    NORMALIZATIONS = {
+      lists:      { from: /^\s*(?:\d\.|\*)\s+([^\n])/, to: '- \1' },
+      https:      { from: /http:/, to: 'https:' },
+      ampersands: { from: '&', to: 'and' },
+      dashes:     { from: /(?<!^)([—–-]+)(?!$)/, to: '-' },
+      quotes:     {
+        from: /#{REGEXES[:quote_begin]}+([\w -]*?\w)#{REGEXES[:quote_end]}+/,
+        to:   '"\1"'
+      }
+    }.freeze
+
+    # Legally equivalent words that schould be ignored for comparison
+    # See https://spdx.org/spdx-license-list/matching-guidelines
+    VARIETAL_WORDS = {
+      'acknowledgment'  => 'acknowledgement',
+      'analogue'        => 'analog',
+      'analyse'         => 'analyze',
+      'artefact'        => 'artifact',
+      'authorisation'   => 'authorization',
+      'authorised'      => 'authorized',
+      'calibre'         => 'caliber',
+      'cancelled'       => 'canceled',
+      'capitalisations' => 'capitalizations',
+      'catalogue'       => 'catalog',
+      'categorise'      => 'categorize',
+      'centre'          => 'center',
+      'emphasised'      => 'emphasized',
+      'favour'          => 'favor',
+      'favourite'       => 'favorite',
+      'fulfil'          => 'fulfill',
+      'fulfilment'      => 'fulfillment',
+      'initialise'      => 'initialize',
+      'judgment'        => 'judgement',
+      'labelling'       => 'labeling',
+      'labour'          => 'labor',
+      'licence'         => 'license',
+      'maximise'        => 'maximize',
+      'modelled'        => 'modeled',
+      'modelling'       => 'modeling',
+      'offence'         => 'offense',
+      'optimise'        => 'optimize',
+      'organisation'    => 'organization',
+      'organise'        => 'organize',
+      'practise'        => 'practice',
+      'programme'       => 'program',
+      'realise'         => 'realize',
+      'recognise'       => 'recognize',
+      'signalling'      => 'signaling',
+      'sub-license'     => 'sublicense',
+      'sub license'     => 'sublicense',
+      'utilisation'     => 'utilization',
+      'whilst'          => 'while',
+      'wilful'          => 'wilfull',
+      'non-commercial'  => 'noncommercial',
+      'cent'            => 'percent',
+      'owner'           => 'holder'
+    }.freeze
+    STRIP_METHODS = %i[
+      hrs markdown_headings borders title version url copyright
+      block_markup span_markup link_markup
+      all_rights_reserved developed_by end_of_terms whitespace
+    ].freeze
 
     # A set of each word in the license, without duplicates
     def wordset
@@ -60,35 +131,21 @@ module Licensee
     # content with attribution first to detect attribuion in LicenseFile
     def content_without_title_and_version
       @content_without_title_and_version ||= begin
-        string = content.strip
-        string = strip_markdown_headings(string)
-        string = strip_hrs(string)
-        string = strip_title(string) while string =~ ContentHelper.title_regex
-        strip_version(string).strip
+        @_content = nil
+        ops = %i[hrs comments markdown_headings title version]
+        ops.each { |op| strip(op) }
+        _content
       end
     end
 
-    # Content without title, version, copyright, whitespace, or insturctions
-    #
-    # wrap - Optional width to wrap the content
-    #
-    # Returns a string
     def content_normalized(wrap: nil)
-      return unless content
-
       @content_normalized ||= begin
-        string = content_without_title_and_version.downcase
-        while string =~ Matchers::Copyright::REGEX
-          string = strip_copyright(string)
-        end
-        string = strip_all_rights_reserved(string)
-        string = strip_developed_by(string)
-        string, _partition, _instructions = string.partition(END_OF_TERMS_REGEX)
-        string = normalize_lists(string)
-        string = normalize_quotes(string)
-        string = normalize_https(string)
-        string = strip_markup(string)
-        strip_whitespace(string)
+        @_content = content_without_title_and_version.downcase
+
+        (NORMALIZATIONS.keys + %i[spelling bullets]).each { |op| normalize(op) }
+        STRIP_METHODS.each { |op| strip(op) }
+
+        _content
       end
 
       if wrap.nil?
@@ -98,15 +155,24 @@ module Licensee
       end
     end
 
+    # Backwards compatibalize constants to avoid a breaking change
+    def self.const_missing(const)
+      key = const.to_s.downcase.gsub('_regex', '').to_sym
+      REGEXES[key] || super
+    end
+
     # Wrap text to the given line length
     def self.wrap(text, line_width = 80)
       return if text.nil?
 
       text = text.clone
+      text.gsub!(REGEXES[:bullet]) { |m| "\n#{m}\n" }
       text.gsub!(/([^\n])\n([^\n])/, '\1 \2')
 
       text = text.split("\n").collect do |line|
-        if line.length > line_width
+        if line =~ REGEXES[:hrs]
+          line
+        elsif line.length > line_width
           line.gsub(/(.{1,#{line_width}})(\s+|$)/, "\\1\n").strip
         else
           line
@@ -121,79 +187,104 @@ module Licensee
     end
 
     def self.title_regex
-      licenses = Licensee::License.all(hidden: true, psuedo: false)
-      titles = licenses.map(&:title_regex)
+      @title_regex ||= begin
+        licenses = Licensee::License.all(hidden: true, psuedo: false)
+        titles = licenses.map(&:title_regex)
 
-      # Title regex must include the version to support matching within
-      # families, but for sake of normalization, we can be less strict
-      without_versions = licenses.map do |license|
-        next if license.title == license.name_without_version
+        # Title regex must include the version to support matching within
+        # families, but for sake of normalization, we can be less strict
+        without_versions = licenses.map do |license|
+          next if license.title == license.name_without_version
 
-        Regexp.new Regexp.escape(license.name_without_version), 'i'
-      end
-      titles.concat(without_versions.compact)
+          Regexp.new Regexp.escape(license.name_without_version), 'i'
+        end
+        titles.concat(without_versions.compact)
 
-      /\A\s*\(?(the )?#{Regexp.union titles}.*$/i
+        /#{START_REGEX}\(?(?:the )?#{Regexp.union titles}.*?$/i
+      end
     end
 
     private
 
-    def strip_title(string)
-      strip(string, ContentHelper.title_regex)
+    # rubocop:disable Naming/MemoizedInstanceVariableName
+    def _content
+      @_content ||= content.to_s.dup.strip
     end
+    # rubocop:enable Naming/MemoizedInstanceVariableName
 
-    def strip_version(string)
-      strip(string, VERSION_REGEX)
-    end
+    def strip(regex_or_sym)
+      return unless _content
 
-    def strip_copyright(string)
-      strip(string, Matchers::Copyright::REGEX)
+      if regex_or_sym.is_a?(Symbol)
+        meth = "strip_#{regex_or_sym}"
+        return send(meth) if respond_to?(meth, true)
+
+        unless REGEXES[regex_or_sym]
+          raise ArgumentError, "#{regex_or_sym} is an invalid regex reference"
+        end
+
+        regex_or_sym = REGEXES[regex_or_sym]
+      end
+
+      @_content = _content.gsub(regex_or_sym, ' ').squeeze(' ').strip
     end
 
-    # Strip HRs from MPL
-    def strip_hrs(string)
-      strip(string, HR_REGEX)
+    def strip_title
+      while _content =~ ContentHelper.title_regex
+        strip(ContentHelper.title_regex)
+      end
     end
 
-    # Strip leading #s from the document
-    def strip_markdown_headings(string)
-      strip(string, MARKDOWN_HEADING_REGEX)
+    def strip_borders
+      normalize(REGEXES[:border_markup], '\1')
     end
 
-    def strip_whitespace(string)
-      strip(string, WHITESPACE_REGEX)
+    def strip_comments
+      lines = _content.split("\n")
+      return if lines.count == 1
+      return unless lines.all? { |line| line =~ REGEXES[:comment_markup] }
+
+      strip(:comment_markup)
     end
 
-    def strip_all_rights_reserved(string)
-      strip(string, ALL_RIGHTS_RESERVED_REGEX)
+    def strip_copyright
+      regex = Matchers::Copyright::REGEX
+      strip(regex) while _content =~ regex
     end
 
-    def strip_markup(string)
-      strip(string, MARKUP_REGEX)
+    def strip_end_of_terms
+      body, _partition, _instructions = _content.partition(END_OF_TERMS_REGEX)
+      @_content = body
     end
 
-    def strip_developed_by(string)
-      strip(string, DEVELOPED_BY_REGEX)
+    def strip_span_markup
+      normalize(REGEXES[:span_markup], '\1')
     end
 
-    def strip(string, regex)
-      string.gsub(regex, ' ').squeeze(' ').strip
+    def strip_link_markup
+      normalize(REGEXES[:link_markup], '\1')
     end
 
-    # Replace all enclosing quotes with double quotes
-    # Single versus double quotes don't alter the meaning, and it's easier to
-    # strip double quotes if we still want to allow possessives
-    def normalize_quotes(string)
-      string.gsub(/#{QUOTE_BEGIN_REGEX}+([\w -]*?\w)#{QUOTE_END_REGEX}+/,
-                  '"\1"')
+    def normalize(from_or_key, to = nil)
+      operation = { from: from_or_key, to: to } if to
+      operation ||= NORMALIZATIONS[from_or_key]
+
+      if operation
+        @_content = _content.gsub operation[:from], operation[:to]
+      elsif respond_to?("normalize_#{from_or_key}", true)
+        send("normalize_#{from_or_key}")
+      else
+        raise ArgumentError, "#{from_or_key} is an invalid normalization"
+      end
     end
 
-    def normalize_https(string)
-      string.gsub(/http:/, 'https:')
+    def normalize_spelling
+      normalize(/\b#{Regexp.union(VARIETAL_WORDS.keys)}\b/, VARIETAL_WORDS)
     end
 
-    def normalize_lists(string)
-      string.gsub(/^\s*(\d\.|\*)/, '-')
+    def normalize_bullets
+      normalize(REGEXES[:bullet], "\n\n* ")
+      normalize(/\)\s+\(/, ')(')
     end
   end
 end

data/lib/licensee/license.rb

@@ -23,6 +23,7 @@ module Licensee
           output = licenses.dup
           output.reject!(&:hidden?) unless options[:hidden]
           output.reject!(&:pseudo_license?) unless options[:pseudo]
+          output.sort_by!(&:key)
           return output if options[:featured].nil?
 
           output.select { |l| l.featured? == options[:featured] }
@@ -95,8 +96,8 @@ module Licensee
       'bsd-3-clause-clear' => /(?:clear bsd|bsd 3-clause(?: clear)?)/i
     }.freeze
 
-    SOURCE_PREFIX = %r{https?://(?:www\.)?}i
-    SOURCE_SUFFIX = %r{(?:\.html?|\.txt|\/)(?:\?[^\s]*)?}i
+    SOURCE_PREFIX = %r{https?://(?:www\.)?}i.freeze
+    SOURCE_SUFFIX = %r{(?:\.html?|\.txt|\/)(?:\?[^\s]*)?}i.freeze
 
     HASH_METHODS = %i[
       key spdx_id meta url rules fields other? gpl? lgpl? cc?
@@ -145,11 +146,11 @@ module Licensee
       title_regex ||= begin
         string = name.downcase.sub('*', 'u')
         string.sub!(/\Athe /i, '')
-        string.sub!(/ license\z/i, '')
         string.sub!(/,? version /, ' ')
         string.sub!(/v(\d+\.\d+)/, '\1')
         string = Regexp.escape(string)
-        string = string.sub(/ (\d+\\.\d+)/, ',?\ (?:version\ |v)?\1')
+        string = string.sub(/\\ licen[sc]e/i, '(?:\ licen[sc]e)?')
+        string = string.sub(/\\ (\d+\\.\d+)/, ',?\s+(?:version\ |v(?:\. )?)?\1')
         string = string.sub(/\bgnu\\ /, '(?:GNU )?')
         Regexp.new string, 'i'
       end

data/lib/licensee/license_field.rb

@@ -45,7 +45,7 @@ module Licensee
     end
 
     alias key name
-    FIELD_REGEX = /\[(#{Regexp.union(LicenseField.keys)})\]/
+    FIELD_REGEX = /\[(#{Regexp.union(LicenseField.keys)})\]/.freeze
 
     # The human-readable field name
     def label

data/lib/licensee/matchers/cabal.rb

@@ -3,7 +3,7 @@ module Licensee
     class Cabal < Licensee::Matchers::Package
       # While we could parse the cabal file, prefer
       # a lenient regex for speed and security. Moar parsing moar problems.
-      LICENSE_REGEX = /^\s*license\s*\:\s*([a-z\-0-9\.]+)\s*$/ix
+      LICENSE_REGEX = /^\s*license\s*\:\s*([a-z\-0-9\.]+)\s*$/ix.freeze
 
       private
 

data/lib/licensee/matchers/cargo.rb

@@ -3,7 +3,7 @@ module Licensee
     class Cargo < Licensee::Matchers::Package
       LICENSE_REGEX = %r{
         ^\s*[\'\"]?license[\'\"]?\s*=\s*[\'\"]([a-z\-0-9\. +()\/]+)[\'\"]\s*
-      }ix
+      }ix.freeze
 
       private
 

data/lib/licensee/matchers/copyright.rb

@@ -5,12 +5,12 @@ module Licensee
 
       # rubocop:disable Metrics/LineLength
       COPYRIGHT_SYMBOLS = Regexp.union([/copyright/i, /\(c\)/i, "\u00A9", "\xC2\xA9"])
-      REGEX = /\A(\s*#{COPYRIGHT_SYMBOLS}.*$)+/i
+      REGEX = /#{ContentHelper::START_REGEX}(?:portions )?(\s*#{COPYRIGHT_SYMBOLS}.*$)+$/i.freeze
       # rubocop:enable Metrics/LineLength
 
       def match
         # Note: must use content, and not content_normalized here
-        if @file.content.strip =~ /\A#{REGEX}\z/i
+        if file.content.strip =~ /#{REGEX}+\z/i
           Licensee::License.find('no-license')
         end
       rescue Encoding::CompatibilityError

data/lib/licensee/matchers/cran.rb

@@ -5,9 +5,9 @@ module Licensee
 
       # While we could parse the DESCRIPTION file, prefer
       # a lenient regex for speed and security. Moar parsing moar problems.
-      LICENSE_FIELD_REGEX = /^license:\s*(.+)/i
-      PLUS_FILE_LICENSE_REGEX = /\s*\+\s*file\s+LICENSE$/i
-      GPL_VERSION_REGEX = /^GPL(?:-([23])|\s*\(\s*>=\s*([23])\s*\))$/i
+      LICENSE_FIELD_REGEX = /^license:\s*(.+)/i.freeze
+      PLUS_FILE_LICENSE_REGEX = /\s*\+\s*file\s+LICENSE$/i.freeze
+      GPL_VERSION_REGEX = /^GPL(?:-([23])|\s*\(\s*>=\s*([23])\s*\))$/i.freeze
 
       private
 

data/lib/licensee/matchers/dist_zilla.rb

@@ -3,7 +3,7 @@ module Licensee
     class DistZilla < Licensee::Matchers::Package
       attr_reader :file
 
-      LICENSE_REGEX = /^license\s*=\s*([a-z\-0-9\._]+)/i
+      LICENSE_REGEX = /^license\s*=\s*([a-z\-0-9\._]+)/i.freeze
 
       private