licensee 9.18.0 → 9.19.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 224c5413c75e78cad466f9d41b8a18a8b0a7ddf348740521ed37dbea4e25e674
-  data.tar.gz: f73bde77074c0e4feff4190551effbaebd685abf539b944ed0509231090fe8be
+  metadata.gz: 4878becb1edfcb446503f89645e4daeeacc60715ca978a8d4391418b0493f255
+  data.tar.gz: 651e41f44d60ab16669584edeb693967e4aa1a6a6c911c7a944fc6b35bd05503
 SHA512:
-  metadata.gz: 75680afe6a50966f190c3748130cfb72b4b4d68a846aec482405e14756a9d1211754b1a94328e79f38619dc3ee3c1999490b9cab157fc7b76b8cc03661316a7b
-  data.tar.gz: 3622c24de4f843dd0f59f37a3db30c8a9e6feba68644f7dce4168adb7a304f97175a41595c70c789acdd4e2166bfa3eb6547b485ce7e0a476719457530652d54
+  metadata.gz: c7e96fea4cd873710794c9e68f93edc8f016a620c55464f0f5c8d8ba82c90de8195376f858d8bc6125bb1ac170e980a98557fd135d68ab38ed98e2ba1d5e030e
+  data.tar.gz: caf4cc661f13ca7ce0ea847e6cd95ec45dd21472eb03cb6862d0cb39c52823818a70373b1133f55570492297543e7db6d4c7adeeefdbef6dd8c5795f67b19919

data/LICENSE.md

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2014-2021 Ben Balter and Licensee contributors
+Copyright (c) Ben Balter and Licensee contributors
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/bin/licensee

@@ -7,6 +7,7 @@ require 'json'
 
 require_relative '../lib/licensee'
 
+# Thor CLI entrypoint for the `licensee` command.
 class LicenseeCLI < Thor
   package_name 'Licensee'
   class_option :remote, type: :boolean, desc: 'Assume PATH is a GitHub owner/repo path'
@@ -30,6 +31,7 @@ class LicenseeCLI < Thor
     @project ||= Licensee.project(path,
                                   detect_packages: options[:packages],
                                   detect_readme:   options[:readme],
+                                  filesystem:      options[:filesystem],
                                   ref:             options[:ref])
   end
 

data/lib/licensee/commands/detect.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
+require_relative 'detect_helpers'
+
+# Implementation of the `licensee detect` command.
 class LicenseeCLI < Thor
-  # Methods to call when displaying information about ProjectFiles
-  MATCHED_FILE_METHODS = %i[
-    content_hash attribution confidence matcher license
-  ].freeze
+  include Licensee::Commands::DetectCLIHelpers
 
   desc 'detect [PATH]', 'Detect the license of the given project'
   option :json, type: :boolean, desc: 'Return output as JSON'
@@ -14,94 +14,14 @@ class LicenseeCLI < Thor
   option :license, type: :string, desc: 'The SPDX ID or key of the license to compare (implies --diff)'
   option :diff, type: :boolean, desc: 'Compare the license to the closest match'
   option :ref, type: :string, desc: 'The name of the commit/branch/tag to search (github.com only)'
+  option :filesystem, type: :boolean, desc: 'Force looking at the filesystem (ignore git data)'
   def detect(_path = nil)
     Licensee.confidence_threshold = options[:confidence]
 
-    if options[:json]
-      say project.to_h.to_json
-      exit !project.licenses.empty?
-    end
-
-    rows = []
-    rows << if project.license
-              ['License:', project.license.spdx_id]
-            elsif !project.licenses.empty?
-              ['Licenses:', project.licenses.map(&:spdx_id)]
-            else
-              ['License:', set_color('None', :red)]
-            end
-
-    rows << ['Matched files:', project.matched_files.map(&:filename).join(', ')] unless project.matched_files.empty?
-
-    print_table rows
-
-    project.matched_files.each do |matched_file|
-      rows = []
-      say "#{matched_file.filename}:"
-
-      MATCHED_FILE_METHODS.each do |method|
-        next unless matched_file.respond_to? method
-
-        value = matched_file.public_send method
-        next if value.nil?
-
-        rows << [humanize(method, :method), humanize(value, method)]
-      end
-      print_table rows, indent: 2
-
-      next unless matched_file.is_a? Licensee::ProjectFiles::LicenseFile
-      next if matched_file.confidence == 100
-
-      licenses = licenses_by_similarity(matched_file)
-      next if licenses.empty?
-
-      say '  Closest non-matching licenses:'
-      rows = licenses[0...3].map do |license, similarity|
-        spdx_id = license.meta['spdx-id']
-        percent = Licensee::ContentHelper.format_percent(similarity)
-        ["#{spdx_id} similarity:", percent]
-      end
-      print_table rows, indent: 4
-    end
-
-    if project.license_file && (options[:license] || options[:diff])
-      license = options[:license] || closest_license_key(project.license_file)
-      if license
-        invoke(:diff, nil,
-               license: license, license_to_diff: project.license_file)
-      end
-    end
-
-    exit !project.licenses.empty?
-  end
-
-  private
-
-  # Given a string or object, prepares it for output and human consumption
-  def humanize(value, type = nil)
-    case type
-    when :license
-      value.spdx_id
-    when :matcher
-      value.class
-    when :confidence
-      Licensee::ContentHelper.format_percent(value)
-    when :method
-      "#{value.to_s.tr('_', ' ').capitalize}:"
-    else
-      value
-    end
-  end
-
-  def licenses_by_similarity(matched_file)
-    matcher = Licensee::Matchers::Dice.new(matched_file)
-    potential_licenses = Licensee.licenses(hidden: true).select(&:wordset)
-    matcher.instance_variable_set(:@potential_licenses, potential_licenses)
-    matcher.licenses_by_similarity
-  end
+    handle_json_output if options[:json]
 
-  def closest_license_key(matched_file)
-    licenses = licenses_by_similarity(matched_file)
-    licenses.first.first.key unless licenses.empty?
+    print_project_summary
+    print_matched_files
+    maybe_diff_license_file
   end
 end

data/lib/licensee/commands/detect_helpers.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Licensee
+  module Commands
+    # Helper methods for formatting `licensee detect` output.
+    module DetectCLIHelpers
+      # Methods to call when displaying information about ProjectFiles
+      MATCHED_FILE_METHODS = %i[
+        content_hash attribution confidence matcher license
+      ].freeze
+
+      private
+
+      # Given a string or object, prepares it for output and human consumption
+      def humanize(value, type = nil)
+        return value.spdx_id if type == :license
+        return value.class if type == :matcher
+        return Licensee::ContentHelper.format_percent(value) if type == :confidence
+        return "#{value.to_s.tr('_', ' ').capitalize}:" if type == :method
+
+        value
+      end
+
+      def licenses_by_similarity(matched_file)
+        matcher = Licensee::Matchers::Dice.new(matched_file)
+        potential_licenses = Licensee.licenses(hidden: true).select(&:wordset)
+        matcher.instance_variable_set(:@potential_licenses, potential_licenses)
+        matcher.licenses_by_similarity
+      end
+
+      def closest_license_key(matched_file)
+        licenses = licenses_by_similarity(matched_file)
+        licenses.first.first.key unless licenses.empty?
+      end
+
+      def handle_json_output
+        say project.to_h.to_json
+        exit !project.licenses.empty?
+      end
+
+      def print_project_summary
+        rows = [license_summary_row]
+        matched_files_row = matched_files_summary_row
+        rows << matched_files_row if matched_files_row
+        print_table rows
+      end
+
+      def license_summary_row
+        if project.license
+          ['License:', project.license.spdx_id]
+        elsif !project.licenses.empty?
+          ['Licenses:', project.licenses.map(&:spdx_id)]
+        else
+          ['License:', set_color('None', :red)]
+        end
+      end
+
+      def matched_files_summary_row
+        return if project.matched_files.empty?
+
+        ['Matched files:', project.matched_files.map(&:path).join(', ')]
+      end
+
+      def print_matched_files
+        project.matched_files.each do |matched_file|
+          print_matched_file_summary(matched_file)
+          print_closest_non_matching_licenses(matched_file)
+        end
+      end
+
+      def print_matched_file_summary(matched_file)
+        say "#{matched_file.path}:"
+        print_table matched_file_rows(matched_file), indent: 2
+      end
+
+      def matched_file_rows(matched_file)
+        MATCHED_FILE_METHODS.filter_map do |method|
+          matched_file_row(matched_file, method)
+        end
+      end
+
+      def matched_file_row(matched_file, method)
+        return unless matched_file.respond_to?(method)
+
+        value = matched_file.public_send(method)
+        return if value.nil?
+
+        [humanize(method, :method), humanize(value, method)]
+      end
+
+      def print_closest_non_matching_licenses(matched_file)
+        licenses = closest_non_matching_licenses(matched_file)
+        return unless licenses
+
+        say '  Closest non-matching licenses:'
+        print_table closest_non_matching_rows(licenses), indent: 4
+      end
+
+      def closest_non_matching_licenses(matched_file)
+        return unless matched_file.is_a?(Licensee::ProjectFiles::LicenseFile)
+        return if matched_file.confidence == 100
+
+        licenses = licenses_by_similarity(matched_file)
+        licenses.empty? ? nil : licenses
+      end
+
+      def closest_non_matching_rows(licenses)
+        licenses[0...3].map do |license, similarity|
+          spdx_id = license.meta['spdx-id']
+          ["#{spdx_id} similarity:", Licensee::ContentHelper.format_percent(similarity)]
+        end
+      end
+
+      def maybe_diff_license_file
+        return unless project.license_file
+        return unless options[:license] || options[:diff]
+
+        license = options[:license] || closest_license_key(project.license_file)
+        return unless license
+
+        invoke(:diff, nil, license: license, license_to_diff: project.license_file)
+      end
+    end
+  end
+end

data/lib/licensee/commands/diff.rb

@@ -2,42 +2,64 @@
 
 require 'tmpdir'
 
+# Implementation of the `licensee diff` command.
 class LicenseeCLI < Thor
   desc 'diff [PATH]', 'Compare the given license text to a known license'
   option :license, type: :string, desc: 'The SPDX ID or key of the license to compare'
   def diff(_path = nil)
     say "Comparing to #{expected_license.name}:"
-    rows = []
+    print_table diff_summary_rows
+    exit_on_exact_match
+    say word_diff
+  end
+
+  private
 
-    left = expected_license.content_normalized(wrap: 80)
-    right = license_to_diff.content_normalized(wrap: 80)
+  def diff_summary_rows
+    [
+      ['Input Length:', license_to_diff.length],
+      ['License length:', expected_license.length],
+      ['Similarity:', formatted_similarity]
+    ]
+  end
+
+  def formatted_similarity
     similarity = expected_license.similarity(license_to_diff)
-    similarity = Licensee::ContentHelper.format_percent(similarity)
-
-    rows << ['Input Length:', license_to_diff.length]
-    rows << ['License length:', expected_license.length]
-    rows << ['Similarity:', similarity]
-    print_table rows
-
-    if left == right
-      say 'Exact match!', :green
-      exit
-    end
-
-    Dir.mktmpdir do |dir|
-      path = File.expand_path 'LICENSE', dir
-      Dir.chdir(dir) do
-        `git init`
-        File.write(path, left)
-        `git add LICENSE`
-        `git commit -m 'left'`
-        File.write(path, right)
-        say `git diff --word-diff`
-      end
-    end
+    Licensee::ContentHelper.format_percent(similarity)
   end
 
-  private
+  def exit_on_exact_match
+    return unless expected_text == input_text
+
+    say 'Exact match!', :green
+    exit
+  end
+
+  def expected_text
+    @expected_text ||= expected_license.content_normalized(wrap: 80)
+  end
+
+  def input_text
+    @input_text ||= license_to_diff.content_normalized(wrap: 80)
+  end
+
+  def word_diff
+    Dir.mktmpdir { |dir| word_diff_in_dir(dir) }
+  end
+
+  def word_diff_in_dir(dir)
+    path = File.expand_path 'LICENSE', dir
+    Dir.chdir(dir) { git_word_diff(path) }
+  end
+
+  def git_word_diff(path)
+    `git init`
+    File.write(path, expected_text)
+    `git add LICENSE`
+    `git commit -m 'left'`
+    File.write(path, input_text)
+    `git diff --word-diff`
+  end
 
   def license_to_diff
     return options[:license_to_diff] if options[:license_to_diff]
@@ -47,16 +69,23 @@ class LicenseeCLI < Thor
   end
 
   def expected_license
-    @expected_license ||= Licensee::License.find options[:license] if options[:license]
-    return @expected_license if @expected_license
+    return @expected_license if defined?(@expected_license)
 
-    if options[:license]
-      error "#{options[:license]} is not a valid license"
-    else
-      error 'Usage: provide a license to diff against with --license (spdx name)'
-    end
+    @expected_license = Licensee::License.find(options[:license]) if options[:license]
+    return @expected_license if @expected_license
 
-    error "Valid licenses: #{Licensee::License.all(hidden: true).map(&:key).join(', ')}"
+    error expected_license_error_message
     exit 1
   end
+
+  def expected_license_error_message
+    message = if options[:license]
+                "#{options[:license]} is not a valid license"
+              else
+                'Usage: provide a license to diff against with --license (spdx name)'
+              end
+
+    valid_licenses = Licensee::License.all(hidden: true).map(&:key).join(', ')
+    "#{message}\nValid licenses: #{valid_licenses}"
+  end
 end

data/lib/licensee/commands/license_path.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+# `licensee license-path` command implementation.
 class LicenseeCLI < Thor
   desc 'license-path [PATH]', "Returns the path to the given project's license file"
   def license_path(path)

data/lib/licensee/commands/version.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+# `licensee version` command implementation.
 class LicenseeCLI < Thor
   desc 'version', 'Return the Licensee version'
   def version

data/lib/licensee/content_helper/constants.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Licensee
+  module ContentHelper
+    module Constants
+      DIGEST = Digest::SHA1
+      START_REGEX = /\A\s*/
+      END_OF_TERMS_REGEX = /^[\s#*_]*end of (the )?terms and conditions[\s#*_]*$/i
+      REGEXES = {
+        bom:                 /#{START_REGEX}\xEF\xBB\xBF/,
+        hrs:                 /^\s*[=\-*]{3,}\s*$/,
+        all_rights_reserved: /#{START_REGEX}all rights reserved\.?$/i,
+        whitespace:          /\s+/,
+        markdown_headings:   /^\s*#+/,
+        version:             /#{START_REGEX}version.*$/i,
+        span_markup:         /[_*~]+(.*?)[_*~]+/,
+        link_markup:         /\[(.+?)\]\(.+?\)/,
+        block_markup:        /^\s*>/,
+        border_markup:       /^[*-](.*?)[*-]$/,
+        comment_markup:      %r{^\s*?[/*]{1,2}},
+        html_comment:        /<!--.*?-->/m,
+        url:                 %r{#{START_REGEX}https?://[^ ]+\n},
+        bullet:              /\n\n\s*(?:[*-]|\(?[\da-z]{1,2}[).])\s+/i,
+        developed_by:        /#{START_REGEX}developed by:.*?\n\n/im,
+        cc_dedication:       /The\s+text\s+of\s+the\s+Creative\s+Commons.*?Public\s+Domain\s+Dedication\./im,
+        cc_wiki:             /wiki\.creativecommons\.org/i,
+        cc_legal_code:       /^\s*Creative Commons Legal Code\s*$/i,
+        cc0_info:            /For more information, please see\s*\S+zero\S+/im,
+        cc0_disclaimer:      /CREATIVE COMMONS CORPORATION.*?\n\n/im,
+        unlicense_info:      /For more information, please.*\S+unlicense\S+/im,
+        mit_optional:        /\(including the next paragraph\)/i
+      }.freeze
+
+      NORMALIZATIONS = {
+        lists:      { from: /^\s*(?:\d\.|[*-])(?: [*_]{0,2}\(?[\da-z]\)[*_]{0,2})?\s+([^\n])/, to: '- \1' },
+        https:      { from: /http:/, to: 'https:' },
+        ampersands: { from: '&', to: 'and' },
+        dashes:     { from: /(?<!^)([—–-]+)(?!$)/, to: '-' },
+        quote:      { from: /[`'"‘“’”]/, to: "'" },
+        hyphenated: { from: /(\w+)-\s*\n\s*(\w+)/, to: '\\1-\\2' }
+      }.freeze
+
+      # Legally equivalent words that schould be ignored for comparison
+      # See https://spdx.org/spdx-license-list/matching-guidelines
+      VARIETAL_WORDS = {
+        'acknowledgment'  => 'acknowledgement',
+        'analogue'        => 'analog',
+        'analyse'         => 'analyze',
+        'artefact'        => 'artifact',
+        'authorisation'   => 'authorization',
+        'authorised'      => 'authorized',
+        'calibre'         => 'caliber',
+        'cancelled'       => 'canceled',
+        'capitalisations' => 'capitalizations',
+        'catalogue'       => 'catalog',
+        'categorise'      => 'categorize',
+        'centre'          => 'center',
+        'emphasised'      => 'emphasized',
+        'favour'          => 'favor',
+        'favourite'       => 'favorite',
+        'fulfil'          => 'fulfill',
+        'fulfilment'      => 'fulfillment',
+        'initialise'      => 'initialize',
+        'judgment'        => 'judgement',
+        'labelling'       => 'labeling',
+        'labour'          => 'labor',
+        'licence'         => 'license',
+        'maximise'        => 'maximize',
+        'modelled'        => 'modeled',
+        'modelling'       => 'modeling',
+        'offence'         => 'offense',
+        'optimise'        => 'optimize',
+        'organisation'    => 'organization',
+        'organise'        => 'organize',
+        'practise'        => 'practice',
+        'programme'       => 'program',
+        'realise'         => 'realize',
+        'recognise'       => 'recognize',
+        'signalling'      => 'signaling',
+        'sub-license'     => 'sublicense',
+        'sub license'     => 'sublicense',
+        'utilisation'     => 'utilization',
+        'whilst'          => 'while',
+        'wilful'          => 'wilfull',
+        'non-commercial'  => 'noncommercial',
+        'per cent'        => 'percent',
+        'copyright owner' => 'copyright holder'
+      }.freeze
+
+      STRIP_METHODS = %i[
+        bom
+        cc_optional
+        cc0_optional
+        unlicense_optional
+        borders
+        title
+        version
+        url
+        copyright
+        title
+        block_markup
+        developed_by
+        end_of_terms
+        whitespace
+        mit_optional
+      ].freeze
+    end
+  end
+end

data/lib/licensee/content_helper/normalization_methods.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Licensee
+  module ContentHelper
+    # Mixin providing content normalization and stripping routines.
+    module NormalizationMethods
+      # Content with the title and version removed
+      # The first time should normally be the attribution line
+      # Used to dry up `content_normalized` but we need the case sensitive
+      # content with attribution first to detect attribuion in LicenseFile
+      def content_without_title_and_version
+        @content_without_title_and_version ||= begin
+          @_content = nil
+          ops = %i[html html_comment hrs comments markdown_headings link_markup title version]
+          ops.each { |op| strip(op) }
+          _content
+        end
+      end
+
+      def content_normalized(wrap: nil)
+        @content_normalized ||= normalize_content
+        wrap ? Licensee::ContentHelper.wrap(@content_normalized, wrap) : @content_normalized
+      end
+
+      def normalize_content
+        @_content = content_without_title_and_version.downcase
+        (ContentHelper::NORMALIZATIONS.keys + %i[spelling span_markup bullets]).each { |op| normalize(op) }
+        ContentHelper::STRIP_METHODS.each { |op| strip(op) }
+        _content
+      end
+
+      private
+
+      def strip(regex_or_sym)
+        return unless _content
+
+        if regex_or_sym.is_a?(Symbol)
+          meth = "strip_#{regex_or_sym}"
+          return send(meth) if respond_to?(meth, true)
+
+          unless ContentHelper::REGEXES[regex_or_sym]
+            raise ArgumentError, "#{regex_or_sym} is an invalid regex reference"
+          end
+
+          regex_or_sym = ContentHelper::REGEXES[regex_or_sym]
+        end
+
+        @_content = _content.gsub(regex_or_sym, ' ').squeeze(' ').strip
+      end
+
+      def strip_title
+        strip(ContentHelper.title_regex) while _content =~ ContentHelper.title_regex
+      end
+
+      def strip_borders
+        normalize(ContentHelper::REGEXES[:border_markup], '\\1')
+      end
+
+      def strip_comments
+        lines = _content.split("\n")
+        return if lines.one?
+        return unless lines.all? { |line| line =~ ContentHelper::REGEXES[:comment_markup] }
+
+        strip(:comment_markup)
+      end
+
+      def strip_copyright
+        regex = Regexp.union(Matchers::Copyright::REGEX, ContentHelper::REGEXES[:all_rights_reserved])
+        strip(regex) while _content =~ regex
+      end
+
+      def strip_cc0_optional
+        return unless _content.include? 'associating cc0'
+
+        strip(ContentHelper::REGEXES[:cc_legal_code])
+        strip(ContentHelper::REGEXES[:cc0_info])
+        strip(ContentHelper::REGEXES[:cc0_disclaimer])
+      end
+
+      def strip_cc_optional
+        return unless _content.include? 'creative commons'
+
+        strip(ContentHelper::REGEXES[:cc_dedication])
+        strip(ContentHelper::REGEXES[:cc_wiki])
+      end
+
+      def strip_unlicense_optional
+        return unless _content.include? 'unlicense'
+
+        strip(ContentHelper::REGEXES[:unlicense_info])
+      end
+
+      def strip_end_of_terms
+        body, _partition, _instructions = _content.partition(ContentHelper::END_OF_TERMS_REGEX)
+        @_content = body
+      end
+
+      def normalize_span_markup
+        normalize(ContentHelper::REGEXES[:span_markup], '\\1')
+      end
+
+      def strip_link_markup
+        normalize(ContentHelper::REGEXES[:link_markup], '\\1')
+      end
+
+      def strip_html
+        return unless respond_to?(:filename) && filename
+        return unless /\.html?/i.match?(File.extname(filename))
+
+        require 'reverse_markdown'
+        @_content = ReverseMarkdown.convert(_content, unknown_tags: :bypass)
+      end
+
+      def normalize(from_or_key, to = nil)
+        operation = { from: from_or_key, to: to } if to
+        operation ||= ContentHelper::NORMALIZATIONS[from_or_key]
+
+        if operation
+          @_content = _content.gsub operation[:from], operation[:to]
+        elsif respond_to?(:"normalize_#{from_or_key}", true)
+          send(:"normalize_#{from_or_key}")
+        else
+          raise ArgumentError, "#{from_or_key} is an invalid normalization"
+        end
+      end
+
+      def normalize_spelling
+        normalize(/\b#{Regexp.union(ContentHelper::VARIETAL_WORDS.keys)}\b/, ContentHelper::VARIETAL_WORDS)
+      end
+
+      def normalize_bullets
+        normalize(ContentHelper::REGEXES[:bullet], "\n\n- ")
+        normalize(/\)\s+\(/, ')(')
+      end
+    end
+  end
+end