licensed 3.7.5 → 3.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c460f79af039dfcd0059dafa8c1ce9ea096f5fc869c05ed374a61e9d1504520
-  data.tar.gz: 98729cb3117d5642fe1bb6b81a8b5f9781578751407f0425ea2ed0c5842ae0ac
+  metadata.gz: f4df54260766353e4cd56b9ae56ded611ed4d6a312469d43e18ab47b6b9cabde
+  data.tar.gz: 8f04c9e9d11bcaf7f47698a174ee1269346e798eaa176e28ac3282de482ef237
 SHA512:
-  metadata.gz: 3f5bd011838dc17f8bb74da74b0799da181419718bf96f55ac097af2a077546bba007a1efa2ec8613bd03f345c1f82bf5b97fcad64ebb140ef06a99210ee45db
-  data.tar.gz: 79b667beb6bde2a62ec1c53c706a5379f761bc9189f2fd6caf4da10e7e2ca51216f44816fc06721c023f81446c7f1a480ca482cc6b77f53af92dcb1943a535b5
+  metadata.gz: 0006a278b5b2a7af75ad7634fe11f418f310e7c7506e1ae3cc68bdfca873cdbf74b9bd359d2a9b917c8c79df3a91b589c0c7e8f0b6ad2c03349a81e3bfbc91cd
+  data.tar.gz: 0c6275c87fe724a747f0432395b568341c495453f0072aa78e0f1ee48e075ebfbd94de5ef85d3eb46adf7dfb16e203c725a90929be6dfe801bf53a7b55c783e8

data/CHANGELOG.md

@@ -6,6 +6,18 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+## 3.9.0
+
+### Added
+
+- `NOTICE` files can now be generated without cached files in a repository (https://github.com/github/licensed/pull/572)
+
+## 3.8.0
+
+### Added
+
+- Licensing compliance status checks can now be used without cached files in a repository (https://github.com/github/licensed/pull/560)
+
 ## 3.7.5
 
 ### Fixed
@@ -643,4 +655,4 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 Initial release :tada:
 
-[Unreleased]: https://github.com/github/licensed/compare/3.7.5...HEAD
+[Unreleased]: https://github.com/github/licensed/compare/3.9.0...HEAD

data/docs/commands/notices.md

@@ -2,7 +2,7 @@
 
 Outputs license and notice text for all dependencies in each app into a `NOTICE` file in the app's `cache_path`.  If an app uses a shared cache path, the file name will contain the app name as well, e.g. `NOTICE.my_app`.
 
-`NOTICE` file contents are retrieved from cached records, with the assumption that cached records have already been reviewed in a compliance workflow.
+`NOTICE` file contents are retrieved from cached records when the `--computed`/`-l` option is not set, with the assumption that cached records have already been reviewed in a compliance workflow.  When the `--computed`/`-l` option is set and a dependency's license is not found, that dependency's license text will be empty in the `NOTICE` file.
 
 ## Options
 
@@ -10,3 +10,5 @@ Outputs license and notice text for all dependencies in each app into a `NOTICE`
    - default value: `./.licensed.yml`
 - `--sources`/`-s`: runtime filter on which dependency sources are run.  Sources must also be enabled in the licensed configuration file.
    - default value: not set, all configured sources
+- `--computed`/`-l`: use live computed when generating a `NOTICE` file
+   - default value: not set, `NOTICE` file generated from cached records

data/docs/commands/status.md

@@ -1,17 +1,36 @@
 # `licensed status`
 
-The status command finds all dependencies and checks whether each dependency has a valid cached record.
+The status command finds all dependencies and checks whether each dependency has a valid record.  There are two methods for checking dependencies' statuses
+
+## Checking status with metadata loaded from cached files
+
+This is the default method for checking the status for dependencies and the recommended method for using licensed.  Checking status from cached metadata files will occur when the `--data-source` CLI flag is either unset, or set to `--data-source=files`.
+
+When using `licensed status` in this scenario, licensed will only compile a minimal amount of dependency metadata like the dependency name and version to match against cached files.  Dependency license and notice texts are loaded from cached files that are created from the [licensed cache](./cache.md) command.
 
 A dependency will fail the status checks if:
 
 1. No cached record is found
 2. The cached record's version is different than the current dependency's version
 3. The cached record's `licenses` data is empty
-4. The cached record's `license` metadata doesn't match an `allowed` license from the dependency's application configuration.
+4. The cached record's `license` metadata doesn't match an `allowed` license from the dependency's application configuration and the dependency has not been marked `reviewed` or `ignored`
    - If `license: other` is specified and all of the `licenses` entries match an `allowed` license a failure will not be logged
 5. The cached record is flagged for re-review.
    - This occurs when the record's license text has changed since the record was reviewed.
 
+## Checking status with computed metadata
+
+Checking status with computed metadata and the licensed configuration will occur when the `--data-source` CLI flag is set to `--data-source=configuration`.
+
+When using `licensed status` in this scenario, licensed will compile all dependency metadata and license text to use in status checks.  There is no need to run `licensed cache` prior to running `licensed status --data-source=configuration`.
+
+A dependency will fail the status checks if:
+
+1. The record's `licenses` data is empty
+2. The record's `license` metadata doesn't match an `allowed` license from the dependency's application configuration and the dependency has not been marked `reviewed` or `ignored`
+   - If `license: other` is specified and all of the `licenses` entries match an `allowed` license a failure will not be logged
+   - A `reviewed` entry must reference a specific version of the depdency, e.g. `<name>@<version>`.  The version identifier must specify a specific dependency version, ranges are not allowed.
+
 ## Options
 
 - `--config`/`-c`: the path to the licensed configuration file
@@ -20,6 +39,9 @@ A dependency will fail the status checks if:
    - default value: not set, all configured sources
 - `--format`/`-f`: the output format
    - default value: `yaml`
+- `--data-source`/`-d`: where to find the data source of records for status checks
+   - available values: `files`, `configuration`
+   - default value: `files`
 - `--force`: if set, forces all dependency metadata files to be recached
    - default value: not set
 
@@ -63,14 +85,16 @@ If the dependency does not include license text but does specify that it uses a
 
 *Resolution:* Review the changes to the license text and classification, along with other metadata contained in the cached file for the dependency.  If the dependency is still allowable for use in your project, remove the `review_changed_license` key from the cached record file.
 
-### license needs review
+### license needs review / dependency needs review
 
 *Cause:* A dependency is using a license that is not in the configured [allowed list of licenses][allowed], and the dependency has not been marked [ignored] or [reviewed].
 *Resolution:* Review the dependency's usage and specified license with someone familiar with OSS licensing and compliance rules to determine whether the dependency is allowable.  Some common resolutions:
 
-1. The dependency's specified license text differed enough from the standard license text that it was not recognized and classified as `other`.  If, with human review, the license text is recognizable then update the `license: other` value in the cached metadata file to the correct license.
-   - An updated classification will persist through version upgrades until the detected license contents have changed.  The determination is made by [licensee/licensee](https://github.com/licensee/licensee), the library which this tool uses to detect and classify license contents.
+1. The dependency's specified license text differed enough from the standard license text that it was not recognized and classified as `other`.  
+   - This resolution only applies when checking dependency status [with cached metadata files](./#checking-status-with-metadata-loaded-from-cached-files).
+   - If the cached license text is recognizable with human review then update the `license: other` value in the cached metadata file to the correct license. An updated classification will persist through version upgrades until the detected license contents have changed.  The determination is made by [licensee/licensee](https://github.com/licensee/licensee), the library which this tool uses to detect and classify license contents.
 1. The dependency might need to be marked as [ignored] or [reviewed] if either of those scenarios are applicable.
+   - When checking status [with computed metadata](./#checking-status-with-computed-metadata), a reviewed entry must include both the dependency's name and the version that it was reviewed at e.g. `licensed@3.8.0`
 1. If the used license should be allowable without review (if your entity has a legal team, they may want to review this assessment), ensure the license SPDX is set as [allowed] in the licensed configuration file.
 
 [allowed]: ../configuration/allowed_licenses.md

data/lib/licensed/cli.rb

@@ -26,8 +26,11 @@ module Licensed
       desc: "Individual source(s) to evaluate.  Must also be enabled via configuration."
     method_option :format, aliases: "-f", enum: ["yaml", "json"],
       desc: "Output format"
+    method_option :data_source, aliases: "-d",
+      enum: ["files", "configuration"], default: "files",
+      desc: "Whether to check compliance status from cached records or the configuration file"
     def status
-      run Licensed::Commands::Status.new(config: config), sources: options[:sources], reporter: options[:format]
+      run Licensed::Commands::Status.new(config: config), sources: options[:sources], reporter: options[:format], data_source: options[:data_source]
     end
 
     desc "list", "List dependencies"
@@ -43,13 +46,15 @@ module Licensed
       run Licensed::Commands::List.new(config: config), sources: options[:sources], reporter: options[:format], licenses: options[:licenses]
     end
 
-    desc "notices", "Generate a NOTICE file from cached records"
+    desc "notices", "Generate a NOTICE file with dependency data"
     method_option :config, aliases: "-c", type: :string,
       desc: "Path to licensed configuration file"
     method_option :sources, aliases: "-s", type: :array,
       desc: "Individual source(s) to evaluate.  Must also be enabled via configuration."
+    method_option :computed, aliases: "-l", type: :boolean,
+      desc: "Whether to generate a NOTICE file using computed data or cached records"
     def notices
-      run Licensed::Commands::Notices.new(config: config), sources: options[:sources]
+      run Licensed::Commands::Notices.new(config: config), sources: options[:sources], computed: options[:computed]
     end
 
     map "-v" => :version

data/lib/licensed/commands/notices.rb

@@ -13,7 +13,7 @@ module Licensed
 
       protected
 
-      # Load stored dependency record data to add to the notices report.
+      # Load a dependency record data and add it to the notices report.
       #
       # app - The application configuration for the dependency
       # source - The dependency source enumerator for the dependency
@@ -22,13 +22,36 @@ module Licensed
       #
       # Returns true.
       def evaluate_dependency(app, source, dependency, report)
+        report["record"] =
+          if load_dependency_record_from_files
+            load_cached_dependency_record(app, source, dependency, report)
+          else
+            dependency.record
+          end
+
+        true
+      end
+
+      # Loads a dependency record from a cached file.
+      #
+      # app - The application configuration for the dependency
+      # source - The dependency source enumerator for the dependency
+      # dependency - An application dependency
+      # report - A report hash for the command to provide extra data for the report output.
+      #
+      # Returns a dependency record or nil if one doesn't exist
+      def load_cached_dependency_record(app, source, dependency, report)
         filename = app.cache_path.join(source.class.type, "#{dependency.name}.#{DependencyRecord::EXTENSION}")
-        report["cached_record"] = Licensed::DependencyRecord.read(filename)
-        if !report["cached_record"]
+        record = Licensed::DependencyRecord.read(filename)
+        if !record
           report.warnings << "expected cached record not found at #{filename}"
         end
 
-        true
+        record
+      end
+
+      def load_dependency_record_from_files
+        !options.fetch(:computed, false)
       end
     end
   end

data/lib/licensed/commands/status.rb

@@ -29,33 +29,39 @@ module Licensed
         end
       end
 
-      # Verifies that a cached record exists, is up to date and
-      # has license data that complies with the licensed configuration.
+      # Evaluates a dependency for any compliance errors.
+      # Checks a dependency against either a cached metadata record or
+      # reviewed entries in the configuration file.
       #
       # app - The application configuration for the dependency
       # source - The dependency source enumerator for the dependency
       # dependency - An application dependency
       # report - A report hash for the command to provide extra data for the report output.
       #
-      # Returns whether the dependency has a cached record that is compliant
+      # Returns whether the dependency is compliant
       # with the licensed configuration.
       def evaluate_dependency(app, source, dependency, report)
-        filename = app.cache_path.join(source.class.type, "#{dependency.name}.#{DependencyRecord::EXTENSION}")
-        report["filename"] = filename
         report["version"] = dependency.version
 
-        cached_record = cached_record(filename)
-        if cached_record.nil?
+        if data_source == "configuration"
+          record = dependency.record
+        else
+          filename = app.cache_path.join(source.class.type, "#{dependency.name}.#{DependencyRecord::EXTENSION}")
+          report["filename"] = filename
+          record = cached_record(filename)
+        end
+
+        if record.nil?
           report["license"] = nil
           report.errors << "cached dependency record not found"
         else
-          report["license"] = cached_record["license"]
-          report.errors << "cached dependency record out of date" if cached_record["version"] != dependency.version
-          report.errors << "missing license text" if cached_record.licenses.empty?
-          if cached_record["review_changed_license"]
+          report["license"] = record["license"]
+          report.errors << "dependency record out of date" if record["version"] != dependency.version
+          report.errors << "missing license text" if record.licenses.empty?
+          if record["review_changed_license"]
             report.errors << "license text has changed and needs re-review. if the new text is ok, remove the `review_changed_license` flag from the cached record"
-          elsif license_needs_review?(app, cached_record)
-            report.errors << "license needs review: #{cached_record["license"]}"
+          elsif license_needs_review?(app, record)
+            report.errors << needs_review_error_message(app, record)
           end
         end
 
@@ -64,19 +70,20 @@ module Licensed
 
       # Returns true if a cached record needs further review based on the
       # record's license(s) and the app's configuration
-      def license_needs_review?(app, cached_record)
+      def license_needs_review?(app, record)
         # review is not needed if the record is set as reviewed
-        return false if app.reviewed?(cached_record)
+        return false if app.reviewed?(record, match_version: data_source == "configuration")
+
         # review is not needed if the top level license is allowed
-        return false if app.allowed?(cached_record["license"])
+        return false if app.allowed?(record["license"])
 
         # the remaining checks are meant to allow records marked as "other"
         # that have multiple licenses, all of which are allowed
 
         # review is needed for non-"other" licenses
-        return true unless cached_record["license"] == "other"
+        return true unless record["license"] == "other"
 
-        licenses = cached_record.licenses.map { |license| license_from_text(license.text) }
+        licenses = record.licenses.map { |license| license_from_text(license.text) }
 
         # review is needed when there is only one license notice
         # this is a performance optimization for the single license case
@@ -86,6 +93,29 @@ module Licensed
         licenses.any? { |license| !app.allowed?(license) }
       end
 
+      def needs_review_error_message(app, record)
+        return "license needs review: #{record["license"]}" if data_source == "files"
+
+        error = "dependency needs review"
+
+        # look for an unversioned reviewed list match
+        if app.reviewed?(record, match_version: false)
+          error += ", unversioned 'reviewed' match found: #{record["name"]}"
+        end
+
+        # look for other version matches in reviewed list
+        possible_matches = app.reviewed_versions(record)
+        if possible_matches.any?
+          error += ", possible 'reviewed' matches found at other versions: #{possible_matches.join(", ")}"
+        end
+
+        error
+      end
+
+      def data_source
+        options[:data_source] || "files"
+      end
+
       def cached_record(filename)
         return nil unless File.exist?(filename)
         DependencyRecord.read(filename)

data/lib/licensed/configuration.rb

@@ -73,17 +73,18 @@ module Licensed
     end
 
     # Is the given dependency reviewed?
-    def reviewed?(dependency)
-      Array(self["reviewed"][dependency["type"]]).any? do |pattern|
-        File.fnmatch?(pattern, dependency["name"], File::FNM_PATHNAME | File::FNM_CASEFOLD)
-      end
+    def reviewed?(dependency, match_version: false)
+      any_list_pattern_matched? self["reviewed"][dependency["type"]], dependency, match_version: match_version
+    end
+
+    # Find all reviewed dependencies that match the provided dependency's name
+    def reviewed_versions(dependency)
+      similar_list_patterns self["reviewed"][dependency["type"]], dependency
     end
 
     # Is the given dependency ignored?
     def ignored?(dependency)
-      Array(self["ignored"][dependency["type"]]).any? do |pattern|
-        File.fnmatch?(pattern, dependency["name"], File::FNM_PATHNAME | File::FNM_CASEFOLD)
-      end
+      any_list_pattern_matched? self["ignored"][dependency["type"]], dependency
     end
 
     # Is the license of the dependency allowed?
@@ -97,8 +98,10 @@ module Licensed
     end
 
     # Set a dependency as reviewed
-    def review(dependency)
-      (self["reviewed"][dependency["type"]] ||= []) << dependency["name"]
+    def review(dependency, at_version: false)
+      id = dependency["name"]
+      id += "@#{dependency["version"]}" if at_version && dependency["version"]
+      (self["reviewed"][dependency["type"]] ||= []) << id
     end
 
     # Set a license as explicitly allowed
@@ -108,6 +111,27 @@ module Licensed
 
     private
 
+    def any_list_pattern_matched?(list, dependency, match_version: false)
+      Array(list).any? do |pattern|
+        if match_version
+          at_version = "@#{dependency["version"]}"
+          pattern, pattern_version = pattern.rpartition(at_version).values_at(0, 1)
+          next false if pattern == "" || pattern_version == ""
+        end
+
+        File.fnmatch?(pattern, dependency["name"], File::FNM_PATHNAME | File::FNM_CASEFOLD)
+      end
+    end
+
+    def similar_list_patterns(list, dependency)
+      Array(list).select do |pattern|
+        pattern, version = pattern.rpartition("@").values_at(0, 2)
+        next if pattern == "" || version == ""
+
+        File.fnmatch?(pattern, dependency["name"], File::FNM_PATHNAME | File::FNM_CASEFOLD)
+      end
+    end
+
     # Returns the cache path for the application based on:
     # 1. An explicitly set cache path for the application, if set
     # 2. An inherited shared cache path

data/lib/licensed/reporters/notices_reporter.rb

@@ -54,11 +54,11 @@ module Licensed
       def notices(report)
         return unless report.target.is_a?(Licensed::Dependency)
 
-        cached_record = report["cached_record"]
-        return unless cached_record
+        record = report["record"]
+        return unless record
 
-        texts = cached_record.licenses.map(&:text)
-        cached_record.notices.each do |notice|
+        texts = record.licenses.map(&:text)
+        record.notices.each do |notice|
           case notice
           when Hash
             texts << notice["text"]
@@ -70,7 +70,7 @@ module Licensed
         end
 
         <<~NOTICE
-          #{cached_record["name"]}@#{cached_record["version"]}
+          #{record["name"]}@#{record["version"]}
 
           #{texts.map(&:strip).reject(&:empty?).compact.join(TEXT_SEPARATOR)}
         NOTICE

data/lib/licensed/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 module Licensed
-  VERSION = "3.7.5".freeze
+  VERSION = "3.9.0".freeze
 
   def self.previous_major_versions
     major_version = Gem::Version.new(Licensed::VERSION).segments.first

data/licensed.gemspec

@@ -35,7 +35,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "rake", ">= 12.3.3"
   spec.add_development_dependency "minitest", "~> 5.8"
-  spec.add_development_dependency "mocha", "~> 1.0"
+  spec.add_development_dependency "mocha", "~> 2.0"
   spec.add_development_dependency "rubocop-github", "~> 0.6"
   spec.add_development_dependency "byebug", "~> 11.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: licensed
 version: !ruby/object:Gem::Version
-  version: 3.7.5
+  version: 3.9.0
 platform: ruby
 authors:
 - GitHub
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-10-20 00:00:00.000000000 Z
+date: 2022-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: licensee
@@ -188,14 +188,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rubocop-github
   requirement: !ruby/object:Gem::Requirement