licensed 2.9.1 → 2.12.0

data/lib/licensed/commands/status.rb

@@ -15,6 +15,25 @@ module Licensed
 
       protected
 
+      # Run the command for all enumerated dependencies found in a dependency source,
+      # recording results in a report.
+      # Enumerating dependencies in the source is skipped if a :sources option
+      # is provided and the evaluated `source.class.type` is not in the :sources values
+      #
+      # app - The application configuration for the source
+      # source - A dependency source enumerator
+      #
+      # Returns whether the command succeeded for the dependency source enumerator
+      def run_source(app, source)
+        super do |report|
+          next if Array(options[:sources]).empty?
+          next if options[:sources].include?(source.class.type)
+
+          report.warnings << "skipped source"
+          :skip
+        end
+      end
+
       # Verifies that a cached record exists, is up to date and
       # has license data that complies with the licensed configuration.
       #

data/lib/licensed/configuration.rb

@@ -108,7 +108,12 @@ module Licensed
     def detect_cache_path(options, inherited_options)
       return options["cache_path"] unless options["cache_path"].to_s.empty?
 
-      cache_path = inherited_options["cache_path"] || DEFAULT_CACHE_PATH
+      # if cache_path and shared_cache are both set in inherited_options,
+      # don't append the app name to the cache path
+      cache_path = inherited_options["cache_path"]
+      return cache_path if cache_path && inherited_options["shared_cache"] == true
+
+      cache_path ||= DEFAULT_CACHE_PATH
       File.join(cache_path, self["name"])
     end
 
@@ -167,7 +172,12 @@ module Licensed
         # will handle configurations that don't have these explicitly set
         dir_name = File.basename(path)
         config["name"] = "#{config["name"]}-#{dir_name}" if config["name"]
-        config["cache_path"] = File.join(config["cache_path"], dir_name) if config["cache_path"]
+
+        # if a cache_path is set and is not marked as shared, append the app name
+        # to the end of the cache path to make a unique cache path for the app
+        if config["cache_path"] && config["shared_cache"] != true
+          config["cache_path"] = File.join(config["cache_path"], dir_name)
+        end
 
         config
       end

data/lib/licensed/dependency.rb

@@ -74,7 +74,7 @@ module Licensed
     def license_contents
       files = matched_files.reject { |f| f == package_file }
                            .group_by(&:content)
-                           .map { |content, files| { "sources" => license_content_sources(files), "text" => content } }
+                           .map { |content, sources| { "sources" => license_content_sources(sources), "text" => content } }
 
       files << generated_license_contents if files.empty?
       files.compact

data/lib/licensed/reporters.rb

@@ -7,5 +7,6 @@ module Licensed
     require "licensed/reporters/list_reporter"
     require "licensed/reporters/json_reporter"
     require "licensed/reporters/yaml_reporter"
+    require "licensed/reporters/notices_reporter"
   end
 end

data/lib/licensed/reporters/cache_reporter.rb

@@ -27,32 +27,32 @@ module Licensed
           shell.info "  #{source.class.type}"
           result = yield report
 
-          warning_reports = report.all_reports.select { |report| report.warnings.any? }.to_a
+          warning_reports = report.all_reports.select { |r| r.warnings.any? }.to_a
           if warning_reports.any?
             shell.newline
             shell.warn "  * Warnings:"
-            warning_reports.each do |report|
-              display_metadata = report.map { |k, v| "#{k}: #{v}" }.join(", ")
+            warning_reports.each do |r|
+              display_metadata = r.map { |k, v| "#{k}: #{v}" }.join(", ")
 
-              shell.warn "    * #{report.name}"
+              shell.warn "    * #{r.name}"
               shell.warn "    #{display_metadata}" unless display_metadata.empty?
-              report.warnings.each do |warning|
+              r.warnings.each do |warning|
                 shell.warn "      - #{warning}"
               end
               shell.newline
             end
           end
 
-          errored_reports = report.all_reports.select { |report| report.errors.any? }.to_a
+          errored_reports = report.all_reports.select { |r| r.errors.any? }.to_a
           if errored_reports.any?
             shell.newline
             shell.error "  * Errors:"
-            errored_reports.each do |report|
-              display_metadata = report.map { |k, v| "#{k}: #{v}" }.join(", ")
+            errored_reports.each do |r|
+              display_metadata = r.map { |k, v| "#{k}: #{v}" }.join(", ")
 
-              shell.error "    * #{report.name}"
+              shell.error "    * #{r.name}"
               shell.error "    #{display_metadata}" unless display_metadata.empty?
-              report.errors.each do |error|
+              r.errors.each do |error|
                 shell.error "      - #{error}"
               end
               shell.newline

data/lib/licensed/reporters/list_reporter.rb

@@ -28,16 +28,32 @@ module Licensed
           shell.info "  #{source.class.type}"
           result = yield report
 
-          errored_reports = report.all_reports.select { |report| report.errors.any? }.to_a
+          warning_reports = report.all_reports.select { |r| r.warnings.any? }.to_a
+          if warning_reports.any?
+            shell.newline
+            shell.warn "  * Warnings:"
+            warning_reports.each do |r|
+              display_metadata = r.map { |k, v| "#{k}: #{v}" }.join(", ")
+
+              shell.warn "    * #{r.name}"
+              shell.warn "    #{display_metadata}" unless display_metadata.empty?
+              r.warnings.each do |warning|
+                shell.warn "      - #{warning}"
+              end
+              shell.newline
+            end
+          end
+
+          errored_reports = report.all_reports.select { |r| r.errors.any? }.to_a
           if errored_reports.any?
             shell.newline
             shell.error "  * Errors:"
-            errored_reports.each do |report|
-              display_metadata = report.map { |k, v| "#{k}: #{v}" }.join(", ")
+            errored_reports.each do |r|
+              display_metadata = r.map { |k, v| "#{k}: #{v}" }.join(", ")
 
-              shell.error "    * #{report.name}"
+              shell.error "    * #{r.name}"
               shell.error "    #{display_metadata}" unless display_metadata.empty?
-              report.errors.each do |error|
+              r.errors.each do |error|
                 shell.error "      - #{error}"
               end
               shell.newline

data/lib/licensed/reporters/notices_reporter.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Licensed
+  module Reporters
+    class NoticesReporter < Reporter
+      TEXT_SEPARATOR = "\n\n#{("-" * 5)}\n\n".freeze
+      LICENSE_SEPARATOR = "\n#{("*" * 5)}\n".freeze
+
+      # Reports on an application configuration in a notices command run
+      #
+      # app - An application configuration
+      #
+      # Returns the result of the yielded method
+      # Note - must be called from inside the `report_run` scope
+      def report_app(app)
+        super do |report|
+          filename = app["shared_cache"] ? "NOTICE.#{app["name"]}" : "NOTICE"
+          path = app.cache_path.join(filename)
+          shell.info "Writing notices for #{app["name"]} to #{path}"
+
+          result = yield report
+
+          File.open(path, "w") do |file|
+            file << "THIRD PARTY NOTICES\n"
+            file << LICENSE_SEPARATOR
+            file << report.all_reports
+                          .map { |r| notices(r) }
+                          .compact
+                          .join(LICENSE_SEPARATOR)
+          end
+
+          result
+        end
+      end
+
+
+      # Reports on a dependency source enumerator in a notices command run.
+      # Shows warnings encountered during the run.
+      #
+      # source - A dependency source enumerator
+      #
+      # Returns the result of the yielded method
+      # Note - must be called from inside the `report_run` scope
+      def report_source(source)
+        super do |report|
+          result = yield report
+
+          report.warnings.each do |warning|
+            shell.warn "* #{report.name}: #{warning}"
+          end
+
+          result
+        end
+      end
+
+      # Reports on a dependency in a notices command run.
+      #
+      # dependency - An application dependency
+      #
+      # Returns the result of the yielded method
+      # Note - must be called from inside the `report_run` scope
+      def report_dependency(dependency)
+        super do |report|
+          result = yield report
+          report.warnings.each do |warning|
+            shell.warn "* #{report.name}: #{warning}"
+          end
+          result
+        end
+      end
+
+      # Returns notices information for a dependency report
+      def notices(report)
+        return unless report.target.is_a?(Licensed::Dependency)
+
+        cached_record = report["cached_record"]
+        return unless cached_record
+
+        texts = cached_record.licenses.map(&:text)
+        cached_record.notices.each do |notice|
+          case notice
+          when Hash
+            texts << notice["text"]
+          when String
+            texts << notice
+          else
+            shell.warn "* unable to parse notices for #{report.target.name}"
+          end
+        end
+
+        <<~NOTICE
+          #{cached_record["name"]}@#{cached_record["version"]}
+
+          #{texts.map(&:strip).reject(&:empty?).compact.join(TEXT_SEPARATOR)}
+        NOTICE
+      end
+    end
+  end
+end

data/lib/licensed/reporters/reporter.rb

@@ -47,6 +47,9 @@ module Licensed
 
       def initialize(shell = Licensed::UI::Shell.new)
         @shell = shell
+        @run_report = nil
+        @app_report = nil
+        @source_report = nil
       end
 
       # Generate a report for a licensed command execution

data/lib/licensed/reporters/status_reporter.rb

@@ -15,20 +15,37 @@ module Licensed
           result = yield report
 
           all_reports = report.all_reports
-          errored_reports = all_reports.select { |report| report.errors.any? }.to_a
 
-          dependency_count = all_reports.select { |report| report.target.is_a?(Licensed::Dependency) }.size
-          error_count = errored_reports.sum { |report| report.errors.size }
+          warning_reports = all_reports.select { |r| r.warnings.any? }.to_a
+          if warning_reports.any?
+            shell.newline
+            shell.warn "Warnings:"
+            warning_reports.each do |r|
+              display_metadata = r.map { |k, v| "#{k}: #{v}" }.join(", ")
+
+              shell.warn "* #{r.name}"
+              shell.warn "  #{display_metadata}" unless display_metadata.empty?
+              r.warnings.each do |warning|
+                shell.warn "    - #{warning}"
+              end
+              shell.newline
+            end
+          end
+
+          errored_reports = all_reports.select { |r| r.errors.any? }.to_a
+
+          dependency_count = all_reports.select { |r| r.target.is_a?(Licensed::Dependency) }.size
+          error_count = errored_reports.sum { |r| r.errors.size }
 
           if error_count > 0
             shell.newline
             shell.error "Errors:"
-            errored_reports.each do |report|
-              display_metadata = report.map { |k, v| "#{k}: #{v}" }.join(", ")
+            errored_reports.each do |r|
+              display_metadata = r.map { |k, v| "#{k}: #{v}" }.join(", ")
 
-              shell.error "* #{report.name}"
+              shell.error "* #{r.name}"
               shell.error "  #{display_metadata}" unless display_metadata.empty?
-              report.errors.each do |error|
+              r.errors.each do |error|
                 shell.error "    - #{error}"
               end
               shell.newline

data/lib/licensed/sources.rb

@@ -11,6 +11,7 @@ module Licensed
     require "licensed/sources/go"
     require "licensed/sources/manifest"
     require "licensed/sources/npm"
+    require "licensed/sources/nuget"
     require "licensed/sources/pip"
     require "licensed/sources/pipenv"
     require "licensed/sources/gradle"

data/lib/licensed/sources/cabal.rb

@@ -111,11 +111,11 @@ module Licensed
         info = package_info_command(id).strip
         return missing_package(id) if info.empty?
 
-        info.lines.each_with_object({}) do |line, info|
+        info.lines.each_with_object({}) do |line, hsh|
           key, value = line.split(":", 2).map(&:strip)
           next unless key && value
 
-          info[key] = value
+          hsh[key] = value
         end
       end
 

data/lib/licensed/sources/go.rb

@@ -120,12 +120,12 @@ module Licensed
         return go_mod["Version"] if go_mod
 
         package_directory = package["Dir"]
-        return unless package_directory
+        return unless package_directory && File.exist?(package_directory)
 
         # find most recent git SHA for a package, or nil if SHA is
         # not available
         Dir.chdir package_directory do
-          contents_version *contents_version_arguments
+          contents_version(*contents_version_arguments)
         end
       end
 

data/lib/licensed/sources/npm.rb

@@ -48,6 +48,7 @@ module Licensed
       # package name to it's metadata
       def recursive_dependencies(dependencies, result = {})
         dependencies.each do |name, dependency|
+          next if dependency["peerMissing"]
           next if yarn_lock_present && dependency["missing"]
           (result[name] ||= []) << dependency
           recursive_dependencies(dependency["dependencies"] || {}, result)

data/lib/licensed/sources/nuget.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+require "json"
+require "reverse_markdown"
+
+module Licensed
+  module Sources
+    # Only supports ProjectReference (project.assets.json) style restore used in .NET Core.
+    # Does not currently support packages.config style restore.
+    class NuGet < Source
+      def self.type
+        "nuget"
+      end
+
+      class NuGetDependency < Licensed::Dependency
+        LICENSE_FILE_REGEX = /<license\s*type\s*=\s*\"\s*file\s*\"\s*>\s*(.*)\s*<\/license>/ix.freeze
+        LICENSE_URL_REGEX = /<licenseUrl>\s*(.*)\s*<\/licenseUrl>/ix.freeze
+        PROJECT_URL_REGEX = /<projectUrl>\s*(.*)\s*<\/projectUrl>/ix.freeze
+        PROJECT_DESC_REGEX = /<description>\s*(.*)\s*<\/description>/ix.freeze
+
+        # Returns the metadata that represents this dependency.  This metadata
+        # is written to YAML in the dependencys cached text file
+        def license_metadata
+          super.tap do |record_metadata|
+            record_metadata["homepage"] = project_url if project_url
+            record_metadata["summary"] = description if description
+          end
+        end
+
+        def nuspec_path
+          name = @metadata["name"]
+          File.join(self.path, "#{name.downcase}.nuspec")
+        end
+
+        def nuspec_contents
+          return @nuspec_contents if defined?(@nuspec_contents)
+          @nuspec_contents = begin
+            return unless nuspec_path && File.exist?(nuspec_path)
+            File.read(nuspec_path)
+          end
+        end
+
+        def project_url
+          return @project_url if defined?(@project_url)
+          @project_url = begin
+            return unless nuspec_contents
+            match = nuspec_contents.match PROJECT_URL_REGEX
+            match[1] if match && match[1]
+          end
+        end
+
+        def description
+          return @description if defined?(@description)
+          @description = begin
+            return unless nuspec_contents
+            match = nuspec_contents.match PROJECT_DESC_REGEX
+            match[1] if match && match[1]
+          end
+        end
+
+        def project_files
+          @nuget_project_files ||= begin
+            files = super().flatten.compact
+
+            # Only include the local file if it's a file licensee didn't already detect
+            nuspec_license_filename = File.basename(nuspec_local_license_file.filename) if nuspec_local_license_file
+            if nuspec_license_filename && files.none? { |file| File.basename(file.filename) == nuspec_license_filename }
+              files.push(nuspec_local_license_file)
+            end
+
+            # Only download licenseUrl if no recognized license was found locally
+            if files.none? { |file| file.license && file.license.key != "other" }
+              files.push(nuspec_remote_license_file)
+            end
+
+            files.compact
+          end
+        end
+
+        # Look for a <license type="file"> element in the nuspec that points to an
+        # on-disk license file (which licensee may not find due to a non-standard filename)
+        def nuspec_local_license_file
+          return @nuspec_local_license_file if defined?(@nuspec_local_license_file)
+          return unless nuspec_contents
+
+          match = nuspec_contents.match LICENSE_FILE_REGEX
+          return unless match && match[1]
+
+          license_path = File.join(File.dirname(nuspec_path), match[1])
+          return unless File.exist?(license_path)
+
+          license_data = File.read(license_path)
+          @nuspec_local_license_file = Licensee::ProjectFiles::LicenseFile.new(license_data, license_path)
+        end
+
+        # Look for a <licenseUrl> element in the nuspec that either is known to contain a license identifier
+        # in the URL, or points to license text on the internet that can be downloaded.
+        def nuspec_remote_license_file
+          return @nuspec_remote_license_file if defined?(@nuspec_remote_license_file)
+          return unless nuspec_contents
+
+          match = nuspec_contents.match LICENSE_URL_REGEX
+          return unless match && match[1]
+
+          # Attempt to fetch the license content
+          license_content = self.class.retrieve_license(match[1])
+          @nuspec_remote_license_file = Licensee::ProjectFiles::LicenseFile.new(license_content, { uri: match[1] }) if license_content
+        end
+
+        class << self
+          def strip_html(html)
+            return unless html
+
+            return html unless html.downcase.include?("<html")
+            ReverseMarkdown.convert(html, unknown_tags: :bypass)
+          end
+
+          def ignored_url?(url)
+            # Many Microsoft packages that now use <license> use this for <licenseUrl>
+            # No need to fetch this page - it just contains NuGet documentation
+            url == "https://aka.ms/deprecateLicenseUrl"
+          end
+
+          def text_content_url(url)
+            # Convert github file URLs to raw URLs
+            return url unless match = url.match(/https?:\/\/(?:www\.)?github.com\/([^\/]+)\/([^\/]+)\/blob\/(.*)/i)
+            "https://github.com/#{match[1]}/#{match[2]}/raw/#{match[3]}"
+          end
+
+          def retrieve_license(url)
+            return unless url
+            return if ignored_url?(url)
+
+            # Transform URLs that are known to return HTML but have a corresponding text-based URL
+            text_url = text_content_url(url)
+
+            raw_content = fetch_content(text_url)
+            strip_html(raw_content)
+          end
+
+          def fetch_content(url, redirect_limit = 5)
+            url = URI.parse(url) if url.instance_of? String
+            return @response_by_url[url] if (@response_by_url ||= {}).key?(url)
+            return if redirect_limit == 0
+
+            begin
+              response = Net::HTTP.get_response(url)
+              case response
+              when Net::HTTPSuccess     then
+                @response_by_url[url] = response.body
+              when Net::HTTPRedirection then
+                redirect_url = URI.parse(response["location"])
+                if redirect_url.relative?
+                  redirect_url = url + redirect_url
+                end
+                # The redirect might be to a URL that requires transformation, i.e. a github file
+                redirect_url = text_content_url(redirect_url.to_s)
+                @response_by_url[url] = fetch_content(redirect_url, redirect_limit - 1)
+              end
+            rescue
+              # Host might no longer exist or some other error, ignore
+            end
+          end
+        end
+      end
+
+      def project_assets_file_path
+        File.join(config.pwd, "project.assets.json")
+      end
+
+      def project_assets_file
+        return @project_assets_file if defined?(@project_assets_file)
+        @project_assets_file = File.read(project_assets_file_path)
+      end
+
+      def enabled?
+        File.exist?(project_assets_file_path)
+      end
+
+      # Inspect project.assets.json files for package references.
+      # Ideally we'd use `dotnet list package` instead, but its output isn't
+      # easily machine readable and doesn't contain everything we need.
+      def enumerate_dependencies
+        json = JSON.parse(project_assets_file)
+        nuget_packages_dir = json["project"]["restore"]["packagesPath"]
+        json["targets"].each_with_object({}) do |(_, target), dependencies|
+          target.each do |reference_key, reference|
+            # Ignore project references
+            next unless reference["type"] == "package"
+            package_id_parts = reference_key.partition("/")
+            name = package_id_parts[0]
+            version = package_id_parts[-1]
+            id = "#{name}-#{version}"
+
+            # Already know this package from another target
+            next if dependencies.key?(id)
+
+            path = File.join(nuget_packages_dir, json["libraries"][reference_key]["path"])
+            dependencies[id] = NuGetDependency.new(
+              name: id,
+              version: version,
+              path: path,
+              metadata: {
+                "type" => NuGet.type,
+                "name" => name
+              }
+            )
+          end
+        end.values
+      end
+    end
+  end
+end