licensed 2.8.0 → 2.11.0

data/lib/licensed/dependency.rb

@@ -74,7 +74,7 @@ module Licensed
     def license_contents
       files = matched_files.reject { |f| f == package_file }
                            .group_by(&:content)
-                           .map { |content, files| { "sources" => license_content_sources(files), "text" => content } }
+                           .map { |content, sources| { "sources" => license_content_sources(sources), "text" => content } }
 
       files << generated_license_contents if files.empty?
       files.compact

data/lib/licensed/git.rb

@@ -11,7 +11,9 @@ module Licensed
       # or nil if not in a git repository.
       def repository_root
         return unless available?
-        Licensed::Shell.execute("git", "rev-parse", "--show-toplevel", allow_failure: true)
+        root = Licensed::Shell.execute("git", "rev-parse", "--show-toplevel", allow_failure: true)
+        return nil if root.empty?
+        root
       end
 
       # Returns true if a git repository is found, false otherwise

data/lib/licensed/reporters.rb

@@ -7,5 +7,6 @@ module Licensed
     require "licensed/reporters/list_reporter"
     require "licensed/reporters/json_reporter"
     require "licensed/reporters/yaml_reporter"
+    require "licensed/reporters/notices_reporter"
   end
 end

data/lib/licensed/reporters/cache_reporter.rb

@@ -27,32 +27,32 @@ module Licensed
           shell.info "  #{source.class.type}"
           result = yield report
 
-          warning_reports = report.all_reports.select { |report| report.warnings.any? }.to_a
+          warning_reports = report.all_reports.select { |r| r.warnings.any? }.to_a
           if warning_reports.any?
             shell.newline
             shell.warn "  * Warnings:"
-            warning_reports.each do |report|
-              display_metadata = report.map { |k, v| "#{k}: #{v}" }.join(", ")
+            warning_reports.each do |r|
+              display_metadata = r.map { |k, v| "#{k}: #{v}" }.join(", ")
 
-              shell.warn "    * #{report.name}"
+              shell.warn "    * #{r.name}"
               shell.warn "    #{display_metadata}" unless display_metadata.empty?
-              report.warnings.each do |warning|
+              r.warnings.each do |warning|
                 shell.warn "      - #{warning}"
               end
               shell.newline
             end
           end
 
-          errored_reports = report.all_reports.select { |report| report.errors.any? }.to_a
+          errored_reports = report.all_reports.select { |r| r.errors.any? }.to_a
           if errored_reports.any?
             shell.newline
             shell.error "  * Errors:"
-            errored_reports.each do |report|
-              display_metadata = report.map { |k, v| "#{k}: #{v}" }.join(", ")
+            errored_reports.each do |r|
+              display_metadata = r.map { |k, v| "#{k}: #{v}" }.join(", ")
 
-              shell.error "    * #{report.name}"
+              shell.error "    * #{r.name}"
               shell.error "    #{display_metadata}" unless display_metadata.empty?
-              report.errors.each do |error|
+              r.errors.each do |error|
                 shell.error "      - #{error}"
               end
               shell.newline

data/lib/licensed/reporters/list_reporter.rb

@@ -28,16 +28,16 @@ module Licensed
           shell.info "  #{source.class.type}"
           result = yield report
 
-          errored_reports = report.all_reports.select { |report| report.errors.any? }.to_a
+          errored_reports = report.all_reports.select { |r| r.errors.any? }.to_a
           if errored_reports.any?
             shell.newline
             shell.error "  * Errors:"
-            errored_reports.each do |report|
-              display_metadata = report.map { |k, v| "#{k}: #{v}" }.join(", ")
+            errored_reports.each do |r|
+              display_metadata = r.map { |k, v| "#{k}: #{v}" }.join(", ")
 
-              shell.error "    * #{report.name}"
+              shell.error "    * #{r.name}"
               shell.error "    #{display_metadata}" unless display_metadata.empty?
-              report.errors.each do |error|
+              r.errors.each do |error|
                 shell.error "      - #{error}"
               end
               shell.newline

data/lib/licensed/reporters/notices_reporter.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Licensed
+  module Reporters
+    class NoticesReporter < Reporter
+      TEXT_SEPARATOR = "\n\n#{("-" * 5)}\n\n".freeze
+      LICENSE_SEPARATOR = "\n#{("*" * 5)}\n".freeze
+
+      # Reports on an application configuration in a notices command run
+      #
+      # app - An application configuration
+      #
+      # Returns the result of the yielded method
+      # Note - must be called from inside the `report_run` scope
+      def report_app(app)
+        super do |report|
+          filename = app["shared_cache"] ? "NOTICE.#{app["name"]}" : "NOTICE"
+          path = app.cache_path.join(filename)
+          shell.info "Writing notices for #{app["name"]} to #{path}"
+
+          result = yield report
+
+          File.open(path, "w") do |file|
+            file << "THIRD PARTY NOTICES\n"
+            file << LICENSE_SEPARATOR
+            file << report.all_reports
+                          .map { |r| notices(r) }
+                          .compact
+                          .join(LICENSE_SEPARATOR)
+          end
+
+          result
+        end
+      end
+
+      # Reports on a dependency in a notices command run.
+      #
+      # dependency - An application dependency
+      #
+      # Returns the result of the yielded method
+      # Note - must be called from inside the `report_run` scope
+      def report_dependency(dependency)
+        super do |report|
+          result = yield report
+          shell.warn "* #{report["warning"]}" if report["warning"]
+          result
+        end
+      end
+
+      # Returns notices information for a dependency report
+      def notices(report)
+        return unless report.target.is_a?(Licensed::Dependency)
+
+        cached_record = report["cached_record"]
+        return unless cached_record
+
+        texts = cached_record.licenses.map(&:text)
+        texts.concat(cached_record.notices)
+
+        <<~NOTICE
+          #{cached_record["name"]}@#{cached_record["version"]}
+
+          #{texts.map(&:strip).reject(&:empty?).compact.join(TEXT_SEPARATOR)}
+        NOTICE
+      end
+    end
+  end
+end

data/lib/licensed/reporters/reporter.rb

@@ -47,6 +47,9 @@ module Licensed
 
       def initialize(shell = Licensed::UI::Shell.new)
         @shell = shell
+        @run_report = nil
+        @app_report = nil
+        @source_report = nil
       end
 
       # Generate a report for a licensed command execution

data/lib/licensed/reporters/status_reporter.rb

@@ -15,20 +15,20 @@ module Licensed
           result = yield report
 
           all_reports = report.all_reports
-          errored_reports = all_reports.select { |report| report.errors.any? }.to_a
+          errored_reports = all_reports.select { |r| r.errors.any? }.to_a
 
-          dependency_count = all_reports.select { |report| report.target.is_a?(Licensed::Dependency) }.size
-          error_count = errored_reports.sum { |report| report.errors.size }
+          dependency_count = all_reports.select { |r| r.target.is_a?(Licensed::Dependency) }.size
+          error_count = errored_reports.sum { |r| r.errors.size }
 
           if error_count > 0
             shell.newline
             shell.error "Errors:"
-            errored_reports.each do |report|
-              display_metadata = report.map { |k, v| "#{k}: #{v}" }.join(", ")
+            errored_reports.each do |r|
+              display_metadata = r.map { |k, v| "#{k}: #{v}" }.join(", ")
 
-              shell.error "* #{report.name}"
+              shell.error "* #{r.name}"
               shell.error "  #{display_metadata}" unless display_metadata.empty?
-              report.errors.each do |error|
+              r.errors.each do |error|
                 shell.error "    - #{error}"
               end
               shell.newline

data/lib/licensed/sources.rb

@@ -11,6 +11,7 @@ module Licensed
     require "licensed/sources/go"
     require "licensed/sources/manifest"
     require "licensed/sources/npm"
+    require "licensed/sources/nuget"
     require "licensed/sources/pip"
     require "licensed/sources/pipenv"
     require "licensed/sources/gradle"

data/lib/licensed/sources/cabal.rb

@@ -111,11 +111,11 @@ module Licensed
         info = package_info_command(id).strip
         return missing_package(id) if info.empty?
 
-        info.lines.each_with_object({}) do |line, info|
+        info.lines.each_with_object({}) do |line, hsh|
           key, value = line.split(":", 2).map(&:strip)
           next unless key && value
 
-          info[key] = value
+          hsh[key] = value
         end
       end
 

data/lib/licensed/sources/go.rb

@@ -15,7 +15,8 @@ module Licensed
       def enumerate_dependencies
         with_configured_gopath do
           packages.map do |package|
-            import_path = non_vendored_import_path(package["ImportPath"])
+            import_path = non_vendored_path(package["ImportPath"], root_package["ImportPath"])
+            import_path ||= package["ImportPath"]
             error = package.dig("Error", "Err") if package["Error"]
 
             Dependency.new(
@@ -86,14 +87,17 @@ module Licensed
         # true if go standard packages includes the import path as given
         return true if go_std_packages.include?(import_path)
         return true if go_std_packages.include?("vendor/#{import_path}")
+        return true if go_std_packages.include?(import_path.sub("golang.org", "internal"))
 
         # additional checks are only for vendored dependencies - return false
         # if package isn't vendored
-        return false unless vendored_path?(import_path)
+        non_vendored_import_path = non_vendored_path(import_path, root_package["ImportPath"])
+        return false unless non_vendored_import_path
 
         # return true if any of the go standard packages matches against
         # the non-vendored import path
-        return true if go_std_packages.include?(non_vendored_import_path(import_path))
+        return true if go_std_packages.include?(non_vendored_import_path)
+        return true if go_std_packages.include?(non_vendored_import_path.sub("golang.org", "internal"))
 
         # modify the import path to look like the import path `go list` returns for vendored std packages
         vendor_path = import_path.sub("#{root_package["ImportPath"]}/", "")
@@ -104,7 +108,7 @@ module Licensed
       def local_package?(package)
         return false unless package && package["ImportPath"]
         import_path = package["ImportPath"]
-        import_path.start_with?(root_package["ImportPath"]) && !vendored_path?(import_path)
+        import_path.start_with?(root_package["ImportPath"]) && !import_path.include?("vendor/")
       end
 
       # Returns the version for a given package
@@ -116,12 +120,12 @@ module Licensed
         return go_mod["Version"] if go_mod
 
         package_directory = package["Dir"]
-        return unless package_directory
+        return unless package_directory && File.exist?(package_directory)
 
         # find most recent git SHA for a package, or nil if SHA is
         # not available
         Dir.chdir package_directory do
-          contents_version *contents_version_arguments
+          contents_version(*contents_version_arguments)
         end
       end
 
@@ -150,34 +154,37 @@ module Licensed
         return if package.nil?
 
         # search root choices:
-        # 1. module directory if using go modules
+        # 1. module directory if using go modules and directory is available
         # 2. vendor folder if package is vendored
         # 3. package root value if available
         # 4. GOPATH if the package directory is under the gopath
         # 5. nil
-        return package.dig("Module", "Dir") if package["Module"]
-        return package["Dir"].match("^(.*/vendor)/.*$")[1] if vendored_path?(package["Dir"])
+        module_dir = package.dig("Module", "Dir")
+        return module_dir if module_dir
+        return package["Dir"].match("^(.*/vendor)/.*$")[1] if vendored_path?(package["Dir"], config.root)
         return package["Root"] if package["Root"]
         return gopath if package["Dir"]&.start_with?(gopath)
         nil
       end
 
-      # Returns whether a package is vendored or not based on the package
-      # import_path
+      # Returns whether a package is vendored or not based on a base path and
+      # whether the path contains a vendor component
       #
       # path - Package path to test
-      def vendored_path?(path)
-        return false if path.nil?
-        path.start_with?(root_package["ImportPath"]) && path.include?("vendor/")
+      # base - The base path that the input must start with
+      def vendored_path?(path, base)
+        return false if path.nil? || base.nil?
+        path.start_with?(base.to_s) && path.include?("vendor/")
       end
 
-      # Returns the import path parameter without the vendor component
+      # Returns the path parameter without the vendor component if one is found
       #
-      # import_path - Package import path with vendor component
-      def non_vendored_import_path(import_path)
-        return unless import_path
-        return import_path unless vendored_path?(import_path)
-        import_path.split("vendor/")[1]
+      # path - Package path with vendor component
+      # base - The base path that the input must start with
+      def non_vendored_path(path, base)
+        return unless path
+        return unless vendored_path?(path, base)
+        path.split("vendor/")[1]
       end
 
       # Returns a hash of information about the package with a given import path

data/lib/licensed/sources/mix.rb

@@ -93,8 +93,12 @@ module Licensed
             :[a-zA-Z0-9_]+    # after an Elixir atom,
             ,\s*              # and skipping a comma and any number of spaces,
             "(?<version>.*?)" # capture the contents of a double-quoted string as the version,
-            .*                # and later
+            .*?\],\s*         # and later
             "(?<repo>.*?)"    # capture the contents of a double-quoted string as the repo
+            (?:
+              ,\s*            # a comma
+              "[a-f0-9]{64}"  # a digest
+            )?
             \},?\s*\Z         # right before the final closing brace.
           /x,
 

data/lib/licensed/sources/npm.rb

@@ -48,6 +48,7 @@ module Licensed
       # package name to it's metadata
       def recursive_dependencies(dependencies, result = {})
         dependencies.each do |name, dependency|
+          next if dependency["peerMissing"]
           next if yarn_lock_present && dependency["missing"]
           (result[name] ||= []) << dependency
           recursive_dependencies(dependency["dependencies"] || {}, result)

data/lib/licensed/sources/nuget.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+require "json"
+require "reverse_markdown"
+
+module Licensed
+  module Sources
+    # Only supports ProjectReference (project.assets.json) style restore used in .NET Core.
+    # Does not currently support packages.config style restore.
+    class NuGet < Source
+      def self.type
+        "nuget"
+      end
+
+      class NuGetDependency < Licensed::Dependency
+        LICENSE_FILE_REGEX = /<license\s*type\s*=\s*\"\s*file\s*\"\s*>\s*(.*)\s*<\/license>/ix.freeze
+        LICENSE_URL_REGEX = /<licenseUrl>\s*(.*)\s*<\/licenseUrl>/ix.freeze
+        PROJECT_URL_REGEX = /<projectUrl>\s*(.*)\s*<\/projectUrl>/ix.freeze
+        PROJECT_DESC_REGEX = /<description>\s*(.*)\s*<\/description>/ix.freeze
+
+        # Returns the metadata that represents this dependency.  This metadata
+        # is written to YAML in the dependencys cached text file
+        def license_metadata
+          super.tap do |record_metadata|
+            record_metadata["homepage"] = project_url if project_url
+            record_metadata["summary"] = description if description
+          end
+        end
+
+        def nuspec_path
+          name = @metadata["name"]
+          File.join(self.path, "#{name.downcase}.nuspec")
+        end
+
+        def nuspec_contents
+          return @nuspec_contents if defined?(@nuspec_contents)
+          @nuspec_contents = begin
+            return unless nuspec_path && File.exist?(nuspec_path)
+            File.read(nuspec_path)
+          end
+        end
+
+        def project_url
+          return @project_url if defined?(@project_url)
+          @project_url = begin
+            return unless nuspec_contents
+            match = nuspec_contents.match PROJECT_URL_REGEX
+            match[1] if match && match[1]
+          end
+        end
+
+        def description
+          return @description if defined?(@description)
+          @description = begin
+            return unless nuspec_contents
+            match = nuspec_contents.match PROJECT_DESC_REGEX
+            match[1] if match && match[1]
+          end
+        end
+
+        def project_files
+          @nuget_project_files ||= begin
+            files = super().flatten.compact
+
+            # Only include the local file if it's a file licensee didn't already detect
+            nuspec_license_filename = File.basename(nuspec_local_license_file.filename) if nuspec_local_license_file
+            if nuspec_license_filename && files.none? { |file| File.basename(file.filename) == nuspec_license_filename }
+              files.push(nuspec_local_license_file)
+            end
+
+            # Only download licenseUrl if no recognized license was found locally
+            if files.none? { |file| file.license && file.license.key != "other" }
+              files.push(nuspec_remote_license_file)
+            end
+
+            files.compact
+          end
+        end
+
+        # Look for a <license type="file"> element in the nuspec that points to an
+        # on-disk license file (which licensee may not find due to a non-standard filename)
+        def nuspec_local_license_file
+          return @nuspec_local_license_file if defined?(@nuspec_local_license_file)
+          return unless nuspec_contents
+
+          match = nuspec_contents.match LICENSE_FILE_REGEX
+          return unless match && match[1]
+
+          license_path = File.join(File.dirname(nuspec_path), match[1])
+          return unless File.exist?(license_path)
+
+          license_data = File.read(license_path)
+          @nuspec_local_license_file = Licensee::ProjectFiles::LicenseFile.new(license_data, license_path)
+        end
+
+        # Look for a <licenseUrl> element in the nuspec that either is known to contain a license identifier
+        # in the URL, or points to license text on the internet that can be downloaded.
+        def nuspec_remote_license_file
+          return @nuspec_remote_license_file if defined?(@nuspec_remote_license_file)
+          return unless nuspec_contents
+
+          match = nuspec_contents.match LICENSE_URL_REGEX
+          return unless match && match[1]
+
+          # Attempt to fetch the license content
+          license_content = self.class.retrieve_license(match[1])
+          @nuspec_remote_license_file = Licensee::ProjectFiles::LicenseFile.new(license_content, { uri: match[1] }) if license_content
+        end
+
+        class << self
+          def strip_html(html)
+            return unless html
+
+            return html unless html.downcase.include?("<html")
+            ReverseMarkdown.convert(html, unknown_tags: :bypass)
+          end
+
+          def ignored_url?(url)
+            # Many Microsoft packages that now use <license> use this for <licenseUrl>
+            # No need to fetch this page - it just contains NuGet documentation
+            url == "https://aka.ms/deprecateLicenseUrl"
+          end
+
+          def text_content_url(url)
+            # Convert github file URLs to raw URLs
+            return url unless match = url.match(/https?:\/\/(?:www\.)?github.com\/([^\/]+)\/([^\/]+)\/blob\/(.*)/i)
+            "https://github.com/#{match[1]}/#{match[2]}/raw/#{match[3]}"
+          end
+
+          def retrieve_license(url)
+            return unless url
+            return if ignored_url?(url)
+
+            # Transform URLs that are known to return HTML but have a corresponding text-based URL
+            text_url = text_content_url(url)
+
+            raw_content = fetch_content(text_url)
+            strip_html(raw_content)
+          end
+
+          def fetch_content(url, redirect_limit = 5)
+            url = URI.parse(url) if url.instance_of? String
+            return @response_by_url[url] if (@response_by_url ||= {}).key?(url)
+            return if redirect_limit == 0
+
+            begin
+              response = Net::HTTP.get_response(url)
+              case response
+              when Net::HTTPSuccess     then
+                @response_by_url[url] = response.body
+              when Net::HTTPRedirection then
+                redirect_url = URI.parse(response["location"])
+                if redirect_url.relative?
+                  redirect_url = url + redirect_url
+                end
+                # The redirect might be to a URL that requires transformation, i.e. a github file
+                redirect_url = text_content_url(redirect_url.to_s)
+                @response_by_url[url] = fetch_content(redirect_url, redirect_limit - 1)
+              end
+            rescue
+              # Host might no longer exist or some other error, ignore
+            end
+          end
+        end
+      end
+
+      def project_assets_file_path
+        File.join(config.pwd, "project.assets.json")
+      end
+
+      def project_assets_file
+        return @project_assets_file if defined?(@project_assets_file)
+        @project_assets_file = File.read(project_assets_file_path)
+      end
+
+      def enabled?
+        File.exist?(project_assets_file_path)
+      end
+
+      # Inspect project.assets.json files for package references.
+      # Ideally we'd use `dotnet list package` instead, but its output isn't
+      # easily machine readable and doesn't contain everything we need.
+      def enumerate_dependencies
+        json = JSON.parse(project_assets_file)
+        nuget_packages_dir = json["project"]["restore"]["packagesPath"]
+        json["targets"].each_with_object({}) do |(_, target), dependencies|
+          target.each do |reference_key, reference|
+            # Ignore project references
+            next unless reference["type"] == "package"
+            package_id_parts = reference_key.partition("/")
+            name = package_id_parts[0]
+            version = package_id_parts[-1]
+            id = "#{name}-#{version}"
+
+            # Already know this package from another target
+            next if dependencies.key?(id)
+
+            path = File.join(nuget_packages_dir, json["libraries"][reference_key]["path"])
+            dependencies[id] = NuGetDependency.new(
+              name: id,
+              version: version,
+              path: path,
+              metadata: {
+                "type" => NuGet.type,
+                "name" => name
+              }
+            )
+          end
+        end.values
+      end
+    end
+  end
+end