licensed 1.5.2 → 2.0.0

data/docs/sources/bundler.md

@@ -1,4 +1,4 @@
-# Bundler (rubygem)
+# Bundler
 
 The bundler source will detect dependencies `Gemfile` and `Gemfile.lock` files are found at an apps `source_path`.  The source uses the `Bundler` API to enumerate dependencies from `Gemfile` and `Gemfile.lock`.
 
@@ -6,13 +6,13 @@ The bundler source will detect dependencies `Gemfile` and `Gemfile.lock` files a
 
 **Note** this content only applies to running licensed from an executable.  It does not apply when using licensed as a gem.
 
-_It is strongly recommended that ruby is always available when running the licensed executable._
+_It is required that the ruby runtime is available when running the licensed executable._
 
 The licensed executable contains and runs a version of ruby.  When using the Bundler APIs, a mismatch between the version of ruby built into the licensed executable and the version of licensed used during `bundle install` can occur.  This mismatch can lead to licensed raising errors due to not finding dependencies.
 
 For example, if `bundle install` was run with ruby 2.5.0 then the bundler specification path would be `<bundle path>/ruby/2.5.0/specifications`.  However, if the licensed executable contains ruby 2.4.0, then licensed will be looking for specifications at `<bundle path>/ruby/2.4.0/specifications`.  That path may not exist, or it may contain invalid or stale content.
 
-If ruby is available when running licensed, licensed will determine the ruby version used during `bundle install` and prefer that version string over the version contained in the executable.  If bundler is also available, then the ruby command will be run from a `bundle exec` context.
+To prevent confusion, licensed uses the local ruby runtime to determine the ruby version for local gems during `bundle install`.  If bundler is also available, then the ruby command will be run from a `bundle exec` context.
 
 ### Excluding gem groups
 
@@ -27,14 +27,14 @@ The bundler source determines which gem groups to include or exclude with the fo
 Strings and string arrays are both :+1:
 
 ```yml
-rubygem:
+bundler:
   without: development
 ```
 
 or
 
 ```yml
-rubygem:
+bundler:
   without:
     - build
     - development

data/lib/licensed.rb

@@ -1,21 +1,12 @@
 # frozen_string_literal: true
 require "licensed/version"
 require "licensed/shell"
-require "licensed/license"
+require "licensed/dependency_record"
 require "licensed/dependency"
 require "licensed/git"
-require "licensed/source/bundler"
-require "licensed/source/bower"
-require "licensed/source/manifest"
-require "licensed/source/npm"
-require "licensed/source/go"
-require "licensed/source/dep"
-require "licensed/source/cabal"
-require "licensed/source/pip"
-require "licensed/source/git_submodule"
+require "licensed/sources"
 require "licensed/configuration"
-require "licensed/command/cache"
-require "licensed/command/status"
-require "licensed/command/list"
-require "licensed/command/version"
+require "licensed/reporters"
+require "licensed/commands"
 require "licensed/ui/shell"
+require "licensed/migrations"

data/lib/licensed/cli.rb

@@ -8,33 +8,48 @@ module Licensed
     desc "cache", "Cache the licenses of dependencies"
     method_option :force, type: :boolean,
       desc: "Overwrite licenses even if version has not changed."
-    method_option :offline, type: :boolean,
-      desc: "This option is deprecated and will be removed in the next major release."
     method_option :config, aliases: "-c", type: :string,
       desc: "Path to licensed configuration file"
     def cache
-      run Licensed::Command::Cache.new(config), force: options[:force]
+      run Licensed::Commands::Cache.new(config: config), force: options[:force]
     end
 
     desc "status", "Check status of dependencies' cached licenses"
     method_option :config, aliases: "-c", type: :string,
       desc: "Path to licensed configuration file"
     def status
-      run Licensed::Command::Status.new(config)
+      run Licensed::Commands::Status.new(config: config)
     end
 
     desc "list", "List dependencies"
     method_option :config, aliases: "-c", type: :string,
       desc: "Path to licensed configuration file"
     def list
-      run Licensed::Command::List.new(config)
+      run Licensed::Commands::List.new(config: config)
     end
 
     map "-v" => :version
     map "--version" => :version
     desc "version", "Show Installed Version of Licensed, [-v, --version]"
     def version
-      run Licensed::Command::Version.new
+      puts Licensed::VERSION
+    end
+
+    desc "migrate", "Migrate from a previous version of licensed"
+    method_option :config, aliases: "-c", type: :string, required: true,
+      desc: "Path to licensed configuration file"
+    method_option :from, aliases: "-f", type: :string, required: true,
+      desc: "Licensed version to migrate from - #{Licensed.previous_major_versions.map { |major| "v#{major}" }.join(", ")}"
+    def migrate
+      case options["from"]
+      when "v1"
+        Licensed::Migrations::V2.migrate(options["config"])
+      else
+        shell = Thor::Base.shell.new
+        shell.say "Unrecognized option from=#{options["from"]}", :red
+        CLI.command_help(shell, "migrate")
+        exit 1
+      end
     end
 
     # If an error occurs (e.g. a missing command or argument), exit 1.
@@ -55,9 +70,8 @@ module Licensed
       options["config"] || Dir.pwd
     end
 
-    def run(command, *args)
-      command.run(*args)
-      exit command.success?
+    def run(command, **args)
+      exit command.run(args)
     end
   end
 end

data/lib/licensed/commands.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+module Licensed
+  module Commands
+    require "licensed/commands/command"
+    require "licensed/commands/cache"
+    require "licensed/commands/status"
+    require "licensed/commands/list"
+  end
+end

data/lib/licensed/commands/cache.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+module Licensed
+  module Commands
+    class Cache < Command
+      def initialize(config:, reporter: Licensed::Reporters::CacheReporter.new)
+        super(config: config, reporter: reporter)
+      end
+
+      protected
+
+      # Run the command for all enumerated dependencies found in a dependency source,
+      # recording results in a report.
+      # Removes any cached records that don't match a current application
+      # dependency.
+      #
+      # app - The application configuration for the source
+      # source - A dependency source enumerator
+      #
+      # Returns whether the command succeeded for the dependency source enumerator
+      def run_source(app, source)
+        result = super
+        clear_stale_cached_records(app, source) if result
+        result
+      end
+
+      # Cache dependency record data.
+      #
+      # app - The application configuration for the dependency
+      # source - The dependency source enumerator for the dependency
+      # dependency - An application dependency
+      # report - A report hash for the command to provide extra data for the report output.
+      #
+      # Returns true.
+      def evaluate_dependency(app, source, dependency, report)
+        filename = app.cache_path.join(source.class.type, "#{dependency.name}.#{DependencyRecord::EXTENSION}")
+        cached_record = Licensed::DependencyRecord.read(filename)
+        if options[:force] || save_dependency_record?(dependency, cached_record)
+          # use the cached license value if the license text wasn't updated
+          dependency.record["license"] = cached_record["license"] if dependency.record.matches?(cached_record)
+          dependency.record.save(filename)
+          report["cached"] = true
+        end
+
+        true
+      end
+
+      # Determine if the current dependency's record should be saved.
+      # The record should be saved if:
+      # 1. there is no cached record
+      # 2. the cached record doesn't have a version set
+      # 3. the cached record version doesn't match the current dependency version
+      #
+      # dependency - An application dependency
+      # cached_record - A dependency record to compare with the dependency
+      #
+      # Returns true if dependency's record should be saved
+      def save_dependency_record?(dependency, cached_record)
+        return true if cached_record.nil?
+
+        cached_version = cached_record["version"]
+        return true if cached_version.nil? || cached_version.empty?
+        return true if dependency.version != cached_version
+        false
+      end
+
+      # Clean up cached files that dont match current dependencies
+      #
+      # app - An application configuration
+      # source - A dependency source enumerator
+      #
+      # Returns nothing
+      def clear_stale_cached_records(app, source)
+        names = source.dependencies.map { |dependency| File.join(source.class.type, dependency.name) }
+        Dir.glob(app.cache_path.join(source.class.type, "**/*.#{DependencyRecord::EXTENSION}")).each do |file|
+          file_path = Pathname.new(file)
+          relative_path = file_path.relative_path_from(app.cache_path).to_s
+          FileUtils.rm(file) unless names.include?(relative_path.chomp(".#{DependencyRecord::EXTENSION}"))
+        end
+      end
+    end
+  end
+end

data/lib/licensed/commands/command.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+module Licensed
+  module Commands
+    class Command
+      attr_reader :config
+      attr_reader :reporter
+      attr_reader :options
+
+      def initialize(config:, reporter:)
+        @config = config
+        @reporter = reporter
+      end
+
+      # Run the command
+      #
+      # options - Options to run the command with
+      #
+      # Returns whether the command was a success
+      def run(**options)
+        @options = options
+        begin
+          result = reporter.report_run(self) do
+            config.apps.map { |app| run_app(app) }.all?
+          end
+        ensure
+          @options = nil
+        end
+
+        result
+      end
+
+      protected
+
+      # Run the command for all enabled sources for an application configuration,
+      # recording results in a report.
+      #
+      # app - An application configuration
+      #
+      # Returns whether the command succeeded for the application.
+      def run_app(app)
+        reporter.report_app(app) do |report|
+          Dir.chdir app.source_path do
+            begin
+              app.sources.select(&:enabled?).map { |source| run_source(app, source) }.all?
+            rescue Licensed::Shell::Error => err
+              report.errors << err.message
+              false
+            end
+          end
+        end
+      end
+
+      # Run the command for all enumerated dependencies found in a dependency source,
+      # recording results in a report.
+      #
+      # app - The application configuration for the source
+      # source - A dependency source enumerator
+      #
+      # Returns whether the command succeeded for the dependency source enumerator
+      def run_source(app, source)
+        reporter.report_source(source) do |report|
+          begin
+            source.dependencies.map { |dependency| run_dependency(app, source, dependency) }.all?
+          rescue Licensed::Shell::Error => err
+            report.errors << err.message
+            false
+          rescue Licensed::Sources::Source::Error => err
+            report.errors << err.message
+            false
+          end
+        end
+      end
+
+      # Run the command for a dependency, evaluating the dependency and
+      # recording results in a report.  Dependencies that were found with errors
+      # are not evaluated and add any errors to the dependency report.
+      #
+      # app - The application configuration for the dependency
+      # source - The dependency source enumerator for the dependency
+      # dependency - An application dependency
+      #
+      # Returns whether the command succeeded for the dependency
+      def run_dependency(app, source, dependency)
+        reporter.report_dependency(dependency) do |report|
+          if dependency.errors?
+            report.errors.concat(dependency.errors)
+            return false
+          end
+
+          begin
+            evaluate_dependency(app, source, dependency, report)
+          rescue Licensed::Shell::Error => err
+            report.errors << err.message
+            false
+          end
+        end
+      end
+
+      # Evaluate a dependency for the command.  Must be implemented by a command implementation.
+      #
+      # app - The application configuration for the dependency
+      # source - The dependency source enumerator for the dependency
+      # dependency - An application dependency
+      # report - A report hash for the command to provide extra data for the report output.
+      #
+      # Returns whether the command succeeded for the dependency
+      def evaluate_dependency(app, source, dependency, report)
+        raise "`evaluate_dependency` must be implemented by a command"
+      end
+    end
+  end
+end

data/lib/licensed/commands/list.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+module Licensed
+  module Commands
+    class List < Command
+      def initialize(config:, reporter: Licensed::Reporters::ListReporter.new)
+        super(config: config, reporter: reporter)
+      end
+
+      protected
+
+      # Listing dependencies requires no extra work.
+      #
+      # app - The application configuration for the dependency
+      # source - The dependency source enumerator for the dependency
+      # dependency - An application dependency
+      # report - A report hash for the command to provide extra data for the report output.
+      #
+      # Returns true.
+      def evaluate_dependency(app, source, dependency, report)
+        true
+      end
+    end
+  end
+end

data/lib/licensed/commands/status.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+require "yaml"
+
+module Licensed
+  module Commands
+    class Status < Command
+      def initialize(config:, reporter: Licensed::Reporters::StatusReporter.new)
+        super(config: config, reporter: reporter)
+      end
+
+      protected
+
+      # Verifies that a cached record exists, is up to date and
+      # has license data that complies with the licensed configuration.
+      #
+      # app - The application configuration for the dependency
+      # source - The dependency source enumerator for the dependency
+      # dependency - An application dependency
+      # report - A report hash for the command to provide extra data for the report output.
+      #
+      # Returns whether the dependency has a cached record that is compliant
+      # with the licensed configuration.
+      def evaluate_dependency(app, source, dependency, report)
+        filename = app.cache_path.join(source.class.type, "#{dependency.name}.#{DependencyRecord::EXTENSION}")
+        report["filename"] = filename
+
+        cached_record = cached_record(filename)
+        if cached_record.nil?
+          report.errors << "cached dependency record not found"
+        else
+          report.errors << "cached dependency record out of date" if cached_record["version"] != dependency.version
+          report.errors << "missing license text" if cached_record.licenses.empty?
+          report.errors << "license needs review: #{cached_record["license"]}" unless allowed_or_reviewed?(app, cached_record)
+        end
+
+        report.errors.empty?
+      end
+
+      def allowed_or_reviewed?(app, dependency)
+        app.allowed?(dependency) || app.reviewed?(dependency)
+      end
+
+      def cached_record(filename)
+        return nil unless File.exist?(filename)
+        DependencyRecord.read(filename)
+      end
+    end
+  end
+end

data/lib/licensed/configuration.rb

@@ -9,15 +9,10 @@ module Licensed
       ".licensed.yaml".freeze,
       ".licensed.json".freeze
     ].freeze
-    SOURCE_TYPES = Source.constants.map { |c| Source.const_get(c) }.freeze
-
-    attr_reader :ui
 
     def initialize(options = {}, inherited_options = {})
       super()
 
-      @ui = Licensed::UI::Shell.new
-
       # update order:
       # 1. anything inherited from root config
       # 2. app defaults
@@ -61,9 +56,9 @@ module Licensed
 
     # Returns an array of enabled app sources
     def sources
-      @sources ||= SOURCE_TYPES.select { |source_class| enabled?(source_class.type) }
-                               .map { |source_class| source_class.new(self) }
-                               .select(&:enabled?)
+      @sources ||= Licensed::Sources::Source.sources
+                                            .select { |source_class| enabled?(source_class.type) }
+                                            .map { |source_class| source_class.new(self) }
     end
 
     # Returns whether a source type is enabled

data/lib/licensed/dependency.rb

@@ -2,90 +2,148 @@
 require "licensee"
 
 module Licensed
-  class Dependency < License
-    LEGAL_FILES = /\A(AUTHORS|NOTICE|LEGAL)(?:\..*)?\z/i
+  class Dependency < Licensee::Projects::FSProject
+    LEGAL_FILES_PATTERN = /(AUTHORS|NOTICE|LEGAL)(?:\..*)?\z/i
 
-    attr_reader :path
-    attr_reader :search_root
     attr_reader :name
+    attr_reader :version
+    attr_reader :errors
+
+    # Create a new project dependency
+    #
+    # name        - unique dependency name
+    # version     - dependency version
+    # path        - absolute file path to the dependency, to find license contents
+    # search_root - (optional) the root location to search for dependency license contents
+    # metadata    - (optional) additional dependency data to cache
+    # errors      - (optional) errors encountered when evaluating dependency
+    #
+    # Returns a new dependency object.  Dependency metadata and license contents
+    # are available if no errors are set on the dependency.
+    def initialize(name:, version:, path:, search_root: nil, metadata: {}, errors: [])
+      # check the path for default errors if no other errors
+      # were found when loading the dependency
+      if errors.empty?
+        path_error = path_error(path, search_root)
+        errors.push(path_error) if path_error
+      end
 
-    def initialize(path, metadata = {})
-      @search_root = metadata.delete("search_root")
-      @name = metadata.delete("path") || metadata["name"]
-      super metadata
-
-      self.path = path
-    end
+      @name = name
+      @version = version
+      @metadata = metadata
+      @errors = errors
 
-    # Returns a Licensee::Projects::FSProject for the dependency path
-    def project
-      @project ||= Licensee::Projects::FSProject.new(path, search_root: search_root, detect_packages: true, detect_readme: true)
-    end
+      # if there are any errors, don't evaluate any dependency contents
+      return if errors.any?
 
-    # Sets the path to source dependency license information
-    def path=(path)
       # enforcing absolute paths makes life much easier when determining
       # an absolute file path in #notices
-      unless Pathname.new(path).absolute?
-        raise "Dependency path #{path} must be absolute"
+      if !Pathname.new(path).absolute?
+        # this is an internal error related to source implementation and
+        # should be raised, not stored to be handled by reporters
+        raise ArgumentError, "dependency path #{path} must be absolute"
       end
 
-      @path = path
-      reset_license!
+      super(path, search_root: search_root, detect_readme: true, detect_packages: true)
     end
 
-    # Detects license information and sets it on this dependency object.
-    #  After calling `detect_license!``, the license is set at
-    # `dependency["license"]` and legal text is set to `dependency.text`
-    def detect_license!
-      self["license"] = license_key
-      self.text = [license_text, *notices].join("\n" + TEXT_SEPARATOR + "\n").rstrip
+    # Returns true if the dependency has any errors, false otherwise
+    def errors?
+      errors.any?
     end
 
-    # Extract legal notices from the dependency source
-    def notices
-      local_files.uniq # unique local file paths
-           .sort # sorted by the path
-           .map { |f| File.read(f) } # read the file contents
-           .map(&:rstrip) # strip whitespace
-           .select { |t| t.length > 0 } # files with content only
+    # Returns a record for this dependency including metadata and legal contents
+    def record
+      return nil if errors?
+      @record ||= DependencyRecord.new(
+        metadata: license_metadata,
+        licenses: license_contents,
+        notices: notice_contents
+      )
     end
 
-    # Returns an array of file paths used to locate legal notices
-    def local_files
-      return [] unless Dir.exist?(path)
+    # Returns a string representing the dependencys license
+    def license_key
+      return "none" if errors? || !license
+      license.key
+    end
 
-      Dir.foreach(path).map do |file|
-        next unless file.match(LEGAL_FILES)
+    # Returns the license text content from all matched sources
+    # except the package file, which doesn't contain license text.
+    def license_contents
+      return [] if errors?
+      matched_files.reject { |f| f == package_file }
+                   .group_by(&:content)
+                   .map { |content, files| { "sources" => content_sources(files), "text" => content } }
+    end
 
-        file_path = File.join(path, file)
-        next unless File.file?(file_path)
+    # Returns legal notices found at the dependency path
+    def notice_contents
+      return [] if errors?
+      notice_files.sort # sorted by the path
+                  .map { |file| { "sources" => content_sources(file), "text" => File.read(file).rstrip } }
+                  .select { |text| text.length > 0 } # files with content only
+    end
+
+    # Returns an array of file paths used to locate legal notices
+    def notice_files
+      return [] if errors?
 
-        file_path
-      end.compact
+      Dir.glob(dir_path.join("*"))
+         .grep(LEGAL_FILES_PATTERN)
+         .select { |path| File.file?(path) }
     end
 
     private
 
-    # Resets all local project and license information
-    def reset_license!
-      @project = nil
-      self.delete("license")
-      self.text = nil
+    def path_error(path, search_root)
+      return "dependency path not found" if path.to_s.empty?
+      return if File.exist?(path)
+      return if search_root && File.exist?(search_root)
+
+      # if the given path doesn't exist
+      # AND a search root isn't given, or the search root doesn't exist
+      # then set an error that the expected dependency path doesn't exist
+      "expected dependency path #{path} does not exist"
     end
 
-    # Regardless of the license detected, try to pull the license content
-    # from the local LICENSE-type files, remote LICENSE, or the README, in that order
-    def license_text
-      content_files = Array(project.license_files)
-      content_files << project.readme_file if content_files.empty? && project.readme_file
-      content_files.map(&:content).join("\n#{LICENSE_SEPARATOR}\n")
+    # Returns the sources for a group of license or notice file contents
+    #
+    # Sources are returned as a single string with sources separated by ", "
+    def content_sources(files)
+      paths = Array(files).map do |file|
+        path = if file.is_a?(Licensee::ProjectFiles::ProjectFile)
+          dir_path.join(file[:dir], file[:name])
+        else
+          Pathname.new(file).expand_path(dir_path)
+        end
+
+        if path.fnmatch?(dir_path.join("**").to_path)
+          # files under the dependency path return the relative path to the file
+          path.relative_path_from(dir_path).to_path
+        else
+          # otherwise return the source_path as the immediate parent folder name
+          # joined with the file name
+          path.dirname.basename.join(path.basename).to_path
+        end
+      end
+
+      paths.join(", ")
     end
 
-    # Returns a string representing the project's license
-    def license_key
-      return "none" unless project.license
-      project.license.key
+    # Returns the metadata that represents this dependency.  This metadata
+    # is written to YAML in the dependencys cached text file
+    def license_metadata
+      {
+        # can be overriden by values in @metadata
+        "name" => name,
+        "version" => version
+      }.merge(
+        @metadata
+      ).merge({
+        # overrides all other values
+        "license" => license_key
+      })
     end
   end
 end