omnibus 5.4.0 → 5.5.0

data/lib/omnibus/licensing.rb

@@ -14,21 +14,56 @@
 # limitations under the License.
 #
 
-require 'uri'
-require 'fileutils'
-require 'omnibus/download_helpers'
+require "uri"
+require "fileutils"
+require "omnibus/download_helpers"
+require "license_scout/collector"
+require "license_scout/options"
 
 module Omnibus
   class Licensing
     include Logging
     include DownloadHelpers
+    include Sugarable
 
     OUTPUT_DIRECTORY = "LICENSES".freeze
+    CACHE_DIRECTORY = "license-cache".freeze
 
     class << self
-      # @see (Licensing#create!)
-      def create!(project)
-        new(project).create!
+
+      # Creates a new instance of Licensing, executes preparation steps, then
+      # yields control to a given block, and then creates a summary of the
+      # included licenses.
+      #
+      # @example Building a project:
+      #
+      #   Licensing.create_incrementally(self) do |license_collector|
+      #     softwares.each do |software|
+      #       software.build_me([license_collector])
+      #     end
+      #   end
+      #
+      # @param [Project] project
+      #   The project being built.
+      #
+      # @yieldparam [Licensing] license_collector
+      #   Yields an instance of Licensing. Call #execute_post_build to copy the
+      #   license files for a Software definition.
+      #
+      # @return [Licensing]
+      #
+      def create_incrementally(project)
+        new(project).tap do |license_collector|
+
+          license_collector.prepare
+          license_collector.validate_license_info
+
+          yield license_collector
+
+          license_collector.process_transitive_dependency_licensing_info
+          license_collector.create_project_license_file
+          license_collector.raise_if_warnings_fatal!
+        end
       end
     end
 
@@ -39,35 +74,80 @@ module Omnibus
     #
     attr_reader :project
 
+    #
+    # The warnings encountered while preparing the licensing information
+    #
+    # @return [Array<String>]
+    #
+    attr_reader :licensing_warnings
+
+    #
+    # The warnings encountered while preparing the licensing information for
+    # transitive dependencies.
+    #
+    # @return [Array<String>]
+    #
+    attr_reader :transitive_dependency_licensing_warnings
+
+    #
+    # Manifest data of transitive dependency licensing information
+    #
+    # @return Hash
+    #
+    attr_reader :dep_license_map
+
     #
     # @param [Project] project
     #   the project to create licenses for.
     #
     def initialize(project)
       @project = project
+      @licensing_warnings = []
+      @transitive_dependency_licensing_warnings = []
+      @dep_license_map = {}
     end
 
     #
-    # Creates the license files for given project.
-    # It is assumed that the project has already been built.
+    # Creates the required directories for licenses.
     #
     # @return [void]
     #
-    def create!
-      prepare
-      validate_license_info
-      create_software_license_files
-      create_project_license_file
+    def prepare
+      FileUtils.rm_rf(output_dir)
+      FileUtils.mkdir_p(output_dir)
+      FileUtils.touch(output_dir_gitkeep_file)
+      FileUtils.rm_rf(cache_dir)
+      FileUtils.mkdir_p(cache_dir)
+      FileUtils.touch(cache_dir_gitkeep_file)
     end
 
+    # Required callback to use instances of this class as a build wrapper for
+    # Software#build_me. Licensing doesn't need to do anything pre-build, so
+    # this does nothing.
     #
-    # Creates the required directories for licenses.
+    # @param [Software] software
     #
     # @return [void]
     #
-    def prepare
-      FileUtils.rm_rf(output_dir)
-      FileUtils.mkdir_p(output_dir)
+    def execute_pre_build(software)
+    end
+
+    # Callback that gets called by Software#build_me after the build is done.
+    # Invokes license copying for the given software. This ensures that
+    # licenses are copied before a git cache snapshot is taken, so that the
+    # license files are correctly restored when a build is skipped due to a
+    # cache hit.
+    #
+    # @param [Software] software
+    #
+    # @return [void]
+    #
+    def execute_post_build(software)
+      collect_licenses_for(software)
+
+      unless software.skip_transitive_dependency_licensing
+        collect_transitive_dependency_licenses_for(software)
+      end
     end
 
     #
@@ -91,7 +171,7 @@ module Omnibus
 
       # Check used license is a standard license
       if project.license != "Unspecified" && !STANDARD_LICENSES.include?(project.license)
-        licensing_warning("Project '#{project.name}' is using '#{project.license}' which is not one of the standard licenses identified in https://opensource.org/licenses/alphabetical. Consider using one of the standard licenses.")
+        licensing_info("Project '#{project.name}' is using '#{project.license}' which is not one of the standard licenses identified in https://opensource.org/licenses/alphabetical. Consider using one of the standard licenses.")
       end
 
       # Now let's check the licensing info for software components
@@ -108,7 +188,7 @@ module Omnibus
 
         # Check if the software license is one of the standard licenses
         if license_info[:license] != "Unspecified" && !STANDARD_LICENSES.include?(license_info[:license])
-          licensing_warning("Software '#{software_name}' uses license '#{license_info[:license]}' which is not one of the standard licenses identified in https://opensource.org/licenses/alphabetical. Consider using one of the standard licenses.")
+          licensing_info("Software '#{software_name}' uses license '#{license_info[:license]}' which is not one of the standard licenses identified in https://opensource.org/licenses/alphabetical. Consider using one of the standard licenses.")
         end
       end
     end
@@ -123,50 +203,14 @@ module Omnibus
     # @return [void]
     #
     def create_project_license_file
-      File.open(project.license_file_path, 'w') do |f|
+      File.open(project.license_file_path, "w") do |f|
         f.puts "#{project.name} #{project.build_version} license: \"#{project.license}\""
         f.puts ""
         f.puts project_license_content
         f.puts ""
         f.puts components_license_summary
-      end
-    end
-
-    #
-    # Copies the license files specified by the software components into the
-    # output directory.
-    #
-    # @return [void]
-    #
-    def create_software_license_files
-      license_map.each do |name, values|
-        license_files = values[:license_files]
-
-        license_files.each do |license_file|
-          if license_file
-            output_file = license_package_location(name, license_file)
-
-            if local?(license_file)
-              input_file = File.expand_path(license_file, values[:project_dir])
-              if File.exist?(input_file)
-                FileUtils.cp(input_file, output_file)
-              else
-                licensing_warning("License file '#{input_file}' does not exist for software '#{name}'.")
-              end
-            else
-              begin
-                download_file!(license_file, output_file, enable_progress_bar: false)
-              rescue SocketError,
-                     Errno::ECONNREFUSED,
-                     Errno::ECONNRESET,
-                     Errno::ENETUNREACH,
-                     Timeout::Error,
-                     OpenURI::HTTPError
-                licensing_warning("Can not download license file '#{license_file}' for software '#{name}'.")
-              end
-            end
-          end
-        end
+        f.puts ""
+        f.puts dependencies_license_summary
       end
     end
 
@@ -176,7 +220,7 @@ module Omnibus
     # @return [String]
     #
     def project_license_content
-      project.license_file.nil? ? "" : IO.read(File.join(Config.project_root,project.license_file))
+      project.license_file.nil? ? "" : IO.read(File.join(Config.project_root, project.license_file))
     end
 
     #
@@ -213,6 +257,41 @@ module Omnibus
       out
     end
 
+    #
+    # Summary of the licenses of the transitive dependencies of the project.
+    # It is in the form of:
+    # ...
+    # This product includes inifile 3.0.0
+    # which is a 'ruby_bundler' dependency of 'chef',
+    # and which is available under a 'MIT' License.
+    # For details, see:
+    # /opt/opscode/LICENSES/ruby_bundler-inifile-3.0.0-README.md
+    # ...
+    #
+    # @return [String]
+    #
+    def dependencies_license_summary
+      out = "\n\n"
+
+      dep_license_map.each do |dep_mgr_name, data|
+        data.each do |dep_name, data|
+          data.each do |dep_version, dep_data|
+            projects = dep_data["dependency_of"].sort.map { |p| "'#{p}'" }.join(", ")
+            files = dep_data["license_files"].map { |f| File.join(output_dir, f) }
+
+            out << "This product includes #{dep_name} #{dep_version}\n"
+            out << "which is a '#{dep_mgr_name}' dependency of #{projects},\n"
+            out << "and which is available under a '#{dep_data["license"]}' License.\n"
+            out << "For details, see:\n"
+            out << files.join("\n")
+            out << "\n\n"
+          end
+        end
+      end
+
+      out
+    end
+
     #
     # Map that collects information about the licenses of the softwares
     # included in the project.
@@ -279,6 +358,34 @@ module Omnibus
       File.expand_path(OUTPUT_DIRECTORY, project.install_dir)
     end
 
+    #
+    # Path to a .gitkeep file we create in the output dir so git caching
+    # doesn't delete the directory.
+    #
+    # @return [String]
+    #
+    def output_dir_gitkeep_file
+      File.join(output_dir, ".gitkeep")
+    end
+
+    # Cache directory where transitive dependency licenses will be collected in.
+    #
+    # @return [String]
+    #
+    def cache_dir
+      File.expand_path(CACHE_DIRECTORY, project.install_dir)
+    end
+
+    #
+    # Path to a .gitkeep file we create in the cache dir so git caching
+    # doesn't delete the directory.
+    #
+    # @return [String]
+    #
+    def cache_dir_gitkeep_file
+      File.join(cache_dir, ".gitkeep")
+    end
+
     #
     # Returns if the given path to a license is local or a remote url.
     #
@@ -290,14 +397,194 @@ module Omnibus
     end
 
     #
-    # Logs the given message as warning.
+    # Logs the given message as info.
+    #
+    # This method should only be used for detecting in a license is known or not.
+    # In the future, we will introduce a configurable way to whitelist or blacklist
+    # the allowed licenses. Once we implement that we need to stop using this method.
+    #
+    # @param [String] message
+    #   message to log as warning
+    def licensing_info(message)
+      log.info(log_key) { message }
+    end
+
+    #
+    # Logs the given message as warning or fails the build depending on the
+    # :fatal_licensing_warnings configuration setting.
     #
     # @param [String] message
     #   message to log as warning
     def licensing_warning(message)
+      licensing_warnings << message
+      log.warn(log_key) { message }
+    end
+
+    #
+    # Logs the given message as warning or fails the build depending on the
+    # :fatal_transitive_dependency_licensing_warnings configuration setting.
+    #
+    # @param [String] message
+    #   message to log as warning
+    def transitive_dependency_licensing_warning(message)
+      transitive_dependency_licensing_warnings << message
       log.warn(log_key) { message }
     end
 
+    def raise_if_warnings_fatal!
+      warnings_to_raise = []
+      if Config.fatal_licensing_warnings && !licensing_warnings.empty?
+        warnings_to_raise << licensing_warnings
+      end
+
+      if Config.fatal_transitive_dependency_licensing_warnings && !transitive_dependency_licensing_warnings.empty?
+        warnings_to_raise << transitive_dependency_licensing_warnings
+      end
+
+      warnings_to_raise.flatten!
+      raise LicensingError.new(warnings_to_raise) unless warnings_to_raise.empty?
+    end
+
+    # 1. Parse all the licensing information for all software from 'cache_dir'
+    # 2. Merge and drop the duplicates
+    # 3. Add these licenses to the main manifest, to be merged with the main
+    # licensing information from software definitions.
+    def process_transitive_dependency_licensing_info
+      Dir.glob("#{cache_dir}/*/*-dependency-licenses.json").each do |license_manifest_path|
+        license_manifest_data = FFI_Yajl::Parser.parse(File.read(license_manifest_path))
+        project_name = license_manifest_data["project_name"]
+        dependency_license_dir = File.dirname(license_manifest_path)
+
+        license_manifest_data["dependency_managers"].each do |dep_mgr_name, dependencies|
+          dep_license_map[dep_mgr_name] ||= {}
+
+          dependencies.each do |dependency|
+            # Copy dependency files
+            dependency["license_files"].each do |f|
+              license_path = File.join(dependency_license_dir, f)
+              output_path = File.join(output_dir, f)
+              FileUtils.cp(license_path, output_path)
+            end
+
+            dep_name = dependency["name"]
+            dep_version = dependency["version"]
+
+            # If we already have this dependency we do not need to add it again.
+            if dep_license_map[dep_mgr_name][dep_name] && dep_license_map[dep_mgr_name][dep_name][dep_version]
+              dep_license_map[dep_mgr_name][dep_name][dep_version]["dependency_of"] << project_name
+            else
+              dep_license_map[dep_mgr_name][dep_name] ||= {}
+              dep_license_map[dep_mgr_name][dep_name][dep_version] = {
+                "license" => dependency["license"],
+                "license_files" => dependency["license_files"],
+                "dependency_of" => [ project_name ],
+              }
+            end
+          end
+        end
+      end
+
+      FileUtils.rm_rf(cache_dir)
+    end
+
+    private
+
+    # Uses license_scout to collect the licenses for transitive dependencies
+    # into #{output_dir}/license-cache/#{software.name}
+    def collect_transitive_dependency_licenses_for(software)
+      # We collect the licenses of the transitive dependencies of this software
+      # with LicenseScout. We place these files under
+      # /opt/project-name/license-cache for them to be cached in git_cache. Once
+      # the build completes we will process these license files but we need to
+      # perform this step after build, before git_cache to be able to operate
+      # correctly with the git_cache.
+      license_output_dir = File.join(cache_dir, software.name)
+
+      collector = LicenseScout::Collector.new(
+        software.project.name,
+        software.project_dir,
+        license_output_dir,
+        LicenseScout::Options.new(
+          environment: software.with_embedded_path,
+          ruby_bin: software.embedded_bin("ruby")
+        )
+      )
+
+      begin
+        collector.run
+        collector.issue_report.each { |i| transitive_dependency_licensing_warning(i) }
+      rescue LicenseScout::Exceptions::UnsupportedProjectType => e
+        # Looks like this project is not supported by LicenseScout. Either the
+        # language and the dependency manager used by the project is not
+        # supported, or the software definition does not have any transitive
+        # dependencies.  In the latter case software definition should set
+        # 'skip_transitive_dependency_licensing' to 'true' to correct this
+        # error.
+        transitive_dependency_licensing_warning(<<-EOH)
+Software '#{software.name}' is not supported project type for transitive \
+dependency license collection. See https://github.com/chef/license_scout for \
+the list of supported languages and dependency managers. If this project does \
+not have any transitive dependencies, consider setting \
+'skip_transitive_dependency_licensing' to 'true' in order to correct this error.
+EOH
+      rescue LicenseScout::Exceptions::Error => e
+        transitive_dependency_licensing_warning(<<-EOH)
+Can not automatically detect licensing information for '#{software.name}' using \
+license_scout. Error is: '#{e}'
+EOH
+      rescue Exception => e
+        transitive_dependency_licensing_warning(<<-EOH)
+Unexpected error while running license_scout for '#{software.name}': '#{e}'
+EOH
+      end
+    end
+
+    # Collect the license files for the software.
+    def collect_licenses_for(software)
+      return nil if software.license == :project_license
+
+      software_name = software.name
+      license_data = license_map[software_name]
+      license_files = license_data[:license_files]
+
+      license_files.each do |license_file|
+        if license_file
+          output_file = license_package_location(software_name, license_file)
+
+          if local?(license_file)
+            input_file = File.expand_path(license_file, license_data[:project_dir])
+            if File.exist?(input_file)
+              FileUtils.cp(input_file, output_file)
+              File.chmod 0644, output_file unless windows?
+            else
+              licensing_warning("License file '#{input_file}' does not exist for software '#{software_name}'.")
+              # If we got here, we need to fail now so we don't take a git
+              # cache snapshot, or else the software build could be restored
+              # from cache without fixing the license issue.
+              raise_if_warnings_fatal!
+            end
+          else
+            begin
+              download_file!(license_file, output_file, enable_progress_bar: false)
+              File.chmod 0644, output_file unless windows?
+            rescue SocketError,
+                   Errno::ECONNREFUSED,
+                   Errno::ECONNRESET,
+                   Errno::ENETUNREACH,
+                   Timeout::Error,
+                   OpenURI::HTTPError,
+                   OpenSSL::SSL::SSLError
+              licensing_warning("Can not download license file '#{license_file}' for software '#{software_name}'.")
+              # If we got here, we need to fail now so we don't take a git
+              # cache snapshot, or else the software build could be restored
+              # from cache without fixing the license issue.
+              raise_if_warnings_fatal!
+            end
+          end
+        end
+      end
+    end
+
     STANDARD_LICENSES = [
       #
       # Below licenses are compiled based on https://opensource.org/licenses/alphabetical
@@ -392,4 +679,5 @@ module Omnibus
       "Chef-MLSA",     # https://www.chef.io/online-master-agreement/
     ].freeze
   end
+
 end

data/lib/omnibus/logger.rb

@@ -16,19 +16,22 @@
 
 module Omnibus
   class Logger
+
+    require "time"
+
     #
     # The amount of padding on the left column.
     #
     # @return [Fixnum]
     #
-    LEFT = 40
+    LEFT = 30
 
     #
     # Our custom log levels, in order of severity
     #
     # @return [Array]
     #
-    LEVELS = %w(UNKNOWN INTERNAL DEBUG INFO WARN ERROR FATAL NOTHING).freeze
+    LEVELS = %w{UNKNOWN INTERNAL DEBUG INFO WARN ERROR FATAL NOTHING}.freeze
 
     #
     # The mutex lock for synchronizing IO writing.
@@ -48,7 +51,7 @@ module Omnibus
     #
     def initialize(io = $stdout)
       @io = io
-      @level = LEVELS.index('WARN')
+      @level = LEVELS.index("WARN")
     end
 
     LEVELS.each.with_index do |level, index|
@@ -66,8 +69,8 @@ module Omnibus
     # @see (Logger#add)
     #
     def deprecated(progname, &block)
-      meta = Proc.new { "DEPRECATED: #{block.call}" }
-      add(LEVELS.index('WARN'), progname, &meta)
+      meta = Proc.new { "DEPRECATED: #{yield}" }
+      add(LEVELS.index("WARN"), progname, &meta)
     end
 
     #
@@ -135,7 +138,7 @@ module Omnibus
       else
         left = "#{format_severity(severity)} | "
       end
-      "#{left.rjust(LEFT)}#{message}\n"
+      "#{left.rjust(LEFT)}#{Time.now.iso8601()} | #{message}\n"
     end
 
     #
@@ -145,9 +148,9 @@ module Omnibus
     #
     def format_severity(severity)
       if severity == 0
-        '_'
+        "_"
       else
-        (LEVELS[severity] || '?')[0]
+        (LEVELS[severity] || "?")[0]
       end
     end
 
@@ -167,7 +170,7 @@ module Omnibus
       def initialize(log, level = :debug)
         @log = log
         @level = level
-        @buffer = ''
+        @buffer = ""
       end
 
       #

data/lib/omnibus/logging.rb

@@ -39,7 +39,7 @@ module Omnibus
       # @return [String]
       #
       def log_key
-        @log_key ||= (name || '(Anonymous)').split('::')[1..-1].join('::')
+        @log_key ||= (name || "(Anonymous)").split("::")[1..-1].join("::")
       end
     end
 

data/lib/omnibus/manifest.rb

@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 
-require 'ffi_yajl'
+require "ffi_yajl"
 
 module Omnibus
   class Manifest
@@ -27,7 +27,7 @@ module Omnibus
     LATEST_MANIFEST_FORMAT = 2
 
     attr_reader :build_version, :build_git_revision, :license
-    def initialize(version=nil, git_rev=nil, license="Unspecified")
+    def initialize(version = nil, git_rev = nil, license = "Unspecified")
       @data = {}
       @build_version = version
       @build_git_revision = git_rev
@@ -68,13 +68,13 @@ module Omnibus
     end
 
     def to_hash
-      software_hash = @data.inject({}) do |memo, (k,v)|
+      software_hash = @data.inject({}) do |memo, (k, v)|
         memo[k] = v.to_hash
         memo
       end
       ret = {
         manifest_format: LATEST_MANIFEST_FORMAT,
-        software: software_hash
+        software: software_hash,
       }
       ret[:build_version] = build_version if build_version
       ret[:build_git_revision] = build_git_revision if build_git_revision

data/lib/omnibus/manifest_diff.rb

@@ -27,8 +27,8 @@ module Omnibus
       @updated ||=
         begin
           (first.entry_names & second.entry_names).collect do |name|
-          diff(first.entry_for(name), second.entry_for(name))
-        end.compact
+            diff(first.entry_for(name), second.entry_for(name))
+          end.compact
         end
     end
 
@@ -36,8 +36,8 @@ module Omnibus
       @removed ||=
         begin
           (first.entry_names - second.entry_names).collect do |name|
-          removed_entry(first.entry_for(name))
-        end
+            removed_entry(first.entry_for(name))
+          end
         end
     end
 
@@ -45,8 +45,8 @@ module Omnibus
       @added ||=
         begin
           (second.entry_names - first.entry_names).collect do |name|
-          new_entry(second.entry_for(name))
-        end
+            new_entry(second.entry_for(name))
+          end
         end
     end
 
@@ -65,7 +65,6 @@ module Omnibus
         source: entry.locked_source }
     end
 
-
     def removed_entry(entry)
       { name: entry.name,
         old_version: entry.locked_version,

data/lib/omnibus/manifest_entry.rb

@@ -32,7 +32,7 @@ module Omnibus
         locked_source: @locked_source,
         source_type: @source_type,
         described_version: @described_version,
-        license: @license
+        license: @license,
       }
     end