chef 16.4.41 → 16.8.9

data/lib/chef/compliance/reporter/chef_server_automate.rb

@@ -0,0 +1,92 @@
+require_relative "automate"
+
+class Chef
+  module Compliance
+    module Reporter
+      #
+      # Used to send inspec reports to Chef Automate server via Chef Server
+      #
+      class ChefServerAutomate < Chef::Compliance::Reporter::Automate
+        def initialize(opts)
+          @entity_uuid           = opts[:entity_uuid]
+          @run_id                = opts[:run_id]
+          @node_name             = opts[:node_info][:node]
+          @insecure              = opts[:insecure]
+          @environment           = opts[:node_info][:environment]
+          @roles                 = opts[:node_info][:roles]
+          @recipes               = opts[:node_info][:recipes]
+          @url                   = opts[:url]
+          @chef_tags             = opts[:node_info][:chef_tags]
+          @policy_group          = opts[:node_info][:policy_group]
+          @policy_name           = opts[:node_info][:policy_name]
+          @source_fqdn           = opts[:node_info][:source_fqdn]
+          @organization_name     = opts[:node_info][:organization_name]
+          @ipaddress             = opts[:node_info][:ipaddress]
+          @fqdn                  = opts[:node_info][:fqdn]
+          @control_results_limit = opts[:control_results_limit]
+          @timestamp             = opts.fetch(:timestamp) { Time.now }
+        end
+
+        def send_report(report)
+          unless @entity_uuid && @run_id
+            Chef::Log.error "entity_uuid(#{@entity_uuid}) or run_id(#{@run_id}) can't be nil, not sending report to #{ChefUtils::Dist::Automate::PRODUCT}"
+            return false
+          end
+
+          automate_report = truncate_controls_results(enriched_report(report), @control_results_limit)
+
+          report_size = Chef::JSONCompat.to_json(automate_report, validate_utf8: false).bytesize
+          # this is set to slightly less than the oc_erchef limit
+          if report_size > 900 * 1024
+            Chef::Log.warn "Generated report size is #{(report_size / (1024 * 1024.0)).round(2)} MB. #{ChefUtils::Dist::Server::PRODUCT} < 13.0 defaults to a limit of ~1MB, 13.0+ defaults to a limit of ~2MB."
+          end
+
+          Chef::Log.info "Report to #{ChefUtils::Dist::Automate::PRODUCT} via #{ChefUtils::Dist::Server::PRODUCT}: #{@url}"
+          with_http_rescue do
+            http_client.post(@url, automate_report)
+            return true
+          end
+          false
+        end
+
+        def http_client
+          config = if @insecure
+                     Chef::Config.merge(ssl_verify_mode: :verify_none)
+                   else
+                     Chef::Config
+                   end
+
+          Chef::ServerAPI.new(@url, config)
+        end
+
+        def with_http_rescue
+          response = yield
+          if response.respond_to?(:code)
+            # handle non 200 error codes, they are not raised as Net::HTTPClientException
+            handle_http_error_code(response.code) if response.code.to_i >= 300
+          end
+          response
+        rescue Net::HTTPClientException => e
+          Chef::Log.error e
+          handle_http_error_code(e.response.code)
+        end
+
+        def handle_http_error_code(code)
+          case code
+          when /401|403/
+            Chef::Log.error "Auth issue: see audit cookbook TROUBLESHOOTING.md"
+          when /404/
+            Chef::Log.error "Object does not exist on remote server."
+          when /413/
+            Chef::Log.error "You most likely hit the erchef request size in #{ChefUtils::Dist::Server::PRODUCT} that defaults to ~2MB. To increase this limit see audit cookbook TROUBLESHOOTING.md OR https://docs.chef.io/config_rb_server.html"
+          when /429/
+            Chef::Log.error "This error typically means the data sent was larger than #{ChefUtils::Dist::Automate::PRODUCT}'s limit (4 MB). Run InSpec locally to identify any controls producing large diffs."
+          end
+          msg = "Received HTTP error #{code}"
+          Chef::Log.error msg
+          raise msg
+        end
+      end
+    end
+  end
+end

data/lib/chef/compliance/reporter/compliance_enforcer.rb

@@ -0,0 +1,20 @@
+class Chef
+  module Compliance
+    module Reporter
+      class AuditEnforcer
+        class ControlFailure < StandardError; end
+
+        def send_report(report)
+          report.fetch(:profiles, []).each do |profile|
+            profile.fetch(:controls, []).each do |control|
+              control.fetch(:results, []).each do |result|
+                raise ControlFailure, "Audit #{control[:id]} has failed. Aborting #{ChefUtils::Dist::Infra::CLIENT} run." if result[:status] == "failed"
+              end
+            end
+          end
+          true
+        end
+      end
+    end
+  end
+end

data/lib/chef/compliance/reporter/json_file.rb

@@ -0,0 +1,19 @@
+require_relative "../../json_compat"
+
+class Chef
+  module Compliance
+    module Reporter
+      class JsonFile
+        def initialize(opts)
+          @path = opts.fetch(:file)
+        end
+
+        def send_report(report)
+          FileUtils.mkdir_p(File.dirname(@path), mode: 0700)
+
+          File.write(@path, Chef::JSONCompat.to_json(report))
+        end
+      end
+    end
+  end
+end

data/lib/chef/compliance/runner.rb

@@ -0,0 +1,250 @@
+autoload :Inspec, "inspec"
+
+require_relative "default_attributes"
+require_relative "reporter/automate"
+require_relative "reporter/chef_server_automate"
+require_relative "reporter/compliance_enforcer"
+require_relative "reporter/json_file"
+
+class Chef
+  module Compliance
+    class Runner < EventDispatch::Base
+      extend Forwardable
+
+      attr_accessor :run_id, :recipes
+      attr_reader :node
+      def_delegators :node, :logger
+
+      def enabled?
+        audit_cookbook_present = recipes.include?("audit::default")
+
+        logger.info("#{self.class}##{__method__}: #{Inspec::Dist::PRODUCT_NAME} profiles? #{inspec_profiles.any?}")
+        logger.info("#{self.class}##{__method__}: audit cookbook? #{audit_cookbook_present}")
+
+        inspec_profiles.any? && !audit_cookbook_present
+      end
+
+      def node=(node)
+        @node = node
+        node.default["audit"] = Chef::Compliance::DEFAULT_ATTRIBUTES.merge(node.default["audit"])
+      end
+
+      def node_load_completed(node, _expanded_run_list, _config)
+        self.node = node
+      end
+
+      def run_started(run_status)
+        self.run_id = run_status.run_id
+      end
+
+      def run_list_expanded(run_list_expansion)
+        self.recipes = run_list_expansion.recipes
+      end
+
+      def run_completed(_node, _run_status)
+        return unless enabled?
+
+        logger.info("#{self.class}##{__method__}: enabling Compliance Phase")
+
+        report
+      end
+
+      def run_failed(_exception, _run_status)
+        return unless enabled?
+
+        logger.info("#{self.class}##{__method__}: enabling Compliance Phase")
+
+        report
+      end
+
+      ### Below code adapted from audit cookbook's files/default/handler/audit_report.rb
+
+      DEPRECATED_CONFIG_VALUES = %w{
+        attributes_save
+        chef_node_attribute_enabled
+        fail_if_not_present
+        inspec_gem_source
+        inspec_version
+        interval
+        owner
+        raise_if_unreachable
+      }.freeze
+
+      def warn_for_deprecated_config_values!
+        deprecated_config_values = (node["audit"].keys & DEPRECATED_CONFIG_VALUES)
+
+        if deprecated_config_values.any?
+          values = deprecated_config_values.sort.map { |v| "'#{v}'" }.join(", ")
+          logger.warn "audit cookbook config values #{values} are not supported in #{ChefUtils::Dist::Infra::PRODUCT}'s Compliance Phase."
+        end
+      end
+
+      def report(report = generate_report)
+        warn_for_deprecated_config_values!
+
+        if report.empty?
+          logger.error "Compliance report was not generated properly, skipped reporting"
+          return
+        end
+
+        Array(node["audit"]["reporter"]).each do |reporter|
+          send_report(reporter, report)
+        end
+      end
+
+      def inspec_opts
+        {
+          backend_cache: node["audit"]["inspec_backend_cache"],
+          inputs: node["audit"]["attributes"],
+          logger: logger,
+          output: node["audit"]["quiet"] ? ::File::NULL : STDOUT,
+          report: true,
+          reporter: ["json-automate"],
+          reporter_backtrace_inclusion: node["audit"]["result_include_backtrace"],
+          reporter_message_truncation: node["audit"]["result_message_limit"],
+          waiver_file: Array(node["audit"]["waiver_file"]),
+        }
+      end
+
+      def inspec_profiles
+        profiles = node["audit"]["profiles"]
+
+        # TODO: Custom exception class here?
+        unless profiles.respond_to?(:map) && profiles.all? { |_, p| p.respond_to?(:transform_keys) && p.respond_to?(:update) }
+          raise "#{Inspec::Dist::PRODUCT_NAME} profiles specified in an unrecognized format, expected a hash of hashes."
+        end
+
+        profiles.map do |name, profile|
+          profile.transform_keys(&:to_sym).update(name: name)
+        end
+      end
+
+      def load_fetchers!
+        case node["audit"]["fetcher"]
+        when "chef-automate"
+          require_relative "fetcher/automate"
+        when "chef-server"
+          require_relative "fetcher/chef_server"
+        when nil
+          # intentionally blank
+        else
+          raise "Invalid value specified for Compliance Phase's fetcher: '#{node["audit"]["fetcher"]}'. Valid values are 'chef-automate', 'chef-server', or nil."
+        end
+      end
+
+      def generate_report(opts: inspec_opts, profiles: inspec_profiles)
+        load_fetchers!
+
+        logger.debug "Options are set to: #{opts}"
+        runner = ::Inspec::Runner.new(opts)
+
+        if profiles.empty?
+          failed_report("No #{Inspec::Dist::PRODUCT_NAME} profiles are defined.")
+          return
+        end
+
+        profiles.each { |target| runner.add_target(target) }
+
+        logger.info "Running profiles from: #{profiles.inspect}"
+        runner.run
+        runner.report.tap do |r|
+          logger.debug "Compliance Report #{r}"
+        end
+      rescue Inspec::FetcherFailure => e
+        failed_report("Cannot fetch all profiles: #{profiles}. Please make sure you're authenticated and the server is reachable. #{e.message}")
+      rescue => e
+        failed_report(e.message)
+      end
+
+      # In case InSpec raises a runtime exception without providing a valid report,
+      # we make one up and add two new fields to it: `status` and `status_message`
+      def failed_report(err)
+        logger.error "#{Inspec::Dist::PRODUCT_NAME} has raised a runtime exception. Generating a minimal failed report."
+        logger.error err
+        {
+          "platform": {
+            "name": "unknown",
+            "release": "unknown",
+          },
+          "profiles": [],
+          "statistics": {
+            "duration": 0.0000001,
+          },
+          "version": Inspec::VERSION,
+          "status": "failed",
+          "status_message": err,
+        }
+      end
+
+      # extracts relevant node data
+      def node_info
+        runlist_roles = node.run_list.select { |item| item.type == :role }.map(&:name)
+        runlist_recipes = node.run_list.select { |item| item.type == :recipe }.map(&:name)
+        {
+          node: node.name,
+          os: {
+            release: node["platform_version"],
+            family: node["platform"],
+          },
+          environment: node.environment,
+          roles: runlist_roles,
+          recipes: runlist_recipes,
+          policy_name: node.policy_name || "",
+          policy_group: node.policy_group || "",
+          chef_tags: node.tags,
+          organization_name: chef_server_uri.path.split("/").last || "",
+          source_fqdn: chef_server_uri.host || "",
+          ipaddress: node["ipaddress"],
+          fqdn: node["fqdn"],
+        }
+      end
+
+      def send_report(reporter, report)
+        logger.info "Reporting to #{reporter}"
+
+        insecure = node["audit"]["insecure"]
+        run_time_limit = node["audit"]["run_time_limit"]
+        control_results_limit = node["audit"]["control_results_limit"]
+
+        case reporter
+        when "chef-automate"
+          opts = {
+            entity_uuid: node["chef_guid"],
+            run_id: run_id,
+            node_info: node_info,
+            insecure: insecure,
+            run_time_limit: run_time_limit,
+            control_results_limit: control_results_limit,
+          }
+          Chef::Compliance::Reporter::Automate.new(opts).send_report(report)
+        when "chef-server-automate"
+          chef_url = node["audit"]["server"] || base_chef_server_url
+          chef_org = Chef::Config[:chef_server_url].split("/").last
+          if chef_url
+            url = construct_url(chef_url, File.join("organizations", chef_org, "data-collector"))
+            opts = {
+              entity_uuid: node["chef_guid"],
+              run_id: run_id,
+              node_info: node_info,
+              insecure: insecure,
+              url: url,
+              run_time_limit: run_time_limit,
+              control_results_limit: control_results_limit,
+            }
+            Chef::Compliance::Reporter::ChefServer.new(opts).send_report(report)
+          else
+            logger.warn "Unable to determine #{ChefUtils::Dist::Server::PRODUCT} url required by #{Inspec::Dist::PRODUCT_NAME} report collector '#{reporter}'. Skipping..."
+          end
+        when "json-file"
+          path = node["audit"]["json_file"]["location"]
+          logger.info "Writing compliance report to #{path}"
+          Chef::Compliance::Reporter::JsonFile.new(file: path).send_report(report)
+        when "audit-enforcer"
+          Chef::Compliance::Reporter::ComplianceEnforcer.new.send_report(report)
+        else
+          logger.warn "#{reporter} is not a supported #{Inspec::Dist::PRODUCT_NAME} report collector"
+        end
+      end
+    end
+  end
+end

data/lib/chef/cookbook/cookbook_version_loader.rb

@@ -17,7 +17,7 @@ require_relative "../cookbook_version"
 require_relative "chefignore"
 require_relative "metadata"
 require_relative "../util/path_helper"
-require "find"
+require "find" unless defined?(Find.find)
 
 class Chef
   class Cookbook

data/lib/chef/cookbook/gem_installer.rb

@@ -47,7 +47,7 @@ class Chef
                 v2
               end
             end
-            cookbook_gems[args.first] += args[1..-1]
+            cookbook_gems[args.first] += args[1..]
           end
         end
 

data/lib/chef/cookbook/synchronizer.rb

@@ -17,7 +17,7 @@ require_relative "../client"
 require_relative "../util/threaded_job_queue"
 require_relative "../server_api"
 require "singleton" unless defined?(Singleton)
-require_relative "../dist"
+require "chef-utils/dist" unless defined?(ChefUtils::Dist)
 
 class Chef
 
@@ -65,7 +65,7 @@ class Chef
         # manifest.
         cache.find(File.join(%w{cookbooks ** {*,.*}})).each do |cache_filename|
           unless @valid_cache_entries[cache_filename]
-            Chef::Log.info("Removing #{cache_filename} from the cache; it is no longer needed by #{Chef::Dist::CLIENT}.")
+            Chef::Log.info("Removing #{cache_filename} from the cache; it is no longer needed by #{ChefUtils::Dist::Infra::CLIENT}.")
             cache.delete(cache_filename)
           end
         end

data/lib/chef/cookbook_manifest.rb

@@ -282,7 +282,7 @@ class Chef
 
         name = File.join(segment, pathname.basename.to_s)
 
-        if segment == "templates" || segment == "files"
+        if %w{templates files}.include?(segment)
           # Check if pathname looks like files/foo or templates/foo (unscoped)
           if pathname.each_filename.to_a.length == 2
             # Use root_default in case the same path exists at root_default and default
@@ -317,6 +317,7 @@ class Chef
     end
 
   end
+
   class CookbookManifestVersions
 
     extend Chef::Mixin::VersionedAPIFactory

data/lib/chef/cookbook_site_streaming_uploader.rb

@@ -18,11 +18,17 @@
 # limitations under the License.
 #
 
-require "uri" unless defined?(URI)
-require "net/http" unless defined?(Net::HTTP)
-require "mixlib/authentication/signedheaderauth"
-require "openssl" unless defined?(OpenSSL)
-require_relative "dist"
+autoload :URI, "uri"
+module Net
+  autoload :HTTP, "net/http"
+end
+autoload :OpenSSL, "openssl"
+module Mixlib
+  module Authentication
+    autoload :SignedHeaderAuth, "mixlib/authentication/signedheaderauth"
+  end
+end
+require "chef-utils/dist" unless defined?(ChefUtils::Dist)
 
 class Chef
   # == Chef::CookbookSiteStreamingUploader
@@ -37,7 +43,7 @@ class Chef
     class << self
 
       def create_build_dir(cookbook)
-        tmp_cookbook_path = Tempfile.new("#{Chef::Dist::SHORT}-#{cookbook.name}-build")
+        tmp_cookbook_path = Tempfile.new("#{ChefUtils::Dist::Infra::SHORT}-#{cookbook.name}-build")
         tmp_cookbook_path.close
         tmp_cookbook_dir = tmp_cookbook_path.path
         File.unlink(tmp_cookbook_dir)
@@ -225,11 +231,7 @@ class Chef
           @part_no += 1
           @part_offset = 0
           next_part = read(how_much_next_part)
-          result = current_part + if next_part
-                                    next_part
-                                  else
-                                    ""
-                                  end
+          result = current_part + (next_part || "")
         else
           @part_offset += how_much_current_part
           result = current_part