chef 16.18.30 → 17.10.163

data/lib/chef/compliance/profile_collection.rb

@@ -0,0 +1,109 @@
+#
+# Copyright:: Copyright (c) Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require_relative "profile"
+
+class Chef
+  module Compliance
+    class ProfileCollection < Array
+
+      # Event dispatcher for this run.
+      #
+      # @return [Chef::EventDispatch::Dispatcher]
+      #
+      attr_reader :events
+
+      def initialize(events)
+        @events = events
+      end
+
+      # Add a profile to the profile collection.  The cookbook_name needs to be determined by the
+      # caller and is used in the `include_profile` API to match on.  The path should be the complete
+      # path on the host of the inspec.yml file, including the filename.
+      #
+      # @param path [String]
+      # @param cookbook_name [String]
+      #
+      def from_file(path, cookbook_name)
+        new_profile = Profile.from_file(events, path, cookbook_name)
+        self << new_profile
+        events&.compliance_profile_loaded(new_profile)
+      end
+
+      # @return [Boolean] if any of the profiles are enabled
+      #
+      def using_profiles?
+        any?(&:enabled?)
+      end
+
+      # @return [Array<Profile>] inspec profiles which are enabled in a form suitable to pass to inspec
+      #
+      def inspec_data
+        select(&:enabled?).each_with_object([]) { |profile, arry| arry << profile.inspec_data }
+      end
+
+      # DSL method to enable profile files.  This matches on the name of the profile being included it
+      # does not match on the filename of the input file.  If the specific profile is omitted then
+      # it uses the default profile. The string supports regular expression matching.
+      #
+      # @example Specific profile in a cookbook
+      #
+      # include_profile "acme_cookbook::ssh-001"
+      #
+      # @example The profile named "default" in a cookbook
+      #
+      # include_profile "acme_cookbook"
+      #
+      # @example Every profile in a cookbook
+      #
+      # include_profile "acme_cookbook::.*"
+      #
+      # @example Matching profiles by regexp in a cookbook
+      #
+      # include_profile "acme_cookbook::ssh.*"
+      #
+      # @example Matching profiles by regexp in any cookbook in the cookbook collection
+      #
+      # include_profile ".*::ssh.*"
+      #
+      def include_profile(arg)
+        (cookbook_name, profile_name) = arg.split("::")
+
+        profile_name = "default" if profile_name.nil?
+
+        profiles = select { |profile| /^#{cookbook_name}$/.match?(profile.cookbook_name) && /^#{profile_name}$/.match?(profile.pathname) }
+
+        if profiles.empty?
+          raise "No inspec profiles matching '#{profile_name}' found in cookbooks matching '#{cookbook_name}'"
+        end
+
+        profiles.each(&:enable!)
+      end
+
+      HIDDEN_IVARS = [ :@events ].freeze
+
+      # Omit the event object from error output
+      #
+      def inspect
+        ivar_string = (instance_variables.map(&:to_sym) - HIDDEN_IVARS).map do |ivar|
+          "#{ivar}=#{instance_variable_get(ivar).inspect}"
+        end.join(", ")
+        "#<#{self.class}:#{object_id} #{ivar_string}>"
+      end
+    end
+  end
+end

data/lib/chef/compliance/reporter/automate.rb

@@ -28,18 +28,28 @@ class Chef
           @token = Chef::Config[:data_collector][:token]
         end
 
-        # Method used in order to send the inspec report to the data_collector server
-        def send_report(report)
-          unless @entity_uuid && @run_id
-            Chef::Log.error "entity_uuid(#{@entity_uuid}) or run_id(#{@run_id}) can't be nil, not sending report to #{ChefUtils::Dist::Automate::PRODUCT}"
-            return false
+        def validate_config!
+          unless @entity_uuid
+            # TODO - this is a weird leakage of naming from the parent class
+            # but entity_uuid is never an attribute that the user can see;
+            # it is sourced from chef_guid, which we don't technically know about in this class -
+            # but telling the operator about a missing chef_guid is more helpful than telling
+            # them about a missing field they've never heard of. Better would be a dock link
+            # that described how to fix this situation.
+            raise "CMPL004: automate_reporter: chef_guid is not available and must be provided. Aborting because we cannot report the scan."
+          end
+
+          unless @run_id
+            raise "CMPL005: automate_reporter: run_id is not available, aborting because we cannot report the scan."
           end
 
           unless @url && @token
-            Chef::Log.warn "data_collector.token and data_collector.server_url must be defined in client.rb! Further information: https://docs.chef.io/chef_compliance_phase/#direct-reporting-to-chef-automate"
-            return false
+            raise "CMPL006: data_collector.token and data_collector.server_url must be configured in client.rb! Further information: https://docs.chef.io/automate/data_collection/#configure-your-chef-infra-client-to-send-data-to-chef-automate-without-chef-infra-server"
           end
+        end
 
+        # Method used in order to send the inspec report to the data_collector server
+        def send_report(report)
           headers = {
             "Content-Type" => "application/json",
             "x-data-collector-auth" => "version=1.0",
@@ -66,7 +76,7 @@ class Chef
 
           begin
             Chef::Log.info "Report to #{ChefUtils::Dist::Automate::PRODUCT}: #{@url}"
-            Chef::Log.debug "Compliance Report: #{json_report}"
+            Chef::Log.debug "Compliance Phase report: #{json_report}"
             http_client.post(nil, json_report, headers)
             true
           rescue => e

data/lib/chef/compliance/reporter/chef_server_automate.rb

@@ -30,11 +30,6 @@ class Chef
         end
 
         def send_report(report)
-          unless @entity_uuid && @run_id
-            Chef::Log.error "entity_uuid(#{@entity_uuid}) or run_id(#{@run_id}) can't be nil, not sending report to #{ChefUtils::Dist::Automate::PRODUCT}"
-            return false
-          end
-
           automate_report = truncate_controls_results(enriched_report(report), @control_results_limit)
 
           report_size = Chef::JSONCompat.to_json(automate_report, validate_utf8: false).bytesize
@@ -51,6 +46,16 @@ class Chef
           false
         end
 
+        def validate_config!
+          unless @entity_uuid
+            raise "CMPL007: chef_server_automate reporter: chef_guid is not available and must be provided. Aborting because we cannot report the scan"
+          end
+
+          unless @run_id
+            raise "CMPL008: chef_server_automate reporter: run_id is not available, aborting because we cannot report the scan."
+          end
+        end
+
         def http_client
           config = if @insecure
                      Chef::Config.merge(ssl_verify_mode: :verify_none)
@@ -80,7 +85,7 @@ class Chef
           when /404/
             Chef::Log.error "Object does not exist on remote server."
           when /413/
-            Chef::Log.error "You most likely hit the erchef request size in #{ChefUtils::Dist::Server::PRODUCT} that defaults to ~2MB. To increase this limit see the Compliance Phase troubleshooting documentation (http://docs.chef.io/chef_compliance_phase/#troubleshooting) or the Chef Infra Server configuration documentation (https://docs.chef.io/server/config_rb_server/)"
+            Chef::Log.error "You most likely hit the request size limit in #{ChefUtils::Dist::Server::PRODUCT} that defaults to ~2MB. To increase this limit see the Compliance Phase troubleshooting documentation (http://docs.chef.io/chef_compliance_phase/#troubleshooting) or the Chef Infra Server configuration documentation (https://docs.chef.io/server/config_rb_server/)"
           when /429/
             Chef::Log.error "This error typically means the data sent was larger than #{ChefUtils::Dist::Automate::PRODUCT}'s limit (4 MB). Run InSpec locally to identify any controls producing large diffs."
           end

data/lib/chef/compliance/reporter/cli.rb

@@ -0,0 +1,77 @@
+class Chef
+  module Compliance
+    module Reporter
+      class Cli
+        def send_report(report)
+          # iterate over each profile and control
+          output = ["\nCompliance report:"]
+          report[:profiles].each do |profile|
+            next if profile[:controls].nil?
+
+            output << " * #{profile[:title]}"
+            profile[:controls].each do |control|
+              next if control[:results].nil?
+
+              output << "#{" " * 6}#{control[:title]}"
+              control[:results].each do |result|
+                output << format_result(result)
+              end
+            end
+          end
+          output << "\n"
+          puts output.join("\n")
+        end
+
+        def validate_config!
+          true
+        end
+
+        private
+
+        # pastel.decorate is a lightweight replacement for highline.color
+        def pastel
+          @pastel ||= begin
+            require "pastel" unless defined?(Pastel)
+            Pastel.new
+          end
+        end
+
+        def format_result(result)
+          output = []
+          found = false
+          if result[:status] == "failed"
+            if result[:code_desc]
+              found = true
+              output << pastel.red("#{" " * 9}- #{result[:code_desc]}")
+            end
+            if result[:message]
+              if found
+                result[:message].split(/\n/).reject(&:empty?).each do |m|
+                  output << pastel.red("#{" " * 12}#{m}")
+                end
+              else
+                result[:message].split(/\n/).reject(&:empty?).each do |m|
+                  output << pastel.red("#{" " * 9}#{m}")
+                end
+              end
+              found = true
+            end
+            unless found
+              output << pastel.red("#{" " * 9}- #{result[:status]}")
+            end
+          else
+            found = false
+            if result[:code_desc]
+              found = true
+              output << pastel.green("#{" " * 9}+ #{result[:code_desc]}")
+            end
+            unless found
+              output << pastel.green("#{" " * 9}+ #{result[:status]}")
+            end
+          end
+          output
+        end
+      end
+    end
+  end
+end

data/lib/chef/compliance/reporter/compliance_enforcer.rb

@@ -1,7 +1,7 @@
 class Chef
   module Compliance
     module Reporter
-      class AuditEnforcer
+      class ComplianceEnforcer
         class ControlFailure < StandardError; end
 
         def send_report(report)
@@ -14,6 +14,10 @@ class Chef
           end
           true
         end
+
+        def validate_config!
+          true
+        end
       end
     end
   end

data/lib/chef/compliance/reporter/json_file.rb

@@ -1,4 +1,5 @@
 require_relative "../../json_compat"
+require_relative "../../log"
 
 class Chef
   module Compliance
@@ -9,10 +10,16 @@ class Chef
         end
 
         def send_report(report)
+          Chef::Log.info "Writing compliance report to #{@path}"
           FileUtils.mkdir_p(File.dirname(@path), mode: 0700)
-
           File.write(@path, Chef::JSONCompat.to_json(report))
         end
+
+        def validate_config!
+          if @path.nil? || @path.class != String || @path.empty?
+            raise "CMPL009: json_file reporter: node['audit']['json_file']['location'] must contain a file path"
+          end
+        end
       end
     end
   end

data/lib/chef/compliance/runner.rb

@@ -1,21 +1,24 @@
 autoload :Inspec, "inspec"
 
 require_relative "default_attributes"
-require_relative "reporter/automate"
-require_relative "reporter/chef_server_automate"
-require_relative "reporter/compliance_enforcer"
-require_relative "reporter/json_file"
 
 class Chef
   module Compliance
     class Runner < EventDispatch::Base
       extend Forwardable
 
+      SUPPORTED_REPORTERS = %w{chef-automate chef-server-automate json-file audit-enforcer compliance-enforcer cli}.freeze
+      SUPPORTED_FETCHERS = %w{chef-automate chef-server}.freeze
+
       attr_accessor :run_id
       attr_reader :node
+      attr_reader :run_context
+
       def_delegators :node, :logger
 
       def enabled?
+        return false if @node.nil?
+
         # Did we parse the libraries file from the audit cookbook?  This class dates back to when Chef Automate was
         # renamed from Chef Visibility in 2017, so should capture all modern versions of the audit cookbook.
         audit_cookbook_present = defined?(::Reporter::ChefAutomate)
@@ -24,7 +27,9 @@ class Chef
         logger.debug("#{self.class}##{__method__}: audit cookbook? #{audit_cookbook_present}")
         logger.debug("#{self.class}##{__method__}: compliance phase attr? #{node["audit"]["compliance_phase"]}")
 
-        if node["audit"]["compliance_phase"].nil?
+        if safe_profile_collection&.using_profiles?
+          true
+        elsif node["audit"]["compliance_phase"].nil?
           inspec_profiles.any? && !audit_cookbook_present
         else
           node["audit"]["compliance_phase"]
@@ -40,24 +45,44 @@ class Chef
         self.node = node
       end
 
+      # This hook gives us the run_context immediately after it is created so that we can wire up this object to it.
+      #
+      # (see EventDispatch::Base#)
+      #
+      def cookbook_compilation_start(run_context)
+        @run_context = run_context
+      end
+
       def run_started(run_status)
         self.run_id = run_status.run_id
       end
 
+      def converge_start(run_context)
+        # With all attributes - including cookbook - loaded, we now have enough data to validate
+        # configuration.  Because the converge is best coupled with the associated compliance run, these validations
+        # will raise (and abort the converge) if the compliance phase configuration is incorrect/will
+        # prevent compliance phase from completing and submitting its report to all configured reporters.
+        # can abort the converge if the compliance phase configuration (node attributes and client config)
+        load_and_validate!
+      end
+
       def run_completed(_node, _run_status)
         return unless enabled?
 
-        logger.info("#{self.class}##{__method__}: enabling Compliance Phase")
+        logger.debug("#{self.class}##{__method__}: enabling Compliance Phase")
 
-        report
+        report_with_interval
       end
 
       def run_failed(_exception, _run_status)
-        return unless enabled?
+        # If the run has failed because our own validation of compliance
+        # phase configuration has failed, we don't want to submit a report
+        # because we're still not configured correctly.
+        return unless enabled? && @validation_passed
 
-        logger.info("#{self.class}##{__method__}: enabling Compliance Phase")
+        logger.debug("#{self.class}##{__method__}: enabling Compliance Phase")
 
-        report
+        report_with_interval
       end
 
       ### Below code adapted from audit cookbook's files/default/handler/audit_report.rb
@@ -67,7 +92,6 @@ class Chef
         fail_if_not_present
         inspec_gem_source
         inspec_version
-        interval
         owner
         raise_if_unreachable
       }.freeze
@@ -81,7 +105,20 @@ class Chef
         end
       end
 
-      def report(report = generate_report)
+      def report_with_interval
+        if interval_seconds_left <= 0
+          create_timestamp_file if interval_enabled
+          report
+        else
+          logger.info "Skipping Chef Infra Compliance Phase due to interval settings (next run in #{interval_seconds_left / 60.0} mins)"
+        end
+      end
+
+      def report(report = nil)
+        logger.info "Starting Chef Infra Compliance Phase"
+        report ||= generate_report
+        # This is invoked at report-time instead of with the normal validations at node loaded,
+        # because we want to ensure that it is visible in the output - and not lost in back-scroll.
         warn_for_deprecated_config_values!
 
         if report.empty?
@@ -89,13 +126,32 @@ class Chef
           return
         end
 
-        Array(node["audit"]["reporter"]).each do |reporter|
-          send_report(reporter, report)
+        requested_reporters.each do |reporter_type|
+          logger.info "Reporting to #{reporter_type}"
+          @reporters[reporter_type].send_report(report)
+        end
+        logger.info "Chef Infra Compliance Phase Complete"
+      end
+
+      def inputs_from_attributes
+        if !node["audit"]["inputs"].empty?
+          node["audit"]["inputs"].to_h
+        else
+          node["audit"]["attributes"].to_h
         end
       end
 
+      def inputs_from_collection
+        safe_input_collection&.inspec_data || {}
+      end
+
+      def waivers_from_collection
+        safe_waiver_collection&.inspec_data || {}
+      end
+
       def inspec_opts
-        inputs = node["audit"]["attributes"].to_h
+        inputs = inputs_from_attributes.merge(inputs_from_collection).merge(waivers_from_collection)
+
         if node["audit"]["chef_node_attribute_enabled"]
           inputs["chef_node"] = node.to_h
           inputs["chef_node"]["chef_environment"] = node.chef_environment
@@ -105,26 +161,34 @@ class Chef
           backend_cache: node["audit"]["inspec_backend_cache"],
           inputs: inputs,
           logger: logger,
+          # output: STDOUT,
           output: node["audit"]["quiet"] ? ::File::NULL : STDOUT,
           report: true,
           reporter: ["json-automate"],
+          # reporter: ["cli"],
           reporter_backtrace_inclusion: node["audit"]["result_include_backtrace"],
           reporter_message_truncation: node["audit"]["result_message_limit"],
-          waiver_file: Array(node["audit"]["waiver_file"]),
+          waiver_file: waiver_files,
         }
       end
 
+      def waiver_files
+        Array(node["audit"]["waiver_file"])
+      end
+
       def inspec_profiles
         profiles = node["audit"]["profiles"]
-
-        # TODO: Custom exception class here?
         unless profiles.respond_to?(:map) && profiles.all? { |_, p| p.respond_to?(:transform_keys) && p.respond_to?(:update) }
-          raise "#{Inspec::Dist::PRODUCT_NAME} profiles specified in an unrecognized format, expected a hash of hashes."
+          raise "CMPL010: #{Inspec::Dist::PRODUCT_NAME} profiles specified in an unrecognized format, expected a hash of hashes."
         end
 
-        profiles.map do |name, profile|
+        from_attributes = profiles.map do |name, profile|
           profile.transform_keys(&:to_sym).update(name: name)
-        end
+        end || []
+
+        from_cookbooks = safe_profile_collection&.inspec_data || []
+
+        from_attributes + from_cookbooks
       end
 
       def load_fetchers!
@@ -135,8 +199,6 @@ class Chef
           require_relative "fetcher/chef_server"
         when nil
           # intentionally blank
-        else
-          raise "Invalid value specified for Compliance Phase's fetcher: '#{node["audit"]["fetcher"]}'. Valid values are 'chef-automate', 'chef-server', or nil."
         end
       end
 
@@ -156,7 +218,7 @@ class Chef
         logger.info "Running profiles from: #{profiles.inspect}"
         runner.run
         runner.report.tap do |r|
-          logger.debug "Compliance Report #{r}"
+          logger.debug "Compliance Phase report #{r}"
         end
       rescue Inspec::FetcherFailure => e
         failed_report("Cannot fetch all profiles: #{profiles}. Please make sure you're authenticated and the server is reachable. #{e.message}")
@@ -209,17 +271,10 @@ class Chef
         }
       end
 
-      def send_report(reporter_type, report)
-        logger.info "Reporting to #{reporter_type}"
-
-        reporter = reporter(reporter_type)
-
-        reporter.send_report(report) if reporter
-      end
-
       def reporter(reporter_type)
         case reporter_type
         when "chef-automate"
+          require_relative "reporter/automate"
           opts = {
             control_results_limit: node["audit"]["control_results_limit"],
             entity_uuid: node["chef_guid"],
@@ -230,6 +285,7 @@ class Chef
           }
           Chef::Compliance::Reporter::Automate.new(opts)
         when "chef-server-automate"
+          require_relative "reporter/chef_server_automate"
           opts = {
             control_results_limit: node["audit"]["control_results_limit"],
             entity_uuid: node["chef_guid"],
@@ -241,13 +297,15 @@ class Chef
           }
           Chef::Compliance::Reporter::ChefServerAutomate.new(opts)
         when "json-file"
-          path = node["audit"]["json_file"]["location"]
-          logger.info "Writing compliance report to #{path}"
+          require_relative "reporter/json_file"
+          path = node.dig("audit", "json_file", "location")
           Chef::Compliance::Reporter::JsonFile.new(file: path)
-        when "audit-enforcer"
+        when "audit-enforcer", "compliance-enforcer"
+          require_relative "reporter/compliance_enforcer"
           Chef::Compliance::Reporter::ComplianceEnforcer.new
-        else
-          raise "'#{reporter_type}' is not a supported reporter for Compliance Phase."
+        when "cli"
+          require_relative "reporter/cli"
+          Chef::Compliance::Reporter::Cli.new
         end
       end
 
@@ -264,6 +322,88 @@ class Chef
         url.path = File.join(url.path, "organizations/#{org}/data-collector")
         url
       end
+
+      # Load the resources required for this runner, and validate configuration
+      # is correct to proceed. Requires node state to be loaded.
+      # Will raise exception if fetcher is not valid, if a reporter is not valid,
+      # or the configuration required by a reporter is not provided.
+      def load_and_validate!
+        return unless enabled?
+
+        @reporters = {}
+        # Note that the docs don't say you can use an array, but our implementation
+        # supports it.
+        requested_reporters.each do |type|
+          unless SUPPORTED_REPORTERS.include? type
+            raise "CMPL003: '#{type}' found in node['audit']['reporter'] is not a supported reporter for Compliance Phase. Supported reporters are: #{SUPPORTED_REPORTERS.join(", ")}. For more information, see the documentation at https://docs.chef.io/chef_compliance_phase#reporters"
+          end
+
+          @reporters[type] = reporter(type)
+          @reporters[type].validate_config!
+        end
+
+        unless (fetcher = node["audit"]["fetcher"]).nil?
+          unless SUPPORTED_FETCHERS.include? fetcher
+            raise "CMPL002: Unrecognized Compliance Phase fetcher (node['audit']['fetcher'] = #{fetcher}). Supported fetchers are: #{SUPPORTED_FETCHERS.join(", ")}, or nil. For more information, see the documentation at https://docs.chef.io/chef_compliance_phase#fetch-profiles"
+          end
+        end
+
+        if !node["audit"]["attributes"].empty? && !node["audit"]["inputs"].empty?
+          raise "CMPL011: both node['audit']['inputs'] and node['audit']['attributes'] are set.  The node['audit']['attributes'] setting is deprecated and should not be used."
+        end
+
+        @validation_passed = true
+      end
+
+      def safe_profile_collection
+        run_context&.profile_collection
+      end
+
+      def safe_waiver_collection
+        run_context&.waiver_collection
+      end
+
+      def safe_input_collection
+        run_context&.input_collection
+      end
+
+      def requested_reporters
+        (Array(node["audit"]["reporter"]) + ["cli"]).uniq
+      end
+
+      def create_timestamp_file
+        FileUtils.touch report_timing_file
+      end
+
+      def report_timing_file
+        ::File.join(Chef::FileCache.create_cache_path("compliance"), "report_timing.json")
+      end
+
+      def interval_time
+        @interval_time ||= node.read("audit", "interval", "time")
+      end
+
+      def interval_enabled
+        @interval_enabled ||= node.read("audit", "interval", "enabled")
+      end
+
+      def interval_seconds
+        @interval_seconds ||=
+          if interval_enabled
+            logger.debug "Running Chef Infra Compliance Phase every #{interval_time} minutes"
+            interval_time * 60
+          else
+            logger.debug "Running Chef Infra Compliance Phase on every run"
+            0
+          end
+      end
+
+      def interval_seconds_left
+        return 0 unless ::File.exist?(report_timing_file)
+
+        seconds_since_last_run = Time.now - ::File.mtime(report_timing_file)
+        interval_seconds - seconds_since_last_run
+      end
     end
   end
 end