chef 16.6.14-universal-mingw32 → 16.9.17-universal-mingw32

data/lib/chef/compliance/reporter/automate.rb

@@ -0,0 +1,201 @@
+class Chef
+  module Compliance
+    module Reporter
+      #
+      # Used to send inspec reports to Chef Automate via the data_collector service
+      #
+      class Automate
+        def initialize(opts)
+          @entity_uuid           = opts[:entity_uuid]
+          @run_id                = opts[:run_id]
+          @node_name             = opts[:node_info][:node]
+          @environment           = opts[:node_info][:environment]
+          @roles                 = opts[:node_info][:roles]
+          @recipes               = opts[:node_info][:recipes]
+          @insecure              = opts[:insecure]
+          @chef_tags             = opts[:node_info][:chef_tags]
+          @policy_group          = opts[:node_info][:policy_group]
+          @policy_name           = opts[:node_info][:policy_name]
+          @source_fqdn           = opts[:node_info][:source_fqdn]
+          @organization_name     = opts[:node_info][:organization_name]
+          @ipaddress             = opts[:node_info][:ipaddress]
+          @fqdn                  = opts[:node_info][:fqdn]
+          @run_time_limit        = opts[:run_time_limit]
+          @control_results_limit = opts[:control_results_limit]
+          @timestamp             = opts.fetch(:timestamp) { Time.now }
+
+          @url = Chef::Config[:data_collector][:server_url]
+          @token = Chef::Config[:data_collector][:token]
+        end
+
+        # Method used in order to send the inspec report to the data_collector server
+        def send_report(report)
+          unless @entity_uuid && @run_id
+            Chef::Log.error "entity_uuid(#{@entity_uuid}) or run_id(#{@run_id}) can't be nil, not sending report to #{ChefUtils::Dist::Automate::PRODUCT}"
+            return false
+          end
+
+          unless @url && @token
+            Chef::Log.warn "data_collector.token and data_collector.server_url must be defined in client.rb! Further information: https://docs.chef.io/chef_compliance_phase/#direct-reporting-to-chef-automate"
+            return false
+          end
+
+          headers = {
+            "Content-Type" => "application/json",
+            "x-data-collector-auth" => "version=1.0",
+            "x-data-collector-token" => @token,
+          }
+
+          all_report_shas = report[:profiles].map { |p| p[:sha256] }
+          missing_report_shas = missing_automate_profiles(headers, all_report_shas)
+
+          full_report = truncate_controls_results(enriched_report(report), @control_results_limit)
+          full_report = strip_profiles_meta(full_report, missing_report_shas, @run_time_limit)
+          json_report = Chef::JSONCompat.to_json(full_report, validate_utf8: false)
+
+          # Automate GRPC currently has a message limit of ~4MB
+          # https://github.com/chef/automate/issues/1417#issuecomment-541908157
+          if json_report.bytesize > 4 * 1024 * 1024
+            Chef::Log.warn "Generated report size is #{(json_report.bytesize / (1024 * 1024.0)).round(2)} MB. #{ChefUtils::Dist::Automate::PRODUCT} has an internal 4MB limit that is not currently configurable."
+          end
+
+          unless json_report
+            Chef::Log.warn "Something went wrong, report can't be nil"
+            return false
+          end
+
+          begin
+            Chef::Log.info "Report to #{ChefUtils::Dist::Automate::PRODUCT}: #{@url}"
+            Chef::Log.debug "Compliance Report: #{json_report}"
+            http_client.post(nil, json_report, headers)
+            true
+          rescue => e
+            Chef::Log.error "send_report: POST to #{@url} returned: #{e.message}"
+            false
+          end
+        end
+
+        def http_client(url = @url)
+          if @insecure
+            Chef::HTTP.new(url, ssl_verify_mode: :verify_none)
+          else
+            Chef::HTTP.new(url)
+          end
+        end
+
+        def enriched_report(final_report)
+          final_report[:profiles].compact!
+
+          # Label this content as an inspec_report
+          final_report[:type] = "inspec_report"
+
+          final_report[:node_name]         = @node_name
+          final_report[:end_time]          = @timestamp.utc.strftime("%FT%TZ")
+          final_report[:node_uuid]         = @entity_uuid
+          final_report[:environment]       = @environment
+          final_report[:roles]             = @roles
+          final_report[:recipes]           = @recipes
+          final_report[:report_uuid]       = @run_id
+          final_report[:source_fqdn]       = @source_fqdn
+          final_report[:organization_name] = @organization_name
+          final_report[:policy_group]      = @policy_group
+          final_report[:policy_name]       = @policy_name
+          final_report[:chef_tags]         = @chef_tags
+          final_report[:ipaddress]         = @ipaddress
+          final_report[:fqdn]              = @fqdn
+
+          final_report
+        end
+
+        CONTROL_RESULT_SORT_ORDER = %w{ failed skipped passed }.freeze
+
+        # Truncates the number of results per control in the report when they exceed max_results.
+        # The truncation prioritizes failed and skipped results over passed ones.
+        # Controls where results have been truncated will get a new object 'removed_results_counts'
+        # with the status counts of the truncated results
+        def truncate_controls_results(report, max_results)
+          return report unless max_results.is_a?(Integer) && max_results > 0
+
+          report.fetch(:profiles, []).each do |profile|
+            profile.fetch(:controls, []).each do |control|
+              # Only bother with truncation if the number of results exceed max_results
+              next unless control[:results].length > max_results
+
+              res = control[:results]
+              res.sort_by! { |r| CONTROL_RESULT_SORT_ORDER.index(r[:status]) }
+
+              # Count the results that will be truncated
+              truncated = { failed: 0, skipped: 0, passed: 0 }
+              (max_results..res.length - 1).each do |i|
+                case res[i][:status]
+                when "failed"
+                  truncated[:failed] += 1
+                when "skipped"
+                  truncated[:skipped] += 1
+                when "passed"
+                  truncated[:passed] += 1
+                end
+              end
+              # Truncate the results array now
+              control[:results] = res[0..max_results - 1]
+              control[:removed_results_counts] = truncated
+            end
+          end
+          report
+        end
+
+        # Contacts the metasearch Automate API to check which of the inspec profile sha256 ids
+        # passed in via `report_shas` are missing from the Automate profiles metadata database.
+        def missing_automate_profiles(headers, report_shas)
+          Chef::Log.debug "Checking the #{ChefUtils::Dist::Automate::PRODUCT} profiles metadata for: #{report_shas}"
+          meta_url = URI(@url)
+          meta_url.path = "/compliance/profiles/metasearch"
+          response_str = http_client(meta_url.to_s).post(nil, "{\"sha256\": #{report_shas}}", headers)
+          missing_shas = Chef::JSONCompat.parse(response_str)["missing_sha256"]
+          unless missing_shas.empty?
+            Chef::Log.info "#{ChefUtils::Dist::Automate::PRODUCT} is missing metadata for the following profile ids: #{missing_shas}"
+          end
+          missing_shas
+        rescue => e
+          Chef::Log.error "missing_automate_profiles error: #{e.message}"
+          # If we get an error it's safer to assume none of the profile shas exist in Automate
+          report_shas
+        end
+
+        # Profile 'name' is a required property.
+        # By not sending it in the report, we make it clear to the ingestion backend that the profile metadata has been stripped from this profile in the report.
+        # Profile 'title' and 'version' are still kept for troubleshooting purposes in the backend.
+        SEEN_PROFILE_UNNECESSARY_FIELDS = %i{ copyright copyright_email groups license maintainer name summary supports}.freeze
+
+        SEEN_PROFILE_UNNECESSARY_CONTROL_FIELDS = %i{ code desc descriptions impact refs source_location tags title }.freeze
+
+        # TODO: This mutates the report and probably doesn't need to.
+        def strip_profiles_meta(report, missing_report_shas, run_time_limit)
+          report[:profiles].each do |p|
+            next if missing_report_shas.include?(p[:sha256])
+
+            p.delete_if { |f| SEEN_PROFILE_UNNECESSARY_FIELDS.include?(f) }
+
+            next unless p[:controls].is_a?(Array)
+
+            p[:controls].each do |c|
+              c.delete_if { |f| SEEN_PROFILE_UNNECESSARY_CONTROL_FIELDS.include?(f) }
+              c.delete(:waiver_data) if c[:waiver_data] == {}
+
+              next unless c[:results].is_a?(Array)
+
+              c[:results].each do |r|
+                if r[:run_time].is_a?(Float) && r[:run_time] < run_time_limit
+                  r.delete(:start_time)
+                  r.delete(:run_time)
+                end
+              end
+            end
+          end
+          report[:run_time_limit] = run_time_limit
+          report
+        end
+      end
+    end
+  end
+end

data/lib/chef/compliance/reporter/chef_server_automate.rb

@@ -0,0 +1,94 @@
+require_relative "automate"
+
+class Chef
+  module Compliance
+    module Reporter
+      #
+      # Used to send inspec reports to Chef Automate server via Chef Server
+      #
+      class ChefServerAutomate < Chef::Compliance::Reporter::Automate
+        attr_reader :url
+
+        def initialize(opts)
+          @entity_uuid           = opts[:entity_uuid]
+          @run_id                = opts[:run_id]
+          @node_name             = opts[:node_info][:node]
+          @insecure              = opts[:insecure]
+          @environment           = opts[:node_info][:environment]
+          @roles                 = opts[:node_info][:roles]
+          @recipes               = opts[:node_info][:recipes]
+          @url                   = opts[:url]
+          @chef_tags             = opts[:node_info][:chef_tags]
+          @policy_group          = opts[:node_info][:policy_group]
+          @policy_name           = opts[:node_info][:policy_name]
+          @source_fqdn           = opts[:node_info][:source_fqdn]
+          @organization_name     = opts[:node_info][:organization_name]
+          @ipaddress             = opts[:node_info][:ipaddress]
+          @fqdn                  = opts[:node_info][:fqdn]
+          @control_results_limit = opts[:control_results_limit]
+          @timestamp             = opts.fetch(:timestamp) { Time.now }
+        end
+
+        def send_report(report)
+          unless @entity_uuid && @run_id
+            Chef::Log.error "entity_uuid(#{@entity_uuid}) or run_id(#{@run_id}) can't be nil, not sending report to #{ChefUtils::Dist::Automate::PRODUCT}"
+            return false
+          end
+
+          automate_report = truncate_controls_results(enriched_report(report), @control_results_limit)
+
+          report_size = Chef::JSONCompat.to_json(automate_report, validate_utf8: false).bytesize
+          # this is set to slightly less than the oc_erchef limit
+          if report_size > 900 * 1024
+            Chef::Log.warn "Generated report size is #{(report_size / (1024 * 1024.0)).round(2)} MB. #{ChefUtils::Dist::Server::PRODUCT} < 13.0 defaults to a limit of ~1MB, 13.0+ defaults to a limit of ~2MB."
+          end
+
+          Chef::Log.info "Report to #{ChefUtils::Dist::Automate::PRODUCT} via #{ChefUtils::Dist::Server::PRODUCT}: #{@url}"
+          with_http_rescue do
+            http_client.post(@url, automate_report)
+            return true
+          end
+          false
+        end
+
+        def http_client
+          config = if @insecure
+                     Chef::Config.merge(ssl_verify_mode: :verify_none)
+                   else
+                     Chef::Config
+                   end
+
+          Chef::ServerAPI.new(@url, config)
+        end
+
+        def with_http_rescue
+          response = yield
+          if response.respond_to?(:code)
+            # handle non 200 error codes, they are not raised as Net::HTTPClientException
+            handle_http_error_code(response.code) if response.code.to_i >= 300
+          end
+          response
+        rescue Net::HTTPClientException => e
+          Chef::Log.error e
+          handle_http_error_code(e.response.code)
+        end
+
+        def handle_http_error_code(code)
+          case code
+          when /401|403/
+            Chef::Log.error "Auth issue: see the Compliance Phase troubleshooting documentation (http://docs.chef.io/chef_compliance_phase/#troubleshooting)."
+          when /404/
+            Chef::Log.error "Object does not exist on remote server."
+          when /413/
+            Chef::Log.error "You most likely hit the erchef request size in #{ChefUtils::Dist::Server::PRODUCT} that defaults to ~2MB. To increase this limit see the Compliance Phase troubleshooting documentation (http://docs.chef.io/chef_compliance_phase/#troubleshooting) or the Chef Infra Server configuration documentation (https://docs.chef.io/server/config_rb_server/)"
+          when /429/
+            Chef::Log.error "This error typically means the data sent was larger than #{ChefUtils::Dist::Automate::PRODUCT}'s limit (4 MB). Run InSpec locally to identify any controls producing large diffs."
+          end
+          msg = "Received HTTP error #{code}"
+          Chef::Log.error msg
+          raise msg
+        end
+      end
+    end
+  end
+end

data/lib/chef/compliance/reporter/compliance_enforcer.rb

@@ -0,0 +1,20 @@
+class Chef
+  module Compliance
+    module Reporter
+      class AuditEnforcer
+        class ControlFailure < StandardError; end
+
+        def send_report(report)
+          report.fetch(:profiles, []).each do |profile|
+            profile.fetch(:controls, []).each do |control|
+              control.fetch(:results, []).each do |result|
+                raise ControlFailure, "Audit #{control[:id]} has failed. Aborting #{ChefUtils::Dist::Infra::CLIENT} run." if result[:status] == "failed"
+              end
+            end
+          end
+          true
+        end
+      end
+    end
+  end
+end

data/lib/chef/compliance/reporter/json_file.rb

@@ -0,0 +1,19 @@
+require_relative "../../json_compat"
+
+class Chef
+  module Compliance
+    module Reporter
+      class JsonFile
+        def initialize(opts)
+          @path = opts.fetch(:file)
+        end
+
+        def send_report(report)
+          FileUtils.mkdir_p(File.dirname(@path), mode: 0700)
+
+          File.write(@path, Chef::JSONCompat.to_json(report))
+        end
+      end
+    end
+  end
+end

data/lib/chef/compliance/runner.rb

@@ -0,0 +1,266 @@
+autoload :Inspec, "inspec"
+
+require_relative "default_attributes"
+require_relative "reporter/automate"
+require_relative "reporter/chef_server_automate"
+require_relative "reporter/compliance_enforcer"
+require_relative "reporter/json_file"
+
+class Chef
+  module Compliance
+    class Runner < EventDispatch::Base
+      extend Forwardable
+
+      attr_accessor :run_id, :recipes
+      attr_reader :node
+      def_delegators :node, :logger
+
+      def enabled?
+        audit_cookbook_present = recipes.include?("audit::default")
+
+        logger.info("#{self.class}##{__method__}: #{Inspec::Dist::PRODUCT_NAME} profiles? #{inspec_profiles.any?}")
+        logger.info("#{self.class}##{__method__}: audit cookbook? #{audit_cookbook_present}")
+
+        inspec_profiles.any? && !audit_cookbook_present
+      end
+
+      def node=(node)
+        @node = node
+        node.default["audit"] = Chef::Compliance::DEFAULT_ATTRIBUTES.merge(node.default["audit"])
+      end
+
+      def node_load_completed(node, _expanded_run_list, _config)
+        self.node = node
+      end
+
+      def run_started(run_status)
+        self.run_id = run_status.run_id
+      end
+
+      def run_list_expanded(run_list_expansion)
+        self.recipes = run_list_expansion.recipes
+      end
+
+      def run_completed(_node, _run_status)
+        return unless enabled?
+
+        logger.info("#{self.class}##{__method__}: enabling Compliance Phase")
+
+        report
+      end
+
+      def run_failed(_exception, _run_status)
+        return unless enabled?
+
+        logger.info("#{self.class}##{__method__}: enabling Compliance Phase")
+
+        report
+      end
+
+      ### Below code adapted from audit cookbook's files/default/handler/audit_report.rb
+
+      DEPRECATED_CONFIG_VALUES = %w{
+        attributes_save
+        fail_if_not_present
+        inspec_gem_source
+        inspec_version
+        interval
+        owner
+        raise_if_unreachable
+      }.freeze
+
+      def warn_for_deprecated_config_values!
+        deprecated_config_values = (node["audit"].keys & DEPRECATED_CONFIG_VALUES)
+
+        if deprecated_config_values.any?
+          values = deprecated_config_values.sort.map { |v| "'#{v}'" }.join(", ")
+          logger.warn "audit cookbook config values #{values} are not supported in #{ChefUtils::Dist::Infra::PRODUCT}'s Compliance Phase."
+        end
+      end
+
+      def report(report = generate_report)
+        warn_for_deprecated_config_values!
+
+        if report.empty?
+          logger.error "Compliance report was not generated properly, skipped reporting"
+          return
+        end
+
+        Array(node["audit"]["reporter"]).each do |reporter|
+          send_report(reporter, report)
+        end
+      end
+
+      def inspec_opts
+        inputs = node["audit"]["attributes"].to_h
+        if node["audit"]["chef_node_attribute_enabled"]
+          inputs["chef_node"] = node.to_h
+          inputs["chef_node"]["chef_environment"] = node.chef_environment
+        end
+
+        {
+          backend_cache: node["audit"]["inspec_backend_cache"],
+          inputs: inputs,
+          logger: logger,
+          output: node["audit"]["quiet"] ? ::File::NULL : STDOUT,
+          report: true,
+          reporter: ["json-automate"],
+          reporter_backtrace_inclusion: node["audit"]["result_include_backtrace"],
+          reporter_message_truncation: node["audit"]["result_message_limit"],
+          waiver_file: Array(node["audit"]["waiver_file"]),
+        }
+      end
+
+      def inspec_profiles
+        profiles = node["audit"]["profiles"]
+
+        # TODO: Custom exception class here?
+        unless profiles.respond_to?(:map) && profiles.all? { |_, p| p.respond_to?(:transform_keys) && p.respond_to?(:update) }
+          raise "#{Inspec::Dist::PRODUCT_NAME} profiles specified in an unrecognized format, expected a hash of hashes."
+        end
+
+        profiles.map do |name, profile|
+          profile.transform_keys(&:to_sym).update(name: name)
+        end
+      end
+
+      def load_fetchers!
+        case node["audit"]["fetcher"]
+        when "chef-automate"
+          require_relative "fetcher/automate"
+        when "chef-server"
+          require_relative "fetcher/chef_server"
+        when nil
+          # intentionally blank
+        else
+          raise "Invalid value specified for Compliance Phase's fetcher: '#{node["audit"]["fetcher"]}'. Valid values are 'chef-automate', 'chef-server', or nil."
+        end
+      end
+
+      def generate_report(opts: inspec_opts, profiles: inspec_profiles)
+        load_fetchers!
+
+        logger.debug "Options are set to: #{opts}"
+        runner = ::Inspec::Runner.new(opts)
+
+        if profiles.empty?
+          failed_report("No #{Inspec::Dist::PRODUCT_NAME} profiles are defined.")
+          return
+        end
+
+        profiles.each { |target| runner.add_target(target) }
+
+        logger.info "Running profiles from: #{profiles.inspect}"
+        runner.run
+        runner.report.tap do |r|
+          logger.debug "Compliance Report #{r}"
+        end
+      rescue Inspec::FetcherFailure => e
+        failed_report("Cannot fetch all profiles: #{profiles}. Please make sure you're authenticated and the server is reachable. #{e.message}")
+      rescue => e
+        failed_report(e.message)
+      end
+
+      # In case InSpec raises a runtime exception without providing a valid report,
+      # we make one up and add two new fields to it: `status` and `status_message`
+      def failed_report(err)
+        logger.error "#{Inspec::Dist::PRODUCT_NAME} has raised a runtime exception. Generating a minimal failed report."
+        logger.error err
+        {
+          "platform": {
+            "name": "unknown",
+            "release": "unknown",
+          },
+          "profiles": [],
+          "statistics": {
+            "duration": 0.0000001,
+          },
+          "version": Inspec::VERSION,
+          "status": "failed",
+          "status_message": err,
+        }
+      end
+
+      # extracts relevant node data
+      def node_info
+        chef_server_uri = URI(Chef::Config[:chef_server_url])
+
+        runlist_roles = node.run_list.select { |item| item.type == :role }.map(&:name)
+        runlist_recipes = node.run_list.select { |item| item.type == :recipe }.map(&:name)
+        {
+          node: node.name,
+          os: {
+            release: node["platform_version"],
+            family: node["platform"],
+          },
+          environment: node.environment,
+          roles: runlist_roles,
+          recipes: runlist_recipes,
+          policy_name: node.policy_name || "",
+          policy_group: node.policy_group || "",
+          chef_tags: node.tags,
+          organization_name: chef_server_uri.path.split("/").last || "",
+          source_fqdn: chef_server_uri.host || "",
+          ipaddress: node["ipaddress"],
+          fqdn: node["fqdn"],
+        }
+      end
+
+      def send_report(reporter_type, report)
+        logger.info "Reporting to #{reporter_type}"
+
+        reporter = reporter(reporter_type)
+
+        reporter.send_report(report) if reporter
+      end
+
+      def reporter(reporter_type)
+        case reporter_type
+        when "chef-automate"
+          opts = {
+            control_results_limit: node["audit"]["control_results_limit"],
+            entity_uuid: node["chef_guid"],
+            insecure: node["audit"]["insecure"],
+            node_info: node_info,
+            run_id: run_id,
+            run_time_limit: node["audit"]["run_time_limit"],
+          }
+          Chef::Compliance::Reporter::Automate.new(opts)
+        when "chef-server-automate"
+          opts = {
+            control_results_limit: node["audit"]["control_results_limit"],
+            entity_uuid: node["chef_guid"],
+            insecure: node["audit"]["insecure"],
+            node_info: node_info,
+            run_id: run_id,
+            run_time_limit: node["audit"]["run_time_limit"],
+            url: chef_server_automate_url,
+          }
+          Chef::Compliance::Reporter::ChefServerAutomate.new(opts)
+        when "json-file"
+          path = node["audit"]["json_file"]["location"]
+          logger.info "Writing compliance report to #{path}"
+          Chef::Compliance::Reporter::JsonFile.new(file: path)
+        when "audit-enforcer"
+          Chef::Compliance::Reporter::ComplianceEnforcer.new
+        else
+          raise "'#{reporter_type}' is not a supported reporter for Compliance Phase."
+        end
+      end
+
+      def chef_server_automate_url
+        url = if node["audit"]["server"]
+                URI(node["audit"]["server"])
+              else
+                URI(Chef::Config[:chef_server_url]).tap do |u|
+                  u.path = ""
+                end
+              end
+
+        org = Chef::Config[:chef_server_url].split("/").last
+        url.path = File.join(url.path, "organizations/#{org}/data-collector")
+        url
+      end
+    end
+  end
+end