chef 16.5.64-universal-mingw32 → 16.8.14-universal-mingw32

data/lib/chef/compliance/fetcher/chef_server.rb

@@ -0,0 +1,134 @@
+require "uri" unless defined?(URI)
+require "plugins/inspec-compliance/lib/inspec-compliance"
+
+# This class implements an InSpec fetcher for Chef Server. The implementation
+# is based on the Chef Compliance fetcher and only adapts the calls to redirect
+# the requests via Chef Server.
+#
+# This implementation depends on chef-client runtime, therefore it is only executable
+# inside of a chef-client run
+
+class Chef
+  module Compliance
+    module Fetcher
+      class ChefServer < ::InspecPlugins::Compliance::Fetcher
+        name "chef-server"
+
+        # it positions itself before `compliance` fetcher
+        # only load it, if the Chef Server is integrated with Chef Compliance
+        priority 501
+
+        CONFIG = { "insecure" => true }.freeze
+
+        # Accepts URLs to compliance profiles in one of two forms:
+        # * a String URL with a compliance scheme, like "compliance://namespace/profile_name"
+        # * a Hash with a key of `compliance` and a value like "compliance/profile_name" and optionally a `version` key with a String value
+        def self.resolve(target)
+          profile_uri = get_target_uri(target)
+          return nil if profile_uri.nil?
+
+          organization = Chef::Config[:chef_server_url].split("/").last
+          owner = profile_uri.user ? "#{profile_uri.user}@#{profile_uri.host}" : profile_uri.host
+          version = target[:version] if target.respond_to?(:key?)
+
+          path_parts = [""]
+          path_parts << "compliance" if chef_server_reporter? || chef_server_fetcher?
+          path_parts << "organizations"
+          path_parts << organization
+          path_parts << "owners"
+          path_parts << owner
+          path_parts << "compliance"
+          path_parts << profile_uri.path
+          path_parts << "version/#{version}" if version
+          path_parts << "tar"
+
+          target_url = URI(Chef::Config[:chef_server_url])
+          target_url.path = File.join(path_parts)
+          Chef::Log.info("Fetching profile from: #{target_url}")
+
+          new(target_url, CONFIG)
+        rescue URI::Error => _e
+          nil
+        end
+
+        #
+        # We want to save compliance: in the lockfile rather than url: to
+        # make sure we go back through the ComplianceAPI handling.
+        #
+        def resolved_source
+          { compliance: chef_server_url }
+        end
+
+        # Downloads archive to temporary file using a Chef::ServerAPI
+        # client so that Chef Server's header-based authentication can be
+        # used.
+        def download_archive_to_temp
+          return @temp_archive_path unless @temp_archive_path.nil?
+
+          rest = Chef::ServerAPI.new(@target, Chef::Config.merge(ssl_verify_mode: :verify_none))
+          archive = with_http_rescue do
+            rest.streaming_request(@target)
+          end
+          @archive_type = ".tar.gz"
+
+          if archive.nil?
+            path = @target.respond_to?(:path) ? @target.path : path
+            raise Inspec::FetcherFailure, "Unable to find requested profile on path: '#{path}' on the #{ChefUtils::Dist::Automate::PRODUCT} system."
+          end
+
+          Inspec::Log.debug("Archive stored at temporary location: #{archive.path}")
+          @temp_archive_path = archive.path
+        end
+
+        def with_http_rescue
+          response = yield
+          if response.respond_to?(:code)
+            # handle non 200 error codes, they are not raised as Net::HTTPClientException
+            handle_http_error_code(response.code) if response.code.to_i >= 300
+          end
+          response
+        rescue Net::HTTPClientException => e
+          Chef::Log.error e
+          handle_http_error_code(e.response.code)
+        end
+
+        def handle_http_error_code(code)
+          case code
+          when /401|403/
+            Chef::Log.error "Auth issue: see audit cookbook TROUBLESHOOTING.md"
+          when /404/
+            Chef::Log.error "Object does not exist on remote server."
+          when /413/
+            Chef::Log.error "You most likely hit the erchef request size in #{ChefUtils::Dist::Server::PRODUCT} that defaults to ~2MB. To increase this limit see audit cookbook TROUBLESHOOTING.md OR https://docs.chef.io/config_rb_server.html"
+          when /429/
+            Chef::Log.error "This error typically means the data sent was larger than #{ChefUtils::Dist::Automate::PRODUCT}'s limit (4 MB). Run InSpec locally to identify any controls producing large diffs."
+          end
+          msg = "Received HTTP error #{code}"
+          Chef::Log.error msg
+          raise Inspec::FetcherFailure, msg
+        end
+
+        def to_s
+          "#{ChefUtils::Dist::Server::PRODUCT}/Compliance Profile Loader"
+        end
+
+        CHEF_SERVER_REPORTERS = %w{chef-server chef-server-compliance chef-server-visibility chef-server-automate}.freeze
+        def self.chef_server_reporter?
+          (Array(Chef.node.attributes["audit"]["reporter"]) & CHEF_SERVER_REPORTERS).any?
+        end
+
+        CHEF_SERVER_FETCHERS = %w{chef-server chef-server-compliance chef-server-visibility chef-server-automate}.freeze
+        def self.chef_server_fetcher?
+          CHEF_SERVER_FETCHERS.include?(Chef.node.attributes["audit"]["fetcher"])
+        end
+
+        private
+
+        def chef_server_url
+          m = %r{^#{@config['server']}/owners/(?<owner>[^/]+)/compliance/(?<id>[^/]+)/tar$}.match(@target)
+          "#{m[:owner]}/#{m[:id]}"
+        end
+      end
+    end
+  end
+end

data/lib/chef/compliance/reporter/automate.rb

@@ -0,0 +1,202 @@
+class Chef
+  module Compliance
+    module Reporter
+      #
+      # Used to send inspec reports to Chef Automate via the data_collector service
+      #
+      class Automate
+        def initialize(opts)
+          @entity_uuid           = opts[:entity_uuid]
+          @run_id                = opts[:run_id]
+          @node_name             = opts[:node_info][:node]
+          @environment           = opts[:node_info][:environment]
+          @roles                 = opts[:node_info][:roles]
+          @recipes               = opts[:node_info][:recipes]
+          @insecure              = opts[:insecure]
+          @chef_tags             = opts[:node_info][:chef_tags]
+          @policy_group          = opts[:node_info][:policy_group]
+          @policy_name           = opts[:node_info][:policy_name]
+          @source_fqdn           = opts[:node_info][:source_fqdn]
+          @organization_name     = opts[:node_info][:organization_name]
+          @ipaddress             = opts[:node_info][:ipaddress]
+          @fqdn                  = opts[:node_info][:fqdn]
+          @run_time_limit        = opts[:run_time_limit]
+          @control_results_limit = opts[:control_results_limit]
+          @timestamp             = opts.fetch(:timestamp) { Time.now }
+
+          @url = Chef::Config[:data_collector][:server_url]
+          @token = Chef::Config[:data_collector][:token]
+        end
+
+        # Method used in order to send the inspec report to the data_collector server
+        def send_report(report)
+          unless @entity_uuid && @run_id
+            Chef::Log.error "entity_uuid(#{@entity_uuid}) or run_id(#{@run_id}) can't be nil, not sending report to #{ChefUtils::Dist::Automate::PRODUCT}"
+            return false
+          end
+
+          unless @url && @token
+            Chef::Log.warn "data_collector.token and data_collector.server_url must be defined in client.rb!"
+            Chef::Log.warn "Further information: https://github.com/chef-cookbooks/audit#direct-reporting-to-chef-automate"
+            return false
+          end
+
+          headers = {
+            "Content-Type" => "application/json",
+            "x-data-collector-auth" => "version=1.0",
+            "x-data-collector-token" => @token,
+          }
+
+          all_report_shas = report[:profiles].map { |p| p[:sha256] }
+          missing_report_shas = missing_automate_profiles(headers, all_report_shas)
+
+          full_report = truncate_controls_results(enriched_report(report), @control_results_limit)
+          full_report = strip_profiles_meta(full_report, missing_report_shas, @run_time_limit)
+          json_report = Chef::JSONCompat.to_json(full_report, validate_utf8: false)
+
+          # Automate GRPC currently has a message limit of ~4MB
+          # https://github.com/chef/automate/issues/1417#issuecomment-541908157
+          if json_report.bytesize > 4 * 1024 * 1024
+            Chef::Log.warn "Generated report size is #{(json_report.bytesize / (1024 * 1024.0)).round(2)} MB. #{ChefUtils::Dist::Automate::PRODUCT} has an internal 4MB limit that is not currently configurable."
+          end
+
+          unless json_report
+            Chef::Log.warn "Something went wrong, report can't be nil"
+            return false
+          end
+
+          begin
+            Chef::Log.info "Report to #{ChefUtils::Dist::Automate::PRODUCT}: #{@url}"
+            Chef::Log.debug "Compliance Report: #{json_report}"
+            http_client.post(nil, json_report, headers)
+            true
+          rescue => e
+            Chef::Log.error "send_report: POST to #{@url} returned: #{e.message}"
+            false
+          end
+        end
+
+        def http_client(url = @url)
+          if @insecure
+            Chef::HTTP.new(url, ssl_verify_mode: :verify_none)
+          else
+            Chef::HTTP.new(url)
+          end
+        end
+
+        def enriched_report(final_report)
+          final_report[:profiles].compact!
+
+          # Label this content as an inspec_report
+          final_report[:type] = "inspec_report"
+
+          final_report[:node_name]         = @node_name
+          final_report[:end_time]          = @timestamp.utc.strftime("%FT%TZ")
+          final_report[:node_uuid]         = @entity_uuid
+          final_report[:environment]       = @environment
+          final_report[:roles]             = @roles
+          final_report[:recipes]           = @recipes
+          final_report[:report_uuid]       = @run_id
+          final_report[:source_fqdn]       = @source_fqdn
+          final_report[:organization_name] = @organization_name
+          final_report[:policy_group]      = @policy_group
+          final_report[:policy_name]       = @policy_name
+          final_report[:chef_tags]         = @chef_tags
+          final_report[:ipaddress]         = @ipaddress
+          final_report[:fqdn]              = @fqdn
+
+          final_report
+        end
+
+        CONTROL_RESULT_SORT_ORDER = %w{ failed skipped passed }.freeze
+
+        # Truncates the number of results per control in the report when they exceed max_results.
+        # The truncation prioritizes failed and skipped results over passed ones.
+        # Controls where results have been truncated will get a new object 'removed_results_counts'
+        # with the status counts of the truncated results
+        def truncate_controls_results(report, max_results)
+          return report unless max_results.is_a?(Integer) && max_results > 0
+
+          report.fetch(:profiles, []).each do |profile|
+            profile.fetch(:controls, []).each do |control|
+              # Only bother with truncation if the number of results exceed max_results
+              next unless control[:results].length > max_results
+
+              res = control[:results]
+              res.sort_by! { |r| CONTROL_RESULT_SORT_ORDER.index(r[:status]) }
+
+              # Count the results that will be truncated
+              truncated = { failed: 0, skipped: 0, passed: 0 }
+              (max_results..res.length - 1).each do |i|
+                case res[i][:status]
+                when "failed"
+                  truncated[:failed] += 1
+                when "skipped"
+                  truncated[:skipped] += 1
+                when "passed"
+                  truncated[:passed] += 1
+                end
+              end
+              # Truncate the results array now
+              control[:results] = res[0..max_results - 1]
+              control[:removed_results_counts] = truncated
+            end
+          end
+          report
+        end
+
+        # Contacts the metasearch Automate API to check which of the inspec profile sha256 ids
+        # passed in via `report_shas` are missing from the Automate profiles metadata database.
+        def missing_automate_profiles(headers, report_shas)
+          Chef::Log.debug "Checking the #{ChefUtils::Dist::Automate::PRODUCT} profiles metadata for: #{report_shas}"
+          meta_url = URI(@url)
+          meta_url.path = "/compliance/profiles/metasearch"
+          response_str = http_client(meta_url.to_s).post(nil, "{\"sha256\": #{report_shas}}", headers)
+          missing_shas = Chef::JSONCompat.parse(response_str)["missing_sha256"]
+          unless missing_shas.empty?
+            Chef::Log.info "#{ChefUtils::Dist::Automate::PRODUCT} is missing metadata for the following profile ids: #{missing_shas}"
+          end
+          missing_shas
+        rescue => e
+          Chef::Log.error "missing_automate_profiles error: #{e.message}"
+          # If we get an error it's safer to assume none of the profile shas exist in Automate
+          report_shas
+        end
+
+        # Profile 'name' is a required property.
+        # By not sending it in the report, we make it clear to the ingestion backend that the profile metadata has been stripped from this profile in the report.
+        # Profile 'title' and 'version' are still kept for troubleshooting purposes in the backend.
+        SEEN_PROFILE_UNNECESSARY_FIELDS = %i{ copyright copyright_email groups license maintainer name summary supports}.freeze
+
+        SEEN_PROFILE_UNNECESSARY_CONTROL_FIELDS = %i{ code desc descriptions impact refs source_location tags title }.freeze
+
+        # TODO: This mutates the report and probably doesn't need to.
+        def strip_profiles_meta(report, missing_report_shas, run_time_limit)
+          report[:profiles].each do |p|
+            next if missing_report_shas.include?(p[:sha256])
+
+            p.delete_if { |f| SEEN_PROFILE_UNNECESSARY_FIELDS.include?(f) }
+
+            next unless p[:controls].is_a?(Array)
+
+            p[:controls].each do |c|
+              c.delete_if { |f| SEEN_PROFILE_UNNECESSARY_CONTROL_FIELDS.include?(f) }
+              c.delete(:waiver_data) if c[:waiver_data] == {}
+
+              next unless c[:results].is_a?(Array)
+
+              c[:results].each do |r|
+                if r[:run_time].is_a?(Float) && r[:run_time] < run_time_limit
+                  r.delete(:start_time)
+                  r.delete(:run_time)
+                end
+              end
+            end
+          end
+          report[:run_time_limit] = run_time_limit
+          report
+        end
+      end
+    end
+  end
+end

data/lib/chef/compliance/reporter/chef_server_automate.rb

@@ -0,0 +1,94 @@
+require_relative "automate"
+
+class Chef
+  module Compliance
+    module Reporter
+      #
+      # Used to send inspec reports to Chef Automate server via Chef Server
+      #
+      class ChefServerAutomate < Chef::Compliance::Reporter::Automate
+        attr_reader :url
+
+        def initialize(opts)
+          @entity_uuid           = opts[:entity_uuid]
+          @run_id                = opts[:run_id]
+          @node_name             = opts[:node_info][:node]
+          @insecure              = opts[:insecure]
+          @environment           = opts[:node_info][:environment]
+          @roles                 = opts[:node_info][:roles]
+          @recipes               = opts[:node_info][:recipes]
+          @url                   = opts[:url]
+          @chef_tags             = opts[:node_info][:chef_tags]
+          @policy_group          = opts[:node_info][:policy_group]
+          @policy_name           = opts[:node_info][:policy_name]
+          @source_fqdn           = opts[:node_info][:source_fqdn]
+          @organization_name     = opts[:node_info][:organization_name]
+          @ipaddress             = opts[:node_info][:ipaddress]
+          @fqdn                  = opts[:node_info][:fqdn]
+          @control_results_limit = opts[:control_results_limit]
+          @timestamp             = opts.fetch(:timestamp) { Time.now }
+        end
+
+        def send_report(report)
+          unless @entity_uuid && @run_id
+            Chef::Log.error "entity_uuid(#{@entity_uuid}) or run_id(#{@run_id}) can't be nil, not sending report to #{ChefUtils::Dist::Automate::PRODUCT}"
+            return false
+          end
+
+          automate_report = truncate_controls_results(enriched_report(report), @control_results_limit)
+
+          report_size = Chef::JSONCompat.to_json(automate_report, validate_utf8: false).bytesize
+          # this is set to slightly less than the oc_erchef limit
+          if report_size > 900 * 1024
+            Chef::Log.warn "Generated report size is #{(report_size / (1024 * 1024.0)).round(2)} MB. #{ChefUtils::Dist::Server::PRODUCT} < 13.0 defaults to a limit of ~1MB, 13.0+ defaults to a limit of ~2MB."
+          end
+
+          Chef::Log.info "Report to #{ChefUtils::Dist::Automate::PRODUCT} via #{ChefUtils::Dist::Server::PRODUCT}: #{@url}"
+          with_http_rescue do
+            http_client.post(@url, automate_report)
+            return true
+          end
+          false
+        end
+
+        def http_client
+          config = if @insecure
+                     Chef::Config.merge(ssl_verify_mode: :verify_none)
+                   else
+                     Chef::Config
+                   end
+
+          Chef::ServerAPI.new(@url, config)
+        end
+
+        def with_http_rescue
+          response = yield
+          if response.respond_to?(:code)
+            # handle non 200 error codes, they are not raised as Net::HTTPClientException
+            handle_http_error_code(response.code) if response.code.to_i >= 300
+          end
+          response
+        rescue Net::HTTPClientException => e
+          Chef::Log.error e
+          handle_http_error_code(e.response.code)
+        end
+
+        def handle_http_error_code(code)
+          case code
+          when /401|403/
+            Chef::Log.error "Auth issue: see audit cookbook TROUBLESHOOTING.md"
+          when /404/
+            Chef::Log.error "Object does not exist on remote server."
+          when /413/
+            Chef::Log.error "You most likely hit the erchef request size in #{ChefUtils::Dist::Server::PRODUCT} that defaults to ~2MB. To increase this limit see audit cookbook TROUBLESHOOTING.md OR https://docs.chef.io/config_rb_server.html"
+          when /429/
+            Chef::Log.error "This error typically means the data sent was larger than #{ChefUtils::Dist::Automate::PRODUCT}'s limit (4 MB). Run InSpec locally to identify any controls producing large diffs."
+          end
+          msg = "Received HTTP error #{code}"
+          Chef::Log.error msg
+          raise msg
+        end
+      end
+    end
+  end
+end

data/lib/chef/compliance/reporter/compliance_enforcer.rb

@@ -0,0 +1,20 @@
+class Chef
+  module Compliance
+    module Reporter
+      class AuditEnforcer
+        class ControlFailure < StandardError; end
+
+        def send_report(report)
+          report.fetch(:profiles, []).each do |profile|
+            profile.fetch(:controls, []).each do |control|
+              control.fetch(:results, []).each do |result|
+                raise ControlFailure, "Audit #{control[:id]} has failed. Aborting #{ChefUtils::Dist::Infra::CLIENT} run." if result[:status] == "failed"
+              end
+            end
+          end
+          true
+        end
+      end
+    end
+  end
+end