chef 17.1.35 → 17.2.29

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b92d0b0a8777142e81fac5c879381f6ea5212a3c068a2e721d736cdc550d1c85
-  data.tar.gz: 742765962f825f43c67be4a2781ea5cca930a2daba4ad69c0a9fbe231423cc44
+  metadata.gz: 3259221c96da998dc7da695df0488bcfbe781ce3764dcd98309c78c1a46cb56c
+  data.tar.gz: 120938b331b2e96306b91c5a192a73ebbfaeebce9b3febfccae32d7e7918e4a6
 SHA512:
-  metadata.gz: 2b2e4f77b2b34aea3209fb5c22453b1238f1b63bcb967cc56d0829371b9bf7a71169e1b3e98dbaf424eb7f1e902393dd18940770f3801242c07c7780adc156f9
-  data.tar.gz: 95ae0ac8453576d18c2e5c5d9a06fd893e3a4e29f06062acea963ecaf8a33b2b32988d24a369f4decae7b17e4840020161f65cdbcd9d8cbd9069397f8f513f90
+  metadata.gz: f1191c217a427591d0cdd0b0223249ed83164f53da4edd463258dfe37d084b9e928627eb9020a20aeb87f141d160eb72b64d4c1d37e47b7206f50e120b7738fe
+  data.tar.gz: '0148bdae1adf6ffa4539aab3c8dbc47fc225cb5d6f1e421967be4d53ce7504f9ff10af21cd0410412847b9e99d6308892b32a0ab673bd657ccb6922f191c91a1'

data/Gemfile

@@ -22,6 +22,7 @@ group(:omnibus_package) do
   gem "rb-readline"
   gem "inspec-core-bin", "~> 4.24" # need to provide the binaries for inspec
   gem "chef-vault"
+  gem "ed25519", "~> 1.2" # to make it possible to install knife into chef. Remove this in Chef 18
 end
 
 group(:omnibus_package, :pry) do

data/lib/chef/action_collection.rb

@@ -87,13 +87,11 @@ class Chef
     attr_reader :action_records
     attr_reader :pending_updates
     attr_reader :run_context
-    attr_reader :consumers
     attr_reader :events
 
     def initialize(events, run_context = nil, action_records = [])
       @action_records  = action_records
       @pending_updates = []
-      @consumers       = []
       @events          = events
       @run_context     = run_context
     end
@@ -118,17 +116,17 @@ class Chef
       self.class.new(events, run_context, subrecords)
     end
 
+    def resources
+      action_records.map(&:new_resource)
+    end
+
     # This hook gives us the run_context immediately after it is created so that we can wire up this object to it.
     #
-    # This also causes the action_collection_registration event to fire, all consumers that have not yet registered with the
-    # action_collection must register via this callback.  This is the latest point before resources actually start to get
-    # evaluated.
-    #
     # (see EventDispatch::Base#)
     #
     def cookbook_compilation_start(run_context)
       run_context.action_collection = self
-      # fire the action_colleciton_registration hook after cookbook_compilation_start -- last chance for consumers to register
+      # this hook is now poorly named since it is just a callback that lets other consumers snag a reference to the action_collection
       run_context.events.enqueue(:action_collection_registration, self)
       @run_context = run_context
     end
@@ -139,7 +137,7 @@ class Chef
     # @params object [Object] callers should call with `self`
     #
     def register(object)
-      consumers << object
+      Chef::Log.warn "the action collection no longer requires registration at #{caller[0]}"
     end
 
     # End of an unsuccessful converge used to fire off detect_unprocessed_resources.
@@ -147,8 +145,6 @@ class Chef
     # (see EventDispatch::Base#)
     #
     def converge_failed(exception)
-      return if consumers.empty?
-
       detect_unprocessed_resources
     end
 
@@ -159,8 +155,6 @@ class Chef
     # (see EventDispatch::Base#)
     #
     def resource_action_start(new_resource, action, notification_type = nil, notifier = nil)
-      return if consumers.empty?
-
       pending_updates << ActionRecord.new(new_resource, action, pending_updates.length)
     end
 
@@ -170,8 +164,6 @@ class Chef
     # (see EventDispatch::Base#)
     #
     def resource_current_state_loaded(new_resource, action, current_resource)
-      return if consumers.empty?
-
       current_record.current_resource = current_resource
     end
 
@@ -181,8 +173,6 @@ class Chef
     # (see EventDispatch::Base#)
     #
     def resource_after_state_loaded(new_resource, action, after_resource)
-      return if consumers.empty?
-
       current_record.after_resource = after_resource
     end
 
@@ -191,8 +181,6 @@ class Chef
     # (see EventDispatch::Base#)
     #
     def resource_up_to_date(new_resource, action)
-      return if consumers.empty?
-
       current_record.status = :up_to_date
     end
 
@@ -201,8 +189,6 @@ class Chef
     # (see EventDispatch::Base#)
     #
     def resource_skipped(resource, action, conditional)
-      return if consumers.empty?
-
       current_record.status = :skipped
       current_record.conditional = conditional
     end
@@ -212,8 +198,6 @@ class Chef
     # (see EventDispatch::Base#)
     #
     def resource_updated(new_resource, action)
-      return if consumers.empty?
-
       current_record.status = :updated
     end
 
@@ -222,8 +206,6 @@ class Chef
     # (see EventDispatch::Base#)
     #
     def resource_failed(new_resource, action, exception)
-      return if consumers.empty?
-
       current_record.status = :failed
       current_record.exception = exception
       current_record.error_description = Formatters::ErrorMapper.resource_failed(new_resource, action, exception).for_json
@@ -234,8 +216,6 @@ class Chef
     # (see EventDispatch::Base#)
     #
     def resource_completed(new_resource)
-      return if consumers.empty?
-
       current_record.elapsed_time = new_resource.elapsed_time
 
       # Verify if the resource has sensitive data and create a new blank resource with only

data/lib/chef/application.rb

@@ -310,7 +310,7 @@ class Chef
       logger.info "Forking #{ChefUtils::Dist::Infra::PRODUCT} instance to converge..."
       pid = fork do
         # Want to allow forked processes to finish converging when
-        # TERM singal is received (exit gracefully)
+        # TERM signal is received (exit gracefully)
         trap("TERM") do
           logger.debug("SIGTERM received during converge," +
             " finishing converge to exit normally (send SIGINT to terminate immediately)")

data/lib/chef/application/base.rb

@@ -297,6 +297,21 @@ class Chef::Application::Base < Chef::Application
     long: "--named-run-list NAMED_RUN_LIST",
     description: "Use a policyfile's named run list instead of the default run list."
 
+  option :slow_report,
+    long: "--[no-]slow-report [COUNT]",
+    description: "List the slowest resources at the end of the run (default: 10).",
+    boolean: true,
+    default: false,
+    proc: lambda { |argument|
+      if argument.nil?
+        true
+      elsif argument == false
+        false
+      else
+        Integer(argument)
+      end
+    }
+
   IMMEDIATE_RUN_SIGNAL = "1".freeze
   RECONFIGURE_SIGNAL = "H".freeze
 

data/lib/chef/client.rb

@@ -863,6 +863,12 @@ class Chef
     end
 
     def start_profiling
+      if Chef::Config[:slow_report]
+        require_relative "handler/slow_report"
+
+        Chef::Config.report_handlers << Chef::Handler::SlowReport.new(Chef::Config[:slow_report])
+      end
+
       return unless Chef::Config[:profile_ruby]
 
       profiling_prereqs!

data/lib/chef/cookbook_version.rb

@@ -138,11 +138,14 @@ class Chef
     end
 
     def recipe_yml_filenames_by_name
-      @recipe_ym_filenames_by_name ||= begin
+      @recipe_yml_filenames_by_name ||= begin
         name_map = yml_filenames_by_name(files_for("recipes"))
-        root_alias = cookbook_manifest.root_files.find { |record| record[:name] == "root_files/recipe.yml" }
+        root_alias = cookbook_manifest.root_files.find { |record|
+          record[:name] == "root_files/recipe.yml" ||
+            record[:name] == "root_files/recipe.yaml"
+        }
         if root_alias
-          Chef::Log.error("Cookbook #{name} contains both recipe.yml and and recipes/default.yml, ignoring recipes/default.yml") if name_map["default"]
+          Chef::Log.error("Cookbook #{name} contains both recipe.yml and recipes/default.yml, ignoring recipes/default.yml") if name_map["default"]
           name_map["default"] = root_alias[:full_path]
         end
         name_map
@@ -582,8 +585,27 @@ class Chef
       records.select { |record| record[:name] =~ /\.rb$/ }.inject({}) { |memo, record| memo[File.basename(record[:name], ".rb")] = record[:full_path]; memo }
     end
 
+    # Filters YAML files from the superset of provided files.
+    # Checks for duplicate basenames with differing extensions (eg yaml v yml)
+    # and raises error if any are detected.
+    # This prevents us from arbitrarily the ".yaml" or ".yml" version when both are present,
+    # because we don't know which is correct.
+    # This method runs in O(n^2) where N = number of yml files present. This number should be consistently
+    # low enough that there's no noticeable perf impact.
     def yml_filenames_by_name(records)
-      records.select { |record| record[:name] =~ /\.yml$/ }.inject({}) { |memo, record| memo[File.basename(record[:name], ".yml")] = record[:full_path]; memo }
+      yml_files = records.select { |record| record[:name] =~ /\.(y[a]?ml)$/ }
+      result = yml_files.inject({}) do |acc, record|
+        filename = record[:name]
+        base_dup_name = File.join(File.dirname(filename), File.basename(filename, File.extname(filename)))
+        yml_files.each do |other|
+          if other[:name] =~ /#{(File.extname(filename) == ".yml") ? "#{base_dup_name}.yaml" : "#{base_dup_name}.yml"}$/
+            raise Chef::Exceptions::AmbiguousYAMLFile.new("Cookbook #{name}@#{version} contains ambiguous files: #{filename} and #{other[:name]}. Please update the cookbook to remove the incorrect file.")
+          end
+        end
+        acc[File.basename(record[:name], File.extname(record[:name]))] = record[:full_path]
+        acc
+      end
+      result
     end
 
     def file_vendor

data/lib/chef/data_bag.rb

@@ -32,7 +32,8 @@ class Chef
     include Chef::Mixin::FromFile
     include Chef::Mixin::ParamsValidate
 
-    VALID_NAME = /^[\.\-[:alnum:]_]+$/.freeze
+    # Regex reference: https://rubular.com/r/oIMySIO4USPm5x
+    VALID_NAME = /^[\-[:alnum:]_]+$/.freeze
     RESERVED_NAMES = /^(node|role|environment|client)$/.freeze
 
     def self.validate_name!(name)

data/lib/chef/data_bag_item.rb

@@ -36,7 +36,8 @@ class Chef
     include Chef::Mixin::FromFile
     include Chef::Mixin::ParamsValidate
 
-    VALID_ID = /^[\.\-[:alnum:]_]+$/.freeze
+    # Regex reference: https://rubular.com/r/oIMySIO4USPm5x
+    VALID_ID = /^[\-[:alnum:]_]+$/.freeze
 
     def self.validate_id!(id_str)
       if id_str.nil? || ( id_str !~ VALID_ID )

data/lib/chef/data_collector.rb

@@ -104,7 +104,6 @@ class Chef
       #
       def action_collection_registration(action_collection)
         @action_collection = action_collection
-        action_collection.register(self)
       end
 
       # - Creates and writes our NodeUUID back to the node object

data/lib/chef/data_collector/run_end_message.rb

@@ -51,7 +51,7 @@ class Chef
             "id" => run_status&.run_id,
             "message_version" => "1.1.0",
             "message_type" => "run_converge",
-            "node" => node || {},
+            "node" => node&.data_for_save || {},
             "node_name" => node&.name || data_collector.node_name,
             "organization_name" => organization,
             "resources" => all_action_records(action_collection),

data/lib/chef/deprecated.rb

@@ -253,6 +253,10 @@ class Chef
       target 33
     end
 
+    class AttributeWhitelistConfiguration < Base
+      target 34
+    end
+
     class Generic < Base
       def url
         "https://docs.chef.io/chef_deprecations_client/"

data/lib/chef/event_dispatch/base.rb

@@ -221,7 +221,8 @@ class Chef
       # Called before convergence starts
       def converge_start(run_context); end
 
-      # Callback hook for handlers to register their interest in the action_collection
+      # Callback hook for handlers to grab a reference to the action_collection
+      # (sent before compiling cookbooks, consumers can also find it off the run_context.action_collection)
       def action_collection_registration(action_collection); end
 
       # Called when the converge phase is finished.

data/lib/chef/exceptions.rb

@@ -174,6 +174,9 @@ class Chef
     class CannotDetermineWindowsInstallerType < Package; end
     class NoWindowsPackageSource < Package; end
 
+    # for example, if both recipes/default.yml, recipes/default.yaml are present
+    class AmbiguousYAMLFile < RuntimeError; end
+
     # Can not create staging file during file deployment
     class FileContentStagingError < RuntimeError
       def initialize(errors)

data/lib/chef/handler.rb

@@ -55,6 +55,12 @@ class Chef
   #
   class Handler
 
+    # FIXME: Chef::Handler should probably inherit from EventDispatch::Base
+    # and should wire up to those events rather than the "notifications" system
+    # which is hanging off of Chef::Client.  Those "notifications" could then be
+    # deprecated in favor of events, and this class could become decoupled from
+    # the Chef::Client object.
+
     def self.handler_for(*args)
       if args.include?(:start)
         Chef::Config[:start_handlers] ||= []
@@ -207,17 +213,45 @@ class Chef
     # The Chef::Node for this client run
     def_delegator :@run_status, :node
 
-    ##
-    # :method: all_resources
+    # @return Array<Chef::Resource> all resources other than unprocessed
     #
-    # An Array containing all resources in the chef run's resource_collection
-    def_delegator :@run_status, :all_resources
+    def all_resources
+      @all_resources ||= action_collection&.filtered_collection(unprocessed: false)&.resources || []
+    end
 
-    ##
-    # :method: updated_resources
+    # @return Array<Chef::Resource> all updated resources
+    #
+    def updated_resources
+      @updated_resources ||= action_collection&.filtered_collection(up_to_date: false, skipped: false, failed: false, unprocessed: false)&.resources || []
+    end
+
+    # @return Array<Chef::Resource> all up_to_date resources
+    #
+    def up_to_date_resources
+      @up_to_date_resources ||= action_collection&.filtered_collection(updated: false, skipped: false, failed: false, unprocessed: false)&.resources || []
+    end
+
+    # @return Array<Chef::Resource> all failed resources
     #
-    # An Array containing all resources that were updated during the chef run
-    def_delegator :@run_status, :updated_resources
+    def failed_resources
+      @failed_resources ||= action_collection&.filtered_collection(updated: false, up_to_date: false, skipped: false, unprocessed: false)&.resources || []
+    end
+
+    # @return Array<Chef::Resource> all skipped resources
+    #
+    def skipped_resources
+      @skipped_resources ||= action_collection&.filtered_collection(updated: false, up_to_date: false, failed: false, unprocessed: false)&.resources || []
+    end
+
+    # Unprocessed resources are those which are left over in the outer recipe context when a run fails.
+    # Sub-resources of unprocessed resourced are impossible to capture because they would require processing
+    # the outer resource.
+    #
+    # @return Array<Chef::Resource> all unprocessed resources
+    #
+    def unprocessed_resources
+      @unprocessed_resources ||= action_collection&.filtered_collection(updated: false, up_to_date: false, failed: false, skipped: false)&.resources || []
+    end
 
     ##
     # :method: success?
@@ -232,6 +266,10 @@ class Chef
     # Did the chef run fail? True if the chef run raised an uncaught exception
     def_delegator :@run_status, :failed?
 
+    def action_collection
+      @run_status.run_context.action_collection
+    end
+
     # The main entry point for report handling. Subclasses should override this
     # method with their own report handling logic.
     def report; end

data/lib/chef/handler/slow_report.rb

@@ -0,0 +1,66 @@
+#
+# Copyright:: Copyright (c) Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require_relative "../handler"
+require "tty/table" unless defined?(TTY::Table)
+
+class Chef
+  class Handler
+    class SlowReport < ::Chef::Handler
+      attr_accessor :amount
+
+      def initialize(amount)
+        @amount = Integer(amount) rescue nil
+        @amount ||= 10
+      end
+
+      def report
+        if count == 0
+          puts "\nNo resources to profile\n\n"
+          return
+        end
+
+        top = all_records.sort_by(&:elapsed_time).last(amount).reverse
+        data = top.map { |r| [ r.new_resource.to_s, r.elapsed_time, r.action, r.new_resource.cookbook_name, r.new_resource.recipe_name, stripped_source_line(r.new_resource) ] }
+        puts "\nTop #{count} slowest #{count == 1 ? "resource" : "resources"}:\n\n"
+        table = TTY::Table.new(%w{resource elapsed_time action cookbook recipe source}, data)
+        rendered = table.render do |renderer|
+          renderer.border do
+            mid          "-"
+            mid_mid      " "
+          end
+        end
+        puts rendered
+        puts "\n"
+      end
+
+      def all_records
+        @all_records ||= action_collection&.filtered_collection(unprocessed: false) || []
+      end
+
+      def count
+        num = all_resources.count
+        num > amount ? amount : num
+      end
+
+      def stripped_source_line(resource)
+        # strip the leading path off of the source line
+        resource.source_line.gsub(%r{.*/cookbooks/}, "").gsub(%r{.*/chef-[0-9\.]+/}, "")
+      end
+    end
+  end
+end