chef 17.2.29 → 17.3.48

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3259221c96da998dc7da695df0488bcfbe781ce3764dcd98309c78c1a46cb56c
-  data.tar.gz: 120938b331b2e96306b91c5a192a73ebbfaeebce9b3febfccae32d7e7918e4a6
+  metadata.gz: 45a45627ed2995c171f44e97e524e17cdcc2b8a3c4566c6cba3e7e57a5810f71
+  data.tar.gz: 5bbe7e7fecc35ddfddc730addb9a944a798ff252a239e433f6894228c9af6359
 SHA512:
-  metadata.gz: f1191c217a427591d0cdd0b0223249ed83164f53da4edd463258dfe37d084b9e928627eb9020a20aeb87f141d160eb72b64d4c1d37e47b7206f50e120b7738fe
-  data.tar.gz: '0148bdae1adf6ffa4539aab3c8dbc47fc225cb5d6f1e421967be4d53ce7504f9ff10af21cd0410412847b9e99d6308892b32a0ab673bd657ccb6922f191c91a1'
+  metadata.gz: 3ef336ebe43aa213491af9e263758101f040ee306537d8d3d0db4a6484fa33daf78c0a524525ee648688cfe9fa2c77d5163ce310d6977b85c0ccf761fcc763ba
+  data.tar.gz: 554e9acfa53f9bc83a3b04cd9efb913b984bf9f69cb58b924a3ad9744fd2af8ff8bf42b6b495ba243ad71364d59ba433f6c440b9768c59aa3a7d9239948e2c7e

data/Gemfile

@@ -22,13 +22,14 @@ group(:omnibus_package) do
   gem "rb-readline"
   gem "inspec-core-bin", "~> 4.24" # need to provide the binaries for inspec
   gem "chef-vault"
-  gem "ed25519", "~> 1.2" # to make it possible to install knife into chef. Remove this in Chef 18
 end
 
 group(:omnibus_package, :pry) do
-  gem "pry"
+  # Locked because pry-byebug is broken with 13+.
+  # some work is ongoing? https://github.com/deivid-rodriguez/pry-byebug/issues/343
+  gem "pry", "= 0.13.0"
   # byebug does not install on freebsd on ruby 3.0
-  # gem "pry-byebug"
+  gem "pry-byebug" unless RUBY_PLATFORM =~ /freebsd/i
   gem "pry-stack_explorer"
 end
 

data/chef.gemspec

@@ -55,6 +55,7 @@ Gem::Specification.new do |s|
 
   s.add_dependency "proxifier", "~> 1.0"
 
+  s.add_dependency "aws-sdk-secretsmanager", "~> 1.46"
   s.bindir       = "bin"
   s.executables  = %w{ }
 

data/lib/chef/client.rb

@@ -751,7 +751,7 @@ class Chef
     end
 
     # Notification registration
-    class<<self
+    class << self
       #
       # Add a listener for the 'client run started' event.
       #

data/lib/chef/data_bag.rb

@@ -32,8 +32,7 @@ class Chef
     include Chef::Mixin::FromFile
     include Chef::Mixin::ParamsValidate
 
-    # Regex reference: https://rubular.com/r/oIMySIO4USPm5x
-    VALID_NAME = /^[\-[:alnum:]_]+$/.freeze
+    VALID_NAME = /^[\.\-[:alnum:]_]+$/.freeze
     RESERVED_NAMES = /^(node|role|environment|client)$/.freeze
 
     def self.validate_name!(name)

data/lib/chef/data_bag_item.rb

@@ -36,8 +36,7 @@ class Chef
     include Chef::Mixin::FromFile
     include Chef::Mixin::ParamsValidate
 
-    # Regex reference: https://rubular.com/r/oIMySIO4USPm5x
-    VALID_ID = /^[\-[:alnum:]_]+$/.freeze
+    VALID_ID = /^[\.\-[:alnum:]_]+$/.freeze
 
     def self.validate_id!(id_str)
       if id_str.nil? || ( id_str !~ VALID_ID )

data/lib/chef/deprecated.rb

@@ -79,10 +79,12 @@ class Chef
         return true if location =~ /^(.*?):(\d+):in/ && begin
           # Don't buffer the whole file in memory, so read it one line at a time.
           line_no = $2.to_i
-          location_file = ::File.open($1)
-          (line_no - 1).times { location_file.readline } # Read all the lines we don't care about.
-          relevant_line = location_file.readline
-          relevant_line.match?(/#.*chef:silence_deprecation($|[^:]|:#{self.class.deprecation_key})/)
+          if File.exist?($1) # some stacktraces come from `eval` and not a file
+            location_file = ::File.open($1)
+            (line_no - 1).times { location_file.readline } # Read all the lines we don't care about.
+            relevant_line = location_file.readline
+            relevant_line.match?(/#.*chef:silence_deprecation($|[^:]|:#{self.class.deprecation_key})/)
+          end
         end
 
         false
@@ -257,6 +259,10 @@ class Chef
       target 34
     end
 
+    class PolicyfileCompatMode < Base
+      target 35
+    end
+
     class Generic < Base
       def url
         "https://docs.chef.io/chef_deprecations_client/"

data/lib/chef/dsl.rb

@@ -4,3 +4,4 @@ require_relative "dsl/data_query"
 require_relative "dsl/include_recipe"
 require_relative "dsl/include_attribute"
 require_relative "dsl/registry_helper"
+require_relative "dsl/secret"

data/lib/chef/dsl/render_helpers.rb

@@ -0,0 +1,44 @@
+#
+# Copyright:: Copyright (c) Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require_relative "toml"
+require_relative "../json_compat"
+autoload :YAML, "yaml"
+
+class Chef
+  module DSL
+    module RenderHelpers
+
+      # pretty-print a hash as a JSON string
+      def render_json(hash)
+        JSON.pretty_generate(hash) + "\n"
+      end
+
+      # pretty-print a hash as a TOML string
+      def render_toml(hash)
+        Chef::DSL::Toml::Dumper.new(hash).toml_str
+      end
+
+      # pretty-print a hash as a YAML string
+      def render_yaml(hash)
+        yaml_content = hash.transform_keys(&:to_s).to_yaml
+        # above replaces first-level keys with strings, below the rest
+        yaml_content.gsub!(" :", " ")
+      end
+
+      extend self
+    end
+  end
+end

data/lib/chef/dsl/secret.rb

@@ -0,0 +1,64 @@
+#
+# Author:: Marc Paradise (<marc@chef.io>)
+# Copyright:: Copyright (c) Chef Software Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require_relative "../secret_fetcher"
+
+class Chef
+  module DSL
+    module Secret
+
+      # Helper method which looks up a secret using the given service and configuration,
+      # and returns the retrieved secret value.
+      # This DSL providers a wrapper around [Chef::SecretFetcher]
+      #
+      # Use of the secret helper in the context of a resource block will automatically mark
+      # that resource as 'sensitive', preventing resource data from being logged.  See [Chef::Resource#sensitive].
+      #
+      # @option name [Object] The identifier or name for this secret
+      # @option version [Object] The secret version. If a service supports versions
+      #                          and no version is provided, the latest version will be fetched.
+      # @option service [Symbol] The service identifier for the service that will
+      #                         perform the secret lookup. See
+      #                         [Chef::SecretFetcher::SECRET_FETCHERS]
+      # @option config [Hash] The configuration that the named service expects
+      #
+      # @return result [Object] The response object type is determined by the fetcher but will usually be a string or a hash.
+      # See individual fetcher documentation to know what to expect for a given service.
+      #
+      # @example
+      #
+      # This example uses the built-in :example secret manager service, which
+      # accepts a hash of secrets.
+      #
+      #   value = secret(name: "test1", service: :example, config: { "test1" => "value1" })
+      #   log "My secret is #{value}"
+      #
+      #   value = secret(name: "test1", service: :aws_secrets_manager, version: "v1", config: { region: "us-west-1" })
+      #   log "My secret is #{value}"
+      def secret(name: nil, version: nil, service: nil, config: nil)
+        Chef::Log.warn <<~EOM.gsub("\n", "")
+          The secrets Chef Infra language helper is currently in beta.
+          This helper will most likely change over time in potentially breaking ways.
+          If you have feedback or you'd like to be part of the future design of this
+          helper e-mail us at secrets_management_beta@progress.com"
+        EOM
+        sensitive(true) if is_a?(Chef::Resource)
+        Chef::SecretFetcher.for_service(service, config).fetch(name, version)
+      end
+    end
+  end
+end

data/lib/chef/dsl/toml.rb

@@ -0,0 +1,116 @@
+require "date"
+
+# imported from https://github.com/chef-cookbooks/habitat
+class Chef
+  module DSL
+    module Toml
+      class Dumper
+        attr_reader :toml_str
+
+        def initialize(hash)
+          @toml_str = ""
+
+          visit(hash, [])
+        end
+
+        private
+
+        def visit(hash, prefix, extra_brackets = false)
+          simple_pairs, nested_pairs, table_array_pairs = sort_pairs hash
+
+          if prefix.any? && (simple_pairs.any? || hash.empty?)
+            print_prefix prefix, extra_brackets
+          end
+
+          dump_pairs simple_pairs, nested_pairs, table_array_pairs, prefix
+        end
+
+        def sort_pairs(hash)
+          nested_pairs = []
+          simple_pairs = []
+          table_array_pairs = []
+
+          hash.keys.sort.each do |key|
+            val = hash[key]
+            element = [key, val]
+
+            if val.is_a? Hash
+              nested_pairs << element
+            elsif val.is_a?(Array) && val.first.is_a?(Hash)
+              table_array_pairs << element
+            else
+              simple_pairs << element
+            end
+          end
+
+          [simple_pairs, nested_pairs, table_array_pairs]
+        end
+
+        def dump_pairs(simple, nested, table_array, prefix = [])
+          # First add simple pairs, under the prefix
+          dump_simple_pairs simple
+          dump_nested_pairs nested, prefix
+          dump_table_array_pairs table_array, prefix
+        end
+
+        def dump_simple_pairs(simple_pairs)
+          simple_pairs.each do |key, val|
+            key = quote_key(key) unless bare_key? key
+            @toml_str << "#{key} = #{to_toml(val)}\n"
+          end
+        end
+
+        def dump_nested_pairs(nested_pairs, prefix)
+          nested_pairs.each do |key, val|
+            key = quote_key(key) unless bare_key? key
+
+            visit val, prefix + [key], false
+          end
+        end
+
+        def dump_table_array_pairs(table_array_pairs, prefix)
+          table_array_pairs.each do |key, val|
+            key = quote_key(key) unless bare_key? key
+            aux_prefix = prefix + [key]
+
+            val.each do |child|
+              print_prefix aux_prefix, true
+              args = sort_pairs(child) << aux_prefix
+
+              dump_pairs(*args)
+            end
+          end
+        end
+
+        def print_prefix(prefix, array = false)
+          new_prefix = prefix.join(".")
+          new_prefix = "[#{new_prefix}]" if array
+
+          @toml_str += "[#{new_prefix}]\n"
+        end
+
+        def to_toml(obj)
+          if obj.is_a?(Time) || obj.is_a?(DateTime)
+            obj.strftime("%Y-%m-%dT%H:%M:%SZ")
+          elsif obj.is_a?(Date)
+            obj.strftime("%Y-%m-%d")
+          elsif obj.is_a? Regexp
+            obj.inspect.inspect
+          elsif obj.is_a? String
+            obj.inspect.gsub(/\\(#[$@{])/, '\1')
+          else
+            obj.inspect
+          end
+        end
+
+        def bare_key?(key)
+          !!key.to_s.match(/^[a-zA-Z0-9_-]*$/)
+        end
+
+        def quote_key(key)
+          '"' + key.gsub('"', '\\"') + '"'
+        end
+      end
+    end
+  end
+end

data/lib/chef/dsl/universal.rb

@@ -22,6 +22,9 @@ require_relative "data_query"
 require_relative "chef_vault"
 require_relative "registry_helper"
 require_relative "powershell"
+require_relative "secret"
+require_relative "render_helpers"
+require_relative "toml"
 require_relative "../mixin/powershell_exec"
 require_relative "../mixin/powershell_out"
 require_relative "../mixin/shell_out"
@@ -47,6 +50,8 @@ class Chef
       include Chef::DSL::ChefVault
       include Chef::DSL::RegistryHelper
       include Chef::DSL::Powershell
+      include Chef::DSL::RenderHelpers
+      include Chef::DSL::Secret
       include Chef::Mixin::PowershellExec
       include Chef::Mixin::PowershellOut
       include Chef::Mixin::ShellOut

data/lib/chef/exceptions.rb

@@ -290,6 +290,28 @@ class Chef
 
     end
 
+    class Secret
+      class RetrievalError < RuntimeError; end
+      class ConfigurationInvalid < RuntimeError; end
+      class FetchFailed < RuntimeError; end
+      class MissingSecretName < RuntimeError; end
+      class InvalidSecretName < RuntimeError; end
+
+      class InvalidFetcherService < RuntimeError
+        def initialize(given, fetcher_service_names)
+          super("#{given} is not a supported secrets service.  Supported services are: :#{fetcher_service_names.join(" :")}")
+        end
+      end
+
+      class MissingFetcher < RuntimeError
+        def initialize(fetcher_service_names)
+          super("No secret service provided. Supported services are: :#{fetcher_service_names.join(" :")}")
+        end
+      end
+
+      class MissingVaultName < RuntimeError; end
+    end
+
     # Exception class for collecting multiple failures. Used when running
     # delayed notifications so that chef can process each delayed
     # notification even if chef client or other notifications fail.

data/lib/chef/handler/slow_report.rb

@@ -59,7 +59,7 @@ class Chef
 
       def stripped_source_line(resource)
         # strip the leading path off of the source line
-        resource.source_line.gsub(%r{.*/cookbooks/}, "").gsub(%r{.*/chef-[0-9\.]+/}, "")
+        resource.source_line&.gsub(%r{.*/cookbooks/}, "")&.gsub(%r{.*/chef-[0-9\.]+/}, "")
       end
     end
   end

data/lib/chef/json_compat.rb

@@ -25,7 +25,7 @@ require "json" unless defined?(JSON)
 class Chef
   class JSONCompat
 
-    class <<self
+    class << self
 
       def parse(source, opts = {})
         FFI_Yajl::Parser.parse(source, opts)

data/lib/chef/policy_builder/policyfile.rb

@@ -32,14 +32,8 @@ class Chef
     # Policyfile is a policy builder implementation that gets run
     # list and cookbook version information from a single document.
     #
-    # == Unsupported Options:
-    # * override_runlist:: This could potentially be integrated into the
-    # policyfile, or replaced with a similar feature that has different
-    # semantics.
-    # * specific_recipes:: put more design thought into this use case.
-    # * run_list in json_attribs:: would be ignored anyway, so it raises an error.
-    # * chef-solo:: not currently supported. Need more design thought around
-    # how this should work.
+    # Does not support legacy chef-solo or roles/environments.
+    #
     class Policyfile
 
       class UnsupportedFeature < StandardError; end
@@ -81,10 +75,12 @@ class Chef
       attr_reader :ohai_data
       attr_reader :json_attribs
       attr_reader :run_context
+      attr_reader :override_runlist
 
       def initialize(node_name, ohai_data, json_attribs, override_runlist, events)
         @node_name = node_name
         @ohai_data = ohai_data
+        @override_runlist = override_runlist
         @json_attribs = json_attribs
         @events = events
 
@@ -94,32 +90,11 @@ class Chef
           raise UnsupportedFeature, "Policyfile does not support chef-solo. Use #{ChefUtils::Dist::Infra::CLIENT} local mode instead."
         end
 
-        if override_runlist
-          raise UnsupportedFeature, "Policyfile does not support override run lists. Use named run_lists instead."
-        end
-
-        if json_attribs && json_attribs.key?("run_list")
-          raise UnsupportedFeature, "Policyfile does not support setting the run_list in json data."
-        end
-
         if Chef::Config[:environment] && !Chef::Config[:environment].chomp.empty?
           raise UnsupportedFeature, "Policyfile does not work with an Environment configured."
         end
       end
 
-      ## API Compat ##
-      # Methods related to unsupported features
-
-      # Override run_list is not supported.
-      def original_runlist
-        nil
-      end
-
-      # Override run_list is not supported.
-      def override_runlist
-        nil
-      end
-
       # Policyfile gives you the run_list already expanded, but users of this
       # class may expect to get a run_list expansion compatible object by
       # calling this method.
@@ -148,17 +123,27 @@ class Chef
         # consume_external_attrs may add items to the run_list. Save the
         # expanded run_list, which we will pass to the server later to
         # determine which versions of cookbooks to use.
+
+        unless Chef::Config[:policy_document_native_api]
+          Chef.deprecated(:policyfile_compat_mode, "The chef-server 11 policyfile compat mode is deprecated, please set policy_document_native_api to true in your config")
+        end
+
         node.reset_defaults_and_overrides
 
         node.consume_external_attrs(ohai_data, json_attribs)
 
+        setup_run_list_override
+
         expand_run_list
         apply_policyfile_attributes
 
+        if persistent_run_list_set?
+          Chef::Log.warn("The node.run_list setting is overriding the Policyfile run_list")
+        end
         Chef::Log.info("Run List is [#{run_list}]")
-        Chef::Log.info("Run List expands to [#{run_list_with_versions_for_display.join(", ")}]")
+        Chef::Log.info("Run List expands to [#{run_list_with_versions_for_display(run_list).join(", ")}]")
 
-        events.node_load_completed(node, run_list_with_versions_for_display, Chef::Config)
+        events.node_load_completed(node, run_list_with_versions_for_display(run_list), Chef::Config)
         events.run_list_expanded(run_list_expansion_ish)
 
         # we must do this after `node.consume_external_attrs`
@@ -194,6 +179,11 @@ class Chef
         events.cookbook_compilation_start(run_context)
 
         run_context.load(run_list_expansion_ish)
+        if specific_recipes
+          specific_recipes.each do |recipe_file|
+            run_context.load_recipe_file(recipe_file)
+          end
+        end
 
         events.cookbook_compilation_complete(run_context)
 
@@ -206,7 +196,7 @@ class Chef
       #
       # @return [RunListExpansionIsh] A RunListExpansion duck-type.
       def expand_run_list
-        CookbookCacheCleaner.instance.skip_removal = true if named_run_list_requested?
+        validate_run_list!(run_list)
 
         node.run_list(run_list)
         node.automatic_attrs[:policy_revision] = revision_id
@@ -231,21 +221,25 @@ class Chef
         cookbooks_to_sync
       end
 
-      # Whether or not this is a temporary policy. Since PolicyBuilder doesn't
-      # support override_runlist, this is always false.
+      ## Internal Public API ##
+
+      # @api private
       #
-      # @return [false]
-      def temporary_policy?
-        false
+      # Validate run_list against policyfile cookbooks
+      #
+      def validate_run_list!(run_list)
+        run_list.map do |recipe_spec|
+          cookbook, recipe = parse_recipe_spec(recipe_spec)
+          lock_data = cookbook_lock_for(cookbook)
+          raise PolicyfileError, "invalid run_list item '#{recipe_spec}' not in cookbook set of PolicyFile #{policyfile_location}" unless lock_data
+        end
       end
 
-      ## Internal Public API ##
-
       # @api private
       #
       # Generates an array of strings with recipe names including version and
       # identifier info.
-      def run_list_with_versions_for_display
+      def run_list_with_versions_for_display(run_list)
         run_list.map do |recipe_spec|
           cookbook, recipe = parse_recipe_spec(recipe_spec)
           lock_data = cookbook_lock_for(cookbook)
@@ -287,9 +281,14 @@ class Chef
 
       # @api private
       def parse_recipe_spec(recipe_spec)
-        rmatch = recipe_spec.match(/recipe\[([^:]+)::([^:]+)\]/)
+        rmatch = recipe_spec.to_s.match(/recipe\[([^:]+)::([^:]+)\]/)
         if rmatch.nil?
-          raise PolicyfileError, "invalid recipe specification #{recipe_spec} in Policyfile from #{policyfile_location}"
+          rmatch = recipe_spec.to_s.match(/recipe\[([^:]+)\]/)
+          if rmatch.nil?
+            raise PolicyfileError, "invalid recipe specification #{recipe_spec} in Policyfile from #{policyfile_location}"
+          else
+            [rmatch[1], "default"]
+          end
         else
           [rmatch[1], rmatch[2]]
         end
@@ -301,8 +300,15 @@ class Chef
       end
 
       # @api private
+      # @return [Array<String>]
       def run_list
-        if named_run_list_requested?
+        return override_runlist.map(&:to_s) if override_runlist
+
+        if json_attribs["run_list"]
+          json_attribs["run_list"]
+        elsif persistent_run_list_set?
+          node.run_list
+        elsif named_run_list_requested?
           named_run_list || raise(ConfigurationError,
             "Policy '#{retrieved_policy_name}' revision '#{revision_id}' does not have named_run_list '#{named_run_list_name}'" +
             "(available named_run_lists: [#{available_named_run_lists.join(", ")}])")
@@ -458,7 +464,7 @@ class Chef
       # should be reduced to a single call.
       def cookbooks_to_sync
         @cookbook_to_sync ||= begin
-          events.cookbook_resolution_start(run_list_with_versions_for_display)
+          events.cookbook_resolution_start(run_list_with_versions_for_display(run_list))
 
           cookbook_versions_by_name = cookbook_locks.inject({}) do |cb_map, (name, lock_data)|
             cb_map[name] = manifest_for(name, lock_data)
@@ -470,7 +476,7 @@ class Chef
         end
       rescue Exception => e
         # TODO: wrap/munge exception to provide helpful error output
-        events.cookbook_resolution_failed(run_list_with_versions_for_display, e)
+        events.cookbook_resolution_failed(run_list_with_versions_for_display(run_list), e)
         raise
       end
 
@@ -509,6 +515,13 @@ class Chef
         Chef::Config
       end
 
+      # Indicates whether the policy is temporary, which means an
+      # override_runlist was provided. Chef::Client uses this to decide whether
+      # to do the final node save at the end of the run or not.
+      def temporary_policy?
+        node.override_runlist_set?
+      end
+
       private
 
       # This method injects the run_context and into the Chef class.
@@ -533,6 +546,10 @@ class Chef
         (policy["named_run_lists"] || {}).keys
       end
 
+      def persistent_run_list_set?
+        Chef::Config[:policy_persist_run_list] && node.run_list && !node.run_list.empty?
+      end
+
       def named_run_list_requested?
         !!Chef::Config[:named_run_list]
       end
@@ -567,6 +584,32 @@ class Chef
         Chef::CookbookVersion.from_cb_artifact_data(raw_manifest)
       end
 
+      def setup_run_list_override
+        unless override_runlist.nil?
+          runlist_override_sanity_check!
+          node.override_runlist = override_runlist
+          Chef::Log.warn "Run List override has been provided."
+          Chef::Log.warn "Original Run List: [#{node.primary_runlist}]"
+          Chef::Log.warn "Overridden Run List: [#{node.run_list}]"
+        end
+      end
+
+      # Ensures runlist override contains RunListItem instances
+      def runlist_override_sanity_check!
+        # Convert to array and remove whitespace
+        if override_runlist.is_a?(String)
+          @override_runlist = override_runlist.split(",").map(&:strip)
+        end
+        @override_runlist = [override_runlist].flatten.compact
+        override_runlist.map! do |item|
+          if item.is_a?(Chef::RunList::RunListItem)
+            item
+          else
+            Chef::RunList::RunListItem.new(item)
+          end
+        end
+      end
+
     end
   end
 end