chef 16.18.30 → 17.10.163

data/lib/chef/node/immutable_collections.rb

@@ -19,6 +19,7 @@ require_relative "common_api"
 require_relative "mixin/state_tracking"
 require_relative "mixin/immutablize_array"
 require_relative "mixin/immutablize_hash"
+require_relative "../delayed_evaluator"
 
 class Chef
   class Node
@@ -109,6 +110,12 @@ class Chef
         key
       end
 
+      def [](*args)
+        value = super
+        value = value.call while value.is_a?(::Chef::DelayedEvaluator)
+        value
+      end
+
       prepend Chef::Node::Mixin::StateTracking
       prepend Chef::Node::Mixin::ImmutablizeArray
     end
@@ -187,6 +194,12 @@ class Chef
         e
       end
 
+      def [](*args)
+        value = super
+        value = value.call while value.is_a?(::Chef::DelayedEvaluator)
+        value
+      end
+
       prepend Chef::Node::Mixin::StateTracking
       prepend Chef::Node::Mixin::ImmutablizeHash
     end

data/lib/chef/node/mixin/deep_merge_cache.rb

@@ -15,6 +15,8 @@
 # limitations under the License.
 #
 
+require_relative "../../delayed_evaluator"
+
 class Chef
   class Node
     module Mixin
@@ -46,13 +48,15 @@ class Chef
         alias :reset :reset_cache
 
         def [](key)
-          if deep_merge_cache.key?(key.to_s)
-            # return the cache of the deep merged values by top-level key
-            deep_merge_cache[key.to_s]
-          else
-            # save all the work of computing node[key]
-            deep_merge_cache.internal_set(key.to_s, merged_attributes(key))
-          end
+          ret = if deep_merge_cache.key?(key.to_s)
+                  # return the cache of the deep merged values by top-level key
+                  deep_merge_cache[key.to_s]
+                else
+                  # save all the work of computing node[key]
+                  deep_merge_cache.internal_set(key.to_s, merged_attributes(key))
+                end
+          ret = ret.call while ret.is_a?(::Chef::DelayedEvaluator)
+          ret
         end
 
       end

data/lib/chef/node.rb

@@ -762,7 +762,7 @@ class Chef
       path = File.expand_path(Chef::Config[:chef_guid_path])
       dir = File.dirname(path)
 
-      unless File.exists?(path)
+      unless File.exist?(path)
         FileUtils.mkdir_p(dir)
         File.write(path, SecureRandom.uuid)
       end

data/lib/chef/org.rb

@@ -1,6 +1,6 @@
 #
 # Author:: Steven Danna (steve@chef.io)
-# Copyright:: Copyright (c) Chef Software Inc.
+# Copyright:: Copyright (c) Chef Software Inc
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,9 +19,10 @@
 require_relative "json_compat"
 require_relative "mixin/params_validate"
 require_relative "server_api"
+require_relative "group"
 
 class Chef
-  class Org
+  class Org < Group
 
     include Chef::Mixin::ParamsValidate
 

data/lib/chef/platform/query_helpers.rb

@@ -58,8 +58,8 @@ class Chef
       end
 
       def dsc_refresh_mode_disabled?(node)
-        require "chef-powershell"
-        exec = ChefPowerShell::PowerShell.new("Get-DscLocalConfigurationManager")
+        require_relative "../powershell"
+        exec = Chef::PowerShell.new("Get-DscLocalConfigurationManager")
         exec.error!
         exec.result["RefreshMode"] == "Disabled"
       end

data/lib/chef/policy_builder/expand_node_object.rb

@@ -248,8 +248,7 @@ class Chef
       end
 
       def api_service
-        @api_service ||= Chef::ServerAPI.new(config[:chef_server_url],
-          { version_class: Chef::CookbookManifestVersions })
+        @api_service ||= Chef::ServerAPI.new(config[:chef_server_url], version_class: Chef::CookbookManifestVersions )
       end
 
       def config

data/lib/chef/policy_builder/policyfile.rb

@@ -32,14 +32,8 @@ class Chef
     # Policyfile is a policy builder implementation that gets run
     # list and cookbook version information from a single document.
     #
-    # == Unsupported Options:
-    # * override_runlist:: This could potentially be integrated into the
-    # policyfile, or replaced with a similar feature that has different
-    # semantics.
-    # * specific_recipes:: put more design thought into this use case.
-    # * run_list in json_attribs:: would be ignored anyway, so it raises an error.
-    # * chef-solo:: not currently supported. Need more design thought around
-    # how this should work.
+    # Does not support legacy chef-solo or roles/environments.
+    #
     class Policyfile
 
       class UnsupportedFeature < StandardError; end
@@ -81,10 +75,12 @@ class Chef
       attr_reader :ohai_data
       attr_reader :json_attribs
       attr_reader :run_context
+      attr_reader :override_runlist
 
       def initialize(node_name, ohai_data, json_attribs, override_runlist, events)
         @node_name = node_name
         @ohai_data = ohai_data
+        @override_runlist = override_runlist
         @json_attribs = json_attribs
         @events = events
 
@@ -94,32 +90,11 @@ class Chef
           raise UnsupportedFeature, "Policyfile does not support chef-solo. Use #{ChefUtils::Dist::Infra::CLIENT} local mode instead."
         end
 
-        if override_runlist
-          raise UnsupportedFeature, "Policyfile does not support override run lists. Use named run_lists instead."
-        end
-
-        if json_attribs && json_attribs.key?("run_list")
-          raise UnsupportedFeature, "Policyfile does not support setting the run_list in json data."
-        end
-
         if Chef::Config[:environment] && !Chef::Config[:environment].chomp.empty?
           raise UnsupportedFeature, "Policyfile does not work with an Environment configured."
         end
       end
 
-      ## API Compat ##
-      # Methods related to unsupported features
-
-      # Override run_list is not supported.
-      def original_runlist
-        nil
-      end
-
-      # Override run_list is not supported.
-      def override_runlist
-        nil
-      end
-
       # Policyfile gives you the run_list already expanded, but users of this
       # class may expect to get a run_list expansion compatible object by
       # calling this method.
@@ -157,13 +132,18 @@ class Chef
 
         node.consume_external_attrs(ohai_data, json_attribs)
 
+        setup_run_list_override
+
         expand_run_list
         apply_policyfile_attributes
 
+        if persistent_run_list_set?
+          Chef::Log.warn("The node.run_list setting is overriding the Policyfile run_list")
+        end
         Chef::Log.info("Run List is [#{run_list}]")
-        Chef::Log.info("Run List expands to [#{run_list_with_versions_for_display.join(", ")}]")
+        Chef::Log.info("Run List expands to [#{run_list_with_versions_for_display(run_list).join(", ")}]")
 
-        events.node_load_completed(node, run_list_with_versions_for_display, Chef::Config)
+        events.node_load_completed(node, run_list_with_versions_for_display(run_list), Chef::Config)
         events.run_list_expanded(run_list_expansion_ish)
 
         # we must do this after `node.consume_external_attrs`
@@ -199,6 +179,11 @@ class Chef
         events.cookbook_compilation_start(run_context)
 
         run_context.load(run_list_expansion_ish)
+        if specific_recipes
+          specific_recipes.each do |recipe_file|
+            run_context.load_recipe_file(recipe_file)
+          end
+        end
 
         events.cookbook_compilation_complete(run_context)
 
@@ -211,7 +196,7 @@ class Chef
       #
       # @return [RunListExpansionIsh] A RunListExpansion duck-type.
       def expand_run_list
-        CookbookCacheCleaner.instance.skip_removal = true if named_run_list_requested?
+        validate_run_list!(run_list)
 
         node.run_list(run_list)
         node.automatic_attrs[:policy_revision] = revision_id
@@ -236,21 +221,25 @@ class Chef
         cookbooks_to_sync
       end
 
-      # Whether or not this is a temporary policy. Since PolicyBuilder doesn't
-      # support override_runlist, this is always false.
+      ## Internal Public API ##
+
+      # @api private
       #
-      # @return [false]
-      def temporary_policy?
-        false
+      # Validate run_list against policyfile cookbooks
+      #
+      def validate_run_list!(run_list)
+        run_list.map do |recipe_spec|
+          cookbook, recipe = parse_recipe_spec(recipe_spec)
+          lock_data = cookbook_lock_for(cookbook)
+          raise PolicyfileError, "invalid run_list item '#{recipe_spec}' not in cookbook set of PolicyFile #{policyfile_location}" unless lock_data
+        end
       end
 
-      ## Internal Public API ##
-
       # @api private
       #
       # Generates an array of strings with recipe names including version and
       # identifier info.
-      def run_list_with_versions_for_display
+      def run_list_with_versions_for_display(run_list)
         run_list.map do |recipe_spec|
           cookbook, recipe = parse_recipe_spec(recipe_spec)
           lock_data = cookbook_lock_for(cookbook)
@@ -292,9 +281,14 @@ class Chef
 
       # @api private
       def parse_recipe_spec(recipe_spec)
-        rmatch = recipe_spec.match(/recipe\[([^:]+)::([^:]+)\]/)
+        rmatch = recipe_spec.to_s.match(/recipe\[([^:]+)::([^:]+)\]/)
         if rmatch.nil?
-          raise PolicyfileError, "invalid recipe specification #{recipe_spec} in Policyfile from #{policyfile_location}"
+          rmatch = recipe_spec.to_s.match(/recipe\[([^:]+)\]/)
+          if rmatch.nil?
+            raise PolicyfileError, "invalid recipe specification #{recipe_spec} in Policyfile from #{policyfile_location}"
+          else
+            [rmatch[1], "default"]
+          end
         else
           [rmatch[1], rmatch[2]]
         end
@@ -306,8 +300,15 @@ class Chef
       end
 
       # @api private
+      # @return [Array<String>]
       def run_list
-        if named_run_list_requested?
+        return override_runlist.map(&:to_s) if override_runlist
+
+        if json_attribs["run_list"]
+          json_attribs["run_list"]
+        elsif persistent_run_list_set?
+          node.run_list
+        elsif named_run_list_requested?
           named_run_list || raise(ConfigurationError,
             "Policy '#{retrieved_policy_name}' revision '#{revision_id}' does not have named_run_list '#{named_run_list_name}'" +
             "(available named_run_lists: [#{available_named_run_lists.join(", ")}])")
@@ -463,7 +464,7 @@ class Chef
       # should be reduced to a single call.
       def cookbooks_to_sync
         @cookbook_to_sync ||= begin
-          events.cookbook_resolution_start(run_list_with_versions_for_display)
+          events.cookbook_resolution_start(run_list_with_versions_for_display(run_list))
 
           cookbook_versions_by_name = cookbook_locks.inject({}) do |cb_map, (name, lock_data)|
             cb_map[name] = manifest_for(name, lock_data)
@@ -475,7 +476,7 @@ class Chef
         end
       rescue Exception => e
         # TODO: wrap/munge exception to provide helpful error output
-        events.cookbook_resolution_failed(run_list_with_versions_for_display, e)
+        events.cookbook_resolution_failed(run_list_with_versions_for_display(run_list), e)
         raise
       end
 
@@ -506,7 +507,7 @@ class Chef
       # @api private
       def api_service
         @api_service ||= Chef::ServerAPI.new(config[:chef_server_url],
-          { version_class: Chef::CookbookManifestVersions })
+          version_class: Chef::CookbookManifestVersions)
       end
 
       # @api private
@@ -514,6 +515,13 @@ class Chef
         Chef::Config
       end
 
+      # Indicates whether the policy is temporary, which means an
+      # override_runlist was provided. Chef::Client uses this to decide whether
+      # to do the final node save at the end of the run or not.
+      def temporary_policy?
+        node.override_runlist_set?
+      end
+
       private
 
       # This method injects the run_context and into the Chef class.
@@ -538,6 +546,10 @@ class Chef
         (policy["named_run_lists"] || {}).keys
       end
 
+      def persistent_run_list_set?
+        Chef::Config[:policy_persist_run_list] && node.run_list && !node.run_list.empty?
+      end
+
       def named_run_list_requested?
         !!Chef::Config[:named_run_list]
       end
@@ -572,6 +584,32 @@ class Chef
         Chef::CookbookVersion.from_cb_artifact_data(raw_manifest)
       end
 
+      def setup_run_list_override
+        unless override_runlist.nil?
+          runlist_override_sanity_check!
+          node.override_runlist = override_runlist
+          Chef::Log.warn "Run List override has been provided."
+          Chef::Log.warn "Original Run List: [#{node.primary_runlist}]"
+          Chef::Log.warn "Overridden Run List: [#{node.run_list}]"
+        end
+      end
+
+      # Ensures runlist override contains RunListItem instances
+      def runlist_override_sanity_check!
+        # Convert to array and remove whitespace
+        if override_runlist.is_a?(String)
+          @override_runlist = override_runlist.split(",").map(&:strip)
+        end
+        @override_runlist = [override_runlist].flatten.compact
+        override_runlist.map! do |item|
+          if item.is_a?(Chef::RunList::RunListItem)
+            item
+          else
+            Chef::RunList::RunListItem.new(item)
+          end
+        end
+      end
+
     end
   end
 end

data/lib/chef/powershell.rb

@@ -33,15 +33,16 @@ class Chef
     # Requires: .NET Framework 4.0 or higher on the target machine.
     #
     # @param script [String] script to run
+    # @param timeout [Integer, nil] timeout in seconds.
     # @return [Object] output
-    def initialize(script)
+    def initialize(script, timeout: -1)
       # This Powershell DLL source lives here: https://github.com/chef/chef-powershell-shim
       # Every merge into that repo triggers a Habitat build and promotion. Running
       # the rake :update_chef_exec_dll task in this (chef/chef) repo will pull down
       # the built packages and copy the binaries to distro/ruby_bin_folder. Bundle install
       # ensures that the correct architecture binaries are installed into the path.
       @dll ||= "Chef.PowerShell.Wrapper.dll"
-      exec(script)
+      exec(script, timeout: timeout)
     end
 
     #
@@ -64,12 +65,13 @@ class Chef
       raise Chef::PowerShell::CommandFailed, "Unexpected exit in PowerShell command: #{@errors}" if error?
     end
 
-    protected
+    private
 
-    def exec(script)
+    def exec(script, timeout: -1)
       FFI.ffi_lib @dll
-      FFI.attach_function :execute_powershell, :ExecuteScript, [:string], :pointer
-      execution = FFI.execute_powershell(script).read_utf16string
+      FFI.attach_function :execute_powershell, :ExecuteScript, %i{string int}, :pointer
+      timeout = -1 if timeout == 0 || timeout.nil?
+      execution = FFI.execute_powershell(script, timeout).read_utf16string
       hashed_outcome = Chef::JSONCompat.parse(execution)
       @result = Chef::JSONCompat.parse(hashed_outcome["result"])
       @errors = hashed_outcome["errors"]

data/lib/chef/property.rb

@@ -412,6 +412,7 @@ class Chef
             # Otherwise, we have to validate it now.
             value = input_to_stored_value(resource, default, is_default: true)
           end
+          value = deep_dup(value)
           value = stored_value_to_output(resource, value)
 
           # If the value is mutable (non-frozen), we set it on the instance
@@ -748,5 +749,22 @@ class Chef
 
       result
     end
+
+    # recursively dup the value
+    def deep_dup(value)
+      return value if value.is_a?(DelayedEvaluator)
+
+      visitor = lambda do |obj|
+        obj = obj.dup rescue obj
+        case obj
+        when Hash
+          obj.each { |k, v| obj[k] = visitor.call(v) }
+        when Array
+          obj.each_with_index { |v, i| obj[i] = visitor.call(v) }
+        end
+        obj
+      end
+      visitor.call(value)
+    end
   end
 end

data/lib/chef/provider/cron.rb

@@ -96,7 +96,7 @@ class Chef
         end
       end
 
-      action :create do
+      action :create, description: "Create an entry in a cron table file (crontab). If an entry already exists (but does not match), update that entry to match." do
         crontab = ""
         newcron = ""
         cron_found = false
@@ -104,8 +104,11 @@ class Chef
         newcron = get_crontab_entry
 
         if @cron_exists
-          unless cron_different?
-            logger.trace("Skipping existing cron entry '#{new_resource.name}'")
+          # Only compare the crontab if the current resource has a set command.
+          # This may not be set in cases where the Chef comment exists but the
+          # crontab command was commented out.
+          if current_resource.property_is_set?(:command) && !cron_different?
+            logger.debug("#{new_resource}: Skipping existing cron entry")
             return
           end
           read_crontab.each_line do |line|
@@ -150,7 +153,7 @@ class Chef
         end
       end
 
-      action :delete do
+      action :delete, description: "Delete an entry from a cron table file (crontab)." do
         if @cron_exists
           crontab = ""
           cron_found = false

data/lib/chef/provider/directory.rb

@@ -32,7 +32,7 @@ class Chef
       def load_current_resource
         @current_resource = Chef::Resource::Directory.new(new_resource.name)
         current_resource.path(new_resource.path)
-        if ::File.exists?(current_resource.path) && @action != :create_if_missing
+        if ::File.exist?(current_resource.path) && @action != :create_if_missing
           load_resource_attributes_from_file(current_resource)
         end
         current_resource
@@ -73,7 +73,7 @@ class Chef
               # make sure we have write permissions to that directory
               is_parent_writable = lambda do |base_dir|
                 base_dir = ::File.dirname(base_dir)
-                if ::File.exists?(base_dir)
+                if ::File.exist?(base_dir)
                   if Chef::FileAccessControl.writable?(base_dir)
                     true
                   elsif Chef::Util::PathHelper.is_sip_path?(base_dir, node)
@@ -89,7 +89,7 @@ class Chef
             else
               # in why run mode & parent directory does not exist no permissions check is required
               # If not in why run, permissions must be valid and we rely on prior assertion that dir exists
-              if !whyrun_mode? || ::File.exists?(parent_directory)
+              if !whyrun_mode? || ::File.exist?(parent_directory)
                 if Chef::FileAccessControl.writable?(parent_directory)
                   true
                 elsif Chef::Util::PathHelper.is_sip_path?(parent_directory, node)
@@ -108,7 +108,7 @@ class Chef
 
         requirements.assert(:delete) do |a|
           a.assertion do
-            if ::File.exists?(new_resource.path)
+            if ::File.exist?(new_resource.path)
               ::File.directory?(new_resource.path) && Chef::FileAccessControl.writable?(new_resource.path)
             else
               true
@@ -121,8 +121,8 @@ class Chef
         end
       end
 
-      action :create do
-        unless ::File.exists?(new_resource.path)
+      action :create, description: "Create a directory. If a directory already exists (but does not match), update that directory to match." do
+        unless ::File.exist?(new_resource.path)
           converge_by("create new directory #{new_resource.path}") do
             if new_resource.recursive == true
               ::FileUtils.mkdir_p(new_resource.path)
@@ -137,8 +137,8 @@ class Chef
         load_resource_attributes_from_file(new_resource) unless Chef::Config[:why_run]
       end
 
-      action :delete do
-        if ::File.exists?(new_resource.path)
+      action :delete, description: "Delete a directory." do
+        if ::File.exist?(new_resource.path)
           converge_by("delete existing directory #{new_resource.path}") do
             if new_resource.recursive == true
               # we don't use rm_rf here because it masks all errors, including

data/lib/chef/provider/execute.rb

@@ -27,7 +27,7 @@ class Chef
 
       provides :execute, target_mode: true
 
-      def_delegators :new_resource, :command, :returns, :environment, :user, :domain, :password, :group, :cwd, :umask, :creates, :elevated, :default_env, :timeout, :input
+      def_delegators :new_resource, :command, :returns, :environment, :user, :domain, :password, :group, :cwd, :umask, :creates, :elevated, :default_env, :timeout, :input, :login
 
       def load_current_resource
         current_resource = Chef::Resource::Execute.new(new_resource.name)
@@ -41,7 +41,7 @@ class Chef
         end
       end
 
-      action :run do
+      action :run, description: "Run a command." do
         if creates && sentinel_file.exist?
           logger.debug("#{new_resource} sentinel file #{sentinel_file} exists - nothing to do")
           return false
@@ -92,6 +92,7 @@ class Chef
         opts[:cwd]         = cwd if cwd
         opts[:umask]       = umask if umask
         opts[:input]       = input if input
+        opts[:login]       = login if login
         opts[:default_env] = default_env
         opts[:log_level]   = :info
         opts[:log_tag]     = new_resource.to_s

data/lib/chef/provider/file.rb

@@ -27,6 +27,8 @@ require_relative "../scan_access_control"
 require_relative "../mixin/checksum"
 require_relative "../mixin/file_class"
 require_relative "../mixin/enforce_ownership_and_permissions"
+require_relative "../resource/file/verification/json"
+require_relative "../resource/file/verification/yaml"
 require_relative "../util/backup"
 require_relative "../util/diff"
 require_relative "../util/selinux"
@@ -152,12 +154,12 @@ class Chef
         unless ::File.exist?(new_resource.path)
           action_create
         else
-          logger.trace("#{new_resource} exists at #{new_resource.path} taking no action.")
+          logger.debug("#{new_resource} exists at #{new_resource.path} taking no action.")
         end
       end
 
       action :delete do
-        if ::File.exists?(new_resource.path)
+        if ::File.exist?(new_resource.path)
           converge_by("delete file #{new_resource.path}") do
             do_backup unless file_class.symlink?(new_resource.path)
             ::File.delete(new_resource.path)
@@ -393,7 +395,7 @@ class Chef
         # a nil tempfile is okay, means the resource has no content or no new content
         return if tempfile.nil?
         # but a tempfile that has no path or doesn't exist should not happen
-        if tempfile.path.nil? || !::File.exists?(tempfile.path)
+        if tempfile.path.nil? || !::File.exist?(tempfile.path)
           raise "#{ChefUtils::Dist::Infra::CLIENT} is confused, trying to deploy a file that has no path or does not exist..."
         end
 

data/lib/chef/provider/git.rb

@@ -28,7 +28,7 @@ class Chef
       extend Forwardable
       provides :git
 
-      GIT_VERSION_PATTERN = Regexp.compile('git version (\d+\.\d+.\d+)')
+      GIT_VERSION_PATTERN = Regexp.compile("git version (\\d+\\.\\d+.\\d+)")
 
       def_delegator :new_resource, :destination, :cwd
 
@@ -94,7 +94,7 @@ class Chef
           enable_submodules
           add_remotes
         else
-          logger.trace "#{new_resource} checkout destination #{cwd} already exists or is a non-empty directory"
+          logger.debug "#{new_resource} checkout destination #{cwd} already exists or is a non-empty directory"
         end
       end
 
@@ -281,12 +281,10 @@ class Chef
 
       def target_revision
         @target_revision ||=
-          begin
-            if sha_hash?(new_resource.revision)
-              @target_revision = new_resource.revision
-            else
-              @target_revision = remote_resolve_reference
-            end
+          if sha_hash?(new_resource.revision)
+            @target_revision = new_resource.revision
+          else
+            @target_revision = remote_resolve_reference
           end
       end
 

data/lib/chef/provider/group/groupadd.rb

@@ -67,7 +67,7 @@ class Chef
                 members_to_be_added << member unless current_resource.members.include?(member)
               end
               members_to_be_added.each do |member|
-                logger.trace("#{new_resource} appending member #{member} to group #{new_resource.group_name}")
+                logger.debug("#{new_resource} appending member #{member} to group #{new_resource.group_name}")
                 add_member(member)
               end
             end
@@ -79,13 +79,13 @@ class Chef
               end
 
               members_to_be_removed.each do |member|
-                logger.trace("#{new_resource} removing member #{member} from group #{new_resource.group_name}")
+                logger.debug("#{new_resource} removing member #{member} from group #{new_resource.group_name}")
                 remove_member(member)
               end
             end
           else
             members_description = new_resource.members.empty? ? "none" : new_resource.members.join(", ")
-            logger.trace("#{new_resource} setting group members to: #{members_description}")
+            logger.debug("#{new_resource} setting group members to: #{members_description}")
             set_members(new_resource.members)
           end
         end