chef 12.5.0.alpha.1 → 12.5.1

data/lib/chef/platform/service_helpers.rb

@@ -19,6 +19,7 @@
 # XXX: mixing shellout into a mixin into classes has to be code smell
 require 'chef/mixin/shell_out'
 require 'chef/mixin/which'
+require 'chef/chef_class'
 
 class Chef
   class Platform
@@ -42,56 +43,56 @@ class Chef
         # different services is NOT a design concern of this module.
         #
         def service_resource_providers
-          @service_resource_providers ||= [].tap do |service_resource_providers|
+          providers = []
 
-            if ::File.exist?("/usr/sbin/update-rc.d")
-              service_resource_providers << :debian
-            end
-
-            if ::File.exist?("/usr/sbin/invoke-rc.d")
-              service_resource_providers << :invokercd
-            end
+          if ::File.exist?(Chef.path_to("/usr/sbin/update-rc.d"))
+            providers << :debian
+          end
 
-            if ::File.exist?("/sbin/insserv")
-              service_resource_providers << :insserv
-            end
+          if ::File.exist?(Chef.path_to("/usr/sbin/invoke-rc.d"))
+            providers << :invokercd
+          end
 
-            # debian >= 6.0 has /etc/init but does not have upstart
-            if ::File.exist?("/etc/init") && ::File.exist?("/sbin/start")
-              service_resource_providers << :upstart
-            end
+          if ::File.exist?(Chef.path_to("/sbin/insserv"))
+            providers << :insserv
+          end
 
-            if ::File.exist?("/sbin/chkconfig")
-              service_resource_providers << :redhat
-            end
+          # debian >= 6.0 has /etc/init but does not have upstart
+          if ::File.exist?(Chef.path_to("/etc/init")) && ::File.exist?(Chef.path_to("/sbin/start"))
+            providers << :upstart
+          end
 
-            if systemd_sanity_check?
-              service_resource_providers << :systemd
-            end
+          if ::File.exist?(Chef.path_to("/sbin/chkconfig"))
+            providers << :redhat
+          end
 
+          if systemd_sanity_check?
+            providers << :systemd
           end
+
+          providers
         end
 
         def config_for_service(service_name)
           configs = []
 
-          if ::File.exist?("/etc/init.d/#{service_name}")
+          if ::File.exist?(Chef.path_to("/etc/init.d/#{service_name}"))
             configs << :initd
           end
 
-          if ::File.exist?("/etc/init/#{service_name}.conf")
+          if ::File.exist?(Chef.path_to("/etc/init/#{service_name}.conf"))
             configs << :upstart
           end
 
-          if ::File.exist?("/etc/xinetd.d/#{service_name}")
+          if ::File.exist?(Chef.path_to("/etc/xinetd.d/#{service_name}"))
             configs << :xinetd
           end
 
-          if ::File.exist?("/etc/rc.d/#{service_name}")
+          if ::File.exist?(Chef.path_to("/etc/rc.d/#{service_name}"))
             configs << :etc_rcd
           end
 
-          if ::File.exist?("/usr/local/etc/rc.d/#{service_name}")
+          if ::File.exist?(Chef.path_to("/usr/local/etc/rc.d/#{service_name}"))
             configs << :usr_local_etc_rcd
           end
 
@@ -105,14 +106,11 @@ class Chef
         private
 
         def systemctl_path
-          if @systemctl_path.nil?
-            @systemctl_path = which("systemctl")
-          end
-          @systemctl_path
+          which("systemctl")
         end
 
         def systemd_sanity_check?
-          systemctl_path && File.exist?("/proc/1/comm") && File.open("/proc/1/comm").gets.chomp == "systemd"
+          systemctl_path && File.exist?(Chef.path_to("/proc/1/comm")) && File.open(Chef.path_to("/proc/1/comm")).gets.chomp == "systemd"
         end
 
         def extract_systemd_services(command)
@@ -126,7 +124,7 @@ class Chef
           # this splits off the suffix after the last dot to return "sshd"
           services += services.select {|s| s.match(/\.service$/) }.map { |s| s.sub(/(.*)\.service$/, '\1') }
         rescue Mixlib::ShellOut::ShellCommandFailed
-          false
+          []
         end
 
         def platform_has_systemd_unit?(service_name)

data/lib/chef/policy_builder.rb

@@ -18,6 +18,7 @@
 
 require 'chef/policy_builder/expand_node_object'
 require 'chef/policy_builder/policyfile'
+require 'chef/policy_builder/dynamic'
 
 class Chef
 
@@ -37,13 +38,5 @@ class Chef
   # * cookbook_hash is stored in run_context
   module PolicyBuilder
 
-    def self.strategy
-      if Chef::Config[:use_policyfile]
-        Policyfile
-      else
-        ExpandNodeObject
-      end
-    end
-
   end
 end

data/lib/chef/policy_builder/dynamic.rb

@@ -0,0 +1,186 @@
+#
+# Author:: Daniel DeLeo (<dan@chef.io>)
+# Copyright:: Copyright 2015 Chef Software, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'forwardable'
+
+require 'chef/log'
+require 'chef/rest'
+require 'chef/run_context'
+require 'chef/config'
+require 'chef/node'
+require 'chef/exceptions'
+
+class Chef
+  module PolicyBuilder
+
+    # PolicyBuilder that selects either a Policyfile or non-Policyfile
+    # implementation based on the content of the node object.
+    class Dynamic
+
+      extend Forwardable
+
+      attr_reader :node
+      attr_reader :node_name
+      attr_reader :ohai_data
+      attr_reader :json_attribs
+      attr_reader :override_runlist
+      attr_reader :events
+
+      def initialize(node_name, ohai_data, json_attribs, override_runlist, events)
+        @implementation = nil
+
+        @node_name = node_name
+        @ohai_data = ohai_data
+        @json_attribs = json_attribs
+        @override_runlist = override_runlist
+        @events = events
+
+        @node = nil
+      end
+
+      ## PolicyBuilder API ##
+
+      # Loads the node state from the server, then picks the correct
+      # implementation class based on the node and json_attribs.
+      #
+      # Calls #finish_load_node on the implementation object to complete the
+      # loading process. All subsequent lifecycle calls are delegated.
+      #
+      # @return [Chef::Node] the loaded node.
+      def load_node
+        events.node_load_start(node_name, config)
+        Chef::Log.debug("Building node object for #{node_name}")
+
+        @node =
+          if Chef::Config[:solo]
+            Chef::Node.build(node_name)
+          else
+            Chef::Node.find_or_create(node_name)
+          end
+        select_implementation(node)
+        implementation.finish_load_node(node)
+        node
+      rescue Exception => e
+        events.node_load_failed(node_name, e, config)
+        raise
+      end
+
+      ## Delegated Public API Methods ##
+
+      ### Accessors ###
+
+      def_delegator :implementation, :original_runlist
+      def_delegator :implementation, :run_context
+      def_delegator :implementation, :run_list_expansion
+
+      ### Lifecycle Methods ###
+
+      # @!method build_node
+      #
+      # Applies external attributes (e.g., from JSON file, environment,
+      # policyfile, etc.) and determines the correct expanded run list for the
+      # run.
+      #
+      # @return [Chef::Node]
+      def_delegator :implementation, :build_node
+
+      # @!method setup_run_context
+      #
+      # Synchronizes cookbooks and initializes the run context object for the
+      # run.
+      #
+      # @return [Chef::RunContext]
+      def_delegator :implementation, :setup_run_context
+
+      # @!method expanded_run_list
+      #
+      # Resolves the run list to a form containing only recipes and sets the
+      # `roles` and `recipes` automatic attributes on the node.
+      #
+      # @return [#recipes, #roles] A RunListExpansion or duck-type.
+      def_delegator :implementation, :expand_run_list
+
+      # @!method sync_cookbooks
+      #
+      # Synchronizes cookbooks. In a normal chef-client run, this is handled by
+      # #setup_run_context, but may be called directly in some circumstances.
+      #
+      # @return [Hash{String => Chef::CookbookManifest}] A map of
+      #   CookbookManifest objects by cookbook name.
+      def_delegator :implementation, :sync_cookbooks
+
+      # @!method temporary_policy?
+      #
+      # Indicates whether the policy is temporary, which means an
+      # override_runlist was provided. Chef::Client uses this to decide whether
+      # to do the final node save at the end of the run or not.
+      #
+      # @return [true,false]
+      def_delegator :implementation, :temporary_policy?
+
+      ## Internal Public API ##
+
+      # Returns the selected implementation, or raises if not set. The
+      # implementation is set when #load_node is called.
+      #
+      # @return [PolicyBuilder::Policyfile, PolicyBuilder::ExpandNodeObject]
+      def implementation
+        @implementation or raise Exceptions::InvalidPolicybuilderCall, "#load_node must be called before other policy builder methods"
+      end
+
+      # @api private
+      #
+      # Sets the implementation based on the content of the node, node JSON
+      # (i.e., the `-j JSON_FILE` data), and config. This is only public for
+      # testing purposes; production code should call #load_node instead.
+      def select_implementation(node)
+        if policyfile_set_in_config? ||
+            policyfile_attribs_in_node_json? ||
+            node_has_policyfile_attrs?(node) ||
+            policyfile_compat_mode_config?
+          @implementation = Policyfile.new(node_name, ohai_data, json_attribs, override_runlist, events)
+        else
+          @implementation = ExpandNodeObject.new(node_name, ohai_data, json_attribs, override_runlist, events)
+        end
+      end
+
+      def config
+        Chef::Config
+      end
+
+      private
+
+      def node_has_policyfile_attrs?(node)
+        node.policy_name || node.policy_group
+      end
+
+      def policyfile_attribs_in_node_json?
+        json_attribs.key?("policy_name") || json_attribs.key?("policy_group")
+      end
+
+      def policyfile_set_in_config?
+        config[:use_policyfile] || config[:policy_name] || config[:policy_group]
+      end
+
+      def policyfile_compat_mode_config?
+        config[:deployment_group] && !config[:policy_document_native_api]
+      end
+
+    end
+  end
+end

data/lib/chef/policy_builder/expand_node_object.rb

@@ -33,6 +33,9 @@ class Chef
     # expands the run_list on a node object and then queries the chef-server
     # to find the correct set of cookbooks, given version constraints of the
     # node's environment.
+    #
+    # Note that this class should only be used via PolicyBuilder::Dynamic and
+    # not instantiated directly.
     class ExpandNodeObject
 
       attr_reader :events
@@ -55,9 +58,10 @@ class Chef
         @run_list_expansion = nil
       end
 
-      # This method injects the run_context and provider and resource priority
-      # maps into the Chef class.  The run_context has to be injected here, the provider and
-      # resource maps could be moved if a better place can be found to do this work.
+      # This method injects the run_context and into the Chef class.
+      #
+      # NOTE: This is duplicated with the Policyfile implementation. If
+      # it gets any more complicated, it needs to be moved elsewhere.
       #
       # @param run_context [Chef::RunContext] the run_context to inject
       def setup_chef_class(run_context)
@@ -93,25 +97,36 @@ class Chef
         run_context
       end
 
-
-      # In client-server operation, loads the node state from the server. In
-      # chef-solo operation, builds a new node object.
+      # DEPRECATED: As of Chef 12.5, chef selects either policyfile mode or
+      # "expand node" mode dynamically, based on the content of the node
+      # object, first boot JSON, and config. This happens in
+      # PolicyBuilder::Dynamic, which selects the implementation during
+      # #load_node and then delegates to either ExpandNodeObject or Policyfile
+      # implementations as appropriate. Tools authors should update their code
+      # to create a PolicyBuilder::Dynamc policy builder and allow it to select
+      # the proper implementation.
       def load_node
-        events.node_load_start(node_name, Chef::Config)
+        Chef.log_deprecation("ExpandNodeObject#load_node is deprecated. Please use Chef::PolicyBuilder::Dynamic instead of using ExpandNodeObject directly")
+
+        events.node_load_start(node_name, config)
         Chef::Log.debug("Building node object for #{node_name}")
 
-        if Chef::Config[:solo]
-          @node = Chef::Node.build(node_name)
-        else
-          @node = Chef::Node.find_or_create(node_name)
-        end
+        @node =
+          if Chef::Config[:solo]
+            Chef::Node.build(node_name)
+          else
+            Chef::Node.find_or_create(node_name)
+          end
+        finish_load_node(node)
+        node
       rescue Exception => e
-        # TODO: wrap this exception so useful error info can be given to the
-        # user.
-        events.node_load_failed(node_name, e, Chef::Config)
+        events.node_load_failed(node_name, e, config)
         raise
       end
 
+      def finish_load_node(node)
+        @node = node
+      end
 
       # Applies environment, external JSON attributes, and override run list to
       # the node, Then expands the run_list.

data/lib/chef/policy_builder/policyfile.rb

@@ -68,22 +68,20 @@ class Chef
 
         @node = nil
 
-        Chef::Log.warn("Using experimental Policyfile feature")
-
         if Chef::Config[:solo]
-          raise UnsupportedFeature, "Policyfile does not support chef-solo at this time."
+          raise UnsupportedFeature, "Policyfile does not support chef-solo. Use chef-client local mode instead."
         end
 
         if override_runlist
-          raise UnsupportedFeature, "Policyfile does not support override run lists at this time"
+          raise UnsupportedFeature, "Policyfile does not support override run lists. Use named run_lists instead."
         end
 
         if json_attribs && json_attribs.key?("run_list")
-          raise UnsupportedFeature, "Policyfile does not support setting the run_list in json data at this time"
+          raise UnsupportedFeature, "Policyfile does not support setting the run_list in json data."
         end
 
         if Chef::Config[:environment] && !Chef::Config[:environment].chomp.empty?
-          raise UnsupportedFeature, "Policyfile does not work with Chef Environments"
+          raise UnsupportedFeature, "Policyfile does not work with Chef Environments."
         end
       end
 
@@ -112,18 +110,11 @@ class Chef
 
       ## PolicyBuilder API ##
 
-      # Loads the node state from the server.
-      def load_node
-        events.node_load_start(node_name, Chef::Config)
-        Chef::Log.debug("Building node object for #{node_name}")
-
-        @node = Chef::Node.find_or_create(node_name)
+      def finish_load_node(node)
+        @node = node
+        select_policy_name_and_group
         validate_policyfile
         events.policyfile_loaded(policy)
-        node
-      rescue Exception => e
-        events.node_load_failed(node_name, e, Chef::Config)
-        raise
       end
 
       # Applies environment, external JSON attributes, and override run list to
@@ -154,25 +145,42 @@ class Chef
         raise
       end
 
+      # Synchronizes cookbooks and initializes the run context object for the
+      # run.
+      #
+      # @return [Chef::RunContext]
       def setup_run_context(specific_recipes=nil)
         Chef::Cookbook::FileVendor.fetch_from_remote(http_api)
         sync_cookbooks
         cookbook_collection = Chef::CookbookCollection.new(cookbooks_to_sync)
         run_context = Chef::RunContext.new(node, cookbook_collection, events)
 
+        setup_chef_class(run_context)
+
         run_context.load(run_list_expansion_ish)
 
+        setup_chef_class(run_context)
         run_context
       end
 
+      # Sets `run_list` on the node from the policy, sets `roles` and `recipes`
+      # attributes on the node accordingly.
+      #
+      # @return [RunListExpansionIsh] A RunListExpansion duck-type.
       def expand_run_list
+        CookbookCacheCleaner.instance.skip_removal = true if named_run_list_requested?
+
         node.run_list(run_list)
         node.automatic_attrs[:roles] = []
         node.automatic_attrs[:recipes] = run_list_expansion_ish.recipes
         run_list_expansion_ish
       end
 
-
+      # Synchronizes cookbooks. In a normal chef-client run, this is handled by
+      # #setup_run_context, but may be called directly in some circumstances.
+      #
+      # @return [Hash{String => Chef::CookbookManifest}] A map of
+      #   CookbookManifest objects by cookbook name.
       def sync_cookbooks
         Chef::Log.debug("Synchronizing cookbooks")
         synchronizer = Chef::CookbookSynchronizer.new(cookbooks_to_sync, events)
@@ -186,12 +194,18 @@ class Chef
 
       # Whether or not this is a temporary policy. Since PolicyBuilder doesn't
       # support override_runlist, this is always false.
+      #
+      # @return [false]
       def temporary_policy?
         false
       end
 
       ## Internal Public API ##
 
+      # @api private
+      #
+      # Generates an array of strings with recipe names including version and
+      # identifier info.
       def run_list_with_versions_for_display
         run_list.map do |recipe_spec|
           cookbook, recipe = parse_recipe_spec(recipe_spec)
@@ -201,6 +215,11 @@ class Chef
         end
       end
 
+      # @api private
+      #
+      # Sets up a RunListExpansionIsh object so that it can be used in place of
+      # a RunListExpansion object, to satisfy the API contract of
+      # #expand_run_list
       def run_list_expansion_ish
         recipes = run_list.map do |recipe_spec|
           cookbook, recipe = parse_recipe_spec(recipe_spec)
@@ -209,11 +228,15 @@ class Chef
         RunListExpansionIsh.new(recipes, [])
       end
 
+      # @api private
+      #
+      # Sets attributes from the policyfile on the node, using the role priority.
       def apply_policyfile_attributes
         node.attributes.role_default = policy["default_attributes"]
         node.attributes.role_override = policy["override_attributes"]
       end
 
+      # @api private
       def parse_recipe_spec(recipe_spec)
         rmatch = recipe_spec.match(/recipe\[([^:]+)::([^:]+)\]/)
         if rmatch.nil?
@@ -223,20 +246,31 @@ class Chef
         end
       end
 
+      # @api private
       def cookbook_lock_for(cookbook_name)
         cookbook_locks[cookbook_name]
       end
 
+      # @api private
       def run_list
-        policy["run_list"]
+        if named_run_list_requested?
+          named_run_list or
+            raise ConfigurationError,
+            "Policy '#{retrieved_policy_name}' revision '#{revision_id}' does not have named_run_list '#{named_run_list_name}'" +
+            "(available named_run_lists: [#{available_named_run_lists.join(', ')}])"
+        else
+          policy["run_list"]
+        end
       end
 
+      # @api private
       def policy
         @policy ||= http_api.get(policyfile_location)
       rescue Net::HTTPServerException => e
         raise ConfigurationError, "Error loading policyfile from `#{policyfile_location}': #{e.class} - #{e.message}"
       end
 
+      # @api private
       def policyfile_location
         if Chef::Config[:policy_document_native_api]
           validate_policy_config!
@@ -273,6 +307,7 @@ class Chef
         end
       end
 
+      # @api private
       def validate_recipe_spec(recipe_spec)
         parse_recipe_spec(recipe_spec)
         nil
@@ -282,11 +317,13 @@ class Chef
 
       class ConfigurationError < StandardError; end
 
+      # @api private
       def deployment_group
         Chef::Config[:deployment_group] or
           raise ConfigurationError, "Setting `deployment_group` is not configured."
       end
 
+      # @api private
       def validate_policy_config!
         policy_group or
           raise ConfigurationError, "Setting `policy_group` is not configured."
@@ -295,14 +332,75 @@ class Chef
           raise ConfigurationError, "Setting `policy_name` is not configured."
       end
 
+      # @api private
       def policy_group
         Chef::Config[:policy_group]
       end
 
+      # @api private
       def policy_name
         Chef::Config[:policy_name]
       end
 
+      # @api private
+      #
+      # Selects the `policy_name` and `policy_group` from the following sources
+      # in priority order:
+      #
+      # 1. JSON attribs (i.e., `-j JSON_FILE`)
+      # 2. `Chef::Config`
+      # 3. The node object
+      #
+      # The selected values are then copied to `Chef::Config` and the node.
+      def select_policy_name_and_group
+        policy_name_to_set =
+          policy_name_from_json_attribs ||
+          policy_name_from_config ||
+          policy_name_from_node
+
+        policy_group_to_set =
+          policy_group_from_json_attribs ||
+          policy_group_from_config ||
+          policy_group_from_node
+
+        node.policy_name = policy_name_to_set
+        node.policy_group = policy_group_to_set
+
+        Chef::Config[:policy_name] = policy_name_to_set
+        Chef::Config[:policy_group] = policy_group_to_set
+      end
+
+      # @api private
+      def policy_group_from_json_attribs
+        json_attribs["policy_group"]
+      end
+
+      # @api private
+      def policy_name_from_json_attribs
+        json_attribs["policy_name"]
+      end
+
+      # @api private
+      def policy_group_from_config
+        Chef::Config[:policy_group]
+      end
+
+      # @api private
+      def policy_name_from_config
+        Chef::Config[:policy_name]
+      end
+
+      # @api private
+      def policy_group_from_node
+        node.policy_group
+      end
+
+      # @api private
+      def policy_name_from_node
+        node.policy_name
+      end
+
+      # @api private
       # Builds a 'cookbook_hash' map of the form
       #   "COOKBOOK_NAME" => "IDENTIFIER"
       #
@@ -330,6 +428,7 @@ class Chef
         raise
       end
 
+      # @api private
       # Fetches the CookbookVersion object for the given name and identifer
       # specified in the lock_data.
       # TODO: This only implements Chef 11 compatibility mode, which means that
@@ -343,20 +442,58 @@ class Chef
         end
       end
 
+      # @api private
       def cookbook_locks
         policy["cookbook_locks"]
       end
 
+      # @api private
+      def revision_id
+        policy["revision_id"]
+      end
+
+      # @api private
       def http_api
         @api_service ||= Chef::REST.new(config[:chef_server_url])
       end
 
+      # @api private
       def config
         Chef::Config
       end
 
       private
 
+      # This method injects the run_context and into the Chef class.
+      #
+      # NOTE: This is duplicated with the ExpandNodeObject implementation. If
+      # it gets any more complicated, it needs to be moved elsewhere.
+      #
+      # @param run_context [Chef::RunContext] the run_context to inject
+      def setup_chef_class(run_context)
+        Chef.set_run_context(run_context)
+      end
+
+      def retrieved_policy_name
+        policy["name"]
+      end
+
+      def named_run_list
+        policy["named_run_lists"] && policy["named_run_lists"][named_run_list_name]
+      end
+
+      def available_named_run_lists
+        (policy["named_run_lists"] || {}).keys
+      end
+
+      def named_run_list_requested?
+        !!Chef::Config[:named_run_list]
+      end
+
+      def named_run_list_name
+        Chef::Config[:named_run_list]
+      end
+
       def compat_mode_manifest_for(cookbook_name, lock_data)
         xyz_version = lock_data["dotted_decimal_identifier"]
         rel_url = "cookbooks/#{cookbook_name}/#{xyz_version}"