chef 10.34.6 → 11.0.0.beta.0

data/lib/chef/run_context.rb

@@ -18,25 +18,46 @@
 # limitations under the License.
 
 require 'chef/resource_collection'
+require 'chef/cookbook_version'
 require 'chef/node'
 require 'chef/role'
 require 'chef/log'
-require 'chef/mixin/language_include_recipe'
+require 'chef/recipe'
+require 'chef/run_context/cookbook_compiler'
 
 class Chef
+
   # == Chef::RunContext
   # Value object that loads and tracks the context of a Chef run
   class RunContext
 
-    # Used to load the node's recipes after expanding its run list
-    include Chef::Mixin::LanguageIncludeRecipe
+    # Chef::Node object for this run
+    attr_reader :node
+
+    # Chef::CookbookCollection for this run
+    attr_reader :cookbook_collection
+
+    # Resource Definitions for this run. Populated when the files in
+    # +definitions/+ are evaluated (this is triggered by #load).
+    attr_reader :definitions
 
-    attr_reader :node, :cookbook_collection, :definitions
+    ###
+    # These need to be settable so deploy can run a resource_collection
+    # independent of any cookbooks via +recipe_eval+
 
-    # Needs to be settable so deploy can run a resource_collection independent
-    # of any cookbooks.
-    attr_accessor :resource_collection, :immediate_notification_collection, :delayed_notification_collection
+    # The Chef::ResourceCollection for this run. Populated by evaluating
+    # recipes, which is triggered by #load. (See also: CookbookCompiler)
+    attr_accessor :resource_collection
 
+    # A Hash containing the immediate notifications triggered by resources
+    # during the converge phase of the chef run.
+    attr_accessor :immediate_notification_collection
+
+    # A Hash containing the delayed (end of run) notifications triggered by
+    # resources during the converge phase of the chef run.
+    attr_accessor :delayed_notification_collection
+
+    # Event dispatcher for this run.
     attr_reader :events
 
     # Creates a new Chef::RunContext object and populates its fields. This object gets
@@ -51,50 +72,23 @@ class Chef
       @immediate_notification_collection = Hash.new {|h,k| h[k] = []}
       @delayed_notification_collection = Hash.new {|h,k| h[k] = []}
       @definitions = Hash.new
+      @loaded_recipes = {}
+      @loaded_attributes = {}
       @events = events
 
-      # TODO: 5/18/2010 cw/timh - See note on Chef::Node's
-      # cookbook_collection attr_accessor
-      node.cookbook_collection = cookbook_collection
+      @node.run_context = self
     end
 
+    # Triggers the compile phase of the chef run. Implemented by
+    # Chef::RunContext::CookbookCompiler
     def load(run_list_expansion)
-      load_libraries
-
-      load_lwrps
-      load_attributes
-      load_resource_definitions
-
-      # Precendence rules state that roles' attributes come after
-      # cookbooks. Now we've loaded attributes from cookbooks with
-      # load_attributes, apply the expansion attributes (loaded from
-      # roles) to the node.
-      @node.apply_expansion_attributes(run_list_expansion)
-
-      @events.recipe_load_start(run_list_expansion.recipes.size)
-      run_list_expansion.recipes.each do |recipe|
-        begin
-          # TODO: timh/cw, 5-14-2010: It's distasteful to be including
-          # the DSL in a class outside the context of the DSL
-          include_recipe(recipe)
-        rescue Chef::Exceptions::RecipeNotFound => e
-          @events.recipe_not_found(e)
-          raise
-        rescue Exception => e
-          path = resolve_recipe(recipe)
-          @events.recipe_file_load_failed(path, e)
-          raise
-        end
-      end
-      @events.recipe_load_complete
-    end
-
-    def resolve_recipe(recipe_name)
-      cookbook_name, recipe_short_name = Chef::Recipe.parse_recipe_name(recipe_name)
-      cookbook = cookbook_collection[cookbook_name]
-      cookbook.recipe_filenames_by_name[recipe_short_name]
+      compiler = CookbookCompiler.new(self, run_list_expansion, events)
+      compiler.compile
     end
 
+    # Adds an immediate notification to the
+    # +immediate_notification_collection+. The notification should be a
+    # Chef::Resource::Notification or duck type.
     def notifies_immediately(notification)
       nr = notification.notifying_resource
       if nr.instance_of?(Chef::Resource)
@@ -104,6 +98,8 @@ class Chef
       end
     end
 
+    # Adds a delayed notification to the +delayed_notification_collection+. The
+    # notification should be a Chef::Resource::Notification or duck type.
     def notifies_delayed(notification)
       nr = notification.notifying_resource
       if nr.instance_of?(Chef::Resource)
@@ -129,107 +125,89 @@ class Chef
       end
     end
 
-    private
-
-    def load_libraries
-      @events.library_load_start(count_files_by_segment(:libraries))
-
-      foreach_cookbook_load_segment(:libraries) do |cookbook_name, filename|
-        begin
-          Chef::Log.debug("Loading cookbook #{cookbook_name}'s library file: #{filename}")
-          Kernel.load(filename)
-          @events.library_file_loaded(filename)
-        rescue Exception => e
-          # TODO wrap/munge exception to highlight syntax/name/no method errors.
-          @events.library_file_load_failed(filename, e)
-          raise
+    # Evaluates the recipes +recipe_names+. Used by DSL::IncludeRecipe
+    def include_recipe(*recipe_names)
+      result_recipes = Array.new
+      recipe_names.flatten.each do |recipe_name|
+        if result = load_recipe(recipe_name)
+          result_recipes << result
         end
       end
+      result_recipes
+    end
+
+    # Evaluates the recipe +recipe_name+. Used by DSL::IncludeRecipe
+    def load_recipe(recipe_name)
+      Chef::Log.debug("Loading Recipe #{recipe_name} via include_recipe")
+
+      cookbook_name, recipe_short_name = Chef::Recipe.parse_recipe_name(recipe_name)
+      if loaded_fully_qualified_recipe?(cookbook_name, recipe_short_name)
+        Chef::Log.debug("I am not loading #{recipe_name}, because I have already seen it.")
+        false
+      else
+        loaded_recipe(cookbook_name, recipe_short_name)
+
+        cookbook = cookbook_collection[cookbook_name]
+        cookbook.load_recipe(recipe_short_name, self)
+      end
+    end
 
-      @events.library_load_complete
+    # Looks up an attribute file given the +cookbook_name+ and
+    # +attr_file_name+. Used by DSL::IncludeAttribute
+    def resolve_attribute(cookbook_name, attr_file_name)
+      cookbook = cookbook_collection[cookbook_name]
+      raise Chef::Exceptions::CookbookNotFound, "could not find cookbook #{cookbook_name} while loading attribute #{name}" unless cookbook
+
+      attribute_filename = cookbook.attribute_filenames_by_short_filename[attr_file_name]
+      raise Chef::Exceptions::AttributeNotFound, "could not find filename for attribute #{attr_file_name} in cookbook #{cookbook_name}" unless attribute_filename
+
+      attribute_filename
     end
 
-    def load_lwrps
-      lwrp_file_count = count_files_by_segment(:providers) + count_files_by_segment(:resources)
-      @events.lwrp_load_start(lwrp_file_count)
-      load_lwrp_providers
-      load_lwrp_resources
-      @events.lwrp_load_complete
+    # An Array of all recipes that have been loaded. This is stored internally
+    # as a Hash, so ordering is not preserved when using ruby 1.8.
+    #
+    # Recipe names are given in fully qualified form, e.g., the recipe "nginx"
+    # will be given as "nginx::default"
+    #
+    # To determine if a particular recipe has been loaded, use #loaded_recipe?
+    def loaded_recipes
+      @loaded_recipes.keys
     end
 
-    def load_lwrp_providers
-      foreach_cookbook_load_segment(:providers) do |cookbook_name, filename|
-        begin
-          Chef::Log.debug("Loading cookbook #{cookbook_name}'s providers from #{filename}")
-          Chef::Provider.build_from_file(cookbook_name, filename, self)
-          @events.lwrp_file_loaded(filename)
-        rescue Exception => e
-          # TODO: wrap exception with helpful info
-          @events.lwrp_file_load_failed(filename, e)
-          raise
-        end
-      end
+    # An Array of all attributes files that have been loaded. Stored internally
+    # using a Hash, so order is not preserved on ruby 1.8.
+    #
+    # Attribute file names are given in fully qualified form, e.g.,
+    # "nginx::default" instead of "nginx".
+    def loaded_attributes
+      @loaded_attributes.keys
     end
 
-    def load_lwrp_resources
-      foreach_cookbook_load_segment(:resources) do |cookbook_name, filename|
-        begin
-          Chef::Log.debug("Loading cookbook #{cookbook_name}'s resources from #{filename}")
-          Chef::Resource.build_from_file(cookbook_name, filename, self)
-          @events.lwrp_file_loaded(filename)
-        rescue Exception => e
-          @events.lwrp_file_load_failed(filename, e)
-          raise
-        end
-      end
+    def loaded_fully_qualified_recipe?(cookbook, recipe)
+      @loaded_recipes.has_key?("#{cookbook}::#{recipe}")
     end
 
-    def load_attributes
-      @events.attribute_load_start(count_files_by_segment(:attributes))
-      foreach_cookbook_load_segment(:attributes) do |cookbook_name, filename|
-        begin
-          Chef::Log.debug("Node #{@node.name} loading cookbook #{cookbook_name}'s attribute file #{filename}")
-          @node.from_file(filename)
-        rescue Exception => e
-          @events.attribute_file_load_failed(filename, e)
-          raise
-        end
-      end
-      @events.attribute_load_complete
-    end
-
-    def load_resource_definitions
-      @events.definition_load_start(count_files_by_segment(:definitions))
-      foreach_cookbook_load_segment(:definitions) do |cookbook_name, filename|
-        begin
-          Chef::Log.debug("Loading cookbook #{cookbook_name}'s definitions from #{filename}")
-          resourcelist = Chef::ResourceDefinitionList.new
-          resourcelist.from_file(filename)
-          definitions.merge!(resourcelist.defines) do |key, oldval, newval|
-            Chef::Log.info("Overriding duplicate definition #{key}, new definition found in #{filename}")
-            newval
-          end
-          @events.definition_file_loaded(filename)
-        rescue Exception => e
-          @events.definition_file_load_failed(filename, e)
-        end
-      end
-      @events.definition_load_complete
+    # Returns true if +recipe+ has been loaded, false otherwise. Default recipe
+    # names are expanded, so `loaded_recipe?("nginx")` and
+    # `loaded_recipe?("nginx::default")` are valid and give identical results.
+    def loaded_recipe?(recipe)
+      cookbook, recipe_name = Chef::Recipe.parse_recipe_name(recipe)
+      loaded_fully_qualified_recipe?(cookbook, recipe_name)
     end
 
-    def count_files_by_segment(segment)
-      cookbook_collection.inject(0) do |count, ( cookbook_name, cookbook )|
-        count + cookbook.segment_filenames(segment).size
-      end
+    def loaded_fully_qualified_attribute?(cookbook, attribute_file)
+      @loaded_attributes.has_key?("#{cookbook}::#{attribute_file}")
     end
 
-    def foreach_cookbook_load_segment(segment, &block)
-      cookbook_collection.each do |cookbook_name, cookbook|
-        segment_filenames = cookbook.segment_filenames(segment)
-        segment_filenames.each do |segment_filename|
-          block.call(cookbook_name, segment_filename)
-        end
-      end
+    def loaded_attribute(cookbook, attribute_file)
+      @loaded_attributes["#{cookbook}::#{attribute_file}"] = true
+    end
+
+    private
+
+    def loaded_recipe(cookbook, recipe)
+      @loaded_recipes["#{cookbook}::#{recipe}"] = true
     end
 
   end

data/lib/chef/run_context/cookbook_compiler.rb

@@ -0,0 +1,280 @@
+#
+# Author:: Daniel DeLeo (<dan@opscode.com>)
+# Copyright:: Copyright (c) 2012 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/log'
+require 'chef/recipe'
+require 'chef/resource/lwrp_base'
+require 'chef/provider/lwrp_base'
+require 'chef/resource_definition_list'
+
+class Chef
+  class RunContext
+
+    # Implements the compile phase of the chef run by loading/eval-ing files
+    # from cookbooks in the correct order and in the correct context.
+    class CookbookCompiler
+      attr_reader :events
+      attr_reader :run_list_expansion
+
+      def initialize(run_context, run_list_expansion, events)
+        @run_context = run_context
+        @events = events
+        @run_list_expansion = run_list_expansion
+        @cookbook_order = nil
+      end
+
+      # Chef::Node object for the current run.
+      def node
+        @run_context.node
+      end
+
+      # Chef::CookbookCollection object for the current run
+      def cookbook_collection
+        @run_context.cookbook_collection
+      end
+
+      # Resource Definitions from the compiled cookbooks. This is populated by
+      # calling #compile_resource_definitions (which is called by #compile)
+      def definitions
+        @run_context.definitions
+      end
+
+      # Run the compile phase of the chef run. Loads files in the following order:
+      # * Libraries
+      # * Attributes
+      # * LWRPs
+      # * Resource Definitions
+      # * Recipes
+      #
+      # Recipes are loaded in precisely the order specified by the expanded run_list.
+      #
+      # Other files are loaded in an order derived from the expanded run_list
+      # and the dependencies declared by cookbooks' metadata. See
+      # #cookbook_order for more information.
+      def compile
+        compile_libraries
+        compile_attributes
+        compile_lwrps
+        compile_resource_definitions
+        compile_recipes
+      end
+
+      # Extracts the cookbook names from the expanded run list, then iterates
+      # over the list, recursing through dependencies to give a run_list
+      # ordered array of cookbook names with no duplicates. Dependencies appear
+      # before the cookbook(s) that depend on them.
+      def cookbook_order
+        @cookbook_order ||= begin
+          ordered_cookbooks = []
+          seen_cookbooks = {}
+          run_list_expansion.recipes.each do |recipe|
+            cookbook = Chef::Recipe.parse_recipe_name(recipe).first
+            add_cookbook_with_deps(ordered_cookbooks, seen_cookbooks, cookbook)
+          end
+          Chef::Log.debug("Cookbooks to compile: #{ordered_cookbooks.inspect}")
+          ordered_cookbooks
+        end
+      end
+
+      # Loads library files from cookbooks according to #cookbook_order.
+      def compile_libraries
+        @events.library_load_start(count_files_by_segment(:libraries))
+        cookbook_order.each do |cookbook|
+          load_libraries_from_cookbook(cookbook)
+        end
+        @events.library_load_complete
+      end
+
+      # Loads attributes files from cookbooks. Attributes files are loaded
+      # according to #cookbook_order; within a cookbook, +default.rb+ is loaded
+      # first, then the remaining attributes files in lexical sort order.
+      def compile_attributes
+        @events.attribute_load_start(count_files_by_segment(:attributes))
+        cookbook_order.each do |cookbook|
+          load_attributes_from_cookbook(cookbook)
+        end
+        @events.attribute_load_complete
+      end
+
+      # Loads LWRPs according to #cookbook_order. Providers are loaded before
+      # resources on a cookbook-wise basis.
+      def compile_lwrps
+        lwrp_file_count = count_files_by_segment(:providers) + count_files_by_segment(:resources)
+        @events.lwrp_load_start(lwrp_file_count)
+        cookbook_order.each do |cookbook|
+          load_lwrps_from_cookbook(cookbook)
+        end
+        @events.lwrp_load_complete
+      end
+
+      # Loads resource definitions according to #cookbook_order
+      def compile_resource_definitions
+        @events.definition_load_start(count_files_by_segment(:definitions))
+        cookbook_order.each do |cookbook|
+          load_resource_definitions_from_cookbook(cookbook)
+        end
+        @events.definition_load_complete
+      end
+
+      # Iterates over the expanded run_list, loading each recipe in turn.
+      def compile_recipes
+        @events.recipe_load_start(run_list_expansion.recipes.size)
+        run_list_expansion.recipes.each do |recipe|
+          begin
+            @run_context.load_recipe(recipe)
+          rescue Chef::Exceptions::RecipeNotFound => e
+            @events.recipe_not_found(e)
+            raise
+          rescue Exception => e
+            path = resolve_recipe(recipe)
+            @events.recipe_file_load_failed(path, e)
+            raise
+          end
+        end
+        @events.recipe_load_complete
+      end
+
+      private
+
+      def load_attributes_from_cookbook(cookbook_name)
+        list_of_attr_files = files_in_cookbook_by_segment(cookbook_name, :attributes).dup
+        if default_file = list_of_attr_files.find {|path| File.basename(path) == "default.rb" }
+          list_of_attr_files.delete(default_file)
+          load_attribute_file(cookbook_name.to_s, default_file)
+        end
+
+        list_of_attr_files.each do |filename|
+          load_attribute_file(cookbook_name.to_s, filename)
+        end
+      end
+
+      def load_attribute_file(cookbook_name, filename)
+        Chef::Log.debug("Node #{node.name} loading cookbook #{cookbook_name}'s attribute file #{filename}")
+        attr_file_basename = ::File.basename(filename, ".rb")
+        node.include_attribute("#{cookbook_name}::#{attr_file_basename}")
+      rescue Exception => e
+        @events.attribute_file_load_failed(filename, e)
+        raise
+      end
+
+      def load_libraries_from_cookbook(cookbook_name)
+        files_in_cookbook_by_segment(cookbook_name, :libraries).each do |filename|
+          begin
+            Chef::Log.debug("Loading cookbook #{cookbook_name}'s library file: #{filename}")
+            Kernel.load(filename)
+            @events.library_file_loaded(filename)
+          rescue Exception => e
+            @events.library_file_load_failed(filename, e)
+            raise
+          end
+        end
+      end
+
+      def load_lwrps_from_cookbook(cookbook_name)
+        files_in_cookbook_by_segment(cookbook_name, :providers).each do |filename|
+          load_lwrp_provider(cookbook_name, filename)
+        end
+        files_in_cookbook_by_segment(cookbook_name, :resources).each do |filename|
+          load_lwrp_resource(cookbook_name, filename)
+        end
+      end
+
+      def load_lwrp_provider(cookbook_name, filename)
+        Chef::Log.debug("Loading cookbook #{cookbook_name}'s providers from #{filename}")
+        Chef::Provider::LWRPBase.build_from_file(cookbook_name, filename, self)
+        @events.lwrp_file_loaded(filename)
+      rescue Exception => e
+        @events.lwrp_file_load_failed(filename, e)
+        raise
+      end
+
+      def load_lwrp_resource(cookbook_name, filename)
+        Chef::Log.debug("Loading cookbook #{cookbook_name}'s resources from #{filename}")
+        Chef::Resource::LWRPBase.build_from_file(cookbook_name, filename, self)
+        @events.lwrp_file_loaded(filename)
+      rescue Exception => e
+        @events.lwrp_file_load_failed(filename, e)
+        raise
+      end
+
+
+      def load_resource_definitions_from_cookbook(cookbook_name)
+        files_in_cookbook_by_segment(cookbook_name, :definitions).each do |filename|
+          begin
+            Chef::Log.debug("Loading cookbook #{cookbook_name}'s definitions from #{filename}")
+            resourcelist = Chef::ResourceDefinitionList.new
+            resourcelist.from_file(filename)
+            definitions.merge!(resourcelist.defines) do |key, oldval, newval|
+              Chef::Log.info("Overriding duplicate definition #{key}, new definition found in #{filename}")
+              newval
+            end
+            @events.definition_file_loaded(filename)
+          rescue Exception => e
+            @events.definition_file_load_failed(filename, e)
+            raise
+          end
+        end
+      end
+
+      # Builds up the list of +ordered_cookbooks+ by first recursing through the
+      # dependencies of +cookbook+, and then adding +cookbook+ to the list of
+      # +ordered_cookbooks+. A cookbook is skipped if it appears in
+      # +seen_cookbooks+, otherwise it is added to the set of +seen_cookbooks+
+      # before its dependencies are processed.
+      def add_cookbook_with_deps(ordered_cookbooks, seen_cookbooks, cookbook)
+        return false if seen_cookbooks.key?(cookbook)
+
+        seen_cookbooks[cookbook] = true
+        each_cookbook_dep(cookbook) do |dependency|
+          add_cookbook_with_deps(ordered_cookbooks, seen_cookbooks, dependency)
+        end
+        ordered_cookbooks << cookbook
+      end
+
+
+      def count_files_by_segment(segment)
+        cookbook_collection.inject(0) do |count, cookbook_by_name|
+          count + cookbook_by_name[1].segment_filenames(segment).size
+        end
+      end
+
+      # Lists the local paths to files in +cookbook+ of type +segment+
+      # (attribute, recipe, etc.), sorted lexically.
+      def files_in_cookbook_by_segment(cookbook, segment)
+        cookbook_collection[cookbook].segment_filenames(segment).sort
+      end
+
+      # Yields the name of each cookbook depended on by +cookbook_name+ in
+      # lexical sort order.
+      def each_cookbook_dep(cookbook_name, &block)
+        cookbook = cookbook_collection[cookbook_name]
+        cookbook.metadata.dependencies.keys.sort.each(&block)
+      end
+
+      # Given a +recipe_name+, finds the file associated with the recipe.
+      def resolve_recipe(recipe_name)
+        cookbook_name, recipe_short_name = Chef::Recipe.parse_recipe_name(recipe_name)
+        cookbook = cookbook_collection[cookbook_name]
+        cookbook.recipe_filenames_by_name[recipe_short_name]
+      end
+
+
+    end
+
+  end
+end