chef 10.12.0 → 10.14.0.beta.1

data/lib/chef/rest.rb

@@ -28,8 +28,11 @@ require 'tempfile'
 require 'chef/rest/auth_credentials'
 require 'chef/rest/rest_request'
 require 'chef/monkey_patches/string'
+require 'chef/monkey_patches/net_http'
 require 'chef/config'
 
+
+
 class Chef
   # == Chef::REST
   # Chef's custom REST client with built-in JSON support and RSA signed header
@@ -240,30 +243,37 @@ class Chef
       headers = build_headers(method, url, headers, json_body)
 
       retriable_rest_request(method, url, json_body, headers) do |rest_request|
-        response = rest_request.call {|r| r.read_body}
+        begin
+          response = rest_request.call {|r| r.read_body}
 
-        response_body = decompress_body(response)
+          response_body = decompress_body(response)
 
-        if response.kind_of?(Net::HTTPSuccess)
-          if response['content-type'] =~ /json/
-            Chef::JSONCompat.from_json(response_body.chomp)
+          if response.kind_of?(Net::HTTPSuccess)
+            if response['content-type'] =~ /json/
+              Chef::JSONCompat.from_json(response_body.chomp)
+            else
+              Chef::Log.warn("Expected JSON response, but got content-type '#{response['content-type']}'")
+              response_body
+            end
+          elsif redirect_location = redirected_to(response)
+            follow_redirect {api_request(:GET, create_url(redirect_location))}
           else
-            Chef::Log.warn("Expected JSON response, but got content-type '#{response['content-type']}'")
-            response_body
+            # have to decompress the body before making an exception for it. But the body could be nil.
+            response.body.replace(decompress_body(response)) if response.body.respond_to?(:replace)
+
+            if response['content-type'] =~ /json/
+              exception = Chef::JSONCompat.from_json(response_body)
+              msg = "HTTP Request Returned #{response.code} #{response.message}: "
+              msg << (exception["error"].respond_to?(:join) ? exception["error"].join(", ") : exception["error"].to_s)
+              Chef::Log.info(msg)
+            end
+            response.error!
           end
-        elsif redirect_location = redirected_to(response)
-          follow_redirect {api_request(:GET, create_url(redirect_location))}
-        else
-          # have to decompress the body before making an exception for it. But the body could be nil.
-          response.body.replace(decompress_body(response)) if response.body.respond_to?(:replace)
-
-          if response['content-type'] =~ /json/
-            exception = Chef::JSONCompat.from_json(response_body)
-            msg = "HTTP Request Returned #{response.code} #{response.message}: "
-            msg << (exception["error"].respond_to?(:join) ? exception["error"].join(", ") : exception["error"].to_s)
-            Chef::Log.info(msg)
+        rescue Exception => e
+          if e.respond_to?(:chef_rest_request=)
+            e.chef_rest_request = rest_request
           end
-          response.error!
+          raise
         end
       end
     end
@@ -295,28 +305,35 @@ class Chef
     def streaming_request(url, headers, &block)
       headers = build_headers(:GET, url, headers, nil, true)
       retriable_rest_request(:GET, url, nil, headers) do |rest_request|
-        tempfile = nil
-        response = rest_request.call do |r|
-          if block_given? && r.kind_of?(Net::HTTPSuccess)
-            begin
-              tempfile = stream_to_tempfile(url, r, &block)
-              yield tempfile
-            ensure
-              tempfile.close!
+        begin
+          tempfile = nil
+          response = rest_request.call do |r|
+            if block_given? && r.kind_of?(Net::HTTPSuccess)
+              begin
+                tempfile = stream_to_tempfile(url, r, &block)
+                yield tempfile
+              ensure
+                tempfile.close!
+              end
+            else
+              tempfile = stream_to_tempfile(url, r)
             end
+          end
+          if response.kind_of?(Net::HTTPSuccess)
+            tempfile
+          elsif redirect_location = redirected_to(response)
+            # TODO: test tempfile unlinked when following redirects.
+            tempfile && tempfile.close!
+            follow_redirect {streaming_request(create_url(redirect_location), {}, &block)}
           else
-            tempfile = stream_to_tempfile(url, r)
+            tempfile && tempfile.close!
+            response.error!
           end
-        end
-        if response.kind_of?(Net::HTTPSuccess)
-          tempfile
-        elsif redirect_location = redirected_to(response)
-          # TODO: test tempfile unlinked when following redirects.
-          tempfile && tempfile.close!
-          follow_redirect {streaming_request(create_url(redirect_location), {}, &block)}
-        else
-          tempfile && tempfile.close!
-          response.error!
+        rescue Exception => e
+          if e.respond_to?(:chef_rest_request=)
+            e.chef_rest_request = rest_request
+          end
+          raise
         end
       end
     end

data/lib/chef/run_context.rb

@@ -37,16 +37,19 @@ class Chef
     # of any cookbooks.
     attr_accessor :resource_collection
 
+    attr_reader :events
+
     # Creates a new Chef::RunContext object and populates its fields. This object gets
     # used by the Chef Server to generate a fully compiled recipe list for a node.
     #
     # === Returns
     # object<Chef::RunContext>:: Duh. :)
-    def initialize(node, cookbook_collection)
+    def initialize(node, cookbook_collection, events)
       @node = node
       @cookbook_collection = cookbook_collection
       @resource_collection = Chef::ResourceCollection.new
       @definitions = Hash.new
+      @events = events
 
       # TODO: 5/18/2010 cw/timh - See note on Chef::Node's
       # cookbook_collection attr_accessor
@@ -55,8 +58,8 @@ class Chef
 
     def load(run_list_expansion)
       load_libraries
-      load_lwrp_providers
-      load_lwrp_resources
+
+      load_lwrps
       load_attributes
       load_resource_definitions
 
@@ -66,51 +69,119 @@ class Chef
       # roles) to the node.
       @node.apply_expansion_attributes(run_list_expansion)
 
+      @events.recipe_load_start(run_list_expansion.recipes.size)
       run_list_expansion.recipes.each do |recipe|
-        # TODO: timh/cw, 5-14-2010: It's distasteful to be including
-        # the DSL in a class outside the context of the DSL
-        include_recipe(recipe)
+        begin
+          # TODO: timh/cw, 5-14-2010: It's distasteful to be including
+          # the DSL in a class outside the context of the DSL
+          include_recipe(recipe)
+        rescue Exception => e
+          path = resolve_recipe(recipe)
+          @events.recipe_file_load_failed(path, e)
+          raise
+        end
       end
+      @events.recipe_load_complete
     end
 
+    def resolve_recipe(recipe_name)
+      cookbook_name, recipe_short_name = Chef::Recipe.parse_recipe_name(recipe_name)
+      cookbook = cookbook_collection[cookbook_name]
+      cookbook.recipe_filenames_by_name[recipe_short_name]
+    end
 
     private
 
     def load_libraries
+      @events.library_load_start(count_files_by_segment(:libraries))
+
       foreach_cookbook_load_segment(:libraries) do |cookbook_name, filename|
-        Chef::Log.debug("Loading cookbook #{cookbook_name}'s library file: #{filename}")
-        Kernel.load(filename)
+        begin
+          Chef::Log.debug("Loading cookbook #{cookbook_name}'s library file: #{filename}")
+          Kernel.load(filename)
+          @events.library_file_loaded(filename)
+        rescue Exception => e
+          # TODO wrap/munge exception to highlight syntax/name/no method errors.
+          @events.library_load_failed(filename, e)
+          raise
+        end
       end
+
+      @events.library_load_complete
+    end
+
+    def load_lwrps
+      lwrp_file_count = count_files_by_segment(:providers) + count_files_by_segment(:resources)
+      @events.lwrp_load_start(lwrp_file_count)
+      load_lwrp_providers
+      load_lwrp_resources
+      @events.lwrp_load_complete
     end
 
     def load_lwrp_providers
       foreach_cookbook_load_segment(:providers) do |cookbook_name, filename|
-        Chef::Log.debug("Loading cookbook #{cookbook_name}'s providers from #{filename}")
-        Chef::Provider.build_from_file(cookbook_name, filename, self)
+        begin
+          Chef::Log.debug("Loading cookbook #{cookbook_name}'s providers from #{filename}")
+          Chef::Provider.build_from_file(cookbook_name, filename, self)
+          @events.lwrp_file_loaded(filename)
+        rescue Exception => e
+          # TODO: wrap exception with helpful info
+          @events.lwrp_file_load_failed(filename, e)
+          raise
+        end
       end
     end
 
     def load_lwrp_resources
       foreach_cookbook_load_segment(:resources) do |cookbook_name, filename|
-        Chef::Log.debug("Loading cookbook #{cookbook_name}'s resources from #{filename}")
-        Chef::Resource.build_from_file(cookbook_name, filename, self)
+        begin
+          Chef::Log.debug("Loading cookbook #{cookbook_name}'s resources from #{filename}")
+          Chef::Resource.build_from_file(cookbook_name, filename, self)
+          @events.lwrp_file_loaded(filename)
+        rescue Exception => e
+          @events.lwrp_file_load_failed(filename, e)
+          raise
+        end
       end
     end
 
     def load_attributes
-      node.load_attributes
+      @events.attribute_load_start(count_files_by_segment(:attributes))
+      foreach_cookbook_load_segment(:attributes) do |cookbook_name, filename|
+        begin
+          Chef::Log.debug("Node #{@node.name} loading cookbook #{cookbook_name}'s attribute file #{filename}")
+          @node.from_file(filename)
+        rescue Exception => e
+          @events.attribute_file_load_failed(filename, e)
+          raise
+        end
+      end
+      @events.attribute_load_complete
     end
 
     def load_resource_definitions
+      @events.definition_load_start(count_files_by_segment(:definitions))
       foreach_cookbook_load_segment(:definitions) do |cookbook_name, filename|
-        Chef::Log.debug("Loading cookbook #{cookbook_name}'s definitions from #{filename}")
-        resourcelist = Chef::ResourceDefinitionList.new
-        resourcelist.from_file(filename)
-        definitions.merge!(resourcelist.defines) do |key, oldval, newval|
-          Chef::Log.info("Overriding duplicate definition #{key}, new definition found in #{filename}")
-          newval
+        begin
+          Chef::Log.debug("Loading cookbook #{cookbook_name}'s definitions from #{filename}")
+          resourcelist = Chef::ResourceDefinitionList.new
+          resourcelist.from_file(filename)
+          definitions.merge!(resourcelist.defines) do |key, oldval, newval|
+            Chef::Log.info("Overriding duplicate definition #{key}, new definition found in #{filename}")
+            newval
+          end
+          @events.definition_file_loaded(filename)
+        rescue Exception => e
+          @events.definition_file_load_failed(filename, e)
         end
       end
+      @events.definition_load_complete
+    end
+
+    def count_files_by_segment(segment)
+      cookbook_collection.inject(0) do |count, ( cookbook_name, cookbook )|
+        count + cookbook.segment_filenames(segment).size
+      end
     end
 
     def foreach_cookbook_load_segment(segment, &block)

data/lib/chef/run_list/run_list_expansion.rb

@@ -40,18 +40,24 @@ class Chef
 
       attr_reader :override_attrs
 
-      attr_reader :errors
-
       attr_reader :environment
 
+      attr_reader :missing_roles_with_including_role
+
       # The data source passed to the constructor. Not used in this class.
       # In subclasses, this is a couchdb or Chef::REST object pre-configured
       # to fetch roles from their correct location.
       attr_reader :source
 
+      # Returns a Hash of the form "including_role" => "included_role_or_recipe".
+      # This can be used to show the expanded run list (ordered) graph.
+      # ==== Caveats
+      # * Duplicate roles are not shown.
+      attr_reader :run_list_trace
+
       def initialize(environment, run_list_items, source=nil)
         @environment = environment
-        @errors = Array.new
+        @missing_roles_with_including_role = Array.new
 
         @run_list_items = run_list_items.dup
         @source = source
@@ -62,11 +68,12 @@ class Chef
         @recipes = Chef::RunList::VersionedRecipeList.new
 
         @applied_roles = {}
+        @run_list_trace = Hash.new {|h, key| h[key] = [] }
       end
 
       # Did we find any errors (expanding roles)?
       def errors?
-        @errors.length > 0
+        @missing_roles_with_including_role.length > 0
       end
 
       alias :invalid? :errors?
@@ -83,10 +90,10 @@ class Chef
       # Chef::Role  in most cases
       # false       if the role has already been applied
       # nil         if the role does not exist
-      def inflate_role(role_name)
+      def inflate_role(role_name, included_by)
         return false if applied_role?(role_name) # Prevent infinite loops
         applied_role(role_name)
-        fetch_role(role_name)
+        fetch_role(role_name, included_by)
       end
 
       def apply_role_attributes(role)
@@ -107,7 +114,7 @@ class Chef
       end
 
       # In subclasses, this method will fetch the role from the data source.
-      def fetch_role(name)
+      def fetch_role(name, included_by)
         raise NotImplementedError
       end
 
@@ -115,12 +122,16 @@ class Chef
       # exception is raised.  We do add an entry in the errors collection.
       # === Returns
       # nil
-      def role_not_found(name)
-        Chef::Log.error("Role #{name} is in the runlist but does not exist. Skipping expand.")
-        @errors << name
+      def role_not_found(name, included_by)
+        Chef::Log.error("Role #{name} (included by '#{included_by}') is in the runlist but does not exist. Skipping expand.")
+        @missing_roles_with_including_role << [name, included_by]
         nil
       end
 
+      def errors
+        @missing_roles_with_including_role.map {|item| item.first }
+      end
+
       private
 
       # these methods modifies internal state based on arguments, so hide it.
@@ -129,18 +140,20 @@ class Chef
         @applied_roles[role_name] = true
       end
 
-      def expand_run_list_items(items)
+      def expand_run_list_items(items, included_by="top level")
         if entry = items.shift
+          @run_list_trace[included_by.to_s] << entry.to_s
+
           case entry.type
           when :recipe
             recipes.add_recipe(entry.name, entry.version)
           when :role
-            if role = inflate_role(entry.name)
-              expand_run_list_items(role.run_list_for(@environment).run_list_items)
+            if role = inflate_role(entry.name, included_by)
+              expand_run_list_items(role.run_list_for(@environment).run_list_items, role)
               apply_role_attributes(role)
             end
           end
-          expand_run_list_items(items)
+          expand_run_list_items(items, included_by)
         end
       end
 
@@ -149,10 +162,10 @@ class Chef
     # Expand a run list from disk. Suitable for chef-solo
     class RunListExpansionFromDisk < RunListExpansion
 
-      def fetch_role(name)
+      def fetch_role(name, included_by)
         Chef::Role.from_disk(name)
       rescue Chef::Exceptions::RoleNotFound
-        role_not_found(name)
+        role_not_found(name, included_by)
       end
 
     end
@@ -164,11 +177,11 @@ class Chef
         @rest ||= (source || Chef::REST.new(Chef::Config[:role_url]))
       end
 
-      def fetch_role(name)
+      def fetch_role(name, included_by)
         rest.get_rest("roles/#{name}")
       rescue Net::HTTPServerException => e
         if e.message == '404 "Not Found"'
-          role_not_found(name)
+          role_not_found(name, included_by)
         else
           raise
         end
@@ -182,10 +195,10 @@ class Chef
         source
       end
 
-      def fetch_role(name)
+      def fetch_role(name, included_by)
         Chef::Role.cdb_load(name, couchdb)
       rescue Chef::Exceptions::CouchDBNotFound
-        role_not_found(name)
+        role_not_found(name, included_by)
       end
 
     end

data/lib/chef/run_status.rb

@@ -23,6 +23,8 @@
 # Chef run.
 class Chef::RunStatus
 
+  attr_reader :events
+
   attr_reader :run_context
 
   attr_writer :run_context
@@ -35,8 +37,9 @@ class Chef::RunStatus
 
   attr_writer :exception
 
-  def initialize(node)
+  def initialize(node, events)
     @node = node
+    @events = events
   end
 
   def node