chef 10.12.0 → 10.14.0.beta.1

data/lib/chef/config.rb

@@ -48,6 +48,9 @@ class Chef
     end
 
     def self.platform_specific_path(path)
+      #10.times { puts "* " * 40}
+      #pp caller
+
       if RUBY_PLATFORM =~ /mswin|mingw|windows/
         # turns /etc/chef/client.rb into C:/chef/client.rb
         path = File.join(ENV['SYSTEMDRIVE'], path.split('/')[2..-1])
@@ -119,6 +122,9 @@ class Chef
     # Turn on "path sanity" by default. See also: http://wiki.opscode.com/display/chef/User+Environment+PATH+Sanity
     enforce_path_sanity(true)
 
+    # Formatted Chef Client output is a beta feature, disabled by default:
+    formatter "null"
+
     # Used when OpenID authentication is enabled in the Web UI
     authorized_openid_identifiers nil
     authorized_openid_providers nil
@@ -133,6 +139,9 @@ class Chef
     cookbook_path [ platform_specific_path("/var/chef/cookbooks"),
                     platform_specific_path("/var/chef/site-cookbooks") ]
 
+    # An array of paths to search for knife exec scripts if they aren't in the current directory
+    script_path []
+
     # Where files are stored temporarily during uploads
     sandbox_path "/var/chef/sandboxes"
 
@@ -184,6 +193,8 @@ class Chef
     run_command_stdout_timeout 120
     solo  false
     splay nil
+    why_run false
+    color false
 
     # Set these to enable SSL authentication / mutual-authentication
     # with the server

data/lib/chef/cookbook/remote_file_vendor.rb

@@ -76,11 +76,7 @@ class Chef
       end
 
       def validate_cached_copy(cache_filename)
-        valid_cache_entries[cache_filename] = true
-      end
-
-      def valid_cache_entries
-        Chef::CookbookVersion.valid_cache_entries
+        CookbookCacheCleaner.instance.mark_file_as_valid(cache_filename)
       end
 
     end

data/lib/chef/cookbook/synchronizer.rb

@@ -0,0 +1,216 @@
+require 'chef/client'
+require 'singleton'
+
+class Chef
+
+  # Keep track of the filenames that we use in both eager cookbook
+  # downloading (during sync_cookbooks) and lazy (during the run
+  # itself, through FileVendor). After the run is over, clean up the
+  # cache.
+  class CookbookCacheCleaner
+
+    # Setup a notification to clear the valid_cache_entries when a Chef client
+    # run starts
+    Chef::Client.when_run_starts do |run_status|
+      instance.reset!
+    end
+
+    # Register a notification to cleanup unused files from cookbooks
+    Chef::Client.when_run_completes_successfully do |run_status|
+      instance.cleanup_file_cache
+    end
+
+    include Singleton
+
+    def initialize
+      reset!
+    end
+
+    def reset!
+      @valid_cache_entries = {}
+    end
+
+    def mark_file_as_valid(cache_path)
+      @valid_cache_entries[cache_path] = true
+    end
+
+    def cache
+      Chef::FileCache
+    end
+
+    def cleanup_file_cache
+      unless Chef::Config[:solo]
+        # Delete each file in the cache that we didn't encounter in the
+        # manifest.
+        cache.find(File.join(%w{cookbooks ** *})).each do |cache_filename|
+          unless @valid_cache_entries[cache_filename]
+            Chef::Log.info("Removing #{cache_filename} from the cache; it is no longer needed by chef-client.")
+            cache.delete(cache_filename)
+          end
+        end
+      end
+    end
+
+  end
+
+  # Synchronizes the locally cached copies of cookbooks with the files on the
+  # server.
+  class CookbookSynchronizer
+    EAGER_SEGMENTS = Chef::CookbookVersion::COOKBOOK_SEGMENTS.dup
+    EAGER_SEGMENTS.delete(:files)
+    EAGER_SEGMENTS.delete(:templates)
+    EAGER_SEGMENTS.freeze
+
+    def initialize(cookbooks_by_name, events)
+      @cookbooks_by_name, @events = cookbooks_by_name, events
+    end
+
+    def cache
+      Chef::FileCache
+    end
+
+    def cookbook_names
+      @cookbooks_by_name.keys
+    end
+
+    def cookbooks
+      @cookbooks_by_name.values
+    end
+
+    def cookbook_count
+      @cookbooks_by_name.size
+    end
+
+    def have_cookbook?(cookbook_name)
+      @cookbooks_by_name.key?(cookbook_name)
+    end
+
+    # Synchronizes all the cookbooks from the chef-server.
+    #)
+    # === Returns
+    # true:: Always returns true
+    def sync_cookbooks
+      Chef::Log.info("Loading cookbooks [#{cookbook_names.sort.join(', ')}]")
+      Chef::Log.debug("Cookbooks detail: #{cookbooks.inspect}")
+
+      clear_obsoleted_cookbooks
+
+      @events.cookbook_sync_start(cookbook_count)
+
+      # Synchronize each of the node's cookbooks, and add to the
+      # valid_cache_entries hash.
+      cookbooks.each do |cookbook|
+        sync_cookbook(cookbook)
+      end
+
+    rescue Exception => e
+      @events.cookbook_sync_failed(cookbooks, e)
+      raise
+    else
+      @events.cookbook_sync_complete
+      true
+    end
+
+    # Iterates over cached cookbooks' files, removing files belonging to
+    # cookbooks that don't appear in +cookbook_hash+
+    def clear_obsoleted_cookbooks
+      @events.cookbook_clean_start
+      # Remove all cookbooks no longer relevant to this node
+      cache.find(File.join(%w{cookbooks ** *})).each do |cache_file|
+        cache_file =~ /^cookbooks\/([^\/]+)\//
+        unless have_cookbook?($1)
+          Chef::Log.info("Removing #{cache_file} from the cache; its cookbook is no longer needed on this client.")
+          cache.delete(cache_file)
+          @events.removed_cookbook_file(cache_file)
+        end
+      end
+      @events.cookbook_clean_complete
+    end
+
+    # Sync the eagerly loaded files contained by +cookbook+
+    #
+    # === Arguments
+    # cookbook<Chef::Cookbook>:: The cookbook to update
+    # valid_cache_entries<Hash>:: Out-param; Added to this hash are the files that
+    # were referred to by this cookbook
+    def sync_cookbook(cookbook)
+      Chef::Log.debug("Synchronizing cookbook #{cookbook.name}")
+
+      # files and templates are lazily loaded, and will be done later.
+
+      EAGER_SEGMENTS.each do |segment|
+        segment_filenames = Array.new
+        cookbook.manifest[segment].each do |manifest_record|
+
+          cache_filename = sync_file_in_cookbook(cookbook, manifest_record)
+          # make the segment filenames a full path.
+          full_path_cache_filename = cache.load(cache_filename, false)
+          segment_filenames << full_path_cache_filename
+        end
+
+        # replace segment filenames with a full-path one.
+        if segment.to_sym == :recipes
+          cookbook.recipe_filenames = segment_filenames
+        elsif segment.to_sym == :attributes
+          cookbook.attribute_filenames = segment_filenames
+        else
+          cookbook.segment_filenames(segment).replace(segment_filenames)
+        end
+      end
+      @events.synchronized_cookbook(cookbook.name)
+    end
+
+    # Sync an individual file if needed. If there is an up to date copy
+    # locally, nothing is done.
+    #
+    # === Arguments
+    # file_manifest::: A Hash of the form {"path" => 'relative/path', "url" => "location to fetch the file"}
+    # === Returns
+    # Path to the cached file as a String
+    def sync_file_in_cookbook(cookbook, file_manifest)
+      cache_filename = File.join("cookbooks", cookbook.name, file_manifest['path'])
+      mark_cached_file_valid(cache_filename)
+
+      # If the checksums are different between on-disk (current) and on-server
+      # (remote, per manifest), do the update. This will also execute if there
+      # is no current checksum.
+      if !cached_copy_up_to_date?(cache_filename, file_manifest['checksum'])
+        download_file(file_manifest['url'], cache_filename)
+        @events.updated_cookbook_file(cookbook.name, cache_filename)
+      else
+        Chef::Log.debug("Not storing #{cache_filename}, as the cache is up to date.")
+      end
+
+      cache_filename
+    end
+
+    def cached_copy_up_to_date?(local_path, expected_checksum)
+      if cache.has_key?(local_path)
+        current_checksum = CookbookVersion.checksum_cookbook_file(cache.load(local_path, false))
+        expected_checksum == current_checksum
+      else
+        false
+      end
+    end
+
+    # Unconditionally download the file from the given URL. File will be
+    # downloaded to the path +destination+ which is relative to the Chef file
+    # cache root.
+    def download_file(url, destination)
+      raw_file = server_api.get_rest(url, true)
+
+      Chef::Log.info("Storing updated #{destination} in the cache.")
+      cache.move_to(raw_file.path, destination)
+    end
+
+    # Marks the given file as valid (non-stale).
+    def mark_cached_file_valid(cache_filename)
+      CookbookCacheCleaner.instance.mark_file_as_valid(cache_filename)
+    end
+
+    def server_api
+      Chef::REST.new(Chef::Config[:chef_server_url])
+    end
+
+  end
+end

data/lib/chef/cookbook_version.rb

@@ -20,7 +20,6 @@
 # limitations under the License.
 
 require 'chef/log'
-require 'chef/client'
 require 'chef/node'
 require 'chef/resource_definition_list'
 require 'chef/recipe'
@@ -104,6 +103,7 @@ class Chef
     end
   end
 
+
   # == Chef::CookbookVersion
   # CookbookVersion is a model object encapsulating the data about a Chef
   # cookbook. Chef supports maintaining multiple versions of a cookbook on a
@@ -284,28 +284,10 @@ class Chef
       nil
     end
 
-    # Keep track of the filenames that we use in both eager cookbook
-    # downloading (during sync_cookbooks) and lazy (during the run
-    # itself, through FileVendor). After the run is over, clean up the
-    # cache.
-    def self.valid_cache_entries
-      @valid_cache_entries ||= {}
-    end
-
-    def self.reset_cache_validity
-      @valid_cache_entries = nil
-    end
-
     def self.cache
       Chef::FileCache
     end
 
-    # Setup a notification to clear the valid_cache_entries when a Chef client
-    # run starts
-    Chef::Client.when_run_starts do |run_status|
-      reset_cache_validity
-    end
-
     # Synchronizes all the cookbooks from the chef-server.
     #
     # === Returns
@@ -351,8 +333,11 @@ class Chef
 
       # files and templates are lazily loaded, and will be done later.
       eager_segments = COOKBOOK_SEGMENTS.dup
-      eager_segments.delete(:files)
-      eager_segments.delete(:templates)
+
+      unless Chef::Config[:no_lazy_load] then
+        eager_segments.delete(:files)
+        eager_segments.delete(:templates)
+      end
 
       eager_segments.each do |segment|
         segment_filenames = Array.new
@@ -412,11 +397,6 @@ class Chef
       end
     end
 
-    # Register a notification to cleanup unused files from cookbooks
-    Chef::Client.when_run_completes_successfully do |run_status|
-      cleanup_file_cache
-    end
-
     # Creates a new Chef::CookbookVersion object.
     #
     # === Returns

data/lib/chef/data_bag.rb

@@ -159,7 +159,7 @@ class Chef
           raise Chef::Exceptions::InvalidDataBagPath, "Data bag path '#{Chef::Config[:data_bag_path]}' is invalid"
         end
 
-        Dir.glob(File.join(Chef::Config[:data_bag_path], name, "*.json")).inject({}) do |bag, f|
+        Dir.glob(File.join(Chef::Config[:data_bag_path], "#{name}", "*.json")).inject({}) do |bag, f|
           item = JSON.parse(IO.read(f))
           bag[item['id']] = item
           bag
@@ -193,7 +193,11 @@ class Chef
     # Save the Data Bag via RESTful API
     def save
       begin
-        chef_server_rest.put_rest("data/#{@name}", self)
+        if Chef::Config[:why_run]
+          Chef::Log.warn("In whyrun mode, so NOT performing data bag save.")
+        else
+          chef_server_rest.put_rest("data/#{@name}", self)
+        end
       rescue Net::HTTPServerException => e
         raise e unless e.response.code == "404"
         chef_server_rest.post_rest("data", self)

data/lib/chef/data_bag_item.rb

@@ -224,7 +224,11 @@ class Chef
     def save(item_id=@raw_data['id'])
       r = chef_server_rest
       begin
-        r.put_rest("data/#{data_bag}/#{item_id}", self)
+        if Chef::Config[:why_run]
+          Chef::Log.warn("In whyrun mode, so NOT performing data bag item save.")
+        else
+          r.put_rest("data/#{data_bag}/#{item_id}", self)
+        end
       rescue Net::HTTPServerException => e
         raise e unless e.response.code == "404"
         r.post_rest("data/#{data_bag}", self)

data/lib/chef/event_dispatch/base.rb

@@ -0,0 +1,303 @@
+class Chef
+
+  # ==EventDispatch
+  # Classes in EventDispatch deal with collecting, distributing, and handling
+  # information in response to events that occur during a chef-client run.
+  #
+  # EventDispatch uses a simple publishing system where data from all events
+  # are forwarded to all subscribers unconditionally.
+  #
+  # EventDispatch is used to implement custom console output formatters so that
+  # users may have more control over the formatting and verbosity of Chef
+  # client output and client-side data collection for server-side client
+  # history storage and reporting.
+  #
+  # === API Stability Status
+  # The EventDispatch API is intended to become a stable, public API upon which
+  # end-users can implement their own custom output formatters, reporting
+  # integration libraries, and more. This is a new feature, however, so
+  # breaking changes may be required as it "bakes" in order to provide a clean,
+  # coherent and supportable API in the long term. Therefore, developers should
+  # consider the feature "beta" for now and be prepared for possible breaking
+  # changes in point releases.
+  module EventDispatch
+
+    # == EventDispatch::Base
+    # EventDispatch::Base is a completely abstract base class that defines the
+    # API used by both the classes that collect event information and those
+    # that process them.
+    class Base
+
+      # Called at the very start of a Chef Run
+      def run_start(version)
+      end
+
+      # Called at the end a successful Chef run.
+      def run_completed(node)
+      end
+
+      # Called at the end of a failed Chef run.
+      def run_failed(exception)
+      end
+
+      # Called right after ohai runs.
+      def ohai_completed(node)
+      end
+
+      # Already have a client key, assuming this node has registered.
+      def skipping_registration(node_name, config)
+      end
+
+      # About to attempt to register as +node_name+
+      def registration_start(node_name, config)
+      end
+
+      def registration_completed
+      end
+
+      # Failed to register this client with the server.
+      def registration_failed(node_name, exception, config)
+      end
+
+      # Called before Chef client loads the node data from the server
+      def node_load_start(node_name, config)
+      end
+
+      # TODO: def node_run_list_overridden(*args)
+
+      # Failed to load node data from the server
+      def node_load_failed(node_name, exception, config)
+      end
+
+      # Error expanding the run list
+      def run_list_expand_failed(node, exception)
+      end
+
+      # Called after Chef client has loaded the node data.
+      # Default and override attrs from roles have been computed, but not yet applied.
+      # Normal attrs from JSON have been added to the node.
+      def node_load_completed(node, expanded_run_list, config)
+      end
+
+      # Called before the cookbook collection is fetched from the server.
+      def cookbook_resolution_start(expanded_run_list)
+      end
+
+      # Called when there is an error getting the cookbook collection from the
+      # server.
+      def cookbook_resolution_failed(expanded_run_list, exception)
+      end
+
+      # Called when the cookbook collection is returned from the server.
+      def cookbook_resolution_complete(cookbook_collection)
+      end
+
+      # Called before unneeded cookbooks are removed
+      def cookbook_clean_start
+      end
+
+      # Called after the file at +path+ is removed. It may be removed if the
+      # cookbook containing it was removed from the run list, or if the file was
+      # removed from the cookbook.
+      def removed_cookbook_file(path)
+      end
+
+      # Called when cookbook cleaning is finished.
+      def cookbook_clean_complete
+      end
+
+      # Called before cookbook sync starts
+      def cookbook_sync_start(cookbook_count)
+      end
+
+      # Called when cookbook +cookbook_name+ has been sync'd
+      def synchronized_cookbook(cookbook_name)
+      end
+
+      # Called when an individual file in a cookbook has been updated
+      def updated_cookbook_file(cookbook_name, path)
+      end
+
+      # Called when an error occurs during cookbook sync
+      def cookbook_sync_failed(cookbooks, exception)
+      end
+
+      # Called after all cookbooks have been sync'd.
+      def cookbook_sync_complete
+      end
+
+      ## TODO: add cookbook name to the API for file load callbacks
+
+      ## TODO: add callbacks for overall cookbook eval start and complete.
+
+      # Called when library file loading starts
+      def library_load_start(file_count)
+      end
+
+      # Called when library file has been loaded
+      def library_file_loaded(path)
+      end
+
+      # Called when a library file has an error on load.
+      def library_file_load_failed(path, exception)
+      end
+
+      # Called when library file loading has finished
+      def library_load_complete
+      end
+
+      # Called when LWRP loading starts
+      def lwrp_load_start(lwrp_file_count)
+      end
+
+      # Called after a LWR or LWP has been loaded
+      def lwrp_file_loaded(path)
+      end
+
+      # Called after a LWR or LWP file errors on load
+      def lwrp_file_load_failed(path, exception)
+      end
+
+      # Called when LWRPs are finished loading
+      def lwrp_load_complete
+      end
+
+      # Called before attribute files are loaded
+      def attribute_load_start(attribute_file_count)
+      end
+
+      # Called after the attribute file is loaded
+      def attribute_file_loaded(path)
+      end
+
+      # Called when an attribute file fails to load.
+      def attribute_file_load_failed(path, exception)
+      end
+
+      # Called when attribute file loading is finished
+      def attribute_load_complete
+      end
+
+      # Called before resource definitions are loaded
+      def definition_load_start(definition_file_count)
+      end
+
+      # Called when a resource definition has been loaded
+      def definition_file_loaded(path)
+      end
+
+      # Called when a resource definition file fails to load
+      def definition_file_load_failed(path, exception)
+      end
+
+      # Called when resource defintions are done loading
+      def definition_load_complete
+      end
+
+      # Called before recipes are loaded
+      def recipe_load_start(recipe_count)
+      end
+
+      # Called after the recipe has been loaded
+      def recipe_file_loaded(path)
+      end
+
+      # Called after a recipe file fails to load
+      def recipe_file_load_failed(path, exception)
+      end
+
+      # Called when recipes have been loaded.
+      def recipe_load_complete
+      end
+
+      # Called before convergence starts
+      def converge_start(run_context)
+      end
+
+      # Called when the converge phase is finished.
+      def converge_complete
+      end
+
+      # TODO: need events for notification resolve?
+      # def notifications_resolved
+      # end
+
+      # Called before action is executed on a resource.
+      def resource_action_start(resource, action, notification_type=nil, notifier=nil)
+      end
+
+      # Called when a resource fails, but will retry.
+      def resource_failed_retriable(resource, action, retry_count, exception)
+      end
+
+      # Called when a resource fails and will not be retried.
+      def resource_failed(resource, action, exception)
+      end
+
+      # Called when a resource action has been skipped b/c of a conditional
+      def resource_skipped(resource, action, conditional)
+      end
+
+      # Called after #load_current_resource has run.
+      def resource_current_state_loaded(resource, action, current_resource)
+      end
+
+      # Called when resource current state load is skipped due to the provider
+      # not supporting whyrun mode.
+      def resource_current_state_load_bypassed(resource, action, current_resource)
+      end
+
+      # Called when evaluating a resource that does not support whyrun in whyrun mode
+      def resource_bypassed(resource, action, current_resource)
+      end
+
+      # Called when a resource has no converge actions, e.g., it was already correct.
+      def resource_up_to_date(resource, action)
+      end
+
+      # Called when a change has been made to a resource. May be called multiple
+      # times per resource, e.g., a file may have its content updated, and then
+      # its permissions updated.
+      def resource_update_applied(resource, action, update)
+      end
+
+      # Called after a resource has been completely converged, but only if
+      # modifications were made.
+      def resource_updated(resource, action)
+      end
+
+      # Called before handlers run
+      def handlers_start(handler_count)
+      end
+
+      # Called after an individual handler has run
+      def handler_executed(handler)
+      end
+
+      # Called after all handlers have executed
+      def handlers_completed
+      end
+
+      # Called when an assertion declared by a provider fails
+      def provider_requirement_failed(action, resource, exception, message)
+      end
+
+      # Called when a provider makes an assumption after a failed assertion
+      # in whyrun mode, in order to allow execution to continue
+      def whyrun_assumption(action, resource, message) 
+      end
+
+      ## TODO: deprecation warning. this way we can queue them up and present
+      #  them all at once.
+
+      # An uncategorized message. This supports the case that a user needs to
+      # pass output that doesn't fit into one of the callbacks above. Note that
+      # there's no semantic information about the content or importance of the
+      # message. That means that if you're using this too often, you should add a
+      # callback for it.
+      def msg(message)
+      end
+
+    end
+  end
+end