chef 0.9.8 → 0.9.10.rc.0

data/lib/chef/cookbook_version.rb

@@ -19,6 +19,7 @@
 # limitations under the License.
 
 require 'chef/log'
+require 'chef/client'
 require 'chef/node'
 require 'chef/resource_definition_list'
 require 'chef/recipe'
@@ -39,7 +40,7 @@ class Chef
     COOKBOOK_SEGMENTS = [ :resources, :providers, :recipes, :definitions, :libraries, :attributes, :files, :templates, :root_files ]
     
     DESIGN_DOCUMENT = {
-      "version" => 5,
+      "version" => 7,
       "language" => "javascript",
       "views" => {
         "all" => {
@@ -89,8 +90,8 @@ class Chef
                 continue;
               }
               
-              var valueParts = value[1].split('.').map(function(v) { return parseInt(v); });
-              var resultParts = result[1].split('.').map(function(v) { return parseInt(v); });
+              var valueParts = value.split('.').map(function(v) { return parseInt(v); });
+              var resultParts = result.split('.').map(function(v) { return parseInt(v); });
 
               if (valueParts[0] != resultParts[0]) {
                 if (valueParts[0] > resultParts[0]) {
@@ -116,7 +117,7 @@ class Chef
           "map" => %q@
           function(doc) {
             if (doc.chef_type == "cookbook_version") {
-              emit(doc.cookbook_name, doc.version);
+              emit(doc.cookbook_name, {version: doc.version, id:doc._id});
             }
           }
           @,
@@ -132,8 +133,8 @@ class Chef
                 continue;
               }
 
-              var valueParts = value[1].split('.').map(function(v) { return parseInt(v); });
-              var resultParts = result[1].split('.').map(function(v) { return parseInt(v); });
+              var valueParts = value.version.split('.').map(function(v) { return parseInt(v); });
+              var resultParts = result.version.split('.').map(function(v) { return parseInt(v); });
 
               if (valueParts[0] != resultParts[0]) {
                 if (valueParts[0] > resultParts[0]) {
@@ -151,13 +152,14 @@ class Chef
                 }
               }
             }
-            return keys[idx][1];
+            return result;
           }
           @
         },
       }
     }
 
+    attr_accessor :root_dir
     attr_accessor :definition_filenames
     attr_accessor :template_filenames
     attr_accessor :file_filenames
@@ -188,12 +190,132 @@ class Chef
     # This is the one and only method that knows how cookbook files'
     # checksums are generated.
     def self.checksum_cookbook_file(filepath)
-      Chef::Cache::Checksum.generate_md5_checksum_for_file(filepath)
+      Chef::ChecksumCache.generate_md5_checksum_for_file(filepath)
     rescue Errno::ENOENT
       Chef::Log.debug("File #{filepath} does not exist, so there is no checksum to generate")
       nil
     end
     
+    # Keep track of the filenames that we use in both eager cookbook
+    # downloading (during sync_cookbooks) and lazy (during the run
+    # itself, through FileVendor). After the run is over, clean up the
+    # cache.
+    def self.valid_cache_entries
+      @valid_cache_entries ||= {}
+    end
+
+    def self.reset_cache_validity
+      @valid_cache_entries = nil
+    end
+
+    # Setup a notification to clear the valid_cache_entries when a Chef client
+    # run starts
+    Chef::Client.when_run_starts do |run_status|
+      reset_cache_validity
+    end
+
+    # Synchronizes all the cookbooks from the chef-server.
+    #
+    # === Returns
+    # true:: Always returns true
+    def self.sync_cookbooks(cookbook_hash)
+      Chef::Log.debug("Cookbooks to load: #{cookbook_hash.inspect}")
+
+      # Remove all cookbooks no longer relevant to this node
+      Chef::FileCache.find(File.join(%w{cookbooks ** *})).each do |cache_file|
+        cache_file =~ /^cookbooks\/([^\/]+)\//
+        unless cookbook_hash.has_key?($1)
+          Chef::Log.info("Removing #{cache_file} from the cache; its cookbook is no longer needed on this client.")
+          Chef::FileCache.delete(cache_file)
+        end
+      end
+
+      # Synchronize each of the node's cookbooks, and add to the
+      # valid_cache_entries hash.
+      cookbook_hash.values.each do |cookbook|
+        sync_cookbook_file_cache(cookbook)
+      end
+
+      true
+    end
+
+    # Update the file caches for a given cache segment.  Takes a segment name
+    # and a hash that matches one of the cookbooks/_attribute_files style
+    # remote file listings.
+    #
+    # === Parameters
+    # cookbook<Chef::Cookbook>:: The cookbook to update
+    # valid_cache_entries<Hash>:: Out-param; Added to this hash are the files that
+    # were referred to by this cookbook
+    def self.sync_cookbook_file_cache(cookbook)
+      Chef::Log.debug("Synchronizing cookbook #{cookbook.name}")
+
+      # files and templates are lazily loaded, and will be done later.
+      eager_segments = COOKBOOK_SEGMENTS.dup
+      eager_segments.delete(:files)
+      eager_segments.delete(:templates)
+
+      eager_segments.each do |segment|
+        segment_filenames = Array.new
+        cookbook.manifest[segment].each do |manifest_record|
+          # segment = cookbook segment
+          # remote_list = list of file hashes
+          #
+          # We need the list of known good attribute files, so we can delete any that are
+          # just laying about.
+
+          cache_filename = File.join("cookbooks", cookbook.name, manifest_record['path'])
+          valid_cache_entries[cache_filename] = true
+
+          current_checksum = nil
+          if Chef::FileCache.has_key?(cache_filename)
+            current_checksum = checksum_cookbook_file(Chef::FileCache.load(cache_filename, false))
+          end
+
+          # If the checksums are different between on-disk (current) and on-server
+          # (remote, per manifest), do the update. This will also execute if there
+          # is no current checksum.
+          if current_checksum != manifest_record['checksum']
+            raw_file = chef_server_rest.get_rest(manifest_record[:url], true)
+
+            Chef::Log.info("Storing updated #{cache_filename} in the cache.")
+            Chef::FileCache.move_to(raw_file.path, cache_filename)
+          else
+            Chef::Log.debug("Not storing #{cache_filename}, as the cache is up to date.")
+          end
+
+          # make the segment filenames a full path.
+          full_path_cache_filename = Chef::FileCache.load(cache_filename, false)
+          segment_filenames << full_path_cache_filename
+        end
+
+        # replace segment filenames with a full-path one.
+        if segment.to_sym == :recipes
+          cookbook.recipe_filenames = segment_filenames
+        elsif segment.to_sym == :attributes
+          cookbook.attribute_filenames = segment_filenames
+        else
+          cookbook.segment_filenames(segment).replace(segment_filenames)
+        end
+      end
+    end
+
+    def self.cleanup_file_cache
+      # Delete each file in the cache that we didn't encounter in the
+      # manifest.
+      Chef::FileCache.find(File.join(%w{cookbooks ** *})).each do |cache_filename|
+        unless valid_cache_entries[cache_filename]
+          Chef::Log.info("Removing #{cache_filename} from the cache; it is no longer on the server.")
+          Chef::FileCache.delete(cache_filename)
+        end
+      end
+    end
+
+    # Register a notification to cleanup unused files from cookbooks
+    Chef::Client.when_run_completes_successfully do |run_status|
+      cleanup_file_cache
+    end
+
     # Creates a new Chef::CookbookVersion object.  
     #
     # === Returns
@@ -210,6 +332,7 @@ class Chef
       @resource_filenames = Array.new
       @provider_filenames = Array.new
       @metadata_filenames = Array.new
+      @root_dir = nil
       @root_filenames = Array.new
       @couchdb_id = nil
       @couchdb = couchdb || Chef::CouchDB.new
@@ -473,113 +596,8 @@ class Chef
 
       records_by_pref[best_pref]
     end
-    
-    # Given a node, segment and path (filename or directory name),
-    # return the priority-ordered list of preference locations to
-    # look.
-    def preferences_for_path(node, segment, path)
-      # only files and templates can be platform-specific
-      if segment.to_sym == :files || segment.to_sym == :templates
-        begin
-          platform, version = Chef::Platform.find_platform_and_version(node)
-        rescue ArgumentError => e
-          # Skip platform/version if they were not found by find_platform_and_version
-          if e.message =~ /Cannot find a (?:platform|version)/
-            platform = "/unknown_platform/"
-            version = "/unknown_platform_version/"
-          else
-            raise
-          end
-        end
-        
-        fqdn = node[:fqdn]
-
-        # Most specific to least specific places to find the path
-        [
-         File.join(segment.to_s, "host-#{fqdn}", path),
-         File.join(segment.to_s, "#{platform}-#{version}", path),
-         File.join(segment.to_s, platform.to_s, path),
-         File.join(segment.to_s, "default", path)
-        ]
-      else
-        [File.join(segment, path)]
-      end
-    end
-    private :preferences_for_path
-
-
-    def relative_filenames_in_preferred_directory(node, segment, dirname)
-      preferences = preferences_for_path(node, segment, dirname)
-      filenames_by_pref = Hash.new
-      preferences.each { |pref| filenames_by_pref[pref] = Array.new }
-
-      manifest[segment].each do |manifest_record|
-        manifest_record_path = manifest_record[:path]
-
-        # find the NON SPECIFIC filenames, but prefer them by filespecificity.
-        # For example, if we have a file:
-        # 'files/default/somedir/somefile.conf' we only keep
-        # 'somedir/somefile.conf'. If there is also
-        # 'files/$hostspecific/somedir/otherfiles' that matches the requested
-        # hostname specificity, that directory will win, as it is more specific.
-        #
-        # This is clearly ugly b/c the use case is for remote directory, where
-        # we're just going to make cookbook_files out of these and make the
-        # cookbook find them by filespecificity again. but it's the shortest
-        # path to "success" for now.
-        if manifest_record_path =~ /(#{Regexp.escape(segment.to_s)}\/[^\/]+\/#{Regexp.escape(dirname)})\/.+$/
-          specificity_dirname = $1
-          non_specific_path = manifest_record_path[/#{Regexp.escape(segment.to_s)}\/[^\/]+\/#{Regexp.escape(dirname)}\/(.+)$/, 1]
-          # Record the specificity_dirname only if it's in the list of
-          # valid preferences
-          if filenames_by_pref[specificity_dirname]
-            filenames_by_pref[specificity_dirname] << non_specific_path
-          end
-        end
-      end
-
-      best_pref = preferences.find { |pref| !filenames_by_pref[pref].empty? }
-
-      raise Chef::Exceptions::FileNotFound, "cookbook #{name} has no directory #{segment}/#{dirname}" unless best_pref
-
-      filenames_by_pref[best_pref]
-
-    end
-
-    # Determine the manifest records from the most specific directory
-    # for the given node. See #preferred_manifest_record for a
-    # description of entries of the returned Array.
-    def preferred_manifest_records_for_directory(node, segment, dirname)
-      preferences = preferences_for_path(node, segment, dirname)
-      records_by_pref = Hash.new
-      preferences.each { |pref| records_by_pref[pref] = Array.new }
-
-      manifest[segment].each do |manifest_record|
-        manifest_record_path = manifest_record[:path]
 
-        # extract the preference part from the path.
-        if manifest_record_path =~ /(#{Regexp.escape(segment.to_s)}\/[^\/]+\/#{Regexp.escape(dirname)})\/.+$/
-          # Note the specificy_dirname includes the segment and
-          # dirname argument as above, which is what
-          # preferences_for_path returns. It could be
-          # "files/ubuntu-9.10/dirname", for example.
-          specificity_dirname = $1
-          
-          # Record the specificity_dirname only if it's in the list of
-          # valid preferences
-          if records_by_pref[specificity_dirname]
-            records_by_pref[specificity_dirname] << manifest_record
-          end
-        end
-      end
-      
-      best_pref = preferences.find { |pref| !records_by_pref[pref].empty? }
-        
-      raise Chef::Exceptions::FileNotFound, "cookbook #{name} has no directory #{segment}/#{dirname}" unless best_pref
 
-      records_by_pref[best_pref]
-    end
-    
     # Given a node, segment and path (filename or directory name),
     # return the priority-ordered list of preference locations to
     # look.
@@ -656,6 +674,20 @@ class Chef
       rendered_manifest
     end
 
+    def metadata_json_file
+      File.join(root_dir, "metadata.json")
+    end
+
+    def metadata_rb_file
+      File.join(root_dir, "metadata.rb")
+    end
+
+    def reload_metadata!
+      if File.exists?(metadata_json_file)
+        metadata.from_json(IO.read(metadata_json_file))
+      end
+    end
+
     ##
     # REST API
     ##
@@ -727,7 +759,7 @@ class Chef
     def self.cdb_list_latest(inflate=false, couchdb=nil)
       couchdb ||= Chef::CouchDB.new
       if inflate
-        doc_ids = cdb_list_latest_ids
+        doc_ids = cdb_list_latest_ids.map {|i|i["id"]}
         couchdb.bulk_get(doc_ids)
       else
         results = couchdb.get_view("cookbooks", "all_latest_version", :group=>true)["rows"]
@@ -761,6 +793,15 @@ class Chef
       (couchdb || Chef::CouchDB.new).delete("cookbook_version", full_name, couchdb_rev)
     end
 
+    # Runs on Chef Server (API); deletes the cookbook from couchdb and also destroys associated
+    # checksum documents 
+    def purge
+      checksums.keys.each do |checksum|
+        Chef::Checksum.cdb_load(checksum, couchdb).purge
+      end
+      cdb_destroy
+    end
+
     def cdb_save
       @couchdb_rev = couchdb.store("cookbook_version", full_name, self)["rev"]
     end

data/lib/chef/exceptions.rb

@@ -57,5 +57,10 @@ class Chef
     class DsclCommandFailed < RuntimeError; end
     class UserIDNotFound < ArgumentError; end
     class GroupIDNotFound < ArgumentError; end
+    class InvalidResourceReference < RuntimeError; end
+    class ResourceNotFound < RuntimeError; end
+    class InvalidResourceSpecification < ArgumentError; end
+    class SolrConnectionError < RuntimeError; end
+    class IllegalChecksumRevert < RuntimeError; end
   end
 end

data/lib/chef/handler.rb

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+require 'chef/client'
 require 'forwardable'
 
 class Chef
@@ -56,8 +57,50 @@ class Chef
   #
   class Handler
 
+    # The list of currently configured report handlers
+    def self.report_handlers
+      Array(Chef::Config[:report_handlers])
+    end
+
+    # Run the report handlers. This will usually be called by a notification
+    # from Chef::Client
+    def self.run_report_handlers(run_status)
+      Chef::Log.info("Running report handlers")
+      report_handlers.each do |handler|
+        handler.run_report_safely(run_status)
+      end
+      Chef::Log.info("Report handlers complete")
+    end
+
+    # Wire up a notification to run the report handlers if the chef run
+    # succeeds.
+    Chef::Client.when_run_completes_successfully do |run_status|
+      run_report_handlers(run_status)
+    end
+
+    # The list of currently configured exception handlers
+    def self.exception_handlers
+      Array(Chef::Config[:exception_handlers])
+    end
+
+    # Run the exception handlers. Usually will be called by a notification
+    # from Chef::Client when the run fails.
+    def self.run_exception_handlers(run_status)
+      Chef::Log.error("Running exception handlers")
+      exception_handlers.each do |handler|
+        handler.run_report_safely(run_status)
+      end
+      Chef::Log.error("Exception handlers complete")
+    end
+
+    # Wire up a notification to run the exception handlers if the chef run fails.
+    Chef::Client.when_run_fails do |run_status|
+      run_exception_handlers(run_status)
+    end
+
     extend Forwardable
 
+    # The Chef::RunStatus object containing data about the Chef run.
     attr_reader :run_status
 
     ##

data/lib/chef/index_queue/consumer.rb

@@ -56,7 +56,7 @@ class Chef
       alias :start :run
       
       def call_action_for_message(message)
-        amqp_payload  = JSON.parse(message[:payload])
+        amqp_payload  = JSON.parse(message[:payload], :create_additions => false, :max_nesting => false)
         action        = amqp_payload["action"].to_sym
         app_payload   = amqp_payload["payload"]
         assert_method_whitelisted(action)

data/lib/chef/index_queue/indexable.rb

@@ -55,7 +55,7 @@ class Chef
         with_metadata["type"]     ||= self.index_object_type
         with_metadata["id"]       ||= self.index_id
         with_metadata["database"] ||= Chef::Config[:couchdb_database]
-        with_metadata["item"]     ||= self
+        with_metadata["item"]     ||= self.to_hash
 
         raise ArgumentError, "Type, Id, or Database missing in index operation: #{with_metadata.inspect}" if (with_metadata["id"].nil? or with_metadata["type"].nil?)
         with_metadata        

data/lib/chef/knife.rb

@@ -151,6 +151,18 @@ class Chef
       subcommand_class || subcommand_not_found!(args)
     end
 
+    protected
+
+    def load_late_dependency(dep, gem_name = nil)
+      begin
+        require dep
+      rescue LoadError
+        gem_name ||= dep.gsub('/', '-')
+        Chef::Log.fatal "#{gem_name} is not installed. run \"gem install #{gem_name}\" to install it."
+        exit 1
+      end
+    end
+
     private
 
     # :nodoc:
@@ -255,7 +267,6 @@ class Chef
       Chef::Config[:node_name] = config[:node_name] if config[:node_name]
       Chef::Config[:client_key] = config[:client_key] if config[:client_key]
       Chef::Config[:chef_server_url] = config[:chef_server_url] if config[:chef_server_url]
-      Mixlib::Log::Formatter.show_time = false
       Chef::Log.init(Chef::Config[:log_location])
       Chef::Log.level(Chef::Config[:log_level])
 
@@ -407,10 +418,12 @@ class Chef
       object = klass.load(name)
 
       output = edit_data(object)
-      
-      output.save
-
-      self.msg("Saved #{output}")
+      if output.to_s != object.to_s
+        output.save
+        self.msg("Saved #{output}")
+      else
+        self.msg("Object unchanged, not saving")
+      end
 
       output(format_for_display(object)) if config[:print_after]
     end