chef 0.9.18 → 0.10.0.beta.0

data/lib/chef/cookbook_version.rb

@@ -1,17 +1,17 @@
-#
 # Author:: Adam Jacob (<adam@opscode.com>)
 # Author:: Nuo Yan (<nuo@opscode.com>)
 # Author:: Christopher Walters (<cw@opscode.com>)
 # Author:: Tim Hinderliter (<tim@opscode.com>)
-# Copyright:: Copyright (c) 2008-2010 Opscode, Inc.
+# Author:: Seth Falcon (<seth@opscode.com>)
+# Copyright:: Copyright 2008-2010 Opscode, Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -25,6 +25,8 @@ require 'chef/resource_definition_list'
 require 'chef/recipe'
 require 'chef/cookbook/file_vendor'
 require 'chef/checksum'
+require 'chef/cookbook/metadata'
+require 'chef/version_class'
 
 class Chef
   # == Chef::CookbookVersion
@@ -37,16 +39,17 @@ class Chef
   # recipe_filenames.insert) should dirty the manifest so it gets regenerated.
   class CookbookVersion
     include Chef::IndexQueue::Indexable
+    include Comparable
 
     COOKBOOK_SEGMENTS = [ :resources, :providers, :recipes, :definitions, :libraries, :attributes, :files, :templates, :root_files ]
-    
+
     DESIGN_DOCUMENT = {
       "version" => 7,
       "language" => "javascript",
       "views" => {
         "all" => {
           "map" => <<-EOJS
-          function(doc) { 
+          function(doc) {
             if (doc.chef_type == "cookbook_version") {
               emit(doc.name, doc);
             }
@@ -55,7 +58,7 @@ class Chef
         },
         "all_id" => {
           "map" => <<-EOJS
-          function(doc) { 
+          function(doc) {
             if (doc.chef_type == "cookbook_version") {
               emit(doc.name, doc.name);
             }
@@ -64,7 +67,7 @@ class Chef
         },
         "all_with_version" => {
           "map" => <<-EOJS
-          function(doc) { 
+          function(doc) {
             if (doc.chef_type == "cookbook_version") {
               emit(doc.cookbook_name, doc.version);
             }
@@ -73,7 +76,7 @@ class Chef
         },
         "all_latest_version" => {
           "map" => %q@
-          function(doc) { 
+          function(doc) {
             if (doc.chef_type == "cookbook_version") {
               emit(doc.cookbook_name, doc.version);
             }
@@ -85,12 +88,12 @@ class Chef
 
             for (var idx in values) {
               var value = values[idx];
-              
+
               if (idx == 0) {
                 result = value;
                 continue;
               }
-              
+
               var valueParts = value.split('.').map(function(v) { return parseInt(v); });
               var resultParts = result.split('.').map(function(v) { return parseInt(v); });
 
@@ -187,7 +190,7 @@ class Chef
 
     attr_reader :recipe_filenames_by_name
     attr_reader :attribute_filenames_by_short_filename
-    
+
     # This is the one and only method that knows how cookbook files'
     # checksums are generated.
     def self.checksum_cookbook_file(filepath)
@@ -196,7 +199,7 @@ class Chef
       Chef::Log.debug("File #{filepath} does not exist, so there is no checksum to generate")
       nil
     end
-    
+
     # Keep track of the filenames that we use in both eager cookbook
     # downloading (during sync_cookbooks) and lazy (during the run
     # itself, through FileVendor). After the run is over, clean up the
@@ -224,7 +227,8 @@ class Chef
     # === Returns
     # true:: Always returns true
     def self.sync_cookbooks(cookbook_hash)
-      Chef::Log.debug("Cookbooks to load: #{cookbook_hash.inspect}")
+      Chef::Log.info("Loading cookbooks [#{cookbook_hash.keys.sort.join(', ')}]")
+      Chef::Log.debug("Cookbooks detail: #{cookbook_hash.inspect}")
 
       clear_obsoleted_cookbooks(cookbook_hash)
 
@@ -317,7 +321,7 @@ class Chef
         # manifest.
         cache.find(File.join(%w{cookbooks ** *})).each do |cache_filename|
           unless valid_cache_entries[cache_filename]
-            Chef::Log.info("Removing #{cache_filename} from the cache; it is no longer on the server.")
+            Chef::Log.info("Removing #{cache_filename} from the cache; it is no longer needed by chef-client.")
             cache.delete(cache_filename)
           end
         end
@@ -329,12 +333,13 @@ class Chef
       cleanup_file_cache
     end
 
-    # Creates a new Chef::CookbookVersion object.  
+    # Creates a new Chef::CookbookVersion object.
     #
     # === Returns
     # object<Chef::CookbookVersion>:: Duh. :)
     def initialize(name, couchdb=nil)
       @name = name
+      @frozen = false
       @attribute_filenames = Array.new
       @definition_filenames = Array.new
       @template_filenames = Array.new
@@ -359,7 +364,18 @@ class Chef
     def version
       metadata.version
     end
-    
+
+    # Indicates if this version is frozen or not. Freezing a coobkook version
+    # indicates that a new cookbook with the same name and version number
+    # shoule
+    def frozen_version?
+      @frozen
+    end
+
+    def freeze_version
+      @frozen = true
+    end
+
     def version=(new_version)
       manifest["version"] = new_version
       metadata.version(new_version)
@@ -375,7 +391,7 @@ class Chef
     #     :version = "1.0",
     #     :name = "Apache 2"
     #     :metadata = ???TODO: timh/cw: 5-24-2010: describe this format,
-    #   
+    #
     #     :files => [
     #       {
     #         :name => "afile.rb",
@@ -393,7 +409,7 @@ class Chef
       end
       @manifest
     end
-    
+
     def manifest=(new_manifest)
       @manifest = Mash.new new_manifest
       @checksums = extract_checksums_from_manifest(@manifest)
@@ -402,7 +418,7 @@ class Chef
       COOKBOOK_SEGMENTS.each do |segment|
         next unless @manifest.has_key?(segment)
         filenames = @manifest[segment].map{|manifest_record| manifest_record['name']}
-        
+
         if segment == :recipes
           self.recipe_filenames = filenames
         elsif segment == :attributes
@@ -413,7 +429,7 @@ class Chef
         end
       end
     end
-    
+
     # Returns a hash of checksums to either nil or the on disk path (which is
     # done by generate_manifest).
     def checksums
@@ -426,17 +442,17 @@ class Chef
     def full_name
       "#{name}-#{version}"
     end
-    
+
     def attribute_filenames=(*filenames)
       @attribute_filenames = filenames.flatten
       @attribute_filenames_by_short_filename = filenames_by_name(attribute_filenames)
       attribute_filenames
     end
-    
+
     ## BACKCOMPAT/DEPRECATED - Remove these and fix breakage before release [DAN - 5/20/2010]##
     alias :attribute_files :attribute_filenames
     alias :attribute_files= :attribute_filenames=
-    
+
     # Return recipe names in the form of cookbook_name::recipe_name
     def fully_qualified_recipe_names
       results = Array.new
@@ -445,17 +461,17 @@ class Chef
       end
       results
     end
-    
+
     def recipe_filenames=(*filenames)
       @recipe_filenames = filenames.flatten
       @recipe_filenames_by_name = filenames_by_name(recipe_filenames)
       recipe_filenames
     end
-    
+
     ## BACKCOMPAT/DEPRECATED - Remove these and fix breakage before release [DAN - 5/20/2010]##
     alias :recipe_files :recipe_filenames
     alias :recipe_files= :recipe_filenames=
-    
+
     # called from DSL
     def load_recipe(recipe_name, run_context)
       unless recipe_filenames_by_name.has_key?(recipe_name)
@@ -469,7 +485,7 @@ class Chef
       unless recipe_filename
         raise Chef::Exceptions::RecipeNotFound, "could not find recipe #{recipe_name} for cookbook #{name}"
       end
-      
+
       recipe.from_file(recipe_filename)
       recipe
     end
@@ -519,7 +535,7 @@ class Chef
       # ensure that we generate the manifest, which will also generate
       # @manifest_records_by_path
       manifest
-      
+
       # in order of prefernce, look for the filename in the manifest
       found_pref = preferences.find {|preferred_filename| @manifest_records_by_path[preferred_filename] }
       if found_pref
@@ -528,7 +544,7 @@ class Chef
         raise Chef::Exceptions::FileNotFound, "cookbook #{name} does not contain file #{segment}/#{filename}"
       end
     end
-    
+
     def preferred_filename_on_disk_location(node, segment, filename, current_filepath=nil)
       manifest_record = preferred_manifest_record(node, segment, filename)
       if current_filepath && (manifest_record['checksum'] == self.class.checksum_cookbook_file(current_filepath))
@@ -594,7 +610,7 @@ class Chef
           # preferences_for_path returns. It could be
           # "files/ubuntu-9.10/dirname", for example.
           specificity_dirname = $1
-          
+
           # Record the specificity_dirname only if it's in the list of
           # valid preferences
           if records_by_pref[specificity_dirname]
@@ -602,9 +618,9 @@ class Chef
           end
         end
       end
-      
+
       best_pref = preferences.find { |pref| !records_by_pref[pref].empty? }
-        
+
       raise Chef::Exceptions::FileNotFound, "cookbook #{name} has no directory #{segment}/#{dirname}" unless best_pref
 
       records_by_pref[best_pref]
@@ -628,7 +644,7 @@ class Chef
             raise
           end
         end
-        
+
         fqdn = node[:fqdn]
 
         # Most specific to least specific places to find the path
@@ -646,6 +662,7 @@ class Chef
 
     def to_hash
       result = manifest.dup
+      result['frozen?'] = frozen_version?
       result['chef_type'] = 'cookbook_version'
       result["_rev"] = couchdb_rev if couchdb_rev
       result.to_hash
@@ -668,12 +685,17 @@ class Chef
         cookbook_version.index_id = cookbook_version.couchdb_id
         o.delete("_id")
       end
-      cookbook_version.manifest = o
       # We want the Chef::Cookbook::Metadata object to always be inflated
       cookbook_version.metadata = Chef::Cookbook::Metadata.from_hash(o["metadata"])
+      cookbook_version.manifest = o
+
+      # We don't need the following step when we decide to stop supporting deprecated operators in the metadata (e.g. <<, >>)
+      cookbook_version.manifest["metadata"] = JSON.parse(cookbook_version.metadata.to_json)
+
+      cookbook_version.freeze_version if o["frozen?"]
       cookbook_version
     end
-    
+
     def generate_manifest_with_urls(&url_generator)
       rendered_manifest = manifest.dup
       COOKBOOK_SEGMENTS.each do |segment|
@@ -712,12 +734,23 @@ class Chef
       self.class.chef_server_rest
     end
 
+    # Save this object to the server via the REST api. If there is an existing
+    # document on the server and it is marked frozen, a
+    # Net::HTTPServerException will be raised for 409 Conflict.
     def save
       chef_server_rest.put_rest("cookbooks/#{name}/#{version}", self)
       self
     end
     alias :create :save
 
+    # Adds the `force=true` parameter to the upload. This allows the user to
+    # overwrite a frozen cookbook (normal #save raises a
+    # Net::HTTPServerException for 409 Conflict in this case).
+    def force_save
+      chef_server_rest.put_rest("cookbooks/#{name}/#{version}?force=true", self)
+      self
+    end
+
     def destroy
       chef_server_rest.delete_rest("cookbooks/#{name}/#{version}")
       self
@@ -757,7 +790,7 @@ class Chef
     ##
     # Couchdb
     ##
-    
+
     def self.cdb_by_name(cookbook_name, couchdb=nil)
       cdb = (couchdb || Chef::CouchDB.new)
       options = { :startkey => cookbook_name, :endkey => cookbook_name }
@@ -787,9 +820,13 @@ class Chef
     end
 
     def self.cdb_list(inflate=false, couchdb=nil)
-      rs = (couchdb || Chef::CouchDB.new).list("cookbooks", inflate)
-      lookup = (inflate ? "value" : "key")
-      rs["rows"].collect { |r| r[lookup] }            
+      couchdb ||= Chef::CouchDB.new
+      if inflate
+        couchdb.list("cookbooks", true)["rows"].collect{|r| r["value"]}
+      else
+        # If you modify this, please make sure the desc sorted order on the versions doesn't get broken.
+        couchdb.get_view("cookbooks", "all_with_version")["rows"].inject({}) { |mapped, row| mapped[row["key"]]||=Array.new; mapped[row["key"]].push(Chef::Version.new(row["value"])); mapped[row["key"]].sort!.reverse!; mapped}
+      end
     end
 
     def self.cdb_load(name, version='latest', couchdb=nil)
@@ -807,7 +844,7 @@ class Chef
     end
 
     # Runs on Chef Server (API); deletes the cookbook from couchdb and also destroys associated
-    # checksum documents 
+    # checksum documents
     def purge
       checksums.keys.each do |checksum|
         begin
@@ -827,8 +864,16 @@ class Chef
       @index_id = value
     end
 
+    def <=>(o)
+      raise Chef::Exceptions::CookbookVersionNameMismatch if self.name != o.name
+      # FIXME: can we change the interface to the Metadata class such
+      # that metadata.version returns a Chef::Version instance instead
+      # of a string?
+      Chef::Version.new(self.version) <=> Chef::Version.new(o.version)
+    end
+
     private
-    
+
     # For each filename, produce a mapping of base filename (i.e. recipe name
     # or attribute file) to on disk location
     def filenames_by_name(filenames)
@@ -858,7 +903,7 @@ class Chef
           file_name = nil
           path = nil
           specificity = "default"
-          
+
           if segment == :root_files
             matcher = segment_file.match(".+/#{Regexp.escape(name.to_s)}/(.+)")
             file_name = matcher[1]
@@ -877,7 +922,7 @@ class Chef
             path = matcher[1]
             file_name = matcher[2]
           end
-          
+
           csum = self.class.checksum_cookbook_file(segment_file)
           checksums_to_on_disk_paths[csum] = segment_file
           rs = Mash.new({
@@ -900,7 +945,7 @@ class Chef
       @manifest = manifest
       @manifest_records_by_path = extract_manifest_records_by_path(manifest)
     end
-    
+
     def file_vendor
       unless @file_vendor
         @file_vendor = Chef::Cookbook::FileVendor.create_from_manifest(manifest)
@@ -918,7 +963,7 @@ class Chef
       end
       checksums
     end
-    
+
     def extract_manifest_records_by_path(manifest)
       manifest_records_by_path = {}
       COOKBOOK_SEGMENTS.each do |segment|
@@ -929,6 +974,6 @@ class Chef
       end
       manifest_records_by_path
     end
-    
+
   end
 end

data/lib/chef/cookbook_version_selector.rb

@@ -0,0 +1,163 @@
+#
+# Author:: Tim Hinderliter (<tim@opscode.com>)
+# Copyright:: Copyright (c) 2011 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'dep_selector'
+
+class Chef
+  module CookbookVersionSelector
+    # This method replaces verbiage from DepSelector messages with
+    # Chef-domain-specific verbiage, such as replacing package with
+    # cookbook.
+    #
+    # TODO [cw, 2011/2/25]: this is a near-term hack. In the long run,
+    # we'll do this better.
+    def self.filter_dep_selector_message(message)
+      m = message
+      m.gsub!("Package", "Cookbook")
+      m.gsub!("package", "cookbook")
+      m.gsub!("Solution constraint", "Run list item")
+      m.gsub!("solution constraint", "run list item")
+      m
+    end
+
+    # all_cookbooks - a hash mapping cookbook names to an array of
+    # available CookbookVersions.
+    #
+    # Creates a DependencyGraph from CookbookVersion objects
+    def self.create_dependency_graph_from_cookbooks(all_cookbooks)
+      dep_graph = DepSelector::DependencyGraph.new
+
+      all_cookbooks.each do |cb_name, cb_versions|
+        cb_versions.each do |cb_version|
+          cb_version_deps = cb_version.metadata.dependencies
+          # TODO [cw. 2011/2/10]: CookbookVersion#version returns a
+          # String even though we're storing as a DepSelector::Version
+          # object underneath. This should be changed so that we
+          # return the object and handle proper serialization and
+          # de-serialization. For now, I'm just going to create a
+          # Version object from the String representation.
+          pv = dep_graph.package(cb_name).add_version(Chef::Version.new(cb_version.version))
+          cb_version_deps.each_pair do |dep_name, constraint_str|
+            # if the dependency is specified as cookbook::recipe,
+            # extract the cookbook component
+            dep_cb_name = dep_name.split("::").first
+            constraint = Chef::VersionConstraint.new(constraint_str)
+            pv.dependencies << DepSelector::Dependency.new(dep_graph.package(dep_cb_name), constraint)
+          end
+        end
+      end
+
+      dep_graph
+    end
+
+    # Return a hash mapping cookbook names to a CookbookVersion
+    # object. If there is no solution that satisfies the constraints,
+    # the first run list item that caused unsatisfiability is
+    # returned.
+    #
+    # This is the final version-resolved list of cookbooks for the
+    # RunList.
+    #
+    # all_cookbooks - a hash mapping cookbook names to an array of
+    # available CookbookVersions.
+    #
+    # recipe_constraints - an array of hashes describing the expanded
+    # run list.  Each element is a hash containing keys :name and
+    # :version_constraint. The :name component is either the
+    # fully-qualified recipe name (e.g. "cookbook1::non_default_recipe")
+    # or just a cookbook name, indicating the default recipe is to be
+    # run (e.g. "cookbook1").
+    def self.constrain(all_cookbooks, recipe_constraints)
+      dep_graph = create_dependency_graph_from_cookbooks(all_cookbooks)
+
+      # extract cookbook names from (possibly) fully-qualified recipe names
+      cookbook_constraints = recipe_constraints.map do |recipe_spec|
+        cookbook_name = (recipe_spec[:name][/^(.+)::/, 1] || recipe_spec[:name])
+        DepSelector::SolutionConstraint.new(dep_graph.package(cookbook_name),
+                                            recipe_spec[:version_constraint])
+      end
+
+      # Pass in the list of all available cookbooks (packages) so that
+      # DepSelector can distinguish between "no version available for
+      # cookbook X" and "no such cookbook X" when an environment
+      # filters out all versions for a given cookbook.
+      all_packages = all_cookbooks.inject([]) do |acc, (cookbook_name, cookbook_versions)|
+        acc << dep_graph.package(cookbook_name)
+        acc
+      end
+
+      # find a valid assignment of CoookbookVersions. If no valid
+      # assignment exists, indicate which run_list_item causes the
+      # unsatisfiability and try to hint at what might be wrong.
+      soln =
+        begin
+          DepSelector::Selector.new(dep_graph).find_solution(cookbook_constraints, all_packages)
+        rescue DepSelector::Exceptions::InvalidSolutionConstraints => e
+          non_existent_cookbooks = e.non_existent_packages.map {|constraint| constraint.package.name}
+          cookbooks_with_no_matching_versions = e.constrained_to_no_versions.map {|constraint| constraint.package.name}
+
+          # Spend a whole lot of effort for pluralizing and
+          # prettifying the message.
+          message = ""
+          if non_existent_cookbooks.length > 0
+            message += "no such " + (non_existent_cookbooks.length > 1 ? "cookbooks" : "cookbook")
+            message += " #{non_existent_cookbooks.join(", ")}"
+          end
+
+          if cookbooks_with_no_matching_versions.length > 0
+            if message.length > 0
+              message += "; "
+            end
+
+            message += "no versions match the constraints on " + (cookbooks_with_no_matching_versions.length > 1 ? "cookbooks" : "cookbook")
+            message += " #{cookbooks_with_no_matching_versions.join(", ")}"
+          end
+
+          message = "Run list contains invalid items: #{message}."
+
+          raise Chef::Exceptions::CookbookVersionSelection::InvalidRunListItems.new(message, non_existent_cookbooks, cookbooks_with_no_matching_versions)
+        rescue DepSelector::Exceptions::NoSolutionExists => e
+          raise Chef::Exceptions::CookbookVersionSelection::UnsatisfiableRunListItem.new(filter_dep_selector_message(e.message), e.unsatisfiable_solution_constraint, e.disabled_non_existent_packages, e.disabled_most_constrained_packages)
+        end
+
+
+      # map assignment back to CookbookVersion objects
+      selected_cookbooks = {}
+      soln.each_pair do |cb_name, cb_version|
+        # TODO [cw, 2011/2/10]: related to the TODO in
+        # create_dependency_graph_from_cookbooks, cbv.version
+        # currently returns a String, so we must compare to
+        # cb_version.to_s, since it's a for-real Version object.
+        selected_cookbooks[cb_name] = all_cookbooks[cb_name].find{|cbv| cbv.version == cb_version.to_s}
+      end
+      selected_cookbooks
+    end
+
+    # Expands the run_list, constrained to the environment's CookbookVersion
+    # constraints.
+    #
+    # Returns:
+    #   Hash of: name to CookbookVersion
+    def self.expand_to_cookbook_versions(run_list, environment, couchdb=nil)
+      # expand any roles in this run_list.
+      expanded_run_list = run_list.expand(environment, 'couchdb', :couchdb => couchdb).recipes.with_version_constraints
+
+      cookbooks_for_environment = Chef::Environment.cdb_load_filtered_cookbook_versions(environment, couchdb)
+      constrain(cookbooks_for_environment, expanded_run_list)
+    end
+  end
+end