chef 10.34.6 → 11.0.0.beta.0

data/lib/chef/cookbook_loader.rb

@@ -25,31 +25,36 @@ require 'chef/cookbook_version'
 require 'chef/cookbook/chefignore'
 require 'chef/cookbook/metadata'
 
+#
+# CookbookLoader class loads the cookbooks lazily as read
+#
 class Chef
   class CookbookLoader
 
-    attr_accessor :metadata
     attr_reader :cookbooks_by_name
     attr_reader :merged_cookbooks
     attr_reader :cookbook_paths
+    attr_reader :metadata
 
     include Enumerable
 
     def initialize(*repo_paths)
-      @repo_paths = repo_paths.flatten
-      raise ArgumentError, "You must specify at least one cookbook repo path" if @repo_paths.empty?
+      repo_paths = repo_paths.flatten
+      raise ArgumentError, "You must specify at least one cookbook repo path" if repo_paths.empty?
       @cookbooks_by_name = Mash.new
       @loaded_cookbooks = {}
       @metadata = Mash.new
       @cookbooks_paths = Hash.new {|h,k| h[k] = []} # for deprecation warnings
+      @chefignores = {}
+      @repo_paths = repo_paths.map do |repo_path|
+        repo_path = File.expand_path(repo_path)
+      end
 
       # Used to track which cookbooks appear in multiple places in the cookbook repos
       # and are merged in to a single cookbook by file shadowing. This behavior is
       # deprecated, so users of this class may issue warnings to the user by checking
       # this variable
       @merged_cookbooks = []
-
-      load_cookbooks
     end
 
     def merged_cookbook_paths # for deprecation warnings
@@ -59,35 +64,43 @@ class Chef
     end
 
     def load_cookbooks
-      cookbook_settings = Hash.new
       @repo_paths.each do |repo_path|
-        repo_path = File.expand_path(repo_path)
-        chefignore = Cookbook::Chefignore.new(repo_path)
         Dir[File.join(repo_path, "*")].each do |cookbook_path|
-          next unless File.directory?(cookbook_path)
-          loader = Cookbook::CookbookVersionLoader.new(cookbook_path, chefignore)
-          loader.load_cookbooks
-          next if loader.empty?
-          @cookbooks_paths[loader.cookbook_name] << cookbook_path # for deprecation warnings
-          if @loaded_cookbooks.key?(loader.cookbook_name)
-            @merged_cookbooks << loader.cookbook_name # for deprecation warnings
-            @loaded_cookbooks[loader.cookbook_name].merge!(loader)
-          else
-            @loaded_cookbooks[loader.cookbook_name] = loader
-          end
+          load_cookbook(File.basename(cookbook_path), [repo_path])
+        end
+      end
+      @cookbooks_by_name
+    end
+
+    def load_cookbook(cookbook_name, repo_paths=nil)
+      repo_paths ||= @repo_paths
+      repo_paths.each do |repo_path|
+        @chefignores[repo_path] ||= Cookbook::Chefignore.new(repo_path)
+        cookbook_path = File.join(repo_path, cookbook_name.to_s)
+        next unless File.directory?(cookbook_path) and Dir[File.join(repo_path, "*")].include?(cookbook_path)
+        loader = Cookbook::CookbookVersionLoader.new(cookbook_path, @chefignores[repo_path])
+        loader.load_cookbooks
+        next if loader.empty?
+        cookbook_name = loader.cookbook_name
+        @cookbooks_paths[cookbook_name] << cookbook_path # for deprecation warnings
+        if @loaded_cookbooks.key?(cookbook_name)
+          @merged_cookbooks << cookbook_name # for deprecation warnings
+          @loaded_cookbooks[cookbook_name].merge!(loader)
+        else
+          @loaded_cookbooks[cookbook_name] = loader
         end
       end
 
-      @loaded_cookbooks.each do |cookbook, loader|
-        cookbook_version = loader.cookbook_version
-        @cookbooks_by_name[cookbook] = cookbook_version
-        @metadata[cookbook] = cookbook_version.metadata
+      if @loaded_cookbooks.has_key?(cookbook_name)
+        cookbook_version = @loaded_cookbooks[cookbook_name].cookbook_version
+        @cookbooks_by_name[cookbook_name] = cookbook_version
+        @metadata[cookbook_name] = cookbook_version.metadata
       end
-      @cookbooks_by_name
+      @cookbooks_by_name[cookbook_name]
     end
 
     def [](cookbook)
-      if @cookbooks_by_name.has_key?(cookbook.to_sym)
+      if @cookbooks_by_name.has_key?(cookbook.to_sym) or load_cookbook(cookbook.to_sym)
         @cookbooks_by_name[cookbook.to_sym]
       else
         raise Exceptions::CookbookNotFoundInRepo, "Cannot find a cookbook named #{cookbook.to_s}; did you forget to add metadata to a cookbook? (http://wiki.opscode.com/display/chef/Metadata)"
@@ -97,7 +110,7 @@ class Chef
     alias :fetch :[]
 
     def has_key?(cookbook_name)
-      @cookbooks_by_name.has_key?(cookbook_name)
+      not self[cookbook_name.to_sym].nil?
     end
     alias :cookbook_exists? :has_key?
     alias :key? :has_key?

data/lib/chef/cookbook_uploader.rb

@@ -3,11 +3,11 @@ require 'set'
 require 'rest_client'
 require 'chef/exceptions'
 require 'chef/knife/cookbook_metadata'
-require 'chef/checksum_cache'
-require 'chef/sandbox'
+require 'chef/digester'
 require 'chef/cookbook_version'
 require 'chef/cookbook/syntax_check'
 require 'chef/cookbook/file_system_file_vendor'
+require 'chef/sandbox'
 
 class Chef
   class CookbookUploader
@@ -110,16 +110,7 @@ class Chef
       # files are uploaded, so save the manifest
       cookbooks.each do |cb|
         save_url = opts[:force] ? cb.force_save_url : cb.save_url
-        begin
-          rest.put_rest(save_url, cb)
-        rescue Net::HTTPServerException => e
-          case e.response.code
-          when "409"
-            raise Chef::Exceptions::CookbookFrozen, "Version #{cb.version} of cookbook #{cb.name} is frozen. Use --force to override."
-          else
-            raise
-          end
-        end
+        rest.put_rest(save_url, cb)
       end
 
       Chef::Log.info("Upload complete!")
@@ -134,17 +125,24 @@ class Chef
         # but we need the base64 encoding for the content-md5
         # header
         checksum64 = Base64.encode64([checksum].pack("H*")).strip
+        timestamp = Time.now.utc.iso8601
         file_contents = File.open(file, "rb") {|f| f.read}
-        headers = { 'content-type' => 'application/x-binary', 'content-md5' => checksum64, 'accept' => 'application/json' }
+        # TODO - 5/28/2010, cw: make signing and sending the request streaming
+        sign_obj = Mixlib::Authentication::SignedHeaderAuth.signing_object(
+                                                                           :http_method => :put,
+                                                                           :path        => URI.parse(url).path,
+                                                                           :body        => file_contents,
+                                                                           :timestamp   => timestamp,
+                                                                           :user_id     => rest.client_name
+                                                                           )
+        headers = { 'content-type' => 'application/x-binary', 'content-md5' => checksum64, :accept => 'application/json' }
+        headers.merge!(sign_obj.sign(OpenSSL::PKey::RSA.new(rest.signing_key)))
 
         begin
-          url = rest.create_url(url)
-          rest.raw_http_request(:PUT, url, headers, file_contents)
+          RestClient::Resource.new(url, :headers=>headers, :timeout=>1800, :open_timeout=>1800).put(file_contents)
           checksums_to_upload.delete(checksum)
-        rescue Net::HTTPServerException, Net::HTTPFatalError, Errno::ECONNREFUSED, Timeout::Error, Errno::ETIMEDOUT, SocketError => e
-          error_message = "Failed to upload #{file} (#{checksum}) to #{url} : #{e.message}"
-          error_message << "\n#{e.response.body}" if e.respond_to?(:response)
-          Chef::Knife.ui.error(error_message)
+        rescue RestClient::Exception => e
+          Chef::Knife.ui.error("Failed to upload #@cookbook : #{e.message}\n#{e.response.body}")
           raise
         end
       end

data/lib/chef/cookbook_version.rb

@@ -24,86 +24,11 @@ require 'chef/node'
 require 'chef/resource_definition_list'
 require 'chef/recipe'
 require 'chef/cookbook/file_vendor'
-require 'chef/checksum'
 require 'chef/cookbook/metadata'
 require 'chef/version_class'
 
 class Chef
 
-  #== Chef::MinimalCookbookVersion
-  # MinimalCookbookVersion is a duck type of CookbookVersion, used
-  # internally by Chef Server as an optimization when determining the
-  # optimal cookbook set for a chef-client.
-  #
-  # MinimalCookbookVersion objects contain only enough information to
-  # solve the cookbook collection for a given run list. They *do not*
-  # contain enough information to generate the response.
-  #
-  # See also: Chef::CookbookVersionSelector
-  class MinimalCookbookVersion
-
-    include Comparable
-
-    ID   = "id".freeze
-    NAME = 'name'.freeze
-    KEY  = 'key'.freeze
-    VERSION = 'version'.freeze
-    VALUE   = 'value'.freeze
-    DEPS    = 'deps'.freeze
-
-    DEPENDENCIES      = 'dependencies'.freeze
-
-    # Loads the full list of cookbooks, using a couchdb view to fetch
-    # only the id, name, version, and dependency constraints. This is
-    # enough information to solve for the cookbook collection for a
-    # given run list. After solving for the cookbook collection, you
-    # need to call +load_full_versions_of+ to convert
-    # MinimalCookbookVersion objects to their non-minimal counterparts
-    def self.load_all(couchdb)
-      # Example:
-      # {"id"=>"1a806f1c-b409-4d8e-abab-fa414ff5b96d", "key"=>"activemq", "value"=>{"version"=>"0.3.3", "deps"=>{"java"=>">= 0.0.0", "runit"=>">= 0.0.0"}}}
-      couchdb ||= Chef::CouchDB.new
-      couchdb.get_view("cookbooks", "all_with_version_and_deps")["rows"].map {|params| self.new(params) }
-    end
-
-    # Loads the non-minimal CookbookVersion objects corresponding to
-    # +minimal_cookbook_versions+ from couchdb using a bulk GET.
-    def self.load_full_versions_of(minimal_cookbook_versions, couchdb)
-      database_ids = Array(minimal_cookbook_versions).map {|mcv| mcv.couchdb_id }
-      couchdb ||= Chef::CouchDB.new
-      couchdb.bulk_get(*database_ids)
-    end
-
-    attr_reader :couchdb_id
-    attr_reader :name
-    attr_reader :version
-    attr_reader :deps
-
-    def initialize(params)
-      @couchdb_id = params[ID]
-      @name = params[KEY]
-      @version = params[VALUE][VERSION]
-      @deps    = params[VALUE][DEPS]
-    end
-
-    # Returns the Cookbook::MinimalMetadata object for this cookbook
-    # version.
-    def metadata
-      @metadata ||= Cookbook::MinimalMetadata.new(@name, DEPENDENCIES => @deps)
-    end
-
-    def legit_version
-      @legit_version ||= Chef::Version.new(@version)
-    end
-
-    def <=>(o)
-      raise Chef::Exceptions::CookbookVersionNameMismatch if self.name != o.name
-      raise "Unexpected comparison to #{o}" unless o.respond_to?(:legit_version)
-      legit_version <=> o.legit_version
-    end
-  end
-
-
   # == Chef::CookbookVersion
   # CookbookVersion is a model object encapsulating the data about a Chef
   # cookbook. Chef supports maintaining multiple versions of a cookbook on a
@@ -113,140 +38,10 @@ class Chef
   # TODO: timh/cw: 5-24-2010: mutators for files (e.g., recipe_filenames=,
   # recipe_filenames.insert) should dirty the manifest so it gets regenerated.
   class CookbookVersion
-    include Chef::IndexQueue::Indexable
     include Comparable
 
     COOKBOOK_SEGMENTS = [ :resources, :providers, :recipes, :definitions, :libraries, :attributes, :files, :templates, :root_files ]
 
-    DESIGN_DOCUMENT = {
-      "version" => 8,
-      "language" => "javascript",
-      "views" => {
-        "all" => {
-          "map" => <<-EOJS
-          function(doc) {
-            if (doc.chef_type == "cookbook_version") {
-              emit(doc.name, doc);
-            }
-          }
-          EOJS
-        },
-        "all_id" => {
-          "map" => <<-EOJS
-          function(doc) {
-            if (doc.chef_type == "cookbook_version") {
-              emit(doc.name, doc.name);
-            }
-          }
-          EOJS
-        },
-        "all_with_version" => {
-          "map" => <<-EOJS
-          function(doc) {
-            if (doc.chef_type == "cookbook_version") {
-              emit(doc.cookbook_name, doc.version);
-            }
-          }
-          EOJS
-        },
-        "all_with_version_and_deps" => {
-          "map" => <<-JS
-          function(doc) {
-            if (doc.chef_type == "cookbook_version") {
-              emit(doc.cookbook_name, {version: doc.version, deps: doc.metadata.dependencies});
-            }
-          }
-          JS
-        },
-        "all_latest_version" => {
-          "map" => %q@
-          function(doc) {
-            if (doc.chef_type == "cookbook_version") {
-              emit(doc.cookbook_name, doc.version);
-            }
-          }
-          @,
-          "reduce" => %q@
-          function(keys, values, rereduce) {
-            var result = null;
-
-            for (var idx in values) {
-              var value = values[idx];
-
-              if (idx == 0) {
-                result = value;
-                continue;
-              }
-
-              var valueParts = value.split('.').map(function(v) { return parseInt(v); });
-              var resultParts = result.split('.').map(function(v) { return parseInt(v); });
-
-              if (valueParts[0] != resultParts[0]) {
-                if (valueParts[0] > resultParts[0]) {
-                  result = value;
-                }
-              }
-              else if (valueParts[1] != resultParts[1]) {
-                if (valueParts[1] > resultParts[1]) {
-                  result = value;
-                }
-              }
-              else if (valueParts[2] != resultParts[2]) {
-                if (valueParts[2] > resultParts[2]) {
-                  result = value;
-                }
-              }
-            }
-            return result;
-          }
-          @
-        },
-        "all_latest_version_by_id" => {
-          "map" => %q@
-          function(doc) {
-            if (doc.chef_type == "cookbook_version") {
-              emit(doc.cookbook_name, {version: doc.version, id:doc._id});
-            }
-          }
-          @,
-          "reduce" => %q@
-          function(keys, values, rereduce) {
-            var result = null;
-
-            for (var idx in values) {
-              var value = values[idx];
-
-              if (idx == 0) {
-                result = value;
-                continue;
-              }
-
-              var valueParts = value.version.split('.').map(function(v) { return parseInt(v); });
-              var resultParts = result.version.split('.').map(function(v) { return parseInt(v); });
-
-              if (valueParts[0] != resultParts[0]) {
-                if (valueParts[0] > resultParts[0]) {
-                  result = value;
-                }
-              }
-              else if (valueParts[1] != resultParts[1]) {
-                if (valueParts[1] > resultParts[1]) {
-                  result = value;
-                }
-              }
-              else if (valueParts[2] != resultParts[2]) {
-                if (valueParts[2] > resultParts[2]) {
-                  result = value;
-                }
-              }
-            }
-            return result;
-          }
-          @
-        },
-      }
-    }
-
     attr_accessor :root_dir
     attr_accessor :definition_filenames
     attr_accessor :template_filenames
@@ -259,10 +54,6 @@ class Chef
     attr_accessor :metadata
     attr_accessor :metadata_filenames
     attr_accessor :status
-    attr_accessor :couchdb_rev
-    attr_accessor :couchdb
-
-    attr_reader :couchdb_id
 
     # attribute_filenames also has a setter that has non-default
     # functionality.
@@ -278,7 +69,7 @@ class Chef
     # This is the one and only method that knows how cookbook files'
     # checksums are generated.
     def self.checksum_cookbook_file(filepath)
-      Chef::ChecksumCache.generate_md5_checksum_for_file(filepath)
+      Chef::Digester.generate_md5_checksum_for_file(filepath)
     rescue Errno::ENOENT
       Chef::Log.debug("File #{filepath} does not exist, so there is no checksum to generate")
       nil
@@ -401,7 +192,7 @@ class Chef
     #
     # === Returns
     # object<Chef::CookbookVersion>:: Duh. :)
-    def initialize(name, couchdb=nil)
+    def initialize(name)
       @name = name
       @frozen = false
       @attribute_filenames = Array.new
@@ -416,9 +207,6 @@ class Chef
       @metadata_filenames = Array.new
       @root_dir = nil
       @root_filenames = Array.new
-      @couchdb_id = nil
-      @couchdb = couchdb || Chef::CouchDB.new
-      @couchdb_rev = nil
       @status = :ready
       @manifest = nil
       @file_vendor = nil
@@ -761,7 +549,6 @@ class Chef
       result = manifest.dup
       result['frozen?'] = frozen_version?
       result['chef_type'] = 'cookbook_version'
-      result["_rev"] = couchdb_rev if couchdb_rev
       result.to_hash
     end
 
@@ -773,21 +560,12 @@ class Chef
 
     def self.json_create(o)
       cookbook_version = new(o["cookbook_name"])
-      if o.has_key?('_rev')
-        cookbook_version.couchdb_rev = o["_rev"] if o.has_key?("_rev")
-        o.delete("_rev")
-      end
-      if o.has_key?("_id")
-        cookbook_version.couchdb_id = o["_id"] if o.has_key?("_id")
-        cookbook_version.index_id = cookbook_version.couchdb_id
-        o.delete("_id")
-      end
       # We want the Chef::Cookbook::Metadata object to always be inflated
       cookbook_version.metadata = Chef::Cookbook::Metadata.from_hash(o["metadata"])
       cookbook_version.manifest = o
 
       # We don't need the following step when we decide to stop supporting deprecated operators in the metadata (e.g. <<, >>)
-      cookbook_version.manifest["metadata"] = Chef::JSONCompat.from_json(cookbook_version.metadata.to_json)
+      cookbook_version.manifest["metadata"] = JSON.parse(cookbook_version.metadata.to_json)
 
       cookbook_version.freeze_version if o["frozen?"]
       cookbook_version
@@ -888,83 +666,6 @@ class Chef
       chef_server_rest.get_rest('cookbooks/_latest')
     end
 
-    ##
-    # Couchdb
-    ##
-
-    def self.cdb_by_name(cookbook_name, couchdb=nil)
-      cdb = (couchdb || Chef::CouchDB.new)
-      options = { :startkey => cookbook_name, :endkey => cookbook_name }
-      rs = cdb.get_view("cookbooks", "all_with_version", options)
-      rs["rows"].inject({}) { |memo, row| memo.has_key?(row["key"]) ? memo[row["key"]] << row["value"] : memo[row["key"]] = [ row["value"] ]; memo }
-    end
-
-    def self.create_design_document(couchdb=nil)
-      (couchdb || Chef::CouchDB.new).create_design_document("cookbooks", DESIGN_DOCUMENT)
-    end
-
-    def self.cdb_list_latest(inflate=false, couchdb=nil)
-      couchdb ||= Chef::CouchDB.new
-      if inflate
-        doc_ids = cdb_list_latest_ids.map {|i|i["id"]}
-        couchdb.bulk_get(doc_ids)
-      else
-        results = couchdb.get_view("cookbooks", "all_latest_version", :group=>true)["rows"]
-        results.inject({}) { |mapped, row| mapped[row["key"]] = row["value"]; mapped}
-      end
-    end
-
-    def self.cdb_list_latest_ids(inflate=false, couchdb=nil)
-      couchdb ||= Chef::CouchDB.new
-      results = couchdb.get_view("cookbooks", "all_latest_version_by_id", :group=>true)["rows"]
-      results.map { |name_and_id| name_and_id["value"]}
-    end
-
-    def self.cdb_list(inflate=false, couchdb=nil)
-      couchdb ||= Chef::CouchDB.new
-      if inflate
-        couchdb.list("cookbooks", true)["rows"].collect{|r| r["value"]}
-      else
-        # If you modify this, please make sure the desc sorted order on the versions doesn't get broken.
-        couchdb.get_view("cookbooks", "all_with_version")["rows"].inject({}) { |mapped, row| mapped[row["key"]]||=Array.new; mapped[row["key"]].push(Chef::Version.new(row["value"])); mapped[row["key"]].sort!.reverse!; mapped}
-      end
-    end
-
-    def self.cdb_load(name, version='latest', couchdb=nil)
-      cdb = couchdb || Chef::CouchDB.new
-      if version == "latest" || version == "_latest"
-        rs = cdb.get_view("cookbooks", "all_latest_version", :key => name, :descending => true, :group => true, :reduce => true)["rows"].first
-        cdb.load("cookbook_version", "#{rs["key"]}-#{rs["value"]}")
-      else
-        cdb.load("cookbook_version", "#{name}-#{version}")
-      end
-    end
-
-    def cdb_destroy
-      (couchdb || Chef::CouchDB.new).delete("cookbook_version", full_name, couchdb_rev)
-    end
-
-    # Runs on Chef Server (API); deletes the cookbook from couchdb and also destroys associated
-    # checksum documents
-    def purge
-      checksums.keys.each do |checksum|
-        begin
-          Chef::Checksum.cdb_load(checksum, couchdb).purge
-        rescue Chef::Exceptions::CouchDBNotFound
-        end
-      end
-      cdb_destroy
-    end
-
-    def cdb_save
-      @couchdb_rev = couchdb.store("cookbook_version", full_name, self)["rev"]
-    end
-
-    def couchdb_id=(value)
-      @couchdb_id = value
-      @index_id = value
-    end
-
     def <=>(o)
       raise Chef::Exceptions::CookbookVersionNameMismatch if self.name != o.name
       # FIXME: can we change the interface to the Metadata class such