chef 0.9.18 → 0.10.0.beta.0

data/lib/chef/cookbook_loader.rb

@@ -9,9 +9,9 @@
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -19,144 +19,71 @@
 # limitations under the License.
 
 require 'chef/config'
+require 'chef/exceptions'
+require 'chef/cookbook/cookbook_version_loader'
 require 'chef/cookbook_version'
+require 'chef/cookbook/chefignore'
 require 'chef/cookbook/metadata'
 
 class Chef
   class CookbookLoader
-    
-    attr_accessor :cookbook, :metadata
-    
+
+    attr_accessor :metadata
+    attr_reader :cookbooks_by_name
+
     include Enumerable
-    
-    def initialize()
+
+    def initialize(*repo_paths)
+      @repo_paths = repo_paths.flatten
+      raise ArgumentError, "You must specify at least one cookbook repo path" if @repo_paths.empty?
       @cookbooks_by_name = Mash.new
-      @metadata = Hash.new
-      @ignore_regexes = Hash.new { |hsh, key| hsh[key] = Array.new }
+      @loaded_cookbooks = {}
+      @metadata = Mash.new
       load_cookbooks
     end
-    
+
     def load_cookbooks
       cookbook_settings = Hash.new
-      [Chef::Config.cookbook_path].flatten.each do |cb_path|
-        cb_path = File.expand_path(cb_path)
-        Dir[File.join(cb_path, "*")].each do |cookbook|
-          next unless File.directory?(cookbook)
-          cookbook_name = File.basename(cookbook).to_sym
-          unless cookbook_settings.has_key?(cookbook_name)
-            cookbook_settings[cookbook_name] = { 
-              :attribute_filenames  => Hash.new,
-              :definition_filenames => Hash.new,
-              :recipe_filenames     => Hash.new,
-              :template_filenames   => Hash.new,
-              :file_filenames       => Hash.new,
-              :library_filenames    => Hash.new,
-              :resource_filenames   => Hash.new,
-              :provider_filenames   => Hash.new,
-              :root_filenames       => Hash.new,
-              :metadata_filenames   => Array.new
-            }
-          end
-          ignore_regexes = load_ignore_file(File.join(cookbook, "ignore"))
-          @ignore_regexes[cookbook_name].concat(ignore_regexes)
-          
-          load_files_unless_basename(
-            File.join(cookbook, "attributes", "*.rb"), 
-            cookbook_settings[cookbook_name][:attribute_filenames]
-          )
-          load_files_unless_basename(
-            File.join(cookbook, "definitions", "*.rb"), 
-            cookbook_settings[cookbook_name][:definition_filenames]
-          )
-          load_files_unless_basename(
-            File.join(cookbook, "recipes", "*.rb"), 
-            cookbook_settings[cookbook_name][:recipe_filenames]
-          )
-          load_files_unless_basename(
-            File.join(cookbook, "libraries", "*.rb"),               
-            cookbook_settings[cookbook_name][:library_filenames]
-          )
-          load_cascading_files(
-            "*",
-            File.join(cookbook, "templates"),
-            cookbook_settings[cookbook_name][:template_filenames]
-          )
-          load_cascading_files(
-            "*",
-            File.join(cookbook, "files"),
-            cookbook_settings[cookbook_name][:file_filenames]
-          )
-          load_cascading_files(
-            "*.rb",
-            File.join(cookbook, "resources"),
-            cookbook_settings[cookbook_name][:resource_filenames]
-          )
-          load_cascading_files(
-            "*.rb",
-            File.join(cookbook, "providers"),
-            cookbook_settings[cookbook_name][:provider_filenames]
-          )
-          load_files(
-            "*",
-            cookbook,
-            cookbook_settings[cookbook_name][:root_filenames]
-          )
-          cookbook_settings[cookbook_name][:root_dir] = cookbook
-          if File.exists?(File.join(cookbook, "metadata.json"))
-            cookbook_settings[cookbook_name][:metadata_filenames] << File.join(cookbook, "metadata.json")
-          end
-
-          empty = cookbook_settings[cookbook_name].inject(true) do |all_empty, files|
-            all_empty && files.last.empty?
-          end
-
-          if empty
-            Chef::Log.warn "found a directory #{cookbook_name} in the cookbook path, but it contains no cookbook files. skipping."
-            cookbook_settings.delete(cookbook_name)
+      @repo_paths.each do |repo_path|
+        repo_path = File.expand_path(repo_path)
+        chefignore = Cookbook::Chefignore.new(repo_path)
+        Dir[File.join(repo_path, "*")].each do |cookbook_path|
+          next unless File.directory?(cookbook_path)
+          loader = Cookbook::CookbookVersionLoader.new(cookbook_path, chefignore)
+          loader.load_cookbooks
+          next if loader.empty?
+          if @loaded_cookbooks.key?(loader.cookbook_name)
+            @loaded_cookbooks[loader.cookbook_name].merge!(loader)
+          else
+            @loaded_cookbooks[loader.cookbook_name] = loader
           end
         end
       end
-      remove_ignored_files_from(cookbook_settings)
 
-      cookbook_settings.each_key do |cookbook|
-        @cookbooks_by_name[cookbook] = Chef::CookbookVersion.new(cookbook)
-        @cookbooks_by_name[cookbook].root_dir = cookbook_settings[cookbook][:root_dir]
-        @cookbooks_by_name[cookbook].attribute_filenames = cookbook_settings[cookbook][:attribute_filenames].values
-        @cookbooks_by_name[cookbook].definition_filenames = cookbook_settings[cookbook][:definition_filenames].values
-        @cookbooks_by_name[cookbook].recipe_filenames = cookbook_settings[cookbook][:recipe_filenames].values
-        @cookbooks_by_name[cookbook].template_filenames = cookbook_settings[cookbook][:template_filenames].values
-        @cookbooks_by_name[cookbook].file_filenames = cookbook_settings[cookbook][:file_filenames].values
-        @cookbooks_by_name[cookbook].library_filenames = cookbook_settings[cookbook][:library_filenames].values
-        @cookbooks_by_name[cookbook].resource_filenames = cookbook_settings[cookbook][:resource_filenames].values
-        @cookbooks_by_name[cookbook].provider_filenames = cookbook_settings[cookbook][:provider_filenames].values
-        @cookbooks_by_name[cookbook].root_filenames = cookbook_settings[cookbook][:root_filenames].values
-        @cookbooks_by_name[cookbook].metadata_filenames = cookbook_settings[cookbook][:metadata_filenames]
-        @metadata[cookbook] = Chef::Cookbook::Metadata.new(@cookbooks_by_name[cookbook])
-        cookbook_settings[cookbook][:metadata_filenames].each do |meta_json|
-          begin
-            @metadata[cookbook].from_json(IO.read(meta_json))
-          rescue JSON::ParserError
-            Chef::Log.fatal("Couldn't parse JSON in " + meta_json)
-            raise
-          end
-        end
-        @cookbooks_by_name[cookbook].metadata = @metadata[cookbook]
+      @loaded_cookbooks.each do |cookbook, loader|
+        cookbook_version = loader.cookbook_version
+        @cookbooks_by_name[cookbook] = cookbook_version
+        @metadata[cookbook] = cookbook_version.metadata
       end
+      @cookbooks_by_name
     end
-    
+
     def [](cookbook)
       if @cookbooks_by_name.has_key?(cookbook.to_sym)
         @cookbooks_by_name[cookbook.to_sym]
       else
-        raise ArgumentError, "Cannot find a cookbook named #{cookbook.to_s}; did you forget to add metadata to a cookbook? (http://wiki.opscode.com/display/chef/Metadata)"
+        raise Exceptions::CookbookNotFoundInRepo, "Cannot find a cookbook named #{cookbook.to_s}; did you forget to add metadata to a cookbook? (http://wiki.opscode.com/display/chef/Metadata)"
       end
     end
 
+    alias :fetch :[]
+
     def has_key?(cookbook_name)
       @cookbooks_by_name.has_key?(cookbook_name)
     end
     alias :cookbook_exists? :has_key?
-    
+    alias :key? :has_key?
+
     def each
       @cookbooks_by_name.keys.sort { |a,b| a.to_s <=> b.to_s }.each do |cname|
         yield(cname, @cookbooks_by_name[cname])
@@ -172,56 +99,5 @@ class Chef
     end
     alias :cookbooks :values
 
-    private
-    
-      def load_ignore_file(ignore_file)
-        results = Array.new
-        if File.exists?(ignore_file) && File.readable?(ignore_file)
-          IO.foreach(ignore_file) do |line|
-            next if line =~ /^#/
-            next if line =~ /^\w*$/
-            line.chomp!
-            results << Regexp.new(line)
-          end
-        end
-        results
-      end
-      
-      def remove_ignored_files_from(cookbook_settings)
-        file_types_to_inspect = [ :attribute_filenames, :definition_filenames, :recipe_filenames, :template_filenames, 
-                                  :file_filenames, :library_filenames, :resource_filenames, :provider_filenames]
-        
-        @ignore_regexes.each do |cookbook_name, regexes|
-          regexes.each do |regex|
-            settings = cookbook_settings[cookbook_name]
-            file_types_to_inspect.each do |file_type|
-              settings[file_type].delete_if { |uniqname, fullpath| fullpath.match(regex) }
-            end
-          end
-        end
-      end
-
-      def load_files(file_glob, base_path, result_hash, recursive=false)
-        rm_base_path = /^#{Regexp.escape(base_path)}\/(.+)$/
-        file_spec = [base_path]
-        file_spec << "**" if recursive
-        file_spec << file_glob
-        # To handle dotfiles like .ssh
-        Dir.glob(File.join(file_spec), File::FNM_DOTMATCH).each do |file|
-          next if File.directory?(file)
-          result_hash[rm_base_path.match(file)[1]] = file
-        end
-      end
-      
-      def load_cascading_files(file_glob, base_path, result_hash)
-        load_files(file_glob, base_path, result_hash, true)
-      end
-      
-      def load_files_unless_basename(file_glob, result_hash)
-        Dir[file_glob].each do |file|
-          result_hash[File.basename(file)] = file
-        end
-      end
-      
   end
 end

data/lib/chef/cookbook_uploader.rb

@@ -1,5 +1,6 @@
 require 'rest_client'
-require 'chef/cookbook_loader'
+require 'chef/exceptions'
+require 'chef/knife/cookbook_metadata'
 require 'chef/checksum_cache'
 require 'chef/sandbox'
 require 'chef/cookbook_version'
@@ -8,96 +9,117 @@ require 'chef/cookbook/file_system_file_vendor'
 
 class Chef
   class CookbookUploader
-    class << self
 
-      def upload_cookbook(cookbook)
-        Chef::Log.info("Saving #{cookbook.name}")
+    attr_reader :cookbook
+    attr_reader :path
+    attr_reader :opts
+    attr_reader :rest
 
-        rest = Chef::REST.new(Chef::Config[:chef_server_url])
+    # Creates a new CookbookUploader.
+    # ===Arguments:
+    # * cookbook::: A Chef::CookbookVersion describing the cookbook to be uploaded
+    # * path::: A String or Array of Strings representing the base paths to the
+    #           cookbook repositories.
+    # * opts::: (optional) An options Hash
+    # ===Options:
+    # * :force  indicates that the uploader should set the force option when
+    #           uploading the cookbook. This allows frozen CookbookVersion
+    #           documents on the server to be overwritten (otherwise a 409 is
+    #           returned by the server)
+    # * :rest   A Chef::REST object that you have configured the way you like it.
+    #           If you don't provide this, one will be created using the values
+    #           in Chef::Config.
+    def initialize(cookbook, path, opts={})
+      @cookbook, @path, @opts = cookbook, path, opts
+      @rest = opts[:rest] || Chef::REST.new(Chef::Config[:chef_server_url])
+    end
 
-        # Syntax Check
-        validate_cookbook(cookbook)
-        # Generate metadata.json from metadata.rb
-        build_metadata(cookbook)
+    def upload_cookbook
+      Chef::Log.info("Saving #{cookbook.name}")
 
-        # generate checksums of cookbook files and create a sandbox
-        checksum_files = cookbook.checksums
-        checksums = checksum_files.inject({}){|memo,elt| memo[elt.first]=nil ; memo}
-        new_sandbox = rest.post_rest("sandboxes", { :checksums => checksums })
+      # Syntax Check
+      validate_cookbook
+      # Generate metadata.json from metadata.rb
+      build_metadata
 
-        Chef::Log.info("Uploading files")
-        # upload the new checksums and commit the sandbox
-        new_sandbox['checksums'].each do |checksum, info|
-          if info['needs_upload'] == true
-            Chef::Log.info("Uploading #{checksum_files[checksum]} (checksum hex = #{checksum}) to #{info['url']}")
+      # generate checksums of cookbook files and create a sandbox
+      checksum_files = cookbook.checksums
+      checksums = checksum_files.inject({}){|memo,elt| memo[elt.first]=nil ; memo}
+      new_sandbox = rest.post_rest("sandboxes", { :checksums => checksums })
 
-            # Checksum is the hexadecimal representation of the md5,
-            # but we need the base64 encoding for the content-md5
-            # header
-            checksum64 = Base64.encode64([checksum].pack("H*")).strip
-            timestamp = Time.now.utc.iso8601
-            file_contents = File.read(checksum_files[checksum])
-            # TODO - 5/28/2010, cw: make signing and sending the request streaming
-            sign_obj = Mixlib::Authentication::SignedHeaderAuth.signing_object(
-                                                                               :http_method => :put,
-                                                                               :path        => URI.parse(info['url']).path,
-                                                                               :body        => file_contents,
-                                                                               :timestamp   => timestamp,
-                                                                               :user_id     => rest.client_name
-                                                                               )
-            headers = { 'content-type' => 'application/x-binary', 'content-md5' => checksum64, :accept => 'application/json' }
-            headers.merge!(sign_obj.sign(OpenSSL::PKey::RSA.new(rest.signing_key)))
-            begin
-              RestClient::Resource.new(info['url'], :headers=>headers, :timeout=>1800, :open_timeout=>1800).put(file_contents)
-            rescue RestClient::Exception => e
-              Chef::Log.error("Upload failed: #{e.message}\n#{e.response.body}")
-              raise
-            end
-          else
-            Chef::Log.debug("#{checksum_files[checksum]} has not changed")
-          end
-        end
-        sandbox_url = new_sandbox['uri']
-        Chef::Log.debug("Committing sandbox")
-        # Retry if S3 is claims a checksum doesn't exist (the eventual
-        # in eventual consistency)
-        retries = 0
-        begin
-          rest.put_rest(sandbox_url, {:is_completed => true})
-        rescue Net::HTTPServerException => e
-          if e.message =~ /^400/ && (retries += 1) <= 5
-            sleep 2
-            retry
-          else
+      Chef::Log.info("Uploading files")
+      # upload the new checksums and commit the sandbox
+      new_sandbox['checksums'].each do |checksum, info|
+        if info['needs_upload'] == true
+          Chef::Log.info("Uploading #{checksum_files[checksum]} (checksum hex = #{checksum}) to #{info['url']}")
+
+          # Checksum is the hexadecimal representation of the md5,
+          # but we need the base64 encoding for the content-md5
+          # header
+          checksum64 = Base64.encode64([checksum].pack("H*")).strip
+          timestamp = Time.now.utc.iso8601
+          file_contents = File.read(checksum_files[checksum])
+          # TODO - 5/28/2010, cw: make signing and sending the request streaming
+          sign_obj = Mixlib::Authentication::SignedHeaderAuth.signing_object(
+                                                                             :http_method => :put,
+                                                                             :path        => URI.parse(info['url']).path,
+                                                                             :body        => file_contents,
+                                                                             :timestamp   => timestamp,
+                                                                             :user_id     => rest.client_name
+                                                                             )
+          headers = { 'content-type' => 'application/x-binary', 'content-md5' => checksum64, :accept => 'application/json' }
+          headers.merge!(sign_obj.sign(OpenSSL::PKey::RSA.new(rest.signing_key)))
+
+          begin
+            RestClient::Resource.new(info['url'], :headers=>headers, :timeout=>1800, :open_timeout=>1800).put(file_contents)
+          rescue RestClient::Exception => e
+            Chef::Log.error("Upload failed: #{e.message}\n#{e.response.body}")
             raise
           end
+        else
+          Chef::Log.debug("#{checksum_files[checksum]} has not changed")
         end
-        # files are uploaded, so save the manifest
-        cookbook.save
-        Chef::Log.info("Upload complete!")
       end
-
-      def build_metadata(cookbook)
-        Chef::Log.debug("Generating metadata")
-        # FIXME: This knife command should be factored out into a
-        # library for use here
-        kcm = Chef::Knife::CookbookMetadata.new
-        kcm.config[:cookbook_path] = Chef::Config[:cookbook_path]
-        kcm.name_args = [ cookbook.name.to_s ]
-        kcm.run
-        cookbook.reload_metadata!
+      sandbox_url = new_sandbox['uri']
+      Chef::Log.debug("Committing sandbox")
+      # Retry if S3 is claims a checksum doesn't exist (the eventual
+      # in eventual consistency)
+      retries = 0
+      begin
+        rest.put_rest(sandbox_url, {:is_completed => true})
+      rescue Net::HTTPServerException => e
+        if e.message =~ /^400/ && (retries += 1) <= 5
+          sleep 2
+          retry
+        else
+          raise
+        end
       end
+      # files are uploaded, so save the manifest
+      opts[:force] ? cookbook.force_save : cookbook.save
+      Chef::Log.info("Upload complete!")
+    end
 
-      def validate_cookbook(cookbook)
-        syntax_checker = Chef::Cookbook::SyntaxCheck.for_cookbook(cookbook.name, @user_cookbook_path)
-        Chef::Log.info("Validating ruby files")
-        exit(1) unless syntax_checker.validate_ruby_files
-        Chef::Log.info("Validating templates")
-        exit(1) unless syntax_checker.validate_templates
-        Chef::Log.info("Syntax OK")
-        true
-      end
+    def build_metadata
+      Chef::Log.debug("Generating metadata")
+      # FIXME: This knife command should be factored out into a
+      # library for use here
+      kcm = Chef::Knife::CookbookMetadata.new
+      kcm.config[:cookbook_path] = path
+      kcm.name_args = [ cookbook.name.to_s ]
+      kcm.run
+      cookbook.reload_metadata!
+    end
 
+    def validate_cookbook
+      syntax_checker = Chef::Cookbook::SyntaxCheck.for_cookbook(cookbook.name, @user_cookbook_path)
+      Chef::Log.info("Validating ruby files")
+      exit(1) unless syntax_checker.validate_ruby_files
+      Chef::Log.info("Validating templates")
+      exit(1) unless syntax_checker.validate_templates
+      Chef::Log.info("Syntax OK")
+      true
     end
+
   end
 end