chef 0.8.16 → 0.9.0.a3

data/lib/chef/knife/cookbook_bulk_delete.rb

@@ -29,7 +29,7 @@ class Chef
           Chef::Log.fatal("You must supply a regular expression to match the results against")
           exit 42
         else
-          bulk_delete(Chef::Cookbook, "cookbook", "cookbook", rest.get_rest("cookbooks"), @name_args[0]) do |name, cookbook|
+          bulk_delete(Chef::CookbookVersion, "cookbook", "cookbook", rest.get_rest("cookbooks"), @name_args[0]) do |name, cookbook|
             rest.delete_rest(cookbook)
           end
         end

data/lib/chef/knife/cookbook_delete.rb

@@ -22,11 +22,11 @@ class Chef
   class Knife
     class CookbookDelete < Knife
 
-      banner "Sub-Command: cookbook delete COOKBOOK (options)"
+      banner "Sub-Command: cookbook delete COOKBOOK VERSION (options)"
 
-      def run 
-        delete_object(Chef::Cookbook, @name_args[0], "cookbook") do
-          rest.delete_rest("cookbooks/#{@name_args[0]}")
+      def run
+        delete_object(Chef::CookbookVersion, "#{@name_args[0]} #{@name_args[1]}", "cookbook") do
+          rest.delete_rest("cookbooks/#{@name_args[0]}/#{@name_args[1]}")
         end
       end
 

data/lib/chef/knife/cookbook_download.rb

@@ -1,6 +1,7 @@
 #
 # Author:: Adam Jacob (<adam@opscode.com>)
-# Copyright:: Copyright (c) 2009 Opscode, Inc.
+# Author:: Christopher Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2009, 2010 Opscode, Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,36 +23,66 @@ class Chef
   class Knife
     class CookbookDownload < Knife
 
-      banner "Sub-Command: cookbook download COOKBOOK (options)"
-
-      option :file,
-       :short => "-f FILE",
-       :long => "--file FILE",
-       :description => "The filename to write to"
-
-      def run 
-        cookbook = rest.get_rest("cookbooks/#{@name_args[0]}")
-        version = cookbook["metadata"]["version"] 
-        Chef::Log.info("Downloading #{@name_args[0]} cookbook version #{version}")
-        rest.sign_on_redirect = false
-        tf = rest.get_rest("cookbooks/#{@name_args[0]}/_content", true)
-        rest.sign_on_redirect = true 
-        unless config[:file]
-          if version
-            config[:file] = File.join(Dir.pwd, "#{@name_args[0]}-#{version}.tar.gz")
+      banner "Sub-Command: cookbook download COOKBOOK VERSION (options)"
+
+      option :version,
+       :short => "-v VERSION",
+       :long => "--version VERSION",
+       :description => "The version of the cookbook to download"
+
+      option :download_directory,
+       :short => "-d DOWNLOAD_DIRECTORY",
+       :long => "--dir DOWNLOAD_DIRECTORY",
+       :description => "The directory to download the cookbook into",
+       :default => Dir.pwd
+      
+      option :force,
+       :short => "-f",
+       :long => "--force",
+       :description => "Force download over the download directory if it exists"
+
+      # TODO: tim/cw: 5-23-2010: need to implement knife-side
+      # specificity for downloads - need to implement --platform and
+      # --fqdn here
+      def run
+        if @name_args.length != 2
+          Chef::Log.fatal("You must supply a cookbook name and version to download!")
+          exit 42
+        end
+          
+        cookbook_name = @name_args[0]
+        cookbook_version = @name_args[1] == 'latest' ? '_latest' : @name_args[1]
+        Chef::Log.info("Downloading #{cookbook_name} cookbook version #{cookbook_version}")
+        
+        cookbook = rest.get_rest("cookbooks/#{cookbook_name}/#{cookbook_version}")
+        manifest = cookbook.manifest
+
+        basedir = File.join(config[:download_directory], "#{cookbook_name}-#{cookbook.version}")
+        if File.exists?(basedir)
+          if config[:force]
+            Chef::Log.debug("Deleting #{basedir}")
+            FileUtils.rm_rf(basedir)
           else
-            config[:file] = File.join(Dir.pwd, "#{@name_args[0]}.tar.gz")
+            Chef::Log.fatal("Directory #{basedir} exists, use --force to overwrite")
+            exit
+          end
+        end
+        
+        Chef::CookbookVersion::COOKBOOK_SEGMENTS.each do |segment|
+          next unless manifest.has_key?(segment)
+          Chef::Log.info("Downloading #{segment}")
+          manifest[segment].each do |segment_file|
+            dest = File.join(basedir, segment_file['path'].gsub('/', File::SEPARATOR))
+            Chef::Log.debug("Downloading #{segment_file['path']} to #{dest}")
+            FileUtils.mkdir_p(File.dirname(dest))
+            rest.sign_on_redirect = false
+            tempfile = rest.get_rest(segment_file['url'], true)
+            FileUtils.mv(tempfile.path, dest)
           end
         end
-        FileUtils.cp(tf.path, config[:file])
-        Chef::Log.info("Cookbook saved: #{config[:file]}")
+        Chef::Log.info("Cookbook downloaded to #{basedir}")
       end
 
     end
   end
 end
-
-
-
-
-

data/lib/chef/knife/cookbook_metadata.rb

@@ -45,8 +45,8 @@ class Chef
 
         if config[:all]
           cl = Chef::CookbookLoader.new
-          cl.each do |cookbook|
-            generate_metadata(cookbook.name.to_s)
+          cl.each do |cname, cookbook|
+            generate_metadata(cname.to_s)
           end
         else
           generate_metadata(@name_args[0])

data/lib/chef/knife/cookbook_show.rb

@@ -24,7 +24,7 @@ class Chef
   class Knife
     class CookbookShow < Knife
 
-      banner "Sub-Command: cookbook show COOKBOOK [PART] [FILENAME] (options)"
+      banner "Sub-Command: cookbook show COOKBOOK [VERSION] [PART] [FILENAME] (options)"
 
       option :fqdn,
        :short => "-f FQDN",
@@ -43,17 +43,36 @@ class Chef
 
       def run 
         case @name_args.length
-        when 3 # We are showing a specific file
-          arguments = { :id => @name_args[2] } 
-          arguments[:fqdn] = config[:fqdn] if config.has_key?(:fqdn)
-          arguments[:platform] = config[:platform] if config.has_key?(:platform)
-          arguments[:version] = config[:platform_version] if config.has_key?(:platform_version)
-          result = rest.get_rest("cookbooks/#{@name_args[0]}/#{@name_args[1]}?#{make_query_params(arguments)}")
+        when 4 # We are showing a specific file
+          node = Hash.new
+          node[:fqdn] = config[:fqdn] if config.has_key?(:fqdn)
+          node[:platform] = config[:platform] if config.has_key?(:platform)
+          node[:platform_version] = config[:platform_version] if config.has_key?(:platform_version)
+
+          class << node
+            def attribute?(name)
+              has_key?(name)
+            end
+          end
+
+          cookbook_name, cookbook_version, segment, filename = @name_args[0..3]
+
+          manifest = rest.get_rest("cookbooks/#{cookbook_name}/#{cookbook_version}")
+          cookbook = Chef::CookbookVersion.new(cookbook_name)
+          cookbook.manifest = manifest
+          
+          manifest_entry = cookbook.preferred_manifest_record(node, segment, filename)
+          result = rest.get_rest("cookbooks/#{cookbook_name}/#{cookbook_version}/files/#{manifest_entry[:checksum]}")
+          
           pretty_print(result)
-        when 2 # We are showing a specific part of the cookbook
-          result = rest.get_rest("cookbooks/#{@name_args[0]}")
-          output(result[@name_args[1]])
-        when 1 # We are showing the whole cookbook data
+        when 3 # We are showing a specific part of the cookbook
+          cookbook_version = @name_args[1] == 'latest' ? '_latest' : @name_args[1]
+          result = rest.get_rest("cookbooks/#{@name_args[0]}/#{cookbook_version}")
+          output(result.manifest[@name_args[2]])
+        when 2 # We are showing the whole cookbook data
+          cookbook_version = @name_args[1] == 'latest' ? '_latest' : @name_args[1]
+          output(rest.get_rest("cookbooks/#{@name_args[0]}/#{cookbook_version}"))
+        when 1 # We are showing the cookbook versions 
           output(rest.get_rest("cookbooks/#{@name_args[0]}"))
         end
       end

data/lib/chef/knife/cookbook_upload.rb

@@ -1,7 +1,7 @@
 #
-#
 # Author:: Adam Jacob (<adam@opscode.com>)
-# Copyright:: Copyright (c) 2009 Opscode, Inc.
+# Author:: Christopher Walters (<cw@opscode.com>)
+# Copyright:: Copyright (c) 2009, 2010 Opscode, Inc.
 # License:: Apache License, Version 2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,8 +17,14 @@
 # limitations under the License.
 #
 
+require 'rest_client'
+
 require 'chef/knife'
 require 'chef/streaming_cookbook_uploader'
+require 'chef/cache/checksum'
+require 'chef/sandbox'
+require 'chef/cookbook_version'
+require 'chef/cookbook/file_system_file_vendor'
 
 class Chef
   class Knife
@@ -37,130 +43,151 @@ class Chef
         :long => "--all",
         :description => "Upload all cookbooks, rather than just a single cookbook"
 
-      def run
-        Chef::Log.debug "Uploading cookbooks from #{config[:cookbook_path]}"
-
+      def run 
         if config[:cookbook_path]
           Chef::Config[:cookbook_path] = config[:cookbook_path]
         else
           config[:cookbook_path] = Chef::Config[:cookbook_path]
         end
 
+        Chef::Cookbook::FileVendor.on_create { |manifest| Chef::Cookbook::FileSystemFileVendor.new(manifest) }
+
+        cl = Chef::CookbookLoader.new
         if config[:all]
-          cl = Chef::CookbookLoader.new
-          cl.each do |cookbook|
+          cl.each do |cookbook_name, cookbook|
             Chef::Log.info("** #{cookbook.name.to_s} **")
-            upload_cookbook(cookbook.name.to_s)
+            upload_cookbook(cookbook)
           end
         else
-          @name_args.each do |cb|
-            Chef::Log.info("** #{cb} **")
-            upload_cookbook(cb)
-          end
+          @name_args.each{|cookbook_name| upload_cookbook(cl[cookbook_name]) }
+        end
+      end
+      
+      def test_ruby(cookbook_dir)
+        Chef::Log.info("Validating ruby files")
+        Dir[File.join(cookbook_dir, '**', '*.rb')].each do |ruby_file|
+          Chef::Log.info("Testing #{ruby_file} for syntax errors...")
+          Chef::Mixin::Command.run_command(:command => "ruby -c #{ruby_file}")
         end
       end
+      
+      def test_templates(cookbook_dir)
+        Chef::Log.info("Validating templates")
+        Dir[File.join(cookbook_dir, '**', '*.erb')].each do |erb_file|
+          Chef::Log.info("Testing template #{erb_file} for syntax errors...")
+          Chef::Mixin::Command.run_command(:command => "sh -c 'erubis -x #{erb_file} | ruby -c'")
+        end
+      end
+      
+      def upload_cookbook(cookbook)
+        Chef::Log.info("Saving #{cookbook.name}")
+        
+        # create build directory
+        tmp_cookbook_dir = create_build_dir(cookbook)
+
+        # create a CookbookLoader that loads a Cookbook from the build directory
+        orig_cookbook_path = nil
+        build_dir_cookbook = nil
+        begin
+          orig_cookbook_path = Chef::Config.cookbook_path
+          Chef::Config.cookbook_path = tmp_cookbook_dir
+          build_dir_cookbook = Chef::CookbookLoader.new[cookbook.name]
+          Chef::Log.debug("Staged cookbook manifest:\n#{JSON.pretty_generate(build_dir_cookbook)}")
+        ensure
+          Chef::Config.cookbook_path = orig_cookbook_path
+        end
 
-      def upload_cookbook(cookbook_name)
-        # Syntax check all cookbook paths rather than tmp_cookbook_dir as to
-        # take advantage of the existing cache used/generated by knife cookbook
-        # test.
-        check = Chef::Knife::CookbookTest.new
-        check.config[:cookbook_path] = config[:cookbook_path]
-        check.name_args = [ cookbook_name ]
-        check.run
-
-        if cookbook_name =~ /^#{File::SEPARATOR}/
-          child_folders = cookbook_name
-          cookbook_name = File.basename(cookbook_name)
-        else
-          child_folders = config[:cookbook_path].inject([]) do |r, e|
-            r << File.join(e, cookbook_name)
-            r
+        # generate checksums of cookbook files and create a sandbox
+        checksum_files = build_dir_cookbook.checksums
+        checksums = checksum_files.inject({}){|memo,elt| memo[elt.first]=nil ; memo}
+        new_sandbox = catch_auth_exceptions{ rest.post_rest("/sandboxes", { :checksums => checksums }) }
+        
+        # upload the new checksums and commit the sandbox
+        new_sandbox['checksums'].each do |checksum, info|
+          if info['needs_upload'] == true
+            Chef::Log.debug("PUTting #{checksum_files[checksum]} (checksum hex = #{checksum}) to #{info['url']}")
+            
+            # Checksum is the hexadecimal representation of the md5,
+            # but we need the base64 encoding for the content-md5
+            # header
+            checksum64 = Base64.encode64([checksum].pack("H*")).strip
+            timestamp = Time.now.utc.iso8601
+            file_contents = File.read(checksum_files[checksum])
+            # TODO - 5/28/2010, cw: make signing and sending the request streaming
+            sign_obj = Mixlib::Authentication::SignedHeaderAuth.signing_object(
+                                                                               :http_method => :put,
+                                                                               :path        => URI.parse(info['url']).path,
+                                                                               :body        => file_contents,
+                                                                               :timestamp   => timestamp,
+                                                                               :user_id     => rest.client_name
+                                                                               )
+            headers = { 'content-type' => 'application/x-binary', 'content-md5' => checksum64, :accept => 'application/json' }
+            headers.merge!(sign_obj.sign(OpenSSL::PKey::RSA.new(rest.signing_key)))
+            begin
+              RestClient::Request.execute(:method => :put, :url => info['url'], :headers => headers, :payload => file_contents)
+            rescue RestClient::RequestFailed => e
+              Chef::Log.error("Upload failed: #{e.message}\n#{e.response.body}")
+              raise
+            end
           end
         end
+        sandbox_url = new_sandbox['uri']
+        Chef::Log.debug("Committing sandbox")
+        catch_auth_exceptions{ rest.put_rest(sandbox_url, {:is_completed => true}) }
 
-        tmp_cookbook_tarball = Tempfile.new("chef-#{cookbook_name}")
-        tmp_cookbook_tarball.close
-        tarball_name = "#{tmp_cookbook_tarball.path}.tar.gz"
-        File.unlink(tmp_cookbook_tarball.path)
+        # files are uploaded, so save the manifest
+        catch_auth_exceptions{ build_dir_cookbook.save }
+        
+        Chef::Log.info("Upload complete!")
+        Chef::Log.debug("Removing local staging directory at #{tmp_cookbook_dir}")
+        FileUtils.rm_rf tmp_cookbook_dir
+      end
 
-        tmp_cookbook_path = Tempfile.new("chef-#{cookbook_name}-build")
+      def create_build_dir(cookbook)
+        tmp_cookbook_path = Tempfile.new("chef-#{cookbook.name}-build")
         tmp_cookbook_path.close
         tmp_cookbook_dir = tmp_cookbook_path.path
         File.unlink(tmp_cookbook_dir)
         FileUtils.mkdir_p(tmp_cookbook_dir)
-
+        
         Chef::Log.debug("Staging at #{tmp_cookbook_dir}")
 
-        found_cookbook = false
+        checksums_to_on_disk_paths = cookbook.checksums
 
-        child_folders.each do |file_path|
-          if File.directory?(file_path)
-            found_cookbook = true
-            Chef::Log.info("Copying from #{file_path} to #{tmp_cookbook_dir}")
-            FileUtils.cp_r(file_path, tmp_cookbook_dir, :remove_destination => true, :preserve => true)
-          else
-            Chef::Log.info("Nothing to copy from #{file_path}")
+        Chef::CookbookVersion::COOKBOOK_SEGMENTS.each do |segment|
+          cookbook.manifest[segment].each do |manifest_record|
+            path_in_cookbook = manifest_record[:path]
+            on_disk_path = checksums_to_on_disk_paths[manifest_record[:checksum]]
+            dest = File.join(tmp_cookbook_dir, cookbook.name.to_s, path_in_cookbook)
+            FileUtils.mkdir_p(File.dirname(dest))
+            FileUtils.cp(on_disk_path, dest)
           end
         end
-
-        unless found_cookbook
-          Chef::Log.fatal("Could not find cookbook #{cookbook_name}!")
-          exit 17
-        end
+        
+        # Validate ruby files and templates
+        test_ruby(tmp_cookbook_dir)
+        test_templates(tmp_cookbook_dir)
 
         # First, generate metadata
+        Chef::Log.debug("Generating metadata")
         kcm = Chef::Knife::CookbookMetadata.new
         kcm.config[:cookbook_path] = [ tmp_cookbook_dir ]
-        kcm.name_args = [ cookbook_name ]
+        kcm.name_args = [ cookbook.name.to_s ]
         kcm.run
+      end
 
-        Chef::Log.info("Creating tarball at #{tarball_name}")
-        Chef::Mixin::Command.run_command(
-          :command => "tar -C #{tmp_cookbook_dir} -cvzf #{tarball_name} ./#{cookbook_name}"
-        )
-
+      def catch_auth_exceptions
         begin
-          cb = rest.get_rest("cookbooks/#{cookbook_name}")
-          cookbook_uploaded = true
+          yield
         rescue Net::HTTPServerException => e
           case e.response.code
-          when "404"
-            cookbook_uploaded = false
           when "401"
-            Chef::Log.fatal "Failed to fetch remote cookbook '#{cookbook_name}' due to authentication failure (#{e}), check your client configuration (username, key)"
+            Chef::Log.fatal "Request failed due to authentication (#{e}), check your client configuration (username, key)"
             exit 18
           end
         end
-
-        if cookbook_uploaded
-          Chef::StreamingCookbookUploader.put(
-            "#{Chef::Config[:chef_server_url]}/cookbooks/#{cookbook_name}/_content",
-            Chef::Config[:node_name],
-            Chef::Config[:client_key],
-            {
-              :file => File.new(tarball_name),
-              :name => cookbook_name
-            }
-          )
-        else
-          Chef::StreamingCookbookUploader.post(
-            "#{Chef::Config[:chef_server_url]}/cookbooks",
-            Chef::Config[:node_name],
-            Chef::Config[:client_key],
-            {
-              :file => File.new(tarball_name),
-              :name => cookbook_name
-            }
-          )
-        end
-        Chef::Log.info("Upload complete!")
-        Chef::Log.debug("Removing local tarball at #{tarball_name}")
-        FileUtils.rm_rf tarball_name
-        Chef::Log.debug("Removing local staging directory at #{tmp_cookbook_dir}")
-        FileUtils.rm_rf tmp_cookbook_dir
       end
-
+      
     end
   end
 end