knife-essentials 0.5.4 → 0.6

data/lib/chef/knife/raw.rb

@@ -64,8 +64,7 @@ class Chef
         chef_rest.retriable_rest_request(method, url, json_body, headers) do |rest_request|
           response = rest_request.call {|r| r.read_body}
 
-#          response_body = chef_rest.decompress_body(response)
-          response_body = response.body
+          response_body = chef_rest.decompress_body(response)
 
           if response.kind_of?(Net::HTTPSuccess)
             if config[:pretty] && response['content-type'] =~ /json/
@@ -78,7 +77,7 @@ class Chef
             follow_redirect {api_request(:GET, create_url(redirect_location))}
           else
             # have to decompress the body before making an exception for it. But the body could be nil.
-#            response.body.replace(decompress_body(response)) if response.body.respond_to?(:replace)
+            response.body.replace(chef_rest.decompress_body(response)) if response.body.respond_to?(:replace)
 
             if response['content-type'] =~ /json/
               exception = response_body
@@ -98,7 +97,7 @@ class Chef
         headers['Accept']       = "application/json" unless raw
         headers["Content-Type"] = 'application/json' if json_body
         headers['Content-Length'] = json_body.bytesize.to_s if json_body
-#        headers[Chef::REST::RESTRequest::ACCEPT_ENCODING] = Chef::REST::RESTRequest::ENCODING_GZIP_DEFLATE
+        headers[Chef::REST::RESTRequest::ACCEPT_ENCODING] = Chef::REST::RESTRequest::ENCODING_GZIP_DEFLATE
         headers.merge!(chef_rest.authentication_headers(method, url, json_body)) if chef_rest.sign_requests?
         headers.merge!(Chef::Config[:custom_http_headers]) if Chef::Config[:custom_http_headers]
         headers

data/lib/chef/sandbox_uploader.rb

@@ -0,0 +1,208 @@
+#
+# Author:: John Keiser (<jkeiser@opscode.com>)
+# Copyright:: Copyright (c) 2009 Opscode, Inc.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+class Chef
+  #
+  # Represents a sandbox uploading session.  You pass in a set of files with
+  #
+  # Much of this is shamelessly stolen from Chef::CookbookUploader, placed at
+  # the sandbox level to allow any set of files to be uploaded and to allow
+  # uploading to plug in to the host's threading model (if any).
+  #
+  # A single-threaded caller will want to:
+  #
+  #     uploader = SandboxUploader.new(checksum_files, rest)
+  #     while uploader.upload_next_file do
+  #     end
+  #     uploader.commit
+  #
+  # A multithreaded caller will want to keep doing this:
+  #
+  #     if !uploader.upload_next_file
+  #       uploader.commit
+  class SandboxUploader
+    # Create a new SandboxUploader.
+    #
+    # ==== Attributes
+    #
+    # * +checksums+ - a list of checksums of files you plan to upload
+    # * +rest+ - the rest client to use
+    #
+    # ==== Throws
+    #
+    # This method will throw an HTTP exception if the sandbox creation fails.
+    def initialize(checksum, rest)
+      # Grab the list of URLs we need to upload to
+      checksums_hash = checksums.inject({}) { |hash,checksum| hash[checksum] = nil; hash }
+      response = rest.post_rest("sandboxes", { :checksums => checksums_hash })
+      @sandbox_url = response['uri']
+      @to_upload = []
+      response['checksums'].each do |checksum, info|
+        if info['needs_upload'] == true
+          @to_upload << {
+            :checksum => checksum,
+            :url => info['url']
+          }
+        else
+          Chef::Log.debug("#{checksum_files[checksum]} has not changed")
+        end
+      end
+      @uploading = 0
+    end
+
+    # Picks a file off the list and uploads it.
+    #
+    # ==== Parameters
+    #
+    # * +&block+ - if there is a file to upload, +block.call(checksum)+ will be
+    #   invoked, and this should return the file contents for the given
+    #   checksum.
+    #
+    # ==== Throws
+    #
+    # An HTTP error is thrown if the upload fails.
+    #
+    # ==== Returns
+    #
+    # +{ :checksum => checksum, :url => url }+ if a file was uploaded.  +nil+ if
+    # there was no file left to upload.
+    def upload_next_file(&block)
+      @uploading += 1
+      begin
+        file = @to_upload.pop
+        return nil if !file
+
+        begin
+          return upload_file(block.call(file[:checksum]), file[:checksum], file[:url])
+        rescue
+          # Stick the work on top so we work on other things first
+          @to_upload.unshift(file)
+          raise
+        end
+      ensure
+        @uploading -= 1
+      end
+    end
+
+    # Commits the sandbox to Chef.
+    #
+    # ==== Throws
+    #
+    # * An HTTP error if the commit fails.
+    # * "Attempt to commit "
+    #
+    # ==== Returns
+    #
+    # +true+ if a commit occurred.  +false+ if commit
+    # has already been called.
+    def commit
+      return false if !@sandbox_url
+      raise "Attempt to commit before uploads are complete!" if !uploads_complete?
+      # nil out sandbox url to avoid race conditions where multiple
+      # callers try to commit
+      sandbox_url = @sandbox_url
+      @sandbox_url = nil
+      @to_upload = nil
+
+      Chef::Log.debug("Committing sandbox")
+      # Retry if S3 is claims a checksum doesn't exist (the eventual
+      # in eventual consistency)
+      retries = 0
+      begin
+        rest.put_rest(sandbox_url, {:is_completed => true})
+      rescue Net::HTTPServerException => e
+        if e.message =~ /^400/ && (retries += 1) <= 5
+          sleep 2
+          retry
+        else
+          raise
+        end
+      end
+      true
+    end
+
+    def cancel
+      # Yeah, so uh, delete doesn't work.  But we'd like to imagine it will.
+      sandbox_url = @sandbox_url
+      @sandbox_url = nil
+      @to_upload = nil
+      rest.delete_rest(sandbox_url)
+    end
+
+    # Tells whether uploads are all complete.
+    def uploads_complete?
+      @to_upload.length == 0 && @uploading == 0
+    end
+
+    private
+
+    def upload_file(file_contents, checksum, url)
+      # Checksum is the hexadecimal representation of the md5,
+      # but we need the base64 encoding for the content-md5
+      # header
+      checksum64 = Base64.encode64([checksum].pack("H*")).strip
+      timestamp = Time.now.utc.iso8601
+      # TODO - 5/28/2010, cw: make signing and sending the request streaming
+      sign_obj = Mixlib::Authentication::SignedHeaderAuth.signing_object(
+                                                                         :http_method => :put,
+                                                                         :path        => URI.parse(url).path,
+                                                                         :body        => file_contents,
+                                                                         :timestamp   => timestamp,
+                                                                         :user_id     => rest.client_name
+                                                                         )
+      headers = { 'content-type' => 'application/x-binary', 'content-md5' => checksum64, :accept => 'application/json' }
+      headers.merge!(sign_obj.sign(OpenSSL::PKey::RSA.new(rest.signing_key)))
+
+      begin
+        RestClient::Resource.new(url, :headers=>headers, :timeout=>1800, :open_timeout=>1800).put(file_contents)
+      rescue RestClient::Exception => e
+        Chef::Knife.ui.error("Failed to upload #@cookbook : #{e.message}\n#{e.response.body}")
+        raise
+      end
+    end
+
+    # For CookbookDir
+
+      private
+
+      def upload(entries, force, freeze)
+        # knife cookbook upload: Validate environment
+        # knife cookbook upload: Warn about cookbook shadowing
+        # knife cookbook upload: Check for broken links
+        # knife cookbook upload: Check for dependencies
+        # CookbookUploader: Validate cookbook syntax, ruby files and templates
+        # SandboxUploader!
+        entries_by_checksum = entries.inject({}) do |hash,entry|
+          if entry.respond_to?(:checksum)
+            checksum = entry.checksum
+          else
+            checksum = ChefFS::Diff.calc_checksum(entry.read)
+          end
+          hash[checksum] = entry
+        end
+
+        uploader = SandboxUploader.new(entries_by_checksum.keys, rest)
+        while uploader.upload_next_file { |checksum| entries_by_checksum[checksum].read }
+        end
+        uploader.commit
+        # CookbookUploader: PUT the manifest to either the force_save_url or save_url for the cookbook
+        # knife cookbook upload: update version constraints for environment
+      end
+  end
+end

data/lib/chef_fs/command_line.rb

@@ -1,115 +1,155 @@
-require 'chef_fs/diff'
+require 'chef_fs/file_system'
 
 module ChefFS
   module CommandLine
     def self.diff(pattern, a_root, b_root, recurse_depth, output_mode)
       found_result = false
-      ChefFS::Diff::diffable_leaves_from_pattern(pattern, a_root, b_root, recurse_depth) do |a_leaf, b_leaf|
+      ChefFS::FileSystem.list_pairs(pattern, a_root, b_root) do |a, b|
         found_result = true
-        diff = diff_leaves(a_leaf, b_leaf, output_mode)
-        yield diff if diff != ''
+        diff_entries(a, b, recurse_depth, output_mode) do |diff|
+          yield diff
+        end
       end
       if !found_result && pattern.exact_path
         yield "#{pattern}: No such file or directory on remote or local"
       end
     end
 
-    private
-
-    # Diff two known leaves (could be files or dirs)
-    def self.diff_leaves(old_file, new_file, output_mode)
-      result = ''
+    # Diff two known entries (could be files or dirs)
+    def self.diff_entries(old_entry, new_entry, recurse_depth, output_mode)
       # If both are directories
-      # If old is a directory and new is a file
-      # If old is a directory and new does not exist
-      if old_file.dir?
-        if new_file.dir?
-          if output_mode != :name_only && output_mode != :name_status
-            result << "Common subdirectories: #{old_file.path}\n"
+      if old_entry.dir?
+        if new_entry.dir?
+          if recurse_depth == 0
+            if output_mode != :name_only && output_mode != :name_status
+              yield "Common subdirectories: #{old_entry.path}\n"
+            end
+          else
+            ChefFS::FileSystem.child_pairs(old_entry, new_entry).each do |old_child,new_child|
+              diff_entries(old_child, new_child,
+                           recurse_depth ? recurse_depth - 1 : nil, output_mode) do |diff|
+                yield diff
+              end
+            end
           end
-        elsif new_file.exists?
+
+      # If old is a directory and new is a file
+        elsif new_entry.exists?
           if output_mode == :name_only
-            result << "#{new_file.path_for_printing}\n"
+            yield "#{new_entry.path_for_printing}\n"
           elsif output_mode == :name_status
-            result << "T\t#{new_file.path_for_printing}\n"
+            yield "T\t#{new_entry.path_for_printing}\n"
           else
-            result << "File #{new_file.path_for_printing} is a directory while file #{new_file.path_for_printing} is a regular file\n"
+            yield "File #{new_entry.path_for_printing} is a directory while file #{new_entry.path_for_printing} is a regular file\n"
           end
-        elsif new_file.parent.can_have_child?(old_file.name, old_file.dir?)
+
+      # If old is a directory and new does not exist
+        elsif new_entry.parent.can_have_child?(old_entry.name, old_entry.dir?)
           if output_mode == :name_only
-            result << "#{new_file.path_for_printing}\n"
+            yield "#{new_entry.path_for_printing}\n"
           elsif output_mode == :name_status
-            result << "D\t#{new_file.path_for_printing}\n"
+            yield "D\t#{new_entry.path_for_printing}\n"
           else
-            result << "Only in #{old_file.parent.path_for_printing}: #{old_file.name}\n"
+            yield "Only in #{old_entry.parent.path_for_printing}: #{old_entry.name}\n"
           end
         end
 
-      # If new is a directory and old does not exist
       # If new is a directory and old is a file
-      elsif new_file.dir?
-        if old_file.exists?
+      elsif new_entry.dir?
+        if old_entry.exists?
           if output_mode == :name_only
-            result << "#{new_file.path_for_printing}\n"
+            yield "#{new_entry.path_for_printing}\n"
           elsif output_mode == :name_status
-            result << "T\t#{new_file.path_for_printing}\n"
+            yield "T\t#{new_entry.path_for_printing}\n"
           else
-            result << "File #{old_file.path_for_printing} is a regular file while file #{old_file.path_for_printing} is a directory\n"
+            yield "File #{old_entry.path_for_printing} is a regular file while file #{old_entry.path_for_printing} is a directory\n"
           end
-        elsif old_file.parent.can_have_child?(new_file.name, new_file.dir?)
+
+      # If new is a directory and old does not exist
+        elsif old_entry.parent.can_have_child?(new_entry.name, new_entry.dir?)
           if output_mode == :name_only
-            result << "#{new_file.path_for_printing}\n"
+            yield "#{new_entry.path_for_printing}\n"
           elsif output_mode == :name_status
-            result << "A\t#{new_file.path_for_printing}\n"
+            yield "A\t#{new_entry.path_for_printing}\n"
           else
-            result << "Only in #{new_file.parent.path_for_printing}: #{new_file.name}\n"
+            yield "Only in #{new_entry.parent.path_for_printing}: #{new_entry.name}\n"
           end
         end
 
       else
         # Neither is a directory, so they are diffable with file diff
-        different, old_value, new_value = ChefFS::Diff::diff_files(old_file, new_file)
-        if different
+        are_same, old_value, new_value = ChefFS::FileSystem.compare(old_entry, new_entry)
+        if !are_same
+          if old_value == :none
+            old_exists = false
+          elsif old_value.nil?
+            old_exists = old_entry.exists?
+          else
+            old_exists = true
+          end
+          if new_value == :none
+            new_exists = false
+          elsif new_value.nil?
+            new_exists = new_entry.exists?
+          else
+            new_exists = true
+          end
+
           # If one of the files doesn't exist, we only want to print the diff if the
           # other file *could be uploaded/downloaded*.
-          if !old_value && !old_file.parent.can_have_child?(new_file.name, new_file.dir?)
-            return result
+          if !old_exists && !old_entry.parent.can_have_child?(new_entry.name, new_entry.dir?)
+            return
           end
-          if !new_value && !new_file.parent.can_have_child?(old_file.name, old_file.dir?)
-            return result
+          if !new_exists && !new_entry.parent.can_have_child?(old_entry.name, old_entry.dir?)
+            return
           end
 
           if output_mode == :name_only
-            result << "#{new_file.path_for_printing}\n"
+            yield "#{new_entry.path_for_printing}\n"
           elsif output_mode == :name_status
-            if !old_value
-              result << "A\t#{new_file.path_for_printing}\n"
-            elsif !new_value
-              result << "D\t#{new_file.path_for_printing}\n"
+            if old_value == :none || (old_value == nil && !old_entry.exists?)
+              yield "A\t#{new_entry.path_for_printing}\n"
+            elsif new_value == :none
+              yield "D\t#{new_entry.path_for_printing}\n"
             else
-              result << "M\t#{new_file.path_for_printing}\n"
+              yield "M\t#{new_entry.path_for_printing}\n"
             end
           else
-            old_path = old_file.path_for_printing
-            new_path = new_file.path_for_printing
+            # If we haven't read the values yet, get them now.
+            begin
+              old_value = old_entry.read if old_value.nil?
+            rescue ChefFS::FileSystem::NotFoundError
+              old_value = :none
+            end
+            begin
+              new_value = new_entry.read if new_value.nil?
+            rescue ChefFS::FileSystem::NotFoundError
+              new_value = :none
+            end
+
+            old_path = old_entry.path_for_printing
+            new_path = new_entry.path_for_printing
+            result = ''
             result << "diff --knife #{old_path} #{new_path}\n"
-            if !old_value
+            if old_value == :none
               result << "new file\n"
               old_path = "/dev/null"
               old_value = ''
             end
-            if !new_value
+            if new_value == :none
               result << "deleted file\n"
               new_path = "/dev/null"
               new_value = ''
             end
             result << diff_text(old_path, new_path, old_value, new_value)
+            yield result
           end
         end
       end
-      return result
     end
 
+    private
+
     def self.sort_keys(json_object)
       if json_object.is_a?(Array)
         json_object.map { |o| sort_keys(o) }

data/lib/chef_fs/file_system.rb

@@ -1,5 +1,4 @@
 require 'chef_fs/path_utils'
-require 'chef_fs/diff'
 
 module ChefFS
   module FileSystem
@@ -90,24 +89,114 @@ module ChefFS
     #
     # ==== Examples
     #
-    #     ChefFS::FileSystem.copy_to(FilePattern.new('/cookbooks', chef_fs, local_fs, nil, true)
+    #     ChefFS::FileSystem.copy_to(FilePattern.new('/cookbooks'),
+    #       chef_fs, local_fs, nil, true) do |message|
+    #       puts message
+    #     end
     #
     def self.copy_to(pattern, src_root, dest_root, recurse_depth, options)
       found_result = false
-      # Find things we might want to copy
-      ChefFS::Diff::diffable_leaves_from_pattern(pattern, src_root, dest_root, recurse_depth) do |src_leaf, dest_leaf, child_recurse_depth|
+      list_pairs(pattern, src_root, dest_root) do |a, b|
         found_result = true
-        copy_leaves(src_leaf, dest_leaf, child_recurse_depth, options)
+        copy_entries(a, b, recurse_depth, options)
       end
       if !found_result && pattern.exact_path
-        yield "#{pattern}: No such file or directory on remote or local"
+        puts "#{pattern}: No such file or directory on remote or local"
       end
     end
 
+    # Yield entries for children that are in either +a_root+ or +b_root+, with
+    # matching pairs matched up.
+    #
+    # ==== Yields
+    #
+    # Yields matching entries in pairs:
+    #
+    #    [ a_entry, b_entry ]
+    #
+    # ==== Example
+    #
+    #     ChefFS::FileSystem.list_pairs(FilePattern.new('**x.txt', a_root, b_root)) do |a, b|
+    #       ...
+    #     end
+    #
+    def self.list_pairs(pattern, a_root, b_root)
+      # Make sure everything on the server is also on the filesystem, and diff
+      found_paths = Set.new
+      ChefFS::FileSystem.list(a_root, pattern) do |a|
+        found_paths << a.path
+        b = ChefFS::FileSystem.resolve_path(b_root, a.path)
+        yield [ a, b ]
+      end
+
+      # Check the outer regex pattern to see if it matches anything on the
+      # filesystem that isn't on the server
+      ChefFS::FileSystem.list(b_root, pattern) do |b|
+        if !found_paths.include?(b.path)
+          a = ChefFS::FileSystem.resolve_path(a_root, b.path)
+          yield [ a, b ]
+        end
+      end
+    end
+
+    # Get entries for children of either a or b, with matching pairs matched up.
+    #
+    # ==== Returns
+    #
+    # An array of child pairs.
+    #
+    #     [ [ a_child, b_child ], ... ]
+    #
+    # If a child is only in a or only in b, the other child entry will be
+    # retrieved by name (and will most likely be a "nonexistent child").
+    #
+    # ==== Example
+    #
+    #     ChefFS::FileSystem.child_pairs(a, b).length
+    #
+    def self.child_pairs(a, b)
+      # If both are directories, recurse into them and diff the children instead of returning ourselves.
+      result = []
+      a_children_names = Set.new
+      a.children.each do |a_child|
+        a_children_names << a_child.name
+        result << [ a_child, b.child(a_child.name) ]
+      end
+
+      # Check b for children that aren't in a
+      b.children.each do |b_child|
+        if !a_children_names.include?(b_child.name)
+          result << [ a.child(b_child.name), b_child ]
+        end
+      end
+      result
+    end
+
+    def self.compare(a, b)
+      are_same, a_value, b_value = a.compare_to(b)
+      if are_same.nil?
+        are_same, b_value, a_value = b.compare_to(a)
+      end
+      if are_same.nil?
+        begin
+          a_value = a.read if a_value.nil?
+        rescue ChefFS::FileSystem::NotFoundError
+          a_value = :none
+        end
+        begin
+          b_value = b.read if b_value.nil?
+        rescue ChefFS::FileSystem::NotFoundError
+          b_value = :none
+        end
+        are_same = (a_value == b_value)
+      end
+      [ are_same, a_value, b_value ]
+    end
+
     private
 
-    # Copy two known leaves (could be files or dirs)
-    def self.copy_leaves(src_entry, dest_entry, recurse_depth, options)
+    # Copy two entries (could be files or dirs)
+    def self.copy_entries(src_entry, dest_entry, recurse_depth, options)
       # A NOTE about this algorithm:
       # There are cases where this algorithm does too many network requests.
       # knife upload with a specific filename will first check if the file
@@ -148,7 +237,7 @@ module ChefFS
             if recurse_depth != 0
               src_entry.children.each do |src_child|
                 new_dest_child = new_dest_dir.child(src_child.name)
-                copy_leaves(src_child, new_dest_child, recurse_depth ? recurse_depth - 1 : recurse_depth, options)
+                copy_entries(src_child, new_dest_child, recurse_depth ? recurse_depth - 1 : recurse_depth, options)
               end
             end
           else
@@ -163,10 +252,30 @@ module ChefFS
 
       else
         # Both exist.
+
+        # If the entry can do a copy directly, do that.
+        if dest_entry.respond_to?(:copy_from)
+          if options[:force] || compare(src_entry, dest_entry)[0] == false
+            if options[:dry_run]
+              puts "Would update #{dest_entry.path_for_printing}"
+            else
+              puts "Updating #{dest_entry.path_for_printing} ..."
+              dest_entry.copy_from(src_entry)
+              puts "Updated #{dest_entry.path_for_printing}"
+            end
+          end
+          return
+        end
+
         # If they are different types, log an error.
         if src_entry.dir?
           if dest_entry.dir?
-            # If they are both directories, we'll end up recursing later.
+            # If both are directories, recurse into their children
+            if recurse_depth != 0
+              child_pairs(src_entry, dest_entry).each do |src_child, dest_child|
+                copy_entries(src_child, dest_child, recurse_depth ? recurse_depth - 1 : recurse_depth, options)
+              end
+            end
           else
             # If they are different types.
             Chef::Log.error("File #{dest_entry.path_for_printing} is a directory while file #{dest_entry.path_for_printing} is a regular file\n")
@@ -180,19 +289,16 @@ module ChefFS
             # Both are files!  Copy them unless we're sure they are the same.
             if options[:force]
               should_copy = true
-              src_value = src_entry.read
+              src_value = nil
             else
-              should_copy, src_value, dest_value = ChefFS::Diff.diff_files(src_entry, dest_entry)
-              src_value = src_entry.read if src_value == :not_retrieved
-              if should_copy == nil
-                should_copy = true
-              end
+              are_same, src_value, dest_value = compare(src_entry, dest_entry)
+              should_copy = !are_same
             end
             if should_copy
               if options[:dry_run]
                 puts "Would update #{dest_entry.path_for_printing}"
               else
-                src_value = src_entry.read if src_value == :not_retrieved
+                src_value = src_entry.read if src_value.nil?
                 dest_entry.write(src_value)
                 puts "Updated #{dest_entry.path_for_printing}"
               end