knife_cookbook_sync 0.0.1 → 0.0.2

data/CHANGELOG.md

@@ -0,0 +1,11 @@
+0.0.2
+=====
+* honor knife.rb configuration of cookbook path
+* add a quiet option (-q). dry run is unaffected.
+* check if the cookbook exists before loading it. vastly improves upload performance for new cookbooks.
+* exploit recent threaded upload improvements in Chef API: 0.1s slower than knife cookbook upload for all new content (worst case for sync).
+
+0.0.1
+=====
+
+* First public release.

data/README.md

@@ -7,7 +7,7 @@ production work with careful coordination with an external cookbook resolver.
 
 Here's the meat. `knife cookbook sync` vs `knife cookbook upload` with a
 pre-uploaded corpus of 39 cookbooks, using the standard unix `time` utility to
-benchmark on MRI 1.9.3-p327:
+benchmark on MRI 1.9.3-p327 and chef 10.16.2:
 
 * `knife cookbook sync -a`: 1.31s user 0.15s system 72% cpu **2.020 total**
 * `knife cookbook upload -a`: 1.34s user 0.15s system 15% cpu **9.684 total**
@@ -18,7 +18,7 @@ to be uploaded, and only uploads what's different.
 
 This means it **does not check versions and dependencies**. It cheats, so you
 should be sure you have your ducks in a row before uploading by using a
-cookbook resolver. This only matters for resolution purposes -- cookbooks
+cookbook resolver. This only matters for determining what to upload -- cookbooks
 uploaded with `knife cookbook sync` are no different otherwise (and in fact use
 chef's own cookbook uploading tooling to do it).
 

data/lib/chef/knife/cookbook_sync.rb

@@ -1,7 +1,5 @@
 #!/usr/bin/env ruby
 
-Thread.abort_on_exception = true
-
 module Knife
   class CookbookSync < Chef::Knife
 
@@ -12,6 +10,7 @@ module Knife
       require 'chef/cookbook_loader'
       require 'chef/cookbook_uploader'
       require 'chef/cookbook_version'
+      require 'chef/log'
     end
 
     banner "knife cookbook sync [COOKBOOKS...]"
@@ -31,36 +30,53 @@ module Knife
     option :cookbook_path,
       :short       => '-o [COOKBOOK PATH]',
       :long        => '--cookbook-path [COOKBOOK PATH]',
-      :default     => %w[cookbooks site-cookbooks],
       :description => "The path that cookbooks should be loaded from (path:path)",
       :proc        => proc { |x| x.split(":") }
-      
+
+    option :quiet,
+      :short        => '-q',
+      :long         => '--quiet',
+      :description  => "Make less noise",
+      :default      => false
+
+    def make_noise(&block)
+      force_make_noise(&block) if block and !config[:quiet]
+    end
+
+    def force_make_noise(&block)
+      @print_mutex.synchronize(&block) if block
+    end
 
     def distill_manifest(cookbook)
       files = { }
       cookbook.manifest.values.select { |x| x.kind_of?(Array) }.flatten.each { |f| files[f['path']] = f['checksum'] }
       # don't check metadata.json since json output is indeterministic, metadata.rb should be all that's needed anw
-      files.delete('metadata.json') 
+      files.delete('metadata.json')
       return files
     end
 
     def sync_cookbooks(cookbooks, cl)
-      uploaded = false 
+      log_level = Chef::Log.level
+
+      # mutes the CookbookVersion noise when the cookbook doesn't exist on the server.
+      Chef::Log.level = :fatal
 
-      print_mutex = Mutex.new
+      to_upload = Queue.new
 
-      cookbooks.each do |cookbook|
+      cookbooks.map(&:to_s).each do |cookbook|
         Thread.new do
-          upload = false 
-          print_mutex.synchronize do
+          upload = false
+
+          make_noise do
             ui.msg "Checking cookbook '#{cookbook}' for sync necessity"
           end
 
-          remote_cookbook = Chef::CookbookVersion.load(cookbook.to_s)
+          remote_cookbook = Chef::CookbookVersion.available_versions(cookbook) &&
+            (Chef::CookbookVersion.load(cookbook.to_s) rescue nil)
           local_cookbook = cl[cookbook.to_s] rescue nil
 
           unless local_cookbook
-            print_mutex.synchronize do
+            make_noise do
               ui.fatal "Cookbook '#{cookbook}' does not exist locally."
             end
             exit 1
@@ -85,57 +101,33 @@ module Knife
           end
 
           if upload
-            print_mutex.synchronize do
+            make_noise do
               ui.msg "sync necessary; uploading '#{cookbook}'"
             end
 
-            retries_left = 5
-
-            begin
-              #
-              # XXX 
-              #
-              # For some godawful reason, if we use the local_cookbook referenced
-              # above, the MD5 sums are off.
-              #
-              # So we reload the cookbooks here because it seems to work, with an
-              # optional retry if the chef server is being pissy. 
-              #
-              cl = Chef::CookbookLoader.new(Chef::Config[:cookbook_path])
-
-              if config[:dry_run]
-                print_mutex.synchronize do
-                  ui.warn "dry run: would sync '#{cookbook}'"
-                end
-              else
-                Chef::CookbookUploader.new(cl[cookbook], Chef::Config[:cookbook_path]).upload_cookbooks
-              end
-
-              uploaded = true
-            rescue Exception => e
-              print_mutex.synchronize do
-                ui.error "Failed to upload; retrying up to #{retries_left} times"
-              end
-
-              retries_left -= 1
-              if retries_left > 0
-                retry
-              else
-                raise e
-              end
-            end
+            to_upload << cl[cookbook]
           end
         end
       end
 
-      Thread.list.reject { |x| x == Thread.current }.each(&:join)
+      Thread.list.reject { |x| x == Thread.current }.each(&:join) # wait for threads to settle
+
+      cookbooks_to_upload = []
+      loop { cookbooks_to_upload << to_upload.shift(true) } rescue nil
+
+      Chef::Log.level = log_level # restore log level now that we're done checking
+      Chef::CookbookUploader.new(cookbooks_to_upload, Chef::Config[:cookbook_path]).upload_cookbooks
 
       # exit with an exit status of 5 if we've uploaded anything.
-      exit uploaded ? 5 : 0
+      exit cookbooks_to_upload.empty? ? 0 : 5
     end
 
     def run
-      Chef::Config[:cookbook_path] = config[:cookbook_path]
+      Thread.abort_on_exception = true
+
+      @print_mutex = Mutex.new
+
+      Chef::Config[:cookbook_path] = config[:cookbook_path] if config[:cookbook_path]
 
       Chef::Cookbook::FileVendor.on_create { |manifest| Chef::Cookbook::FileSystemFileVendor.new(manifest) }
 

data/lib/knife_cookbook_sync/version.rb

@@ -1,3 +1,3 @@
 module KnifeCookbookSync
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: knife_cookbook_sync
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-09 00:00:00.000000000 Z
+date: 2013-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chef
@@ -35,6 +35,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md