knife_cookbook_sync 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in knife_cookbook_sync.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Erik Hollensbe
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,71 @@
+# knife cookbook sync
+
+Sync your cookbooks faster than `knife cookbook upload` or alternatives.
+
+`knife cookbook sync` is primarily a development tool, but can be used for
+production work with careful coordination with an external cookbook resolver.
+
+Here's the meat. `knife cookbook sync` vs `knife cookbook upload` with a
+pre-uploaded corpus of 39 cookbooks, using the standard unix `time` utility to
+benchmark on MRI 1.9.3-p327:
+
+* `knife cookbook sync -a`: 1.31s user 0.15s system 72% cpu **2.020 total**
+* `knife cookbook upload -a`: 1.34s user 0.15s system 15% cpu **9.684 total**
+
+Instead of resolving and uploading everything (or even what you ask to upload),
+it uses the cryptographic sums chef already generates to determine what needs
+to be uploaded, and only uploads what's different.
+
+This means it **does not check versions and dependencies**. It cheats, so you
+should be sure you have your ducks in a row before uploading by using a
+cookbook resolver. This only matters for resolution purposes -- cookbooks
+uploaded with `knife cookbook sync` are no different otherwise (and in fact use
+chef's own cookbook uploading tooling to do it).
+
+Unsurprisingly, the more that has changed, the more the performance will
+decrease slowly towards `knife cookbook upload` performance. The gains are
+really only seen when you need to sync whole repositories where most or all of
+the product that's on-disk has already been uploaded. It's particularly nice
+for fast test cycles where you just don't want to care just yet what cookbooks
+have changed.
+
+`knife cookbook sync` has no dependencies other than chef, and should be
+forward compatible with all versions of chef in the 10.x series, and probably
+beyond.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'knife_cookbook_sync'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install knife_cookbook_sync
+
+## Usage
+
+`knife cookbook sync` takes a list of cookbooks or `-a` to upload everything.
+It uses your `cookbook_path` `knife.rb` settings to determine what's available
+to upload which is overridable by `-o`.
+
+If you pass `-d`, it'll perform a "dry run" and just show you what it would
+upload.
+
+For more information, use `knife cookbook sync --help`.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Authors
+
+* Erik Hollensbe <erik+github@hollensbe.org>

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/knife_cookbook_sync.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'knife_cookbook_sync/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "knife_cookbook_sync"
+  gem.version       = KnifeCookbookSync::VERSION
+  gem.authors       = ["Erik Hollensbe"]
+  gem.email         = ["erik+github@hollensbe.org"]
+  gem.description   = %q{Sync only what's changed -- faster than cookbook upload}
+  gem.summary       = %q{Sync only what's changed -- faster than cookbook upload}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency 'chef', '~> 10.0'
+end

data/lib/chef/knife/cookbook_sync.rb

@@ -0,0 +1,151 @@
+#!/usr/bin/env ruby
+
+Thread.abort_on_exception = true
+
+module Knife
+  class CookbookSync < Chef::Knife
+
+    deps do
+      require 'chef/knife/cookbook_metadata'
+      require 'chef/rest'
+      require 'chef/checksum_cache'
+      require 'chef/cookbook_loader'
+      require 'chef/cookbook_uploader'
+      require 'chef/cookbook_version'
+    end
+
+    banner "knife cookbook sync [COOKBOOKS...]"
+
+    option :dry_run,
+      :short       => '-d',
+      :long        => '--dry-run',
+      :boolean     => true,
+      :description => "Do a dry run -- do not sync anything, just show what would be synced"
+
+    option :all,
+      :short       => '-a',
+      :long        => '--all',
+      :boolean     => true,
+      :description => "Sync all cookbooks"
+
+    option :cookbook_path,
+      :short       => '-o [COOKBOOK PATH]',
+      :long        => '--cookbook-path [COOKBOOK PATH]',
+      :default     => %w[cookbooks site-cookbooks],
+      :description => "The path that cookbooks should be loaded from (path:path)",
+      :proc        => proc { |x| x.split(":") }
+      
+
+    def distill_manifest(cookbook)
+      files = { }
+      cookbook.manifest.values.select { |x| x.kind_of?(Array) }.flatten.each { |f| files[f['path']] = f['checksum'] }
+      # don't check metadata.json since json output is indeterministic, metadata.rb should be all that's needed anw
+      files.delete('metadata.json') 
+      return files
+    end
+
+    def sync_cookbooks(cookbooks, cl)
+      uploaded = false 
+
+      print_mutex = Mutex.new
+
+      cookbooks.each do |cookbook|
+        Thread.new do
+          upload = false 
+          print_mutex.synchronize do
+            ui.msg "Checking cookbook '#{cookbook}' for sync necessity"
+          end
+
+          remote_cookbook = Chef::CookbookVersion.load(cookbook.to_s)
+          local_cookbook = cl[cookbook.to_s] rescue nil
+
+          unless local_cookbook
+            print_mutex.synchronize do
+              ui.fatal "Cookbook '#{cookbook}' does not exist locally."
+            end
+            exit 1
+          end
+
+          if local_cookbook and !remote_cookbook
+            upload = true
+          else
+            remote_files = distill_manifest(remote_cookbook) rescue { }
+            local_files  = distill_manifest(local_cookbook) rescue { }
+
+            if local_files.keys.length != remote_files.keys.length
+              upload = true
+            else
+              (local_files.keys + remote_files.keys).uniq.each do |filename|
+                if local_files[filename] != remote_files[filename]
+                  upload = true
+                  break
+                end
+              end
+            end
+          end
+
+          if upload
+            print_mutex.synchronize do
+              ui.msg "sync necessary; uploading '#{cookbook}'"
+            end
+
+            retries_left = 5
+
+            begin
+              #
+              # XXX 
+              #
+              # For some godawful reason, if we use the local_cookbook referenced
+              # above, the MD5 sums are off.
+              #
+              # So we reload the cookbooks here because it seems to work, with an
+              # optional retry if the chef server is being pissy. 
+              #
+              cl = Chef::CookbookLoader.new(Chef::Config[:cookbook_path])
+
+              if config[:dry_run]
+                print_mutex.synchronize do
+                  ui.warn "dry run: would sync '#{cookbook}'"
+                end
+              else
+                Chef::CookbookUploader.new(cl[cookbook], Chef::Config[:cookbook_path]).upload_cookbooks
+              end
+
+              uploaded = true
+            rescue Exception => e
+              print_mutex.synchronize do
+                ui.error "Failed to upload; retrying up to #{retries_left} times"
+              end
+
+              retries_left -= 1
+              if retries_left > 0
+                retry
+              else
+                raise e
+              end
+            end
+          end
+        end
+      end
+
+      Thread.list.reject { |x| x == Thread.current }.each(&:join)
+
+      # exit with an exit status of 5 if we've uploaded anything.
+      exit uploaded ? 5 : 0
+    end
+
+    def run
+      Chef::Config[:cookbook_path] = config[:cookbook_path]
+
+      Chef::Cookbook::FileVendor.on_create { |manifest| Chef::Cookbook::FileSystemFileVendor.new(manifest) }
+
+      cl = Chef::CookbookLoader.new(Chef::Config[:cookbook_path])
+
+      if config[:all]
+        sync_cookbooks cl.cookbooks.map(&:name), cl
+      else
+        sync_cookbooks name_args, cl
+      end
+    end
+  end
+end

data/lib/knife_cookbook_sync.rb

@@ -0,0 +1,4 @@
+require "knife_cookbook_sync/version"
+
+module KnifeCookbookSync
+end

data/lib/knife_cookbook_sync/version.rb

@@ -0,0 +1,3 @@
+module KnifeCookbookSync
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: knife_cookbook_sync
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Erik Hollensbe
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-12-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: chef
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Sync only what's changed -- faster than cookbook upload
+email:
+- erik+github@hollensbe.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- knife_cookbook_sync.gemspec
+- lib/chef/knife/cookbook_sync.rb
+- lib/knife_cookbook_sync.rb
+- lib/knife_cookbook_sync/version.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Sync only what's changed -- faster than cookbook upload
+test_files: []
+has_rdoc: