knife-cookbook-sync 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    YzM3ZTUyYjk4MjBjZDRiY2ZlMzYzNGZjOTU1YzYwNTk2MmI0YTI0MA==
+  data.tar.gz: !binary |-
+    YjkwNzQ4ODFhNzc5MzNmN2EyYTEwYWI0ZjM3NTVkMGEwMzdkM2Q4MQ==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    NmJjNDAwOTJkMzY2ZTQxNzgzMmUxNTM5ZTYyMGZmYWUzNjAzNjQzOTIxMTAy
+    ZWE3YWEyMDE4OTdjZmRjYTM0N2NiN2YzMmU1MDdhYTNhNWU1YmRlMzhiMGRi
+    MTU5MTIzOGM0ZjhkMWJhNDUzODA2NDRlZWZhZTdkMTZkZTU0Zjg=
+  data.tar.gz: !binary |-
+    YTVmNTA3MWI4MDRiOTI0NTEwNjJjOTIyNmY3ZmMwZTExYTlkZDNkNGQxZjY4
+    NjVmMjZkOTRkNTdhNmE1NzRhMDVkNjZiMTY4NTkyZjBiNjgyOTRmOGEwNTEx
+    ZjRjMTUxZTUzYWE5OWM3ZWJmZTU4NzE3NTZhMDhmY2RhYzM5M2M=

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/CHANGELOG.md

@@ -0,0 +1,36 @@
+0.1.0 (04/13/2013)
+==================
+
+* Gem has been renamed -- is now `knife-cookbook-sync`.
+* Chef 11 now works, Chef 10 still works.
+* Clarified exit statuses for various behavior.
+
+0.0.5
+=====
+
+* Deal with resolver bad behavior in a more appropriate way (abort).
+
+0.0.4
+=====
+
+* Deal with cookbook sync's exit statuses a little more clearly in the
+  chef-workflow task.
+
+0.0.3
+=====
+
+* Add chef-workflow support
+
+0.0.2
+=====
+* honor knife.rb configuration of cookbook path
+* add a quiet option (-q). dry run is unaffected.
+* check if the cookbook exists before loading it. vastly improves upload
+  performance for new cookbooks.
+* exploit recent threaded upload improvements in Chef API: 0.1s slower than
+  knife cookbook upload for all new content (worst case for sync).
+
+0.0.1
+=====
+
+* First public release.

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in knife_cookbook_sync.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Erik Hollensbe
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,102 @@
+# knife cookbook sync
+
+*Note*: this gem has been renamed to be more conformant with knife plugin
+naming on rubygems.org. The original gem was named
+[knife_cookbook_sync](https://rubygems.org/gems/knife_cookbook_sync).
+
+Sync your cookbooks faster than `knife cookbook upload` or alternatives.
+
+`knife cookbook sync` is primarily a development tool, but can be used for
+production work with careful coordination with an external cookbook resolver.
+
+Here's the meat. `knife cookbook sync` vs `knife cookbook upload` with a
+pre-uploaded corpus of 39 cookbooks, using the standard unix `time` utility to
+benchmark on MRI 1.9.3-p327 and chef 10.16.2:
+
+* `knife cookbook sync -a`: 1.31s user 0.15s system 72% cpu **2.020 total**
+* `knife cookbook upload -a`: 1.34s user 0.15s system 15% cpu **9.684 total**
+
+Instead of resolving and uploading everything (or even what you ask to upload),
+it uses the cryptographic sums chef already generates to determine what needs
+to be uploaded, and only uploads what's different.
+
+This means it **does not check versions and dependencies**. It cheats, so you
+should be sure you have your ducks in a row before uploading by using a
+cookbook resolver. This only matters for determining what to upload -- cookbooks
+uploaded with `knife cookbook sync` are no different otherwise (and in fact use
+chef's own cookbook uploading tooling to do it).
+
+Unsurprisingly, the more that has changed, the more the performance will
+decrease slowly towards `knife cookbook upload` performance. The gains are
+really only seen when you need to sync whole repositories where most or all of
+the product that's on-disk has already been uploaded. It's particularly nice
+for fast test cycles where you just don't want to care just yet what cookbooks
+have changed. However, you can use it for all uploading without any real issue.
+
+`knife cookbook sync` has no dependencies other than chef, and is compatible
+with both chef 10.x and 11.x. 
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'knife-cookbook-sync'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install knife-cookbook-sync
+
+## Usage
+
+`knife cookbook sync` takes a list of cookbooks or `-a` to upload everything.
+It uses your `cookbook_path` `knife.rb` settings to determine what's available
+to upload which is overridable by `-o`.
+
+If you pass `-d`, it'll perform a "dry run" and just show you what it would
+upload.
+
+See "Exit Statuses" below for information on how `knife cookbook sync` can let
+you know what it did (you know, for scripting yo).
+
+For more information, use `knife cookbook sync --help`.
+
+## Exit Statuses
+
+The following exit codes are used in various situations:
+
+* 0: nothing went wrong, but we did not see anything different. Works in dry run and sync mode.
+* 1: Something is not right -- usually this means you have a broken
+  `cookbook_path` or did not supply `-o`. Whereever Chef is sending its UI
+  output will have the information you seek (usually Standard Error).
+* 5: nothing went wrong, but there are differences between what's in your
+  `cookbook_path` and the chef server. Works in dry run and sync mode.
+
+## Chef-Workflow support
+
+We support [chef-workflow](https://github.com/chef-workflow/chef-workflow) by
+way of a task you can use.
+
+Add the 'knife-cookbook-sync' gem to your `Gemfile` (version `0.1.0` or later),
+and this to your `Rakefile`:
+
+```ruby
+chef_workflow_task 'chef/cookbooks/sync'
+```
+
+And you'll have a `chef:cookbooks:sync` rake target you can use.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Authors
+
+* Erik Hollensbe <erik+github@hollensbe.org>

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/knife-cookbook-sync.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'knife-cookbook-sync/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "knife-cookbook-sync"
+  gem.version       = KnifeCookbookSync::VERSION
+  gem.authors       = ["Erik Hollensbe"]
+  gem.email         = ["erik+github@hollensbe.org"]
+  gem.description   = %q{Sync only what's changed -- faster than cookbook upload}
+  gem.summary       = %q{Sync only what's changed -- faster than cookbook upload}
+  gem.homepage      = "https://github.com/erikh/knife-cookbook-sync"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency 'chef'
+end

data/lib/chef-workflow/tasks/chef/cookbooks/sync.rb

@@ -0,0 +1,15 @@
+require 'knife/dsl'
+require 'chef-workflow/tasks/bootstrap/knife'
+
+namespace :chef do
+  namespace :cookbooks do
+    desc "Upload your cookbooks to the chef server"
+    task :sync => [ "bootstrap:knife" ] do
+      resolve_task = Rake::Task["chef:cookbooks:resolve"] rescue nil
+      resolve_task.invoke if resolve_task
+
+      result = knife %W[cookbook sync -a]
+      fail unless [0, 5].include?(result)
+    end
+  end
+end

data/lib/chef/knife/cookbook_sync.rb

@@ -0,0 +1,169 @@
+#!/usr/bin/env ruby
+
+module Knife
+  class CookbookSync < Chef::Knife
+
+    deps do
+      require 'chef/knife/cookbook_metadata'
+      require 'chef/rest'
+
+      begin
+        require 'chef/checksum_cache'
+      rescue LoadError
+      end
+
+      require 'chef/cookbook_loader'
+      require 'chef/cookbook_uploader'
+      require 'chef/cookbook_version'
+      require 'chef/log'
+    end
+
+    banner "knife cookbook sync [COOKBOOKS...]"
+
+    option :dry_run,
+      :short       => '-d',
+      :long        => '--dry-run',
+      :boolean     => true,
+      :description => "Do a dry run -- do not sync anything, just show what would be synced"
+
+    option :all,
+      :short       => '-a',
+      :long        => '--all',
+      :boolean     => true,
+      :description => "Sync all cookbooks"
+
+    option :cookbook_path,
+      :short       => '-o [COOKBOOK PATH]',
+      :long        => '--cookbook-path [COOKBOOK PATH]',
+      :description => "The path that cookbooks should be loaded from (path:path)",
+      :proc        => proc { |x| x.split(":") }
+
+    option :quiet,
+      :short        => '-q',
+      :long         => '--quiet',
+      :description  => "Make less noise",
+      :default      => false
+
+    def make_noise(&block)
+      force_make_noise(&block) if block and !config[:quiet]
+    end
+
+    def force_make_noise(&block)
+      @print_mutex.synchronize(&block) if block
+    end
+
+    def distill_manifest(cookbook)
+      files = { }
+      cookbook.manifest.values.select { |x| x.kind_of?(Array) }.flatten.each { |f| files[f['path']] = f['checksum'] }
+      # don't check metadata.json since json output is indeterministic, metadata.rb should be all that's needed anw
+      files.delete('metadata.json')
+      return files
+    end
+
+    def sync_cookbooks(cookbooks, cl)
+      log_level = Chef::Log.level
+
+      # mutes the CookbookVersion noise when the cookbook doesn't exist on the server.
+      Chef::Log.level = :fatal
+
+      to_upload = Queue.new
+
+      cookbooks.map(&:to_s).map do |cookbook|
+        Thread.new do
+          upload = false
+
+          make_noise do
+            ui.msg "Checking cookbook '#{cookbook}' for sync necessity"
+          end
+
+          remote_cookbook = Chef::CookbookVersion.available_versions(cookbook) &&
+            (Chef::CookbookVersion.load(cookbook.to_s) rescue nil)
+          local_cookbook = cl[cookbook.to_s] rescue nil
+
+          unless local_cookbook
+            make_noise do
+              ui.fatal "Cookbook '#{cookbook}' does not exist locally."
+            end
+            exit 1
+          end
+
+          if local_cookbook and !remote_cookbook
+            upload = true
+          else
+            remote_files = distill_manifest(remote_cookbook) rescue { }
+            local_files  = distill_manifest(local_cookbook) rescue { }
+
+            if local_files.keys.length != remote_files.keys.length
+              upload = true
+            else
+              (local_files.keys + remote_files.keys).uniq.each do |filename|
+                if local_files[filename] != remote_files[filename]
+                  upload = true
+                  break
+                end
+              end
+            end
+          end
+
+          if upload
+            make_noise do
+              ui.msg "sync necessary; uploading '#{cookbook}'"
+            end
+
+            to_upload << cl[cookbook]
+          end
+        end
+      end.each(&:join)
+
+      cookbooks_to_upload = []
+      loop { cookbooks_to_upload << to_upload.shift(true) } rescue nil
+
+      # exit 0 if there's nothing to upload
+      if cookbooks_to_upload.empty?
+        exit 0
+      end
+
+      Chef::Log.level = log_level # restore log level now that we're done checking
+      Chef::CookbookUploader.new(cookbooks_to_upload, Chef::Config[:cookbook_path]).upload_cookbooks
+
+      # exit with an exit status of 5 if we've uploaded anything.
+      exit 5
+    end
+
+    def run
+      Thread.abort_on_exception = true
+
+      @print_mutex = Mutex.new
+
+      Chef::Config[:cookbook_path] = config[:cookbook_path] if config[:cookbook_path]
+
+      if !Chef::Config[:cookbook_path] or Chef::Config[:cookbook_path].empty?
+        ui.msg "No cookbook path in config file or provided on command-line. Aborting."
+        exit 1
+      end
+
+      Chef::Cookbook::FileVendor.on_create { |manifest| Chef::Cookbook::FileSystemFileVendor.new(manifest) }
+
+      cl = Chef::CookbookLoader.new(Chef::Config[:cookbook_path])
+      cl.load_cookbooks if cl.respond_to?(:load_cookbooks)
+
+      if config[:all]
+        if cl.respond_to?(:cookbook_names)
+          names = cl.cookbook_names
+        else
+          names = cl.cookbooks.map(&:name)
+        end
+
+        if !names or names.empty?
+          ui.msg "No cookbooks found to upload in cookbook path: #{Chef::Config[:cookbook_path].inspect} -- not continuing"
+          ui.msg "Set cookbook_path in knife.rb or pass the -o option."
+          exit 1
+        end
+
+        sync_cookbooks names, cl
+      else
+        sync_cookbooks name_args, cl
+      end
+    end
+  end
+end

data/lib/knife-cookbook-sync.rb

@@ -0,0 +1,4 @@
+require "knife-cookbook-sync/version"
+
+module KnifeCookbookSync
+end

data/lib/knife-cookbook-sync/version.rb

@@ -0,0 +1,3 @@
+module KnifeCookbookSync
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification
+name: knife-cookbook-sync
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Erik Hollensbe
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: chef
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Sync only what's changed -- faster than cookbook upload
+email:
+- erik+github@hollensbe.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- knife-cookbook-sync.gemspec
+- lib/chef-workflow/tasks/chef/cookbooks/sync.rb
+- lib/chef/knife/cookbook_sync.rb
+- lib/knife-cookbook-sync.rb
+- lib/knife-cookbook-sync/version.rb
+homepage: https://github.com/erikh/knife-cookbook-sync
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Sync only what's changed -- faster than cookbook upload
+test_files: []