chefdepartie 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ffda5a4e9b1258d25ba59354dbdb73148af675b4
+  data.tar.gz: ad14a0b279ec6541be45be51ef4fbbbd350c6367
+SHA512:
+  metadata.gz: ff88fbed52f7d75d518a8b1924bf9c4b179bd0d67ee110dcc778d21040658f3c5d5b4663df905c05c883d260ad04eb207c341670ba0a535b9770b49966200323
+  data.tar.gz: abf7fa5b9c495221ae3425e7150716ef8f93120eb7a0164772f8294d270f9f972066143798892a481dd25fa79ec0eb72cae2eb3be56acee4da992f104b177325

data/.gitignore

@@ -0,0 +1,2 @@
+cookbooks
+*.gem

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :test do
+  gem 'simplecov', '=0.10.0'
+end
+

data/Gemfile.lock

@@ -0,0 +1,142 @@
+PATH
+  remote: .
+  specs:
+    chefdepartie (0.0.3)
+      chef (~> 12.3.0)
+      chef-zero (~> 4.2.2)
+      librarian-chef (~> 0.0.4)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    builder (3.2.2)
+    chef (12.3.0)
+      chef-zero (~> 4.1)
+      diff-lcs (~> 1.2, >= 1.2.4)
+      erubis (~> 2.7)
+      ffi-yajl (>= 1.2, < 3.0)
+      highline (~> 1.6, >= 1.6.9)
+      mixlib-authentication (~> 1.3)
+      mixlib-cli (~> 1.4)
+      mixlib-config (~> 2.0)
+      mixlib-log (~> 1.3)
+      mixlib-shellout (>= 2.0.0.rc.0, < 3.0)
+      net-ssh (~> 2.6)
+      net-ssh-multi (~> 1.1)
+      ohai (~> 8.0)
+      plist (~> 3.1.0)
+      pry (~> 0.9)
+      rspec-core (~> 3.2)
+      rspec-expectations (~> 3.2)
+      rspec-mocks (~> 3.2)
+      rspec_junit_formatter (~> 0.2.0)
+      serverspec (~> 2.7)
+      specinfra (~> 2.10)
+    chef-zero (4.2.2)
+      ffi-yajl (>= 1.1, < 3.0)
+      hashie (~> 2.0)
+      mixlib-log (~> 1.3)
+      rack
+      uuidtools (~> 2.1)
+    coderay (1.1.0)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    erubis (2.7.0)
+    ffi (1.9.8)
+    ffi-yajl (2.2.0)
+      libyajl2 (~> 1.2)
+    hashie (2.1.2)
+    highline (1.7.2)
+    ipaddress (0.8.0)
+    json (1.8.2)
+    librarian (0.1.2)
+      highline
+      thor (~> 0.15)
+    librarian-chef (0.0.4)
+      chef (>= 0.10)
+      librarian (~> 0.1.0)
+      minitar (>= 0.5.2)
+    libyajl2 (1.2.0)
+    method_source (0.8.2)
+    mime-types (2.6.1)
+    minitar (0.5.4)
+    minitest (5.6.1)
+    mixlib-authentication (1.3.0)
+      mixlib-log
+    mixlib-cli (1.5.0)
+    mixlib-config (2.2.1)
+    mixlib-log (1.6.0)
+    mixlib-shellout (2.1.0)
+    multi_json (1.11.1)
+    net-scp (1.2.1)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.9.2)
+    net-ssh-gateway (1.2.0)
+      net-ssh (>= 2.6.5)
+    net-ssh-multi (1.2.1)
+      net-ssh (>= 2.6.5)
+      net-ssh-gateway (>= 1.2.0)
+    ohai (8.4.0)
+      ffi (~> 1.9)
+      ffi-yajl (>= 1.1, < 3.0)
+      ipaddress
+      mime-types (~> 2.0)
+      mixlib-cli
+      mixlib-config (~> 2.0)
+      mixlib-log
+      mixlib-shellout (~> 2.0)
+      rake (~> 10.1)
+      systemu (~> 2.6.4)
+      wmi-lite (~> 1.0)
+    plist (3.1.0)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rack (1.6.2)
+    rake (10.4.2)
+    rspec (3.3.0)
+      rspec-core (~> 3.3.0)
+      rspec-expectations (~> 3.3.0)
+      rspec-mocks (~> 3.3.0)
+    rspec-core (3.3.0)
+      rspec-support (~> 3.3.0)
+    rspec-expectations (3.3.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-its (1.2.0)
+      rspec-core (>= 3.0.0)
+      rspec-expectations (>= 3.0.0)
+    rspec-mocks (3.3.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-support (3.3.0)
+    rspec_junit_formatter (0.2.3)
+      builder (< 4)
+      rspec-core (>= 2, < 4, != 2.12.0)
+    serverspec (2.18.0)
+      multi_json
+      rspec (~> 3.0)
+      rspec-its
+      specinfra (~> 2.35)
+    simplecov (0.10.0)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    slop (3.6.0)
+    specinfra (2.35.1)
+      net-scp
+      net-ssh
+    systemu (2.6.5)
+    thor (0.19.1)
+    uuidtools (2.1.5)
+    wmi-lite (1.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  chefdepartie!
+  minitest (~> 5.6)
+  simplecov (= 0.10.0)

data/README.md

@@ -0,0 +1,56 @@
+# A quick little helper
+
+In a kitchen, a chef de partie is a line cook who helps prepare the ingredients the real chef will use.
+
+Chefdepartie is a wrapper around a chef-zero server, allowing you to easily upload all of your:
+
+* Cookbooks (site)
+* Librarian cookbooks
+* Roles
+* Data bags
+
+This allows you to test out all of your cookbooks together without putting them on your actual chef server.
+
+## Chefdepartie vs chef-solo
+
+Since chef-zero provides the same interface as **a real chef server**, you can actually use knife with it!
+
+This gives you a full chef sandbox, and can be useful for things like CI or packer image provisioning.
+
+* View and edit data bags, without an internet connection
+* Bootstrap test nodes (vagrant images, cloud servers, etc)
+
+## Limitations
+
+Since chefdepartie is just a wrapper around chef-zero, it runs entirely in memory and has no persistence. This makes it a useful sandbox, but it shouldn't be relied on for anything non-ephemeral.
+
+chefdepartie really doesn't work with environments. Everything is assumed to be on the default environment.
+
+# Configuration
+
+create a normal chef config file, but you only need to specify the following values:
+
+```
+chef_server_url  'http://localhost:4000'
+client_key 'path_to_any_pemfile'
+encrypted_data_bag_secret 'path_to_your_real_databag_secret'
+cookbook_path 'path_to_your_cookbooks'
+node_name 'any_name'
+```
+
+Then, provide this file to chefdepartie as an environment variable (yeah, this needs some tweaking)
+
+
+# Invocation
+
+Currently a little rough around the edges. You specify the chef config file to use, and then just run chefdepartie.
+
+```
+CHEFDEPARTIE_CONFIG=~/workspace/shopify/chefdepartie/config.rb  be ruby lib/chefdepartie.rb
+```
+
+# To do:
+
+* Wrap launching server with something like thor
+* Tests to ensure all uploads are working as expected
+* CI for tests

data/chefdepartie.gemspec

@@ -0,0 +1,25 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'chefdepartie/version'
+
+Gem::Specification.new do |spec|
+  spec.name        = 'chefdepartie'
+  spec.version     = Chefdepartie::VERSION
+  spec.summary     = 'chefdepartie helps you test you cookbooks locally'
+  spec.description = 'chefdepartie uses chef-zero to provide a local, testing chef server'
+  spec.authors     = ['Dale Hamel']
+  spec.email       = 'dale.hamel@srvthe.net'
+  spec.files       = Dir['lib/**/*']
+  spec.homepage    = 'https://rubygems.org/gems/chefdepartie'
+  spec.license     = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'chef', '~> 12.3.0'
+  spec.add_runtime_dependency 'chef-zero', '~> 4.2.2'
+  spec.add_runtime_dependency 'librarian-chef', '~> 0.0.4'
+  spec.add_development_dependency 'minitest', '~> 5.6'
+end

data/config.rb

@@ -0,0 +1,5 @@
+chef_server_url  'http://localhost:4000'
+client_key '/home/dale.hamel/.chef/configs/shopify/client.pem'
+encrypted_data_bag_secret '/home/dale.hamel/.chef/configs/shopify/encrypted_data_bag_secret'
+cookbook_path '/home/dale.hamel/workspace/shopify/cookbooks/cookbooks'
+node_name 'foo'

data/lib/chefdepartie/cookbook.rb

@@ -0,0 +1,72 @@
+require 'chef'
+require 'chef/cookbook/metadata'
+require 'chef/cookbook_uploader'
+require 'librarian'
+require 'librarian/chef'
+
+module Chefdepartie
+  module Cookbooks
+
+    def self.upload_all
+      cookbooks = File.dirname(Chef::Config[:cookbook_path])
+      Dir.chdir(cookbooks) do
+        puts "Uploading librarian cookbooks"
+        upload_cheffile
+
+        puts "Uploading site cookbooks"
+        books = Dir["cookbooks/*"]
+        upload_site_cookbooks(books)
+      end
+    end
+
+  private
+
+    def self.upload_cookbooks(path, books)
+      loader = Chef::CookbookLoader.new(path)
+
+      books = books.collect do |name|
+        status = :new
+        cookbook = loader.load_cookbook(name)
+        raise "could not load cookbook #{name} " if cookbook.nil?
+        cookbook
+      end.compact
+
+      rest = Chef::REST.new(Chef::Config[:chef_server_url])
+
+      begin
+        Chef::CookbookUploader.new(books, {force: true, rest: rest}).upload_cookbooks
+      rescue SystemExit => e
+        raise "Cookbook upload exited with status #{e.status}"
+      end
+
+      books.map(&:name).map(&:to_s)
+    end
+
+    def self.upload_site_cookbooks(books)
+      paths = {}
+      books.each do |book|
+        path, book = book.split('/', 2)
+        paths[path] ||= []
+        paths[path] << book
+      end
+
+      paths.each do |path, books|
+        upload_cookbooks(path, books)
+      end
+
+      paths.values.flatten.uniq.sort
+    end
+
+    def self.upload_cheffile
+      unless ENV['NO_LIBRARIAN']
+        system("librarian-chef install")
+        upload_cookbooks("tmp/librarian/cookbooks", cheffile_cookbooks.map(&:name))
+      end
+    end
+
+    def self.cheffile_cookbooks
+      librarian = ::Librarian::Chef.environment_class.new
+      librarian.lock.manifests
+    end
+  end
+end

data/lib/chefdepartie/databag.rb

@@ -0,0 +1,103 @@
+require 'chef'
+
+# We monkeypatch chefzero because it does something really stupid with databags, not sure why.
+# It prevents them from being mashes client side. This allows databags to by serialized
+# So that chef-client will load it as a mash instead of a hash
+# The change here is we comment out:
+#
+# data_bag_item = data_bag_item['raw_data']
+
+module ChefZero
+  module ChefData
+    class DataNormalizer
+      def self.normalize_data_bag_item(data_bag_item, data_bag_name, id, method)
+        if method == 'DELETE'
+          # TODO SERIOUSLY, WHO DOES THIS MANY EXCEPTIONS IN THEIR INTERFACE
+          if !(data_bag_item['json_class'] == 'Chef::DataBagItem' && data_bag_item['raw_data'])
+            data_bag_item['id'] ||= id
+            data_bag_item = { 'raw_data' => data_bag_item }
+            data_bag_item['chef_type'] ||= 'data_bag_item'
+            data_bag_item['json_class'] ||= 'Chef::DataBagItem'
+            data_bag_item['data_bag'] ||= data_bag_name
+            data_bag_item['name'] ||= "data_bag_item_#{data_bag_name}_#{id}"
+          end
+        else
+          # If it's not already wrapped with raw_data, wrap it.
+          if data_bag_item['json_class'] == 'Chef::DataBagItem' && data_bag_item['raw_data']
+            #data_bag_item = data_bag_item['raw_data']
+          end
+          # Argh.  We don't do this on GET, but we do on PUT and POST????
+          if %w(PUT POST).include?(method)
+            data_bag_item['chef_type'] ||= 'data_bag_item'
+            data_bag_item['data_bag'] ||= data_bag_name
+          end
+          data_bag_item['id'] ||= id
+        end
+        data_bag_item
+      end
+    end
+  end
+end
+
+module Chefdepartie
+  module Databags
+    def self.upload_all
+      puts "Uploading databags"
+      cookbooks = File.dirname(Chef::Config[:cookbook_path])
+      bags = Dir[File.join(cookbooks,'data_bags','/*')]
+      upload_all_data_bags(bags)
+    end
+
+  private
+
+    def self.upload_data_bag_items(items)
+      current_bags = Chef::DataBag.list.keys.to_set
+
+      items.each do |item|
+        bag_name = item.data_bag
+        unless current_bags.include?(bag_name)
+          bag = Chef::DataBag.new
+          bag.name(bag_name)
+          bag.create
+          puts "Created databag #{bag_name.inspect}."
+          current_bags << bag_name
+        end
+
+        item.save
+        puts "Uploaded databag #{bag_name.inspect}, item #{item.id.inspect}."
+      end
+    end
+
+    # Takes a list of data bag names, and uploads all data bag items from within that bag.
+    def self.upload_all_data_bags(data_bags)
+      items = []
+      secret = Chef::EncryptedDataBagItem.load_secret(Chef::Config[:encrypted_data_bag_secret])
+
+      data_bags.each do |data_bag|
+        bag_name = File.basename(data_bag)
+        files = Dir.glob(File.join(data_bag,"*.json")).flatten
+
+        files.each do |item_file|
+          raw_data = Chef::JSONCompat.from_json(IO.read(item_file))
+
+          item = Chef::DataBagItem.new
+          item.data_bag bag_name
+          if is_encrypted_data_bag?(raw_data)
+            item.raw_data = Chef::EncryptedDataBagItem.new(raw_data, secret).to_hash
+          else
+            item.raw_data = raw_data
+          end
+          items << item
+        end
+      end
+
+      upload_data_bag_items(items)
+    end
+
+    def self.is_encrypted_data_bag?(raw_data)
+      # TODO: use Chef::EncryptedDataBagItem::CheckEncrypted when we move to Chef 12
+      first_sub_item = Array(raw_data.find { |k, v| k != 'id' }).last
+      !!(first_sub_item.kind_of?(Hash) && first_sub_item['encrypted_data'] && first_sub_item['cipher'])
+    end
+  end
+end

data/lib/chefdepartie/role.rb

@@ -0,0 +1,44 @@
+require 'chef'
+require 'chef/role'
+
+module Chefdepartie
+  module Roles
+
+    def self.upload_all
+      puts "Uploading roles"
+      cookbooks = File.dirname(Chef::Config[:cookbook_path])
+      roles = []
+      Find.find(File.join(cookbooks,'roles')){ |f| roles << f if f =~ /\.rb$/}
+      upload_site_roles(roles)
+    end
+
+  private
+
+    def self.upload_site_roles(files)
+      roles = {}
+      files.each do |f|
+        role = role_from_file(f)
+        roles[role.name] = role # original name as hash key
+        role.name(role.name)
+      end
+
+      upload_roles(roles.values)
+
+      roles
+    end
+
+    def self.role_from_file(file)
+      role = Chef::Role.new
+      puts file
+      role.from_file(file)
+      role
+    end
+
+    def self.upload_roles(roles)
+      roles.each do |role|
+        role.save
+        puts "Uploaded #{role.name.inspect} role."
+      end
+    end
+  end
+end

data/lib/chefdepartie/server.rb

@@ -0,0 +1,11 @@
+require 'chef_zero/server'
+require 'uri/generic'
+
+def start_server
+  port = URI(Chef::Config[:chef_server_url]).port
+  server = ChefZero::Server.new(host: '0.0.0.0', port: port)
+  Thread.new do
+    server.start#_background
+  end
+end
+

data/lib/chefdepartie/version.rb

@@ -0,0 +1,3 @@
+module Chefdepartie
+  VERSION='0.0.3'
+end

data/lib/chefdepartie.rb

@@ -0,0 +1,35 @@
+require 'chef'
+
+require_relative 'chefdepartie/server'
+require_relative 'chefdepartie/role'
+require_relative 'chefdepartie/cookbook'
+require_relative 'chefdepartie/databag'
+
+module Chefdepartie
+  def self.run(**kwargs)
+
+    # Load the configuration
+    config = kwargs[:config]
+    Chef::Config.from_file(config)
+
+    # Start the chef-zero server
+    server_thread = start_server
+
+    # Upload everything
+    upload_all
+
+    # Now that everything has been uploaded, we'll join the server thread
+    puts "Ready"
+    server_thread.join
+  end
+
+private
+
+  def self.upload_all
+    Chefdepartie::Roles.upload_all
+    Chefdepartie::Databags.upload_all
+    Chefdepartie::Cookbooks.upload_all
+  end
+end
+
+Chefdepartie.run(config: ENV['CHEFDEPARTIE_CONFIG']) # FIXME: use something better than an env var to get the config

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: chefdepartie
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+platform: ruby
+authors:
+- Dale Hamel
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-07-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: chef
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 12.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 12.3.0
+- !ruby/object:Gem::Dependency
+  name: chef-zero
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.2.2
+- !ruby/object:Gem::Dependency
+  name: librarian-chef
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.6'
+description: chefdepartie uses chef-zero to provide a local, testing chef server
+email: dale.hamel@srvthe.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- README.md
+- chefdepartie.gemspec
+- config.rb
+- lib/chefdepartie.rb
+- lib/chefdepartie/cookbook.rb
+- lib/chefdepartie/databag.rb
+- lib/chefdepartie/role.rb
+- lib/chefdepartie/server.rb
+- lib/chefdepartie/version.rb
+homepage: https://rubygems.org/gems/chefdepartie
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: chefdepartie helps you test you cookbooks locally
+test_files: []
+has_rdoc: