chefdepartie 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a6cce015b3b98610b59d135891bb7e7273b9f625
-  data.tar.gz: a2d809eec199079dc747e8ac2f18cd4b3d3bb68c
+  metadata.gz: 26e730293bec2be0202f17b109278f8b6efc1277
+  data.tar.gz: 422e0da3b37f7020368887e012b0fca69e52867b
 SHA512:
-  metadata.gz: f043835243ecbf22dc13000806d27e254b262c63b4e3bc733dc97a525ccb91977c7742ad53d5f78835e9c3235758c3fea0b5a8c9a0aa9d21e2076c5b4f0fbaaa
-  data.tar.gz: 0181eb915b23f3999860f4ecd613f5a4c3dd25cc86ebd49bae46dbf3d0c75257f4fd50c62e6a6f2681b1eb4a488c956e307cbf80ff59890b6cd039b42e1f4526
+  metadata.gz: 4a7079ac8b938ef935af14f6468a6c8495f302e54d6046f0a9db037794f7314f921e9d976630066b2bedf3b35804209bef2aea7a154ee9e9cde34320ee39a936
+  data.tar.gz: 87dba1d350ddb0828cb4b852a114375d156708aaf488c50414698fe9453d4f104de48c83830df997449450c58683cf00b9bb6b54c24679ae2731eb9f1ed5d2b6

data/lib/chefdepartie.rb

@@ -1,61 +1,8 @@
-require 'chef'
-
-require_relative 'chefdepartie/server'
-require_relative 'chefdepartie/role'
-require_relative 'chefdepartie/cookbook'
-require_relative 'chefdepartie/databag'
-
-# Chefdepartie root namespace for core module commands
-module Chefdepartie
-  def self.run(**kwargs)
-    # Load the configuration
-    config_file = kwargs[:config_file] || ENV['CHEFDEPARTIE_CONFIG'] || ''
-    load_config(config_file, kwargs[:config])
-    background = kwargs[:background]
-
-    # Start the chef-zero server
-    self.server_thread = start_server(background)
-
-    # Upload everything
-    upload_all
-
-    # Notify that the chef server is ready
-    puts 'Ready'
-
-    # Join the chef server thread now that everything has been uploaded
-    self.server_thread.join unless background
-  end
-
-  def self.stop
-    puts 'Stopping server'
-    server_thread.stop
-  end
-
-  private
-
-  def self.server_thread=(thread)
-    @@server_thread = thread
-  end
-
-  def self.server_thread
-    @@server_thread
-  end
-
-  def self.upload_all
-    Chefdepartie::Roles.upload_all
-    Chefdepartie::Databags.upload_all
-    Chefdepartie::Cookbooks.upload_all
-  end
-
-  def self.load_config(config_file, config)
-    # Load config from config file if provided
-    Chef::Config.from_file(config_file) if (!config_file.empty? && File.exist?(config_file))
-
-    # Load config from hash
-    if config && config.is_a?(Hash)
-      config.each do |k, v|
-        Chef::Config.send(k.to_sym, v)
-      end
-    end
-  end
-end
+$LOAD_PATH.unshift(File.expand_path('../lib', __FILE__))
+
+require 'chefdepartie/runner'
+require 'chefdepartie/server'
+require 'chefdepartie/role'
+require 'chefdepartie/cookbook'
+require 'chefdepartie/databag'
+require 'chefdepartie/cache'

data/lib/chefdepartie/cache.rb

@@ -0,0 +1,60 @@
+require 'fileutils'
+require 'chef'
+require 'cityhash'
+require 'chef_zero/data_store/raw_file_store'
+require 'tmpdir'
+
+module Chefdepartie
+  module Cache
+    extend self
+    def setup(path)
+      FileUtils.mkdir_p(File.join(path, 'organizations', 'chef'))
+      @path = File.join(path, 'cache.dat')
+      if File.exist?(@path) then restore  else @cache = {} end
+      ds = ChefZero::DataStore::RawFileStore.new(File.join(path))
+      ChefZero::DataStore::DefaultFacade.new(ds, false, false)
+    end
+
+    def cache(path)
+      return false unless cache?
+      hash = File.file?(path) ? CityHash.hash128(File.read(path)) : hashdir(path)
+      hit = @cache[to_key(path)] == hash
+      unless hit
+        @cache[to_key(path)] = hash
+        dump
+      end
+      hit
+    end
+
+    def cache?
+      !@cache.nil?
+    end
+
+    def flush
+      @cache = {}
+      dump
+    end
+
+    private
+
+    def hashdir(path)
+      hash = ''
+      Dir["#{path}/**/*"].each do |path|
+        hash += CityHash.hash128(File.read(path)).to_s if File.file?(path)
+      end
+      CityHash.hash128(hash)
+    end
+
+    def restore
+      @cache = Marshal.load(File.binread(@path))
+    end
+
+    def dump
+      File.binwrite(@path, Marshal.dump(@cache))
+    end
+
+    def to_key(file)
+      file.gsub(/^#{File.dirname(Chef::Config[:cookbook_path])}/, '')
+    end
+  end
+end

data/lib/chefdepartie/cookbook.rb

@@ -1,8 +1,10 @@
 require 'chef'
 require 'chef/cookbook/metadata'
 require 'chef/cookbook_uploader'
+require "chef/cookbook/syntax_check"
 require 'librarian'
 require 'librarian/chef'
+require 'fileutils'
 
 module Chefdepartie
   # Handle finding and uploading cookbooks
@@ -25,8 +27,11 @@ module Chefdepartie
       loader = Chef::CookbookLoader.new(path)
 
       books = books.collect do |name|
-        status = :new
+        next if Cache.cache(File.join(path, name))
+        puts "Will upload #{name}"
         cookbook = loader.load_cookbook(name)
+        c = Chef::Cookbook::SyntaxCheck.for_cookbook(name, path)
+        c.ruby_files.concat(c.template_files).each { |f| c.validated(f) }
         fail "could not load cookbook #{name} " if cookbook.nil?
         cookbook
       end.compact
@@ -59,7 +64,8 @@ module Chefdepartie
 
     def self.upload_cheffile
       unless ENV['NO_LIBRARIAN']
-        system('librarian-chef install')
+        FileUtils.mkdir_p('tmp/librarian/cookbooks')
+        system('librarian-chef install --quiet --path tmp/librarian/cookbooks')
         upload_cookbooks('tmp/librarian/cookbooks', cheffile_cookbooks.map(&:name))
       end
     end

data/lib/chefdepartie/databag.rb

@@ -78,6 +78,7 @@ module Chefdepartie
         files = Dir.glob(File.join(data_bag, '*.json')).flatten
 
         files.each do |item_file|
+          next if Cache.cache(item_file)
           raw_data = Chef::JSONCompat.from_json(IO.read(item_file))
 
           item = Chef::DataBagItem.new

data/lib/chefdepartie/role.rb

@@ -7,7 +7,7 @@ module Chefdepartie
       puts 'Uploading roles'
       cookbooks = File.dirname(Chef::Config[:cookbook_path])
       roles = []
-      Find.find(File.join(cookbooks, 'roles')) { |f| roles << f if f =~ /\.rb$/ }
+      Find.find(File.join(cookbooks, 'roles')) { |f| roles << f if f =~ /\.rb$/ && !Cache.cache(f) }
       upload_site_roles(roles)
     end
 
@@ -28,7 +28,6 @@ module Chefdepartie
 
     def self.role_from_file(file)
       role = Chef::Role.new
-      puts file
       role.from_file(file)
       role
     end

data/lib/chefdepartie/runner.rb

@@ -0,0 +1,34 @@
+require 'chef'
+require 'openssl'
+require 'tempfile'
+
+module Chefdepartie
+  def self.run(**kwargs)
+    Runner.run(kwargs)
+  end
+
+  def self.stop
+    Server.stop
+  end
+
+  module Runner
+    extend self
+
+    def run(**kwargs)
+      # Load the configuration
+      Server.configure(kwargs)
+
+      # Start the chef-zero server
+      Server.start
+
+      # Upload everything
+      Server.upload_all
+
+      # Notify that the chef server is ready
+      puts 'Ready'
+
+      # Join the chef server thread now that everything has been uploaded
+      Server.join
+    end
+  end
+end

data/lib/chefdepartie/server.rb

@@ -1,16 +1,68 @@
 require 'chef_zero/server'
 require 'uri/generic'
 
-def start_server(background)
-  port = URI(Chef::Config[:chef_server_url]).port
-  server = ChefZero::Server.new(host: '0.0.0.0', port: port)
-
-  if background
-    server.start_background
-    server
-  else
-    Thread.new do
-      server.start
+module Chefdepartie
+  module Server
+    extend self
+
+    def start
+      @server = ChefZero::Server.new(@opts)
+
+      if @background
+        @server.start_background
+      else
+        @server_thread = Thread.new do
+          server.start
+        end
+      end
+    end
+
+    def join
+      @server_thread.join unless @background
+    end
+
+    def stop
+      puts 'Stopping server'
+      @server_thread.nil? ? @server.stop : @server_thread.stop
+    end
+
+    def upload_all
+      Chefdepartie::Roles.upload_all
+      Chefdepartie::Databags.upload_all
+      Chefdepartie::Cookbooks.upload_all
+    end
+
+    # Load chef config from hash or file
+    def configure(kwargs)
+      config_file = kwargs[:config_file] || ENV['CHEFDEPARTIE_CONFIG'] || ''
+      # Load config from config file if provided
+      Chef::Config.from_file(config_file) if !config_file.empty? && File.exist?(config_file)
+
+      config = kwargs[:config]
+      # Load config from hash
+      if config && config.is_a?(Hash)
+        config[:node_name] ||= 'chef-zero'
+        config[:client_key] ||= fake_key
+        config.each do |k, v|
+          Chef::Config.send(k.to_sym, v)
+        end
+      end
+
+      @background = kwargs[:background]
+      @cache = kwargs[:cache]
+
+      port = URI(Chef::Config[:chef_server_url]).port
+      @opts = { host: '0.0.0.0', port: port } # , log_level: :debug }
+      @opts.merge!(data_store: Cache.setup(@cache)) if @cache
+    end
+
+  private
+
+    def fake_key
+      client_key = Tempfile.new('chef-zero-client')
+      client_key.write(OpenSSL::PKey::RSA.new(2048).to_s)
+      client_key.close
+      client_key.path
     end
   end
 end

data/lib/chefdepartie/version.rb

@@ -1,4 +1,4 @@
 # Store version info
 module Chefdepartie
-  VERSION = '0.0.5'
+  VERSION = '0.0.6'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chefdepartie
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Dale Hamel
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-25 00:00:00.000000000 Z
+date: 2015-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chef
@@ -16,19 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.0
+        version: '12.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.4.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.0
+        version: '12.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.4.3
 - !ruby/object:Gem::Dependency
   name: chef-zero
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 4.2.2
   type: :runtime
@@ -36,6 +45,9 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 4.2.2
 - !ruby/object:Gem::Dependency
@@ -53,35 +65,73 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.0.4
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: cityhash
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.6'
-  type: :development
+        version: 0.8.1
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.6'
+        version: 0.8.1
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 10.4.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 10.4.2
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.4.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.4.0
 description: chefdepartie uses chef-zero to provide a local, testing chef server
 email: dale.hamel@srvthe.net
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- Gemfile
-- Gemfile.lock
-- README.md
-- chefdepartie.gemspec
-- config.rb
 - lib/chefdepartie.rb
+- lib/chefdepartie/cache.rb
 - lib/chefdepartie/cookbook.rb
 - lib/chefdepartie/databag.rb
 - lib/chefdepartie/role.rb
+- lib/chefdepartie/runner.rb
 - lib/chefdepartie/server.rb
 - lib/chefdepartie/version.rb
 homepage: https://rubygems.org/gems/chefdepartie

data/.gitignore

@@ -1,2 +0,0 @@
-cookbooks
-*.gem

data/Gemfile

@@ -1,7 +0,0 @@
-source 'https://rubygems.org'
-
-gemspec
-
-group :test do
-  gem 'simplecov', '=0.10.0'
-end

data/Gemfile.lock

@@ -1,153 +0,0 @@
-PATH
-  remote: .
-  specs:
-    chefdepartie (0.0.5)
-      chef (~> 12.3.0)
-      chef-zero (~> 4.2.2)
-      librarian-chef (~> 0.0.4)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    builder (3.2.2)
-    chef (12.3.0)
-      chef-zero (~> 4.1)
-      diff-lcs (~> 1.2, >= 1.2.4)
-      erubis (~> 2.7)
-      ffi-yajl (>= 1.2, < 3.0)
-      highline (~> 1.6, >= 1.6.9)
-      mixlib-authentication (~> 1.3)
-      mixlib-cli (~> 1.4)
-      mixlib-config (~> 2.0)
-      mixlib-log (~> 1.3)
-      mixlib-shellout (>= 2.0.0.rc.0, < 3.0)
-      net-ssh (~> 2.6)
-      net-ssh-multi (~> 1.1)
-      ohai (~> 8.0)
-      plist (~> 3.1.0)
-      pry (~> 0.9)
-      rspec-core (~> 3.2)
-      rspec-expectations (~> 3.2)
-      rspec-mocks (~> 3.2)
-      rspec_junit_formatter (~> 0.2.0)
-      serverspec (~> 2.7)
-      specinfra (~> 2.10)
-    chef-config (12.5.1)
-      mixlib-config (~> 2.0)
-      mixlib-shellout (~> 2.0)
-    chef-zero (4.2.3)
-      ffi-yajl (>= 1.1, < 3.0)
-      hashie (~> 2.0)
-      mixlib-log (~> 1.3)
-      rack
-      uuidtools (~> 2.1)
-    coderay (1.1.0)
-    diff-lcs (1.2.5)
-    docile (1.1.5)
-    erubis (2.7.0)
-    ffi (1.9.10)
-    ffi-yajl (2.2.2)
-      libyajl2 (~> 1.2)
-    hashie (2.1.2)
-    highline (1.7.8)
-    ipaddress (0.8.0)
-    json (1.8.2)
-    librarian (0.1.2)
-      highline
-      thor (~> 0.15)
-    librarian-chef (0.0.4)
-      chef (>= 0.10)
-      librarian (~> 0.1.0)
-      minitar (>= 0.5.2)
-    libyajl2 (1.2.0)
-    method_source (0.8.2)
-    mime-types (2.99)
-    minitar (0.5.4)
-    minitest (5.6.1)
-    mixlib-authentication (1.3.0)
-      mixlib-log
-    mixlib-cli (1.5.0)
-    mixlib-config (2.2.1)
-    mixlib-log (1.6.0)
-    mixlib-shellout (2.2.3)
-    multi_json (1.11.2)
-    net-scp (1.2.1)
-      net-ssh (>= 2.6.5)
-    net-ssh (2.9.2)
-    net-ssh-gateway (1.2.0)
-      net-ssh (>= 2.6.5)
-    net-ssh-multi (1.2.1)
-      net-ssh (>= 2.6.5)
-      net-ssh-gateway (>= 1.2.0)
-    net-telnet (0.1.1)
-    ohai (8.7.0)
-      chef-config (>= 12.5.0.alpha.1, < 13)
-      ffi (~> 1.9)
-      ffi-yajl (~> 2.2)
-      ipaddress
-      mime-types (~> 2.0)
-      mixlib-cli
-      mixlib-config (~> 2.0)
-      mixlib-log
-      mixlib-shellout (~> 2.0)
-      rake (~> 10.1)
-      systemu (~> 2.6.4)
-      wmi-lite (~> 1.0)
-    plist (3.1.0)
-    pry (0.10.3)
-      coderay (~> 1.1.0)
-      method_source (~> 0.8.1)
-      slop (~> 3.4)
-    rack (1.6.4)
-    rake (10.4.2)
-    rspec (3.4.0)
-      rspec-core (~> 3.4.0)
-      rspec-expectations (~> 3.4.0)
-      rspec-mocks (~> 3.4.0)
-    rspec-core (3.4.1)
-      rspec-support (~> 3.4.0)
-    rspec-expectations (3.4.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.4.0)
-    rspec-its (1.2.0)
-      rspec-core (>= 3.0.0)
-      rspec-expectations (>= 3.0.0)
-    rspec-mocks (3.4.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.4.0)
-    rspec-support (3.4.1)
-    rspec_junit_formatter (0.2.3)
-      builder (< 4)
-      rspec-core (>= 2, < 4, != 2.12.0)
-    serverspec (2.24.3)
-      multi_json
-      rspec (~> 3.0)
-      rspec-its
-      specinfra (~> 2.43)
-    sfl (2.2)
-    simplecov (0.10.0)
-      docile (~> 1.1.0)
-      json (~> 1.8)
-      simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.0)
-    slop (3.6.0)
-    specinfra (2.44.3)
-      net-scp
-      net-ssh (~> 2.7)
-      net-telnet
-      sfl
-    systemu (2.6.5)
-    thor (0.19.1)
-    uuidtools (2.1.5)
-    wmi-lite (1.0.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  chefdepartie!
-  minitest (~> 5.6)
-  simplecov (= 0.10.0)
-
-BUNDLED WITH
-   1.10.6

data/README.md

@@ -1,56 +0,0 @@
-# A quick little helper
-
-In a kitchen, a chef de partie is a line cook who helps prepare the ingredients the real chef will use.
-
-Chefdepartie is a wrapper around a chef-zero server, allowing you to easily upload all of your:
-
-* Cookbooks (site)
-* Librarian cookbooks
-* Roles
-* Data bags
-
-This allows you to test out all of your cookbooks together without putting them on your actual chef server.
-
-## Chefdepartie vs chef-solo
-
-Since chef-zero provides the same interface as **a real chef server**, you can actually use knife with it!
-
-This gives you a full chef sandbox, and can be useful for things like CI or packer image provisioning.
-
-* View and edit data bags, without an internet connection
-* Bootstrap test nodes (vagrant images, cloud servers, etc)
-
-## Limitations
-
-Since chefdepartie is just a wrapper around chef-zero, it runs entirely in memory and has no persistence. This makes it a useful sandbox, but it shouldn't be relied on for anything non-ephemeral.
-
-chefdepartie really doesn't work with environments. Everything is assumed to be on the default environment.
-
-# Configuration
-
-create a normal chef config file, but you only need to specify the following values:
-
-```
-chef_server_url  'http://localhost:4000'
-client_key 'path_to_any_pemfile'
-encrypted_data_bag_secret 'path_to_your_real_databag_secret'
-cookbook_path 'path_to_your_cookbooks'
-node_name 'any_name'
-```
-
-Then, provide this file to chefdepartie as an environment variable (yeah, this needs some tweaking)
-
-
-# Invocation
-
-Currently a little rough around the edges. You specify the chef config file to use, and then just run chefdepartie.
-
-```
-CHEFDEPARTIE_CONFIG=~/workspace/shopify/chefdepartie/config.rb  be ruby lib/chefdepartie.rb
-```
-
-# To do:
-
-* Wrap launching server with something like thor
-* Tests to ensure all uploads are working as expected
-* CI for tests

data/chefdepartie.gemspec

@@ -1,25 +0,0 @@
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'chefdepartie/version'
-
-Gem::Specification.new do |spec|
-  spec.name        = 'chefdepartie'
-  spec.version     = Chefdepartie::VERSION
-  spec.summary     = 'chefdepartie helps you test you cookbooks locally'
-  spec.description = 'chefdepartie uses chef-zero to provide a local, testing chef server'
-  spec.authors     = ['Dale Hamel']
-  spec.email       = 'dale.hamel@srvthe.net'
-  spec.files       = Dir['lib/**/*']
-  spec.homepage    = 'https://rubygems.org/gems/chefdepartie'
-  spec.license     = 'MIT'
-
-  spec.files         = `git ls-files`.split($INPUT_RECORD_SEPARATOR)
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ['lib']
-
-  spec.add_runtime_dependency 'chef', '~> 12.3.0'
-  spec.add_runtime_dependency 'chef-zero', '~> 4.2.2'
-  spec.add_runtime_dependency 'librarian-chef', '~> 0.0.4'
-  spec.add_development_dependency 'minitest', '~> 5.6'
-end

data/config.rb

@@ -1,5 +0,0 @@
-chef_server_url  'http://localhost:4000'
-client_key '/home/dale.hamel/.chef/configs/shopify/client.pem'
-encrypted_data_bag_secret '/home/dale.hamel/.chef/configs/shopify/encrypted_data_bag_secret'
-cookbook_path '/home/dale.hamel/workspace/shopify/cookbooks/cookbooks'
-node_name 'foo'