chef-workflow 0.1.0

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.chef-workflow
+html
+tasks.txt

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+if ENV["ERIKH_LOCAL"]
+  gem "vagrant-prison", :path => "~/repos/ht/vagrant-prison"
+end
+
+# Specify your gem's dependencies in chef-workflow.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Erik Hollensbe
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,161 @@
+Chef Workflow Toolkit
+---------------------
+
+This code is the common base of
+[chef-workflow-tasklib](https://github.com/chef-workflow/chef-workflow-tasklib)
+and
+[chef-workflow-testlib](https://github.com/chef-workflow/chef-workflow-testlib).
+Unless you are looking to create extensions to these two systems, or use the
+libraries contained within, you would be better served (with rare exception) by
+visiting these two projects for your testing and workflow needs.
+
+Environment Variables
+---------------------
+
+This toolkit exposes a number of environment variables that you may want to set
+to affect your experience using it:
+
+* `CHEF_WORKFLOW_DEBUG` - set to an integer of 1-3, controls the amount of
+  verbosity of reporting done in the libraries themselves. 1 usually amounts to
+  diagnostic messages, 2 to full transactions (converges, bootstraps, VM
+  provision detail), and 3 to more chatty individual pain points.
+* `CHEF_CONFIG` - the path to your knife.rb. Note that if you've set this up in
+  `KnifeSupport` or with `configure_knife` you should almost never need this.
+* `TEST_CHEF_SUBNET` - specifies a /24 network that IP addresses are drawn from
+  for local testing with VM systems. Note that this support is fairly crude --
+  it's strongly suggested you use a proper /24 and make your last octet `0`.
+  :)
+
+Classes
+-------
+
+Expect this to be supplemented with linked RDoc that describes the API for each
+class. For now, though, you'll have to generate the docs yourself or read the
+source comments.
+
+At the time of this writing there is not a very consistent namespacing method,
+expect this to be corrected before the first release.
+
+Utility libraries:
+==================
+
+* `Chef::Workflow::ConfigureHelper` - a mixin which provides easy-to-use methods
+  of driving the various support configuration systems (see below).
+* `AttrSupport` - a small mixin to supply chef-style instance variable mutators;
+  the kind that are a little more convenient to use with `instance_eval`'d
+  blocks.
+* `DebugSupport` - mixin which defines a method called `if_debug` which is our
+  gating mechanism for the `CHEF_WORKFLOW_DEBUG` environment variable.
+* `GenericSupport` - mixin which keeps the configuration interface consistent by
+  providing a `configure` class method that uses `instance_eval` and exposes a
+  pre-configured object under `singleton` which can be manipulated.
+* `KnifePluginSupport` - mixin which contains a routine called
+  `init_knife_plugin` to simplify configuration of knife plugins, which can
+  then be used as normal objects. Also configures a UI object with `StringIO`
+  so that it can be communicated with optionally.
+
+Configuration libraries:
+========================
+
+These all mixin `GenericSupport` and as a result are expected to be treated as
+singletons, by accessing their `singleton` class method and configured with the
+`configure` class method which `instance_eval`'s a block.
+
+If you are using `chef-workflow-tasklib`, most of the bits here as you have
+configured them can be described to you via `bundle exec rake chef:show_config`.
+
+* `GeneralSupport` - "General" configuration attributes that are global to the
+  entire system.
+* `IPSupport` - Database for associating IP addresses with a server group. See
+  discussion on the scheduler below for more information on server groups. This
+  is generally not configured externally, but by tooling within the system.
+* `KnifeSupport` - Most configuration regarding chef lives here, and additional
+  network access.
+* `VagrantSupport` - Specific bits related to using Vagrant, such as the box to
+  be used for provisioning. 
+* `EC2Support` - bits related to driving AWS EC2.
+
+Scheduler and VM
+----------------
+
+The Scheduler and VM system work together to make groups of machines easy to
+provision. 
+
+The scheduler is responsible for scheduling provisioning of groups of machines
+which are interdependent with other machines. In other words when machine C
+depends on B and A to be provisioned, and B and A have no dependencies, and
+they are all scheduled at the same time, the scheduler will determine that A
+and B have to be provisioned immediately and as soon as they are provisioned
+successfully, it will attempt to provision C. Depending on the system
+controlling the scheduler and its constraints, it can do this in a serial or
+parallel fashion, the latter of which will attempt to provision as much as
+possible at the same time, and as things finish will provision things that are
+satisfied by what finished.
+
+In other words, provisioning takes a lot of time in a test run, and the
+scheduler tries very hard to make it take as little time as is reasonably
+possible given your constraints and the constraints of the system.
+
+It manages the actual act of provisioning through the VM system, which tracks
+the state of what's currently provisioned, what has already successfully
+provisioned (and presumed alive), and what is waiting to be provisoned. The VM
+class itself is largely responsible for exposing this data to the scheduler,
+and marshalling its state to disk in the event of a failure so things can be
+cleaned up or resumed in the case of resources that will always be depended on
+for a test run.
+
+In other words, provisioning takes a lot of time in a test run, and the VM
+system tries very hard to not add more time to this by tracking machines that
+are already provisioned so they don't have to be re-provisioned, even between
+runs. It also makes it easy to clean up a bad or stale test run.
+
+Server Groups
+=============
+
+The VM system itself is a mapping of server groups to an array of provisioning
+"commands", which are implemented as classes with a consistent interface
+(visitors). A provisioning command may create a [vagrant
+prison](https://github.com/chef-workflow/vagrant-prison) which contains all the
+servers for that server group, complete with assigning them a host-only
+interface and storing that with `IPSupport` so that it can be retrieved by
+other bits of the test system or task system. Another provisioning command may
+execute the in-code equivalent of `knife bootstrap` to build out your servers
+with a role named after the server group. For de-provisioning, the provisioning
+commands are played in reverse with a `shutdown` call applied to them.
+
+Scheduler and VM libraries
+==========================
+
+* `Scheduler` - this is the meat; if you're driving a new testing system such
+  as rspec, you'll want to get real familiar with the interface presented.
+* `VM` - marshalling and delegation interface. Most of this is exposed to the
+  scheduler interface.
+* `VM::VagrantProvisioner` - creates a vagrant prison composed of n servers for
+  a server group with a unique host-only ip for each server.
+* `VM::EC2Provisioner` - creates a group of servers provisioned on EC2.
+* `VM::KnifeProvisioner` - the provisioner equivalent of `knife bootstrap`, with
+  additional sanity checks for both converge success and a waiting period for
+  search indexing. On deprovision, deletes the nodes that were created. Will
+  always try to bootstrap a group in parallel.
+* `VM::ChefServerProvisioner` - similar to `VM::KnifeProvisioner`, it wraps the
+  [knife-server](https://github.com/fnichol/knife-server) toolkit to create a
+  chef server.
+
+Contributing
+------------
+
+* fork the project
+* make a branch
+* add your stuff
+* push your branch to your repo
+* send a pull request
+
+**Note:** modifications to gem metadata, author lists, and other credits
+without rationale will be rejected immediately.
+
+Credits
+-------
+
+This work was partially sponsored by [Hotel Tonight](http://hoteltonight.com)
+and is what they use to test our infrastructure internally. Primarily authored by
+[Erik Hollensbe](https://github.com/erikh).

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require 'rdoc/task'
+
+RDoc::Task.new do |r|
+  r.main = "README.md"
+  r.rdoc_files.include("README.md", "LICENSE.txt", "lib/**/*.rb")
+  r.options << "--all"
+end

data/bin/chef-workflow-bootstrap

@@ -0,0 +1,149 @@
+#!ruby
+
+require 'fileutils'
+
+unless $stdin.tty?
+  $stderr.puts "Please run this from a terminal in your git repository's root."
+  exit 1
+end
+
+if ARGV[0] =~ /^--?h(?:elp)?$/
+  $stderr.puts 'Creates the necessary files needed for bootstrapping chef-workflow.'
+  $stderr.puts 'Ignores any files that already exist.'
+  exit 1
+end
+
+def create(filename, content)
+  print "Trying to create #{filename}..."
+  if File.exist?(filename)
+    puts "ignoring, file already exists."
+  else
+    FileUtils.mkdir_p(File.dirname(filename))
+    File.binwrite(filename, content)
+    puts "done."
+  end
+end
+
+RAKEFILE_CONTENT = <<-EOF
+require 'chef-workflow-tasklib'
+#
+# If you want to change the tasks, remove this next line, and see the README:
+#
+# https://github.com/chef-workflow/chef-workflow-tasklib#picking-your-own-workflow
+#
+chef_workflow_task 'default'
+EOF
+
+CONFIG_CONTENT = <<-EOF
+#
+# chef-workflow configuration. You can inspect the full set of configured values
+# with: `bundle exec rake chef:show_config`
+#
+
+#
+# This is general configuration -- related to no specific component. Use with
+# caution! Most of these settings don't need to be adjusted normally.
+#
+
+configure_general do
+  # change the location of the chef-workflow generated files
+  # workflow_dir File.join(Dir.pwd, '.chef-workflow')
+  #
+  # change the location of the VM database
+  # vm_file '.chef-workflow/vm'
+  #
+  # change the location of the chef-server prison file (vagrant only)
+  # chef_server_prison '.chef-workflow/chef-server'
+  #
+  # change the type of machine provisioner. You can use a class or :ec2 or
+  # :vagrant which map to the appropriate provisioner classes. If you use a
+  # custom class, make sure you require it first!
+  # machine_provisioner :vagrant 
+end
+
+#
+# This section configures all things related to vagrant usage
+#
+configure_vagrant do
+  # if you wish to use a different box, supply it as a url here.
+  # box_url "http://files.vagrantup.com/precise32.box"
+end
+
+configure_knife do
+  #
+  # you shouldn't need to modify most of these! be sure to 'bundle exec rake
+  # chef:show_config' to see the standard values, as well as others that are
+  # not listed here.
+  #
+  # cookbooks_path 'cookbooks'
+  # roles_path 'roles'
+  # environments_path 'environments'
+  # data_bags_path 'data_bags'
+  # ssh_user 'vagrant'
+  # ssh_password 'vagrant'
+  # ssh_identity_file nil
+  # use_sudo true
+  # test_environment 'vagrant'
+  
+  # search_index_wait is used for provisioning with VM::KnifeProvisioner. After
+  # provisioning, it waits a set value of seconds (60 default) for the metadata
+  # to appear in chef's index. If this fails to appear in this time an
+  # exception will be thrown. If you have a slow chef server or find yourself
+  # hitting this limit a lot, increase this value.
+  #
+  # search_index_wait 60
+end
+
+#
+# EC2 settings.
+#
+configure_ec2 do
+  #
+  # You'll want to heavily edit these if you use the EC2 support. Additionally,
+  # it is *strongly advised* to use a separate AWS account for your testing.
+  #
+  # The access key ID - no default, uses environment if it doesn't exist.
+  # access_key_id "my_access_key"
+  #
+  # The secret access key - no default, uses environment if it doesn't exist.
+  # secret_access_key "my_secret"
+  #
+  # The AMI of provisioned boxes - no default!
+  # ami "ami-abcdef123"
+  #
+  # The instance type of provisioned boxes - no default!
+  # instance_type "m1.small"
+  #
+  # The region where the instances should be allocated - no default!
+  # region "us-west-1"
+  #
+  # The ssh key used for creating machines (you still have to configure knife
+  # separately) - no default!
+  # ssh_key "test"
+  #
+  # The security groups to put all machines in; if set to :auto (the default),
+  # creates a new one until chef:clean is run.
+  # security_groups :auto
+  #
+  # The open ports in the security groups that are created dynamically. If you
+  # need something more complicated, create your own groups by hand and set
+  # `security_groups` above to those values. 
+  # security_group_open_ports [22, 4000]
+end
+
+#
+# This is largely important for local VM work. If you're using something like
+# EC2 support (which determines its own IPs, for obvious reasons) these
+# configuration settings are meaningless.
+#
+configure_ips do
+  # this is treated as a /24, walked incrementally, and with vagrant at least,
+  # .1 is spoken for. You'll want 0 as your last octet. Expect this to be less
+  # fail in the future. :)
+  #
+  # subnet "10.10.10.0"
+end
+EOF
+
+create('Rakefile', RAKEFILE_CONTENT)
+create('lib/chef-workflow-config.rb', CONFIG_CONTENT)

data/chef-workflow.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'chef-workflow/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "chef-workflow"
+  gem.version       = Chef::Workflow::VERSION
+  gem.authors       = ["Erik Hollensbe"]
+  gem.email         = ["erik+github@hollensbe.org"]
+  gem.description   = %q{A comprehensive rake-based workflow for chef}
+  gem.summary       = %q{A comprehensive rake-based workflow for chef}
+  gem.homepage      = "https://github.com/chef-workflow/chef-workflow"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency 'vagrant-prison', '~> 0.1.0'
+  gem.add_dependency 'chef', '~> 10.0'
+  gem.add_dependency 'aws-sdk', '~> 1.7.0'
+  gem.add_dependency 'net-ssh', '~> 2.6.0'
+
+  gem.add_development_dependency 'rdoc'
+  gem.add_development_dependency 'rake'
+end

data/lib/chef-workflow.rb

@@ -0,0 +1,73 @@
+require "chef-workflow/version"
+
+require 'chef-workflow/support/general'
+require 'chef-workflow/support/knife'
+require 'chef-workflow/support/vagrant'
+require 'chef-workflow/support/ip'
+require 'chef-workflow/support/debug'
+require 'chef-workflow/support/ec2'
+
+class Chef
+  module Workflow
+    #
+    # Basic helpers (intended to be mixed in elsewhere) to configure the
+    # various support configuration systems.
+    #
+    module ConfigureHelper
+      #
+      # Configure 'GeneralSupport'
+      #
+      def configure_general(&block)
+        GeneralSupport.configure(&block)
+      end
+
+      #
+      # Configure 'KnifeSupport'
+      #
+      def configure_knife(&block)
+        KnifeSupport.configure(&block)
+      end
+
+      #
+      # Configure 'VagrantSupport'
+      #
+      def configure_vagrant(&block)
+        VagrantSupport.configure(&block)
+      end
+      
+      #
+      # Configure 'IPSupport' - you probably don't need to do this.
+      #
+      def configure_ips(&block)
+        IPSupport.configure(&block)
+      end
+
+      #
+      # Configure 'EC2Support'
+      #
+      def configure_ec2(&block)
+        EC2Support.configure(&block)
+      end
+    end
+  end
+end
+
+class << eval("self", TOPLEVEL_BINDING)
+  include Chef::Workflow::ConfigureHelper
+end
+
+if defined? Rake::DSL
+  module Rake::DSL
+    include Chef::Workflow::ConfigureHelper
+  end
+end
+
+$:.unshift 'lib'
+
+begin
+  require 'chef-workflow-config'
+rescue LoadError
+  $stderr.puts "There is no chef-workflow-config in your lib directory."
+  $stderr.puts "Please run chef-workflow-bootstrap or add one."
+end
+