chef-workflow 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +20 -0
- data/Gemfile +8 -0
- data/LICENSE.txt +22 -0
- data/README.md +161 -0
- data/Rakefile +8 -0
- data/bin/chef-workflow-bootstrap +149 -0
- data/chef-workflow.gemspec +27 -0
- data/lib/chef-workflow.rb +73 -0
- data/lib/chef-workflow/support/attr.rb +31 -0
- data/lib/chef-workflow/support/debug.rb +51 -0
- data/lib/chef-workflow/support/ec2.rb +170 -0
- data/lib/chef-workflow/support/general.rb +63 -0
- data/lib/chef-workflow/support/generic.rb +30 -0
- data/lib/chef-workflow/support/ip.rb +129 -0
- data/lib/chef-workflow/support/knife-plugin.rb +33 -0
- data/lib/chef-workflow/support/knife.rb +122 -0
- data/lib/chef-workflow/support/scheduler.rb +403 -0
- data/lib/chef-workflow/support/vagrant.rb +41 -0
- data/lib/chef-workflow/support/vm.rb +71 -0
- data/lib/chef-workflow/support/vm/chef_server.rb +31 -0
- data/lib/chef-workflow/support/vm/ec2.rb +146 -0
- data/lib/chef-workflow/support/vm/knife.rb +217 -0
- data/lib/chef-workflow/support/vm/vagrant.rb +95 -0
- data/lib/chef-workflow/version.rb +6 -0
- metadata +167 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Erik Hollensbe
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,161 @@
|
|
1
|
+
Chef Workflow Toolkit
|
2
|
+
---------------------
|
3
|
+
|
4
|
+
This code is the common base of
|
5
|
+
[chef-workflow-tasklib](https://github.com/chef-workflow/chef-workflow-tasklib)
|
6
|
+
and
|
7
|
+
[chef-workflow-testlib](https://github.com/chef-workflow/chef-workflow-testlib).
|
8
|
+
Unless you are looking to create extensions to these two systems, or use the
|
9
|
+
libraries contained within, you would be better served (with rare exception) by
|
10
|
+
visiting these two projects for your testing and workflow needs.
|
11
|
+
|
12
|
+
Environment Variables
|
13
|
+
---------------------
|
14
|
+
|
15
|
+
This toolkit exposes a number of environment variables that you may want to set
|
16
|
+
to affect your experience using it:
|
17
|
+
|
18
|
+
* `CHEF_WORKFLOW_DEBUG` - set to an integer of 1-3, controls the amount of
|
19
|
+
verbosity of reporting done in the libraries themselves. 1 usually amounts to
|
20
|
+
diagnostic messages, 2 to full transactions (converges, bootstraps, VM
|
21
|
+
provision detail), and 3 to more chatty individual pain points.
|
22
|
+
* `CHEF_CONFIG` - the path to your knife.rb. Note that if you've set this up in
|
23
|
+
`KnifeSupport` or with `configure_knife` you should almost never need this.
|
24
|
+
* `TEST_CHEF_SUBNET` - specifies a /24 network that IP addresses are drawn from
|
25
|
+
for local testing with VM systems. Note that this support is fairly crude --
|
26
|
+
it's strongly suggested you use a proper /24 and make your last octet `0`.
|
27
|
+
:)
|
28
|
+
|
29
|
+
Classes
|
30
|
+
-------
|
31
|
+
|
32
|
+
Expect this to be supplemented with linked RDoc that describes the API for each
|
33
|
+
class. For now, though, you'll have to generate the docs yourself or read the
|
34
|
+
source comments.
|
35
|
+
|
36
|
+
At the time of this writing there is not a very consistent namespacing method,
|
37
|
+
expect this to be corrected before the first release.
|
38
|
+
|
39
|
+
Utility libraries:
|
40
|
+
==================
|
41
|
+
|
42
|
+
* `Chef::Workflow::ConfigureHelper` - a mixin which provides easy-to-use methods
|
43
|
+
of driving the various support configuration systems (see below).
|
44
|
+
* `AttrSupport` - a small mixin to supply chef-style instance variable mutators;
|
45
|
+
the kind that are a little more convenient to use with `instance_eval`'d
|
46
|
+
blocks.
|
47
|
+
* `DebugSupport` - mixin which defines a method called `if_debug` which is our
|
48
|
+
gating mechanism for the `CHEF_WORKFLOW_DEBUG` environment variable.
|
49
|
+
* `GenericSupport` - mixin which keeps the configuration interface consistent by
|
50
|
+
providing a `configure` class method that uses `instance_eval` and exposes a
|
51
|
+
pre-configured object under `singleton` which can be manipulated.
|
52
|
+
* `KnifePluginSupport` - mixin which contains a routine called
|
53
|
+
`init_knife_plugin` to simplify configuration of knife plugins, which can
|
54
|
+
then be used as normal objects. Also configures a UI object with `StringIO`
|
55
|
+
so that it can be communicated with optionally.
|
56
|
+
|
57
|
+
Configuration libraries:
|
58
|
+
========================
|
59
|
+
|
60
|
+
These all mixin `GenericSupport` and as a result are expected to be treated as
|
61
|
+
singletons, by accessing their `singleton` class method and configured with the
|
62
|
+
`configure` class method which `instance_eval`'s a block.
|
63
|
+
|
64
|
+
If you are using `chef-workflow-tasklib`, most of the bits here as you have
|
65
|
+
configured them can be described to you via `bundle exec rake chef:show_config`.
|
66
|
+
|
67
|
+
* `GeneralSupport` - "General" configuration attributes that are global to the
|
68
|
+
entire system.
|
69
|
+
* `IPSupport` - Database for associating IP addresses with a server group. See
|
70
|
+
discussion on the scheduler below for more information on server groups. This
|
71
|
+
is generally not configured externally, but by tooling within the system.
|
72
|
+
* `KnifeSupport` - Most configuration regarding chef lives here, and additional
|
73
|
+
network access.
|
74
|
+
* `VagrantSupport` - Specific bits related to using Vagrant, such as the box to
|
75
|
+
be used for provisioning.
|
76
|
+
* `EC2Support` - bits related to driving AWS EC2.
|
77
|
+
|
78
|
+
Scheduler and VM
|
79
|
+
----------------
|
80
|
+
|
81
|
+
The Scheduler and VM system work together to make groups of machines easy to
|
82
|
+
provision.
|
83
|
+
|
84
|
+
The scheduler is responsible for scheduling provisioning of groups of machines
|
85
|
+
which are interdependent with other machines. In other words when machine C
|
86
|
+
depends on B and A to be provisioned, and B and A have no dependencies, and
|
87
|
+
they are all scheduled at the same time, the scheduler will determine that A
|
88
|
+
and B have to be provisioned immediately and as soon as they are provisioned
|
89
|
+
successfully, it will attempt to provision C. Depending on the system
|
90
|
+
controlling the scheduler and its constraints, it can do this in a serial or
|
91
|
+
parallel fashion, the latter of which will attempt to provision as much as
|
92
|
+
possible at the same time, and as things finish will provision things that are
|
93
|
+
satisfied by what finished.
|
94
|
+
|
95
|
+
In other words, provisioning takes a lot of time in a test run, and the
|
96
|
+
scheduler tries very hard to make it take as little time as is reasonably
|
97
|
+
possible given your constraints and the constraints of the system.
|
98
|
+
|
99
|
+
It manages the actual act of provisioning through the VM system, which tracks
|
100
|
+
the state of what's currently provisioned, what has already successfully
|
101
|
+
provisioned (and presumed alive), and what is waiting to be provisoned. The VM
|
102
|
+
class itself is largely responsible for exposing this data to the scheduler,
|
103
|
+
and marshalling its state to disk in the event of a failure so things can be
|
104
|
+
cleaned up or resumed in the case of resources that will always be depended on
|
105
|
+
for a test run.
|
106
|
+
|
107
|
+
In other words, provisioning takes a lot of time in a test run, and the VM
|
108
|
+
system tries very hard to not add more time to this by tracking machines that
|
109
|
+
are already provisioned so they don't have to be re-provisioned, even between
|
110
|
+
runs. It also makes it easy to clean up a bad or stale test run.
|
111
|
+
|
112
|
+
Server Groups
|
113
|
+
=============
|
114
|
+
|
115
|
+
The VM system itself is a mapping of server groups to an array of provisioning
|
116
|
+
"commands", which are implemented as classes with a consistent interface
|
117
|
+
(visitors). A provisioning command may create a [vagrant
|
118
|
+
prison](https://github.com/chef-workflow/vagrant-prison) which contains all the
|
119
|
+
servers for that server group, complete with assigning them a host-only
|
120
|
+
interface and storing that with `IPSupport` so that it can be retrieved by
|
121
|
+
other bits of the test system or task system. Another provisioning command may
|
122
|
+
execute the in-code equivalent of `knife bootstrap` to build out your servers
|
123
|
+
with a role named after the server group. For de-provisioning, the provisioning
|
124
|
+
commands are played in reverse with a `shutdown` call applied to them.
|
125
|
+
|
126
|
+
Scheduler and VM libraries
|
127
|
+
==========================
|
128
|
+
|
129
|
+
* `Scheduler` - this is the meat; if you're driving a new testing system such
|
130
|
+
as rspec, you'll want to get real familiar with the interface presented.
|
131
|
+
* `VM` - marshalling and delegation interface. Most of this is exposed to the
|
132
|
+
scheduler interface.
|
133
|
+
* `VM::VagrantProvisioner` - creates a vagrant prison composed of n servers for
|
134
|
+
a server group with a unique host-only ip for each server.
|
135
|
+
* `VM::EC2Provisioner` - creates a group of servers provisioned on EC2.
|
136
|
+
* `VM::KnifeProvisioner` - the provisioner equivalent of `knife bootstrap`, with
|
137
|
+
additional sanity checks for both converge success and a waiting period for
|
138
|
+
search indexing. On deprovision, deletes the nodes that were created. Will
|
139
|
+
always try to bootstrap a group in parallel.
|
140
|
+
* `VM::ChefServerProvisioner` - similar to `VM::KnifeProvisioner`, it wraps the
|
141
|
+
[knife-server](https://github.com/fnichol/knife-server) toolkit to create a
|
142
|
+
chef server.
|
143
|
+
|
144
|
+
Contributing
|
145
|
+
------------
|
146
|
+
|
147
|
+
* fork the project
|
148
|
+
* make a branch
|
149
|
+
* add your stuff
|
150
|
+
* push your branch to your repo
|
151
|
+
* send a pull request
|
152
|
+
|
153
|
+
**Note:** modifications to gem metadata, author lists, and other credits
|
154
|
+
without rationale will be rejected immediately.
|
155
|
+
|
156
|
+
Credits
|
157
|
+
-------
|
158
|
+
|
159
|
+
This work was partially sponsored by [Hotel Tonight](http://hoteltonight.com)
|
160
|
+
and is what they use to test our infrastructure internally. Primarily authored by
|
161
|
+
[Erik Hollensbe](https://github.com/erikh).
|
data/Rakefile
ADDED
@@ -0,0 +1,149 @@
|
|
1
|
+
#!ruby
|
2
|
+
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
unless $stdin.tty?
|
6
|
+
$stderr.puts "Please run this from a terminal in your git repository's root."
|
7
|
+
exit 1
|
8
|
+
end
|
9
|
+
|
10
|
+
if ARGV[0] =~ /^--?h(?:elp)?$/
|
11
|
+
$stderr.puts 'Creates the necessary files needed for bootstrapping chef-workflow.'
|
12
|
+
$stderr.puts 'Ignores any files that already exist.'
|
13
|
+
exit 1
|
14
|
+
end
|
15
|
+
|
16
|
+
def create(filename, content)
|
17
|
+
print "Trying to create #{filename}..."
|
18
|
+
if File.exist?(filename)
|
19
|
+
puts "ignoring, file already exists."
|
20
|
+
else
|
21
|
+
FileUtils.mkdir_p(File.dirname(filename))
|
22
|
+
File.binwrite(filename, content)
|
23
|
+
puts "done."
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
RAKEFILE_CONTENT = <<-EOF
|
28
|
+
require 'chef-workflow-tasklib'
|
29
|
+
#
|
30
|
+
# If you want to change the tasks, remove this next line, and see the README:
|
31
|
+
#
|
32
|
+
# https://github.com/chef-workflow/chef-workflow-tasklib#picking-your-own-workflow
|
33
|
+
#
|
34
|
+
chef_workflow_task 'default'
|
35
|
+
EOF
|
36
|
+
|
37
|
+
CONFIG_CONTENT = <<-EOF
|
38
|
+
#
|
39
|
+
# chef-workflow configuration. You can inspect the full set of configured values
|
40
|
+
# with: `bundle exec rake chef:show_config`
|
41
|
+
#
|
42
|
+
|
43
|
+
#
|
44
|
+
# This is general configuration -- related to no specific component. Use with
|
45
|
+
# caution! Most of these settings don't need to be adjusted normally.
|
46
|
+
#
|
47
|
+
|
48
|
+
configure_general do
|
49
|
+
# change the location of the chef-workflow generated files
|
50
|
+
# workflow_dir File.join(Dir.pwd, '.chef-workflow')
|
51
|
+
#
|
52
|
+
# change the location of the VM database
|
53
|
+
# vm_file '.chef-workflow/vm'
|
54
|
+
#
|
55
|
+
# change the location of the chef-server prison file (vagrant only)
|
56
|
+
# chef_server_prison '.chef-workflow/chef-server'
|
57
|
+
#
|
58
|
+
# change the type of machine provisioner. You can use a class or :ec2 or
|
59
|
+
# :vagrant which map to the appropriate provisioner classes. If you use a
|
60
|
+
# custom class, make sure you require it first!
|
61
|
+
# machine_provisioner :vagrant
|
62
|
+
end
|
63
|
+
|
64
|
+
#
|
65
|
+
# This section configures all things related to vagrant usage
|
66
|
+
#
|
67
|
+
configure_vagrant do
|
68
|
+
# if you wish to use a different box, supply it as a url here.
|
69
|
+
# box_url "http://files.vagrantup.com/precise32.box"
|
70
|
+
end
|
71
|
+
|
72
|
+
configure_knife do
|
73
|
+
#
|
74
|
+
# you shouldn't need to modify most of these! be sure to 'bundle exec rake
|
75
|
+
# chef:show_config' to see the standard values, as well as others that are
|
76
|
+
# not listed here.
|
77
|
+
#
|
78
|
+
# cookbooks_path 'cookbooks'
|
79
|
+
# roles_path 'roles'
|
80
|
+
# environments_path 'environments'
|
81
|
+
# data_bags_path 'data_bags'
|
82
|
+
# ssh_user 'vagrant'
|
83
|
+
# ssh_password 'vagrant'
|
84
|
+
# ssh_identity_file nil
|
85
|
+
# use_sudo true
|
86
|
+
# test_environment 'vagrant'
|
87
|
+
|
88
|
+
# search_index_wait is used for provisioning with VM::KnifeProvisioner. After
|
89
|
+
# provisioning, it waits a set value of seconds (60 default) for the metadata
|
90
|
+
# to appear in chef's index. If this fails to appear in this time an
|
91
|
+
# exception will be thrown. If you have a slow chef server or find yourself
|
92
|
+
# hitting this limit a lot, increase this value.
|
93
|
+
#
|
94
|
+
# search_index_wait 60
|
95
|
+
end
|
96
|
+
|
97
|
+
#
|
98
|
+
# EC2 settings.
|
99
|
+
#
|
100
|
+
configure_ec2 do
|
101
|
+
#
|
102
|
+
# You'll want to heavily edit these if you use the EC2 support. Additionally,
|
103
|
+
# it is *strongly advised* to use a separate AWS account for your testing.
|
104
|
+
#
|
105
|
+
# The access key ID - no default, uses environment if it doesn't exist.
|
106
|
+
# access_key_id "my_access_key"
|
107
|
+
#
|
108
|
+
# The secret access key - no default, uses environment if it doesn't exist.
|
109
|
+
# secret_access_key "my_secret"
|
110
|
+
#
|
111
|
+
# The AMI of provisioned boxes - no default!
|
112
|
+
# ami "ami-abcdef123"
|
113
|
+
#
|
114
|
+
# The instance type of provisioned boxes - no default!
|
115
|
+
# instance_type "m1.small"
|
116
|
+
#
|
117
|
+
# The region where the instances should be allocated - no default!
|
118
|
+
# region "us-west-1"
|
119
|
+
#
|
120
|
+
# The ssh key used for creating machines (you still have to configure knife
|
121
|
+
# separately) - no default!
|
122
|
+
# ssh_key "test"
|
123
|
+
#
|
124
|
+
# The security groups to put all machines in; if set to :auto (the default),
|
125
|
+
# creates a new one until chef:clean is run.
|
126
|
+
# security_groups :auto
|
127
|
+
#
|
128
|
+
# The open ports in the security groups that are created dynamically. If you
|
129
|
+
# need something more complicated, create your own groups by hand and set
|
130
|
+
# `security_groups` above to those values.
|
131
|
+
# security_group_open_ports [22, 4000]
|
132
|
+
end
|
133
|
+
|
134
|
+
#
|
135
|
+
# This is largely important for local VM work. If you're using something like
|
136
|
+
# EC2 support (which determines its own IPs, for obvious reasons) these
|
137
|
+
# configuration settings are meaningless.
|
138
|
+
#
|
139
|
+
configure_ips do
|
140
|
+
# this is treated as a /24, walked incrementally, and with vagrant at least,
|
141
|
+
# .1 is spoken for. You'll want 0 as your last octet. Expect this to be less
|
142
|
+
# fail in the future. :)
|
143
|
+
#
|
144
|
+
# subnet "10.10.10.0"
|
145
|
+
end
|
146
|
+
EOF
|
147
|
+
|
148
|
+
create('Rakefile', RAKEFILE_CONTENT)
|
149
|
+
create('lib/chef-workflow-config.rb', CONFIG_CONTENT)
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'chef-workflow/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |gem|
|
7
|
+
gem.name = "chef-workflow"
|
8
|
+
gem.version = Chef::Workflow::VERSION
|
9
|
+
gem.authors = ["Erik Hollensbe"]
|
10
|
+
gem.email = ["erik+github@hollensbe.org"]
|
11
|
+
gem.description = %q{A comprehensive rake-based workflow for chef}
|
12
|
+
gem.summary = %q{A comprehensive rake-based workflow for chef}
|
13
|
+
gem.homepage = "https://github.com/chef-workflow/chef-workflow"
|
14
|
+
|
15
|
+
gem.files = `git ls-files`.split($/)
|
16
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
17
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
18
|
+
gem.require_paths = ["lib"]
|
19
|
+
|
20
|
+
gem.add_dependency 'vagrant-prison', '~> 0.1.0'
|
21
|
+
gem.add_dependency 'chef', '~> 10.0'
|
22
|
+
gem.add_dependency 'aws-sdk', '~> 1.7.0'
|
23
|
+
gem.add_dependency 'net-ssh', '~> 2.6.0'
|
24
|
+
|
25
|
+
gem.add_development_dependency 'rdoc'
|
26
|
+
gem.add_development_dependency 'rake'
|
27
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
require "chef-workflow/version"
|
2
|
+
|
3
|
+
require 'chef-workflow/support/general'
|
4
|
+
require 'chef-workflow/support/knife'
|
5
|
+
require 'chef-workflow/support/vagrant'
|
6
|
+
require 'chef-workflow/support/ip'
|
7
|
+
require 'chef-workflow/support/debug'
|
8
|
+
require 'chef-workflow/support/ec2'
|
9
|
+
|
10
|
+
class Chef
|
11
|
+
module Workflow
|
12
|
+
#
|
13
|
+
# Basic helpers (intended to be mixed in elsewhere) to configure the
|
14
|
+
# various support configuration systems.
|
15
|
+
#
|
16
|
+
module ConfigureHelper
|
17
|
+
#
|
18
|
+
# Configure 'GeneralSupport'
|
19
|
+
#
|
20
|
+
def configure_general(&block)
|
21
|
+
GeneralSupport.configure(&block)
|
22
|
+
end
|
23
|
+
|
24
|
+
#
|
25
|
+
# Configure 'KnifeSupport'
|
26
|
+
#
|
27
|
+
def configure_knife(&block)
|
28
|
+
KnifeSupport.configure(&block)
|
29
|
+
end
|
30
|
+
|
31
|
+
#
|
32
|
+
# Configure 'VagrantSupport'
|
33
|
+
#
|
34
|
+
def configure_vagrant(&block)
|
35
|
+
VagrantSupport.configure(&block)
|
36
|
+
end
|
37
|
+
|
38
|
+
#
|
39
|
+
# Configure 'IPSupport' - you probably don't need to do this.
|
40
|
+
#
|
41
|
+
def configure_ips(&block)
|
42
|
+
IPSupport.configure(&block)
|
43
|
+
end
|
44
|
+
|
45
|
+
#
|
46
|
+
# Configure 'EC2Support'
|
47
|
+
#
|
48
|
+
def configure_ec2(&block)
|
49
|
+
EC2Support.configure(&block)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
class << eval("self", TOPLEVEL_BINDING)
|
56
|
+
include Chef::Workflow::ConfigureHelper
|
57
|
+
end
|
58
|
+
|
59
|
+
if defined? Rake::DSL
|
60
|
+
module Rake::DSL
|
61
|
+
include Chef::Workflow::ConfigureHelper
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
$:.unshift 'lib'
|
66
|
+
|
67
|
+
begin
|
68
|
+
require 'chef-workflow-config'
|
69
|
+
rescue LoadError
|
70
|
+
$stderr.puts "There is no chef-workflow-config in your lib directory."
|
71
|
+
$stderr.puts "Please run chef-workflow-bootstrap or add one."
|
72
|
+
end
|
73
|
+
|