chef-workflow 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,33 @@
1
+ require 'chef/application/knife'
2
+ require 'chef/knife'
3
+ require 'stringio'
4
+
5
+ #
6
+ # Mixin to add methods to assist with creating knife plugins.
7
+ #
8
+ module KnifePluginSupport
9
+
10
+ #
11
+ # Given a class name for a plugin compatible with the Chef::Knife interface,
12
+ # initializes it and makes it available for execution. It also overrides the
13
+ # `ui` object to use `StringIO` objects, which allow you to choose when and
14
+ # if you display the output of the commands by referencing
15
+ # `obj.ui.stdout.string` and similar calls.
16
+ #
17
+ # The second argument is an array of arguments to the command, such as they
18
+ # would be presented to a command line tool as `ARGV`.
19
+ #
20
+ def init_knife_plugin(klass, args)
21
+ klass.options = Chef::Application::Knife.options.merge(klass.options)
22
+ klass.load_deps
23
+ cli = klass.new(args)
24
+ cli.ui = Chef::Knife::UI.new(
25
+ StringIO.new('', 'w'),
26
+ StringIO.new('', 'w'),
27
+ StringIO.new('', 'r'),
28
+ cli.config
29
+ )
30
+
31
+ return cli
32
+ end
33
+ end
@@ -0,0 +1,122 @@
1
+ require 'fileutils'
2
+ require 'erb'
3
+ require 'chef-workflow/support/generic'
4
+ require 'chef-workflow/support/general'
5
+ require 'chef-workflow/support/debug'
6
+
7
+ #
8
+ # Configuration class for chef tooling and SSH interaction. Uses `GenericSupport`.
9
+ #
10
+ class KnifeSupport
11
+ include GenericSupport
12
+ include DebugSupport
13
+
14
+ # defaults, yo
15
+ DEFAULTS = {
16
+ :search_index_wait => 60,
17
+ :cookbooks_path => File.join(Dir.pwd, 'cookbooks'),
18
+ :chef_config_path => File.join(GeneralSupport.singleton.workflow_dir, 'chef'),
19
+ :knife_config_path => File.join(GeneralSupport.singleton.workflow_dir, 'chef', 'knife.rb'),
20
+ :roles_path => File.join(Dir.pwd, 'roles'),
21
+ :environments_path => File.join(Dir.pwd, 'environments'),
22
+ :data_bags_path => File.join(Dir.pwd, 'data_bags'),
23
+ :ssh_user => "vagrant",
24
+ :ssh_password => "vagrant",
25
+ :ssh_identity_file => nil,
26
+ :use_sudo => true,
27
+ :test_environment => "vagrant",
28
+ :test_recipes => []
29
+ }
30
+
31
+ DEFAULTS[:knife_config_template] = <<-EOF
32
+ log_level :info
33
+ log_location STDOUT
34
+ node_name 'test-user'
35
+ client_key File.join('<%= KnifeSupport.singleton.chef_config_path %>', 'admin.pem')
36
+ validation_client_name 'chef-validator'
37
+ validation_key File.join('<%= KnifeSupport.singleton.chef_config_path %>', 'validation.pem')
38
+ chef_server_url 'http://<%= IPSupport.singleton.get_role_ips("chef-server").first %>:4000'
39
+ environment '<%= KnifeSupport.singleton.test_environment %>'
40
+ cache_type 'BasicFile'
41
+ cache_options( :path => File.join('<%= KnifeSupport.singleton.chef_config_path %>', 'checksums' ))
42
+ cookbook_path [ '<%= KnifeSupport.singleton.cookbooks_path %>' ]
43
+ EOF
44
+
45
+ #
46
+ # Helper method to allow extensions to add attributes to this class. Could
47
+ # probably be replaced by `AttrSupport`. Takes an attribute name and a
48
+ # default which will be set initially, intended to be overridden by the user
49
+ # if necessary.
50
+ #
51
+ def self.add_attribute(attr_name, default)
52
+ KnifeSupport.configure
53
+
54
+ DEFAULTS[attr_name] = default # a little inelegant, but it works.
55
+
56
+ # HACK: no good way to hook this right now, revisit later.
57
+ str = ""
58
+ if attr_name.to_s == "knife_config_path"
59
+ str = <<-EOF
60
+ def #{attr_name}=(arg)
61
+ @#{attr_name} = arg
62
+ ENV["CHEF_CONFIG"] = arg
63
+ end
64
+
65
+ def #{attr_name}(arg=nil)
66
+ if arg
67
+ @#{attr_name} = arg
68
+ ENV["CHEF_CONFIG"] = arg
69
+ end
70
+
71
+ @#{attr_name}
72
+ end
73
+ EOF
74
+ else
75
+ str = <<-EOF
76
+ def #{attr_name}=(arg)
77
+ @#{attr_name} = arg
78
+ end
79
+
80
+ def #{attr_name}(arg=nil)
81
+ if arg
82
+ @#{attr_name} = arg
83
+ end
84
+
85
+ @#{attr_name}
86
+ end
87
+ EOF
88
+ end
89
+
90
+ KnifeSupport.singleton.instance_eval str
91
+ KnifeSupport.singleton.send(attr_name, default)
92
+ end
93
+
94
+ DEFAULTS.each { |key, value| add_attribute(key, value) }
95
+
96
+ def initialize(options={})
97
+ DEFAULTS.each do |key, value|
98
+ instance_variable_set(
99
+ "@#{key}",
100
+ options.has_key?(key) ? options[key] : DEFAULTS[key]
101
+ )
102
+ end
103
+ end
104
+
105
+ def method_missing(sym, *args)
106
+ if_debug(2) do
107
+ $stderr.puts "#{self.class.name}'s #{sym} method was referenced while trying to configure #{self.class.name}"
108
+ $stderr.puts "#{self.class.name} has not been configured to support this feature."
109
+ $stderr.puts "This is probably due to it being dynamically added from a rake task, and you're running the test suite."
110
+ $stderr.puts "It's probably harmless. Expect a better solution than this debug message soon."
111
+ end
112
+ end
113
+
114
+ #
115
+ # Writes out a knife.rb based on the settings in this configuration. Uses the
116
+ # `knife_config_path` and `chef_config_path` to determine where to write it.
117
+ #
118
+ def build_knife_config
119
+ FileUtils.mkdir_p(chef_config_path)
120
+ File.binwrite(knife_config_path, ERB.new(knife_config_template).result(binding))
121
+ end
122
+ end
@@ -0,0 +1,403 @@
1
+ require 'set'
2
+ require 'thread'
3
+ require 'timeout'
4
+ require 'chef-workflow/support/attr'
5
+ require 'chef-workflow/support/debug'
6
+ require 'chef-workflow/support/vm'
7
+
8
+ #
9
+ # This is a scheduler for provisioners. It can run in parallel or serial mode,
10
+ # and is dependency-based, that is, it will only schedule items for execution
11
+ # which have all their dependencies satisfied and items that haven't will wait
12
+ # to execute until that happens.
13
+ #
14
+ class Scheduler
15
+ extend AttrSupport
16
+ include DebugSupport
17
+
18
+ ##
19
+ # :attr:
20
+ #
21
+ # Turn serial mode on (off by default). This forces the scheduler to execute
22
+ # every provision in order, even if it could handle multiple provisions at
23
+ # the same time.
24
+ #
25
+ fancy_attr :serial
26
+
27
+ ##
28
+ # :attr:
29
+ #
30
+ # Ignore exceptions while deprovisioning. Default is false.
31
+ #
32
+
33
+ fancy_attr :force_deprovision
34
+
35
+ #
36
+ # Constructor. If the first argument is true, will install an `at_exit` hook
37
+ # to write out the VM and IP databases.
38
+ #
39
+ def initialize(at_exit_hook=true)
40
+ @force_deprovision = false
41
+ @solved_mutex = Mutex.new
42
+ @waiters_mutex = Mutex.new
43
+ @serial = false
44
+ @solver_thread = nil
45
+ @working = { }
46
+ @waiters = Set.new
47
+ @queue = Queue.new
48
+ @vm = VM.load_from_file || VM.new
49
+
50
+ if at_exit_hook
51
+ at_exit { write_state }
52
+ end
53
+ end
54
+
55
+ #
56
+ # Write out the VM and IP databases.
57
+ #
58
+ def write_state
59
+ @vm.save_to_file
60
+ # FIXME not the best place to do this, but we have additional problems if
61
+ # we don't
62
+ IPSupport.singleton.write
63
+ end
64
+
65
+ #
66
+ # Helper to assist with dealing with a VM object
67
+ #
68
+ def solved
69
+ @vm.provisioned
70
+ end
71
+
72
+ #
73
+ # Helper to assist with dealing with a VM object
74
+ #
75
+ def vm_groups
76
+ @vm.groups
77
+ end
78
+
79
+ #
80
+ # Helper to assist with dealing with a VM object
81
+ #
82
+ def vm_dependencies
83
+ @vm.dependencies
84
+ end
85
+
86
+ #
87
+ # Helper to assist with dealing with a VM object
88
+ #
89
+ def vm_working
90
+ @vm.working
91
+ end
92
+
93
+ #
94
+ # Schedule a group of VMs for provision. This takes a group name, which is a
95
+ # string, an array of provisioner objects, and a list of string dependencies.
96
+ # If anything in the dependencies list hasn't been pre-declared, it refuses
97
+ # to continue.
98
+ #
99
+ # This method will return nil if the server group is already provisioned.
100
+ #
101
+ def schedule_provision(group_name, provisioner, dependencies=[])
102
+ return nil if vm_groups[group_name]
103
+ provisioner = [provisioner] unless provisioner.kind_of?(Array)
104
+ provisioner.each { |x| x.name = group_name }
105
+ vm_groups[group_name] = provisioner
106
+
107
+ unless dependencies.all? { |x| vm_groups.has_key?(x) }
108
+ raise "One of your dependencies for #{group_name} has not been pre-declared. Cannot continue"
109
+ end
110
+
111
+ vm_dependencies[group_name] = dependencies.to_set
112
+ @waiters_mutex.synchronize do
113
+ @waiters.add(group_name)
114
+ end
115
+ end
116
+
117
+ #
118
+ # Sleep until this list of dependencies are resolved. In parallel mode, will
119
+ # raise if an exeception occurred while waiting for these resources. In
120
+ # serial mode, wait_for just returns nil.
121
+ #
122
+ def wait_for(*dependencies)
123
+ return nil if @serial
124
+ return nil if dependencies.empty?
125
+
126
+ dep_set = dependencies.to_set
127
+ until dep_set & solved == dep_set
128
+ sleep 1
129
+ @solver_thread.join unless @solver_thread.alive?
130
+ end
131
+ end
132
+
133
+ #
134
+ # Helper method for scheduling. Wraps items in a timeout and immediately
135
+ # checks all running workers for exceptions, which are immediately bubbled up
136
+ # if there are any. If do_loop is true, it will retry the timeout.
137
+ #
138
+ def with_timeout(do_loop=true)
139
+ Timeout.timeout(10) do
140
+ dead_working = @working.values.reject(&:alive?)
141
+ if dead_working.size > 0
142
+ dead_working.map(&:join)
143
+ end
144
+
145
+ yield
146
+ end
147
+ rescue TimeoutError
148
+ retry if do_loop
149
+ end
150
+
151
+ #
152
+ # Start the scheduler. In serial mode this call will block until the whole
153
+ # dependency graph is satisfied, or one of the provisions fails, at which
154
+ # point an exception will be raised. In parallel mode, this call completes
155
+ # immediately, and you should use #wait_for to control main thread flow.
156
+ #
157
+ # This call also installs a SIGINFO (Ctrl+T in the terminal on macs) and
158
+ # SIGUSR2 handler which can be used to get information on the status of
159
+ # what's solved and what's working.
160
+ #
161
+ # Immediately returns if in threaded mode and the solver is already running.
162
+ #
163
+ def run
164
+ # short circuit if we're not serial and already running
165
+ return if @solver_thread and !@serial
166
+
167
+ handler = lambda do |*args|
168
+ p ["solved:", solved]
169
+ p ["working:", @working]
170
+ p ["waiting:", @waiters]
171
+ end
172
+
173
+ %w[USR2 INFO].each { |sig| trap(sig, &handler) if Signal.list[sig] }
174
+
175
+ queue_runner = lambda do
176
+ run = true
177
+
178
+ while run
179
+ service_resolved_waiters
180
+
181
+ ready = []
182
+
183
+ if @queue.empty?
184
+ if @serial
185
+ return
186
+ else
187
+ with_timeout do
188
+ # this is where most of the execution time is spent, so ensure
189
+ # waiters get considered here.
190
+ service_resolved_waiters
191
+ ready << @queue.shift
192
+ end
193
+ end
194
+ end
195
+
196
+ while !@queue.empty?
197
+ ready << @queue.shift
198
+ end
199
+
200
+ ready.each do |r|
201
+ if r
202
+ @solved_mutex.synchronize do
203
+ solved.add(r)
204
+ @working.delete(r)
205
+ vm_working.delete(r)
206
+ end
207
+ else
208
+ run = false
209
+ end
210
+ end
211
+ end
212
+ end
213
+
214
+ if @serial
215
+ service_resolved_waiters
216
+ queue_runner.call
217
+ else
218
+ @solver_thread = Thread.new do
219
+ with_timeout(false) { service_resolved_waiters }
220
+ queue_runner.call
221
+ end
222
+
223
+ # we depend on at_exit hooks being fired, and Thread#abort_on_exception
224
+ # doesn't fire them. This solution bubbles up the exceptions in a similar
225
+ # fashion without actually sacrificing the at_exit functionality.
226
+ Thread.new do
227
+ begin
228
+ @solver_thread.join
229
+ rescue Exception => e
230
+ $stderr.puts "Solver thread encountered an exception:"
231
+ $stderr.puts "#{e.class.name}: #{e.message}"
232
+ $stderr.puts e.backtrace.join("\n")
233
+ Kernel.exit 1
234
+ end
235
+ end
236
+ end
237
+ end
238
+
239
+ #
240
+ # Instructs the scheduler to stop. Note that this is not an interrupt, and
241
+ # the queue will still be exhausted before terminating.
242
+ #
243
+ def stop
244
+ if @serial
245
+ @queue << nil
246
+ else
247
+ @working.values.map { |v| v.join rescue nil }
248
+ @queue << nil
249
+ @solver_thread.join rescue nil
250
+ end
251
+ end
252
+
253
+ #
254
+ # This method determines what 'waiters', or provisioners that cannot
255
+ # provision yet because of unresolved dependencies, can be executed.
256
+ #
257
+ def service_resolved_waiters
258
+ @waiters_mutex.synchronize do
259
+ @waiters -= (@working.keys.to_set + solved)
260
+ end
261
+
262
+ waiter_iteration = lambda do
263
+ @waiters.each do |group_name|
264
+ if (solved & vm_dependencies[group_name]) == vm_dependencies[group_name]
265
+ if_debug do
266
+ $stderr.puts "Provisioning #{group_name}"
267
+ end
268
+
269
+ provisioner = vm_groups[group_name]
270
+
271
+ provision_block = lambda do
272
+ # FIXME maybe a way to specify initial args?
273
+ args = nil
274
+ provisioner.each do |this_prov|
275
+ unless args = this_prov.startup(args)
276
+ $stderr.puts "Could not provision #{group_name} with provisioner #{this_prov.class.name}"
277
+ raise "Could not provision #{group_name} with provisioner #{this_prov.class.name}"
278
+ end
279
+ end
280
+ @queue << group_name
281
+ end
282
+
283
+ vm_working.add(group_name)
284
+
285
+ if @serial
286
+ # HACK: just give the working check something that will always work.
287
+ # Probably should just mock it.
288
+ @working[group_name] = Thread.new { sleep }
289
+ provision_block.call
290
+ else
291
+ @working[group_name] = Thread.new(&provision_block)
292
+ end
293
+ end
294
+ end
295
+ end
296
+
297
+ if @serial
298
+ waiter_iteration.call
299
+ else
300
+ @waiters_mutex.synchronize(&waiter_iteration)
301
+ end
302
+ end
303
+
304
+ #
305
+ # Teardown a single group -- modifies the solved formula. Be careful to
306
+ # resupply dependencies if you use this, as nothing will resolve until you
307
+ # resupply it.
308
+ #
309
+ # This takes an optional argument to wait for the group to be solved before
310
+ # attempting to tear it down. Setting this to false effectively says, "I know
311
+ # what I'm doing", and you should feel bad if you file an issue because you
312
+ # supplied it.
313
+ #
314
+
315
+ def teardown_group(group_name, wait=true)
316
+ wait_for(group_name) if wait
317
+
318
+ dependent_items = vm_dependencies.partition { |k,v| v.include?(group_name) }.first.map(&:first)
319
+
320
+ if_debug do
321
+ if dependent_items.length > 0
322
+ $stderr.puts "Trying to terminate #{group_name}, found #{dependent_items.inspect} depending on it"
323
+ end
324
+ end
325
+
326
+ @solved_mutex.synchronize do
327
+ dependent_and_working = @working.keys & dependent_items
328
+
329
+ if dependent_and_working.count > 0
330
+ $stderr.puts "#{dependent_and_working.inspect} are depending on #{group_name}, which you are trying to deprovision."
331
+ $stderr.puts "We can't resolve this problem for you, and future converges may fail during this run that would otherwise work."
332
+ $stderr.puts "Consider using wait_for to better control the dependencies, or turning serial provisioning on."
333
+ end
334
+
335
+ deprovision_group(group_name)
336
+ end
337
+
338
+ end
339
+
340
+ #
341
+ # Performs the deprovision of a group by replaying its provision strategy
342
+ # backwards and applying the #shutdown method instead of the #startup method.
343
+ # Removes it from the various state tables if true is set as the second
344
+ # argument, which is the default.
345
+ #
346
+ def deprovision_group(group_name, clean_state=true)
347
+ provisioner = vm_groups[group_name]
348
+
349
+ # if we can't find the provisioner, we probably got asked to clean up
350
+ # something we never scheduled. Just ignore that.
351
+ if provisioner
352
+ if_debug do
353
+ $stderr.puts "Attempting to deprovision group #{group_name}"
354
+ end
355
+
356
+ perform_deprovision = lambda do |this_prov|
357
+ unless this_prov.shutdown
358
+ if_debug do
359
+ $stderr.puts "Could not deprovision group #{group_name}."
360
+ end
361
+ end
362
+ end
363
+
364
+ provisioner.reverse.each do |this_prov|
365
+ if @force_deprovision
366
+ begin
367
+ perform_deprovision.call(this_prov)
368
+ rescue Exception => e
369
+ if_debug do
370
+ $stderr.puts "Deprovision #{this_prov.class.name}/#{group_name} had errors:"
371
+ $stderr.puts "#{e.message}"
372
+ end
373
+ end
374
+ else
375
+ perform_deprovision.call(this_prov)
376
+ end
377
+ end
378
+ end
379
+
380
+ if clean_state
381
+ solved.delete(group_name)
382
+ vm_working.delete(group_name)
383
+ vm_dependencies.delete(group_name)
384
+ vm_groups.delete(group_name)
385
+ end
386
+ end
387
+
388
+ #
389
+ # Instruct all provisioners except ones in the exception list to tear down.
390
+ # Calls #stop as its first action.
391
+ #
392
+ # This is always done serially. For sanity.
393
+ #
394
+ def teardown(exceptions=[])
395
+ stop
396
+
397
+ (vm_groups.keys.to_set - exceptions.to_set).each do |group_name|
398
+ deprovision_group(group_name) # clean this after everything finishes
399
+ end
400
+
401
+ write_state
402
+ end
403
+ end