chef-workflow 0.1.0

data/lib/chef-workflow/support/knife-plugin.rb

@@ -0,0 +1,33 @@
+require 'chef/application/knife'
+require 'chef/knife'
+require 'stringio'
+
+#
+# Mixin to add methods to assist with creating knife plugins.
+#
+module KnifePluginSupport
+
+  #
+  # Given a class name for a plugin compatible with the Chef::Knife interface,
+  # initializes it and makes it available for execution. It also overrides the
+  # `ui` object to use `StringIO` objects, which allow you to choose when and
+  # if you display the output of the commands by referencing
+  # `obj.ui.stdout.string` and similar calls.
+  #
+  # The second argument is an array of arguments to the command, such as they
+  # would be presented to a command line tool as `ARGV`.
+  #
+  def init_knife_plugin(klass, args)
+    klass.options = Chef::Application::Knife.options.merge(klass.options)
+    klass.load_deps
+    cli = klass.new(args)
+    cli.ui = Chef::Knife::UI.new(
+      StringIO.new('', 'w'), 
+      StringIO.new('', 'w'),
+      StringIO.new('', 'r'),
+      cli.config
+    )
+
+    return cli
+  end
+end

data/lib/chef-workflow/support/knife.rb

@@ -0,0 +1,122 @@
+require 'fileutils'
+require 'erb'
+require 'chef-workflow/support/generic'
+require 'chef-workflow/support/general'
+require 'chef-workflow/support/debug'
+
+#
+# Configuration class for chef tooling and SSH interaction. Uses `GenericSupport`.
+#
+class KnifeSupport
+  include GenericSupport
+  include DebugSupport
+
+  # defaults, yo
+  DEFAULTS = {
+    :search_index_wait      => 60,
+    :cookbooks_path         => File.join(Dir.pwd, 'cookbooks'),
+    :chef_config_path       => File.join(GeneralSupport.singleton.workflow_dir, 'chef'),
+    :knife_config_path      => File.join(GeneralSupport.singleton.workflow_dir, 'chef', 'knife.rb'),
+    :roles_path             => File.join(Dir.pwd, 'roles'),
+    :environments_path      => File.join(Dir.pwd, 'environments'),
+    :data_bags_path         => File.join(Dir.pwd, 'data_bags'),
+    :ssh_user               => "vagrant",
+    :ssh_password           => "vagrant",
+    :ssh_identity_file      => nil,
+    :use_sudo               => true,
+    :test_environment       => "vagrant",
+    :test_recipes           => []
+  }
+
+  DEFAULTS[:knife_config_template] = <<-EOF
+  log_level                :info
+  log_location             STDOUT
+  node_name                'test-user'
+  client_key               File.join('<%= KnifeSupport.singleton.chef_config_path %>', 'admin.pem')
+  validation_client_name   'chef-validator'
+  validation_key           File.join('<%= KnifeSupport.singleton.chef_config_path %>', 'validation.pem')
+  chef_server_url          'http://<%= IPSupport.singleton.get_role_ips("chef-server").first %>:4000'
+  environment              '<%= KnifeSupport.singleton.test_environment %>'
+  cache_type               'BasicFile'
+  cache_options( :path => File.join('<%= KnifeSupport.singleton.chef_config_path %>', 'checksums' ))
+  cookbook_path            [ '<%= KnifeSupport.singleton.cookbooks_path %>' ]
+  EOF
+
+  #
+  # Helper method to allow extensions to add attributes to this class. Could
+  # probably be replaced by `AttrSupport`. Takes an attribute name and a
+  # default which will be set initially, intended to be overridden by the user
+  # if necessary.
+  #
+  def self.add_attribute(attr_name, default)
+    KnifeSupport.configure
+
+    DEFAULTS[attr_name] = default # a little inelegant, but it works.
+
+    # HACK: no good way to hook this right now, revisit later.
+    str = ""
+    if attr_name.to_s == "knife_config_path"
+      str = <<-EOF
+        def #{attr_name}=(arg)
+          @#{attr_name} = arg
+          ENV["CHEF_CONFIG"] = arg
+        end
+
+        def #{attr_name}(arg=nil)
+          if arg
+            @#{attr_name} = arg
+            ENV["CHEF_CONFIG"] = arg
+          end
+          
+          @#{attr_name}
+        end
+      EOF
+    else
+      str = <<-EOF
+        def #{attr_name}=(arg)
+          @#{attr_name} = arg
+        end
+
+        def #{attr_name}(arg=nil)
+          if arg
+            @#{attr_name} = arg
+          end
+          
+          @#{attr_name}
+        end
+      EOF
+    end
+
+    KnifeSupport.singleton.instance_eval str
+    KnifeSupport.singleton.send(attr_name, default)
+  end
+  
+  DEFAULTS.each { |key, value| add_attribute(key, value) }
+
+  def initialize(options={})
+    DEFAULTS.each do |key, value|
+      instance_variable_set(
+        "@#{key}", 
+        options.has_key?(key) ? options[key] : DEFAULTS[key]
+      )
+    end
+  end
+
+  def method_missing(sym, *args)
+    if_debug(2) do
+      $stderr.puts "#{self.class.name}'s #{sym} method was referenced while trying to configure #{self.class.name}"
+      $stderr.puts "#{self.class.name} has not been configured to support this feature."
+      $stderr.puts "This is probably due to it being dynamically added from a rake task, and you're running the test suite."
+      $stderr.puts "It's probably harmless. Expect a better solution than this debug message soon."
+    end
+  end
+
+  #
+  # Writes out a knife.rb based on the settings in this configuration. Uses the
+  # `knife_config_path` and `chef_config_path` to determine where to write it.
+  #
+  def build_knife_config
+    FileUtils.mkdir_p(chef_config_path)
+    File.binwrite(knife_config_path, ERB.new(knife_config_template).result(binding))
+  end
+end

data/lib/chef-workflow/support/scheduler.rb

@@ -0,0 +1,403 @@
+require 'set'
+require 'thread'
+require 'timeout'
+require 'chef-workflow/support/attr'
+require 'chef-workflow/support/debug'
+require 'chef-workflow/support/vm'
+
+#
+# This is a scheduler for provisioners. It can run in parallel or serial mode,
+# and is dependency-based, that is, it will only schedule items for execution
+# which have all their dependencies satisfied and items that haven't will wait
+# to execute until that happens.
+#
+class Scheduler
+  extend AttrSupport
+  include DebugSupport
+
+  ##
+  # :attr:
+  #
+  # Turn serial mode on (off by default). This forces the scheduler to execute
+  # every provision in order, even if it could handle multiple provisions at
+  # the same time.
+  #
+  fancy_attr :serial
+
+  ##
+  # :attr:
+  #
+  # Ignore exceptions while deprovisioning. Default is false.
+  #
+
+  fancy_attr :force_deprovision
+
+  #
+  # Constructor. If the first argument is true, will install an `at_exit` hook
+  # to write out the VM and IP databases.
+  #
+  def initialize(at_exit_hook=true)
+    @force_deprovision  = false
+    @solved_mutex       = Mutex.new
+    @waiters_mutex      = Mutex.new
+    @serial             = false
+    @solver_thread      = nil
+    @working            = { }
+    @waiters            = Set.new
+    @queue              = Queue.new
+    @vm                 = VM.load_from_file || VM.new 
+
+    if at_exit_hook
+      at_exit { write_state }
+    end
+  end
+
+  #
+  # Write out the VM and IP databases.
+  #
+  def write_state 
+    @vm.save_to_file
+    # FIXME not the best place to do this, but we have additional problems if
+    #       we don't
+    IPSupport.singleton.write
+  end
+
+  #
+  # Helper to assist with dealing with a VM object
+  #
+  def solved
+    @vm.provisioned
+  end
+
+  #
+  # Helper to assist with dealing with a VM object
+  #
+  def vm_groups
+    @vm.groups
+  end
+  
+  #
+  # Helper to assist with dealing with a VM object
+  #
+  def vm_dependencies
+    @vm.dependencies
+  end
+
+  #
+  # Helper to assist with dealing with a VM object
+  #
+  def vm_working
+    @vm.working
+  end
+
+  #
+  # Schedule a group of VMs for provision. This takes a group name, which is a
+  # string, an array of provisioner objects, and a list of string dependencies.
+  # If anything in the dependencies list hasn't been pre-declared, it refuses
+  # to continue.
+  #
+  # This method will return nil if the server group is already provisioned.
+  #
+  def schedule_provision(group_name, provisioner, dependencies=[])
+    return nil if vm_groups[group_name]
+    provisioner = [provisioner] unless provisioner.kind_of?(Array)
+    provisioner.each { |x| x.name = group_name }
+    vm_groups[group_name] = provisioner
+
+    unless dependencies.all? { |x| vm_groups.has_key?(x) }
+      raise "One of your dependencies for #{group_name} has not been pre-declared. Cannot continue"
+    end
+
+    vm_dependencies[group_name] = dependencies.to_set
+    @waiters_mutex.synchronize do
+      @waiters.add(group_name)
+    end
+  end
+
+  #
+  # Sleep until this list of dependencies are resolved. In parallel mode, will
+  # raise if an exeception occurred while waiting for these resources. In
+  # serial mode, wait_for just returns nil.
+  #
+  def wait_for(*dependencies)
+    return nil if @serial
+    return nil if dependencies.empty?
+
+    dep_set = dependencies.to_set
+    until dep_set & solved == dep_set
+      sleep 1
+      @solver_thread.join unless @solver_thread.alive?
+    end
+  end
+
+  #
+  # Helper method for scheduling. Wraps items in a timeout and immediately
+  # checks all running workers for exceptions, which are immediately bubbled up
+  # if there are any. If do_loop is true, it will retry the timeout.
+  #
+  def with_timeout(do_loop=true)
+    Timeout.timeout(10) do
+      dead_working = @working.values.reject(&:alive?)
+      if dead_working.size > 0
+        dead_working.map(&:join)
+      end
+
+      yield
+    end
+  rescue TimeoutError
+    retry if do_loop
+  end
+
+  #
+  # Start the scheduler. In serial mode this call will block until the whole
+  # dependency graph is satisfied, or one of the provisions fails, at which
+  # point an exception will be raised. In parallel mode, this call completes
+  # immediately, and you should use #wait_for to control main thread flow.
+  #
+  # This call also installs a SIGINFO (Ctrl+T in the terminal on macs) and
+  # SIGUSR2 handler which can be used to get information on the status of
+  # what's solved and what's working. 
+  #
+  # Immediately returns if in threaded mode and the solver is already running.
+  #
+  def run
+    # short circuit if we're not serial and already running
+    return if @solver_thread and !@serial
+
+    handler = lambda do |*args|
+      p ["solved:", solved]
+      p ["working:", @working]
+      p ["waiting:", @waiters]
+    end
+
+    %w[USR2 INFO].each { |sig| trap(sig, &handler) if Signal.list[sig] }
+
+    queue_runner = lambda do
+      run = true
+
+      while run
+        service_resolved_waiters
+
+        ready = []
+
+        if @queue.empty?
+          if @serial
+            return
+          else
+            with_timeout do
+              # this is where most of the execution time is spent, so ensure
+              # waiters get considered here.
+              service_resolved_waiters
+              ready << @queue.shift
+            end
+          end
+        end
+
+        while !@queue.empty?
+          ready << @queue.shift
+        end
+
+        ready.each do |r|
+          if r
+            @solved_mutex.synchronize do
+              solved.add(r)
+              @working.delete(r)
+              vm_working.delete(r)
+            end
+          else
+            run = false
+          end
+        end
+      end
+    end
+
+    if @serial
+      service_resolved_waiters
+      queue_runner.call
+    else
+      @solver_thread = Thread.new do
+        with_timeout(false) { service_resolved_waiters }
+        queue_runner.call
+      end
+
+      # we depend on at_exit hooks being fired, and Thread#abort_on_exception
+      # doesn't fire them. This solution bubbles up the exceptions in a similar
+      # fashion without actually sacrificing the at_exit functionality.
+      Thread.new do
+        begin
+          @solver_thread.join
+        rescue Exception => e
+          $stderr.puts "Solver thread encountered an exception:"
+          $stderr.puts "#{e.class.name}: #{e.message}"
+          $stderr.puts e.backtrace.join("\n")
+          Kernel.exit 1
+        end
+      end
+    end
+  end
+
+  #
+  # Instructs the scheduler to stop. Note that this is not an interrupt, and
+  # the queue will still be exhausted before terminating.
+  #
+  def stop
+    if @serial
+      @queue << nil
+    else
+      @working.values.map { |v| v.join rescue nil }
+      @queue << nil
+      @solver_thread.join rescue nil
+    end
+  end
+
+  #
+  # This method determines what 'waiters', or provisioners that cannot
+  # provision yet because of unresolved dependencies, can be executed.
+  #
+  def service_resolved_waiters
+    @waiters_mutex.synchronize do
+      @waiters -= (@working.keys.to_set + solved)
+    end
+
+    waiter_iteration = lambda do
+      @waiters.each do |group_name|
+        if (solved & vm_dependencies[group_name]) == vm_dependencies[group_name]
+          if_debug do
+            $stderr.puts "Provisioning #{group_name}"
+          end
+
+          provisioner = vm_groups[group_name]
+
+          provision_block = lambda do
+            # FIXME maybe a way to specify initial args?
+            args = nil
+            provisioner.each do |this_prov|
+              unless args = this_prov.startup(args)
+                $stderr.puts "Could not provision #{group_name} with provisioner #{this_prov.class.name}"
+                raise "Could not provision #{group_name} with provisioner #{this_prov.class.name}"
+              end
+            end
+            @queue << group_name
+          end
+
+          vm_working.add(group_name)
+
+          if @serial
+            # HACK: just give the working check something that will always work.
+            #       Probably should just mock it.
+            @working[group_name] = Thread.new { sleep }
+            provision_block.call
+          else
+            @working[group_name] = Thread.new(&provision_block)
+          end
+        end
+      end
+    end
+
+    if @serial
+      waiter_iteration.call
+    else
+      @waiters_mutex.synchronize(&waiter_iteration)
+    end
+  end
+
+  #
+  # Teardown a single group -- modifies the solved formula. Be careful to
+  # resupply dependencies if you use this, as nothing will resolve until you
+  # resupply it.
+  #
+  # This takes an optional argument to wait for the group to be solved before
+  # attempting to tear it down. Setting this to false effectively says, "I know
+  # what I'm doing", and you should feel bad if you file an issue because you
+  # supplied it.
+  #
+
+  def teardown_group(group_name, wait=true)
+    wait_for(group_name) if wait
+
+    dependent_items = vm_dependencies.partition { |k,v| v.include?(group_name) }.first.map(&:first)
+
+    if_debug do
+      if dependent_items.length > 0
+        $stderr.puts "Trying to terminate #{group_name}, found #{dependent_items.inspect} depending on it"
+      end
+    end
+
+    @solved_mutex.synchronize do
+      dependent_and_working = @working.keys & dependent_items
+
+      if dependent_and_working.count > 0
+        $stderr.puts "#{dependent_and_working.inspect} are depending on #{group_name}, which you are trying to deprovision."
+        $stderr.puts "We can't resolve this problem for you, and future converges may fail during this run that would otherwise work."
+        $stderr.puts "Consider using wait_for to better control the dependencies, or turning serial provisioning on."
+      end
+
+      deprovision_group(group_name)
+    end
+
+  end
+
+  #
+  # Performs the deprovision of a group by replaying its provision strategy
+  # backwards and applying the #shutdown method instead of the #startup method.
+  # Removes it from the various state tables if true is set as the second
+  # argument, which is the default.
+  #
+  def deprovision_group(group_name, clean_state=true)
+    provisioner = vm_groups[group_name]
+
+    # if we can't find the provisioner, we probably got asked to clean up
+    # something we never scheduled. Just ignore that.
+    if provisioner
+      if_debug do
+        $stderr.puts "Attempting to deprovision group #{group_name}"
+      end
+
+      perform_deprovision = lambda do |this_prov|
+        unless this_prov.shutdown
+          if_debug do
+            $stderr.puts "Could not deprovision group #{group_name}."
+          end
+        end
+      end
+
+      provisioner.reverse.each do |this_prov|
+        if @force_deprovision
+          begin
+            perform_deprovision.call(this_prov)
+          rescue Exception => e
+            if_debug do
+              $stderr.puts "Deprovision #{this_prov.class.name}/#{group_name} had errors:"
+              $stderr.puts "#{e.message}"
+            end
+          end
+        else
+          perform_deprovision.call(this_prov)
+        end
+      end
+    end
+
+    if clean_state
+      solved.delete(group_name)
+      vm_working.delete(group_name)
+      vm_dependencies.delete(group_name)
+      vm_groups.delete(group_name)
+    end
+  end
+
+  #
+  # Instruct all provisioners except ones in the exception list to tear down.
+  # Calls #stop as its first action.
+  #
+  # This is always done serially. For sanity.
+  #
+  def teardown(exceptions=[])
+    stop
+
+    (vm_groups.keys.to_set - exceptions.to_set).each do |group_name|
+      deprovision_group(group_name) # clean this after everything finishes
+    end
+
+    write_state
+  end
+end