chef-workflow 0.1.0

data/lib/chef-workflow/support/vagrant.rb

@@ -0,0 +1,41 @@
+require 'fileutils'
+require 'chef-workflow/support/generic'
+
+#
+# Vagrant configuration settings. Uses `GenericSupport`.
+#
+class VagrantSupport
+  # The default vagrant box we use for provisioning.
+  DEFAULT_VAGRANT_BOX = "http://files.vagrantup.com/precise32.box"
+
+  # the calculated box, currently taken from the box_url. Expect this to change.
+  attr_reader :box
+
+  # FIXME: support non-url boxes and ram configurations
+  def initialize(box_url=DEFAULT_VAGRANT_BOX)
+    self.box_url = box_url
+  end
+
+  #
+  # Set or retrieve the box_url. See #box_url=.
+  #
+  def box_url(arg=nil)
+    if arg
+      self.box_url = arg
+    end
+
+    @box_url
+  end
+
+  #
+  # Set the box_url. The box name is derived from the url currently.
+  #
+  def box_url=(url)
+    @box_url = url
+    @box = File.basename(url).gsub('\.box', '')
+  end
+
+  include GenericSupport
+end
+
+VagrantSupport.configure

data/lib/chef-workflow/support/vm.rb

@@ -0,0 +1,71 @@
+require 'set'
+require 'fileutils'
+require 'chef-workflow/support/general'
+require 'chef-workflow/support/attr'
+require 'chef-workflow/support/debug'
+
+#--
+# XXX see the dynamic require at the bottom
+#++
+
+#
+# This class mainly exists to track the run state of the Scheduler, and is kept
+# simple so that the contents can be marshalled and restored from a file.
+#
+class VM
+  class << self
+    extend AttrSupport
+    fancy_attr :vm_file
+  end
+
+  include DebugSupport
+  extend AttrSupport
+
+  #
+  # If a file exists that contains a VM object, load it. Use VM.vm_file to
+  # control the location of this file.
+  #
+  def self.load_from_file
+    vm_file = GeneralSupport.singleton.vm_file
+
+    if File.file?(vm_file)
+      return Marshal.load(File.binread(vm_file || DEFAULT_VM_FILE))
+    end
+
+    return nil
+  end
+ 
+  #
+  # Save the marshalled representation to a file. Use VM.vm_file to control the
+  # location of this file.
+  #
+  def save_to_file
+    vm_file = GeneralSupport.singleton.vm_file
+    marshalled = Marshal.dump(self)
+    FileUtils.mkdir_p(File.dirname(vm_file))
+    File.binwrite(vm_file, marshalled)
+  end
+
+  # the vm groups and their provisioning lists.
+  attr_reader :groups
+  # the dependencies that each vm group depends on
+  attr_reader :dependencies
+  # the set of provisioned (solved) groups
+  attr_reader :provisioned
+  # the set of provisioning (working) groups
+  attr_reader :working
+
+  def clean
+    @groups        = { }
+    @dependencies  = { }
+    @provisioned   = Set.new
+    @working       = Set.new
+  end
+
+  alias initialize clean
+end
+
+# XXX require all the provisioners -- marshal will blow up unless this is done.
+Dir[File.join(File.expand_path(File.dirname(__FILE__)), 'vm', '*')].each do |x|
+  require x
+end

data/lib/chef-workflow/support/vm/chef_server.rb

@@ -0,0 +1,31 @@
+require 'chef-workflow/support/knife'
+require 'chef-workflow/support/knife-plugin'
+require 'chef/knife/server_bootstrap_standalone'
+
+class VM
+  class ChefServerProvisioner
+    include DebugSupport
+    include KnifePluginSupport
+
+    attr_accessor :name
+
+    def startup(*args)
+      ip = args.first.first #arg
+
+      raise "No IP to use for the chef server" unless ip
+
+      args = %W[--node-name test-chef-server --host #{ip}]
+
+      args += %W[--ssh-user #{KnifeSupport.singleton.ssh_user}]                 if KnifeSupport.singleton.ssh_user
+      args += %W[--ssh-password #{KnifeSupport.singleton.ssh_password}]         if KnifeSupport.singleton.ssh_password
+      args += %W[--identity-file #{KnifeSupport.singleton.ssh_identity_file}]   if KnifeSupport.singleton.ssh_identity_file
+
+      init_knife_plugin(Chef::Knife::ServerBootstrapStandalone, args).run
+      true
+    end
+
+    def shutdown
+      true
+    end
+  end
+end

data/lib/chef-workflow/support/vm/ec2.rb

@@ -0,0 +1,146 @@
+require 'chef-workflow/support/ec2'
+require 'chef-workflow/support/ip'
+require 'chef-workflow/support/debug'
+require 'net/ssh'
+require 'timeout'
+
+class VM
+  class EC2Provisioner
+    include DebugSupport
+
+    attr_accessor :name
+
+    def initialize(name, number_of_servers)
+      @name = name
+      @number_of_servers = number_of_servers
+      @instance_ids = []
+    end
+
+    def ssh_connection_check(ip)
+      Net::SSH.start(ip, KnifeSupport.singleton.ssh_user, { :keys => [KnifeSupport.singleton.ssh_identity_file] }) do |ssh|
+        ssh.open_channel do |ch|
+          ch.on_open_failed do
+            return false
+          end
+
+          ch.exec("exit 0") do
+            return true
+          end
+        end
+      end
+    rescue
+      return false
+    end
+
+    def ec2
+      EC2Support.singleton
+    end
+
+    def startup(*args)
+      aws_ec2 = ec2.ec2_obj
+
+      ec2.assert_security_groups
+
+      instances = aws_ec2.instances.create(
+        :count            => @number_of_servers,
+        :image_id         => ec2.ami,
+        :security_groups  => ec2.security_groups,
+        :key_name         => ec2.ssh_key,
+        :instance_type    => ec2.instance_type
+      )
+
+      #
+      # instances isn't actually an array above -- see this url:
+      #
+      # https://github.com/aws/aws-sdk-ruby/issues/100
+      #
+      # Actually make it a real array here so it's useful.
+      #
+
+      if instances.kind_of?(Array)
+        new_instances = []
+
+        instances.each do |instance|
+          new_instances.push(instance)
+        end
+
+        instances = new_instances
+      else
+        instances = [instances]
+      end
+
+      #
+      # There are instances where AWS won't acknowledge a created instance
+      # right away. Let's make sure the API server knows they all exist before
+      # moving forward.
+      #
+
+      unresolved_instances = instances.dup
+
+      until unresolved_instances.empty?
+        instance = unresolved_instances.shift
+
+        unless (instance.status rescue nil)
+          if_debug(3) do
+            $stderr.puts "API server doesn't think #{instance.id} exists yet."
+          end
+
+          sleep 0.3
+          unresolved_instances.push(instance)
+        end
+      end
+
+      ip_addresses = []
+      
+      instances.each do |instance|
+        @instance_ids.push(instance.id)
+      end
+
+      begin
+        Timeout.timeout(ec2.provision_wait) do
+          until instances.empty?
+            instance = instances.shift
+
+            ready = false
+
+            if instance.status == :running
+              ready = ssh_connection_check(instance.ip_address)
+              unless ready
+                if_debug(3) do
+                  $stderr.puts "Instance #{instance.id} running, but ssh isn't up yet."
+                end
+              end
+            else
+              if_debug(3) do
+                $stderr.puts "#{instance.id} isn't running yet -- scheduling for re-check"
+              end
+            end
+
+            if ready
+              ip_addresses.push(instance.ip_address)
+              IPSupport.singleton.assign_role_ip(name, instance.ip_address)
+            else
+              sleep 0.3
+              instances.push(instance)
+            end
+          end
+        end
+      rescue TimeoutError
+        raise "instances timed out waiting for ec2"
+      end
+
+      return ip_addresses
+    end
+
+    def shutdown
+      aws_ec2 = ec2.ec2_obj
+
+      @instance_ids.each do |instance_id|
+        aws_ec2.instances[instance_id].terminate
+      end
+
+      IPSupport.singleton.delete_role(name)
+      return true
+    end
+  end
+end

data/lib/chef-workflow/support/vm/knife.rb

@@ -0,0 +1,217 @@
+require 'chef-workflow/support/debug'
+require 'chef-workflow/support/knife'
+require 'chef-workflow/support/knife-plugin'
+require 'chef/node'
+require 'chef/search/query'
+require 'chef/knife/bootstrap'
+require 'chef/knife/client_delete'
+require 'chef/knife/node_delete'
+require 'timeout'
+
+class VM
+  #
+  # The Knife Provisioner does three major things:
+  #
+  # * Bootstraps a series of machines living on IP addresses supplied to it
+  # * Ensures that they converged successfully (if not, raises and displays output)
+  # * Waits until chef has indexed their metadata
+  #
+  # On deprovision, it deletes the nodes and clients related to this server group.
+  #
+  # Machines are named as such: $server_group-$number, where $number starts at 0
+  # and increases with the number of servers requested. Your node names will be
+  # named this as well as the clients associated with them.
+  #
+  # It does as much of this as it can in parallel, but stalls the current thread
+  # until the subthreads complete. This allows is to work as quickly as possible
+  # in a 'serial' scheduling scenario as we know bootstrapping can always occur
+  # in parallel for the group.
+  #
+  class KnifeProvisioner
+
+    include DebugSupport
+    include KnifePluginSupport
+  
+    # the username for SSH.
+    attr_accessor :username
+    # the password for SSH.
+    attr_accessor :password
+    # drive knife bootstrap's sudo functionality.
+    attr_accessor :use_sudo
+    # the ssh key to be used for SSH
+    attr_accessor :ssh_key
+    # the bootstrap template to be used.
+    attr_accessor :template_file
+    # the chef environment to be used.
+    attr_accessor :environment
+    # the port to contact for SSH
+    attr_accessor :port
+    # the list of IPs to provision.
+    attr_accessor :ips
+    # the run list of this server group.
+    attr_accessor :run_list
+    # the name of this server group.
+    attr_accessor :name
+    # perform the solr check to ensure the instance has converged and its
+    # metadata is ready for searching.
+    attr_accessor :solr_check
+
+    # constructor.
+    def initialize
+      @ips            = []
+      @username       = nil
+      @password       = nil
+      @ssh_key        = nil
+      @port           = nil
+      @use_sudo       = nil
+      @run_list       = nil
+      @template_file  = nil
+      @environment    = nil
+      @node_names     = []
+      @solr_check     = true
+    end
+
+    #
+    # Runs the provisioner. Accepts an array of IP addresses as its first
+    # argument, intended to be provided by provisioners that ran before it as
+    # their return value.
+    #
+    # Will raise if the IPs are not supplied or the provisioner is not named with
+    # a server group.
+    #
+    def startup(*args)
+      @ips = args.first #argh
+      raise "This provisioner is unnamed, cannot continue" unless name
+      raise "This provisioner requires ip addresses which were not supplied" unless ips
+
+      @run_list ||= ["role[#{name}]"]
+
+      t = []
+      ips.each_with_index do |ip, index|
+        node_name = "#{name}-#{index}"
+        @node_names.push(node_name)
+        t.push bootstrap(node_name, ip)
+      end
+
+      t.each(&:join)
+
+      return solr_check ? check_nodes : true
+    end
+
+    #
+    # Deprovisions the server group. Runs node delete and client delete on all
+    # nodes that were created by this provisioner.
+    #
+    def shutdown
+      t = []
+
+      @node_names.each do |node_name|
+        t.push(
+          Thread.new do
+            client_delete(node_name)
+            node_delete(node_name)
+          end
+        )
+      end
+
+      t.each(&:join)
+
+      return true
+    end
+
+    #
+    # Checks that the nodes have made it into the search index. Will block until
+    # all nodes in this server group are found, or a 60 second timeout is
+    # reached, at which point it will raise.
+    #
+    def check_nodes
+      q = Chef::Search::Query.new
+      unchecked_node_names = @node_names.dup
+
+      # this dirty hack turns 'role[foo]' into 'roles:foo', but also works on
+      # recipe[] too. Then joins the whole thing with AND
+      search_query = run_list.
+        map { |s| s.gsub(/\[/, 's:"').gsub(/\]/, '"') }.
+        join(" AND ")
+
+      Timeout.timeout(KnifeSupport.singleton.search_index_wait) do
+        until unchecked_node_names.empty?
+          node_name = unchecked_node_names.shift
+          if_debug(3) do
+            $stderr.puts "Checking search validity for node #{node_name}"
+          end
+
+          result = q.search(
+            :node, 
+            search_query + %Q[ AND name:"#{node_name}"]
+          ).first
+
+          unless result and result.count == 1 and result.first.name == node_name
+            unchecked_node_names << node_name
+          end
+          
+          # unfortunately if this isn't here you might as well issue kill -9 to
+          # the rake process
+          sleep 0.3
+        end
+      end
+
+      return true
+    rescue Timeout::Error
+      raise "Bootstrapped nodes for #{name} did not appear in Chef search index after 60 seconds."
+    end
+
+    #
+    # Bootstraps a single node. Validates bootstrap by checking the node metadata
+    # directly and ensuring it made it into the chef server.
+    #
+    def bootstrap(node_name, ip)
+      args = []
+
+      args += %W[-x #{username}]                    if username
+      args += %W[-P #{password}]                    if password
+      args += %w[--sudo]                            if use_sudo
+      args += %W[-i #{ssh_key}]                     if ssh_key
+      args += %W[--template-file #{template_file}]  if template_file
+      args += %W[-p #{port}]                        if port 
+      args += %W[-E #{environment}]                 if environment
+
+      args += %W[-r #{run_list.join(",")}]
+      args += %W[-N '#{node_name}']
+      args += [ip]
+
+      bootstrap_cli = init_knife_plugin(Chef::Knife::Bootstrap, args)
+
+      Thread.new do
+        bootstrap_cli.run
+        # knife bootstrap is the honey badger when it comes to exit status.
+        # We can't rely on it, so we examine the run_list of the node instead
+        # to ensure it converged.
+        run_list_size = Chef::Node.load(node_name).run_list.to_a.size
+        unless run_list_size > 0
+          puts bootstrap_cli.ui.stdout.string
+          puts bootstrap_cli.ui.stderr.string
+          raise "bootstrap for #{node_name}/#{ip} wasn't successful."
+        end
+        if_debug(2) do
+          puts bootstrap_cli.ui.stdout.string
+          puts bootstrap_cli.ui.stderr.string
+        end
+      end
+    end
+
+    #
+    # Deletes a chef client.
+    #
+    def client_delete(node_name)
+      init_knife_plugin(Chef::Knife::ClientDelete, [node_name, '-y']).run
+    end
+
+    #
+    # Deletes a chef node.
+    #
+    def node_delete(node_name)
+      init_knife_plugin(Chef::Knife::NodeDelete, [node_name, '-y']).run
+    end
+  end
+end