chef-workflow 0.1.1 → 0.2.0

data/lib/chef-workflow/support/ssh.rb

@@ -0,0 +1,100 @@
+require 'chef-workflow/support/ip'
+require 'chef-workflow/support/knife'
+require 'chef-workflow/support/debug'
+require 'net/ssh'
+
+module ChefWorkflow
+  #
+  # Helper for performing SSH on groups of servers. Intended to be mixed into
+  # test case classes.
+  #
+  module SSHHelper
+    include ChefWorkflow::DebugSupport
+
+    #
+    # run a command against a group of servers. These commands are run in
+    # parallel, but the command itself does not complete until all the threads
+    # have finished running.
+    #
+    def ssh_role_command(role, command)
+      t = []
+      ChefWorkflow::IPSupport.get_role_ips(role).each do |ip|
+        t.push(
+          Thread.new do
+            ssh_command(ip, command)
+          end
+        )
+      end
+      t.each(&:join)
+    end
+
+    #
+    # takes a block which it uses inside of the open_channel block that Net::SSH
+    # uses. Intended to provide a consistent way of setting up Net::SSH  Makes
+    # heavy use of KnifeSupport to determine how to drive the command.
+    #
+    def configure_ssh_command(ip, command)
+      command = "#{ChefWorkflow::KnifeSupport.use_sudo ? 'sudo ': ''}#{command}"
+
+      options = { }
+
+      options[:password] = ChefWorkflow::KnifeSupport.ssh_password          if ChefWorkflow::KnifeSupport.ssh_password
+      options[:keys]     = [ChefWorkflow::KnifeSupport.ssh_identity_file]   if ChefWorkflow::KnifeSupport.ssh_identity_file
+
+      Net::SSH.start(ip, ChefWorkflow::KnifeSupport.ssh_user, options) do |ssh|
+        ssh.open_channel do |ch|
+          ch.on_open_failed do |ch, code, desc|
+            raise "Connection Error to #{ip}: #{desc}"
+          end
+
+          ch.exec(command) do |ch, success|
+            yield ch, success
+          end
+        end
+
+        ssh.loop
+      end
+    end
+
+    #
+    # Run a command against a single IP. Returns the exit status.
+    # 
+    #
+    def ssh_command(ip, command)
+      configure_ssh_command(ip, command) do |ch, success|
+        return 1 unless success
+
+        if_debug(2) do
+          ch.on_data do |ch, data|
+            $stderr.puts data
+          end
+        end
+
+        ch.on_request("exit-status") do |ch, data|
+          return data.read_long
+        end
+      end
+    end
+
+    #
+    # run a command, and instead of capturing the exit status, return the data
+    # captured during the command run.
+    #
+    def ssh_capture(ip, command)
+      retval = ""
+      configure_ssh_command(ip, command) do |ch, success|
+        return "" unless success
+
+        ch.on_data do |ch, data|
+          retval << data
+        end
+
+        ch.on_request("exit-status") do |ch, data|
+          return retval
+        end
+      end
+
+      return retval
+    end
+  end
+end

data/lib/chef-workflow/support/vagrant.rb

@@ -1,41 +1,45 @@
 require 'fileutils'
 require 'chef-workflow/support/generic'
 
-#
-# Vagrant configuration settings. Uses `GenericSupport`.
-#
-class VagrantSupport
-  # The default vagrant box we use for provisioning.
-  DEFAULT_VAGRANT_BOX = "http://files.vagrantup.com/precise32.box"
-
-  # the calculated box, currently taken from the box_url. Expect this to change.
-  attr_reader :box
-
-  # FIXME: support non-url boxes and ram configurations
-  def initialize(box_url=DEFAULT_VAGRANT_BOX)
-    self.box_url = box_url
-  end
-
+module ChefWorkflow
   #
-  # Set or retrieve the box_url. See #box_url=.
+  # Vagrant configuration settings. Uses `GenericSupport`.
   #
-  def box_url(arg=nil)
-    if arg
-      self.box_url = arg
+  class VagrantSupport
+    include ChefWorkflow::GenericSupport
+
+    # The default vagrant box we use for provisioning.
+    DEFAULT_VAGRANT_BOX = "http://files.vagrantup.com/precise32.box"
+
+    # the calculated box, currently taken from the box_url. Expect this to change.
+    attr_reader :box
+
+    #--
+    # FIXME: support non-url boxes and ram configurations
+    #++
+    def initialize(box_url=DEFAULT_VAGRANT_BOX)
+      self.box_url = box_url
     end
 
-    @box_url
-  end
+    #
+    # Set or retrieve the box_url. See #box_url=.
+    #
+    def box_url(arg=nil)
+      if arg
+        self.box_url = arg
+      end
 
-  #
-  # Set the box_url. The box name is derived from the url currently.
-  #
-  def box_url=(url)
-    @box_url = url
-    @box = File.basename(url).gsub('\.box', '')
-  end
+      @box_url
+    end
 
-  include GenericSupport
+    #
+    # Set the box_url. The box name is derived from the url currently.
+    #
+    def box_url=(url)
+      @box_url = url
+      @box = File.basename(url).gsub(/\.box$/, '')
+    end
+  end
 end
 
-VagrantSupport.configure
+ChefWorkflow::VagrantSupport.configure

data/lib/chef-workflow/support/vm.rb

@@ -3,69 +3,40 @@ require 'fileutils'
 require 'chef-workflow/support/general'
 require 'chef-workflow/support/attr'
 require 'chef-workflow/support/debug'
+require 'chef-workflow/support/db/group'
+require 'chef-workflow/support/db/basic'
 
 #--
 # XXX see the dynamic require at the bottom
 #++
 
-#
-# This class mainly exists to track the run state of the Scheduler, and is kept
-# simple so that the contents can be marshalled and restored from a file.
-#
-class VM
-  class << self
-    extend AttrSupport
-    fancy_attr :vm_file
-  end
-
-  include DebugSupport
-  extend AttrSupport
-
+module ChefWorkflow
   #
-  # If a file exists that contains a VM object, load it. Use VM.vm_file to
-  # control the location of this file.
+  # This class mainly exists to track the run state of the Scheduler, and is kept
+  # simple so that the contents can be marshalled and restored from a file.
   #
-  def self.load_from_file
-    vm_file = GeneralSupport.singleton.vm_file
-
-    if File.file?(vm_file)
-      return Marshal.load(File.binread(vm_file || DEFAULT_VM_FILE))
+  class VM
+    include ChefWorkflow::DebugSupport
+
+    # the vm groups and their provisioning lists.
+    attr_reader :groups
+    # the dependencies that each vm group depends on
+    attr_reader :dependencies
+    # the set of provisioned (solved) groups
+    attr_reader :provisioned
+    # the set of provisioning (working) groups
+    attr_reader :working
+
+    def initialize
+      @groups        = ChefWorkflow::DatabaseSupport::VMGroup.new('vm_groups', false)
+      @dependencies  = ChefWorkflow::DatabaseSupport::VMGroup.new('vm_dependencies', true)
+      @provisioned   = ChefWorkflow::DatabaseSupport::Set.new('vm_scheduler', 'provisioned')
+      @working       = ChefWorkflow::DatabaseSupport::Set.new('vm_scheduler', 'working')
     end
-
-    return nil
-  end
- 
-  #
-  # Save the marshalled representation to a file. Use VM.vm_file to control the
-  # location of this file.
-  #
-  def save_to_file
-    vm_file = GeneralSupport.singleton.vm_file
-    marshalled = Marshal.dump(self)
-    FileUtils.mkdir_p(File.dirname(vm_file))
-    File.binwrite(vm_file, marshalled)
   end
 
-  # the vm groups and their provisioning lists.
-  attr_reader :groups
-  # the dependencies that each vm group depends on
-  attr_reader :dependencies
-  # the set of provisioned (solved) groups
-  attr_reader :provisioned
-  # the set of provisioning (working) groups
-  attr_reader :working
-
-  def clean
-    @groups        = { }
-    @dependencies  = { }
-    @provisioned   = Set.new
-    @working       = Set.new
+  # XXX require all the provisioners -- marshal will blow up unless this is done.
+  Dir[File.join(File.expand_path(File.dirname(__FILE__)), 'vm', '*')].each do |x|
+    require x if File.file?(x)
   end
-
-  alias initialize clean
-end
-
-# XXX require all the provisioners -- marshal will blow up unless this is done.
-Dir[File.join(File.expand_path(File.dirname(__FILE__)), 'vm', '*')].each do |x|
-  require x
 end

data/lib/chef-workflow/support/vm/chef_server.rb

@@ -1,31 +1,40 @@
-require 'chef-workflow/support/knife'
+require 'chef-workflow/support/debug'
 require 'chef-workflow/support/knife-plugin'
-require 'chef/knife/server_bootstrap_standalone'
 
-class VM
-  class ChefServerProvisioner
-    include DebugSupport
-    include KnifePluginSupport
+module ChefWorkflow
+  class VM
+    class ChefServerProvisioner
+      include ChefWorkflow::DebugSupport
+      include ChefWorkflow::KnifePluginSupport
 
-    attr_accessor :name
+      attr_accessor :name
 
-    def startup(*args)
-      ip = args.first.first #arg
+      def startup(*args)
+        require 'chef-workflow/support/knife'
+        require 'chef/knife/ssh' # required for chef 10.12
+        require 'chef/knife/server_bootstrap_standalone'
 
-      raise "No IP to use for the chef server" unless ip
+        ip = args.first.first #arg
 
-      args = %W[--node-name test-chef-server --host #{ip}]
+        raise "No IP to use for the chef server" unless ip
 
-      args += %W[--ssh-user #{KnifeSupport.singleton.ssh_user}]                 if KnifeSupport.singleton.ssh_user
-      args += %W[--ssh-password #{KnifeSupport.singleton.ssh_password}]         if KnifeSupport.singleton.ssh_password
-      args += %W[--identity-file #{KnifeSupport.singleton.ssh_identity_file}]   if KnifeSupport.singleton.ssh_identity_file
+        args = %W[--node-name test-chef-server --host #{ip}]
 
-      init_knife_plugin(Chef::Knife::ServerBootstrapStandalone, args).run
-      true
-    end
+        args += %W[--ssh-user #{ChefWorkflow::KnifeSupport.ssh_user}]                 if ChefWorkflow::KnifeSupport.ssh_user
+        args += %W[--ssh-password #{ChefWorkflow::KnifeSupport.ssh_password}]         if ChefWorkflow::KnifeSupport.ssh_password
+        args += %W[--identity-file #{ChefWorkflow::KnifeSupport.ssh_identity_file}]   if ChefWorkflow::KnifeSupport.ssh_identity_file
+
+        init_knife_plugin(Chef::Knife::ServerBootstrapStandalone, args).run
+        true
+      end
+
+      def shutdown
+        true
+      end
 
-    def shutdown
-      true
+      def report
+        [name]
+      end
     end
   end
 end

data/lib/chef-workflow/support/vm/ec2.rb

@@ -1,146 +1,175 @@
-require 'chef-workflow/support/ec2'
-require 'chef-workflow/support/ip'
 require 'chef-workflow/support/debug'
-require 'net/ssh'
-require 'timeout'
 
-class VM
-  class EC2Provisioner
-    include DebugSupport
+module ChefWorkflow
+  class VM
+    class EC2Provisioner
+      include ChefWorkflow::DebugSupport
 
-    attr_accessor :name
+      attr_accessor :name
 
-    def initialize(name, number_of_servers)
-      @name = name
-      @number_of_servers = number_of_servers
-      @instance_ids = []
-    end
+      def initialize(name, number_of_servers)
+        require 'chef-workflow/support/ec2'
+        require 'chef-workflow/support/ip'
+        require 'net/ssh'
+        require 'timeout'
 
-    def ssh_connection_check(ip)
-      Net::SSH.start(ip, KnifeSupport.singleton.ssh_user, { :keys => [KnifeSupport.singleton.ssh_identity_file] }) do |ssh|
-        ssh.open_channel do |ch|
-          ch.on_open_failed do
-            return false
-          end
+        @name = name
+        @number_of_servers = number_of_servers
+        init_instance_ids
+      end
+
+      def init_instance_ids
+        @instance_ids = ChefWorkflow::DatabaseSupport::Set.new("vm_ec2_instances", name)
+      end
 
-          ch.exec("exit 0") do
-            return true
+      def ssh_connection_check(ip)
+        Net::SSH.start(ip, ChefWorkflow::KnifeSupport.ssh_user, { :keys => [ChefWorkflow::KnifeSupport.ssh_identity_file] }) do |ssh|
+          ssh.open_channel do |ch|
+            ch.on_open_failed do
+              return false
+            end
+
+            ch.exec("exit 0") do
+              return true
+            end
           end
+          ssh.loop
         end
+      rescue Exception => e
+        return false
       end
-    rescue
-      return false
-    end
 
-    def ec2
-      EC2Support.singleton
-    end
+      def ec2
+        ChefWorkflow::EC2Support
+      end
 
-    def startup(*args)
-      aws_ec2 = ec2.ec2_obj
+      def startup(*args)
+        aws_ec2 = ec2.ec2_obj
 
-      ec2.assert_security_groups
+        ec2.assert_security_groups
 
-      instances = aws_ec2.instances.create(
-        :count            => @number_of_servers,
-        :image_id         => ec2.ami,
-        :security_groups  => ec2.security_groups,
-        :key_name         => ec2.ssh_key,
-        :instance_type    => ec2.instance_type
-      )
+        instances = aws_ec2.instances.create(
+          :count            => @number_of_servers,
+          :image_id         => ec2.ami,
+          :security_groups  => ec2.security_groups,
+          :key_name         => ec2.ssh_key,
+          :instance_type    => ec2.instance_type
+        )
 
-      #
-      # instances isn't actually an array above -- see this url:
-      #
-      # https://github.com/aws/aws-sdk-ruby/issues/100
-      #
-      # Actually make it a real array here so it's useful.
-      #
+        #
+        # instances isn't actually an array above -- see this url:
+        #
+        # https://github.com/aws/aws-sdk-ruby/issues/100
+        #
+        # Actually make it a real array here so it's useful.
+        #
 
-      if instances.kind_of?(Array)
-        new_instances = []
+        if instances.kind_of?(Array)
+          new_instances = []
 
-        instances.each do |instance|
-          new_instances.push(instance)
+          instances.each do |instance|
+            new_instances.push(instance)
+          end
+
+          instances = new_instances
+        else
+          instances = [instances]
         end
 
-        instances = new_instances
-      else
-        instances = [instances]
-      end
+        #
+        # There are instances where AWS won't acknowledge a created instance
+        # right away. Let's make sure the API server knows they all exist before
+        # moving forward.
+        #
 
-      #
-      # There are instances where AWS won't acknowledge a created instance
-      # right away. Let's make sure the API server knows they all exist before
-      # moving forward.
-      #
+        unresolved_instances = instances.dup
 
-      unresolved_instances = instances.dup
+        until unresolved_instances.empty?
+          instance = unresolved_instances.shift
 
-      until unresolved_instances.empty?
-        instance = unresolved_instances.shift
+          unless (instance.status rescue nil)
+            if_debug(3) do
+              $stderr.puts "API server doesn't think #{instance.id} exists yet."
+            end
 
-        unless (instance.status rescue nil)
-          if_debug(3) do
-            $stderr.puts "API server doesn't think #{instance.id} exists yet."
+            sleep 0.3
+            unresolved_instances.push(instance)
           end
-
-          sleep 0.3
-          unresolved_instances.push(instance)
         end
-      end
-
-      ip_addresses = []
-      
-      instances.each do |instance|
-        @instance_ids.push(instance.id)
-      end
 
-      begin
-        Timeout.timeout(ec2.provision_wait) do
-          until instances.empty?
-            instance = instances.shift
+        ip_addresses = []
 
-            ready = false
+        instances.each do |instance|
+          @instance_ids.add(instance.id)
+        end
 
-            if instance.status == :running
-              ready = ssh_connection_check(instance.ip_address)
-              unless ready
-                if_debug(3) do
-                  $stderr.puts "Instance #{instance.id} running, but ssh isn't up yet."
+        ipaddress_mutex = Mutex.new
+        debug_mutex = Mutex.new
+
+        begin
+          Timeout.timeout(ec2.provision_wait) do
+            instances.map do |instance|
+              Thread.new do
+                loop do
+                  ready = false
+
+                  if instance.status == :running
+                    ready = ssh_connection_check(instance.ip_address)
+                    unless ready
+                      if_debug(3) do
+                        debug_mutex.synchronize do
+                          $stderr.puts "Instance #{instance.id} running, but ssh isn't up yet."
+                        end
+                      end
+                    end
+                  else
+                    if_debug(3) do
+                      debug_mutex.synchronize do
+                        $stderr.puts "#{instance.id} isn't running yet -- scheduling for re-check"
+                      end
+                    end
+                  end
+
+                  if ready
+                    ipaddress_mutex.synchronize do
+                      ip_addresses.push(instance.ip_address)
+                    end
+                    break
+                  end
+
+                  sleep 2
                 end
               end
-            else
-              if_debug(3) do
-                $stderr.puts "#{instance.id} isn't running yet -- scheduling for re-check"
-              end
-            end
-
-            if ready
-              ip_addresses.push(instance.ip_address)
-              IPSupport.singleton.assign_role_ip(name, instance.ip_address)
-            else
-              sleep 0.3
-              instances.push(instance)
-            end
+            end.each(&:join)
           end
+        rescue TimeoutError
+          raise "instances timed out waiting for ec2"
+        end
+
+        ip_addresses.each do |ipaddr|
+          ChefWorkflow::IPSupport.assign_role_ip(name, ipaddr)
         end
-      rescue TimeoutError
-        raise "instances timed out waiting for ec2"
+
+        return ip_addresses
       end
 
-      return ip_addresses
-    end
+      def shutdown
+        aws_ec2 = ec2.ec2_obj
+
+        @instance_ids.each do |instance_id|
+          aws_ec2.instances[instance_id].terminate
+        end
 
-    def shutdown
-      aws_ec2 = ec2.ec2_obj
+        @instance_ids.clear
 
-      @instance_ids.each do |instance_id|
-        aws_ec2.instances[instance_id].terminate
+        ChefWorkflow::IPSupport.delete_role(name)
+        return true
       end
 
-      IPSupport.singleton.delete_role(name)
-      return true
+      def report
+        init_instance_ids
+        ["#{@number_of_servers} servers; instance ids: #{@instance_ids.to_a.join(" ")}"]
+      end
     end
   end
 end