rspec-system 2.4.0 → 2.5.0

data/CHANGELOG.md

@@ -1,3 +1,18 @@
+2.5.0
+=====
+
+This feature release improves the VSPhere provider.
+
+* Additional settings have been provided to manage the deployment of nodes in VSphere
+* Extra settings added to control timeouts and retries
+* The main resiliency code has been overhauled and now we have retry handling for most external calls
+
+#### Detailed Changes
+
+* Repair vsphere provider (Ken Barber)
+
+-------------------------------
+
 2.4.0
 =====
 

data/README.md

@@ -278,6 +278,17 @@ This provider has a lot more options for setup, in the form of environment varia
 * *RSPEC_VSPHERE_DEST_DIR* - destination path to launch vm's
 * *RSPEC_VSPHERE_TEMPLATE_DIR* - path to where you deployed the templates from the OVF files described above
 * *RSPEC_VSPHERE_RPOOL* - name of resource pool to use
+* *RSPEC_VSPHERE_CLUSTER* - name of the cluster to use
+* *RSPEC_VSPHERE_SSH_KEYS* - path to private key for authentication. Multiple paths may be provided using a colon separator.
+* *RSPEC_VSPHERE_DATACENTER* - optional name of VSphere data centre
+* *RSPEC_VSPHERE_NODE_TIMEOUT* - amount of seconds before trying to relaunch a node. Defaults to 1200.
+* *RSPEC_VSPHERE_NODE_TRIES* - amount of attempts to relaunch a node. Defaults to 10.
+* *RSPEC_VSPHERE_NODE_SLEEP* - amount of seconds to sleep for before trying again. Defaults to a random number between 30 and 90.
+* *RSPEC_VSPHERE_SSH_TIMEOUT* - amount of seconds before retrying to SSH into a node. Default to 60.
+* *RSPEC_VSPHERE_SSH_TRIES* - amount of attempts to SSH into a node. Defaults to 10.
+* *RSPEC_VSPHERE_SSH_SLEEP* - amount of seconds to sleep before trying to SSH again. Defaults to 4.
+* *RSPEC_VSPHERE_CONNECT_TIMEOUT* - amount of seconds before retrying to connect to the VSphere API. Defaults to 60.
+* *RSPEC_VSPHERE_CONNECT_TRIES* - amount of attempts to connect to the VSphere API. Defaults to 10.
 
 Set these variables, and run the usual rake command:
 

data/lib/rspec-system/node_set/vsphere.rb

@@ -12,6 +12,8 @@ module RSpecSystem
 
     ENV_TYPE = 'vsphere'
 
+    attr_reader :vmconf
+
     # Creates a new instance of RSpecSystem::NodeSet::Vsphere
     #
     # @param setname [String] name of the set to instantiate
@@ -20,96 +22,242 @@ module RSpecSystem
     # @param options [Hash] options Hash
     def initialize(setname, config, custom_prefabs_path, options)
       super
-      @vim = RbVmomi::VIM.connect(
-        :host => ENV["RSPEC_VSPHERE_HOST"],
-        :user => ENV["RSPEC_VSPHERE_USER"],
-        :password => ENV["RSPEC_VSPHERE_PASS"],
-        :ssl => true,
-        :insecure => true
-      )
+
+      # Valid supported ENV variables
+      options = [:host, :user, :pass, :dest_dir, :template_dir, :rpool,
+        :cluster, :ssh_keys, :datacenter, :node_timeout, :node_tries,
+        :node_sleep, :ssh_timeout, :ssh_tries, :ssh_sleep, :connect_timeout,
+        :connect_tries]
+
+      # Devise defaults, use fog configuration from file system if it exists
+      defaults = load_fog_config()
+      defaults = defaults.merge({
+        :node_timeout => 1200,
+        :node_tries => 10,
+        :node_sleep => 30 + rand(60),
+        :ssh_timeout => 60,
+        :ssh_tries => 10,
+        :ssh_sleep => 4,
+        :connect_timeout => 60,
+        :connect_tries => 10,
+      })
+
+      # Traverse the ENV variables and load them into our config automatically
+      @vmconf = defaults
+      ENV.each do |k,v|
+        next unless k =~/^RSPEC_VSPHERE_/
+        var = k.sub(/^RSPEC_VSPHERE_/, '').downcase.to_sym
+        unless options.include?(var)
+          log.info("Ignoring unknown environment variable #{k}")
+          next
+        end
+        @vmconf[var] = v
+      end
 
       # Initialize node storage if not already
       RSpec.configuration.rspec_storage[:nodes] ||= {}
     end
 
-    # @!group NodeSet Methods
-
-    # Setup the NodeSet by starting all nodes.
+    # Retrieves fog configuration if it exists
     #
-    # @return [void]
-    def setup
-      dest_dir = ENV['RSPEC_VSPHERE_DEST_DIR']
-      template_dir = ENV['RSPEC_VSPHERE_TEMPLATE_DIR']
-
-      si = @vim.serviceInstance
-      dc = si.find_datacenter
-
-      rp = dc.find_compute_resource('general').resourcePool.find(ENV["RSPEC_VSPHERE_RPOOL"])
-      relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec(:pool => rp)
-      spec = RbVmomi::VIM.VirtualMachineCloneSpec(
-        :location => relocateSpec,
-        :powerOn => true,
-        :template => false
-      )
-
-      vm_folder = dc.vmFolder
-      vm_newfolder = vm_folder.find(dest_dir)
-
-      log.info "Launching VSphere instances one by one"
-      nodes.each do |k,v|
-        ps = v.provider_specifics['vsphere']
-
-        raise 'No provider specifics for this prefab' if ps.nil?
-
-        template = ps['template']
-
-        raise "No template specified for this prefab" if template.nil?
-
-        log.info "Launching VSphere instance #{k} with template #{template}"
-
-        vm = vm_folder.find(ENV['RSPEC_VSPHERE_TEMPLATE_DIR']).find(template)
+    # @api private
+    def load_fog_config(path = ENV['HOME'] + '/.fog')
+      creds = {}
+      if File.exists?(path)
+        fog = YAML.load_file(path)
+        fog[:default] ||= {}
+        creds = {
+          :host => fog[:default][:vsphere_server],
+          :user => fog[:default][:vsphere_username],
+          :pass => fog[:default][:vsphere_password],
+        }
+      end
 
-        raise "No template found" if vm.nil?
+      return creds
+    end
 
-        vm_name = "rspec-system-#{k}-#{random_string(10)}"
+    # This is a DSL based wrapper that provides connection and disconnection
+    # handling for the VSphere client API.
+    #
+    # The connection handling automatically retries upon failure.
+    #
+    # @api private
+    def with_connection(&block)
+      vim = nil
+      dc = nil
+
+      tries = 0
+      begin
+        timeout(vmconf[:connect_timeout]) do
+          vim = RbVmomi::VIM.connect(
+            :host => vmconf[:host],
+            :user => vmconf[:user],
+            :password => vmconf[:pass],
+            :ssl => true,
+            :insecure => true
+          )
+        end
+      rescue => e
+        tries += 1
+        log.error("Failure to connect (attempt #{tries})")
+        if tries < vmconf[:connect_tries]
+          log.info("Retry connection")
+          retry
+        end
+        log.info("Failed to connect after #{tries} attempts, throwing exception")
+        raise e
+      end
 
-        log.info "Cloning new VSphere vm #{vm_name} in folder #{dest_dir}"
+      begin
+        dc = vim.serviceInstance.find_datacenter(vmconf[:datacenter])
+      rescue => e
+        log.error("Unable to retrieve datacenter #{vmconf[:datacenter]}")
+        raise e
+      end
 
-        vm.CloneVM_Task(
-          :folder => vm_newfolder,
-          :name => vm_name,
-          :spec => spec
-        ).wait_for_completion
+      block.call(dc)
 
-        log.info "Cloning complete"
+      vim.close
+    end
 
-        newvm = vm_newfolder.find(vm_name)
-        guest_info = newvm.guest
+    # @!group NodeSet Methods
 
-        timeout(60) do
-          while(newvm.guest.guestState != 'running') do
-            sleep 2
-            log.info "#{k}> Waiting for vm to run ..."
+    # Setup the NodeSet by starting all nodes.
+    #
+    # @return [void]
+    def setup
+      with_connection do |dc|
+        # Traverse folders to find target folder for new vm's and template
+        # folders. Automatically create the destination folder if it doesn't
+        # exist.
+        dest_folder = dc.vmFolder.traverse!(vmconf[:dest_dir], RbVmomi::VIM::Folder)
+        raise "Destination folder #{vmconf[:dest_dir]} not found" if dest_folder.nil?
+        template_folder = dc.vmFolder.traverse(vmconf[:template_dir], RbVmomi::VIM::Folder)
+        raise "Template folder #{vmconf[:template_dir]} not found" if template_folder.nil?
+
+        # Find resource pool and prepare clone spec for cloning further down.
+        rp = dc.find_compute_resource(vmconf[:cluster]).
+                resourcePool.
+                traverse(vmconf[:rpool])
+        relocateSpec = RbVmomi::VIM.VirtualMachineRelocateSpec(:pool => rp)
+        spec = RbVmomi::VIM.VirtualMachineCloneSpec(
+          :location => relocateSpec,
+          :powerOn => true,
+          :template => false
+        )
+
+        log.info "Launching VSphere instances one by one"
+        nodes.each do |k,v|
+          #####################
+          # Node launching step
+          #####################
+          RSpec.configuration.rspec_storage[:nodes][k] ||= {}
+
+          # Obtain the template name to use
+          ps = v.provider_specifics['vsphere']
+          raise 'No provider specifics for this prefab' if ps.nil?
+          template = ps['template']
+          raise "No template specified for this prefab" if template.nil?
+
+          # Traverse to find template VM object
+          vm = template_folder.find(template, RbVmomi::VIM::VirtualMachine)
+          raise "Cannot template find template #{template} in folder #{vmconf[:template_dir]}" if vm.nil?
+
+          # Create a random name for the new VM
+          vm_name = "rspec-system-#{k}-#{random_string(10)}"
+          RSpec.configuration.rspec_storage[:nodes][k][:vm] = vm_name
+
+          log.info "Launching VSphere instance #{k} with template #{vmconf[:template_dir]}/#{template} as #{vmconf[:dest_dir]}/#{vm_name}"
+
+          ipaddress = nil
+          newvm = nil
+          tries = 0
+          start_time = Time.now
+          begin
+            timeout(vmconf[:node_timeout]) do
+              log.info "Cloning new VSphere vm #{vm_name} in folder #{vmconf[:dest_dir]}"
+              vm.CloneVM_Task(
+                :folder => dest_folder,
+                :name => vm_name,
+                :spec => spec
+              ).wait_for_completion
+              time1 = Time.now
+              log.info "Cloning complete, took #{time1 - start_time} seconds"
+
+              newvm = dest_folder.find(vm_name, RbVmomi::VIM::VirtualMachine)
+              raise "Cannot find newly built virtual machine #{vm_name} in folder #{vmconf[:dest_dir]}" if newvm.nil?
+
+              while(newvm.guest.guestState != 'running') do
+                sleep 4
+                log.info "#{k}> Waiting for vm to run ..."
+              end
+              time2 = Time.now
+              log.info "#{k}> Time in seconds for VM to run: #{time2 - time1}"
+
+              while((ipaddress = newvm.guest.ipAddress) == nil) do
+                sleep 4
+                log.info "#{k}> Waiting for ip address ..."
+              end
+              time3 = Time.now
+              log.info "#{k}> Time in seconds waiting for IP: #{time3 - time2}"
+            end
+            RSpec.configuration.rspec_storage[:nodes][k][:ipaddress] = ipaddress
+          rescue Timeout::Error, SystemCallError => e
+            tries += 1
+            log.error("VM launch attempt #{tries} failed with: " + e.message)
+            if tries < vmconf[:node_tries]
+              log.info("Destroying any VM's, sleeping then trying again ...")
+              begin
+                newvm.PowerOffVM_Task.wait_for_completion
+              rescue RbVmomi::Fault => e
+                log.error "Fault attempting to power off node #{k}, #{e.message}"
+              ensure
+                begin
+                  newvm.Destroy_Task.wait_for_completion
+                rescue RbVmomi::Fault => e
+                  log.error "Fault attempting to destroy node #{k}, #{e.message}"
+                end
+              end
+              sleep_time = vmconf[:node_sleep]
+              log.info("Sleeping #{sleep_time} seconds before trying again ...")
+              sleep sleep_time
+              retry
+            else
+              log.error("Failed to create VM and already retried #{tries} times, throwing exception")
+              raise e
+            end
           end
-        end
-
-        timeout(60) do
-          while(newvm.guest.ipAddress == nil) do
-            sleep 2
-            log.info "#{k}> Waiting for ip address ..."
+          time2 = Time.now
+          log.info "#{k}> Took #{time2 - start_time} seconds to boot instance"
+
+          #####################
+          # SSH Step
+          #####################
+          tries = 0
+          begin
+            timeout(vmconf[:ssh_timeout]) do
+              output << bold(color("localhost$", :green)) << " ssh #{k}\n"
+              chan = Net::SSH.start(ipaddress, 'root', {
+                :keys => vmconf[:ssh_keys].split(":"),
+              })
+
+              RSpec.configuration.rspec_storage[:nodes][k][:ssh] = chan
+            end
+          rescue Timeout::Error, SystemCallError => e
+            tries += 1
+            output << e.message << "\n"
+            if tries < vmconf[:ssh_tries]
+              log.info("Sleeping for #{vmconf[:ssh_sleep]} seconds then trying again ...")
+              sleep vmconf[:ssh_sleep]
+              retry
+            else
+              log.error("Inability to connect to host, already tried #{tries} times, throwing exception")
+              raise e
+            end
           end
+          time3 = Time.now
+          log.info "#{k}> Took #{time3 - start_time} seconds for instance to be ready"
         end
-
-        ipaddress = newvm.guest.ipAddress
-
-        output << bold(color("localhost$", :green)) << " ssh #{k}"
-        chan = Net::SSH.start(ipaddress, 'vagrant', :password => 'vagrant')
-
-        RSpec.configuration.rspec_storage[:nodes][k] = {
-          :ipaddress => ipaddress,
-          :ssh => chan,
-          :vm => newvm
-        }
       end
 
       nil
@@ -119,28 +267,48 @@ module RSpecSystem
     #
     # @return [void]
     def teardown
-      nodes.each do |k,v|
-        storage = RSpec.configuration.rspec_storage[:nodes][k]
+      with_connection do |dc|
+        nodes.each do |k,v|
+          storage = RSpec.configuration.rspec_storage[:nodes][k]
 
-        if storage.nil?
-          log.info "No entry for node #{k}, no teardown necessary"
-          next
-        end
+          if storage.nil?
+            log.info "No entry for node #{k}, no teardown necessary"
+            next
+          end
 
-        ssh = storage[:ssh]
-        ssh.close unless ssh.closed?
+          ssh = storage[:ssh]
+          unless ssh.nil? or ssh.closed?
+            ssh.close
+          end
 
-        if destroy
-          log.info "Destroying instance #{k}"
-          vm = storage[:vm]
-          if vm == nil
-            log.error "No vm object for #{k}"
+          if destroy
+            log.info "Destroying instance #{k}"
+            vm_name = storage[:vm]
+            if vm_name == nil
+              log.error "No vm object for #{k}"
+              next
+            end
+
+            # Traverse folders to find target folder for new vm's
+            vm_folder = dc.vmFolder.traverse(vmconf[:dest_dir], RbVmomi::VIM::Folder)
+            raise "VirtualMachine folder #{vmconf[:dest_dir]} not found" if vm_folder.nil?
+            vm = vm_folder.find(vm_name, RbVmomi::VIM::VirtualMachine)
+            raise "VirtualMachine #{vm_name} not found in #{vmconf[:dest_dir]}" if vm.nil?
+
+            begin
+              vm.PowerOffVM_Task.wait_for_completion
+            rescue RbVmomi::Fault => e
+              log.error "Fault attempting to power off node #{k}, #{e.message}"
+            ensure
+              begin
+                vm.Destroy_Task.wait_for_completion
+              rescue RbVmomi::Fault => e
+                log.error "Fault attempting to destroy node #{k}, #{e.message}"
+              end
+            end
+          else
             next
           end
-          vm.PowerOffVM_Task.wait_for_completion
-          vm.Destroy_Task.wait_for_completion
-        else
-          next
         end
       end
 
@@ -156,7 +324,7 @@ module RSpecSystem
       cmd = opts[:c]
 
       ssh = RSpec.configuration.rspec_storage[:nodes][dest][:ssh]
-      ssh_exec!(ssh, "cd /tmp && sudo sh -c '#{cmd}'")
+      ssh_exec!(ssh, cmd)
     end
 
     # Transfer files to a host in the NodeSet.
@@ -171,20 +339,17 @@ module RSpecSystem
       source = opts[:sp]
       dest_path = opts[:dp]
 
-      # Grab a remote path for temp transfer
-      tmpdest = tmppath
-
       # Do the copy and print out results for debugging
       ssh = RSpec.configuration.rspec_storage[:nodes][dest][:ssh]
-      ssh.scp.upload! source.to_s, tmpdest.to_s, :recursive => true
-
-      # Now we move the file into their final destination
-      result = run(:n => opts[:d], :c => "mv #{tmpdest} #{dest_path}")
-      if result[:exit_code] == 0
-        return true
-      else
-        return false
+
+      begin
+        ssh.scp.upload! source.to_s, dest_path.to_s, :recursive => true
+      rescue => e
+        log.error("Error with scp of file #{source} to #{dest}:#{dest_path}")
+        raise e
       end
+
+      true
     end
 
   end

data/resources/prefabs.yml

@@ -46,7 +46,7 @@
       box: 'centos-59-x64-vbox4210-nocm'
       box_url: 'http://puppet-vagrant-boxes.puppetlabs.com/centos-59-x64-vbox4210-nocm.box'
     vsphere:
-      template: 'centos-59-x64-vs51-nocm'
+      template: 'centos-5-x86_64'
 'centos-63-x64':
   description: "Vagrant images obtained from http://puppet-vagrant-boxes.puppetlabs.com"
   facts:
@@ -92,7 +92,7 @@
       box: 'centos-64-x64-vbox4210-nocm'
       box_url: 'http://puppet-vagrant-boxes.puppetlabs.com/centos-64-x64-vbox4210-nocm.box'
     vsphere:
-      template: 'centos-64-x64-vs51-nocm'
+      template: 'centos-6-x86_64'
 'fedora-18-x64':
   description: ""
   facts:
@@ -114,7 +114,7 @@
       box: 'fedora-18-x64-vbox4210-nocm'
       box_url: 'http://puppet-vagrant-boxes.puppetlabs.com/fedora-18-x64-vbox4210-nocm.box'
     vsphere:
-      template: 'fedora-18-x64-vs51-nocm'
+      template: 'fedora-18-x86_64'
 'debian-606-x64':
   description: "Vagrant images obtained from http://puppet-vagrant-boxes.puppetlabs.com"
   facts:
@@ -157,7 +157,7 @@
       box: 'debian-607-x64-vbox4210-nocm'
       box_url: 'http://puppet-vagrant-boxes.puppetlabs.com/debian-607-x64-vbox4210-nocm.box'
     vsphere:
-      template: 'debian-607-x64-vs51-nocm'
+      template: 'debian-6-x86_64'
 'debian-70rc1-x64':
   description: ""
   facts:
@@ -181,7 +181,7 @@
       box: 'debian-70rc1-x64-vbox4210-nocm'
       box_url: 'http://puppet-vagrant-boxes.puppetlabs.com/debian-70rc1-x64-vbox4210-nocm.box'
     vsphere:
-      template: 'debian-70rc1-x64-vs51-nocm'
+      template: 'debian-7-x86_64'
 'ubuntu-server-1004-x64':
   description: ''
   facts:
@@ -227,7 +227,7 @@
       box: 'ubuntu-server-10044-x64-vbox4210-nocm'
       box_url: 'http://puppet-vagrant-boxes.puppetlabs.com/ubuntu-server-10044-x64-vbox4210-nocm.box'
     vsphere:
-      template: 'ubuntu-server-10044-x64-vs51-nocm'
+      template: 'ubuntu-1004-x86_64'
 'ubuntu-server-1204-x64':
   description: ''
   facts:
@@ -273,7 +273,7 @@
       box: 'ubuntu-server-12042-x64-vbox4210-nocm'
       box_url: 'http://puppet-vagrant-boxes.puppetlabs.com/ubuntu-server-12042-x64-vbox4210-nocm.box'
     vsphere:
-      template: 'ubuntu-server-12042-x64-vs51-nocm'
+      template: 'ubuntu-1204-x86_64'
 'sles-11sp1-x64':
   description: ''
   facts:
@@ -292,4 +292,4 @@
       box: 'sles-11sp1-x64-vbox4210-nocm'
       box_url: 'http://puppet-vagrant-boxes.puppetlabs.com/sles-11sp1-x64-vbox4210-nocm.box'
     vsphere:
-      template: 'sles-11sp1-x64-vs51-nocm'
+      template: 'sles-11-x86_64'

data/rspec-system.gemspec

@@ -2,7 +2,7 @@
 Gem::Specification.new do |s|
   # Metadata
   s.name        = "rspec-system"
-  s.version     = "2.4.0"
+  s.version     = "2.5.0"
   s.authors     = ["Ken Barber"]
   s.email       = ["info@puppetlabs.com"]
   s.homepage    = "https://github.com/puppetlabs/rspec-system"

data/spec/spec_helper_system.rb

@@ -21,6 +21,7 @@ RSpec.configure do |c|
   c.include ::LocalHelpers
 
   c.before :suite do
+    shell 'echo "mesg n" >> ~/.profile'
     shell 'echo foobar > /tmp/setupblock'
   end
 end

data/spec/system/shell_spec.rb

@@ -56,17 +56,17 @@ describe "shell:" do
   end
 
   it 'escape all quotes properly' do
-    shell 'rm -f ~vagrant/foo'
-    shell "su - vagrant -c 'echo \"foo bar baz\" > ~/foo'" do |r|
+    shell 'rm -f ~/foo'
+    shell "bash -c 'echo \"foo bar baz\" > ~/foo'" do |r|
       r.stderr.should be_empty
       r.exit_code.should be_zero
     end
 
-    shell 'cat ~vagrant/foo' do |r|
+    shell 'cat ~/foo' do |r|
       r.stdout.should =~ /foo bar baz/
       r.exit_code.should be_zero
     end
-    shell 'rm -f ~vagrant/foo'
+    shell 'rm -f ~/foo'
   end
 
   it 'a string of commands should succeed' do
@@ -140,7 +140,7 @@ describe "shell:" do
     end
 
     it 'should complete fine if command completed within timeout' do
-      shell(:c => 'sleep 1', :timeout => 5) do |r|
+      shell(:c => 'sleep 1', :timeout => 10) do |r|
         r.stdout.should == ''
         r.stderr.should == ''
         r.exit_code.should == 0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rspec-system
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.5.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-19 00:00:00.000000000 Z
+date: 2013-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec