beaker 1.16.0 → 1.17.0

data/lib/beaker/dsl/helpers.rb

@@ -3,6 +3,9 @@ require 'resolv'
 require 'inifile'
 require 'timeout'
 require 'beaker/dsl/outcomes'
+require 'beaker/options'
+require 'hocon'
+require 'hocon/config_error'
 
 module Beaker
   module DSL
@@ -74,19 +77,16 @@ module Beaker
       # @return [Result]   An object representing the outcome of *command*.
       # @raise  [FailTest] Raises an exception if *command* obviously fails.
       def on(host, command, opts = {}, &block)
-        unless command.is_a? Command
+        block_on host do | host |
+          if command.is_a? Command
+            command = command.cmd_line(host)
+          end
           cmd_opts = {}
+          #add any additional environment variables to the command
           if opts[:environment]
             cmd_opts['ENV'] = opts[:environment]
           end
           command = Command.new(command.to_s, [], cmd_opts)
-        end
-        if host.is_a? String or host.is_a? Symbol
-          host = hosts_as(host) #check by role
-        end
-        if host.is_a? Array
-          host.map { |h| on h, command, opts, &block }
-        else
           @result = host.exec(command, opts)
 
           # Also, let additional checking be performed by the caller.
@@ -100,8 +100,7 @@ module Beaker
                 yield @result
             end
           end
-
-          return @result
+          @result
         end
       end
 
@@ -179,11 +178,10 @@ module Beaker
       #
       # @return [Result] Returns the result of the SCP operation
       def scp_from host, from_path, to_path, opts = {}
-        if host.is_a? Array
-          host.each { |h| scp_from h, from_path, to_path, opts }
-        else
+        block_on host do | host |
           @result = host.do_scp_from(from_path, to_path, opts)
           @result.log logger
+          @result
         end
       end
 
@@ -201,11 +199,10 @@ module Beaker
       #
       # @return [Result] Returns the result of the SCP operation
       def scp_to host, from_path, to_path, opts = {}
-        if host.is_a? Array
-          host.each { |h| scp_to h, from_path, to_path, opts }
-        else
+        block_on host do | host |
           @result = host.do_scp_to(from_path, to_path, opts)
           @result.log logger
+          @result
         end
       end
 
@@ -349,7 +346,7 @@ module Beaker
       # @option opts [String] :source The location on the test runners box where the files are found
       # @option opts [String] :module_name The name of the module to be copied over
       def puppet_module_install_on(host, opts = {})
-        Array(host).each do |h|
+        block_on host do | h |
           on h, puppet("module install #{opts[:module_name]}")
         end
       end
@@ -405,31 +402,14 @@ module Beaker
       # @raise [SkipTest] Raises skip test if there are no valid hosts for
       #   this test case after confinement.
       def confine(type, criteria, host_array = nil, &block)
-        provided_hosts = host_array ? true : false
         hosts_to_modify = host_array || hosts
-        criteria.each_pair do |property, value|
-          case type
-          when :except
-            hosts_to_modify = hosts_to_modify.reject do |host|
-              inspect_host host, property, value
-            end
-            if block_given?
-              hosts_to_modify = hosts_to_modify.reject do |host|
-                yield host
-              end
-            end
-          when :to
-            hosts_to_modify = hosts_to_modify.select do |host|
-              inspect_host host, property, value
-            end
-            if block_given?
-              hosts_to_modify = hosts_to_modify.select do |host|
-                yield host
-              end
-            end
-          else
-            raise "Unknown option #{type}"
-          end
+        case type
+        when :except
+          hosts_to_modify = hosts_to_modify - select_hosts(criteria, hosts_to_modify, &block)
+        when :to
+          hosts_to_modify = select_hosts(criteria, hosts_to_modify, &block)
+        else
+          raise "Unknown option #{type}"
         end
         if hosts_to_modify.empty?
           logger.warn "No suitable hosts with: #{criteria.inspect}"
@@ -456,6 +436,59 @@ module Beaker
         end
       end
 
+      #Return a set of hosts that meet the given criteria
+      # @param [Hash{Symbol,String=>String,Regexp,Array<String,Regexp>}]
+      #   criteria Specify the criteria with which a host should be
+      #   considered for inclusion.  The key is any attribute
+      #   of the host that will be yielded by {Beaker::Host#[]}.
+      #   The value can be any string/regex or array of strings/regexp.
+      #   The values are compared using [Enumerable#any?] so that if one
+      #   value of an array matches the host is considered a match for that
+      #   criteria.
+      # @param [Array<Host>] host_array This creatively named parameter is
+      #   an optional array of hosts to confine to.  If not passed in, this
+      #   method will modify {Beaker::TestCase#hosts} in place.
+      # @param [Proc] block Addition checks to determine suitability of hosts
+      #   for selection.  Each host that is still valid after checking
+      #   *criteria* is then passed in turn into this block.  The block
+      #   should return true if the host matches this additional criteria.
+      #
+      # @return [Array<Host>] Returns an array of hosts that meet the provided criteria
+      def select_hosts(criteria, host_array = nil, &block)
+        hosts_to_select_from = host_array || hosts
+        criteria.each_pair do |property, value|
+          hosts_to_select_from = hosts_to_select_from.select do |host|
+            inspect_host host, property, value
+          end
+        end
+        if block_given?
+          hosts_to_select_from = hosts_to_select_from.select do |host|
+            yield host
+          end
+        end
+        hosts_to_select_from
+      end
+
+      # Return the name of the puppet user.
+      #
+      # @param [Host] host One object that acts like a Beaker::Host
+      #
+      # @note This method assumes puppet is installed on the host.
+      #
+      def puppet_user(host)
+        return host.puppet('master')['group']
+      end
+
+      # Return the name of the puppet group.
+      #
+      # @param [Host] host One object that acts like a Beaker::Host
+      #
+      # @note This method assumes puppet is installed on the host.
+      #
+      def puppet_group(host)
+        return host.puppet('master')['user']
+      end
+
       # @!visibility private
       def inspect_host(host, property, one_or_more_values)
         values = Array(one_or_more_values)
@@ -489,16 +522,24 @@ module Beaker
       #                            specified, if no section is specified the
       #                            a puppet.conf file will be written with the
       #                            options put in a section named after [mode]
-      #
-      #                            There is a special setting for command_line
-      #                            arguments such as --debug or --logdest, which
-      #                            cannot be set in puppet.conf.   For example:
+      # @option conf_opts [String] :__commandline_args__  A special setting for
+      #                            command_line arguments such as --debug or
+      #                            --logdest, which cannot be set in
+      #                            puppet.conf. For example:
       #
       #                            :__commandline_args__ => '--logdest /tmp/a.log'
       #
       #                            These will only be applied when starting a FOSS
       #                            master, as a pe master is just bounced.
-      #
+      # @option conf_opts [Hash]   :__service_args__  A special setting of options
+      #                            for controlling how the puppet master service is
+      #                            handled. The only setting currently is
+      #                            :bypass_service_script, which if set true will
+      #                            force stopping and starting a webrick master
+      #                            using the start_puppet_from_source_* methods,
+      #                            even if it seems the host has passenger.
+      #                            This is needed in FOSS tests to initialize
+      #                            SSL.
       # @param [File] testdir      The temporary directory which will hold backup
       #                            configuration, and other test artifacts.
       #
@@ -524,16 +565,43 @@ module Beaker
       def with_puppet_running_on host, conf_opts, testdir = host.tmpdir(File.basename(@path)), &block
         raise(ArgumentError, "with_puppet_running_on's conf_opts must be a Hash. You provided a #{conf_opts.class}: '#{conf_opts}'") if !conf_opts.kind_of?(Hash)
         cmdline_args = conf_opts[:__commandline_args__]
-        conf_opts = conf_opts.reject { |k,v| k == :__commandline_args__ }
+        service_args = conf_opts[:__service_args__] || {}
+        conf_opts = conf_opts.reject { |k,v| [:__commandline_args__, :__service_args__].include?(k) }
 
         curl_retries = host['master-start-curl-retries'] || options['master-start-curl-retries']
         logger.debug "Setting curl retries to #{curl_retries}"
 
+        if options[:is_jvm_puppet]
+          confdir = host.puppet('master')['confdir']
+          vardir = host.puppet('master')['vardir']
+
+          if cmdline_args
+            split_args = cmdline_args.split()
+
+            split_args.each do |arg|
+              case arg
+              when /--confdir=(.*)/
+                confdir = $1
+              when /--vardir=(.*)/
+                vardir = $1
+              end
+            end
+          end
+
+          jvm_puppet_opts = { "jruby-puppet" => {
+            "master-conf-dir" => confdir,
+            "master-var-dir" => vardir,
+          }}
+
+          jvm_puppet_conf = File.join("#{host['jvm-puppet-confdir']}", "jvm-puppet.conf")
+          modify_tk_config(host, jvm_puppet_conf, jvm_puppet_opts)
+        end
+
         begin
           backup_file = backup_the_file(host, host['puppetpath'], testdir, 'puppet.conf')
           lay_down_new_puppet_conf host, conf_opts, testdir
 
-          if host['puppetservice']
+          if host.use_service_scripts? && !service_args[:bypass_service_script]
             bounce_service( host, host['puppetservice'], curl_retries )
           else
             puppet_master_started = start_puppet_from_source_on!( host, cmdline_args )
@@ -547,9 +615,9 @@ module Beaker
 
         ensure
           begin
-            restore_puppet_conf_from_backup( host, backup_file )
 
-            if host['puppetservice']
+            if host.use_service_scripts? && !service_args[:bypass_service_script]
+              restore_puppet_conf_from_backup( host, backup_file )
               bounce_service( host, host['puppetservice'], curl_retries )
             else
               if puppet_master_started
@@ -557,6 +625,7 @@ module Beaker
               else
                 dump_puppet_log(host)
               end
+              restore_puppet_conf_from_backup( host, backup_file )
             end
 
           rescue Exception => teardown_exception
@@ -655,7 +724,7 @@ module Beaker
       # @!visibility private
       def dump_puppet_log(host)
         syslogfile = case host['platform']
-          when /fedora|centos|el/ then '/var/log/messages'
+          when /fedora|centos|el|redhat|scientific/ then '/var/log/messages'
           when /ubuntu|debian/ then '/var/log/syslog'
           else return
         end
@@ -687,10 +756,89 @@ module Beaker
         new_conf
       end
 
+      # Modify the given TrapperKeeper config file.
+      #
+      # @param [Host] host  A host object
+      # @param [OptionsHash] options_hash  New hash which will be merged into
+      #                                    the given TrapperKeeper config.
+      # @param [String] config_file_path  Path to the TrapperKeeper config on
+      #                                   the given host which is to be
+      #                                   modified.
+      # @param [Bool] replace  If set true, instead of updating the existing
+      #                        TrapperKeeper configuration, replace it entirely
+      #                        with the contents of the given hash.
+      #
+      # @note TrapperKeeper config files can be HOCON, JSON, or Ini. We don't
+      # particularly care which of these the file named by `config_file_path` on
+      # the SUT actually is, just that the contents can be parsed into a map.
+      #
+      def modify_tk_config(host, config_file_path, options_hash, replace=false)
+        if options_hash.empty?
+          return nil
+        end
+
+        new_hash = Beaker::Options::OptionsHash.new
+
+        if replace
+          new_hash.merge!(options_hash)
+        else
+          if not host.file_exist?( config_file_path )
+            raise "Error: #{config_file_path} does not exist on #{host}"
+          end
+          file_string = host.exec( Command.new( "cat #{config_file_path}" )).stdout
+
+          begin
+            tk_conf_hash = read_tk_config_string(file_string)
+          rescue RuntimeError
+            raise "Error reading trapperkeeper config: #{config_file_path} at host: #{host}"
+          end
+
+          new_hash.merge!(tk_conf_hash)
+          new_hash.merge!(options_hash)
+        end
+
+        file_string = JSON.dump(new_hash)
+        create_remote_file host, config_file_path, file_string
+      end
+
+      # The Trapperkeeper config service will accept HOCON (aka typesafe), JSON,
+      # or Ini configuration files which means we need to safely handle the the
+      # exceptions that might come from parsing the given string with the wrong
+      # parser and fall back to the next valid parser in turn. We finally raise
+      # a RuntimeException if none of the parsers succeed.
+      #
+      # @!visibility private
+      def read_tk_config_string( string )
+          begin
+            return Hocon.parse(string)
+          rescue Hocon::ConfigError
+            nil
+          end
+
+          begin
+            return JSON.parse(string)
+          rescue JSON::JSONError
+            nil
+          end
+
+          begin
+            return IniFile.new(string)
+          rescue IniFile::Error
+            nil
+          end
+
+          raise "Failed to read TrapperKeeper config!"
+      end
+
       # @!visibility private
       def bounce_service host, service, curl_retries = 120
-        host.exec puppet_resource( 'service', service, 'ensure=stopped' )
-        host.exec puppet_resource( 'service', service, 'ensure=running' )
+        if host.graceful_restarts?
+          apachectl_path = host.is_pe? ? "#{host['puppetsbindir']}/apache2ctl" : 'apache2ctl'
+          host.exec(Command.new("#{apachectl_path} graceful"))
+        else
+          host.exec puppet_resource('service', service, 'ensure=stopped')
+          host.exec puppet_resource('service', service, 'ensure=running')
+        end
         curl_with_retries(" #{service} ", host, "https://localhost:8140", [35, 60], curl_retries)
       end
 
@@ -769,79 +917,76 @@ module Beaker
       #                      validation, etc.
       #
       def apply_manifest_on(host, manifest, opts = {}, &block)
-        if host.is_a?(Array)
-          return host.map do |h|
-            apply_manifest_on(h, manifest, opts, &block)
+        block_on host do | host |
+
+          on_options = {}
+          on_options[:acceptable_exit_codes] = Array(opts[:acceptable_exit_codes])
+
+          puppet_apply_opts = {}
+          puppet_apply_opts[:verbose] = nil
+          puppet_apply_opts[:parseonly] = nil if opts[:parseonly]
+          puppet_apply_opts[:trace] = nil if opts[:trace]
+          puppet_apply_opts[:parser] = 'future' if opts[:future_parser]
+          puppet_apply_opts[:modulepath] = opts[:modulepath] if opts[:modulepath]
+          puppet_apply_opts[:noop] = nil if opts[:noop]
+
+          # From puppet help:
+          # "... an exit code of '2' means there were changes, an exit code of
+          # '4' means there were failures during the transaction, and an exit
+          # code of '6' means there were both changes and failures."
+          if [opts[:catch_changes],opts[:catch_failures],opts[:expect_failures],opts[:expect_changes]].compact.length > 1
+            raise(ArgumentError,
+                  'Cannot specify more than one of `catch_failures`, ' +
+                  '`catch_changes`, `expect_failures`, or `expect_changes` ' +
+                  'for a single manifest')
           end
-        end
 
-        on_options = {}
-        on_options[:acceptable_exit_codes] = Array(opts[:acceptable_exit_codes])
-
-        puppet_apply_opts = {}
-        puppet_apply_opts[:verbose] = nil
-        puppet_apply_opts[:parseonly] = nil if opts[:parseonly]
-        puppet_apply_opts[:trace] = nil if opts[:trace]
-        puppet_apply_opts[:parser] = 'future' if opts[:future_parser]
-        puppet_apply_opts[:modulepath] = opts[:modulepath] if opts[:modulepath]
-        puppet_apply_opts[:noop] = nil if opts[:noop]
-
-        # From puppet help:
-        # "... an exit code of '2' means there were changes, an exit code of
-        # '4' means there were failures during the transaction, and an exit
-        # code of '6' means there were both changes and failures."
-        if [opts[:catch_changes],opts[:catch_failures],opts[:expect_failures],opts[:expect_changes]].compact.length > 1
-          raise(ArgumentError,
-                'Cannot specify more than one of `catch_failures`, ' +
-                '`catch_changes`, `expect_failures`, or `expect_changes` ' +
-                'for a single manifest')
-        end
+          if opts[:catch_changes]
+            puppet_apply_opts['detailed-exitcodes'] = nil
 
-        if opts[:catch_changes]
-          puppet_apply_opts['detailed-exitcodes'] = nil
+            # We're after idempotency so allow exit code 0 only.
+            on_options[:acceptable_exit_codes] |= [0]
+          elsif opts[:catch_failures]
+            puppet_apply_opts['detailed-exitcodes'] = nil
 
-          # We're after idempotency so allow exit code 0 only.
-          on_options[:acceptable_exit_codes] |= [0]
-        elsif opts[:catch_failures]
-          puppet_apply_opts['detailed-exitcodes'] = nil
+            # We're after only complete success so allow exit codes 0 and 2 only.
+            on_options[:acceptable_exit_codes] |= [0, 2]
+          elsif opts[:expect_failures]
+            puppet_apply_opts['detailed-exitcodes'] = nil
 
-          # We're after only complete success so allow exit codes 0 and 2 only.
-          on_options[:acceptable_exit_codes] |= [0, 2]
-        elsif opts[:expect_failures]
-          puppet_apply_opts['detailed-exitcodes'] = nil
+            # We're after failures specifically so allow exit codes 1, 4, and 6 only.
+            on_options[:acceptable_exit_codes] |= [1, 4, 6]
+          elsif opts[:expect_changes]
+            puppet_apply_opts['detailed-exitcodes'] = nil
 
-          # We're after failures specifically so allow exit codes 1, 4, and 6 only.
-          on_options[:acceptable_exit_codes] |= [1, 4, 6]
-        elsif opts[:expect_changes]
-          puppet_apply_opts['detailed-exitcodes'] = nil
+            # We're after changes specifically so allow exit code 2 only.
+            on_options[:acceptable_exit_codes] |= [2]
+          else
+            # Either use the provided acceptable_exit_codes or default to [0]
+            on_options[:acceptable_exit_codes] |= [0]
+          end
 
-          # We're after changes specifically so allow exit code 2 only.
-          on_options[:acceptable_exit_codes] |= [2]
-        else
-          # Either use the provided acceptable_exit_codes or default to [0]
-          on_options[:acceptable_exit_codes] |= [0]
-        end
+          # Not really thrilled with this implementation, might want to improve it
+          # later. Basically, there is a magic trick in the constructor of
+          # PuppetCommand which allows you to pass in a Hash for the last value in
+          # the *args Array; if you do so, it will be treated specially. So, here
+          # we check to see if our caller passed us a hash of environment variables
+          # that they want to set for the puppet command. If so, we set the final
+          # value of *args to a new hash with just one entry (the value of which
+          # is our environment variables hash)
+          if opts.has_key?(:environment)
+            puppet_apply_opts['ENV'] = opts[:environment]
+          end
 
-        # Not really thrilled with this implementation, might want to improve it
-        # later.  Basically, there is a magic trick in the constructor of
-        # PuppetCommand which allows you to pass in a Hash for the last value in
-        # the *args Array; if you do so, it will be treated specially.  So, here
-        # we check to see if our caller passed us a hash of environment variables
-        # that they want to set for the puppet command.  If so, we set the final
-        # value of *args to a new hash with just one entry (the value of which
-        # is our environment variables hash)
-        if opts.has_key?(:environment)
-          puppet_apply_opts['ENV'] = opts[:environment]
-        end
+          file_path = host.tmpfile('apply_manifest.pp')
+          create_remote_file(host, file_path, manifest + "\n")
 
-        file_path = host.tmpfile('apply_manifest.pp')
-        create_remote_file(host, file_path, manifest + "\n")
+          if host[:default_apply_opts].respond_to? :merge
+            puppet_apply_opts = host[:default_apply_opts].merge( puppet_apply_opts )
+          end
 
-        if host[:default_apply_opts].respond_to? :merge
-          puppet_apply_opts = host[:default_apply_opts].merge( puppet_apply_opts )
+          on host, puppet('apply', file_path, puppet_apply_opts), on_options, &block
         end
-
-        on host, puppet('apply', file_path, puppet_apply_opts), on_options, &block
       end
 
       # Runs 'puppet apply' on default host, piping manifest through stdin
@@ -853,9 +998,7 @@ module Beaker
       # @deprecated
       def run_agent_on(host, arg='--no-daemonize --verbose --onetime --test',
                        options={}, &block)
-        if host.is_a? Array
-          host.each { |h| run_agent_on h, arg, options, &block }
-        else
+        block_on host do | host |
           on host, puppet_agent(arg), options, &block
         end
       end
@@ -863,32 +1006,34 @@ module Beaker
       # FIX: this should be moved into host/platform
       # @visibility private
       def run_cron_on(host, action, user, entry="", &block)
-        platform = host['platform']
-        if platform.include?('solaris') || platform.include?('aix') then
-          case action
-            when :list   then args = '-l'
-            when :remove then args = '-r'
-            when :add
-              on( host,
-                 "echo '#{entry}' > /var/spool/cron/crontabs/#{user}",
-                  &block )
-          end
+        block_on host do | host |
+          platform = host['platform']
+          if platform.include?('solaris') || platform.include?('aix') then
+            case action
+              when :list   then args = '-l'
+              when :remove then args = '-r'
+              when :add
+                on( host,
+                   "echo '#{entry}' > /var/spool/cron/crontabs/#{user}",
+                    &block )
+            end
 
-        else # default for GNU/Linux platforms
-          case action
-            when :list   then args = '-l -u'
-            when :remove then args = '-r -u'
-            when :add
-               on( host,
-                  "echo '#{entry}' > /tmp/#{user}.cron && " +
-                  "crontab -u #{user} /tmp/#{user}.cron",
-                   &block )
+          else # default for GNU/Linux platforms
+            case action
+              when :list   then args = '-l -u'
+              when :remove then args = '-r -u'
+              when :add
+                 on( host,
+                    "echo '#{entry}' > /tmp/#{user}.cron && " +
+                    "crontab -u #{user} /tmp/#{user}.cron",
+                     &block )
+            end
           end
-        end
 
-        if args
-          case action
-            when :list, :remove then on(host, "crontab #{args} #{user}", &block)
+          if args
+            case action
+              when :list, :remove then on(host, "crontab #{args} #{user}", &block)
+            end
           end
         end
       end
@@ -899,23 +1044,24 @@ module Beaker
       # A teardown step is also added to make sure unstubbing of the host is
       # removed always.
       #
-      # @param machine [String] the host to execute this stub
+      # @param [Host, Array<Host>, String, Symbol] machine    One or more hosts to act upon,
+      #                            or a role (String or Symbol) that identifies one or more hosts.
       # @param ip_spec [Hash{String=>String}] a hash containing the host to ip
       #   mappings
       # @example Stub puppetlabs.com on the master to 127.0.0.1
       #   stub_hosts_on(master, 'puppetlabs.com' => '127.0.0.1')
       def stub_hosts_on(machine, ip_spec)
-        ip_spec.each do |host, ip|
-          logger.notify("Stubbing host #{host} to IP #{ip} on machine #{machine}")
-          on( machine,
-              puppet('resource', 'host', host, 'ensure=present', "ip=#{ip}") )
-        end
+        block_on machine do | host |
+          ip_spec.each do |address, ip|
+            logger.notify("Stubbing address #{address} to IP #{ip} on machine #{host}")
+            on( host, puppet('resource', 'host', address, 'ensure=present', "ip=#{ip}") )
+          end
 
-        teardown do
-          ip_spec.each do |host, ip|
-            logger.notify("Unstubbing host #{host} to IP #{ip} on machine #{machine}")
-            on( machine,
-                puppet('resource', 'host', host, 'ensure=absent') )
+          teardown do
+            ip_spec.each do |address, ip|
+              logger.notify("Unstubbing address #{address} to IP #{ip} on machine #{host}")
+              on( host, puppet('resource', 'host', address, 'ensure=absent') )
+            end
           end
         end
       end
@@ -943,8 +1089,10 @@ module Beaker
         #use global options hash
         forge_host ||= options[:forge_host]
         @forge_ip ||= Resolv.getaddress(forge_host)
-        stub_hosts_on(machine, 'forge.puppetlabs.com' => @forge_ip)
-        stub_hosts_on(machine, 'forgeapi.puppetlabs.com' => @forge_ip)
+        block_on machine do | host |
+          stub_hosts_on(host, 'forge.puppetlabs.com' => @forge_ip)
+          stub_hosts_on(host, 'forgeapi.puppetlabs.com' => @forge_ip)
+        end
       end
 
       # This wraps the method `stub_hosts` and makes the stub specific to
@@ -1024,32 +1172,36 @@ module Beaker
       end
 
       #stops the puppet agent running on the host
+      # @param [Host, Array<Host>, String, Symbol] agent    One or more hosts to act upon,
+      #                            or a role (String or Symbol) that identifies one or more hosts.
       def stop_agent_on(agent)
-        vardir = agent.puppet['vardir']
-        agent_running = true
-        while agent_running
-          result = on agent, "[ -e '#{vardir}/state/agent_catalog_run.lock' ]", :acceptable_exit_codes => [0,1]
-          agent_running = (result.exit_code == 0)
-          sleep 2 unless agent_running
-        end
+        block_on agent do | host |
+          vardir = agent.puppet['vardir']
+          agent_running = true
+          while agent_running
+            result = on host, "[ -e '#{vardir}/state/agent_catalog_run.lock' ]", :acceptable_exit_codes => [0,1]
+            agent_running = (result.exit_code == 0)
+            sleep 2 unless agent_running
+          end
 
-        # The agent service is `pe-puppet` everywhere EXCEPT certain linux distros on PE 2.8
-        # In all the case that it is different, this init script will exist. So we can assume
-        # that if the script doesn't exist, we should just use `pe-puppet`
-        result = on agent, "[ -e /etc/init.d/pe-puppet-agent ]", :acceptable_exit_codes => [0,1]
-        agent_service = (result.exit_code == 0) ? 'pe-puppet-agent' : 'pe-puppet'
-
-        # Under a number of stupid circumstances, we can't stop the
-        # agent using puppet.  This is usually because of issues with
-        # the init script or system on that particular configuration.
-        avoid_puppet_at_all_costs = false
-        avoid_puppet_at_all_costs ||= agent['platform'] =~ /el-4/
-        avoid_puppet_at_all_costs ||= agent['pe_ver'] && version_is_less(agent['pe_ver'], '3.2') && agent['platform'] =~ /sles/
-
-        if avoid_puppet_at_all_costs
-          on agent, "/etc/init.d/#{agent_service} stop"
-        else
-          on agent, puppet_resource('service', agent_service, 'ensure=stopped')
+          # The agent service is `pe-puppet` everywhere EXCEPT certain linux distros on PE 2.8
+          # In all the case that it is different, this init script will exist. So we can assume
+          # that if the script doesn't exist, we should just use `pe-puppet`
+          result = on agent, "[ -e /etc/init.d/pe-puppet-agent ]", :acceptable_exit_codes => [0,1]
+          agent_service = (result.exit_code == 0) ? 'pe-puppet-agent' : 'pe-puppet'
+
+          # Under a number of stupid circumstances, we can't stop the
+          # agent using puppet.  This is usually because of issues with
+          # the init script or system on that particular configuration.
+          avoid_puppet_at_all_costs = false
+          avoid_puppet_at_all_costs ||= agent['platform'] =~ /el-4/
+          avoid_puppet_at_all_costs ||= agent['pe_ver'] && version_is_less(agent['pe_ver'], '3.2') && agent['platform'] =~ /sles/
+
+          if avoid_puppet_at_all_costs
+            on agent, "/etc/init.d/#{agent_service} stop"
+          else
+            on agent, puppet_resource('service', agent_service, 'ensure=stopped')
+          end
         end
       end
 
@@ -1068,31 +1220,34 @@ module Beaker
 
       # Ensure the host has requested a cert, then sign it
       #
-      # @param [Host] host  The host to sign for
+      # @param [Host, Array<Host>, String, Symbol] host    One or more hosts to act upon,
+      #                            or a role (String or Symbol) that identifies one or more hosts.
       #
       # @return nil
       # @raise [FailTest] if process times out
       def sign_certificate_for(host)
-        if [master, dashboard, database].include? host
+        block_on host do | host |
+          if [master, dashboard, database].include? host
 
-          on host, puppet( 'agent -t' ), :acceptable_exit_codes => [0,1,2]
-          on master, puppet( "cert --allow-dns-alt-names sign #{host}" ), :acceptable_exit_codes => [0,24]
+            on host, puppet( 'agent -t' ), :acceptable_exit_codes => [0,1,2]
+            on master, puppet( "cert --allow-dns-alt-names sign #{host}" ), :acceptable_exit_codes => [0,24]
 
-        else
+          else
 
-          hostname = Regexp.escape host.node_name
+            hostname = Regexp.escape host.node_name
 
-          last_sleep = 0
-          next_sleep = 1
-          (0..10).each do |i|
-            fail_test("Failed to sign cert for #{hostname}") if i == 10
+            last_sleep = 0
+            next_sleep = 1
+            (0..10).each do |i|
+              fail_test("Failed to sign cert for #{hostname}") if i == 10
 
-            on master, puppet("cert --sign --all"), :acceptable_exit_codes => [0,24]
-            break if on(master, puppet("cert --list --all")).stdout =~ /\+ "?#{hostname}"?/
-            sleep next_sleep
-            (last_sleep, next_sleep) = next_sleep, last_sleep+next_sleep
-          end
+              on master, puppet("cert --sign --all"), :acceptable_exit_codes => [0,24]
+              break if on(master, puppet("cert --list --all")).stdout =~ /\+ "?#{hostname}"?/
+              sleep next_sleep
+              (last_sleep, next_sleep) = next_sleep, last_sleep+next_sleep
+            end
 
+          end
         end
       end
 
@@ -1104,7 +1259,8 @@ module Beaker
 
       # Get a facter fact from a provided host
       #
-      # @param [Host] host  The host to query the fact for
+      # @param [Host, Array<Host>, String, Symbol] host    One or more hosts to act upon,
+      #                            or a role (String or Symbol) that identifies one or more hosts.
       # @param [String] name The name of the fact to query for
       # @!macro common_opts
       #
@@ -1112,7 +1268,11 @@ module Beaker
       # @raise  [FailTest] Raises an exception if call to facter fails
       def fact_on(host, name, opts = {})
         result = on host, facter(name, opts)
-        result.stdout.chomp if result.stdout
+        if result.kind_of?(Array)
+          result.map { |res| res.stdout.chomp }
+        else
+          result.stdout.chomp
+        end
       end
 
       # Get a facter fact from the default host
@@ -1200,7 +1360,7 @@ module Beaker
         module_name = nil
         if File.exists?("#{root_module_dir}/metadata.json")
           logger.debug "Attempting to parse Modulename from metadata.json"
-          module_json = JSON.parse (File.read "#{root_module_dir}/metadata.json")
+          module_json = JSON.parse(File.read "#{root_module_dir}/metadata.json")
           if(module_json.has_key?('name'))
             module_name = get_module_name(module_json['name'])
           end