beaker 2.52.0 → 3.0.0

data/lib/beaker/dsl/helpers/puppet_helpers.rb

@@ -323,7 +323,8 @@ module Beaker
           curl_retries = 120 if curl_retries.nil?
           port = options[:puppetserver_port] if port.nil?
           if host.graceful_restarts?
-            apachectl_path = host.is_pe? ? "#{host['puppetsbindir']}/apache2ctl" : 'apache2ctl'
+            service = host.check_for_command('apache2ctl') ? 'apache2ctl' : 'apachectl'
+            apachectl_path = host.is_pe? ? "#{host['puppetsbindir']}/#{service}" : service
             host.exec(Command.new("#{apachectl_path} graceful"))
           else
             host.exec puppet_resource('service', service, 'ensure=stopped')

data/lib/beaker/dsl/install_utils/windows_utils.rb

@@ -139,7 +139,33 @@ exit /B %errorlevel%
             # if puppet service exists, then pe-puppet is not queried
             # if puppet service does not exist, pe-puppet is queried and that exit code is used
             # therefore, this command will always exit 0 if either service is installed
-            on host, Command.new("sc query puppet || sc query pe-puppet", [], { :cmdexe => true })
+            #
+            # We also take advantage of this output to verify the startup
+            # settings are honored as supplied to the MSI
+            on host, Command.new("sc qc puppet || sc qc pe-puppet", [], { :cmdexe => true }) do |result|
+              output = result.stdout
+              startup_mode = msi_opts['PUPPET_AGENT_STARTUP_MODE'].upcase
+
+              search = case startup_mode
+                when 'AUTOMATIC'
+                  { :code => 2, :name => 'AUTO_START' }
+                when 'MANUAL'
+                  { :code => 3, :name => 'DEMAND_START' }
+                when 'DISABLED'
+                  { :code => 4, :name => 'DISABLED' }
+                end
+
+              if output !~ /^\s+START_TYPE\s+:\s+#{search[:code]}\s+#{search[:name]}/
+                raise "puppet service startup mode did not match supplied MSI option '#{startup_mode}'"
+              end
+            end
+
+            # (PA-514) value for PUPPET_AGENT_STARTUP_MODE should be present in
+            # registry and honored after install/upgrade.
+            reg_key = host.is_x86_64? ? "HKLM\\SOFTWARE\\Wow6432Node\\Puppet Labs\\PuppetInstaller" :
+                                        "HKLM\\SOFTWARE\\Puppet Labs\\PuppetInstaller"
+            reg_query_command = %Q(reg query "#{reg_key}" /v "RememberedPuppetAgentStartupMode" | findstr #{msi_opts['PUPPET_AGENT_STARTUP_MODE']})
+            on host, Command.new(reg_query_command, [], { :cmdexe => true })
 
             # emit the misc/versions.txt file which contains component versions for
             # puppet, facter, hiera, pxp-agent, packaging and vendored Ruby

data/lib/beaker/host.rb

@@ -92,11 +92,26 @@ module Beaker
           TCPSocket.new(reachable_name, port).close
           return true
         end
-      rescue Errno::ECONNREFUSED, Timeout::Error, Errno::ETIMEDOUT
+      rescue Errno::ECONNREFUSED, Timeout::Error, Errno::ETIMEDOUT, Errno::EHOSTUNREACH
         return false
       end
     end
 
+    # Wait for a port on the host.  Useful for those occasions when you've called
+    # host.reboot and want to avoid spam from subsequent SSH connections retrying
+    # to connect from say retry_on()
+    def wait_for_port(port, attempts=15)
+      @logger.debug("  Waiting for port #{port} ... ", false)
+      start = Time.now
+      done = repeat_fibonacci_style_for(attempts) { port_open?(port) }
+      if done
+        @logger.debug('connected in %0.2f seconds' % (Time.now - start))
+      else
+        @logger.debug('timeout')
+      end
+      done
+    end
+
     def up?
       begin
         Socket.getaddrinfo( reachable_name, nil )

data/lib/beaker/host/unix/exec.rb

@@ -106,7 +106,7 @@ module Unix::Exec
     if exec(Beaker::Command.new("grep ^#{key}=.*#{escaped_val} #{env_file}"), :accept_all_exit_codes => true ).exit_code == 0
       return #nothing to do here, key value pair already exists
     #see if the key already exists
-    elsif exec(Beaker::Command.new("grep ^#{key} #{env_file}"), :accept_all_exit_codes => true ).exit_code == 0
+    elsif exec(Beaker::Command.new("grep ^#{key}= #{env_file}"), :accept_all_exit_codes => true ).exit_code == 0
       exec(Beaker::SedCommand.new(self['platform'], "s/^#{key}=/#{key}=#{escaped_val}:/", env_file))
     else
       exec(Beaker::Command.new("echo \"#{key}=#{val}\" >> #{env_file}"))
@@ -142,7 +142,7 @@ module Unix::Exec
   #  host.get_env_var('path')
   def get_env_var key
     key = key.to_s
-    exec(Beaker::Command.new("env | grep #{key}"), :accept_all_exit_codes => true).stdout.chomp
+    exec(Beaker::Command.new("env | grep ^#{key}="), :accept_all_exit_codes => true).stdout.chomp
   end
 
   #Delete the environment variable from the current ssh environment
@@ -153,7 +153,7 @@ module Unix::Exec
     key = key.to_s
     env_file = self[:ssh_env_file]
     #remove entire line
-    exec(Beaker::SedCommand.new(self['platform'], "/#{key}=.*$/d", env_file))
+    exec(Beaker::SedCommand.new(self['platform'], "/^#{key}=.*$/d", env_file))
     #update the profile.d to current state
     #match it to the contents of ssh_env_file
     mirror_env_to_profile_d(env_file)
@@ -296,7 +296,7 @@ module Unix::Exec
 
   # Checks if selinux is enabled
   #
-  # @return [Boolean] true if selinux is enabled, false otherwise 
+  # @return [Boolean] true if selinux is enabled, false otherwise
   def selinux_enabled?()
     exec(Beaker::Command.new("sudo selinuxenabled"), :accept_all_exit_codes => true).exit_code == 0
   end

data/lib/beaker/host/unix/pkg.rb

@@ -95,8 +95,9 @@ module Unix::Pkg
         if ! check_for_command('pkgutil')
           # https://www.opencsw.org/package/pkgutil/
           noask_text = self.noask_file_text
-          create_remote_file self, File.join(noask_directory, 'noask'), noask_text
-          execute('pkgadd -d http://get.opencsw.org/now -a noask -n all', opts)
+          noask_file = File.join(external_copy_base, 'noask')
+          create_remote_file(self, noask_file, noask_text)
+          execute("pkgadd -d http://get.opencsw.org/now -a #{noask_file} -n all", opts)
           execute('/opt/csw/bin/pkgutil -U', opts)
           execute('/opt/csw/bin/pkgutil -y -i pkgutil', opts)
         end

data/lib/beaker/host_prebuilt_steps.rb

@@ -161,41 +161,6 @@ module Beaker
       report_and_raise(logger, e, "sync_root_keys")
     end
 
-    #Determine the Extra Packages for Enterprise Linux URL for the provided Enterprise Linux host.
-    # @param [Host, Array<Host>] host One host to act on.  Will use host epel_url, epel_arch and epel_pkg
-    #                                 before using defaults provided in opts.
-    # @return [String, String, String] The URL, arch  and package name for EPL for the provided host
-    # @param [Hash{Symbol=>String}] opts Options to alter execution.
-    # @option opts [String] :epel_url Link to download
-    # @option opts [String] :epel_arch Architecture to download (i386, x86_64, etc), defaults to i386 for 5 and 6, x86_64 for 7
-    # @option opts [String] :epel_6_pkg Package to download from provided link for el-6
-    # @option opts [String] :epel_5_pkg Package to download from provided link for el-5
-    # @raise [Exception] Raises an error if the host provided's platform != /el-(5|6)/
-    def epel_info_for host, opts
-      if !el_based?(host)
-        raise "epel_info_for! not available for #{host.name} on platform #{host['platform']}"
-      end
-
-      version = host['platform'].version
-      url = "#{host[:epel_url] || opts[:epel_url]}/#{version}"
-      if version == '7'
-        if opts[:epel_7_arch] == 'i386'
-          raise ArgumentError.new("epel-7 does not provide packages for i386")
-        end
-        pkg = host[:epel_pkg] || opts[:epel_7_pkg]
-        arch = opts[:epel_7_arch] || 'x86_64'
-      elsif version == '6'
-        pkg = host[:epel_pkg] || opts[:epel_6_pkg]
-        arch = host[:epel_arch] || opts[:epel_6_arch] || 'i386'
-      elsif version == '5'
-        pkg = host[:epel_pkg] || opts[:epel_5_pkg]
-        arch = host[:epel_arch] || opts[:epel_5_arch] || 'i386'
-      else
-        raise ArgumentError.new("epel_info_for does not support el version #{version}, on #{host.name}")
-      end
-      return url, arch, pkg
-    end
-
     # Run 'apt-get update' on the provided host or hosts.
     # If the platform of the provided host is not ubuntu, debian or cumulus: do nothing.
     #
@@ -258,10 +223,6 @@ module Beaker
     # @option opts [Boolean] :debug If true, print verbose rpm information when installing EPEL
     # @option opts [Beaker::Logger] :logger A {Beaker::Logger} object
     # @option opts [String] :epel_url Link to download from
-    # @option opts [String] :epel_arch Architecture of epel to download (i386, x86_64, etc)
-    # @option opts [String] :epel_7_pkg Package to download from provided link for el-7
-    # @option opts [String] :epel_6_pkg Package to download from provided link for el-6
-    # @option opts [String] :epel_5_pkg Package to download from provided link for el-5
     def add_el_extras( host, opts )
       #add_el_extras
       #only supports el-* platforms
@@ -272,14 +233,9 @@ module Beaker
         when el_based?(host) && ['5','6','7'].include?(host['platform'].version)
           result = host.exec(Command.new('rpm -qa | grep epel-release'), :acceptable_exit_codes => [0,1])
           if result.exit_code == 1
-            url, arch, pkg = epel_info_for host, opts
-            if host['platform'].version == '7'
-              host.exec(Command.new("rpm -i#{debug_opt} #{url}/#{arch}/e/#{pkg}"))
-            else
-              host.exec(Command.new("rpm -i#{debug_opt} #{url}/#{arch}/#{pkg}"))
-            end
+            host.exec(Command.new("rpm -i#{debug_opt} #{opts[:epel_url]}/epel-release-latest-#{host['platform'].version}.noarch.rpm"))
             #update /etc/yum.repos.d/epel.repo for new baseurl
-            host.exec(Command.new("sed -i -e 's;#baseurl.*$;baseurl=#{Regexp.escape(url)}/\$basearch;' /etc/yum.repos.d/epel.repo"))
+            host.exec(Command.new("sed -i -e 's;#baseurl.*$;baseurl=#{Regexp.escape("#{opts[:epel_url]}/#{host['platform'].version}")}/\$basearch;' /etc/yum.repos.d/epel.repo"))
             #remove mirrorlist
             host.exec(Command.new("sed -i -e '/mirrorlist/d' /etc/yum.repos.d/epel.repo"))
             host.exec(Command.new('yum clean all && yum makecache'))

data/lib/beaker/hypervisor.rb

@@ -149,6 +149,6 @@ module Beaker
   end
 end
 
-[ 'vsphere_helper', 'vagrant', 'vagrant_custom', 'vagrant_virtualbox', 'vagrant_parallels', 'vagrant_libvirt', 'vagrant_fusion', 'vagrant_workstation', 'fusion', 'aws_sdk', 'vsphere', 'vmpooler', 'vcloud', 'aixer', 'solaris', 'docker', 'google_compute', 'openstack', 'noop' ].each do |lib|
+[ 'vsphere_helper', 'vagrant', 'vagrant_custom', 'vagrant_virtualbox', 'vagrant_parallels', 'vagrant_libvirt', 'vagrant_fusion', 'vagrant_workstation', 'fusion', 'aws_sdk', 'vsphere', 'vmpooler', 'vcloud', 'docker', 'google_compute', 'openstack', 'noop' ].each do |lib|
     require "beaker/hypervisor/#{lib}"
 end

data/lib/beaker/hypervisor/openstack.rb

@@ -171,23 +171,33 @@ module Beaker
       end
     end
 
+    # Get a floating IP address to associate with the instance, and
+    # allocate a new one if none are available
     def get_ip
+      @logger.debug "Finding IP"
       ip = @compute_client.addresses.find { |ip| ip.instance_id.nil? }
       if ip.nil?
-        @logger.debug "Creating IP"
-        ip = @compute_client.addresses.create
+        begin
+          @logger.debug "Creating IP"
+          ip = @compute_client.addresses.create
+        rescue Fog::Compute::OpenStack::NotFound
+          # If there are no more floating IP addresses, allocate a
+          # new one and try again. 
+          @compute_client.allocate_address(@options[:floating_ip_pool])
+          ip = @compute_client.addresses.find { |ip| ip.instance_id.nil? }
+        end
       end
+      raise 'Could not find or allocate an address' if not ip
       ip
     end
 
-
     #Create new instances in OpenStack
     def provision
       @logger.notify "Provisioning OpenStack"
 
       @hosts.each do |host|
         ip = get_ip
-        hostname = ip.ip.gsub! '.','-'
+        hostname = ip.ip.gsub '.','-'
         host[:vmhostname] = hostname+'.rfc1918.puppetlabs.net'
         @logger.debug "Provisioning #{host.name} (#{host[:vmhostname]})"
         options = {
@@ -223,36 +233,8 @@ module Beaker
         end
 
         # Associate a public IP to the server
-        # Create if there are no floating ips available
-        #
-        # Do we already have an address?
-        @logger.debug vm.addresses
-        address=nil
-        begin
-          # Here we try and assign an address from a floating IP pool
-          # This seems to fail on some implementations (FloatingIpPoolNotFound)
-          ip.ip.gsub! '-','.'
-          ip.server = vm
-          address = ip.ip
-
-        rescue Fog::Compute::OpenStack::NotFound
-          # Here, we fail to just trying to use an address that's already assigned if there is one
-          # There may be better logic, but this worked in the original implementation
-          # There might be an argument for checking whether an address is reachable a la
-          # port_open? logic in host.rb but maybe race conditions
-
-          begin
-            if vm.addresses[@options[:openstack_network]]
-              address = vm.addresses[@options[:openstack_network]].map{ |network| network['addr'] }.first
-            end
-          rescue NoMethodError
-            @logger.debug "No current address retrievable from OpenStack data"
-          end
-
-        end
-
-        raise 'Could not find or assign an address to the instance' unless address
-        host[:ip] = address
+        ip.server = vm
+        host[:ip] = ip.ip
 
         @logger.debug "OpenStack host #{host.name} (#{host[:vmhostname]}) assigned ip: #{host[:ip]}"
 
@@ -262,6 +244,9 @@ module Beaker
                             :project           => @options[:project].to_s })
         @vms << vm
 
+        # Wait for the host to accept ssh logins
+        host.wait_for_port(22)
+
         #enable root if user is not root
         enable_root(host)
 

data/lib/beaker/options/presets.rb

@@ -170,10 +170,6 @@ module Beaker
           :package_proxy          => false,
           :add_el_extras          => false,
           :epel_url               => "http://dl.fedoraproject.org/pub/epel",
-          :epel_arch              => "i386",
-          :epel_7_pkg             => "epel-release-7-8.noarch.rpm",
-          :epel_6_pkg             => "epel-release-6-8.noarch.rpm",
-          :epel_5_pkg             => "epel-release-5-4.noarch.rpm",
           :consoleport            => 443,
           :pe_dir                 => '/opt/enterprise/dists',
           :pe_version_file        => 'LATEST',
@@ -190,7 +186,7 @@ module Beaker
           :puppetdb_port_nonssl   => 8080,
           :puppetserver_port      => 8140,
           :nodeclassifier_port    => 4433,
-          :cache_files_locally    => true, # TODO change to false in next major version
+          :cache_files_locally    => false,
           :aws_keyname_modifier   => rand(10 ** 10).to_s.rjust(10,'0'), # 10 digit random number string
           :run_in_parallel        => [],
           :ssh                    => {

data/lib/beaker/version.rb

@@ -1,5 +1,5 @@
 module Beaker
   module Version
-    STRING = '2.52.0'
+    STRING = '3.0.0'
   end
 end

data/spec/beaker/dsl/helpers/puppet_helpers_spec.rb

@@ -775,28 +775,42 @@ describe ClassMixedWithDSLHelpers do
           host.uses_passenger!
         end
         it 'bounces puppet twice' do
+          allow( subject ).to receive(:curl_with_retries)
+          subject.with_puppet_running_on(host, {})
+          expect(host).to execute_commands_matching(/apachectl graceful/).exactly(2).times
+        end
+
+        it 'gracefully restarts using apache2ctl' do
+          allow(host).to receive( :check_for_command ).and_return( true )
           allow( subject ).to receive(:curl_with_retries)
           subject.with_puppet_running_on(host, {})
           expect(host).to execute_commands_matching(/apache2ctl graceful/).exactly(2).times
         end
 
+        it 'gracefully restarts using apachectl' do
+          allow(host).to receive( :check_for_command ).and_return( false )
+          allow( subject ).to receive(:curl_with_retries)
+          subject.with_puppet_running_on(host, {})
+          expect(host).to execute_commands_matching(/apachectl graceful/).exactly(2).times
+        end
+
         it 'yields to a block after bouncing service' do
           execution = 0
           allow( subject ).to receive(:curl_with_retries)
           expect do
             subject.with_puppet_running_on(host, {}) do
-              expect(host).to execute_commands_matching(/apache2ctl graceful/).once
+              expect(host).to execute_commands_matching(/apachectl graceful/).once
               execution += 1
             end
           end.to change { execution }.by(1)
-          expect(host).to execute_commands_matching(/apache2ctl graceful/).exactly(2).times
+          expect(host).to execute_commands_matching(/apachectl graceful/).exactly(2).times
         end
 
         context ':restart_when_done flag set false' do
           it 'bounces puppet once' do
             allow( subject ).to receive(:curl_with_retries)
             subject.with_puppet_running_on(host, { :restart_when_done => false })
-            expect(host).to execute_commands_matching(/apache2ctl graceful/).once
+            expect(host).to execute_commands_matching(/apachectl graceful/).once
           end
 
           it 'yields to a block after bouncing service' do
@@ -804,7 +818,7 @@ describe ClassMixedWithDSLHelpers do
             allow( subject ).to receive(:curl_with_retries)
             expect do
               subject.with_puppet_running_on(host, { :restart_when_done => false }) do
-                expect(host).to execute_commands_matching(/apache2ctl graceful/).once
+                expect(host).to execute_commands_matching(/apachectl graceful/).once
                 execution += 1
               end
             end.to change { execution }.by(1)

data/spec/beaker/dsl/helpers/web_helpers_spec.rb

@@ -36,14 +36,13 @@ describe ClassMixedWithDSLHelpers do
       it "returns its second and third arguments concatenated." do
         concat_path = "#{destdir}/#{name}"
         create_files([concat_path])
-        expect( logger ).to receive( :notify ).with( /^Already\ fetched\ / )
+        allow(logger).to receive(:notify)
+        allow(subject).to receive(:open)
         result = subject.fetch_http_file url, name, destdir
         expect(result).to eq(concat_path)
       end
 
-      it 'doesn\'t cache if :cache_files_locally set false' do
-        options[:cache_files_locally] = false
-
+      it 'doesn\'t cache by default' do
         expect( logger ).to receive( :notify ).with( /^Fetching/ ).ordered
         expect( logger ).to receive( :notify ).with( /^\ \ and\ saving\ to\ / ).ordered
         expect( subject ).to receive( :open )
@@ -51,6 +50,29 @@ describe ClassMixedWithDSLHelpers do
         subject.fetch_http_file( url, name, destdir )
       end
 
+      context ':cache_files_locally option is set' do
+        it 'caches if the file exists locally' do
+          options[:cache_files_locally] = true
+          allow(File).to receive(:exists?).and_return(true)
+
+          expect( logger ).to receive( :notify ).with( /^Already\ fetched\ / )
+          expect( subject ).not_to receive( :open )
+
+          subject.fetch_http_file( url, name, destdir )
+        end
+
+        it 'doesn\'t cache if the file doesn\'t exist locally' do
+          options[:cache_files_locally] = true
+          allow(File).to receive(:exists?).and_return(false)
+
+          expect( logger ).to receive( :notify ).with( /^Fetching/ ).ordered
+          expect( logger ).to receive( :notify ).with( /^\ \ and\ saving\ to\ / ).ordered
+          expect( subject ).to receive( :open )
+
+          subject.fetch_http_file( url, name, destdir )
+        end
+      end
+
     end
 
     describe 'given invalid arguments' do

data/spec/beaker/dsl/install_utils/windows_utils_spec.rb

@@ -36,7 +36,7 @@ describe ClassMixedWithDSLInstallUtils do
 
   def expect_status_called(times = hosts.length)
     expect( Beaker::Command ).to receive( :new )
-      .with( "sc query puppet || sc query pe-puppet", [], {:cmdexe => true} )
+      .with( "sc qc puppet || sc qc pe-puppet", [], {:cmdexe => true} )
       .exactly( times ).times
   end
 
@@ -61,6 +61,16 @@ describe ClassMixedWithDSLInstallUtils do
     end
   end
 
+  def expect_reg_query_called(times = hosts.length)
+    hosts.each do |host|
+      expect(host).to receive(:is_x86_64?).and_return(:true)
+    end
+
+    expect( Beaker::Command ).to receive( :new )
+      .with(%r{reg query "HKLM\\SOFTWARE\\Wow6432Node\\Puppet Labs\\PuppetInstaller}, [], {:cmdexe => true})
+      .exactly(times).times
+  end
+
   describe "#install_msi_on" do
     let( :log_file )    { '/fake/log/file.log' }
     before :each do
@@ -71,6 +81,7 @@ describe ClassMixedWithDSLInstallUtils do
     it "will specify a PUPPET_AGENT_STARTUP_MODE of Manual (disabling the service) by default" do
       expect_install_called
       expect_status_called
+      expect_reg_query_called
       expect_version_log_called
       expect( subject ).to receive( :create_install_msi_batch_on ).with(
           anything, anything,
@@ -81,6 +92,7 @@ describe ClassMixedWithDSLInstallUtils do
     it "allows configuration of PUPPET_AGENT_STARTUP_MODE" do
       expect_install_called
       expect_status_called
+      expect_reg_query_called
       expect_version_log_called
       value = 'Automatic'
       expect( subject ).to receive( :create_install_msi_batch_on ).with(
@@ -109,6 +121,7 @@ describe ClassMixedWithDSLInstallUtils do
 
     it "will generate a command to emit a log file with the :debug option set" do
       expect_install_called
+      expect_reg_query_called
       expect_status_called
       expect_version_log_called
 
@@ -118,6 +131,7 @@ describe ClassMixedWithDSLInstallUtils do
 
     it 'will pass msi_path to #create_install_msi_batch_on as-is' do
       expect_install_called
+      expect_reg_query_called
       expect_status_called
       expect_version_log_called
       test_path = 'test/path'
@@ -125,6 +139,38 @@ describe ClassMixedWithDSLInstallUtils do
           anything, test_path, anything)
       subject.install_msi_on(hosts, test_path)
     end
+
+    it 'will search in Wow6432Node for the remembered startup setting on 64-bit hosts' do
+      expect_install_called
+      expect_status_called
+      expect_version_log_called
+
+      hosts.each do |host|
+        expect(host).to receive(:is_x86_64?).and_return(true)
+      end
+
+      expect( Beaker::Command ).to receive( :new )
+        .with('reg query "HKLM\\SOFTWARE\\Wow6432Node\\Puppet Labs\\PuppetInstaller" /v "RememberedPuppetAgentStartupMode" | findstr Foo', [], {:cmdexe => true})
+        .exactly(hosts.length).times
+
+      subject.install_msi_on(hosts, msi_path, {'PUPPET_AGENT_STARTUP_MODE' => "Foo"})
+    end
+
+    it 'will omit Wow6432Node in the registry search for remembered startup setting on 32-bit hosts' do
+      expect_install_called
+      expect_status_called
+      expect_version_log_called
+
+      hosts.each do |host|
+        expect(host).to receive(:is_x86_64?).and_return(false)
+      end
+
+      expect( Beaker::Command ).to receive( :new )
+        .with('reg query "HKLM\\SOFTWARE\\Puppet Labs\\PuppetInstaller" /v "RememberedPuppetAgentStartupMode" | findstr Foo', [], {:cmdexe => true})
+        .exactly(hosts.length).times
+
+      subject.install_msi_on(hosts, msi_path, {'PUPPET_AGENT_STARTUP_MODE' => "Foo"})
+    end
   end
 
   describe '#create_install_msi_batch_on' do