beaker-pe 1.26.0 → 1.27.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NjRjNWVlOTY5MzVhNjQyZWJmNDE0OTU0OTQ5Mjc3NzIzOGNjNThjZA==
+    ZGIxMzQ2NjExMDg5MGJlODU2YWYxNmFiYTZjYmE0MDcxNTVkZGRjYw==
   data.tar.gz: !binary |-
-    MTAzNDEyYTMzNzFmYmMzOTdmZDRiNjRmYWQ3ODhjYzdhOWY1YmIwOQ==
+    OGQzMmQ1N2FiNjNjNjMwYjIwYTdkZDIzMzgwYjkwMDNjOGU5ZmJlOQ==
 SHA512:
   metadata.gz: !binary |-
-    ZDQwMDdiZWI0NjliMzA1ZDQ5ZDJmNDcxMGFlNDUzNDA3MTI4ZThkMzRkM2E4
-    NzQyYjE3NjlmMmY1OGY0ZWVmMTY4M2Q5NDEyN2MxM2JmY2FjN2Y3NzNiZTVh
-    ZGUzOTZhNWI1MjdhYTk2MTEwNmJlZWY1Njg1YmRlZTI0ZmQ2NjE=
+    ZDc4N2Y3OTk3MTc3ZDMyMThiMTdjOGU5ZmY3MGYxYTdlMGYwM2EyNWE4NzRh
+    NzMyNTY5NzM4OTBjODgyMjc2ZjExMTFhZDQyZDc3ZDU3ZGJkZTc3ZWM5MTBh
+    ZWY1NTQ3M2NiNzAxYWFhMTFmZjk1NjQ1ZGVjOGNhMzAxYWFlZmQ=
   data.tar.gz: !binary |-
-    OTBjNmY0MWI1NGIwNzU0MDVhNzhjNmI5MmJlNGFhNDNiOTkzNTI5MWFjYTBh
-    OWY0ZjcxZGY2NWM2NGQxZmIyMzU0NzBlY2FlNWQxOWYxYzE1ZjQyYWU5ZWU3
-    ZTAwMDAxOTU4ZGQ4YmQ5ZDJlZTMyYjZkZmMyMWU4ZTJmZWNjNzU=
+    Y2MwOWVhNjFiNzBhNzEwN2U5OGZlYzgwOWY1MGE0NTUxM2YyYWRlYmNjZGY5
+    YjE2MjM1NDdiOWU3NDgyNTk3YTZiYzZlZjlhN2RkMTIwZDdlYjVlOTU5MjA2
+    YjdiOTRjOGI2OTBiNDMyOTljNGIzNThlZjJmY2QyYTRiYzUwZWQ=

data/lib/beaker-pe/install/pe_utils.rb

@@ -101,6 +101,30 @@ module Beaker
           end
         end
 
+        # Return agent nodes with 'lb_connect' role that are not loadbalancers
+        def loadbalancer_connecting_agents
+          lb_connect_nodes = select_hosts(roles: ['lb_connect'])
+          lb_connect_agents = lb_connect_nodes.reject { |h| h['roles'].include?('loadbalancer')}
+        end
+
+        # Returns true if loadbalncer exists and is configured with 'lb_connect' role
+        def lb_connect_loadbalancer_exists?
+          if any_hosts_as?('loadbalancer')
+            lb_node = select_hosts(roles: ['loadbalancer'])
+            lb_node.first['roles'].include?('lb_connect')
+          end
+        end
+
+        #Returns loadbalancer if host is an agent and loadbalancer has lb_connect role
+        #@param [Host] agent host with lb_connect role
+        def get_lb_downloadhost(host)
+          downloadhost = master
+          if !host['roles'].include?('loadbalancer') &&  lb_connect_loadbalancer_exists?
+            downloadhost = loadbalancer
+          end
+          downloadhost
+        end
+
         # Generate the command line string needed to from a frictionless puppet-agent
         # install on this host in a PE environment.
         #
@@ -124,6 +148,14 @@ module Beaker
             end
           end
 
+          # If this is an agent node configured to connect to the loadbalancer
+          # using 'lb_connect' role, then use loadbalancer in the download url
+          # instead of master
+          downloadhost = master
+          if host['roles'].include?('lb_connect')
+            downloadhost = get_lb_downloadhost(host)
+          end
+
           pe_debug = host[:pe_debug] || opts[:pe_debug] ? ' -x' : ''
           use_puppet_ca_cert = host[:use_puppet_ca_cert] || opts[:use_puppet_ca_cert]
 
@@ -135,7 +167,7 @@ module Beaker
               cert_validator = '[Net.ServicePointManager]::ServerCertificateValidationCallback = {\\$true}'
             end
 
-            cmd = %Q{powershell -c "cd #{host['working_dir']};#{cert_validator};\\$webClient = New-Object System.Net.WebClient;\\$webClient.DownloadFile('https://#{master}:8140/packages/current/install.ps1', '#{host['working_dir']}/install.ps1');#{host['working_dir']}/install.ps1 -verbose #{frictionless_install_opts.join(' ')}"}
+            cmd = %Q{powershell -c "cd #{host['working_dir']};#{cert_validator};\\$webClient = New-Object System.Net.WebClient;\\$webClient.DownloadFile('https://#{downloadhost}:8140/packages/current/install.ps1', '#{host['working_dir']}/install.ps1');#{host['working_dir']}/install.ps1 -verbose #{frictionless_install_opts.join(' ')}"}
           else
             curl_opts = %w{--tlsv1 -O}
             if use_puppet_ca_cert
@@ -144,7 +176,7 @@ module Beaker
               curl_opts << '-k'
             end
 
-            cmd = "FRICTIONLESS_TRACE='true'; export FRICTIONLESS_TRACE; cd #{host['working_dir']} && curl #{curl_opts.join(' ')} https://#{master}:8140/packages/current/install.bash && bash#{pe_debug} install.bash #{frictionless_install_opts.join(' ')}".strip
+            cmd = "FRICTIONLESS_TRACE='true'; export FRICTIONLESS_TRACE; cd #{host['working_dir']} && curl #{curl_opts.join(' ')} https://#{downloadhost}:8140/packages/current/install.bash && bash#{pe_debug} install.bash #{frictionless_install_opts.join(' ')}".strip
           end
 
           return cmd
@@ -376,33 +408,37 @@ module Beaker
           else
             _console_dispatcher = get_console_dispatcher_for_beaker_pe!
 
-            # Check if we've already created a frictionless agent node group
-            # to avoid errors creating the same node group when the beaker hosts file contains
-            # multiple hosts with the same platform
-            node_group = _console_dispatcher.get_node_group_by_name('Beaker Frictionless Agent')
-            if node_group.nil? || node_group.empty?
-              node_group = {}
-              node_group['name'] = "Beaker Frictionless Agent"
-              # Pin the master to the node
-              node_group['rule'] = [ "and",  [ '=', 'name', master.to_s ]]
-              node_group['classes'] ||= {}
-            end
+            # Add pe_repo packages to 'PE Master' group
+            node_group = _console_dispatcher.get_node_group_by_name('PE Master')
 
             # add the pe_repo platform class if it's not already present
-            if ! node_group['classes'].include?(klass)
-              node_group['classes'][klass] = {}
-
-              _console_dispatcher.create_new_node_group_model(node_group)
-              # The puppet agent run that will download the agent tarballs to the master can sometimes fail with
-              # curl errors if there is a network hiccup. Use beakers `retry_on` method to retry up to
-              # three times to avoid failing the entire test pipeline due to a network blip
-              retry_opts = {
-                :desired_exit_codes => [0,2],
-                :max_retries => 3,
-                # Beakers retry_on method wants the verbose value to be a string, not a bool.
-                :verbose => 'true'
-              }
-              retry_on(master, puppet("agent -t"), retry_opts)
+            if node_group
+              if !node_group['classes'].include?(klass)
+                node_group['classes'][klass] = {}
+                _console_dispatcher.create_new_node_group_model(node_group)
+
+                # The puppet agent run that will download the agent tarballs to the master can sometimes fail with
+                # curl errors if there is a network hiccup. Use beakers `retry_on` method to retry up to
+                # three times to avoid failing the entire test pipeline due to a network blip
+                retry_opts = {
+                  :desired_exit_codes => [0,2],
+                  :max_retries => 3,
+                  # Beakers retry_on method wants the verbose value to be a string, not a bool.
+                  :verbose => 'true'
+                }
+                retry_on(master, puppet("agent -t"), retry_opts)
+
+                # If we are connecting through loadbalancer, download the agent tarballs to compile masters
+                if lb_connect_loadbalancer_exists?
+                  hosts.each do |h|
+                    if h['roles'].include?('compile_master')
+                      retry_on(h, puppet("agent -t"), retry_opts)
+                    end
+                  end
+                end
+              end
+            else
+              raise "Failed to add pe_repo packages, PE Master node group is not available"
             end
           end
         end
@@ -507,7 +543,7 @@ module Beaker
         def simple_monolithic_install(master, agents, opts={})
           step "Performing a standard monolithic install with frictionless agents"
           all_hosts = [master, *agents]
-          configure_type_defaults_on(all_hosts)
+          configure_type_defaults_on([master])
 
           # Set PE distribution on the agents, creates working directories
           prepare_hosts(all_hosts, opts)
@@ -518,41 +554,11 @@ module Beaker
             on master, installer_cmd(master, opts)
           end
 
-          step "Setup frictionless installer on the master" do
-            agents.each do |agent|
-              # If We're *not* running the classic installer, we want
-              # to make sure the master has packages for us.
-              if agent['platform'] != master['platform'] # only need to do this if platform differs
-                deploy_frictionless_to_master(agent)
-              end
-            end
-          end
-
-          step "Install agents" do
-            block_on(agents, {:run_in_parallel => true}) do |host|
-              install_ca_cert_on(host, opts)
-              on(host, installer_cmd(host, opts))
-            end
-          end
-
-          step "Sign agent certificates" do
-            # This will sign all cert requests
-            sign_certificate_for(agents)
-          end
-
-          step "Stop puppet agents to avoid interfering with tests" do
-            stop_agent_on(all_hosts, :run_in_parallel => true)
-          end
+          install_agents_only_on(agents, opts)
 
           step "Run puppet to setup mcollective and pxp-agent" do
             on(master, puppet_agent('-t'), :acceptable_exit_codes => [0,2])
             run_puppet_on_non_infrastructure_nodes(all_hosts)
-
-            #Workaround for windows frictionless install, see BKR-943 for the reason
-            agents.select {|agent| agent['platform'] =~ /windows/}.each do |agent|
-              client_datadir = agent.puppet['client_datadir']
-              on(agent, puppet("resource file \"#{client_datadir}\" ensure=absent force=true"))
-            end
           end
 
           step "Run puppet a second time on the primary to populate services.conf (PE-19054)" do
@@ -1549,6 +1555,52 @@ module Beaker
             end
           end
         end
+
+        # Method to install just the agent nodes
+        # This method can be called only after installing PE on infrastructure nodes
+        # @param [Array] agent only nodes from Beaker hosts
+        # @param [Hash] opts The Beaker options hash
+        def install_agents_only_on(agent_nodes, opts)
+          unless agent_nodes.empty?
+            configure_type_defaults_on(agent_nodes)
+
+             step "Setup frictionless installer on the master" do
+               agent_nodes.each do |agent|
+                 # If We're *not* running the classic installer, we want
+                 # to make sure the master has packages for us.
+                 if agent['platform'] != master['platform'] # only need to do this if platform differs
+                   deploy_frictionless_to_master(agent)
+                 end
+               end
+             end
+
+             step "Install agents" do
+               block_on(agent_nodes, {:run_in_parallel => true}) do |host|
+                 install_ca_cert_on(host, opts)
+                 on(host, installer_cmd(host, opts))
+               end
+             end
+
+             step "Sign agent certificates" do
+               # This will sign all cert requests
+               sign_certificate_for(agent_nodes)
+             end
+
+             step "Stop puppet agents to avoid interfering with tests" do
+               stop_agent_on(agent_nodes, :run_in_parallel => true)
+             end
+
+             step "Run puppet on all agent nodes" do
+               on agent_nodes, puppet_agent('-t'), :acceptable_exit_codes => [0,2], :run_in_parallel => true
+             end
+
+             #Workaround for windows frictionless install, see BKR-943
+             agent_nodes.select {|agent| agent['platform'] =~ /windows/}.each do |agent|
+               client_datadir = agent.puppet['client_datadir']
+               on(agent, puppet("resource file \"#{client_datadir}\" ensure=absent force=true"))
+             end
+          end
+        end
       end
     end
   end

data/lib/beaker-pe/version.rb

@@ -3,7 +3,7 @@ module Beaker
     module PE
 
       module Version
-        STRING = '1.26.0'
+        STRING = '1.27.0'
       end
 
     end

data/spec/beaker-pe/install/pe_utils_spec.rb

@@ -60,6 +60,17 @@ describe ClassMixedWithDSLInstallUtils do
                                                 :working_dir => '/tmp',
                                                 :dist => 'puppet-enterprise-3.7.1-rc0-78-gffc958f-eos-4-i386' } ) }
 
+  let(:lei_hosts)     { make_hosts( { :pe_ver => '3.0',
+                                      :platform => 'linux',
+                                      :roles => [ 'agent' ],
+                                      :type => 'pe'}, 4 ) }
+  let(:lb_test_hosts) { lei_hosts[0][:roles] = ['master', 'database', 'dashboard']
+                        lei_hosts[1][:roles] = ['loadbalancer', 'lb_connect']
+                        lei_hosts[2][:roles] = ['compile_master']
+                        lei_hosts[3][:roles] = ['frictionless', 'lb_connect']
+                        lei_hosts[3][:working_dir] = '/tmp'
+                        lei_hosts }
+
   context '#prep_host_for_upgrade' do
 
     it 'sets per host options before global options' do
@@ -188,6 +199,26 @@ describe ClassMixedWithDSLInstallUtils do
     end
   end
 
+  describe 'loadbalancer_connecting_agents' do
+    it 'no hosts are chosen if there are no agents with lb_connect role' do
+      allow( subject ).to receive(:hosts).and_return([])
+    end
+    it 'chooses agents with lb_connect role' do
+      allow( subject ).to receive(:lb_test_hosts).and_return([lb_test_hosts[3]])
+    end
+
+  end
+
+  describe 'get_lb_downloadhost' do
+    it 'choose lb_connect loadbalancer as downloadhost, if there is one' do
+      allow( subject ).to receive(:lb_test_hosts[3]).and_return([lb_test_hosts[1]])
+    end
+    it 'if there is no lb_connect loadbalancer, return master' do
+      lei_hosts[1][:roles] = ['loadbalancer']
+      allow( subject ).to receive(:lb_test_hosts[3]).and_return([lb_test_hosts[0]])
+    end
+  end
+
   describe 'frictionless_agent_installer_cmd' do
     let(:host) do
       the_host = unixhost.dup
@@ -265,6 +296,18 @@ describe ClassMixedWithDSLInstallUtils do
       "\""
       expect( subject.frictionless_agent_installer_cmd( host, {}, '2016.4.0' ) ).to eq(expecting)
     end
+
+    it 'generates a frictionless install command with loadbalancer as download host' do
+      hosts = lb_test_hosts
+      expect( subject ).to receive( :get_lb_downloadhost ).with(lb_test_hosts[3]).and_return( 'testloadbalancer' )
+      expecting = [
+        "FRICTIONLESS_TRACE='true'",
+        "export FRICTIONLESS_TRACE",
+        "cd /tmp && curl --tlsv1 -O -k https://testloadbalancer:8140/packages/current/install.bash && bash install.bash"
+      ].join("; ")
+
+      expect( subject.frictionless_agent_installer_cmd( lb_test_hosts[3], {}, '2016.4.0' ) ).to eq(expecting)
+    end
   end
 
   describe 'install_ca_cert_on' do
@@ -1079,8 +1122,9 @@ describe ClassMixedWithDSLInstallUtils do
   describe '#deploy_frictionless_to_master' do
     let(:master) { make_host('master', :pe_ver => '2017.2', :platform => 'ubuntu-16.04-x86_64', :roles => ['master', 'database', 'dashboard']) }
     let(:agent) { make_host('agent', :pe_ver => '2017.2', :platform => 'el-7-x86_64', :roles => ['frictionless']) }
+    let(:compile_master) { make_host('agent', :pe_ver => '2017.2', :roles => ['frictionless', 'compile_master']) }
     let(:dispatcher) { double('dispatcher') }
-    let(:node_group) { {} }
+    let(:node_group) { { 'classes' => {} } }
 
     before :each do
       allow(subject).to receive(:retry_on)
@@ -1090,12 +1134,12 @@ describe ClassMixedWithDSLInstallUtils do
 
       allow(dispatcher).to receive(:get_node_group_by_name).and_return(node_group)
       allow(dispatcher).to receive(:create_new_node_group_model) {|model| node_group.update(model)}
+      allow(subject).to receive(:compile_masters).and_return([compile_master])
     end
 
-    it 'adds the right pe_repo class to the Beaker Frictionless Agent group' do
+    it 'adds the right pe_repo class to the PE Master group' do
       subject.deploy_frictionless_to_master(agent)
 
-      expect(node_group['rule']).to eq(['and', ['=', 'name', 'master']])
       expect(node_group['classes']).to include('pe_repo::platform::el_7_x86_64')
     end
 
@@ -1449,6 +1493,7 @@ describe ClassMixedWithDSLInstallUtils do
       allow(subject).to receive(:prepare_host_installer_options)
       allow(subject).to receive(:generate_installer_conf_file_for)
       allow(subject).to receive(:deploy_frictionless_to_master)
+      allow(subject).to receive(:install_agents_only_on)
 
       allow(subject).to receive(:installer_cmd).with(monolithic, anything()).and_return("install master")
       allow(subject).to receive(:installer_cmd).with(el_agent, anything()).and_return("install el agent")
@@ -1458,44 +1503,66 @@ describe ClassMixedWithDSLInstallUtils do
       allow(subject).to receive(:sign_certificate_for)
     end
 
-    describe 'configuring frictionless installer' do
-      it "skips the master's platform" do
-        expect(subject).not_to receive(:deploy_frictionless_to_master)
-
-        subject.simple_monolithic_install(monolithic, [el_agent, el_agent, el_agent])
-      end
-
-      it "adds frictionless install classes for other platforms" do
-        expect(subject).to receive(:deploy_frictionless_to_master).with(deb_agent)
-
-        subject.simple_monolithic_install(monolithic, [el_agent, deb_agent])
-      end
-    end
-
     it 'installs on the master then on the agents' do
       expect(subject).to receive(:on).with(monolithic, "install master").ordered
-      expect(subject).to receive(:block_on).with([el_agent, el_agent], :run_in_parallel => true).ordered
-
+      expect(subject).to receive(:install_agents_only_on).with([el_agent, el_agent], {}).ordered
       subject.simple_monolithic_install(monolithic, [el_agent, el_agent])
     end
 
-    it 'installs agents in parallel' do
-      expect(subject).to receive(:block_on).with([el_agent, el_agent, deb_agent, deb_agent], :run_in_parallel => true)
+    it "calls prepare_hosts on all hosts instead of just master" do
+      expect(subject).to receive(:prepare_hosts).with([monolithic] + [el_agent, el_agent, el_agent], {})
+      subject.simple_monolithic_install(monolithic, [el_agent, el_agent, el_agent])
+    end
+  end
 
-      subject.simple_monolithic_install(monolithic, [el_agent, el_agent, deb_agent, deb_agent])
+  describe 'install_agents_only_on' do
+    let(:monolithic) { make_host('monolithic',
+                                 :pe_ver => '2016.4',
+                                 :platform => 'el-7-x86_64',
+                                 :roles => ['master', 'database', 'dashboard']) }
+    let(:agent) { make_host('agent',
+                            :pe_ver => '2016.4',
+                            :platform => 'el-7-x86_64',
+                            :roles => ['frictionless']) }
+    before :each do
+      allow(subject).to receive(:on)
+      allow(subject).to receive(:hosts).and_return([monolithic, agent, agent])
+      allow(subject).to receive(:configure_type_defaults_on)
+      allow(subject).to receive(:deploy_frictionless_to_master)
+      allow(subject).to receive(:stop_agent_on)
+      allow(subject).to receive(:sign_certificate_for)
+      allow(subject).to receive(:installer_cmd).with(agent, anything()).and_return("install agent")
     end
 
-    it 'signs certificates then stops agents to avoid interference with tests' do
-      agents = [el_agent, el_agent, deb_agent, deb_agent]
-      expect(subject).to receive(:sign_certificate_for).with(agents).ordered
-      expect(subject).to receive(:stop_agent_on).with([monolithic, *agents], :run_in_parallel => true).ordered
+    it 'does not call deploy_frictionless_to_master if agent platform is same as master' do
+      expect(subject).not_to receive(:deploy_frictionless_to_master)
+      subject.install_agents_only_on([agent], opts)
+    end
 
-      subject.simple_monolithic_install(monolithic, agents)
+    it 'calls deploy_frictionless_to_master if agent platform is different from master' do
+      agent['platform'] = 'deb-7-x86_64'
+      expect(subject).to receive(:deploy_frictionless_to_master)
+      subject.install_agents_only_on([agent], opts)
     end
 
-    it "calls prepare_hosts on all hosts instead of just master" do
-      expect(subject).to receive(:prepare_hosts).with([monolithic] + [el_agent, el_agent, el_agent], {})
-      subject.simple_monolithic_install(monolithic, [el_agent, el_agent, el_agent])
+    it 'installs agent on agent hosts' do
+      agents = [agent, agent]
+      expect(subject).to receive(:block_on).with(agents, :run_in_parallel => true)
+      subject.install_agents_only_on(agents, opts)
+    end
+
+    it 'signs certificate and stops agent on agent host' do
+      agents = [agent, agent]
+      expect(subject).to receive(:sign_certificate_for).with(agents)
+      expect(subject).to receive(:stop_agent_on).with(agents, :run_in_parallel => true)
+      subject.install_agents_only_on(agents, opts)
+    end
+
+    it 'runs puppet on agent hosts' do
+      agents = [agent, agent]
+      expect(subject).to receive(:on).with(agents, proc {
+        |cmd| cmd.command == "puppet agent"}, hash_including(:run_in_parallel => true)).once
+      subject.install_agents_only_on(agents, opts)
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: beaker-pe
 version: !ruby/object:Gem::Version
-  version: 1.26.0
+  version: 1.27.0
 platform: ruby
 authors:
 - Puppetlabs
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-23 00:00:00.000000000 Z
+date: 2017-12-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec