beaker-pe 1.11.0 → 1.12.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    N2IyMTYxOWMzNzU2MzU0ZmRmNTM2NjQ2ODRiNTk1OWYyMzk0N2YzNA==
+    YTI4NjI0OGUyOGY5ZGMyZDljNTBhMjA1NDhhMzYxY2I5M2JjYmVkNQ==
   data.tar.gz: !binary |-
-    ZjBiMWIwYjY3NjI5MTc0NTQ4NjFjN2EzMWU2N2I0ZDYwZTZiMWRkOA==
+    YWIyYThhODVhMmU0OGQ4ZjA2ZmM1NmQ3MzBhYjdiMTViMzhjZjgzMQ==
 SHA512:
   metadata.gz: !binary |-
-    ZDdhZDZlYjI1OTYzMzc3M2I3YjJmMmMzMDAyOTVkM2Q4NTQ3OTY0M2IwOTZk
-    NmNlYTcxOWVkYjVjOGQ0ZjcyMGNiN2NjMjM1NWUxYjYzYjBmNGIwN2I1M2Uw
-    MGE2NWUyNWZmMDM1MmMzOGI4ZGZlODQ0NzlmMmRjZWNiZmJjNWY=
+    MjkzODE5YmVlMzU1YjUzMTVkYzhiMTlhNjgyMTQ4MDkxOGMxMDcxMmU2ZjJj
+    NmI0MWE2MGJmNGFkNDExMDFlMzgzMmRiMDEyODJmNzI0MmZkMjc5Zjc2MmUz
+    OWIxYjIzYzRjMzFkMDJhZTE0MmFmZDFjZGFhMTFmZDU2Y2MxNjI=
   data.tar.gz: !binary |-
-    MTEzMWRjMWM0ZTM1MDdiNTM2NTcwYjlhZDkxMGQwZmYwYWYzNjU3ZWNiOTYw
-    NTg1ZGNjMTM5NzJmNjQzMjRhODNkYWNkMTFkNGRkYTBiZDJhYjJlY2U3YzQ0
-    NDM2NzVmMTliMzNiNDM0YTZjZWQ2YmY4MGY2YTE3N2I0YTVjMDg=
+    YzdjYTQyOTI5ZDBmYmVjMzZiYjRmNmJmZmE1NmRmZTY5ZDY4OTA3ZGMyMzcx
+    YTAyMWM1ZmVjNTIzMzk2MDdmYzhmZTc1OWVmNDg3MWJjMjBjYWY2NDUwZDM1
+    NzZlYWRjMGJhNGJiMmMwNjU2YzJiY2JmN2E5NzVjMjZiNGQ2YzY=

data/HISTORY.md

@@ -1,6 +1,7 @@
 # worker - History
 ## Tags
-* [LATEST - 23 Mar, 2017 (bce6add6)](#LATEST)
+* [LATEST - 23 Mar, 2017 (dd68cfa4)](#LATEST)
+* [1.11.0 - 23 Mar, 2017 (6c3b0067)](#1.11.0)
 * [1.10.0 - 20 Mar, 2017 (22e22ca8)](#1.10.0)
 * [1.9.1 - 22 Feb, 2017 (3b0bd457)](#1.9.1)
 * [1.9.0 - 7 Feb, 2017 (efae323b)](#1.9.0)
@@ -33,7 +34,55 @@
 * [0.1.0 - 29 Feb, 2016 (4fc88d8c)](#0.1.0)
 
 ## Details
-### <a name = "LATEST">LATEST - 23 Mar, 2017 (bce6add6)
+### <a name = "LATEST">LATEST - 23 Mar, 2017 (dd68cfa4)
+
+* (GEM) update beaker-pe version to 1.12.0 (dd68cfa4)
+
+* Merge pull request #57 from nicklewis/improved-monolithic-install (07aa286e)
+
+
+```
+Merge pull request #57 from nicklewis/improved-monolithic-install
+
+(BKR-1058) Implement a simpler monolithic install method
+```
+* (BKR-1058) Implement a simpler monolithic install method (691a101e)
+
+
+```
+(BKR-1058) Implement a simpler monolithic install method
+
+Currently, the `do_install` method is a mega-method responsible for
+installing and upgrading every version, layout, and platform of PE. It
+knows how to handle masterless installs, legacy agent installs, and
+extremely old versions of PE. Because of that legacy, it has some
+inefficiencies and is inflexible.
+
+Since the most common case by far is a simple monolithic or split PE
+install with a set of frictionless agents, we can dramatically simplify
+and therefore speed up the basic case. This commit implements a
+determine_install_type method that will detect whether the task being
+asked of it is one of those simple cases. The `do_install` method now
+branches based on the type of install, calling
+`simple_monolithic_install` for a basic monolithic + agents install, or
+`generic_install` for everything else. Simple split installs are
+currently detected, but not handled specially (they still fall back to
+the generic install).
+
+Doing away with some of the legacy concerns allows this method to be
+much simpler and more efficient. In particular, because the method has a
+defined task (mono master + agents, rather than a generic list of
+hosts), it can be optimized around that task. This is accomplished now
+by first installing the master, and then installing all the agents in
+parallel. This method also removes an extra agent run that hasn't been
+necessary since PE 3.3.
+
+For a monolithic master with four frictionless agents, this new method
+takes ~6 min 30 sec, compared to ~11 minutes before.
+```
+### <a name = "1.11.0">1.11.0 - 23 Mar, 2017 (6c3b0067)
+
+* (HISTORY) update beaker-pe history for gem release 1.11.0 (6c3b0067)
 
 * (GEM) update beaker-pe version to 1.11.0 (bce6add6)
 

data/lib/beaker-pe/install/pe_utils.rb

@@ -378,6 +378,122 @@ module Beaker
         # @api private
         #
         def do_install hosts, opts = {}
+          # detect the kind of install we're doing
+          install_type = determine_install_type(hosts, opts)
+          case install_type
+          when :simple_monolithic
+            simple_monolithic_install(hosts.first, hosts.drop(1), opts)
+          when :simple_split
+            # This isn't implemented yet, so just do a generic install instead
+            #simple_split_install(hosts, opts)
+            generic_install(hosts, opts)
+          else
+            generic_install(hosts, opts)
+          end
+        end
+
+        def has_all_roles?(host, roles)
+          roles.all? {|role| host['roles'].include?(role)}
+        end
+
+        # Determine what kind of install is being performed
+        # @param [Array<Host>] hosts The sorted hosts to install or upgrade PE on
+        # @param [Hash{Symbol=>Symbol, String}] opts The options for how to install or upgrade PE
+        #
+        # @example
+        #   determine_install_type(hosts, {:type => :install, :pe_ver => '2017.2.0'})
+        #
+        # @return [Symbol]
+        #   One of :generic, :simple_monolithic, :simple_split
+        #   :simple_monolithic
+        #     returned when installing >=2016.4 with a monolithic master and
+        #     any number of frictionless agents
+        #   :simple_split
+        #     returned when installing >=2016.4 with a split install and any
+        #     number of frictionless agents
+        #   :generic
+        #     returned for any other install or upgrade
+        #
+        # @api private
+        def determine_install_type(hosts, opts)
+          # Do a generic install if this is masterless, not all the same PE version, an upgrade, or earlier than 2016.4
+          return :generic if opts[:masterless]
+          return :generic if hosts.map {|host| host['pe_ver']}.uniq.length > 1
+          return :generic if opts[:type] == :upgrade
+          return :generic if version_is_less(opts[:pe_ver] || hosts.first['pe_ver'], '2016.4')
+
+          mono_roles = ['master', 'database', 'dashboard']
+          if has_all_roles?(hosts.first, mono_roles) && hosts.drop(1).all? {|host| host['roles'].include?('frictionless')}
+            :simple_monolithic
+          elsif hosts[0]['roles'].include?('master') && hosts[1]['roles'].include?('database') && hosts[2]['roles'].include?('dashboard') && hosts.drop(3).all? {|host| host['roles'].include?('frictionless')}
+            :simple_split
+          else
+            :generic
+          end
+        end
+
+        # Install PE on a monolithic master and some number of frictionless agents.
+        # @param [Host] master The node to install the master on
+        # @param [Array<Host>] agents The nodes to install agents on
+        # @param [Hash{Symbol=>Symbol, String}] opts The options for how to install or upgrade PE
+        #
+        # @example
+        #   simple_monolithic_install(master, agents, {:type => :install, :pe_ver => '2017.2.0'})
+        #
+        # @return nil
+        #
+        # @api private
+        def simple_monolithic_install(master, agents, opts={})
+          step "Performing a standard monolithic install with frictionless agents"
+          all_hosts = [master, *agents]
+          configure_type_defaults_on(all_hosts)
+
+          # Set PE distribution on the master, create working dir
+          prepare_hosts([master], opts)
+          fetch_pe([master], opts)
+          prepare_host_installer_options(master)
+          generate_installer_conf_file_for(master, [master], opts)
+          on master, installer_cmd(master, opts)
+
+          step "Setup frictionless installer on the master" do
+            agents.each do |agent|
+              # If We're *not* running the classic installer, we want
+              # to make sure the master has packages for us.
+              if agent['platform'] != master['platform'] # only need to do this if platform differs
+                deploy_frictionless_to_master(agent)
+              end
+            end
+          end
+
+          step "Install agents" do
+            agents.group_by {|agent| installer_cmd(agent, opts)}.each do |cmd, agents|
+              on agents, cmd, :run_in_parallel => true
+            end
+          end
+
+          step "Stop puppet agents to avoid interfering with tests" do
+            stop_agent_on(all_hosts, :run_in_parallel => true)
+          end
+
+          step "Sign agent certificates" do
+            # This will sign all cert requests
+            sign_certificate_for(agents)
+          end
+
+          step "Run puppet to setup mcollective and pxp-agent" do
+            on all_hosts, puppet_agent('-t'), :acceptable_exit_codes => [0,2], :run_in_parallel => true
+
+            #Workaround for windows frictionless install, see BKR-943 for the reason
+            agents.select {|agent| agent['platform'] =~ /windows/}.each do |agent|
+              client_datadir = agent.puppet['client_datadir']
+              on(agent, puppet("resource file \"#{client_datadir}\" ensure=absent force=true"))
+            end
+          end
+        end
+
+        def generic_install hosts, opts = {}
+          step "Installing PE on a generic set of hosts"
+
           masterless = opts[:masterless]
           opts[:type] = opts[:type] || :install
           unless masterless

data/lib/beaker-pe/version.rb

@@ -3,7 +3,7 @@ module Beaker
     module PE
 
       module Version
-        STRING = '1.11.0'
+        STRING = '1.12.0'
       end
 
     end

data/spec/beaker-pe/install/pe_utils_spec.rb

@@ -878,7 +878,64 @@ describe ClassMixedWithDSLInstallUtils do
     end
   end
 
+  describe "#determine_install_type" do
+    let(:monolithic) { make_host('monolithic', :pe_ver => '2016.4', :roles => [ 'master', 'database', 'dashboard' ]) }
+    let(:master) { make_host('master', :pe_ver => '2016.4', :roles => [ 'master' ]) }
+    let(:puppetdb) { make_host('puppetdb', :pe_ver => '2016.4', :roles => [ 'database' ]) }
+    let(:console) { make_host('console', :pe_ver => '2016.4', :roles => [ 'dashboard' ]) }
+    let(:agent) { make_host('agent', :pe_ver => '2016.4', :roles => ['frictionless']) }
+
+    it 'identifies a monolithic install with frictionless agents' do
+      hosts = [monolithic, agent, agent, agent]
+      expect(subject.determine_install_type(hosts, {})).to eq(:simple_monolithic)
+    end
+
+    it 'identifies a monolithic install without frictionless agents' do
+      expect(subject.determine_install_type([monolithic], {})).to eq(:simple_monolithic)
+    end
+
+    it 'identifies a split install with frictionless agents' do
+      hosts = [master, puppetdb, console, agent, agent, agent]
+      expect(subject.determine_install_type(hosts, {})).to eq(:simple_split)
+    end
+
+    it 'identifies a split install without frictionless agents' do
+      hosts = [master, puppetdb, console]
+      expect(subject.determine_install_type(hosts, {})).to eq(:simple_split)
+    end
+
+    it 'identifies an install with multiple agent versions as generic' do
+      new_agent = make_host('agent', :pe_ver => '2017.2', :roles => ['frictionless'])
+      hosts = [monolithic, agent, new_agent]
+      expect(subject.determine_install_type(hosts, {})).to eq(:generic)
+    end
+
+    it 'identifies an upgrade as generic' do
+      hosts = [monolithic, agent, agent, agent]
+      expect(subject.determine_install_type(hosts, {:type => :upgrade})).to eq(:generic)
+    end
+
+    it 'identifies a legacy PE version as generic' do
+      old_monolithic = make_host('monolithic', :pe_ver => '3.8', :roles => [ 'master', 'database', 'dashboard' ])
+      old_agent = make_host('agent', :pe_ver => '3.8', :roles => ['frictionless'])
+      hosts = [old_monolithic, old_agent, old_agent, old_agent]
+      expect(subject.determine_install_type(hosts, {})).to eq(:generic)
+    end
+
+    it 'identifies a non-standard install as generic' do
+      hosts = [monolithic, master, agent, agent, agent]
+      expect(subject.determine_install_type(hosts, {})).to eq(:generic)
+    end
+  end
+
   describe 'do_install' do
+    it 'chooses to do a simple monolithic install when appropriate' do
+      expect(subject).to receive(:simple_monolithic_install)
+      allow(subject).to receive(:determine_install_type).and_return(:simple_monolithic)
+
+      subject.do_install([])
+    end
+
     it 'can perform a simple installation' do
       allow( subject ).to receive( :on ).and_return( Beaker::Result.new( {}, '' ) )
       allow( subject ).to receive( :fetch_pe ).and_return( true )
@@ -1186,6 +1243,78 @@ describe ClassMixedWithDSLInstallUtils do
 
   end
 
+  describe 'simple_monolithic_install' do
+    let(:monolithic) { make_host('monolithic', :pe_ver => '2016.4', :platform => 'el-7-x86_64', :roles => [ 'master', 'database', 'dashboard' ]) }
+    let(:el_agent) { make_host('agent', :pe_ver => '2016.4', :platform => 'el-7-x86_64', :roles => ['frictionless']) }
+    let(:deb_agent) { make_host('agent', :pe_ver => '2016.4', :platform => 'debian-7-x86_64', :roles => ['frictionless']) }
+
+    before :each do
+      allow(subject).to receive(:on)
+      allow(subject).to receive(:configure_type_defaults_on)
+      allow(subject).to receive(:prepare_hosts)
+      allow(subject).to receive(:fetch_pe)
+      allow(subject).to receive(:prepare_host_installer_options)
+      allow(subject).to receive(:generate_installer_conf_file_for)
+      allow(subject).to receive(:deploy_frictionless_to_master)
+
+      allow(subject).to receive(:installer_cmd).with(monolithic, anything()).and_return("install master")
+      allow(subject).to receive(:installer_cmd).with(el_agent, anything()).and_return("install el agent")
+      allow(subject).to receive(:installer_cmd).with(deb_agent, anything()).and_return("install deb agent")
+
+      allow(subject).to receive(:stop_agent_on)
+      allow(subject).to receive(:sign_certificate_for)
+    end
+
+    describe 'configuring frictionless installer' do
+      it "skips the master's platform" do
+        expect(subject).not_to receive(:deploy_frictionless_to_master)
+
+        subject.simple_monolithic_install(monolithic, [el_agent, el_agent, el_agent])
+      end
+
+      it "adds frictionless install classes for other platforms" do
+        expect(subject).to receive(:deploy_frictionless_to_master).with(deb_agent)
+
+        subject.simple_monolithic_install(monolithic, [el_agent, deb_agent])
+      end
+    end
+
+    it 'installs on the master then on the agents' do
+      expect(subject).to receive(:on).with(monolithic, "install master").ordered
+      expect(subject).to receive(:on).with([el_agent, el_agent], "install el agent", anything()).ordered
+
+      subject.simple_monolithic_install(monolithic, [el_agent, el_agent])
+    end
+
+    it 'installs agents in parallel if their install command is the same' do
+      expect(subject).to receive(:on).with([el_agent, el_agent], "install el agent", :run_in_parallel => true)
+      expect(subject).to receive(:on).with([deb_agent, deb_agent], "install deb agent", :run_in_parallel => true)
+
+      subject.simple_monolithic_install(monolithic, [el_agent, el_agent, deb_agent, deb_agent])
+    end
+
+    it 'signs all certificates at once' do
+      agents = [el_agent, el_agent, deb_agent, deb_agent]
+      expect(subject).to receive(:sign_certificate_for).with(agents)
+
+      subject.simple_monolithic_install(monolithic, agents)
+    end
+
+    it 'stops the agents in parallel to avoid interference with tests' do
+      agents = [el_agent, el_agent, deb_agent, deb_agent]
+      expect(subject).to receive(:stop_agent_on).with([monolithic, *agents], :run_in_parallel => true)
+
+      subject.simple_monolithic_install(monolithic, agents)
+    end
+
+    it 'runs agents in parallel, only one time' do
+      agents = [el_agent, el_agent, deb_agent, deb_agent]
+      expect(subject).to receive(:on).with([monolithic, *agents], proc {|cmd| cmd.command == "puppet agent"}, hash_including(:run_in_parallel => true)).once
+
+      subject.simple_monolithic_install(monolithic, agents)
+    end
+  end
+
   describe 'do_higgs_install' do
 
     before :each do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: beaker-pe
 version: !ruby/object:Gem::Version
-  version: 1.11.0
+  version: 1.12.0
 platform: ruby
 authors:
 - Puppetlabs