hybrid_platforms_conductor 32.3.6

data/spec/hybrid_platforms_conductor_test/helpers/deployer_helpers.rb

@@ -0,0 +1,149 @@
+module HybridPlatformsConductorTest
+
+  module Helpers
+
+    module DeployerHelpers
+
+      # Expect a given action to be setting up the mutex on a given node
+      #
+      # Parameters::
+      # * *action* (Hash<Symbol,Object>): The action to check
+      # * *node* (String): The concerned node
+      # * *sudo* (Boolean): Is sudo supposed to be used? [default: true]
+      def expect_action_to_lock_node(action, node, sudo: true)
+        expect(action[:scp].size).to eq 1
+        expect(action[:scp].first[0]).to match /^.+\/mutex_dir$/
+        expect(action[:scp].first[1]).to eq '.'
+        expect(action[:remote_bash]).to eq "while ! #{sudo ? 'sudo ' : ''}./mutex_dir lock /tmp/hybrid_platforms_conductor_deploy_lock \"$(ps -o ppid= -p $$)\"; do echo -e 'Another deployment is running on #{node}. Waiting for it to finish to continue...' ; sleep 5 ; done"
+      end
+
+      # Expect a given action to be releasing the mutex on a given node
+      #
+      # Parameters::
+      # * *action* (Hash<Symbol,Object>): The action to check
+      # * *node* (String): The concerned node
+      # * *sudo* (Boolean): Is sudo supposed to be used? [default: true]
+      def expect_action_to_unlock_node(action, node, sudo: true)
+        expect(action).to eq(remote_bash: "#{sudo ? 'sudo ' : ''}./mutex_dir unlock /tmp/hybrid_platforms_conductor_deploy_lock")
+      end
+
+      # Expect a given set of actions to be a deployment
+      #
+      # Parameters::
+      # * *actions* (Object): Actions
+      # * *nodes* (String or Array<String>): Node (or list of nodes) that should be checked
+      # * *check* (Boolean): Is the deploy only a check? [default: false]
+      # * *sudo* (Boolean): Is sudo supposed to be used? [default: true]
+      # * *expected_actions* (Array<Object>): Additional expected actions [default: []]
+      # * *mocked_result* (Hash<String, [Object, String, String]>): Expected result of the actions, per node, or nil for success [default: nil]
+      # Result::
+      # * Hash<String, [Integer or Symbol, String, String] >: Expected result of those expected actions
+      def expect_actions_to_deploy_on(actions, nodes, check: false, sudo: true, expected_actions: [], mocked_result: nil)
+        nodes = [nodes] if nodes.is_a?(String)
+        mocked_result = Hash[nodes.map { |node| [node, [0, "#{check ? 'Check' : 'Deploy'} successful", '']] }] if mocked_result.nil?
+        expect(actions.size).to eq nodes.size
+        nodes.each do |node|
+          expect(actions.key?(node)).to eq true
+          expect(actions[node].size).to eq(2 + expected_actions.size)
+          expect_action_to_lock_node(actions[node][0], node, sudo: sudo)
+          expect(actions[node][1..-2]).to eq expected_actions
+          expect(actions[node][-1]).to eq(bash: "echo \"#{check ? 'Checking' : 'Deploying'} on #{node}\"")
+        end
+        mocked_result
+      end
+
+      # Expect a given set of actions to be unlock deployments to a list of nodes
+      #
+      # Parameters::
+      # * *actions* (Object): Actions
+      # * *nodes* (String or Array<String>): Node (or list of nodes) that should be checked
+      # * *sudo* (Boolean): Is sudo supposed to be used? [default: true]
+      def expect_actions_to_unlock(actions, nodes, sudo: true)
+        nodes = [nodes] if nodes.is_a?(String)
+        expect(actions.size).to eq nodes.size
+        nodes.each do |node|
+          expect(actions.key?(node)).to eq true
+          expect_action_to_unlock_node(actions[node], node, sudo: sudo)
+        end
+        Hash[nodes.map { |node| [node, [0, 'Release mutex successful', '']] }]
+      end
+
+      # Expect a given set of actions to upload log files on a list of nodes
+      #
+      # Parameters::
+      # * *actions* (Object): Actions
+      # * *nodes* (String or Array<String>): Node (or list of nodes) that should be checked
+      # * *sudo* (Boolean): Is sudo supposed to be used? [default: true]
+      def expect_actions_to_upload_logs(actions, nodes, sudo: true)
+        nodes = [nodes] if nodes.is_a?(String)
+        expect(actions.size).to eq nodes.size
+        nodes.each do |node|
+          expect(actions.key?(node)).to eq true
+          expect(actions[node][:remote_bash]).to eq "#{sudo ? 'sudo ' : ''}mkdir -p /var/log/deployments"
+          expect(actions[node][:scp].first[1]).to eq '/var/log/deployments'
+          expect(actions[node][:scp][:group]).to eq 'root'
+          expect(actions[node][:scp][:owner]).to eq 'root'
+          expect(actions[node][:scp][:sudo]).to eq sudo
+        end
+        Hash[nodes.map { |node| [node, [0, 'Logs uploaded', '']] }]
+      end
+
+      # Expect some logs to have the following information.
+      # Expected logs format:
+      #
+      # date: 2019-08-14 17:02:57
+      # user: muriel
+      # debug: Yes
+      # repo_name: my_remote_platform
+      # commit_id: c0d16b1b7ae286ae4a059185957e08f0ddc95517
+      # commit_message: Test commit
+      # diff_files: 
+      # ===== STDOUT =====
+      # Deploy successful
+      # ===== STDERR =====
+      #
+      # Parameters::
+      # * *logs* (String): The logs content
+      # * *stdout* (String): Expected STDOUT
+      # * *stderr* (String): Expected STDERR
+      # * *properties* (Hash<Symbol, String or Regexp>): Expected properties values, per name. Values can be exact strings or regexps.
+      def expect_logs_to_be(logs, stdout, stderr, properties)
+        lines = logs.split("\n")
+        idx_stdout = lines.index('===== STDOUT =====')
+        expect(idx_stdout).not_to eq nil
+        idx_stderr = lines.index('===== STDERR =====')
+        expect(idx_stderr).not_to eq nil
+        logs_properties = Hash[lines[0..idx_stdout - 1].map do |property_line|
+          property_fields = property_line.split(': ')
+          [
+            property_fields.first.to_sym,
+            property_fields[1..-1].join(': ')
+          ]
+        end]
+        expect(logs_properties.size).to eq properties.size
+        properties.each do |expected_property, expected_property_value|
+          expect(logs_properties.key?(expected_property)).to eq true
+          if expected_property_value.is_a?(String)
+            expect(logs_properties[expected_property]).to eq expected_property_value
+          else
+            expect(logs_properties[expected_property]).to match expected_property_value
+          end
+        end
+        expect(lines[idx_stdout + 1..idx_stderr - 1].join("\n")).to eq stdout
+        expect(lines[idx_stderr + 1..-1].join("\n")).to eq stderr
+      end
+
+      # Get a test Deployer
+      #
+      # Result::
+      # * Deployer: Deployer on which we can do testing
+      def test_deployer
+        @deployer = HybridPlatformsConductor::Deployer.new logger: logger, logger_stderr: logger, config: test_config, cmd_runner: test_cmd_runner, nodes_handler: test_nodes_handler, actions_executor: test_actions_executor, services_handler: test_services_handler unless @deployer
+        @deployer
+      end
+
+    end
+
+  end
+
+end

data/spec/hybrid_platforms_conductor_test/helpers/deployer_test_helpers.rb

@@ -0,0 +1,812 @@
+module HybridPlatformsConductorTest
+
+  module Helpers
+
+    module DeployerTestHelpers
+
+      # Define deployment specs that are common for check mode and real deployment
+      #
+      # Parameters::
+      # * *check_mode* (Boolean): Are we in check-mode? [default: true]
+      def deploy_specs_for(check_mode: true)
+        expected_deploy_result = [0, "#{check_mode ? 'Check' : 'Deploy'} successful", '']
+        platform_name = check_mode ? 'platform' : 'my_remote_platform'
+
+        context "testing deployment#{check_mode ? ' in why-run mode' : ''}" do
+
+          # Get expected actions for a deployment
+          #
+          # Parameters::
+          # * *services* (Hash<String, Array<String> >): Expected nodes that should be deployed, with their corresponding services [default: { 'node' => %w[service] }]
+          # * *sudo* (Boolean): Do we expect sudo to be used in commands? [default: true]
+          # * *check_mode* (Boolean): Are we testing in check mode? [default: @check_mode]
+          # * *mocked_deploy_result* (Hash or nil): Mocked result of the deployment actions, or nil to use the helper's default [default: nil]
+          # * *additional_expected_actions* (Array): Additional expected actions [default: []]
+          # * *expect_concurrent_actions* (Boolean): Are actions expected to be run in parallel? [default: false]
+          # * *expect_actions_timeout* (Integer or nil): Expected timeout in actions, or nil for none [default: nil]
+          def expected_actions_for_deploy_on(
+            services: { 'node' => %w[service] },
+            sudo: true,
+            check_mode: @check_mode,
+            mocked_deploy_result: nil,
+            additional_expected_actions: [],
+            expect_concurrent_actions: false,
+            expect_actions_timeout: nil
+          )
+            actions = [
+              # First run, we expect the mutex to be setup, and the deployment actions to be run
+              proc do |actions_per_nodes, timeout: nil, concurrent: false, log_to_dir: 'run_logs', log_to_stdout: true|
+                expect(timeout).to eq expect_actions_timeout
+                expect(concurrent).to eq expect_concurrent_actions
+                expect(log_to_dir).to eq 'run_logs'
+                expect_actions_to_deploy_on(
+                  actions_per_nodes,
+                  services.keys,
+                  check: check_mode,
+                  sudo: sudo,
+                  mocked_result: mocked_deploy_result,
+                  expected_actions: additional_expected_actions
+                )
+              end,
+              # Second run, we expect the mutex to be released
+              proc { |actions_per_nodes| expect_actions_to_unlock(actions_per_nodes, services.keys, sudo: sudo) }
+            ]
+            services.each do |node, node_services|
+              expect(test_services_handler).to receive(:actions_to_deploy_on).with(node, node_services, check_mode) do
+                [{ bash: "echo \"#{check_mode ? 'Checking' : 'Deploying'} on #{node}\"" }]
+              end
+            end
+            # Third run, we expect logs to be uploaded on the node (only if not check mode)
+            unless check_mode
+              services.each do |node, node_services|
+                expect(test_services_handler).to receive(:log_info_for).with(node, node_services) do
+                  {
+                    repo_name_0: 'platform',
+                    commit_id_0: '123456',
+                    commit_message_0: 'Test commit'
+                  }
+                end
+              end
+              actions << proc { |actions_per_nodes| expect_actions_to_upload_logs(actions_per_nodes, services.keys, sudo: sudo) }
+            end
+            actions
+          end
+
+          # Prepare a platform ready to test deployments on.
+          #
+          # Parameters::
+          # * *nodes_info* (Hash): Node info to give the platform [default: 1 node having 1 service]
+          # * *expect_default_actions* (Boolean): Should we expect default actions? [default: true]
+          # * *expect_sudo* (Boolean): Do we expect sudo to be used in commands? [default: true]
+          # * *expect_secrets* (Hash): Secrets to be expected during deployment [default: {}]
+          # * *expect_local_environment* (Boolean): Expected local environment flag [default: false]
+          # * *expect_additional_actions* (Array): Additional expected actions [default: []]
+          # * *expect_concurrent_actions* (Boolean): Are actions expected to be run in parallel? [default: false]
+          # * *expect_actions_timeout* (Integer or nil): Expected timeout in actions, or nil for none [default: nil]
+          # * *check_mode* (Boolean): Are we testing in check mode? [default: @check_mode]
+          # * *additional_config* (String): Additional configuration to set [default: '']
+          # * Proc: Code called once the platform is ready for testing the deployer
+          #   * Parameters::
+          #     * *repository* (String): Path to the repository
+          def with_platform_to_deploy(
+            nodes_info: { nodes: { 'node' => { services: %w[service] } } },
+            expect_default_actions: true,
+            expect_sudo: true,
+            expect_secrets: {},
+            expect_local_environment: false,
+            expect_additional_actions: [],
+            expect_concurrent_actions: false,
+            expect_actions_timeout: nil,
+            check_mode: @check_mode,
+            additional_config: ''
+          )
+            platform_name = check_mode ? 'platform' : 'my_remote_platform'
+            with_test_platform(nodes_info, !check_mode, additional_config) do |repository|
+              with_connections_mocked_on(nodes_info[:nodes].keys) do
+                # Mock the ServicesHandler accesses
+                expect_services_to_deploy = Hash[nodes_info[:nodes].map do |node, node_info|
+                  [node, node_info[:services]]
+                end]
+                unless check_mode
+                  expect(test_services_handler).to receive(:deploy_allowed?).with(
+                    services: expect_services_to_deploy,
+                    secrets: expect_secrets,
+                    local_environment: expect_local_environment
+                  ) do
+                    nil
+                  end
+                end
+                expect(test_services_handler).to receive(:package).with(
+                  services: expect_services_to_deploy,
+                  secrets: expect_secrets,
+                  local_environment: expect_local_environment
+                )
+                expect(test_services_handler).to receive(:prepare_for_deploy).with(
+                  services: expect_services_to_deploy,
+                  secrets: expect_secrets,
+                  local_environment: expect_local_environment,
+                  why_run: check_mode
+                )
+                expect_actions_executor_runs(expected_actions_for_deploy_on(
+                  services: expect_services_to_deploy,
+                  check_mode: check_mode,
+                  sudo: expect_sudo,
+                  additional_expected_actions: expect_additional_actions,
+                  expect_concurrent_actions: expect_concurrent_actions,
+                  expect_actions_timeout: expect_actions_timeout
+                )) if expect_default_actions
+                test_deployer.use_why_run = true if check_mode
+                yield repository
+              end
+            end
+          end
+
+          # Prepare a directory with certificates
+          #
+          # Parameters::
+          # * Proc: Code called with the directory created with a mocked certificate
+          #   * Parameters::
+          #     * *certs_dir* (String): Directory containing certificates
+          def with_certs_dir
+            with_repository do |repository|
+              certs_dir = "#{repository}/certificates"
+              FileUtils.mkdir_p certs_dir
+              File.write("#{certs_dir}/test_cert.crt", 'Hello')
+              yield certs_dir
+            end
+          end
+
+          before :each do
+            @check_mode = check_mode
+          end
+
+          it 'deploys on 1 node' do
+            with_platform_to_deploy do
+              expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+            end
+          end
+
+          it 'deploys on 1 node having several services' do
+            with_platform_to_deploy(nodes_info: { nodes: { 'node' => { services: %w[service1 service2 service3] } } }) do
+              expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+            end
+          end
+
+          it 'deploys on 1 node in a local environment' do
+            with_platform_to_deploy(expect_local_environment: true) do
+              test_deployer.local_environment = true
+              expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+              expect(test_deployer.local_environment).to eq true
+            end
+          end
+
+          it 'deploys on 1 node using root' do
+            with_platform_to_deploy(expect_sudo: false) do
+              test_actions_executor.connector(:ssh).ssh_user = 'root'
+              expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+            end
+          end
+
+          it 'deploys on 1 node using 1 secret' do
+            with_platform_to_deploy(expect_secrets: { 'secret1' => 'password1' }) do
+              test_deployer.secrets = [{ 'secret1' => 'password1' }]
+              expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+            end
+          end
+
+          it 'deploys on 1 node using several secrets' do
+            with_platform_to_deploy(expect_secrets: { 'secret1' => 'password1', 'secret2' => 'password2' }) do
+              test_deployer.secrets = [{ 'secret1' => 'password1' }, { 'secret2' => 'password2' }]
+              expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+            end
+          end
+
+          it 'deploys on 1 node in local environment with certificates to install using hpc_certificates on Debian' do
+            with_certs_dir do |certs_dir|
+              with_platform_to_deploy(
+                nodes_info: { nodes: { 'node' => { meta: { image: 'debian_9' }, services: %w[service] } } },
+                expect_local_environment: true,
+                expect_additional_actions: [
+                  { remote_bash: 'sudo apt update && sudo apt install -y ca-certificates' },
+                  {
+                    remote_bash: 'sudo update-ca-certificates',
+                    scp:  {
+                      certs_dir => '/usr/local/share/ca-certificates',
+                      :sudo => true
+                    }
+                  }
+                ]
+              ) do
+                ENV['hpc_certificates'] = certs_dir
+                test_deployer.local_environment = true
+                expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+              end
+            end
+          end
+
+          it 'deploys on 1 node with certificates to install using hpc_certificates on Debian but ignores them in non-local environment' do
+            with_certs_dir do |certs_dir|
+              with_platform_to_deploy(nodes_info: { nodes: { 'node' => { meta: { image: 'debian_9' }, services: %w[service] } } }) do
+                ENV['hpc_certificates'] = certs_dir
+                expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+              end
+            end
+          end
+
+          it 'deploys on 1 node with certificates to install using hpc_certificates on Debian using root' do
+            with_certs_dir do |certs_dir|
+              with_platform_to_deploy(
+                nodes_info: { nodes: { 'node' => { meta: { image: 'debian_9' }, services: %w[service] } } },
+                expect_sudo: false,
+                expect_local_environment: true,
+                expect_additional_actions: [
+                  { remote_bash: 'apt update && apt install -y ca-certificates' },
+                  {
+                    remote_bash: 'update-ca-certificates',
+                    scp:  {
+                      certs_dir => '/usr/local/share/ca-certificates',
+                      :sudo => false
+                    }
+                  }
+                ]
+              ) do
+                ENV['hpc_certificates'] = certs_dir
+                test_actions_executor.connector(:ssh).ssh_user = 'root'
+                test_deployer.local_environment = true
+                expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+              end
+            end
+          end
+
+          it 'deploys on 1 node with certificates to install using hpc_certificates on CentOS' do
+            with_certs_dir do |certs_dir|
+              with_platform_to_deploy(
+                nodes_info: { nodes: { 'node' => { meta: { image: 'centos_7' }, services: %w[service] } } },
+                expect_local_environment: true,
+                expect_additional_actions: [
+                  { remote_bash: 'sudo yum install -y ca-certificates' },
+                  {
+                    remote_bash: ['sudo update-ca-trust enable', 'sudo update-ca-trust extract'],
+                    scp: {
+                      "#{certs_dir}/test_cert.crt" => '/etc/pki/ca-trust/source/anchors',
+                      :sudo => true
+                    }
+                  }
+                ]
+              ) do
+                ENV['hpc_certificates'] = certs_dir
+                test_deployer.local_environment = true
+                expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+              end
+            end
+          end
+
+          it 'deploys on 1 node with certificates to install using hpc_certificates on CentOS using root' do
+            with_certs_dir do |certs_dir|
+              with_platform_to_deploy(
+                nodes_info: { nodes: { 'node' => { meta: { image: 'centos_7' }, services: %w[service] } } },
+                expect_sudo: false,
+                expect_local_environment: true,
+                expect_additional_actions: [
+                  { remote_bash: 'yum install -y ca-certificates' },
+                  {
+                    remote_bash: ['update-ca-trust enable', 'update-ca-trust extract'],
+                    scp: {
+                      "#{certs_dir}/test_cert.crt" => '/etc/pki/ca-trust/source/anchors',
+                      :sudo => false
+                    }
+                  }
+                ]
+              ) do
+                ENV['hpc_certificates'] = certs_dir
+                test_actions_executor.connector(:ssh).ssh_user = 'root'
+                test_deployer.local_environment = true
+                expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+              end
+            end
+          end
+
+          it 'deploys on several nodes' do
+            with_platform_to_deploy(nodes_info: { nodes: {
+              'node1' => { services: %w[service1] },
+              'node2' => { services: %w[service2] },
+              'node3' => { services: %w[service3] }
+            } }) do
+              expect(test_deployer.deploy_on(%w[node1 node2 node3])).to eq(
+                'node1' => expected_deploy_result,
+                'node2' => expected_deploy_result,
+                'node3' => expected_deploy_result
+              )
+            end
+          end
+
+          it 'deploys on several nodes in parallel' do
+            with_platform_to_deploy(
+              nodes_info: {
+                nodes: {
+                  'node1' => { services: %w[service1] },
+                  'node2' => { services: %w[service2] },
+                  'node3' => { services: %w[service3] }
+                }
+              },
+              expect_concurrent_actions: true
+            ) do
+              test_deployer.concurrent_execution = true
+              expect(test_deployer.deploy_on(%w[node1 node2 node3])).to eq(
+                'node1' => expected_deploy_result,
+                'node2' => expected_deploy_result,
+                'node3' => expected_deploy_result
+              )
+            end
+          end
+
+          it 'deploys on several nodes with timeout' do
+            with_platform_to_deploy(
+              nodes_info: {
+                nodes: {
+                  'node1' => { services: %w[service1] },
+                  'node2' => { services: %w[service2] },
+                  'node3' => { services: %w[service3] }
+                }
+              },
+              expect_actions_timeout: 5
+            ) do
+              test_deployer.timeout = 5
+              expect(test_deployer.deploy_on(%w[node1 node2 node3])).to eq(
+                'node1' => expected_deploy_result,
+                'node2' => expected_deploy_result,
+                'node3' => expected_deploy_result
+              )
+            end
+          end
+
+          context 'checking deployment retries' do
+
+            # Prepare a platform ready to test deployments' retries on.
+            #
+            # Parameters::
+            # * *nodes_info* (Hash): Node info to give the platform [default: { nodes: { 'node' => {} } }]
+            # * Proc: Code called once the platform is ready for testing the deployer
+            #   * Parameters::
+            #     * *repository* (String): Path to the repository
+            def with_platform_to_retry_deploy(nodes_info: { nodes: { 'node' => { services: %w[service] } } })
+              with_platform_to_deploy(
+                nodes_info: nodes_info,
+                expect_default_actions: false,
+                additional_config: "
+                for_nodes([#{nodes_info[:nodes].keys.map { |node| "'#{node}'" }.join(', ')}]) do
+                  retry_deploy_for_errors_on_stdout [
+                    'stdout non-deterministic error'
+                  ]
+                  retry_deploy_for_errors_on_stderr [
+                    'stderr non-deterministic error',
+                    /stderr regexp error \\d+/
+                  ]
+                end
+                for_nodes([#{nodes_info[:nodes].keys.map { |node| "'#{node}'" }.join(', ')}]) do
+                  retry_deploy_for_errors_on_stdout [
+                    /stdout regexp error \\d+/
+                  ]
+                end
+                "
+              ) do |repository|
+                yield repository
+              end
+            end
+
+            # Mock a sequential list of deployments
+            #
+            # Parameters::
+            # * *statuses* (Array<Hash<String,Status> or Status>)>): List of mocked deployment statuses per node name, or just the status for the default node.
+            #   A status is a triplet [Integer or Symbol, String, String]: exit status, stdout and stderr.
+            def mock_deploys_with(statuses)
+              expect_actions_executor_runs(statuses.map do |status|
+                status = { 'node' => status } if status.is_a?(Array)
+                expected_actions_for_deploy_on(
+                  services: Hash[status.keys.map { |node| [node, %w[service]] }],
+                  mocked_deploy_result: status
+                )
+              end.flatten)
+            end
+
+            it 'restarts deployment for a non-deterministic error' do
+              with_platform_to_retry_deploy do
+                test_deployer.nbr_retries_on_error = 1
+                mock_deploys_with [
+                  [1, "Error: This is a stdout non-deterministic error\nDeploy failed\n", ''],
+                  [0, 'Deploy ok', '']
+                ]
+                expect(test_deployer.deploy_on('node')).to eq('node' => [
+                  0,
+                  <<~EOS,
+                    Error: This is a stdout non-deterministic error
+                    Deploy failed
+
+                    Deployment exit status code: 1
+                    !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+                    Deploy ok
+                  EOS
+                  <<~EOS
+                    !!! 1 retriable errors detected in this deployment:
+                    * stdout non-deterministic error
+
+                    !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+
+                  EOS
+                ])
+              end
+            end
+
+            it 'restarts deployment for a non-deterministic error matched with a Regexp' do
+              with_platform_to_retry_deploy do
+                test_deployer.nbr_retries_on_error = 1
+                mock_deploys_with [
+                  [1, "Error: This is a stdout regexp error 42\nDeploy failed\n", ''],
+                  [0, 'Deploy ok', '']
+                ]
+                expect(test_deployer.deploy_on('node')).to eq('node' => [
+                  0,
+                  <<~EOS,
+                    Error: This is a stdout regexp error 42
+                    Deploy failed
+
+                    Deployment exit status code: 1
+                    !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+                    Deploy ok
+                  EOS
+                  <<~EOS
+                    !!! 1 retriable errors detected in this deployment:
+                    * /stdout regexp error \\d+/ matched 'stdout regexp error 42'
+
+                    !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+
+                  EOS
+                ])
+              end
+            end
+
+            it 'restarts deployment for a non-deterministic error on stderr' do
+              with_platform_to_retry_deploy do
+                test_deployer.nbr_retries_on_error = 1
+                mock_deploys_with [
+                  [1, '', "Error: This is a stderr non-deterministic error\nDeploy failed\n"],
+                  [0, 'Deploy ok', '']
+                ]
+                expect(test_deployer.deploy_on('node')).to eq('node' => [
+                  0,
+                  <<~EOS,
+
+                    Deployment exit status code: 1
+                    !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+                    Deploy ok
+                  EOS
+                  <<~EOS
+                    Error: This is a stderr non-deterministic error
+                    Deploy failed
+                    !!! 1 retriable errors detected in this deployment:
+                    * stderr non-deterministic error
+
+                    !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+
+                  EOS
+                ])
+              end
+            end
+
+            it 'restarts deployment for a non-deterministic error on stderr matched with a Regexp' do
+              with_platform_to_retry_deploy do
+                test_deployer.nbr_retries_on_error = 1
+                mock_deploys_with [
+                  [1, '', "Error: This is a stderr regexp error 42\nDeploy failed\n"],
+                  [0, 'Deploy ok', '']
+                ]
+                expect(test_deployer.deploy_on('node')).to eq('node' => [
+                  0,
+                  <<~EOS,
+
+                    Deployment exit status code: 1
+                    !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+                    Deploy ok
+                  EOS
+                  <<~EOS
+                    Error: This is a stderr regexp error 42
+                    Deploy failed
+                    !!! 1 retriable errors detected in this deployment:
+                    * /stderr regexp error \\d+/ matched 'stderr regexp error 42'
+
+                    !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+
+                  EOS
+                ])
+              end
+            end
+
+            it 'stops restarting deployments for a non-deterministic error when errors has disappeared, even if retries were remaining' do
+              with_platform_to_retry_deploy do
+                test_deployer.nbr_retries_on_error = 5
+                mock_deploys_with [
+                  [1, "Error: This is a stdout non-deterministic error 1\nDeploy failed", ''],
+                  [1, "Error: This is a stdout non-deterministic error 2\nDeploy failed", ''],
+                  [0, 'Deploy ok', '']
+                ]
+                expect(test_deployer.deploy_on('node')).to eq('node' => [
+                  0,
+                  <<~EOS,
+                    Error: This is a stdout non-deterministic error 1
+                    Deploy failed
+                    Deployment exit status code: 1
+                    !!! Retry deployment due to non-deterministic error (4 remaining attempts)...
+                    Error: This is a stdout non-deterministic error 2
+                    Deploy failed
+
+                    Deployment exit status code: 1
+                    !!! Retry deployment due to non-deterministic error (3 remaining attempts)...
+                    Deploy ok
+                  EOS
+                  <<~EOS
+                    !!! 1 retriable errors detected in this deployment:
+                    * stdout non-deterministic error
+
+                    !!! Retry deployment due to non-deterministic error (4 remaining attempts)...
+                    !!! 1 retriable errors detected in this deployment:
+                    * stdout non-deterministic error
+
+
+                    !!! Retry deployment due to non-deterministic error (3 remaining attempts)...
+
+                  EOS
+                ])
+              end
+            end
+
+            it 'stops restarting deployments for a non-deterministic error that became deterministic, even if retries were remaining' do
+              with_platform_to_retry_deploy do
+                test_deployer.nbr_retries_on_error = 5
+                mock_deploys_with [
+                  [1, "Error: This is a stdout non-deterministic error 1\nDeploy failed", ''],
+                  [1, "Error: This is a stdout non-deterministic error 2\nDeploy failed", ''],
+                  [1, "Error: This is a stdout deterministic error 3\nDeploy failed", '']
+                ]
+                expect(test_deployer.deploy_on('node')).to eq('node' => [
+                  1,
+                  <<~EOS,
+                    Error: This is a stdout non-deterministic error 1
+                    Deploy failed
+                    Deployment exit status code: 1
+                    !!! Retry deployment due to non-deterministic error (4 remaining attempts)...
+                    Error: This is a stdout non-deterministic error 2
+                    Deploy failed
+
+                    Deployment exit status code: 1
+                    !!! Retry deployment due to non-deterministic error (3 remaining attempts)...
+                    Error: This is a stdout deterministic error 3
+                    Deploy failed
+                  EOS
+                  <<~EOS
+                    !!! 1 retriable errors detected in this deployment:
+                    * stdout non-deterministic error
+
+                    !!! Retry deployment due to non-deterministic error (4 remaining attempts)...
+                    !!! 1 retriable errors detected in this deployment:
+                    * stdout non-deterministic error
+
+
+                    !!! Retry deployment due to non-deterministic error (3 remaining attempts)...
+
+                  EOS
+                ])
+              end
+            end
+
+            it 'does not restart deployment for a deterministic error' do
+              with_platform_to_retry_deploy do
+                test_deployer.nbr_retries_on_error = 5
+                mock_deploys_with [
+                  [1, "Error: This is a stdout deterministic error\nDeploy failed\n", '']
+                ]
+                expect(test_deployer.deploy_on('node')).to eq('node' => [
+                  1,
+                  <<~EOS,
+                    Error: This is a stdout deterministic error
+                    Deploy failed
+                  EOS
+                  ''
+                ])
+              end
+            end
+
+            it 'does not restart deployment for a non-deterministic error logged during a successful deploy' do
+              with_platform_to_retry_deploy do
+                test_deployer.nbr_retries_on_error = 5
+                mock_deploys_with [
+                  [0, "Error: This is a stdout non-deterministic error\nDeploy failed\n", '']
+                ]
+                expect(test_deployer.deploy_on('node')).to eq('node' => [
+                  0,
+                  <<~EOS,
+                    Error: This is a stdout non-deterministic error
+                    Deploy failed
+                  EOS
+                  ''
+                ])
+              end
+            end
+
+            it 'does not restart deployment for a non-deterministic error if retries are 0' do
+              with_platform_to_retry_deploy do
+                test_deployer.nbr_retries_on_error = 0
+                mock_deploys_with [
+                  [1, "Error: This is a stdout non-deterministic error\nDeploy failed\n", '']
+                ]
+                expect(test_deployer.deploy_on('node')).to eq('node' => [
+                  1,
+                  <<~EOS,
+                    Error: This is a stdout non-deterministic error
+                    Deploy failed
+                  EOS
+                  ''
+                ])
+              end
+            end
+
+            it 'restarts deployment for non-deterministic errors with a limited amount of retries' do
+              with_platform_to_retry_deploy do
+                test_deployer.nbr_retries_on_error = 2
+                mock_deploys_with [
+                  [1, "Error: This is a stdout non-deterministic error 1\nDeploy failed", ''],
+                  [1, "Error: This is a stdout non-deterministic error 2\nDeploy failed", ''],
+                  [1, "Error: This is a stdout non-deterministic error 3\nDeploy failed", '']
+                ]
+                expect(test_deployer.deploy_on('node')).to eq('node' => [
+                  1,
+                  <<~EOS,
+                    Error: This is a stdout non-deterministic error 1
+                    Deploy failed
+                    Deployment exit status code: 1
+                    !!! Retry deployment due to non-deterministic error (1 remaining attempts)...
+                    Error: This is a stdout non-deterministic error 2
+                    Deploy failed
+
+                    Deployment exit status code: 1
+                    !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+                    Error: This is a stdout non-deterministic error 3
+                    Deploy failed
+                  EOS
+                  <<~EOS
+                    !!! 1 retriable errors detected in this deployment:
+                    * stdout non-deterministic error
+
+                    !!! Retry deployment due to non-deterministic error (1 remaining attempts)...
+                    !!! 1 retriable errors detected in this deployment:
+                    * stdout non-deterministic error
+
+
+                    !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+
+                  EOS
+                ])
+              end
+            end
+
+            it 'restarts deployment for non-deterministic errors only on nodes needing it' do
+              with_platform_to_retry_deploy(nodes_info: { nodes: {
+                'node1' => { services: %w[service] },
+                'node2' => { services: %w[service] },
+                'node3' => { services: %w[service] },
+                'node4' => { services: %w[service] }
+              } }) do
+                test_deployer.nbr_retries_on_error = 2
+                # Some nodes deploy successfully,
+                # others have deterministic errors,
+                # others have non-deterministic errors being corrected
+                # others have non-deterministic errors not being corrected
+                mock_deploys_with [
+                  {
+                    'node1' => [1, "Error: This is a stdout non-deterministic error\n[node1] Deploy failed\n", ''],
+                    'node2' => [0, '[node2] Deploy ok', ''],
+                    'node3' => [1, "Error: This is a stdout non-deterministic error\n[node3] Deploy failed\n", ''],
+                    'node4' => [1, "Error: This is a stdout non-deterministic error\n[node4] Deploy failed\n", '']
+                  },
+                  {
+                    'node1' => [0, '[node1] Deploy ok', ''],
+                    'node3' => [1, "Error: This is a stdout deterministic error\n[node3] Deploy failed\n", ''],
+                    'node4' => [1, "Error: This is a stdout non-deterministic error\n[node4] Deploy failed\n", '']
+                  },
+                  {
+                    'node4' => [1, "Error: This is a stdout non-deterministic error\n[node4] Deploy failed\n", '']
+                  }
+                ]
+                expect(test_deployer.deploy_on(%w[node1 node2 node3 node4])).to eq(
+                  'node1' => [
+                    0,
+                    <<~EOS,
+                      Error: This is a stdout non-deterministic error
+                      [node1] Deploy failed
+
+                      Deployment exit status code: 1
+                      !!! Retry deployment due to non-deterministic error (1 remaining attempts)...
+                      [node1] Deploy ok
+                    EOS
+                    <<~EOS
+                      !!! 1 retriable errors detected in this deployment:
+                      * stdout non-deterministic error
+
+                      !!! Retry deployment due to non-deterministic error (1 remaining attempts)...
+
+                    EOS
+                  ],
+                  'node2' => [
+                    0,
+                    '[node2] Deploy ok',
+                    ''
+                  ],
+                  'node3' => [
+                    1,
+                    <<~EOS,
+                      Error: This is a stdout non-deterministic error
+                      [node3] Deploy failed
+
+                      Deployment exit status code: 1
+                      !!! Retry deployment due to non-deterministic error (1 remaining attempts)...
+                      Error: This is a stdout deterministic error
+                      [node3] Deploy failed
+
+                    EOS
+                    <<~EOS
+                      !!! 1 retriable errors detected in this deployment:
+                      * stdout non-deterministic error
+
+                      !!! Retry deployment due to non-deterministic error (1 remaining attempts)...
+
+                    EOS
+                  ],
+                  'node4' => [
+                    1,
+                    <<~EOS,
+                      Error: This is a stdout non-deterministic error
+                      [node4] Deploy failed
+
+                      Deployment exit status code: 1
+                      !!! Retry deployment due to non-deterministic error (1 remaining attempts)...
+                      Error: This is a stdout non-deterministic error
+                      [node4] Deploy failed
+
+
+                      Deployment exit status code: 1
+                      !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+                      Error: This is a stdout non-deterministic error
+                      [node4] Deploy failed
+
+                    EOS
+                    <<~EOS
+                      !!! 1 retriable errors detected in this deployment:
+                      * stdout non-deterministic error
+
+                      !!! Retry deployment due to non-deterministic error (1 remaining attempts)...
+                      !!! 1 retriable errors detected in this deployment:
+                      * stdout non-deterministic error
+
+
+                      !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+
+                    EOS
+                  ]
+                )
+              end
+            end
+
+          end
+
+        end
+
+      end
+
+    end
+
+  end
+
+end
+
+RSpec.configure do |c|
+  c.extend HybridPlatformsConductorTest::Helpers::DeployerTestHelpers
+end