hybrid_platforms_conductor 32.17.0 → 33.0.2

data/spec/hybrid_platforms_conductor_test/helpers/reports_handler_helpers.rb

@@ -18,7 +18,7 @@ module HybridPlatformsConductorTest
       # Result::
       # * ReportsHandler: ReportsHandler on which we can do testing
       def test_reports_handler
-        @reports_handler = HybridPlatformsConductor::ReportsHandler.new logger: logger, logger_stderr: logger, config: test_config, platforms_handler: test_platforms_handler, nodes_handler: test_nodes_handler unless @reports_handler
+        @reports_handler ||= HybridPlatformsConductor::ReportsHandler.new logger: logger, logger_stderr: logger, config: test_config, platforms_handler: test_platforms_handler, nodes_handler: test_nodes_handler
         @reports_handler
       end
 

data/spec/hybrid_platforms_conductor_test/helpers/serverless_chef_helpers.rb

@@ -27,18 +27,18 @@ module HybridPlatformsConductorTest
             FileUtils.cp_r "#{__dir__}/../serverless_chef_repositories/#{names[name]}/.", repository
           end
           with_platforms(repositories.values.map { |repository| "serverless_chef_platform path: '#{repository}'\n" }.join + additional_config) do
-            repositories = Hash[names.keys.map do |name|
+            repositories = names.keys.map do |name|
               [
                 test_platforms_handler.platform(name),
                 repositories[name]
               ]
-            end]
+            end.to_h
             test_platforms_handler.inject_dependencies(
               nodes_handler: test_nodes_handler,
               actions_executor: test_actions_executor
             )
             if repositories.size == 1
-              yield *repositories.first
+              yield(*repositories.first)
             else
               yield repositories
             end

data/spec/hybrid_platforms_conductor_test/helpers/services_handler_helpers.rb

@@ -9,7 +9,7 @@ module HybridPlatformsConductorTest
       # Result::
       # * ServicesHandler: ServicesHandler on which we can do testing
       def test_services_handler
-        @services_handler = HybridPlatformsConductor::ServicesHandler.new logger: logger, logger_stderr: logger, config: test_config, cmd_runner: test_cmd_runner, platforms_handler: test_platforms_handler, nodes_handler: test_nodes_handler, actions_executor: test_actions_executor unless @services_handler
+        @services_handler ||= HybridPlatformsConductor::ServicesHandler.new logger: logger, logger_stderr: logger, config: test_config, cmd_runner: test_cmd_runner, platforms_handler: test_platforms_handler, nodes_handler: test_nodes_handler, actions_executor: test_actions_executor
         @services_handler
       end
 

data/spec/hybrid_platforms_conductor_test/helpers/tests_runner_helpers.rb

@@ -27,7 +27,7 @@ module HybridPlatformsConductorTest
       # Result::
       # * Deployer: Tests Runner on which we can do testing
       def test_tests_runner
-        @tests_runner = HybridPlatformsConductor::TestsRunner.new logger: logger, logger_stderr: logger, config: test_config, cmd_runner: test_cmd_runner, platforms_handler: test_platforms_handler, nodes_handler: test_nodes_handler, actions_executor: test_actions_executor, deployer: test_deployer unless @tests_runner
+        @tests_runner ||= HybridPlatformsConductor::TestsRunner.new logger: logger, logger_stderr: logger, config: test_config, cmd_runner: test_cmd_runner, platforms_handler: test_platforms_handler, nodes_handler: test_nodes_handler, actions_executor: test_actions_executor, deployer: test_deployer
         @tests_runner
       end
 

data/spec/hybrid_platforms_conductor_test/platform_handler_plugins/test.rb

@@ -202,7 +202,7 @@ module HybridPlatformsConductorTest
       def platform_info
         {
           nodes: {},
-          nodes_lists: {},
+          nodes_lists: {}
         }.merge(HybridPlatformsConductorTest::PlatformHandlerPlugins::Test.platforms_info[name])
       end
 

data/spec/hybrid_platforms_conductor_test/rubocop_spec.rb

@@ -0,0 +1,31 @@
+require 'json'
+
+describe 'Coding guidelines' do
+
+  it 'makes sure code style follow Rubocop guides' do
+    rubocop_report = JSON.parse(`bundle exec rubocop --format json`)
+    expect(rubocop_report['summary']['offense_count']).to(
+      eq(0),
+      proc do
+        # Format a great error message to help
+        wrong_files = rubocop_report['files'].reject { |file_info| file_info['offenses'].empty? }
+        <<~EO_ERROR
+          #{wrong_files.size} files have Rubocop issues:
+          #{
+            wrong_files.map do |file_info|
+              offenses = file_info['offenses'].map { |offense_info| "L#{offense_info['location']['start_line']}: #{offense_info['cop_name']} - #{offense_info['message']}" }
+              "* #{file_info['path']}:#{
+                if offenses.size == 1
+                  " #{offenses.first}"
+                else
+                  " #{offenses.size} offenses:\n#{offenses.map { |offense| "  - #{offense}" }.join("\n")}"
+                end
+              }"
+            end.join("\n")
+          }
+        EO_ERROR
+      end
+    )
+  end
+
+end

data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/1_node/nodes/node.json

@@ -4,10 +4,10 @@
     "description": "Single test node",
     "image": "debian_9",
     "private_ips": ["172.16.0.1"],
-    "property1": {
-      "property11": "value11"
+    "property_1": {
+      "property_11": "value11"
     },
-    "property2": "value2"
+    "property_2": "value2"
   },
   "policy_name": "test_policy",
   "policy_group": "test_group"

data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/data_bags/nodes/node.json

@@ -4,10 +4,10 @@
     "description": "Single test node",
     "image": "debian_9",
     "private_ips": ["172.16.0.1"],
-    "property1": {
-      "property11": "value11"
+    "property_1": {
+      "property_11": "value11"
     },
-    "property2": "value2"
+    "property_2": "value2"
   },
   "policy_name": "test_policy",
   "policy_group": "test_group"

data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/recipes/cookbooks/test_cookbook_2/libraries/default.rb

@@ -1,4 +1,5 @@
 # Library code
 
 def my_library_helper(arg)
+  puts arg
 end

data/spec/hybrid_platforms_conductor_test/serverless_chef_repositories/several_cookbooks/other_cookbooks/test_cookbook_2/libraries/default.rb

@@ -1,4 +1,5 @@
 # Library code
 
 def my_library_helper(arg)
+  puts arg
 end

data/spec/hybrid_platforms_conductor_test/shared_examples/deployer.rb

@@ -0,0 +1,952 @@
+require 'timeout'
+
+shared_examples 'a deployer' do
+
+  let(:expected_deploy_result) { [0, "#{check_mode ? 'Check' : 'Deploy'} successful", ''] }
+
+  before do
+    @check_mode = check_mode
+  end
+
+  it 'deploys on 1 node' do
+    with_platform_to_deploy do
+      expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+    end
+  end
+
+  it 'deploys on 1 node having several services' do
+    with_platform_to_deploy(nodes_info: { nodes: { 'node' => { services: %w[service1 service2 service3] } } }) do
+      expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+    end
+  end
+
+  it 'deploys on 1 node in a local environment' do
+    with_platform_to_deploy(expect_local_environment: true) do
+      test_deployer.local_environment = true
+      expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+      expect(test_deployer.local_environment).to eq true
+    end
+  end
+
+  it 'deploys on 1 node using root' do
+    with_platform_to_deploy(expect_sudo: nil) do
+      test_actions_executor.connector(:ssh).ssh_user = 'root'
+      expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+    end
+  end
+
+  it 'deploys on 1 node using an alternate sudo' do
+    with_platform_to_deploy(
+      expect_sudo: 'other_sudo --user root',
+      additional_config: <<~'EO_CONFIG'
+        sudo_for { |user| "other_sudo --user #{user}" }
+      EO_CONFIG
+    ) do
+      expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+    end
+  end
+
+  it 'deploys on 1 node using 1 secret' do
+    with_platform_to_deploy(expect_secrets: { 'secret1' => 'password1' }) do
+      test_deployer.override_secrets('secret1' => 'password1')
+      expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+    end
+  end
+
+  it 'deploys on 1 node in local environment with certificates to install using hpc_certificates on Debian' do
+    with_certs_dir do |certs_dir|
+      with_platform_to_deploy(
+        nodes_info: { nodes: { 'node' => { meta: { image: 'debian_9' }, services: %w[service] } } },
+        expect_local_environment: true,
+        expect_additional_actions: [
+          { remote_bash: 'sudo -u root apt update && sudo -u root apt install -y ca-certificates' },
+          {
+            remote_bash: 'sudo -u root update-ca-certificates',
+            scp: {
+              certs_dir => '/usr/local/share/ca-certificates',
+              :sudo => true
+            }
+          }
+        ]
+      ) do
+        ENV['hpc_certificates'] = certs_dir
+        test_deployer.local_environment = true
+        expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+      end
+    end
+  end
+
+  it 'deploys on 1 node in local environment with certificates to install using hpc_certificates on Debian and an alternate sudo' do
+    with_certs_dir do |certs_dir|
+      with_platform_to_deploy(
+        nodes_info: { nodes: { 'node' => { meta: { image: 'debian_9' }, services: %w[service] } } },
+        expect_sudo: 'other_sudo --user root',
+        expect_local_environment: true,
+        expect_additional_actions: [
+          { remote_bash: 'other_sudo --user root apt update && other_sudo --user root apt install -y ca-certificates' },
+          {
+            remote_bash: 'other_sudo --user root update-ca-certificates',
+            scp: {
+              certs_dir => '/usr/local/share/ca-certificates',
+              :sudo => true
+            }
+          }
+        ],
+        additional_config: <<~'EO_CONFIG'
+          sudo_for { |user| "other_sudo --user #{user}" }
+        EO_CONFIG
+      ) do
+        ENV['hpc_certificates'] = certs_dir
+        test_deployer.local_environment = true
+        expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+      end
+    end
+  end
+
+  it 'deploys on 1 node with certificates to install using hpc_certificates on Debian but ignores them in non-local environment' do
+    with_certs_dir do |certs_dir|
+      with_platform_to_deploy(nodes_info: { nodes: { 'node' => { meta: { image: 'debian_9' }, services: %w[service] } } }) do
+        ENV['hpc_certificates'] = certs_dir
+        expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+      end
+    end
+  end
+
+  it 'deploys on 1 node with certificates to install using hpc_certificates on Debian using root' do
+    with_certs_dir do |certs_dir|
+      with_platform_to_deploy(
+        nodes_info: { nodes: { 'node' => { meta: { image: 'debian_9' }, services: %w[service] } } },
+        expect_sudo: nil,
+        expect_local_environment: true,
+        expect_additional_actions: [
+          { remote_bash: 'apt update && apt install -y ca-certificates' },
+          {
+            remote_bash: 'update-ca-certificates',
+            scp: {
+              certs_dir => '/usr/local/share/ca-certificates',
+              :sudo => false
+            }
+          }
+        ]
+      ) do
+        ENV['hpc_certificates'] = certs_dir
+        test_actions_executor.connector(:ssh).ssh_user = 'root'
+        test_deployer.local_environment = true
+        expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+      end
+    end
+  end
+
+  it 'deploys on 1 node with certificates to install using hpc_certificates on CentOS' do
+    with_certs_dir do |certs_dir|
+      with_platform_to_deploy(
+        nodes_info: { nodes: { 'node' => { meta: { image: 'centos_7' }, services: %w[service] } } },
+        expect_local_environment: true,
+        expect_additional_actions: [
+          { remote_bash: 'sudo -u root yum install -y ca-certificates' },
+          {
+            remote_bash: ['sudo -u root update-ca-trust enable', 'sudo -u root update-ca-trust extract'],
+            scp: {
+              "#{certs_dir}/test_cert.crt" => '/etc/pki/ca-trust/source/anchors',
+              :sudo => true
+            }
+          }
+        ]
+      ) do
+        ENV['hpc_certificates'] = certs_dir
+        test_deployer.local_environment = true
+        expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+      end
+    end
+  end
+
+  it 'deploys on 1 node with certificates to install using hpc_certificates on CentOS and an alternate sudo' do
+    with_certs_dir do |certs_dir|
+      with_platform_to_deploy(
+        nodes_info: { nodes: { 'node' => { meta: { image: 'centos_7' }, services: %w[service] } } },
+        expect_sudo: 'other_sudo --user root',
+        expect_local_environment: true,
+        expect_additional_actions: [
+          { remote_bash: 'other_sudo --user root yum install -y ca-certificates' },
+          {
+            remote_bash: ['other_sudo --user root update-ca-trust enable', 'other_sudo --user root update-ca-trust extract'],
+            scp: {
+              "#{certs_dir}/test_cert.crt" => '/etc/pki/ca-trust/source/anchors',
+              :sudo => true
+            }
+          }
+        ],
+        additional_config: <<~'EO_CONFIG'
+          sudo_for { |user| "other_sudo --user #{user}" }
+        EO_CONFIG
+      ) do
+        ENV['hpc_certificates'] = certs_dir
+        test_deployer.local_environment = true
+        expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+      end
+    end
+  end
+
+  it 'deploys on 1 node with certificates to install using hpc_certificates on CentOS using root' do
+    with_certs_dir do |certs_dir|
+      with_platform_to_deploy(
+        nodes_info: { nodes: { 'node' => { meta: { image: 'centos_7' }, services: %w[service] } } },
+        expect_sudo: nil,
+        expect_local_environment: true,
+        expect_additional_actions: [
+          { remote_bash: 'yum install -y ca-certificates' },
+          {
+            remote_bash: ['update-ca-trust enable', 'update-ca-trust extract'],
+            scp: {
+              "#{certs_dir}/test_cert.crt" => '/etc/pki/ca-trust/source/anchors',
+              :sudo => false
+            }
+          }
+        ]
+      ) do
+        ENV['hpc_certificates'] = certs_dir
+        test_actions_executor.connector(:ssh).ssh_user = 'root'
+        test_deployer.local_environment = true
+        expect(test_deployer.deploy_on('node')).to eq('node' => expected_deploy_result)
+      end
+    end
+  end
+
+  it 'deploys on several nodes' do
+    with_platform_to_deploy(
+      nodes_info: {
+        nodes: {
+          'node1' => { services: %w[service1] },
+          'node2' => { services: %w[service2] },
+          'node3' => { services: %w[service3] }
+        }
+      }
+    ) do
+      expect(test_deployer.deploy_on(%w[node1 node2 node3])).to eq(
+        'node1' => expected_deploy_result,
+        'node2' => expected_deploy_result,
+        'node3' => expected_deploy_result
+      )
+    end
+  end
+
+  it 'deploys on several nodes in parallel' do
+    with_platform_to_deploy(
+      nodes_info: {
+        nodes: {
+          'node1' => { services: %w[service1] },
+          'node2' => { services: %w[service2] },
+          'node3' => { services: %w[service3] }
+        }
+      },
+      expect_concurrent_actions: true
+    ) do
+      test_deployer.concurrent_execution = true
+      expect(test_deployer.deploy_on(%w[node1 node2 node3])).to eq(
+        'node1' => expected_deploy_result,
+        'node2' => expected_deploy_result,
+        'node3' => expected_deploy_result
+      )
+    end
+  end
+
+  it 'deploys on several nodes with timeout' do
+    with_platform_to_deploy(
+      nodes_info: {
+        nodes: {
+          'node1' => { services: %w[service1] },
+          'node2' => { services: %w[service2] },
+          'node3' => { services: %w[service3] }
+        }
+      },
+      expect_actions_timeout: 5
+    ) do
+      test_deployer.timeout = 5
+      expect(test_deployer.deploy_on(%w[node1 node2 node3])).to eq(
+        'node1' => expected_deploy_result,
+        'node2' => expected_deploy_result,
+        'node3' => expected_deploy_result
+      )
+    end
+  end
+
+  it 'fails when packaging timeout has been reached while taking the futex' do
+    with_platform_to_deploy(
+      additional_config: 'packaging_timeout 1',
+      expect_package: false,
+      expect_prepare_for_deploy: false,
+      expect_connections_to_nodes: false
+    ) do
+      # Simulate another process taking the packaging futex
+      futex_file = HybridPlatformsConductor::Deployer.const_get(:PACKAGING_FUTEX_FILE)
+      Futex.new(futex_file).open do
+        # Expect the error to be raised within 2 seconds (as it should timeout after 1 second)
+        Timeout.timeout(2) do
+          expect { test_deployer.deploy_on('node') }.to raise_error(
+            Futex::CantLock,
+            /can't get exclusive access to the file #{Regexp.escape(futex_file)} because of the lock at #{Regexp.escape(futex_file)}\.lock, after 1\.\d+s of waiting/
+          )
+        end
+      rescue Timeout::Error
+        raise 'The packaging timeout (set to 1 seconds) did not fire within 2 seconds. Looks like it is not working properly.'
+      end
+    end
+  end
+
+  context 'when checking deployment retries' do
+
+    it 'restarts deployment for a non-deterministic error' do
+      with_platform_to_retry_deploy do
+        test_deployer.nbr_retries_on_error = 1
+        mock_deploys_with [
+          [1, "Error: This is a stdout non-deterministic error\nDeploy failed\n", ''],
+          [0, 'Deploy ok', '']
+        ]
+        expect(test_deployer.deploy_on('node')).to eq(
+          'node' => [
+            0,
+            <<~EO_STDOUT,
+              Error: This is a stdout non-deterministic error
+              Deploy failed
+
+              Deployment exit status code: 1
+              !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+              Deploy ok
+            EO_STDOUT
+            <<~EO_STDERR
+              !!! 1 retriable errors detected in this deployment:
+              * stdout non-deterministic error
+
+              !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+
+            EO_STDERR
+          ]
+        )
+      end
+    end
+
+    it 'restarts deployment for a non-deterministic error matched with a Regexp' do
+      with_platform_to_retry_deploy do
+        test_deployer.nbr_retries_on_error = 1
+        mock_deploys_with [
+          [1, "Error: This is a stdout regexp error 42\nDeploy failed\n", ''],
+          [0, 'Deploy ok', '']
+        ]
+        expect(test_deployer.deploy_on('node')).to eq(
+          'node' => [
+            0,
+            <<~EO_STDOUT,
+              Error: This is a stdout regexp error 42
+              Deploy failed
+
+              Deployment exit status code: 1
+              !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+              Deploy ok
+            EO_STDOUT
+            <<~EO_STDERR
+              !!! 1 retriable errors detected in this deployment:
+              * /stdout regexp error \\d+/ matched 'stdout regexp error 42'
+
+              !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+
+            EO_STDERR
+          ]
+        )
+      end
+    end
+
+    it 'restarts deployment for a non-deterministic error on stderr' do
+      with_platform_to_retry_deploy do
+        test_deployer.nbr_retries_on_error = 1
+        mock_deploys_with [
+          [1, '', "Error: This is a stderr non-deterministic error\nDeploy failed\n"],
+          [0, 'Deploy ok', '']
+        ]
+        expect(test_deployer.deploy_on('node')).to eq(
+          'node' => [
+            0,
+            <<~EO_STDOUT,
+
+              Deployment exit status code: 1
+              !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+              Deploy ok
+            EO_STDOUT
+            <<~EO_STDERR
+              Error: This is a stderr non-deterministic error
+              Deploy failed
+              !!! 1 retriable errors detected in this deployment:
+              * stderr non-deterministic error
+
+              !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+
+            EO_STDERR
+          ]
+        )
+      end
+    end
+
+    it 'restarts deployment for a non-deterministic error on stderr matched with a Regexp' do
+      with_platform_to_retry_deploy do
+        test_deployer.nbr_retries_on_error = 1
+        mock_deploys_with [
+          [1, '', "Error: This is a stderr regexp error 42\nDeploy failed\n"],
+          [0, 'Deploy ok', '']
+        ]
+        expect(test_deployer.deploy_on('node')).to eq(
+          'node' => [
+            0,
+            <<~EO_STDOUT,
+
+              Deployment exit status code: 1
+              !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+              Deploy ok
+            EO_STDOUT
+            <<~EO_STDERR
+              Error: This is a stderr regexp error 42
+              Deploy failed
+              !!! 1 retriable errors detected in this deployment:
+              * /stderr regexp error \\d+/ matched 'stderr regexp error 42'
+
+              !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+
+            EO_STDERR
+          ]
+        )
+      end
+    end
+
+    it 'stops restarting deployments for a non-deterministic error when errors has disappeared, even if retries were remaining' do
+      with_platform_to_retry_deploy do
+        test_deployer.nbr_retries_on_error = 5
+        mock_deploys_with [
+          [1, "Error: This is a stdout non-deterministic error 1\nDeploy failed", ''],
+          [1, "Error: This is a stdout non-deterministic error 2\nDeploy failed", ''],
+          [0, 'Deploy ok', '']
+        ]
+        expect(test_deployer.deploy_on('node')).to eq(
+          'node' => [
+            0,
+            <<~EO_STDOUT,
+              Error: This is a stdout non-deterministic error 1
+              Deploy failed
+              Deployment exit status code: 1
+              !!! Retry deployment due to non-deterministic error (4 remaining attempts)...
+              Error: This is a stdout non-deterministic error 2
+              Deploy failed
+
+              Deployment exit status code: 1
+              !!! Retry deployment due to non-deterministic error (3 remaining attempts)...
+              Deploy ok
+            EO_STDOUT
+            <<~EO_STDERR
+              !!! 1 retriable errors detected in this deployment:
+              * stdout non-deterministic error
+
+              !!! Retry deployment due to non-deterministic error (4 remaining attempts)...
+              !!! 1 retriable errors detected in this deployment:
+              * stdout non-deterministic error
+
+
+              !!! Retry deployment due to non-deterministic error (3 remaining attempts)...
+
+            EO_STDERR
+          ]
+        )
+      end
+    end
+
+    it 'stops restarting deployments for a non-deterministic error that became deterministic, even if retries were remaining' do
+      with_platform_to_retry_deploy do
+        test_deployer.nbr_retries_on_error = 5
+        mock_deploys_with [
+          [1, "Error: This is a stdout non-deterministic error 1\nDeploy failed", ''],
+          [1, "Error: This is a stdout non-deterministic error 2\nDeploy failed", ''],
+          [1, "Error: This is a stdout deterministic error 3\nDeploy failed", '']
+        ]
+        expect(test_deployer.deploy_on('node')).to eq(
+          'node' => [
+            1,
+            <<~EO_STDOUT,
+              Error: This is a stdout non-deterministic error 1
+              Deploy failed
+              Deployment exit status code: 1
+              !!! Retry deployment due to non-deterministic error (4 remaining attempts)...
+              Error: This is a stdout non-deterministic error 2
+              Deploy failed
+
+              Deployment exit status code: 1
+              !!! Retry deployment due to non-deterministic error (3 remaining attempts)...
+              Error: This is a stdout deterministic error 3
+              Deploy failed
+            EO_STDOUT
+            <<~EO_STDERR
+              !!! 1 retriable errors detected in this deployment:
+              * stdout non-deterministic error
+
+              !!! Retry deployment due to non-deterministic error (4 remaining attempts)...
+              !!! 1 retriable errors detected in this deployment:
+              * stdout non-deterministic error
+
+
+              !!! Retry deployment due to non-deterministic error (3 remaining attempts)...
+
+            EO_STDERR
+          ]
+        )
+      end
+    end
+
+    it 'does not restart deployment for a deterministic error' do
+      with_platform_to_retry_deploy do
+        test_deployer.nbr_retries_on_error = 5
+        mock_deploys_with [
+          [1, "Error: This is a stdout deterministic error\nDeploy failed\n", '']
+        ]
+        expect(test_deployer.deploy_on('node')).to eq(
+          'node' => [
+            1,
+            <<~EO_STDOUT,
+              Error: This is a stdout deterministic error
+              Deploy failed
+            EO_STDOUT
+            ''
+          ]
+        )
+      end
+    end
+
+    it 'does not restart deployment for a non-deterministic error logged during a successful deploy' do
+      with_platform_to_retry_deploy do
+        test_deployer.nbr_retries_on_error = 5
+        mock_deploys_with [
+          [0, "Error: This is a stdout non-deterministic error\nDeploy failed\n", '']
+        ]
+        expect(test_deployer.deploy_on('node')).to eq(
+          'node' => [
+            0,
+            <<~EO_STDOUT,
+              Error: This is a stdout non-deterministic error
+              Deploy failed
+            EO_STDOUT
+            ''
+          ]
+        )
+      end
+    end
+
+    it 'does not restart deployment for a non-deterministic error if retries are 0' do
+      with_platform_to_retry_deploy do
+        test_deployer.nbr_retries_on_error = 0
+        mock_deploys_with [
+          [1, "Error: This is a stdout non-deterministic error\nDeploy failed\n", '']
+        ]
+        expect(test_deployer.deploy_on('node')).to eq(
+          'node' => [
+            1,
+            <<~EO_STDOUT,
+              Error: This is a stdout non-deterministic error
+              Deploy failed
+            EO_STDOUT
+            ''
+          ]
+        )
+      end
+    end
+
+    it 'restarts deployment for non-deterministic errors with a limited amount of retries' do
+      with_platform_to_retry_deploy do
+        test_deployer.nbr_retries_on_error = 2
+        mock_deploys_with [
+          [1, "Error: This is a stdout non-deterministic error 1\nDeploy failed", ''],
+          [1, "Error: This is a stdout non-deterministic error 2\nDeploy failed", ''],
+          [1, "Error: This is a stdout non-deterministic error 3\nDeploy failed", '']
+        ]
+        expect(test_deployer.deploy_on('node')).to eq(
+          'node' => [
+            1,
+            <<~EO_STDOUT,
+              Error: This is a stdout non-deterministic error 1
+              Deploy failed
+              Deployment exit status code: 1
+              !!! Retry deployment due to non-deterministic error (1 remaining attempts)...
+              Error: This is a stdout non-deterministic error 2
+              Deploy failed
+
+              Deployment exit status code: 1
+              !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+              Error: This is a stdout non-deterministic error 3
+              Deploy failed
+            EO_STDOUT
+            <<~EO_STDERR
+              !!! 1 retriable errors detected in this deployment:
+              * stdout non-deterministic error
+
+              !!! Retry deployment due to non-deterministic error (1 remaining attempts)...
+              !!! 1 retriable errors detected in this deployment:
+              * stdout non-deterministic error
+
+
+              !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+
+            EO_STDERR
+          ]
+        )
+      end
+    end
+
+    it 'restarts deployment for non-deterministic errors only on nodes needing it' do
+      with_platform_to_retry_deploy(
+        nodes_info: {
+          nodes: {
+            'node1' => { services: %w[service] },
+            'node2' => { services: %w[service] },
+            'node3' => { services: %w[service] },
+            'node4' => { services: %w[service] }
+          }
+        }
+      ) do
+        test_deployer.nbr_retries_on_error = 2
+        # Some nodes deploy successfully,
+        # others have deterministic errors,
+        # others have non-deterministic errors being corrected
+        # others have non-deterministic errors not being corrected
+        mock_deploys_with [
+          {
+            'node1' => [1, "Error: This is a stdout non-deterministic error\n[node1] Deploy failed\n", ''],
+            'node2' => [0, '[node2] Deploy ok', ''],
+            'node3' => [1, "Error: This is a stdout non-deterministic error\n[node3] Deploy failed\n", ''],
+            'node4' => [1, "Error: This is a stdout non-deterministic error\n[node4] Deploy failed\n", '']
+          },
+          {
+            'node1' => [0, '[node1] Deploy ok', ''],
+            'node3' => [1, "Error: This is a stdout deterministic error\n[node3] Deploy failed\n", ''],
+            'node4' => [1, "Error: This is a stdout non-deterministic error\n[node4] Deploy failed\n", '']
+          },
+          {
+            'node4' => [1, "Error: This is a stdout non-deterministic error\n[node4] Deploy failed\n", '']
+          }
+        ]
+        expect(test_deployer.deploy_on(%w[node1 node2 node3 node4])).to eq(
+          'node1' => [
+            0,
+            <<~EO_STDOUT,
+              Error: This is a stdout non-deterministic error
+              [node1] Deploy failed
+
+              Deployment exit status code: 1
+              !!! Retry deployment due to non-deterministic error (1 remaining attempts)...
+              [node1] Deploy ok
+            EO_STDOUT
+            <<~EO_STDERR
+              !!! 1 retriable errors detected in this deployment:
+              * stdout non-deterministic error
+
+              !!! Retry deployment due to non-deterministic error (1 remaining attempts)...
+
+            EO_STDERR
+          ],
+          'node2' => [
+            0,
+            '[node2] Deploy ok',
+            ''
+          ],
+          'node3' => [
+            1,
+            <<~EO_STDOUT,
+              Error: This is a stdout non-deterministic error
+              [node3] Deploy failed
+
+              Deployment exit status code: 1
+              !!! Retry deployment due to non-deterministic error (1 remaining attempts)...
+              Error: This is a stdout deterministic error
+              [node3] Deploy failed
+
+            EO_STDOUT
+            <<~EO_STDERR
+              !!! 1 retriable errors detected in this deployment:
+              * stdout non-deterministic error
+
+              !!! Retry deployment due to non-deterministic error (1 remaining attempts)...
+
+            EO_STDERR
+          ],
+          'node4' => [
+            1,
+            <<~EO_STDOUT,
+              Error: This is a stdout non-deterministic error
+              [node4] Deploy failed
+
+              Deployment exit status code: 1
+              !!! Retry deployment due to non-deterministic error (1 remaining attempts)...
+              Error: This is a stdout non-deterministic error
+              [node4] Deploy failed
+
+
+              Deployment exit status code: 1
+              !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+              Error: This is a stdout non-deterministic error
+              [node4] Deploy failed
+
+            EO_STDOUT
+            <<~EO_STDERR
+              !!! 1 retriable errors detected in this deployment:
+              * stdout non-deterministic error
+
+              !!! Retry deployment due to non-deterministic error (1 remaining attempts)...
+              !!! 1 retriable errors detected in this deployment:
+              * stdout non-deterministic error
+
+
+              !!! Retry deployment due to non-deterministic error (0 remaining attempts)...
+
+            EO_STDERR
+          ]
+        )
+      end
+    end
+
+  end
+
+  context 'when checking secrets handling' do
+
+    it 'calls secrets readers only for nodes and services to be deployed and merges their secrets' do
+      register_plugins(
+        :secrets_reader,
+        {
+          secrets_reader_1: HybridPlatformsConductorTest::TestSecretsReaderPlugin,
+          secrets_reader_2: HybridPlatformsConductorTest::TestSecretsReaderPlugin,
+          secrets_reader_3: HybridPlatformsConductorTest::TestSecretsReaderPlugin
+        }
+      )
+      with_platform_to_deploy(
+        nodes_info: {
+          nodes: {
+            'node1' => { services: %w[service1 service2] },
+            'node2' => { services: %w[service2 service3] },
+            'node3' => { services: %w[service3] },
+            'node4' => { services: %w[service1 service3] }
+          }
+        },
+        expect_services_to_deploy: {
+          'node1' => %w[service1 service2],
+          'node2' => %w[service2 service3],
+          'node3' => %w[service3]
+        },
+        expect_secrets: {
+          'node1' => {
+            'service1' => {
+              'secrets_reader_1' => 'Secret value',
+              'secrets_reader_2' => 'Secret value'
+            },
+            'service2' => {
+              'secrets_reader_1' => 'Secret value',
+              'secrets_reader_2' => 'Secret value'
+            }
+          },
+          'node2' => {
+            'service2' => {
+              'secrets_reader_1' => 'Secret value',
+              'secrets_reader_2' => 'Secret value',
+              'secrets_reader_3' => 'Secret value'
+            },
+            'service3' => {
+              'secrets_reader_1' => 'Secret value',
+              'secrets_reader_2' => 'Secret value',
+              'secrets_reader_3' => 'Secret value'
+            }
+          },
+          'node3' => {
+            'service3' => {
+              'secrets_reader_1' => 'Secret value',
+              'secrets_reader_2' => 'Secret value'
+            }
+          }
+        },
+        additional_config: <<~EO_CONFIG
+          read_secrets_from %i[secrets_reader_1 secrets_reader_2]
+          for_nodes('node2') { read_secrets_from :secrets_reader_3 }
+        EO_CONFIG
+      ) do
+        HybridPlatformsConductorTest::TestSecretsReaderPlugin.deployer = test_deployer
+        expect(test_deployer.deploy_on(%w[node1 node2 node3])).to eq(
+          'node1' => expected_deploy_result,
+          'node2' => expected_deploy_result,
+          'node3' => expected_deploy_result
+        )
+        expect(HybridPlatformsConductorTest::TestSecretsReaderPlugin.calls).to eq [
+          { instance: :secrets_reader_1, node: 'node1', service: 'service1' },
+          { instance: :secrets_reader_1, node: 'node1', service: 'service2' },
+          { instance: :secrets_reader_2, node: 'node1', service: 'service1' },
+          { instance: :secrets_reader_2, node: 'node1', service: 'service2' },
+          { instance: :secrets_reader_1, node: 'node2', service: 'service2' },
+          { instance: :secrets_reader_1, node: 'node2', service: 'service3' },
+          { instance: :secrets_reader_2, node: 'node2', service: 'service2' },
+          { instance: :secrets_reader_2, node: 'node2', service: 'service3' },
+          { instance: :secrets_reader_3, node: 'node2', service: 'service2' },
+          { instance: :secrets_reader_3, node: 'node2', service: 'service3' },
+          { instance: :secrets_reader_1, node: 'node3', service: 'service3' },
+          { instance: :secrets_reader_2, node: 'node3', service: 'service3' }
+        ]
+      end
+    end
+
+    it 'merges secrets having same values' do
+      register_plugins(
+        :secrets_reader,
+        {
+          secrets_reader_1: HybridPlatformsConductorTest::TestSecretsReaderPlugin,
+          secrets_reader_2: HybridPlatformsConductorTest::TestSecretsReaderPlugin
+        }
+      )
+      with_platform_to_deploy(
+        nodes_info: {
+          nodes: {
+            'node1' => { services: %w[service1] },
+            'node2' => { services: %w[service2] }
+          }
+        },
+        expect_secrets: {
+          'global1' => 'value1',
+          'global2' => 'value2',
+          'global3' => 'value3',
+          'global4' => 'value4'
+        },
+        additional_config: <<~EO_CONFIG
+          read_secrets_from :secrets_reader_1
+          for_nodes('node2') { read_secrets_from :secrets_reader_2 }
+        EO_CONFIG
+      ) do
+        HybridPlatformsConductorTest::TestSecretsReaderPlugin.deployer = test_deployer
+        HybridPlatformsConductorTest::TestSecretsReaderPlugin.mocked_secrets = {
+          'node1' => {
+            'service1' => {
+              secrets_reader_1: {
+                'global1' => 'value1',
+                'global2' => 'value2'
+              }
+            }
+          },
+          'node2' => {
+            'service2' => {
+              secrets_reader_1: {
+                'global2' => 'value2',
+                'global3' => 'value3'
+              },
+              secrets_reader_2: {
+                'global3' => 'value3',
+                'global4' => 'value4'
+              }
+            }
+          }
+        }
+        expect(test_deployer.deploy_on(%w[node1 node2])).to eq(
+          'node1' => expected_deploy_result,
+          'node2' => expected_deploy_result
+        )
+        expect(HybridPlatformsConductorTest::TestSecretsReaderPlugin.calls).to eq [
+          { instance: :secrets_reader_1, node: 'node1', service: 'service1' },
+          { instance: :secrets_reader_1, node: 'node2', service: 'service2' },
+          { instance: :secrets_reader_2, node: 'node2', service: 'service2' }
+        ]
+      end
+    end
+
+    it 'fails when merging secrets having different values' do
+      register_plugins(
+        :secrets_reader,
+        {
+          secrets_reader_1: HybridPlatformsConductorTest::TestSecretsReaderPlugin,
+          secrets_reader_2: HybridPlatformsConductorTest::TestSecretsReaderPlugin
+        }
+      )
+      with_platform_to_deploy(
+        nodes_info: {
+          nodes: {
+            'node1' => { services: %w[service1] },
+            'node2' => { services: %w[service2] }
+          }
+        },
+        expect_deploy_allowed: false,
+        expect_package: false,
+        expect_prepare_for_deploy: false,
+        expect_connections_to_nodes: false,
+        additional_config: <<~EO_CONFIG
+          read_secrets_from :secrets_reader_1
+          for_nodes('node2') { read_secrets_from :secrets_reader_2 }
+        EO_CONFIG
+      ) do
+        HybridPlatformsConductorTest::TestSecretsReaderPlugin.deployer = test_deployer
+        HybridPlatformsConductorTest::TestSecretsReaderPlugin.mocked_secrets = {
+          'node1' => {
+            'service1' => {
+              secrets_reader_1: {
+                'global1' => 'value1',
+                'global2' => 'value2'
+              }
+            }
+          },
+          'node2' => {
+            'service2' => {
+              secrets_reader_1: {
+                'global2' => 'value2',
+                'global3' => {
+                  'sub_key' => 'value3'
+                }
+              },
+              secrets_reader_2: {
+                'global3' => {
+                  'sub_key' => 'Other value'
+                },
+                'global4' => 'value4'
+              }
+            }
+          }
+        }
+        expect { test_deployer.deploy_on(%w[node1 node2]) }.to raise_error 'Secret set at path global3->sub_key by secrets_reader_2 for service service2 on node node2 has conflicting values (set debug for value details).'
+        expect(HybridPlatformsConductorTest::TestSecretsReaderPlugin.calls).to eq [
+          { instance: :secrets_reader_1, node: 'node1', service: 'service1' },
+          { instance: :secrets_reader_1, node: 'node2', service: 'service2' },
+          { instance: :secrets_reader_2, node: 'node2', service: 'service2' }
+        ]
+      end
+    end
+
+    it 'does not call secrets readers when secrets are overridden' do
+      register_plugins(
+        :secrets_reader,
+        {
+          secrets_reader_1: HybridPlatformsConductorTest::TestSecretsReaderPlugin,
+          secrets_reader_2: HybridPlatformsConductorTest::TestSecretsReaderPlugin,
+          secrets_reader_3: HybridPlatformsConductorTest::TestSecretsReaderPlugin
+        }
+      )
+      with_platform_to_deploy(
+        nodes_info: {
+          nodes: {
+            'node1' => { services: %w[service1] },
+            'node2' => { services: %w[service2] },
+            'node3' => { services: %w[service3] }
+          }
+        },
+        expect_secrets: {
+          'overridden_secrets' => 'value'
+        },
+        additional_config: <<~EO_CONFIG
+          read_secrets_from %i[secrets_reader_1 secrets_reader_2]
+          for_nodes('node2') { read_secrets_from :secrets_reader_3 }
+        EO_CONFIG
+      ) do
+        HybridPlatformsConductorTest::TestSecretsReaderPlugin.deployer = test_deployer
+        test_deployer.override_secrets('overridden_secrets' => 'value')
+        expect(test_deployer.deploy_on(%w[node1 node2 node3])).to eq(
+          'node1' => expected_deploy_result,
+          'node2' => expected_deploy_result,
+          'node3' => expected_deploy_result
+        )
+        expect(HybridPlatformsConductorTest::TestSecretsReaderPlugin.calls).to eq []
+      end
+    end
+
+  end
+
+end