hybrid_platforms_conductor 32.3.6

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connection_spec.rb

@@ -0,0 +1,212 @@
+describe HybridPlatformsConductor::ActionsExecutor do
+
+  context 'checking connections handling' do
+
+    # Get a test platform to test connection, using the test_connector
+    #
+    # Parameters::
+    # * Proc: Code called with platform setup
+    #   * Parameters::
+    #     * *repository* (String): Repository where the platform has been setup
+    def with_test_platform_for_connections
+      with_test_platform_for_executor(nodes: {
+        'node1' => {},
+        'node2' => {},
+        'node3' => {},
+        'node4' => {}
+      }) do |repository|
+        yield repository
+      end
+    end
+
+    it 'connects on a node before executing actions needing connection' do
+      with_test_platform_for_connections do
+        test_actions_executor.connector(:test_connector).accept_nodes = ['node1']
+        test_actions_executor.execute_actions('node1' => { test_action: { need_connector: true } })
+        expect(action_executions).to eq [{ node: 'node1', message: 'Action executed' }]
+        expect(test_actions_executor.connector(:test_connector).calls).to eq [
+         [:connectable_nodes_from, ['node1']],
+         [:with_connection_to, ['node1'], { no_exception: true }]
+        ]
+      end
+    end
+
+    it 'returns an error when no connector can connect to the needed node' do
+      with_test_platform_for_connections do
+        expect(test_actions_executor.execute_actions('node1' => { test_action: { need_connector: true } })).to eq(
+          'node1' => [:no_connector, '', 'Unable to get a connector to node1']
+        )
+      end
+    end
+
+    it 'connects on several nodes before executing actions needing connection' do
+      with_test_platform_for_connections do
+        test_actions_executor.connector(:test_connector).accept_nodes = %w[node1 node2 node3 node4]
+        test_actions_executor.execute_actions(%w[node1 node2 node3 node4] => { test_action: { need_connector: true } })
+        expect(action_executions).to eq [
+          { node: 'node1', message: 'Action executed' },
+          { node: 'node2', message: 'Action executed' },
+          { node: 'node3', message: 'Action executed' },
+          { node: 'node4', message: 'Action executed' }
+        ]
+        expect(test_actions_executor.connector(:test_connector).calls).to eq [
+         [:connectable_nodes_from, %w[node1 node2 node3 node4]],
+         [:with_connection_to, %w[node1 node2 node3 node4], { no_exception: true }]
+        ]
+      end
+    end
+
+    it 'connects on several nodes before executing actions needing connection only on nodes that could be connected' do
+      with_test_platform_for_connections do
+        test_actions_executor.connector(:test_connector).accept_nodes = %w[node1 node2 node3 node4]
+        test_actions_executor.connector(:test_connector).connected_nodes = %w[node1 node2 node4]
+        test_actions_executor.execute_actions(%w[node1 node2 node3 node4] => { test_action: { need_connector: true } })
+        expect(action_executions).to eq [
+          { node: 'node1', message: 'Action executed' },
+          { node: 'node2', message: 'Action executed' },
+          { node: 'node4', message: 'Action executed' }
+        ]
+        expect(test_actions_executor.connector(:test_connector).calls).to eq [
+         [:connectable_nodes_from, %w[node1 node2 node3 node4]],
+         [:with_connection_to, %w[node1 node2 node3 node4], { no_exception: true }]
+        ]
+      end
+    end
+
+    it 'connects only on nodes having an action needing connection' do
+      with_test_platform_for_connections do
+        test_actions_executor.connector(:test_connector).accept_nodes = %w[node1 node2 node3 node4]
+        test_actions_executor.execute_actions(
+          %w[node1 node3] => { test_action: { need_connector: true } },
+          %w[node2 node4] => { test_action: { need_connector: false } }
+        )
+        expect(action_executions).to eq [
+          { node: 'node1', message: 'Action executed' },
+          { node: 'node2', message: 'Action executed' },
+          { node: 'node3', message: 'Action executed' },
+          { node: 'node4', message: 'Action executed' }
+        ]
+        expect(test_actions_executor.connector(:test_connector).calls).to eq [
+         [:connectable_nodes_from, %w[node1 node3]],
+         [:with_connection_to, %w[node1 node3], { no_exception: true }]
+        ]
+      end
+    end
+
+    it 'does not ask for any connection if actions don\'t need remote' do
+      with_test_platform_for_connections do
+        test_actions_executor.connector(:test_connector).accept_nodes = %w[node1 node2 node3 node4]
+        test_actions_executor.execute_actions(%w[node1 node2 node3 node4] => { test_action: { need_connector: false } })
+        expect(action_executions).to eq [
+          { node: 'node1', message: 'Action executed' },
+          { node: 'node2', message: 'Action executed' },
+          { node: 'node3', message: 'Action executed' },
+          { node: 'node4', message: 'Action executed' }
+        ]
+        expect(test_actions_executor.connector(:test_connector).calls).to eq []
+      end
+    end
+
+    it 'uses proper connectors for each node needing connection' do
+      with_test_platform_for_connections do
+        test_actions_executor.connector(:test_connector).accept_nodes = %w[node1 node3]
+        test_actions_executor.connector(:test_connector_2).accept_nodes = %w[node2 node4]
+        test_actions_executor.execute_actions(%w[node1 node2 node3 node4] => { test_action: { need_connector: true } })
+        expect(action_executions).to eq [
+          { node: 'node1', message: 'Action executed' },
+          { node: 'node2', message: 'Action executed' },
+          { node: 'node3', message: 'Action executed' },
+          { node: 'node4', message: 'Action executed' }
+        ]
+        expect(test_actions_executor.connector(:test_connector).calls).to eq [
+         [:connectable_nodes_from, %w[node1 node2 node3 node4]],
+         [:with_connection_to, %w[node1 node3], { no_exception: true }]
+        ]
+        expect(test_actions_executor.connector(:test_connector_2).calls).to eq [
+         [:connectable_nodes_from, %w[node2 node4]],
+         [:with_connection_to, %w[node2 node4], { no_exception: true }]
+        ]
+      end
+    end
+
+    it 'uses the first connector available to connect on nodes' do
+      with_test_platform_for_connections do
+        test_actions_executor.connector(:test_connector).accept_nodes = %w[node1 node2 node3]
+        test_actions_executor.connector(:test_connector_2).accept_nodes = %w[node2 node4]
+        test_actions_executor.execute_actions(%w[node1 node2 node3 node4] => { test_action: { need_connector: true } })
+        expect(action_executions).to eq [
+          { node: 'node1', message: 'Action executed' },
+          { node: 'node2', message: 'Action executed' },
+          { node: 'node3', message: 'Action executed' },
+          { node: 'node4', message: 'Action executed' }
+        ]
+        expect(test_actions_executor.connector(:test_connector).calls).to eq [
+         [:connectable_nodes_from, %w[node1 node2 node3 node4]],
+         [:with_connection_to, %w[node1 node2 node3], { no_exception: true }]
+        ]
+        expect(test_actions_executor.connector(:test_connector_2).calls).to eq [
+         [:connectable_nodes_from, %w[node4]],
+         [:with_connection_to, %w[node4], { no_exception: true }]
+        ]
+      end
+    end
+
+    it 'can prepare connections to nodes' do
+      with_test_platform_for_connections do
+        test_actions_executor.connector(:test_connector).accept_nodes = %w[node1 node3]
+        test_actions_executor.connector(:test_connector_2).accept_nodes = %w[node2 node4]
+        test_actions_executor.with_connections_prepared_to(%w[node1 node2 node3 node4]) do |connected_nodes|
+          expect(test_actions_executor.connector(:test_connector).calls).to eq [
+           [:connectable_nodes_from, %w[node1 node2 node3 node4]],
+           [:with_connection_to, %w[node1 node3], { no_exception: false }]
+          ]
+          expect(test_actions_executor.connector(:test_connector_2).calls).to eq [
+           [:connectable_nodes_from, %w[node2 node4]],
+           [:with_connection_to, %w[node2 node4], { no_exception: false }]
+          ]
+          expect(connected_nodes).to eq(
+            'node1' => test_actions_executor.connector(:test_connector),
+            'node2' => test_actions_executor.connector(:test_connector_2),
+            'node3' => test_actions_executor.connector(:test_connector),
+            'node4' => test_actions_executor.connector(:test_connector_2)
+          )
+        end
+      end
+    end
+
+    it 'fails to prepare connections to nodes when connectors are not available' do
+      with_test_platform_for_connections do
+        test_actions_executor.connector(:test_connector).accept_nodes = %w[node1 node3]
+        test_actions_executor.connector(:test_connector_2).accept_nodes = %w[node2]
+        expect do
+          test_actions_executor.with_connections_prepared_to(%w[node1 node2 node3 node4]) { |connected_nodes| }
+        end.to raise_error(/The following nodes have no possible connector to them: node4/)
+      end
+    end
+
+    it 'can prepare connections to nodes ignoring failures if needed' do
+      with_test_platform_for_connections do
+        test_actions_executor.connector(:test_connector).accept_nodes = %w[node1 node3]
+        test_actions_executor.connector(:test_connector_2).accept_nodes = %w[node2]
+        test_actions_executor.with_connections_prepared_to(%w[node1 node2 node3 node4], no_exception: true) do |connected_nodes|
+          expect(test_actions_executor.connector(:test_connector).calls).to eq [
+           [:connectable_nodes_from, %w[node1 node2 node3 node4]],
+           [:with_connection_to, %w[node1 node3], { no_exception: true }]
+          ]
+          expect(test_actions_executor.connector(:test_connector_2).calls).to eq [
+           [:connectable_nodes_from, %w[node2 node4]],
+           [:with_connection_to, %w[node2], { no_exception: true }]
+          ]
+          expect(connected_nodes).to eq(
+            'node1' => test_actions_executor.connector(:test_connector),
+            'node2' => test_actions_executor.connector(:test_connector_2),
+            'node3' => test_actions_executor.connector(:test_connector),
+            'node4' => :no_connector
+          )
+        end
+      end
+    end
+
+  end
+
+end

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/cli_options_spec.rb

@@ -0,0 +1,125 @@
+describe HybridPlatformsConductor::ActionsExecutor do
+
+  context 'checking connector plugin ssh' do
+
+    context 'checking command-line interface options' do
+
+      # Setup a platform for tests
+      #
+      # Parameters::
+      # * Proc: Code called when the platform is setup
+      #   * Parameters::
+      #     * *repository* (String): Platform's repository
+      def with_test_platform_for_cli
+        with_test_platform(
+          { nodes: { 'node' => {} } },
+          false,
+          "
+            gateway :test_gateway, 'Host test_gateway'
+            gateway :test_gateway2, 'Host test_gateway2'
+          "
+        ) do |repository|
+          ENV['hpc_ssh_gateways_conf'] = 'test_gateway'
+          yield repository
+        end
+      end
+
+      # Return the connector to be tested
+      #
+      # Result::
+      # * Connector: Connector to be tested
+      def test_connector
+        test_actions_executor.connector(:ssh)
+      end
+
+      it 'selects the correct gateway user' do
+        with_test_platform_for_cli do
+          expect_actions_executor_runs([proc do |actions, timeout: nil, concurrent: false, log_to_dir: 'run_logs', log_to_stdout: true|
+            expect(test_connector.ssh_gateway_user).to eq 'another_user'
+            {}
+          end])
+          exit_code, stdout, stderr = run 'run', '--node', 'node', '--command', 'echo Hello', '--ssh-gateway-user', 'another_user'
+          expect(exit_code).to eq 0
+          expect(stdout).to eq ''
+          expect(stderr).to eq ''
+        end
+      end
+
+      it 'selects the correct gateway conf' do
+        with_test_platform_for_cli do
+          expect_actions_executor_runs([proc do |actions, timeout: nil, concurrent: false, log_to_dir: 'run_logs', log_to_stdout: true|
+            expect(test_connector.ssh_gateways_conf).to eq :test_gateway2
+            {}
+          end])
+          exit_code, stdout, stderr = run 'run', '--node', 'node', '--command', 'echo Hello', '--ssh-gateways-conf', 'test_gateway2'
+          expect(exit_code).to eq 0
+          expect(stdout).to eq ''
+          expect(stderr).to eq ''
+        end
+      end
+
+      it 'does not use the SSH control master' do
+        with_test_platform_for_cli do
+          expect_actions_executor_runs([proc do |actions, timeout: nil, concurrent: false, log_to_dir: 'run_logs', log_to_stdout: true|
+            expect(test_connector.ssh_use_control_master).to eq false
+            {}
+          end])
+          exit_code, stdout, stderr = run 'run', '--node', 'node', '--command', 'echo Hello', '--ssh-no-control-master'
+          expect(exit_code).to eq 0
+          expect(stdout).to eq ''
+          expect(stderr).to eq ''
+        end
+      end
+
+      it 'does not use strict host key checking' do
+        with_test_platform_for_cli do
+          expect_actions_executor_runs([proc do |actions, timeout: nil, concurrent: false, log_to_dir: 'run_logs', log_to_stdout: true|
+            expect(test_connector.ssh_strict_host_key_checking).to eq false
+            {}
+          end])
+          exit_code, stdout, stderr = run 'run', '--node', 'node', '--command', 'echo Hello', '--ssh-no-host-key-checking'
+          expect(exit_code).to eq 0
+          expect(stdout).to eq ''
+          expect(stderr).to eq ''
+        end
+      end
+
+      it 'uses a different SSH user name' do
+        with_test_platform_for_cli do
+          expect_actions_executor_runs([proc do |actions, timeout: nil, concurrent: false, log_to_dir: 'run_logs', log_to_stdout: true|
+            expect(test_connector.ssh_user).to eq 'ssh_new_user'
+            {}
+          end])
+          exit_code, stdout, stderr = run 'run', '--node', 'node', '--command', 'echo Hello', '--ssh-user', 'ssh_new_user'
+          expect(exit_code).to eq 0
+          expect(stdout).to eq ''
+          expect(stderr).to eq ''
+        end
+      end
+
+      it 'fails if no user name has been given, either through environment or command-line' do
+        ENV.delete 'hpc_ssh_user'
+        ENV.delete 'USER'
+        with_test_platform_for_cli do
+          expect { run 'run', '--node', 'node', '--command', 'echo Hello' }.to raise_error(RuntimeError, 'No SSH user name specified. Please use --ssh-user option or hpc_ssh_user environment variable to set it.')
+        end
+      end
+
+      it 'expects passwords to be input' do
+        with_test_platform_for_cli do
+          expect_actions_executor_runs([proc do |actions, timeout: nil, concurrent: false, log_to_dir: 'run_logs', log_to_stdout: true|
+            expect(test_connector.auth_password).to eq true
+            {}
+          end])
+          exit_code, stdout, stderr = run 'run', '--node', 'node', '--command', 'echo Hello', '--password'
+          expect(exit_code).to eq 0
+          expect(stdout).to eq ''
+          expect(stderr).to eq ''
+        end
+      end
+
+    end
+
+  end
+
+end

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/config_dsl_spec.rb

@@ -0,0 +1,50 @@
+describe HybridPlatformsConductor::ActionsExecutor do
+
+  context 'checking connector plugin ssh' do
+
+    context 'checking Config DSL extensions' do
+
+      it 'returns 1 defined gateway' do
+        with_repository do |repository|
+          with_platforms 'gateway :gateway1, \'\'' do
+            expect(test_config.known_gateways).to eq [:gateway1]
+          end
+        end
+      end
+
+      it 'returns 1 defined gateway with its content' do
+        ssh_gateway = '
+          Host gateway
+            Hostname mygateway.com
+        '
+        with_repository do |repository|
+          with_platforms "gateway :gateway1, '#{ssh_gateway}'" do
+            expect(test_config.ssh_for_gateway(:gateway1)).to eq ssh_gateway
+          end
+        end
+      end
+
+      it 'returns 1 defined gateway with its content and replacing ERB template correctly' do
+        with_repository do |repository|
+          with_platforms 'gateway :gateway1, \'Host gateway_<%= @user %>\'' do
+            expect(test_config.ssh_for_gateway(:gateway1, user: 'test_user')).to eq 'Host gateway_test_user'
+          end
+        end
+      end
+
+      it 'returns several defined gateways' do
+        with_repository do |repository|
+          with_platforms '
+            gateway :gateway1, \'\'
+            gateway :gateway2, \'\'
+          ' do
+            expect(test_config.known_gateways.sort).to eq %i[gateway1 gateway2].sort
+          end
+        end
+      end
+
+    end
+
+  end
+
+end

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connectable_nodes_spec.rb

@@ -0,0 +1,28 @@
+describe HybridPlatformsConductor::ActionsExecutor do
+
+  context 'checking connector plugin ssh' do
+
+    context 'checking connectable nodes selection' do
+
+      # Return the connector to be tested
+      #
+      # Result::
+      # * Connector: Connector to be tested
+      def test_connector
+        test_actions_executor.connector(:ssh)
+      end
+
+      it 'selects connectable nodes correctly' do
+        with_test_platform(nodes: {
+          'node1' => { meta: { host_ip: '192.168.42.42' } },
+          'node2' => {}
+        }) do
+          expect(test_connector.connectable_nodes_from(%w[node1 node2])).to eq ['node1']
+        end
+      end
+
+    end
+
+  end
+
+end

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connections_spec.rb

@@ -0,0 +1,448 @@
+describe HybridPlatformsConductor::ActionsExecutor do
+
+  context 'checking connector plugin ssh' do
+
+    context 'checking connections preparations' do
+
+      # Return the connector to be tested
+      #
+      # Result::
+      # * Connector: Connector to be tested
+      def test_connector
+        test_actions_executor.connector(:ssh)
+      end
+
+      it 'creates an SSH master to 1 node' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42' } } }) do
+          with_cmd_runner_mocked(
+            [
+              ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+              ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }]
+            ] + ssh_expected_commands_for('node' => { connection: '192.168.42.42', user: 'test_user' })
+          ) do
+            test_connector.ssh_user = 'test_user'
+            test_connector.with_connection_to(['node']) do |connected_nodes|
+              expect(connected_nodes).to eq ['node']
+            end
+          end
+        end
+      end
+
+      it 'creates SSH master to several nodes' do
+        with_test_platform(nodes: {
+          'node1' => { meta: { host_ip: '192.168.42.1' } },
+          'node2' => { meta: { host_ip: '192.168.42.2' } },
+          'node3' => { meta: { host_ip: '192.168.42.3' } }
+        }) do
+          with_cmd_runner_mocked(
+            [
+              ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+              ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }]
+            ] + ssh_expected_commands_for(
+              'node1' => { connection: '192.168.42.1', user: 'test_user' },
+              'node2' => { connection: '192.168.42.2', user: 'test_user' },
+              'node3' => { connection: '192.168.42.3', user: 'test_user' }
+            )
+          ) do
+            test_connector.ssh_user = 'test_user'
+            test_connector.with_connection_to(%w[node1 node2 node3]) do |connected_nodes|
+              expect(connected_nodes.sort).to eq %w[node1 node2 node3].sort
+            end
+          end
+        end
+      end
+
+      it 'fails when an SSH master can\'t be created' do
+        with_test_platform(nodes: {
+          'node1' => { meta: { host_ip: '192.168.42.1' } },
+          'node2' => { meta: { host_ip: '192.168.42.2' } },
+          'node3' => { meta: { host_ip: '192.168.42.3' } }
+        }) do
+          with_cmd_runner_mocked(
+            [
+              ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+              ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }]
+            ] + ssh_expected_commands_for(
+              'node1' => { connection: '192.168.42.1', user: 'test_user' },
+              'node3' => { connection: '192.168.42.3', user: 'test_user' }
+            ) + ssh_expected_commands_for(
+              {
+                'node2' => { connection: '192.168.42.2', user: 'test_user', control_master_create_error: 'Can\'t connect to 192.168.42.2' }
+              },
+              with_control_master_destroy: false
+            )
+          ) do
+            test_connector.ssh_user = 'test_user'
+            expect { test_connector.with_connection_to(%w[node1 node2 node3]) }.to raise_error(/^Error while starting SSH Control Master with .+\/ssh -o BatchMode=yes -o ControlMaster=yes -o ControlPersist=yes test_user@hpc.node2 true: Can't connect to 192.168.42.2$/)
+          end
+        end
+      end
+
+      it 'fails without throwing exception when an SSH master can\'t be created and we use no_exception' do
+        with_test_platform(nodes: {
+          'node1' => { meta: { host_ip: '192.168.42.1' } },
+          'node2' => { meta: { host_ip: '192.168.42.2' } },
+          'node3' => { meta: { host_ip: '192.168.42.3' } }
+        }) do
+          with_cmd_runner_mocked(
+            [
+              ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+              ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }]
+            ] + ssh_expected_commands_for(
+              'node1' => { connection: '192.168.42.1', user: 'test_user' },
+              'node3' => { connection: '192.168.42.3', user: 'test_user' }
+            ) + ssh_expected_commands_for(
+              {
+                'node2' => { connection: '192.168.42.2', user: 'test_user', control_master_create_error: 'Can\'t connect to 192.168.42.2' }
+              },
+              with_control_master_destroy: false
+            )
+          ) do
+            test_connector.ssh_user = 'test_user'
+            test_connector.with_connection_to(%w[node1 node2 node3], no_exception: true) do |connected_nodes|
+              expect(connected_nodes.sort).to eq %w[node1 node3].sort
+            end
+          end
+        end
+      end
+
+      it 'reuses SSH master already created to 1 node' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42' } } }) do
+          with_cmd_runner_mocked(
+            [
+              ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+              ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }]
+            ] +
+              ssh_expected_commands_for('node' => { connection: '192.168.42.42', user: 'test_user' }) +
+              ssh_expected_commands_for(
+                { 'node' => { connection: '192.168.42.42', user: 'test_user' } },
+                with_strict_host_key_checking: false,
+                with_control_master_create: false,
+                with_control_master_check: true,
+                with_control_master_destroy: false
+              )
+          ) do
+            test_connector.ssh_user = 'test_user'
+            test_connector.with_connection_to(['node']) do
+              test_connector.with_connection_to(['node']) do
+              end
+            end
+          end
+        end
+      end
+
+      it 'reuses SSH masters already created to some nodes and create new ones if needed' do
+        with_test_platform(nodes: {
+          'node1' => { meta: { host_ip: '192.168.42.1' } },
+          'node2' => { meta: { host_ip: '192.168.42.2' } },
+          'node3' => { meta: { host_ip: '192.168.42.3' } },
+          'node4' => { meta: { host_ip: '192.168.42.4' } }
+        }) do
+          with_cmd_runner_mocked(
+            [
+              ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+              ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }]
+            ] +
+              ssh_expected_commands_for(
+                'node1' => { connection: '192.168.42.1', user: 'test_user' },
+                'node3' => { connection: '192.168.42.3', user: 'test_user' }
+              ) +
+              ssh_expected_commands_for(
+                'node2' => { connection: '192.168.42.2', user: 'test_user' },
+                'node4' => { connection: '192.168.42.4', user: 'test_user' }
+              ) +
+              ssh_expected_commands_for(
+                {
+                  'node1' => { connection: '192.168.42.1', user: 'test_user' },
+                  'node3' => { connection: '192.168.42.3', user: 'test_user' }
+                },
+                with_strict_host_key_checking: false,
+                with_control_master_create: false,
+                with_control_master_check: true,
+                with_control_master_destroy: false
+              )
+          ) do
+            test_connector.ssh_user = 'test_user'
+            test_connector.with_connection_to(%w[node1 node3]) do
+              test_connector.with_connection_to(%w[node1 node2 node3 node4]) do
+              end
+            end
+          end
+        end
+      end
+
+      it 'makes sure the last client using ControlMaster destroys it, even using a different environment' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42' } } }) do
+          # 1. Current thread creates the ControlMaster.
+          # 2. Second thread connects to it.
+          # 3. Current thread releases it.
+          # 4. Second thread releases it, hence destroying it.
+          init_commands = [
+            ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+            ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }],
+          ]
+          nodes_connections_to_mock = { 'node' => { connection: '192.168.42.42', user: 'test_user' } }
+          step = 0
+          second_thread = Thread.new do
+            # Use a different environment: CmdRunner, NodesHandler, ActionsExecutor
+            second_cmd_runner = HybridPlatformsConductor::CmdRunner.new logger: logger, logger_stderr: logger
+            with_cmd_runner_mocked(
+              init_commands +
+                ssh_expected_commands_for(
+                  nodes_connections_to_mock,
+                  with_control_master_create: false,
+                  with_control_master_check: true
+                ),
+              cmd_runner: second_cmd_runner
+            ) do
+              second_config = HybridPlatformsConductor::Config.new logger: logger, logger_stderr: logger
+              second_platforms_handler = HybridPlatformsConductor::PlatformsHandler.new logger: logger, logger_stderr: logger, config: second_config, cmd_runner: second_cmd_runner
+              second_nodes_handler = HybridPlatformsConductor::NodesHandler.new logger: logger, logger_stderr: logger, config: second_config, cmd_runner: second_cmd_runner, platforms_handler: second_platforms_handler
+              second_actions_executor = HybridPlatformsConductor::ActionsExecutor.new logger: logger, logger_stderr: logger, config: second_config, cmd_runner: second_cmd_runner, nodes_handler: second_nodes_handler
+              second_actions_executor.connector(:ssh).ssh_user = 'test_user'
+              # Wait for the first thread to create ControlMaster
+              sleep 0.1 while step == 0
+              second_actions_executor.connector(:ssh).with_connection_to(['node']) do
+                step = 2
+                # Wait for the first thread to release the ControlMaster
+                sleep 0.1 while step == 2
+              end
+            end
+          end
+          with_cmd_runner_mocked(
+            init_commands +
+              ssh_expected_commands_for(nodes_connections_to_mock, with_control_master_destroy: false)
+          ) do
+            test_connector.ssh_user = 'test_user'
+            test_connector.with_connection_to(['node']) do
+              step = 1
+              # Now wait for the second thread to also acquire it
+              sleep 0.1 while step == 1
+            end
+            step = 3
+          end
+          second_thread.join
+        end
+      end
+
+      it 'does not create SSH master if asked' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42' } } }) do
+          with_cmd_runner_mocked(
+            [
+              ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+              ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }]
+            ] + ssh_expected_commands_for(
+              { 'node' => { connection: '192.168.42.42', user: 'test_user' } },
+              with_control_master_create: false,
+              with_control_master_destroy: false
+            )
+          ) do
+            test_connector.ssh_use_control_master = false
+            test_connector.ssh_user = 'test_user'
+            test_connector.with_connection_to(['node']) do
+            end
+          end
+        end
+      end
+
+      it 'does not check host keys if asked' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42' } } }) do
+          with_cmd_runner_mocked(
+            [
+              ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+              ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }]
+            ] + ssh_expected_commands_for(
+              { 'node' => { connection: '192.168.42.42', user: 'test_user' } },
+              with_strict_host_key_checking: false
+            )
+          ) do
+            test_connector.ssh_strict_host_key_checking = false
+            test_connector.ssh_user = 'test_user'
+            test_connector.with_connection_to(['node']) do
+            end
+          end
+        end
+      end
+
+      it 'does not use batch mode when passwords are to be expected' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42' } } }) do
+          with_cmd_runner_mocked(
+            [
+              ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+              ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }]
+            ] + ssh_expected_commands_for(
+              { 'node' => { connection: '192.168.42.42', user: 'test_user' } },
+              with_batch_mode: false
+            )
+          ) do
+            test_connector.auth_password = true
+            test_connector.ssh_user = 'test_user'
+            test_connector.with_connection_to(['node']) do
+            end
+          end
+        end
+      end
+
+      it 'uses sshpass to prepare connections needing passwords' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42' } } }) do
+          with_cmd_runner_mocked(
+            [
+              ['sshpass -V', proc { [0, "sshpass 1.06\n", ''] }],
+              ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+              ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }]
+            ] + ssh_expected_commands_for(
+              { 'node' => { connection: '192.168.42.42', user: 'test_user' } },
+              with_batch_mode: false
+            )
+          ) do
+            test_connector.passwords['node'] = 'PaSsWoRd'
+            test_connector.ssh_user = 'test_user'
+            test_connector.with_connection_to(['node']) do
+            end
+          end
+        end
+      end
+
+      it 'does not reuse provided SSH executables and configs' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42' } } }) do
+          with_cmd_runner_mocked(
+            [
+              ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+              ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }]
+            ] +
+              ssh_expected_commands_for('node' => { connection: '192.168.42.42', user: 'test_user' }) +
+              ssh_expected_commands_for(
+                { 'node' => { connection: '192.168.42.42', user: 'test_user' } },
+                with_strict_host_key_checking: false,
+                with_control_master_create: false,
+                with_control_master_check: true,
+                with_control_master_destroy: false
+              )
+          ) do
+            test_connector.ssh_user = 'test_user'
+            test_connector.with_connection_to(['node']) do
+              stdout = ''
+              stderr = ''
+              test_connector.prepare_for('node', nil, stdout, stderr)
+              first_ssh_exec = test_connector.ssh_exec
+              test_connector.with_connection_to(['node']) do
+                test_connector.prepare_for('node', nil, stdout, stderr)
+                expect(test_connector.ssh_exec).not_to eq first_ssh_exec
+              end
+            end
+          end
+        end
+      end
+
+      it 'cleans provided SSH executables and configs after use' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42' } } }) do
+          with_cmd_runner_mocked(
+            [
+              ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+              ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }]
+            ] +
+              ssh_expected_commands_for('node' => { connection: '192.168.42.42', user: 'test_user' }) +
+              ssh_expected_commands_for(
+                { 'node' => { connection: '192.168.42.42', user: 'test_user' } },
+                with_strict_host_key_checking: false,
+                with_control_master_create: false,
+                with_control_master_check: true,
+                with_control_master_destroy: false
+              )
+          ) do
+            ssh_exec_1 = nil
+            ssh_exec_2 = nil
+            test_connector.ssh_user = 'test_user'
+            test_connector.with_connection_to(['node']) do
+              stdout = ''
+              stderr = ''
+              test_connector.prepare_for('node', nil, stdout, stderr)
+              ssh_exec_1 = test_connector.ssh_exec
+              test_connector.with_connection_to(['node']) do
+                test_connector.prepare_for('node', nil, stdout, stderr)
+                ssh_exec_2 = test_connector.ssh_exec
+                expect(File.exist?(ssh_exec_1)).to eq true
+                expect(File.exist?(ssh_exec_2)).to eq true
+              end
+              expect(File.exist?(ssh_exec_1)).to eq true
+              expect(File.exist?(ssh_exec_2)).to eq false
+            end
+            expect(File.exist?(ssh_exec_1)).to eq false
+            expect(File.exist?(ssh_exec_2)).to eq false
+          end
+        end
+      end
+
+      it 'creates an SSH master to 1 node even when there is a stalled ControlMaster file' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42' } } }) do
+          with_cmd_runner_mocked(
+            [
+              ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+              ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }]
+            ] + ssh_expected_commands_for('node' => { connection: '192.168.42.42', user: 'test_user' })
+          ) do
+            test_connector.ssh_user = 'test_user'
+            # Fake a ControlMaster file that is stalled
+            File.write(test_actions_executor.connector(:ssh).send(:control_master_file, '192.168.42.42', '22', 'test_user'), '')
+            test_connector.with_connection_to(['node']) do
+            end
+          end
+        end
+      end
+
+      it 'creates an SSH master to 1 node even when there is a left-over user for the ControlMaster file that has not been unregistered' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42' } } }) do
+          with_cmd_runner_mocked(
+            [
+              ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+              ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }]
+            ] + ssh_expected_commands_for('node' => { connection: '192.168.42.42', user: 'test_user' })
+          ) do
+            test_connector.ssh_user = 'test_user'
+            # Fake a user that was not cleaned correctly
+            File.write('/tmp/hpc_ssh/test_user.node.users', "unregistered_user\n")
+            test_connector.with_connection_to(['node']) do
+            end
+          end
+        end
+      end
+
+      it 'retries when the remote node is booting up' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42' } } }) do
+          nbr_boot_messages = 0
+          with_cmd_runner_mocked(
+            [
+              ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+              ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }],
+            ] +
+              [[
+                /^.+\/ssh -o BatchMode=yes -o ControlMaster=yes -o ControlPersist=yes test_user@hpc\.node true$/,
+                proc do
+                  nbr_boot_messages += 1
+                  [255, '', "System is booting up. See pam_nologin(8)\nAuthentication failed.\n"]
+                end
+              ]] * 3 +
+              ssh_expected_commands_for('node' => { connection: '192.168.42.42', user: 'test_user' })
+          ) do
+            test_connector.ssh_user = 'test_user'
+            # To speed up the test, alter the wait time between retries.
+            old_wait = HybridPlatformsConductor::HpcPlugins::Connector::Ssh.const_get(:WAIT_TIME_FOR_BOOT)
+            begin
+              HybridPlatformsConductor::HpcPlugins::Connector::Ssh.send(:remove_const, :WAIT_TIME_FOR_BOOT)
+              HybridPlatformsConductor::HpcPlugins::Connector::Ssh.const_set(:WAIT_TIME_FOR_BOOT, 1)
+              test_connector.with_connection_to(['node']) do
+              end
+              expect(nbr_boot_messages).to eq 3
+            ensure
+              HybridPlatformsConductor::HpcPlugins::Connector::Ssh.send(:remove_const, :WAIT_TIME_FOR_BOOT)
+              HybridPlatformsConductor::HpcPlugins::Connector::Ssh.const_set(:WAIT_TIME_FOR_BOOT, old_wait)
+            end
+          end
+        end
+      end
+
+    end
+
+  end
+
+end