hybrid_platforms_conductor 32.6.0 → 32.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6b6259b1f05bb082b4f2c6820f74b6c90dad15f5868a311337429b80d233500
-  data.tar.gz: c5f58e3ba104ba2365addea64ab08c43a4fe9797430c950bf01258eb7bfcdc1b
+  metadata.gz: 539ccbcddce0dda10361f172eb18b5db378aa6e299117e4c25f3416a0e5eb3f8
+  data.tar.gz: 9e3f5c649cbc70399b462698d80794a56ce4e141b552d88df809373cb0e33b3c
 SHA512:
-  metadata.gz: b079e89a17630ed994614bc24e76ba2154ae52fcaf3712feaaef9e823487d933d6b33b87b3aa253e8d11270d5765d061349c4a37f467e81d729d994ed1f7b72e
-  data.tar.gz: 4a89e197bed2c87dbc10d66d8a3f6240c68deb7c2799962be282634c3c109b653c2a66725baa5ca44e38fd0bf1ec992148720047668c1c7786bd494a8e3836e7
+  metadata.gz: f665a9dd0e2aed5c3e78fee2840db66eaa67374e81609042c42687ec65a86621ad1d6f5d0c25578526b19b8f3f534dc1c4330b5956348b83928138d0a6e30509
+  data.tar.gz: 4d7cc60f030cfe9ab7c24053b4c1fa2ef440c95a8170685157de7c4f25d54fc8d2b469303baffb98214d58e39613ae116bf42df0991832e68b8046c9b7eccae1

data/lib/hybrid_platforms_conductor/deployer.rb

@@ -311,13 +311,24 @@ module HybridPlatformsConductor
           environment: environment,
           logger: @logger,
           logger_stderr: @logger_stderr,
-          config: @config,
+          config: sub_executable.config,
           cmd_runner: @cmd_runner,
           # Here we use the NodesHandler that will be bound to the sub-Deployer only, as the node's metadata might be modified by the Provisioner.
           nodes_handler: sub_executable.nodes_handler,
           actions_executor: @actions_executor
         )
         instance.with_running_instance(stop_on_exit: true, destroy_on_exit: !reuse_instance, port: 22) do
+          # Test-provisioned nodes have SSH Session Exec capabilities
+          sub_executable.nodes_handler.override_metadata_of node, :ssh_session_exec, 'true'
+          # Test-provisioned nodes use default sudo
+          sub_executable.config.sudo_procs.replace(sub_executable.config.sudo_procs.map do |sudo_proc_info|
+            {
+              nodes_selectors_stack: sudo_proc_info[:nodes_selectors_stack].map do |nodes_selector|
+                @nodes_handler.select_nodes(nodes_selector).select { |selected_node| selected_node != node }
+              end,
+              sudo_proc: sudo_proc_info[:sudo_proc]
+            }
+          end)
           actions_executor = sub_executable.actions_executor
           deployer = sub_executable.deployer
           # Setup test environment for this container

data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb

@@ -233,7 +233,13 @@ module HybridPlatformsConductor
         # Parameters::
         # * *bash_cmds* (String): Bash commands to execute
         def remote_bash(bash_cmds)
-          ssh_cmd = "#{ssh_exec} #{ssh_url} /bin/bash <<'EOF'\n#{bash_cmds}\nEOF"
+          ssh_cmd =
+            if @nodes_handler.get_ssh_session_exec_of(@node) == 'false'
+              # When ExecSession is disabled we need to use stdin directly
+              "{ cat | #{ssh_exec} #{ssh_url} -T; } <<'EOF'\n#{bash_cmds}\nEOF"
+            else
+              "#{ssh_exec} #{ssh_url} /bin/bash <<'EOF'\n#{bash_cmds}\nEOF"
+            end
           # Due to a limitation of Process.spawn, each individual argument is limited to 128KB of size.
           # Therefore we need to make sure that if bash_cmds exceeds MAX_CMD_ARG_LENGTH bytes (considering EOF chars) then we use an intermediary shell script to store the commands.
           if bash_cmds.size > MAX_CMD_ARG_LENGTH
@@ -290,25 +296,30 @@ module HybridPlatformsConductor
         # * *owner* (String or nil): Owner to be used when copying the files, or nil for current one [default: nil]
         # * *group* (String or nil): Group to be used when copying the files, or nil for current one [default: nil]
         def remote_copy(from, to, sudo: false, owner: nil, group: nil)
-          run_cmd <<~EOS
-            cd #{File.dirname(from)} && \
-            tar \
-              --create \
-              --gzip \
-              --file - \
-              #{owner.nil? ? '' : "--owner #{owner}"} \
-              #{group.nil? ? '' : "--group #{group}"} \
-              #{File.basename(from)} | \
-            #{ssh_exec} \
-              #{ssh_url} \
-              \"#{sudo ? "#{@nodes_handler.sudo_on(@node)} " : ''}tar \
-                --extract \
-                --gunzip \
+          if @nodes_handler.get_ssh_session_exec_of(@node) == 'false'
+            # We don't have ExecSession, so don't use ssh, but scp instead.
+            run_cmd "scp -S #{ssh_exec} #{from} #{ssh_url}:#{to}"
+          else
+            run_cmd <<~EOS
+              cd #{File.dirname(from)} && \
+              tar \
+                --create \
+                --gzip \
                 --file - \
-                --directory #{to} \
-                --owner root \
-              \"
-          EOS
+                #{owner.nil? ? '' : "--owner #{owner}"} \
+                #{group.nil? ? '' : "--group #{group}"} \
+                #{File.basename(from)} | \
+              #{ssh_exec} \
+                #{ssh_url} \
+                \"#{sudo ? "#{@nodes_handler.sudo_on(@node)} " : ''}tar \
+                  --extract \
+                  --gunzip \
+                  --file - \
+                  --directory #{to} \
+                  --owner root \
+                \"
+            EOS
+          end
         end
 
         # Get the ssh executable to be used when connecting to the current node
@@ -490,31 +501,46 @@ module HybridPlatformsConductor
                     ssh_url = "hpc.#{node}"
                     if current_users.empty?
                       log_debug "[ ControlMaster - #{ssh_url} ] - Creating SSH ControlMaster..."
-                      # Create the control master
-                      ssh_control_master_start_cmd = "#{ssh_exec}#{@passwords.key?(node) || @auth_password ? '' : ' -o BatchMode=yes'} -o ControlMaster=yes -o ControlPersist=yes #{ssh_url} true"
                       exit_status = nil
-                      idx_try = 0
-                      loop do
-                        stderr = nil
-                        exit_status, _stdout, stderr = @cmd_runner.run_cmd ssh_control_master_start_cmd, log_to_stdout: log_debug?, no_exception: true, timeout: timeout
-                        if exit_status == 0
-                          break
-                        elsif stderr =~ /System is booting up/
-                          if idx_try == MAX_RETRIES_FOR_BOOT
-                            if no_exception
-                              break
-                            else
-                              raise ActionsExecutor::ConnectionError, "Tried #{idx_try} times to create SSH Control Master with #{ssh_control_master_start_cmd} but system says it's booting up."
+                      if @nodes_handler.get_ssh_session_exec_of(node) == 'false'
+                        # Here we have to create a ControlMaster using an interactive session, as the SSH server prohibits ExecSession, and so command executions.
+                        # We'll do that using another terminal spawned in the background.
+                        Thread.new do
+                          log_debug "[ ControlMaster - #{ssh_url} ] - Spawn interactive ControlMaster in separate terminal"
+                          @cmd_runner.run_cmd "xterm -e '#{ssh_exec} -o ControlMaster=yes -o ControlPersist=yes #{ssh_url}'", log_to_stdout: log_debug?
+                          log_debug "[ ControlMaster - #{ssh_url} ] - Separate interactive ControlMaster closed"
+                        end
+                        out 'External ControlMaster has been spawned.'
+                        out 'Please login into it, keep its session opened and press enter here when done...'
+                        raise "Can't spawn interactive ControlMaster to #{node} in non-interactive mode. You may want to change the hpc_interactive env variable." if ENV['hpc_interactive'] == 'false'
+                        $stdin.gets
+                        exit_status = 0
+                      else
+                        # Create the control master
+                        ssh_control_master_start_cmd = "#{ssh_exec}#{@passwords.key?(node) || @auth_password ? '' : ' -o BatchMode=yes'} -o ControlMaster=yes -o ControlPersist=yes #{ssh_url} true"
+                        idx_try = 0
+                        loop do
+                          stderr = nil
+                          exit_status, _stdout, stderr = @cmd_runner.run_cmd ssh_control_master_start_cmd, log_to_stdout: log_debug?, no_exception: true, timeout: timeout
+                          if exit_status == 0
+                            break
+                          elsif stderr =~ /System is booting up/
+                            if idx_try == MAX_RETRIES_FOR_BOOT
+                              if no_exception
+                                break
+                              else
+                                raise ActionsExecutor::ConnectionError, "Tried #{idx_try} times to create SSH Control Master with #{ssh_control_master_start_cmd} but system says it's booting up."
+                              end
                             end
+                            # Wait a bit and try again
+                            idx_try += 1
+                            log_debug "[ ControlMaster - #{ssh_url} ] - System is booting up (try ##{idx_try}). Wait #{WAIT_TIME_FOR_BOOT} seconds before trying ControlMaster's creation again."
+                            sleep WAIT_TIME_FOR_BOOT
+                          elsif no_exception
+                            break
+                          else
+                            raise ActionsExecutor::ConnectionError, "Error while starting SSH Control Master with #{ssh_control_master_start_cmd}: #{stderr.strip}"
                           end
-                          # Wait a bit and try again
-                          idx_try += 1
-                          log_debug "[ ControlMaster - #{ssh_url} ] - System is booting up (try ##{idx_try}). Wait #{WAIT_TIME_FOR_BOOT} seconds before trying ControlMaster's creation again."
-                          sleep WAIT_TIME_FOR_BOOT
-                        elsif no_exception
-                          break
-                        else
-                          raise ActionsExecutor::ConnectionError, "Error while starting SSH Control Master with #{ssh_control_master_start_cmd}: #{stderr.strip}"
                         end
                       end
                       if exit_status == 0
@@ -549,28 +575,33 @@ module HybridPlatformsConductor
                     end
                   end
                 end
+              else
+                # We have not created any ControlMaster, but still consider the nodes to be ready to connect
+                user_locks = Hash[nodes.map { |node| [node, nil]} ]
               end
               yield user_locks.keys
             ensure
-              user_locks_mutex.synchronize do
-                user_locks.each do |node, user_id|
-                  with_lock_on_control_master_for(node, user_id: user_id) do |current_users, user_id|
-                    ssh_url = "hpc.#{node}"
-                    log_warn "[ ControlMaster - #{ssh_url} ] - Current process/thread was not part of the ControlMaster users anymore whereas it should have been" unless current_users.include?(user_id)
-                    remaining_users = current_users - [user_id]
-                    if remaining_users.empty?
-                      # Stop the ControlMaster
-                      log_debug "[ ControlMaster - #{ssh_url} ] - Stopping ControlMaster..."
-                      # Dumb verbose ssh! Tricky trick to just silence what is useless.
-                      # Don't fail if the connection close fails (but still log the error), as it can be seen as only a warning: it means the connection was closed anyway.
-                      @cmd_runner.run_cmd "#{ssh_exec_for(node)} -O exit #{ssh_url} 2>&1 | grep -v 'Exit request sent.'", log_to_stdout: log_debug?, expected_code: 1, timeout: timeout, no_exception: true
-                      log_debug "[ ControlMaster - #{ssh_url} ] - ControlMaster stopped"
-                      # Uncomment if you want to test that the connection has been closed
-                      # @cmd_runner.run_cmd "#{ssh_exec_for(node)} -O check #{ssh_url}", log_to_stdout: log_debug?, expected_code: 255, timeout: timeout
-                    else
-                      log_debug "[ ControlMaster - #{ssh_url} ] - Leaving ControlMaster started as #{remaining_users.size} processes/threads are still using it."
+              if @ssh_use_control_master
+                user_locks_mutex.synchronize do
+                  user_locks.each do |node, user_id|
+                    with_lock_on_control_master_for(node, user_id: user_id) do |current_users, user_id|
+                      ssh_url = "hpc.#{node}"
+                      log_warn "[ ControlMaster - #{ssh_url} ] - Current process/thread was not part of the ControlMaster users anymore whereas it should have been" unless current_users.include?(user_id)
+                      remaining_users = current_users - [user_id]
+                      if remaining_users.empty?
+                        # Stop the ControlMaster
+                        log_debug "[ ControlMaster - #{ssh_url} ] - Stopping ControlMaster..."
+                        # Dumb verbose ssh! Tricky trick to just silence what is useless.
+                        # Don't fail if the connection close fails (but still log the error), as it can be seen as only a warning: it means the connection was closed anyway.
+                        @cmd_runner.run_cmd "#{ssh_exec_for(node)} -O exit #{ssh_url} 2>&1 | grep -v 'Exit request sent.'", log_to_stdout: log_debug?, expected_code: 1, timeout: timeout, no_exception: true
+                        log_debug "[ ControlMaster - #{ssh_url} ] - ControlMaster stopped"
+                        # Uncomment if you want to test that the connection has been closed
+                        # @cmd_runner.run_cmd "#{ssh_exec_for(node)} -O check #{ssh_url}", log_to_stdout: log_debug?, expected_code: 255, timeout: timeout
+                      else
+                        log_debug "[ ControlMaster - #{ssh_url} ] - Leaving ControlMaster started as #{remaining_users.size} processes/threads are still using it."
+                      end
+                      false
                     end
-                    false
                   end
                 end
               end

data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_deploy_and_idempotence.rb

@@ -54,17 +54,19 @@ module HybridPlatformsConductor
                 instance.stop
                 instance.with_running_instance(port: 22) do
 
-                  # ===== Deploy removes root access
-                  # Check that we can't connect with root
-                  ssh_ok = false
-                  begin
-                    Net::SSH.start(instance.ip, 'root', password: 'root_pwd', auth_methods: ['password'], verify_host_key: :never) do |ssh|
-                      ssh_ok = ssh.exec!('echo Works').strip == 'Works'
+                  unless @nodes_handler.get_root_access_allowed_of(@node) == 'true'
+                    # ===== Deploy removes root access
+                    # Check that we can't connect with root
+                    ssh_ok = false
+                    begin
+                      Net::SSH.start(instance.ip, 'root', password: 'root_pwd', auth_methods: ['password'], verify_host_key: :never) do |ssh|
+                        ssh_ok = ssh.exec!('echo Works').strip == 'Works'
+                      end
+                    rescue
                     end
-                  rescue
+                    assert_equal ssh_ok, false, 'Root can still connect on the image after deployment'
+                    # Even if we can connect using root, run the idempotence test
                   end
-                  assert_equal ssh_ok, false, 'Root can still connect on the image after deployment'
-                  # Even if we can connect using root, run the idempotence test
 
                   # ===== Idempotence
                   unless ssh_ok

data/lib/hybrid_platforms_conductor/provisioner.rb

@@ -76,6 +76,15 @@ module HybridPlatformsConductor
             # Make sure we update it.
             @nodes_handler.override_metadata_of @node, :host_ip, instance_ip
             @nodes_handler.invalidate_metadata_of @node, :host_keys
+            # Make sure the SSH transformations don't apply to this node
+            @config.ssh_connection_transforms.replace(@config.ssh_connection_transforms.map do |ssh_transform_info|
+              {
+                nodes_selectors_stack: ssh_transform_info[:nodes_selectors_stack].map do |nodes_selector|
+                  @nodes_handler.select_nodes(nodes_selector).select { |selected_node| selected_node != @node }
+                end,
+                transform: ssh_transform_info[:transform]
+              }
+            end)
           end
           wait_for_port!(port) if port
           yield

data/lib/hybrid_platforms_conductor/version.rb

@@ -1,5 +1,5 @@
 module HybridPlatformsConductor
 
-  VERSION = '32.6.0'
+  VERSION = '32.8.0'
 
 end

data/spec/hybrid_platforms_conductor_test.rb

@@ -94,6 +94,7 @@ module HybridPlatformsConductorTest
         ENV.delete 'hpc_password_for_thycotic'
         ENV.delete 'hpc_domain_for_thycotic'
         ENV.delete 'hpc_certificates'
+        ENV.delete 'hpc_interactive'
         # Set the necessary Hybrid Platforms Conductor environment variables
         ENV['hpc_ssh_user'] = 'test_user'
         HybridPlatformsConductor::ServicesHandler.packaged_deployments.clear

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connections_spec.rb

@@ -28,6 +28,45 @@ describe HybridPlatformsConductor::ActionsExecutor do
         end
       end
 
+      it 'creates an SSH master to 1 node not having Session Exec capabilities' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42', ssh_session_exec: 'false' } } }) do
+          with_cmd_runner_mocked(
+            [
+              ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+              ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }]
+            ] + ssh_expected_commands_for({ 'node' => { connection: '192.168.42.42', user: 'test_user' } }, with_session_exec: false)
+          ) do
+            test_connector.ssh_user = 'test_user'
+            test_connector.with_connection_to(['node']) do |connected_nodes|
+              expect(connected_nodes).to eq ['node']
+            end
+          end
+        end
+      end
+
+      it 'can\'t create an SSH master to 1 node not having Session Exec capabilities when hpc_interactive is false' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42', ssh_session_exec: 'false' } } }) do
+          ENV['hpc_interactive'] = 'false'
+          with_cmd_runner_mocked(
+            [
+              ['which env', proc { [0, "/usr/bin/env\n", ''] }],
+              ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }]
+            ] + ssh_expected_commands_for(
+              { 'node' => { connection: '192.168.42.42', user: 'test_user' } },
+              with_control_master_create_optional: true,
+              with_control_master_destroy: false,
+              with_session_exec: false
+            )
+          ) do
+            test_connector.ssh_user = 'test_user'
+            expect do
+              test_connector.with_connection_to(['node']) do
+              end
+            end.to raise_error 'Can\'t spawn interactive ControlMaster to node in non-interactive mode. You may want to change the hpc_interactive env variable.'
+          end
+        end
+      end
+
       it 'creates SSH master to several nodes' do
         with_test_platform(nodes: {
           'node1' => { meta: { host_ip: '192.168.42.1' } },
@@ -102,8 +141,14 @@ describe HybridPlatformsConductor::ActionsExecutor do
               ['which env', proc { [0, "/usr/bin/env\n", ''] }],
               ['ssh -V 2>&1', proc { [0, "OpenSSH_7.4p1 Debian-10+deb9u7, OpenSSL 1.0.2u  20 Dec 2019\n", ''] }]
             ] + ssh_expected_commands_for(
-              'node1' => { connection: '192.168.42.1', user: 'test_user' },
-              'node3' => { connection: '192.168.42.3', user: 'test_user' }
+              {
+                'node1' => { connection: '192.168.42.1', user: 'test_user' },
+                'node3' => { connection: '192.168.42.3', user: 'test_user' }
+              },
+              # Here the threads for node1's and node3's ControlMasters might not trigger before the one for node2, so they will not destroy it.
+              # Sometimes they don't even have time to create the Control Masters that node2 has already failed.
+              with_control_master_create_optional: true,
+              with_control_master_destroy_optional: true
             ) + ssh_expected_commands_for(
               {
                 'node2' => { connection: '192.168.42.2', user: 'test_user', control_master_create_error: 'Can\'t connect to 192.168.42.2' }
@@ -278,7 +323,8 @@ describe HybridPlatformsConductor::ActionsExecutor do
           ) do
             test_connector.ssh_use_control_master = false
             test_connector.ssh_user = 'test_user'
-            test_connector.with_connection_to(['node']) do
+            test_connector.with_connection_to(['node']) do |connected_nodes|
+              expect(connected_nodes).to eq %w[node]
             end
           end
         end

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/remote_actions_spec.rb

@@ -141,6 +141,30 @@ describe HybridPlatformsConductor::ActionsExecutor do
         end
       end
 
+      it 'executes bash commands remotely without Session Exec capabilities' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [[/^\{ cat \| .+\/ssh hpc\.node -T; } <<'EOF'\nbash_cmd.bash\nEOF$/, proc { [0, 'Bash commands executed on node', ''] }]],
+          expected_stdout: 'Bash commands executed on node',
+          session_exec: false
+        ) do
+          test_connector.remote_bash('bash_cmd.bash')
+        end
+      end
+
+      it 'copies files remotely without Session Exec capabilities' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [
+            [
+              /^scp -S .+\/ssh \/path\/to\/src.file hpc\.node:\/remote_path\/to\/dst.dir$/,
+              proc { [0, '', ''] }
+            ]
+          ],
+          session_exec: false
+        ) do
+          test_connector.remote_copy('/path/to/src.file', '/remote_path/to/dst.dir')
+        end
+      end
+
     end
 
   end

data/spec/hybrid_platforms_conductor_test/api/deployer/provisioner_spec.rb

@@ -17,8 +17,8 @@ describe HybridPlatformsConductor::Deployer do
           block.call
         end
         provisioner = nil
-        test_deployer.with_test_provisioned_instance(:test_provisioner, 'node', environment: 'hpc_testing_provisioner') do |test_deployer, test_instance|
-          expect(test_deployer.local_environment).to eq true
+        test_deployer.with_test_provisioned_instance(:test_provisioner, 'node', environment: 'hpc_testing_provisioner') do |sub_test_deployer, test_instance|
+          expect(sub_test_deployer.local_environment).to eq true
           provisioner = test_instance
           expect(test_instance.node).to eq 'node'
           expect(test_instance.environment).to match /^#{Regexp.escape(`whoami`.strip)}_hpc_testing_provisioner_\d+_\d+_\w+$/
@@ -40,8 +40,8 @@ describe HybridPlatformsConductor::Deployer do
           block.call
         end
         provisioner = nil
-        test_deployer.with_test_provisioned_instance(:test_provisioner, 'node', environment: 'hpc_testing_provisioner') do |test_deployer, test_instance|
-          expect(test_deployer.local_environment).to eq true
+        test_deployer.with_test_provisioned_instance(:test_provisioner, 'node', environment: 'hpc_testing_provisioner') do |sub_test_deployer, test_instance|
+          expect(sub_test_deployer.local_environment).to eq true
           provisioner = test_instance
           expect(test_instance.node).to eq 'node'
           expect(test_instance.environment).to match /^#{Regexp.escape(`whoami`.strip)}_hpc_testing_provisioner_\d+_\d+_\w+$/
@@ -50,6 +50,70 @@ describe HybridPlatformsConductor::Deployer do
       end
     end
 
+    it 'gives a new test instance ready to be used in place of the node without SSH transformations' do
+      with_test_platform(
+        {
+          nodes: {
+            'node1' => { meta: { host_ip: '192.168.42.1', ssh_session_exec: 'false' } },
+            'node2' => { meta: { host_ip: '192.168.42.2', ssh_session_exec: 'false' } }
+          }
+        },
+        false,
+        '
+          for_nodes(%w[node1 node2]) do
+            transform_ssh_connection do |node, connection, connection_user, gateway, gateway_user|
+              ["#{connection}_#{node}", "#{connection_user}_#{node}", "#{gateway}_#{node}", "#{gateway_user}_#{node}"]
+            end
+          end
+        '
+      ) do |repository|
+        register_plugins(:provisioner, { test_provisioner: HybridPlatformsConductorTest::TestProvisioner })
+        File.write("#{test_config.hybrid_platforms_dir}/dummy_secrets.json", '{}')
+        HybridPlatformsConductorTest::TestProvisioner.mocked_states = %i[created created running exited]
+        HybridPlatformsConductorTest::TestProvisioner.mocked_ip = '172.17.0.1'
+        expect(Socket).to receive(:tcp).with('172.17.0.1', 22, { connect_timeout: 1 }) do |&block|
+          block.call
+        end
+        test_deployer.with_test_provisioned_instance(:test_provisioner, 'node1', environment: 'hpc_testing_provisioner') do |sub_test_deployer, test_instance|
+          expect(sub_test_deployer.instance_eval { @nodes_handler.get_ssh_session_exec_of('node1') }).to eq 'true'
+          expect(sub_test_deployer.instance_eval { @nodes_handler.get_ssh_session_exec_of('node2') }).to eq 'false'
+          ssh_transforms = test_instance.instance_eval { @config.ssh_connection_transforms }
+          expect(ssh_transforms.size).to eq 1
+          expect(ssh_transforms[0][:nodes_selectors_stack]).to eq [%w[node2]]
+        end
+      end
+    end
+
+    it 'gives a new test instance ready to be used in place of the node without sudo specificities' do
+      with_test_platform(
+        {
+          nodes: {
+            'node1' => { meta: { host_ip: '192.168.42.1' } },
+            'node2' => { meta: { host_ip: '192.168.42.2' } }
+          }
+        },
+        false,
+        '
+          for_nodes(%w[node1 node2]) do
+            sudo_for { |user| "other_sudo --user #{user}" }
+          end
+        '
+      ) do |repository|
+        register_plugins(:provisioner, { test_provisioner: HybridPlatformsConductorTest::TestProvisioner })
+        File.write("#{test_config.hybrid_platforms_dir}/dummy_secrets.json", '{}')
+        HybridPlatformsConductorTest::TestProvisioner.mocked_states = %i[created created running exited]
+        HybridPlatformsConductorTest::TestProvisioner.mocked_ip = '172.17.0.1'
+        expect(Socket).to receive(:tcp).with('172.17.0.1', 22, { connect_timeout: 1 }) do |&block|
+          block.call
+        end
+        test_deployer.with_test_provisioned_instance(:test_provisioner, 'node1', environment: 'hpc_testing_provisioner') do |sub_test_deployer, test_instance|
+          sudo_procs = test_instance.instance_eval { @config.sudo_procs }
+          expect(sudo_procs.size).to eq 1
+          expect(sudo_procs[0][:nodes_selectors_stack]).to eq [%w[node2]]
+        end
+      end
+    end
+
     it 'does not destroy instances when asked to reuse' do
       with_test_platform(
         nodes: { 'node' => { meta: { host_ip: '192.168.42.42' } } }
@@ -62,8 +126,8 @@ describe HybridPlatformsConductor::Deployer do
           block.call
         end
         provisioner = nil
-        test_deployer.with_test_provisioned_instance(:test_provisioner, 'node', environment: 'hpc_testing_provisioner', reuse_instance: true) do |test_deployer, test_instance|
-          expect(test_deployer.local_environment).to eq true
+        test_deployer.with_test_provisioned_instance(:test_provisioner, 'node', environment: 'hpc_testing_provisioner', reuse_instance: true) do |sub_test_deployer, test_instance|
+          expect(sub_test_deployer.local_environment).to eq true
           provisioner = test_instance
           expect(test_instance.node).to eq 'node'
           expect(test_instance.environment).to eq "#{`whoami`.strip}_hpc_testing_provisioner"
@@ -84,8 +148,8 @@ describe HybridPlatformsConductor::Deployer do
           block.call
         end
         provisioner = nil
-        test_deployer.with_test_provisioned_instance(:test_provisioner, 'node', environment: 'hpc_testing_provisioner', reuse_instance: true) do |test_deployer, test_instance|
-          expect(test_deployer.local_environment).to eq true
+        test_deployer.with_test_provisioned_instance(:test_provisioner, 'node', environment: 'hpc_testing_provisioner', reuse_instance: true) do |sub_test_deployer, test_instance|
+          expect(sub_test_deployer.local_environment).to eq true
           provisioner = test_instance
           expect(test_instance.node).to eq 'node'
           expect(test_instance.environment).to eq "#{`whoami`.strip}_hpc_testing_provisioner"
@@ -102,7 +166,7 @@ describe HybridPlatformsConductor::Deployer do
         File.write("#{test_config.hybrid_platforms_dir}/dummy_secrets.json", '{}')
         HybridPlatformsConductorTest::TestProvisioner.mocked_states = %i[created created created exited exited]
         expect do
-          test_deployer.with_test_provisioned_instance(:test_provisioner, 'node', environment: 'hpc_testing_provisioner') do |test_deployer, test_instance|
+          test_deployer.with_test_provisioned_instance(:test_provisioner, 'node', environment: 'hpc_testing_provisioner') do |sub_test_deployer, test_instance|
           end
         end.to raise_error /\[ node\/#{Regexp.escape(`whoami`.strip)}_hpc_testing_provisioner_\d+_\d+_\w+ \] - Instance fails to be in a state among \(running\) with timeout 1\. Currently in state exited/
       end
@@ -120,7 +184,7 @@ describe HybridPlatformsConductor::Deployer do
           raise Errno::ETIMEDOUT, 'Timeout while reading from port 22'
         end
         expect do
-          test_deployer.with_test_provisioned_instance(:test_provisioner, 'node', environment: 'hpc_testing_provisioner') do |test_deployer, test_instance|
+          test_deployer.with_test_provisioned_instance(:test_provisioner, 'node', environment: 'hpc_testing_provisioner') do |sub_test_deployer, test_instance|
           end
         end.to raise_error /\[ node\/#{Regexp.escape(`whoami`.strip)}_hpc_testing_provisioner_\d+_\d+_\w+ \] - Instance fails to have port 22 opened with timeout 1\./
       end

data/spec/hybrid_platforms_conductor_test/executables/options/common_spec.rb

@@ -50,7 +50,8 @@ describe 'executables\' common options' do
         with_test_platform_for_common_options do
           exit_code, stdout, stderr = run executable, *(['--debug'] + default_options)
           expect(exit_code).to eq 0
-          expect(stderr).to eq ''
+          # Make sure to ignore the deployment markers from stderr.
+          expect(stderr.gsub("===== [ node1 / node1_service ] - HPC Service Check ===== Begin\n===== [ node1 / node1_service ] - HPC Service Check ===== End\n", '')).to eq ''
         end
       end
 

data/spec/hybrid_platforms_conductor_test/helpers/cmd_runner_helpers.rb

@@ -8,14 +8,27 @@ module HybridPlatformsConductorTest
       # Run expectations on the expected commands to be called.
       #
       # Parameters::
-      # * *commands* (Array< [String or Regexp, Proc] >): Expected commands that should be called on CmdRunner: the command name or regexp and the corresponding mocked code
-      #   * Parameters::
-      #     * Same parameters as CmdRunner@run_cmd
+      # * *commands* (Array<Array>): List of expected commands that should be called on CmdRunner. Each specification is a list containing those items:
+      #   * *0* (String or Regexp): The command name or regexp matching the command name
+      #   * *1* (Proc): The mocking code to be called in place of the real command:
+      #     * Parameters::
+      #       * Same parameters as CmdRunner@run_cmd
+      #     * Result::
+      #       * Same results as CmdRunner@run_cmd
+      #   * *2* (Hash): Optional hash of options. Can be ommited. [default = {}]
+      #     * *optional* (Boolean): If true then don't fail if the command to be mocked has not been called [default: false]
       # * *cmd_runner* (CmdRunner): The CmdRunner to mock [default: test_cmd_runner]
       # * Proc: Code called with the command runner mocked
       def with_cmd_runner_mocked(commands, cmd_runner: test_cmd_runner)
-        unexpected_commands = []
-        remaining_expected_commands = commands.clone
+        remaining_expected_commands = commands.map do |(expected_command, command_code, options)|
+          [
+            expected_command,
+            command_code,
+            {
+              optional: false
+            }.merge(options || {})
+          ]
+        end
         # We need to protect the access to this array as the mocked commands can be called by competing threads
         remaining_expected_commands_mutex = Mutex.new
         allow(cmd_runner).to receive(:run_cmd) do |cmd, log_to_file: nil, log_to_stdout: true, log_stdout_to_io: nil, log_stderr_to_io: nil, expected_code: 0, timeout: nil, no_exception: false|
@@ -23,7 +36,7 @@ module HybridPlatformsConductorTest
           found_command = nil
           found_command_code = nil
           remaining_expected_commands_mutex.synchronize do
-            remaining_expected_commands.delete_if do |(expected_command, command_code)|
+            remaining_expected_commands.delete_if do |(expected_command, command_code, _options)|
               break unless found_command.nil?
               if (expected_command.is_a?(String) && expected_command == cmd) || (expected_command.is_a?(Regexp) && cmd =~ expected_command)
                 found_command = expected_command
@@ -64,14 +77,23 @@ module HybridPlatformsConductorTest
             log_stderr_to_io << mocked_stderr if !mocked_stderr.empty? && !log_stderr_to_io.nil?
             [mocked_exit_status, mocked_stdout, mocked_stderr]
           else
-            logger.error "[ Mocked CmdRunner ] - !!! Unexpected command run: #{cmd}"
-            unexpected_commands << cmd
-            [:unexpected_command_to_mock, '', "Could not mock unexpected command #{cmd}"]
+            raise "Unexpected command run:\n#{cmd}\nRemaining expected commands:\n#{
+              remaining_expected_commands.map do |(expected_command, _command_code, _options)|
+                expected_command
+              end.join("\n")
+            }"
           end
         end
         yield
-        expect(unexpected_commands).to eq []
-        expect(remaining_expected_commands).to eq([]), "Expected CmdRunner commands were not run:\n#{remaining_expected_commands.map(&:first).join("\n")}"
+        expect(
+          remaining_expected_commands.select do |(_expected_command, _command_code, options)|
+            !options[:optional]
+          end
+        ).to eq([]), "Expected CmdRunner commands were not run:\n#{
+          remaining_expected_commands.map do |(expected_command, _command_code, options)|
+            "#{options[:optional] ? '[Optional] ' : ''}#{expected_command}"
+          end.join("\n")
+        }"
         # Un-mock the command runner
         allow(cmd_runner).to receive(:run_cmd).and_call_original
       end

data/spec/hybrid_platforms_conductor_test/helpers/connector_ssh_helpers.rb

@@ -15,19 +15,25 @@ module HybridPlatformsConductorTest
       #   * *times* (Integer): Number of times this connection should be used [default: 1]
       #   * *control_master_create_error* (String or nil): Error to simulate during the SSH ControlMaster creation, or nil for none [default: nil]
       # * *with_control_master_create* (Boolean): Do we create the control master? [default: true]
+      # * *with_control_master_create_optional* (Boolean): If true, then consider the ControlMaster creation to be optional [default: false]
       # * *with_control_master_check* (Boolean): Do we check the control master? [default: false]
       # * *with_control_master_destroy* (Boolean): Do we destroy the control master? [default: true]
+      # * *with_control_master_destroy_optional* (Boolean): If true, then consider the ControlMaster destruction to be optional [default: false]
       # * *with_strict_host_key_checking* (Boolean): Do we use strict host key checking? [default: true]
       # * *with_batch_mode* (Boolean): Do we use BatchMode when creating the control master? [default: true]
+      # * *with_session_exec* (Boolean): Do we use Sessien Exec capabilities when creating the control master? [default: true]
       # Result::
-      # * Array< [String or Regexp, Proc] >: The expected commands that should be used, and their corresponding mocked code
+      # * Array<Array>: The expected commands that should be used, and their corresponding mocked code and options
       def ssh_expected_commands_for(
         nodes_connections,
         with_control_master_create: true,
+        with_control_master_create_optional: false,
         with_control_master_check: false,
         with_control_master_destroy: true,
+        with_control_master_destroy_optional: false,
         with_strict_host_key_checking: true,
-        with_batch_mode: true
+        with_batch_mode: true,
+        with_session_exec: true
       )
         nodes_connections.map do |node, node_connection_info|
           node_connection_info[:times] = 1 unless node_connection_info.key?(:times)
@@ -43,8 +49,23 @@ module HybridPlatformsConductorTest
             ])
           end
           if with_control_master_create
+            control_master_created = false
             ssh_commands_per_connection << [
-              /^.+\/ssh #{with_batch_mode ? '-o BatchMode=yes ' : ''}-o ControlMaster=yes -o ControlPersist=yes hpc\.#{Regexp.escape(node)} true$/,
+              if with_session_exec
+                /^.+\/ssh #{with_batch_mode ? '-o BatchMode=yes ' : ''}-o ControlMaster=yes -o ControlPersist=yes hpc\.#{Regexp.escape(node)} true$/
+              else
+                unless ENV['hpc_interactive'] == 'false'
+                  # Mock the user hitting enter as the Control Master will be created in another thread and the main thread waits for user input.
+                  expect($stdin).to receive(:gets) do
+                    # We have to wait for the Control Master creation thread to actually create the Control Master before hitting Enter.
+                    while !control_master_created do
+                      sleep 0.1
+                    end
+                    "\n"
+                  end
+                end
+                /^xterm -e '.+\/ssh -o ControlMaster=yes -o ControlPersist=yes hpc\.#{Regexp.escape(node)}'$/
+              end,
               proc do
                 control_file = test_actions_executor.connector(:ssh).send(:control_master_file, node_connection_info[:connection], '22', node_connection_info[:user])
                 # Fail if the ControlMaster file already exists, as would SSH do if the file is stalled
@@ -53,11 +74,16 @@ module HybridPlatformsConductorTest
                 elsif node_connection_info[:control_master_create_error].nil?
                   # Really touch a fake control file, as ssh connector checks for its existence
                   File.write(control_file, '')
+                  control_master_created = true
+                  # If there is no Session Exec, this is done in a separate thread.
+                  # So keep it alive until the user wants to stop it (which is done using an ssh -O exit command).
+                  loop { sleep 0.1 } unless with_session_exec
                   [0, '', '']
                 else
                   [255, '', node_connection_info[:control_master_create_error]]
                 end
-              end
+              end,
+              { optional: with_control_master_create_optional }
             ]
           end
           if with_control_master_check
@@ -73,7 +99,8 @@ module HybridPlatformsConductorTest
                 # Really mock the control file deletion
                 File.unlink(test_actions_executor.connector(:ssh).send(:control_master_file, node_connection_info[:connection], '22', node_connection_info[:user]))
                 [1, '', '']
-              end
+              end,
+              { optional: with_control_master_destroy_optional }
             ]
           end
           ssh_commands_once + ssh_commands_per_connection * node_connection_info[:times]
@@ -97,6 +124,7 @@ module HybridPlatformsConductorTest
       # * *timeout* (Integer or nil): Timeout to prepare the connector for [default: nil]
       # * *password* (String or nil): Password to set for the node, or nil for none [default: nil]
       # * *additional_config* (String): Additional config [default: '']
+      # * *session_exec* (Boolean): Do we mock a node having an SSH connection accepting Session Exec? [default: true]
       # * Proc: Client code to execute testing
       def with_test_platform_for_remote_testing(
         expected_cmds: [],
@@ -104,9 +132,14 @@ module HybridPlatformsConductorTest
         expected_stderr: '',
         timeout: nil,
         password: nil,
-        additional_config: ''
+        additional_config: '',
+        session_exec: true
       )
-        with_test_platform({ nodes: { 'node' => { meta: { host_ip: '192.168.42.42' } } } }, false, additional_config) do
+        with_test_platform(
+          { nodes: { 'node' => { meta: { host_ip: '192.168.42.42', ssh_session_exec: session_exec ? 'true' : 'false' } } } },
+          false,
+          additional_config
+        ) do
           with_cmd_runner_mocked(
             [
               ['which env', proc { [0, "/usr/bin/env\n", ''] }],
@@ -115,7 +148,8 @@ module HybridPlatformsConductorTest
               (password ? [['sshpass -V', proc { [0, "sshpass 1.06\n", ''] }]] : []) +
               ssh_expected_commands_for(
                 { 'node' => { connection: '192.168.42.42', user: 'test_user' } },
-                with_batch_mode: password.nil?
+                with_batch_mode: password.nil?,
+                with_session_exec: session_exec
               ) +
               expected_cmds
           ) do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hybrid_platforms_conductor
 version: !ruby/object:Gem::Version
-  version: 32.6.0
+  version: 32.8.0
 platform: ruby
 authors:
 - Muriel Salvan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-11 00:00:00.000000000 Z
+date: 2021-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: range_operators