hybrid_platforms_conductor 32.3.6

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/global_helpers_spec.rb

@@ -0,0 +1,313 @@
+describe HybridPlatformsConductor::ActionsExecutor do
+
+  context 'checking connector plugin ssh' do
+
+    context 'checking additional helpers' do
+
+      # Return the connector to be tested
+      #
+      # Result::
+      # * Connector: Connector to be tested
+      def test_connector
+        test_actions_executor.connector(:ssh)
+      end
+
+      # Get the SSH config for a given node.
+      # Don't return comments and empty lines.
+      #
+      # Parameters::
+      # * *node* (String or nil): The node we look the SSH config for, or nil for the global configuration
+      # * *ssh_config* (String or nil): The SSH config, or nil to get it from the test_actions_executor [default: nil]
+      # * *nodes* (Array<String> or nil): List of nodes to give ssh_config, or nil for none. Used only if ssh_config is nil. [default: nil]
+      # * *ssh_exec* (String or nil): SSH executable, or nil to keep default. Used only if ssh_config is nil. [default: nil]
+      # * *known_hosts_file* (String or nil): Known host file to give. Used only if ssh_config is nil. [default: nil]
+      # Result::
+      # * String or nil: Corresponding SSH config, or nil if none
+      def ssh_config_for(node, ssh_config: nil, nodes: nil, ssh_exec: nil, known_hosts_file: nil)
+        if ssh_config.nil?
+          params = {}
+          params[:nodes] = nodes unless nodes.nil?
+          params[:ssh_exec] = ssh_exec unless ssh_exec.nil?
+          params[:known_hosts_file] = known_hosts_file unless known_hosts_file.nil?
+          ssh_config = test_connector.ssh_config(**params)
+        end
+        ssh_config_lines = ssh_config.split("\n")
+        begin_marker = node.nil? ? /^Host \*$/ : /^# #{Regexp.escape(node)} - .+$/
+        start_idx = ssh_config_lines.index { |line| line =~ begin_marker }
+        return nil if start_idx.nil?
+        end_marker = /^# \w+ - .+$/
+        end_idx = ssh_config_lines[start_idx + 1..-1].index { |line| line =~ end_marker }
+        end_idx = end_idx.nil? ? -1 : start_idx + end_idx
+        ssh_config_lines[start_idx..end_idx].select do |line|
+          stripped_line = line.strip
+          !stripped_line.empty? && stripped_line[0] != '#'
+        end.join("\n") + "\n"
+      end
+
+      it 'generates a global configuration with user from environment' do
+        with_test_platform do
+          ENV['hpc_ssh_user'] = 'test_user'
+          expect(ssh_config_for(nil)).to eq <<~EOS
+            Host *
+              User test_user
+              ControlPath #{Dir.tmpdir}/hpc_ssh/hpc_actions_executor_mux_%h_%p_%r
+              PubkeyAcceptedKeyTypes +ssh-dss
+          EOS
+        end
+      end
+
+      it 'generates a global configuration with user from setting' do
+        with_test_platform do
+          test_connector.ssh_user = 'test_user'
+          expect(ssh_config_for(nil)).to eq <<~EOS
+            Host *
+              User test_user
+              ControlPath #{Dir.tmpdir}/hpc_ssh/hpc_actions_executor_mux_%h_%p_%r
+              PubkeyAcceptedKeyTypes +ssh-dss
+          EOS
+        end
+      end
+
+      it 'generates a global configuration with known hosts file' do
+        with_test_platform do
+          test_connector.ssh_user = 'test_user'
+          expect(ssh_config_for(nil, known_hosts_file: '/path/to/known_hosts')).to eq <<~EOS
+            Host *
+              User test_user
+              ControlPath #{Dir.tmpdir}/hpc_ssh/hpc_actions_executor_mux_%h_%p_%r
+              PubkeyAcceptedKeyTypes +ssh-dss
+              UserKnownHostsFile /path/to/known_hosts
+          EOS
+        end
+      end
+
+      it 'generates a global configuration without strict host key checking' do
+        with_test_platform do
+          test_connector.ssh_user = 'test_user'
+          test_connector.ssh_strict_host_key_checking = false
+          expect(ssh_config_for(nil)).to eq <<~EOS
+            Host *
+              User test_user
+              ControlPath #{Dir.tmpdir}/hpc_ssh/hpc_actions_executor_mux_%h_%p_%r
+              PubkeyAcceptedKeyTypes +ssh-dss
+              StrictHostKeyChecking no
+          EOS
+        end
+      end
+
+      it 'includes the gateway definition from environment' do
+        with_test_platform({}, false, 'gateway :gateway1, \'Host my_gateway\'') do
+          ENV['hpc_ssh_gateways_conf'] = 'gateway1'
+          expect(test_connector.ssh_config).to match /^Host my_gateway$/
+        end
+      end
+
+      it 'includes the gateway definition from setting' do
+        with_test_platform({}, false, 'gateway :gateway1, \'Host my_gateway\'') do
+          test_connector.ssh_gateways_conf = :gateway1
+          expect(test_connector.ssh_config).to match /^Host my_gateway$/
+        end
+      end
+
+      it 'includes the gateway definition with a different ssh executable' do
+        with_test_platform({}, false, 'gateway :gateway1, \'Host my_gateway_<%= @ssh_exec %>\'') do
+          test_connector.ssh_gateways_conf = :gateway1
+          expect(test_connector.ssh_config(ssh_exec: 'new_ssh')).to match /^Host my_gateway_new_ssh$/
+        end
+      end
+
+      it 'does not include the gateway definition if it is not selected' do
+        with_test_platform({}, false, 'gateway :gateway2, \'Host my_gateway\'') do
+          test_connector.ssh_gateways_conf = :gateway1
+          expect(test_connector.ssh_config).not_to match /^Host my_gateway$/
+        end
+      end
+
+      it 'generates a simple config for a node with host_ip' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42' } } }) do
+          expect(ssh_config_for('node')).to eq <<~EOS
+            Host hpc.node
+              Hostname 192.168.42.42
+          EOS
+        end
+      end
+
+      it 'generates a simple config for several nodes' do
+        with_test_platform(nodes: {
+          'node1' => { meta: { host_ip: '192.168.42.1' } },
+          'node2' => { meta: { host_ip: '192.168.42.2' } },
+          'node3' => { meta: { host_ip: '192.168.42.3' } }
+        }) do
+          expect(ssh_config_for('node1')).to eq <<~EOS
+            Host hpc.node1
+              Hostname 192.168.42.1
+          EOS
+          expect(ssh_config_for('node2')).to eq <<~EOS
+            Host hpc.node2
+              Hostname 192.168.42.2
+          EOS
+          expect(ssh_config_for('node3')).to eq <<~EOS
+            Host hpc.node3
+              Hostname 192.168.42.3
+          EOS
+        end
+      end
+
+      it 'selects nodes when generating the config' do
+        with_test_platform(nodes: {
+          'node1' => { meta: { host_ip: '192.168.42.1' } },
+          'node2' => { meta: { host_ip: '192.168.42.2' } },
+          'node3' => { meta: { host_ip: '192.168.42.3' } }
+        }) do
+          expect(ssh_config_for('node1', nodes: %w[node1 node3])).to eq <<~EOS
+            Host hpc.node1
+              Hostname 192.168.42.1
+          EOS
+          expect(ssh_config_for('node2', nodes: %w[node1 node3])).to eq nil
+        end
+      end
+
+      it 'fails if a node can\'t be connected to' do
+        with_test_platform(nodes: { 'node' => {} }) do
+          expect { ssh_config_for('node') }.to raise_error(/No connection possible to node/)
+        end
+      end
+
+      it 'generates an alias if the node has a hostname' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42', hostname: 'my_hostname.my_domain' } } }) do
+          expect(ssh_config_for('node')).to eq <<~EOS
+            Host hpc.node my_hostname.my_domain
+              Hostname 192.168.42.42
+          EOS
+        end
+      end
+
+      it 'generates aliases if the node has private ips' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42', private_ips: ['192.168.42.1', '192.168.42.2'] } } }) do
+          expect(ssh_config_for('node')).to eq <<~EOS
+            Host hpc.node hpc.192.168.42.1 hpc.192.168.42.2
+              Hostname 192.168.42.42
+          EOS
+        end
+      end
+
+      it 'generates a simple config for a node with hostname' do
+        with_test_platform(nodes: { 'node' => { meta: { hostname: 'my_hostname.my_domain' } } }) do
+          expect(ssh_config_for('node')).to eq <<~EOS
+            Host hpc.node my_hostname.my_domain
+              Hostname my_hostname.my_domain
+          EOS
+        end
+      end
+
+      it 'generates a simple config for a node with private_ips' do
+        with_test_platform(nodes: { 'node' => { meta: { private_ips: ['192.168.42.1', '192.168.42.2'] } } }) do
+          expect(ssh_config_for('node')).to eq <<~EOS
+            Host hpc.node hpc.192.168.42.1 hpc.192.168.42.2
+              Hostname 192.168.42.1
+          EOS
+        end
+      end
+
+      it 'uses node forced gateway information' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42', gateway: 'test_gateway', gateway_user: 'test_gateway_user' } } }) do
+          expect(ssh_config_for('node')).to eq <<~EOS
+            Host hpc.node
+              Hostname 192.168.42.42
+              ProxyCommand ssh -q -W %h:%p test_gateway_user@test_gateway
+          EOS
+        end
+      end
+
+      it 'uses node default gateway information and user from environment' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42', gateway: 'test_gateway' } } }) do
+          ENV['hpc_ssh_gateway_user'] = 'test_gateway_user'
+          expect(ssh_config_for('node')).to eq <<~EOS
+            Host hpc.node
+              Hostname 192.168.42.42
+              ProxyCommand ssh -q -W %h:%p test_gateway_user@test_gateway
+          EOS
+        end
+      end
+
+      it 'uses node default gateway information and user from setting' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42', gateway: 'test_gateway' } } }) do
+          test_connector.ssh_gateway_user = 'test_gateway_user'
+          expect(ssh_config_for('node')).to eq <<~EOS
+            Host hpc.node
+              Hostname 192.168.42.42
+              ProxyCommand ssh -q -W %h:%p test_gateway_user@test_gateway
+          EOS
+        end
+      end
+
+      it 'uses node forced gateway information with a different ssh executable' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42', gateway: 'test_gateway', gateway_user: 'test_gateway_user' } } }) do
+          expect(ssh_config_for('node', ssh_exec: 'new_ssh')).to eq <<~EOS
+            Host hpc.node
+              Hostname 192.168.42.42
+              ProxyCommand new_ssh -q -W %h:%p test_gateway_user@test_gateway
+          EOS
+        end
+      end
+
+      it 'uses node default gateway information with a different ssh executable' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42', gateway: 'test_gateway' } } }) do
+          test_connector.ssh_gateway_user = 'test_gateway_user'
+          expect(ssh_config_for('node', ssh_exec: 'new_ssh')).to eq <<~EOS
+            Host hpc.node
+              Hostname 192.168.42.42
+              ProxyCommand new_ssh -q -W %h:%p test_gateway_user@test_gateway
+          EOS
+        end
+      end
+
+      it 'generates a config compatible for passwords authentication' do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42' } } }) do
+          test_connector.passwords['node'] = 'PaSsWoRd'
+          expect(ssh_config_for('node')).to eq <<~EOS
+            Host hpc.node
+              Hostname 192.168.42.42
+              PreferredAuthentications password
+              PubkeyAuthentication no
+          EOS
+        end
+      end
+
+      it 'generates a config compatible for passwords authentication only for marked nodes' do
+        with_test_platform(nodes: {
+          'node1' => { meta: { host_ip: '192.168.42.1' } },
+          'node2' => { meta: { host_ip: '192.168.42.2' } },
+          'node3' => { meta: { host_ip: '192.168.42.3' } },
+          'node4' => { meta: { host_ip: '192.168.42.4' } }
+        }) do
+          test_connector.passwords['node1'] = 'PaSsWoRd1'
+          test_connector.passwords['node3'] = 'PaSsWoRd3'
+          expect(ssh_config_for('node1')).to eq <<~EOS
+            Host hpc.node1
+              Hostname 192.168.42.1
+              PreferredAuthentications password
+              PubkeyAuthentication no
+          EOS
+          expect(ssh_config_for('node2')).to eq <<~EOS
+            Host hpc.node2
+              Hostname 192.168.42.2
+          EOS
+          expect(ssh_config_for('node3')).to eq <<~EOS
+            Host hpc.node3
+              Hostname 192.168.42.3
+              PreferredAuthentications password
+              PubkeyAuthentication no
+          EOS
+          expect(ssh_config_for('node4')).to eq <<~EOS
+            Host hpc.node4
+              Hostname 192.168.42.4
+          EOS
+        end
+      end
+
+   end
+
+  end
+
+end

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/node_helpers_spec.rb

@@ -0,0 +1,32 @@
+describe HybridPlatformsConductor::ActionsExecutor do
+
+  context 'checking connector plugin ssh' do
+
+    context 'checking additional helpers on prepared nodes' do
+
+      it 'provides an SSH executable wrapping the node\'s SSH config' do
+        with_test_platform_for_remote_testing do
+          expect(`#{test_connector.ssh_exec} -V 2>&1`).to eq `ssh -V 2>&1`
+          expect(`#{test_connector.ssh_exec} -G hpc.node`.split("\n").find { |line| line =~ /^hostname .+$/ }).to eq 'hostname 192.168.42.42'
+        end
+      end
+
+      it 'provides an SSH URL that can be used by other processes to connect to this node' do
+        with_test_platform_for_remote_testing do
+          expect(test_connector.ssh_url).to eq 'test_user@hpc.node'
+        end
+      end
+
+      it 'uses sshpass in the provided SSH executable if needed' do
+        with_test_platform_for_remote_testing(password: 'PaSsWoRd') do
+          expect(`#{test_connector.ssh_exec} -V 2>&1`).to eq `ssh -V 2>&1`
+          expect(`#{test_connector.ssh_exec} -G hpc.node`.split("\n").find { |line| line =~ /^hostname .+$/ }).to eq 'hostname 192.168.42.42'
+          expect(File.read(test_connector.ssh_exec)).to match /^sshpass -pPaSsWoRd ssh .+$/
+        end
+      end
+
+    end
+
+  end
+
+end

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/remote_actions_spec.rb

@@ -0,0 +1,134 @@
+describe HybridPlatformsConductor::ActionsExecutor do
+
+  context 'checking connector plugin ssh' do
+
+    context 'checking remote actions' do
+
+      it 'executes bash commands remotely' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [[/.+\/ssh test_user@hpc\.node \/bin\/bash <<'EOF'\nbash_cmd.bash\nEOF/, proc { [0, 'Bash commands executed on node', ''] }]],
+          expected_stdout: 'Bash commands executed on node'
+        ) do
+          test_connector.remote_bash('bash_cmd.bash')
+        end
+      end
+
+      it 'executes bash commands remotely with timeout' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [
+            [
+              /.+\/ssh test_user@hpc\.node \/bin\/bash <<'EOF'\nbash_cmd.bash\nEOF/,
+              proc do |cmd, log_to_file: nil, log_to_stdout: true, log_stdout_to_io: nil, log_stderr_to_io: nil, expected_code: 0, timeout: nil, no_exception: false|
+                expect(timeout).to eq 5
+                [0, '', '']
+              end
+            ]
+          ],
+          timeout: 5
+        ) do
+          test_connector.remote_bash('bash_cmd.bash')
+        end
+      end
+
+      it 'executes interactive commands remotely' do
+        with_test_platform_for_remote_testing do
+          expect(test_connector).to receive(:system) do |cmd|
+            expect(cmd).to match /^.+\/ssh test_user@hpc\.node$/
+          end
+          test_connector.remote_interactive
+        end
+      end
+
+      it 'copies files remotely' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [
+            [
+              /cd \/path\/to && tar\s+--create\s+--gzip\s+--file -\s+src.file \| \/.+\/ssh\s+test_user@hpc\.node\s+"tar\s+--extract\s+--gunzip\s+--file -\s+--directory \/remote_path\/to\/dst.dir\s+--owner root\s+"/,
+              proc { [0, '', ''] }
+            ]
+          ]
+        ) do
+          test_connector.remote_copy('/path/to/src.file', '/remote_path/to/dst.dir')
+        end
+      end
+
+      it 'copies files remotely with timeout' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [
+            [
+              /cd \/path\/to && tar\s+--create\s+--gzip\s+--file -\s+src.file \| \/.+\/ssh\s+test_user@hpc\.node\s+"tar\s+--extract\s+--gunzip\s+--file -\s+--directory \/remote_path\/to\/dst.dir\s+--owner root\s+"/,
+              proc do |cmd, log_to_file: nil, log_to_stdout: true, log_stdout_to_io: nil, log_stderr_to_io: nil, expected_code: 0, timeout: nil, no_exception: false|
+                expect(timeout).to eq 5
+                [0, '', '']
+              end
+            ]
+          ],
+          timeout: 5
+        ) do
+          test_connector.remote_copy('/path/to/src.file', '/remote_path/to/dst.dir')
+        end
+      end
+
+      it 'executes really big bash commands remotely' do
+        cmd = "echo #{'1' * 131_060}"
+        with_test_platform_for_remote_testing(
+          expected_cmds: [
+            [
+              /.+\/hpc_temp_cmds_.+\.sh$/,
+              proc do |received_cmd|
+                expect(File.read(received_cmd)).to match /.+\/ssh test_user@hpc\.node \/bin\/bash <<'EOF'\n#{Regexp.escape(cmd)}\nEOF/
+                [0, 'Bash commands executed on node', '']
+              end
+            ]
+          ],
+          expected_stdout: 'Bash commands executed on node'
+        ) do
+          # Use an argument that exceeds the max arg length limit
+          test_connector.remote_bash(cmd)
+        end
+      end
+
+      it 'copies files remotely with sudo' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [
+            [
+              /cd \/path\/to && tar\s+--create\s+--gzip\s+--file -\s+src.file \| \/.+\/ssh\s+test_user@hpc\.node\s+"sudo tar\s+--extract\s+--gunzip\s+--file -\s+--directory \/remote_path\/to\/dst.dir\s+--owner root\s+"/,
+              proc { [0, '', ''] }
+            ]
+          ]
+        ) do
+          test_connector.remote_copy('/path/to/src.file', '/remote_path/to/dst.dir', sudo: true)
+        end
+      end
+
+      it 'copies files remotely with a different owner' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [
+            [
+              /cd \/path\/to && tar\s+--create\s+--gzip\s+--file -\s+--owner remote_user\s+src.file \| \/.+\/ssh\s+test_user@hpc\.node\s+"tar\s+--extract\s+--gunzip\s+--file -\s+--directory \/remote_path\/to\/dst.dir\s+--owner root\s+"/,
+              proc { [0, '', ''] }
+            ]
+          ]
+        ) do
+          test_connector.remote_copy('/path/to/src.file', '/remote_path/to/dst.dir', owner: 'remote_user')
+        end
+      end
+
+      it 'copies files remotely with a different group' do
+        with_test_platform_for_remote_testing(
+          expected_cmds: [
+            [
+              /cd \/path\/to && tar\s+--create\s+--gzip\s+--file -\s+--group remote_group\s+src.file \| \/.+\/ssh\s+test_user@hpc\.node\s+"tar\s+--extract\s+--gunzip\s+--file -\s+--directory \/remote_path\/to\/dst.dir\s+--owner root\s+"/,
+              proc { [0, '', ''] }
+            ]
+          ]
+        ) do
+          test_connector.remote_copy('/path/to/src.file', '/remote_path/to/dst.dir', group: 'remote_group')
+        end
+      end
+
+    end
+
+  end
+
+end