hybrid_platforms_conductor 32.5.0 → 32.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 834a3af157221b88e6a150738b146f90fdf832c43fe5e8471619879c47a22d5b
-  data.tar.gz: c4f54b879fbfba923fda2a8aee1b299cb846ed9478ca9bd81535f220ec622f69
+  metadata.gz: 83352b7821e41bc5b0282c693c47f0cac0f5a9915a45dfa37affd3d1fcaadd8c
+  data.tar.gz: fba8bf35c2569716b2b8f934a46684c8b29843e50fe49b8133d3eeae580938d1
 SHA512:
-  metadata.gz: 42ae65da627c812b096c0e9f9b4bb8c9ac4dde3185e1f6cf8d1405acbab960d5c5c88050c25e3d45906acb6054c73fd7ec29b2e5751b1ec38734f83a33f34bef
-  data.tar.gz: 41e641381b34d9d0841770974c4401612bfc3a4d11a5ec2fe85969f40ab1bfce0952cb4bbf5b7d2fce5420d9090a4962e278b9fb84d4251e0f13ee84db405e6c
+  metadata.gz: 69c667f1f6c626372ecef7c21c9afdb2865a38e418ad6add2ec4d04d9a0eeda6892fed168d546d89d8805d843c31945eb885d53b0cf5208a65a7eb89ba75ea74
+  data.tar.gz: b9478d38660f80b7c9f62411d72d0e890e74ec509973ceb3d417721d21f5f9923ffc5dfbc96b857af6301d7bd708630638d393dc3bbf2abcd4cf7d12e22a7af6

data/lib/hybrid_platforms_conductor/deployer.rb

@@ -311,13 +311,24 @@ module HybridPlatformsConductor
           environment: environment,
           logger: @logger,
           logger_stderr: @logger_stderr,
-          config: @config,
+          config: sub_executable.config,
           cmd_runner: @cmd_runner,
           # Here we use the NodesHandler that will be bound to the sub-Deployer only, as the node's metadata might be modified by the Provisioner.
           nodes_handler: sub_executable.nodes_handler,
           actions_executor: @actions_executor
         )
         instance.with_running_instance(stop_on_exit: true, destroy_on_exit: !reuse_instance, port: 22) do
+          # Test-provisioned nodes have SSH Session Exec capabilities
+          sub_executable.nodes_handler.override_metadata_of node, :ssh_session_exec, 'true'
+          # Test-provisioned nodes use default sudo
+          sub_executable.config.sudo_procs.replace(sub_executable.config.sudo_procs.map do |sudo_proc_info|
+            {
+              nodes_selectors_stack: sudo_proc_info[:nodes_selectors_stack].map do |nodes_selector|
+                @nodes_handler.select_nodes(nodes_selector).select { |selected_node| selected_node != node }
+              end,
+              sudo_proc: sudo_proc_info[:sudo_proc]
+            }
+          end)
           actions_executor = sub_executable.actions_executor
           deployer = sub_executable.deployer
           # Setup test environment for this container

data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb

@@ -16,11 +16,40 @@ module HybridPlatformsConductor
 
         module PlatformsDslSsh
 
+          # List of SSH connection transformations:
+          # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
+          # * *transform* (Proc): Code called to transform SSH connection info:
+          #   Parameters::
+          #   * *node* (String): Node for which we transform the SSH connection
+          #   * *connection* (String or nil): The connection host or IP, or nil if none
+          #   * *connection_user* (String): The connection user
+          #   * *gateway* (String or nil): The gateway name, or nil if none
+          #   * *gateway_user* (String or nil): The gateway user, or nil if none
+          #   Result::
+          #   * String: The transformed connection host or IP, or nil if none
+          #   * String: The transformed connection user
+          #   * String or nil: The transformed gateway name, or nil if none
+          #   * String or nil: The transformed gateway user, or nil if none
+          # Array< Hash<Symbol, Object> >
+          attr_reader :ssh_connection_transforms
+
           # Initialize the DSL
           def init_ssh
             # List of gateway configurations, per gateway config name
             # Hash<Symbol, String>
             @gateways = {}
+            @ssh_connection_transforms = []
+          end
+
+          # Define a transformation of SSH connection.
+          #
+          # Parameters::
+          # * *transform* (Proc): Code to be called to transform an SSH connection (see ssh_connection_transforms signature for details)
+          def transform_ssh_connection(&transform)
+            @ssh_connection_transforms << {
+              nodes_selectors_stack: current_nodes_selectors_stack,
+              transform: transform
+            }
           end
 
           # Register a new gateway configuration
@@ -204,7 +233,13 @@ module HybridPlatformsConductor
         # Parameters::
         # * *bash_cmds* (String): Bash commands to execute
         def remote_bash(bash_cmds)
-          ssh_cmd = "#{ssh_exec} #{ssh_url} /bin/bash <<'EOF'\n#{bash_cmds}\nEOF"
+          ssh_cmd =
+            if @nodes_handler.get_ssh_session_exec_of(@node) == 'false'
+              # When ExecSession is disabled we need to use stdin directly
+              "{ cat | #{ssh_exec} #{ssh_url} -T; } <<'EOF'\n#{bash_cmds}\nEOF"
+            else
+              "#{ssh_exec} #{ssh_url} /bin/bash <<'EOF'\n#{bash_cmds}\nEOF"
+            end
           # Due to a limitation of Process.spawn, each individual argument is limited to 128KB of size.
           # Therefore we need to make sure that if bash_cmds exceeds MAX_CMD_ARG_LENGTH bytes (considering EOF chars) then we use an intermediary shell script to store the commands.
           if bash_cmds.size > MAX_CMD_ARG_LENGTH
@@ -261,25 +296,30 @@ module HybridPlatformsConductor
         # * *owner* (String or nil): Owner to be used when copying the files, or nil for current one [default: nil]
         # * *group* (String or nil): Group to be used when copying the files, or nil for current one [default: nil]
         def remote_copy(from, to, sudo: false, owner: nil, group: nil)
-          run_cmd <<~EOS
-            cd #{File.dirname(from)} && \
-            tar \
-              --create \
-              --gzip \
-              --file - \
-              #{owner.nil? ? '' : "--owner #{owner}"} \
-              #{group.nil? ? '' : "--group #{group}"} \
-              #{File.basename(from)} | \
-            #{ssh_exec} \
-              #{ssh_url} \
-              \"#{sudo ? "#{@nodes_handler.sudo_on(@node)} " : ''}tar \
-                --extract \
-                --gunzip \
+          if @nodes_handler.get_ssh_session_exec_of(@node) == 'false'
+            # We don't have ExecSession, so don't use ssh, but scp instead.
+            run_cmd "scp -S #{ssh_exec} #{from} #{ssh_url}:#{to}"
+          else
+            run_cmd <<~EOS
+              cd #{File.dirname(from)} && \
+              tar \
+                --create \
+                --gzip \
                 --file - \
-                --directory #{to} \
-                --owner root \
-              \"
-          EOS
+                #{owner.nil? ? '' : "--owner #{owner}"} \
+                #{group.nil? ? '' : "--group #{group}"} \
+                #{File.basename(from)} | \
+              #{ssh_exec} \
+                #{ssh_url} \
+                \"#{sudo ? "#{@nodes_handler.sudo_on(@node)} " : ''}tar \
+                  --extract \
+                  --gunzip \
+                  --file - \
+                  --directory #{to} \
+                  --owner root \
+                \"
+            EOS
+          end
         end
 
         # Get the ssh executable to be used when connecting to the current node
@@ -295,7 +335,7 @@ module HybridPlatformsConductor
         # Result::
         # * String: The ssh URL connecting to the current node
         def ssh_url
-          "#{@ssh_user}@hpc.#{@node}"
+          "hpc.#{@node}"
         end
 
         # Get an SSH configuration content giving access to nodes of the platforms with the current configuration
@@ -318,14 +358,6 @@ module HybridPlatformsConductor
             # ENDPOINTS #
             #############
 
-            Host *
-              User #{@ssh_user}
-              # Default control socket path to be used when multiplexing SSH connections
-              ControlPath #{control_master_file('%h', '%p', '%r')}
-              #{open_ssh_major_version >= 7 ? 'PubkeyAcceptedKeyTypes +ssh-dss' : ''}
-              #{known_hosts_file.nil? ? '' : "UserKnownHostsFile #{known_hosts_file}"}
-              #{@ssh_strict_host_key_checking ? '' : 'StrictHostKeyChecking no'}
-
           EOS
 
           # Add each node
@@ -333,21 +365,37 @@ module HybridPlatformsConductor
           @nodes_handler.prefetch_metadata_of nodes, %i[private_ips hostname host_ip description]
           nodes.sort.each do |node|
             # Generate the conf for the node
-            begin
-              connection, gateway, gateway_user = connection_info_for(node)
+            connection, connection_user, gateway, gateway_user = connection_info_for(node, no_exception: true)
+            if connection.nil?
+              config_content << "# #{node} - Not connectable using SSH - #{@nodes_handler.get_description_of(node) || ''}\n"
+            else
               config_content << "# #{node} - #{connection} - #{@nodes_handler.get_description_of(node) || ''}\n"
               config_content << "Host #{ssh_aliases_for(node).join(' ')}\n"
               config_content << "  Hostname #{connection}\n"
+              config_content << "  User \"#{connection_user}\"\n" if connection_user != @ssh_user
               config_content << "  ProxyCommand #{ssh_exec} -q -W %h:%p #{gateway_user}@#{gateway}\n" unless gateway.nil?
               if @passwords.key?(node)
                 config_content << "  PreferredAuthentications password\n"
                 config_content << "  PubkeyAuthentication no\n"
               end
-            rescue NotConnectableError
-              config_content << "# #{node} - Not connectable using SSH - #{@nodes_handler.get_description_of(node) || ''}\n"
             end
             config_content << "\n"
           end
+          # Add global definitions at the end of the SSH config, as they might be overriden by previous ones, and first match wins.
+          config_content << <<~EOS
+            ###########
+            # GLOBALS #
+            ###########
+
+            Host *
+              User #{@ssh_user}
+              # Default control socket path to be used when multiplexing SSH connections
+              ControlPath #{control_master_file('%h', '%p', '%r')}
+              #{open_ssh_major_version >= 7 ? 'PubkeyAcceptedKeyTypes +ssh-dss' : ''}
+              #{known_hosts_file.nil? ? '' : "UserKnownHostsFile #{known_hosts_file}"}
+              #{@ssh_strict_host_key_checking ? '' : 'StrictHostKeyChecking no'}
+
+          EOS
           config_content
         end
 
@@ -450,34 +498,48 @@ module HybridPlatformsConductor
                   with_lock_on_control_master_for(node) do |current_users, user_id|
                     working_master = false
                     ssh_exec = ssh_exec_for(node)
-                    ssh_url = "#{@ssh_user}@hpc.#{node}"
+                    ssh_url = "hpc.#{node}"
                     if current_users.empty?
                       log_debug "[ ControlMaster - #{ssh_url} ] - Creating SSH ControlMaster..."
-                      # Create the control master
-                      ssh_control_master_start_cmd = "#{ssh_exec}#{@passwords.key?(node) || @auth_password ? '' : ' -o BatchMode=yes'} -o ControlMaster=yes -o ControlPersist=yes #{ssh_url} true"
                       exit_status = nil
-                      idx_try = 0
-                      loop do
-                        stderr = nil
-                        exit_status, _stdout, stderr = @cmd_runner.run_cmd ssh_control_master_start_cmd, log_to_stdout: log_debug?, no_exception: true, timeout: timeout
-                        if exit_status == 0
-                          break
-                        elsif stderr =~ /System is booting up/
-                          if idx_try == MAX_RETRIES_FOR_BOOT
-                            if no_exception
-                              break
-                            else
-                              raise ActionsExecutor::ConnectionError, "Tried #{idx_try} times to create SSH Control Master with #{ssh_control_master_start_cmd} but system says it's booting up."
+                      if @nodes_handler.get_ssh_session_exec_of(node) == 'false'
+                        # Here we have to create a ControlMaster using an interactive session, as the SSH server prohibits ExecSession, and so command executions.
+                        # We'll do that using another terminal spawned in the background.
+                        Thread.new do
+                          log_debug "[ ControlMaster - #{ssh_url} ] - Spawn interactive ControlMaster in separate terminal"
+                          @cmd_runner.run_cmd "xterm -e '#{ssh_exec} -o ControlMaster=yes -o ControlPersist=yes #{ssh_url}'", log_to_stdout: log_debug?
+                          log_debug "[ ControlMaster - #{ssh_url} ] - Separate interactive ControlMaster closed"
+                        end
+                        out 'External ControlMaster has been spawned.'
+                        out 'Please login into it, keep its session opened and press enter here when done...'
+                        $stdin.gets
+                        exit_status = 0
+                      else
+                        # Create the control master
+                        ssh_control_master_start_cmd = "#{ssh_exec}#{@passwords.key?(node) || @auth_password ? '' : ' -o BatchMode=yes'} -o ControlMaster=yes -o ControlPersist=yes #{ssh_url} true"
+                        idx_try = 0
+                        loop do
+                          stderr = nil
+                          exit_status, _stdout, stderr = @cmd_runner.run_cmd ssh_control_master_start_cmd, log_to_stdout: log_debug?, no_exception: true, timeout: timeout
+                          if exit_status == 0
+                            break
+                          elsif stderr =~ /System is booting up/
+                            if idx_try == MAX_RETRIES_FOR_BOOT
+                              if no_exception
+                                break
+                              else
+                                raise ActionsExecutor::ConnectionError, "Tried #{idx_try} times to create SSH Control Master with #{ssh_control_master_start_cmd} but system says it's booting up."
+                              end
                             end
+                            # Wait a bit and try again
+                            idx_try += 1
+                            log_debug "[ ControlMaster - #{ssh_url} ] - System is booting up (try ##{idx_try}). Wait #{WAIT_TIME_FOR_BOOT} seconds before trying ControlMaster's creation again."
+                            sleep WAIT_TIME_FOR_BOOT
+                          elsif no_exception
+                            break
+                          else
+                            raise ActionsExecutor::ConnectionError, "Error while starting SSH Control Master with #{ssh_control_master_start_cmd}: #{stderr.strip}"
                           end
-                          # Wait a bit and try again
-                          idx_try += 1
-                          log_debug "[ ControlMaster - #{ssh_url} ] - System is booting up (try ##{idx_try}). Wait #{WAIT_TIME_FOR_BOOT} seconds before trying ControlMaster's creation again."
-                          sleep WAIT_TIME_FOR_BOOT
-                        elsif no_exception
-                          break
-                        else
-                          raise ActionsExecutor::ConnectionError, "Error while starting SSH Control Master with #{ssh_control_master_start_cmd}: #{stderr.strip}"
                         end
                       end
                       if exit_status == 0
@@ -512,28 +574,33 @@ module HybridPlatformsConductor
                     end
                   end
                 end
+              else
+                # We have not created any ControlMaster, but still consider the nodes to be ready to connect
+                user_locks = Hash[nodes.map { |node| [node, nil]} ]
               end
               yield user_locks.keys
             ensure
-              user_locks_mutex.synchronize do
-                user_locks.each do |node, user_id|
-                  with_lock_on_control_master_for(node, user_id: user_id) do |current_users, user_id|
-                    ssh_url = "#{@ssh_user}@hpc.#{node}"
-                    log_warn "[ ControlMaster - #{ssh_url} ] - Current process/thread was not part of the ControlMaster users anymore whereas it should have been" unless current_users.include?(user_id)
-                    remaining_users = current_users - [user_id]
-                    if remaining_users.empty?
-                      # Stop the ControlMaster
-                      log_debug "[ ControlMaster - #{ssh_url} ] - Stopping ControlMaster..."
-                      # Dumb verbose ssh! Tricky trick to just silence what is useless.
-                      # Don't fail if the connection close fails (but still log the error), as it can be seen as only a warning: it means the connection was closed anyway.
-                      @cmd_runner.run_cmd "#{ssh_exec_for(node)} -O exit #{ssh_url} 2>&1 | grep -v 'Exit request sent.'", log_to_stdout: log_debug?, expected_code: 1, timeout: timeout, no_exception: true
-                      log_debug "[ ControlMaster - #{ssh_url} ] - ControlMaster stopped"
-                      # Uncomment if you want to test that the connection has been closed
-                      # @cmd_runner.run_cmd "#{ssh_exec_for(node)} -O check #{ssh_url}", log_to_stdout: log_debug?, expected_code: 255, timeout: timeout
-                    else
-                      log_debug "[ ControlMaster - #{ssh_url} ] - Leaving ControlMaster started as #{remaining_users.size} processes/threads are still using it."
+              if @ssh_use_control_master
+                user_locks_mutex.synchronize do
+                  user_locks.each do |node, user_id|
+                    with_lock_on_control_master_for(node, user_id: user_id) do |current_users, user_id|
+                      ssh_url = "hpc.#{node}"
+                      log_warn "[ ControlMaster - #{ssh_url} ] - Current process/thread was not part of the ControlMaster users anymore whereas it should have been" unless current_users.include?(user_id)
+                      remaining_users = current_users - [user_id]
+                      if remaining_users.empty?
+                        # Stop the ControlMaster
+                        log_debug "[ ControlMaster - #{ssh_url} ] - Stopping ControlMaster..."
+                        # Dumb verbose ssh! Tricky trick to just silence what is useless.
+                        # Don't fail if the connection close fails (but still log the error), as it can be seen as only a warning: it means the connection was closed anyway.
+                        @cmd_runner.run_cmd "#{ssh_exec_for(node)} -O exit #{ssh_url} 2>&1 | grep -v 'Exit request sent.'", log_to_stdout: log_debug?, expected_code: 1, timeout: timeout, no_exception: true
+                        log_debug "[ ControlMaster - #{ssh_url} ] - ControlMaster stopped"
+                        # Uncomment if you want to test that the connection has been closed
+                        # @cmd_runner.run_cmd "#{ssh_exec_for(node)} -O check #{ssh_url}", log_to_stdout: log_debug?, expected_code: 255, timeout: timeout
+                      else
+                        log_debug "[ ControlMaster - #{ssh_url} ] - Leaving ControlMaster started as #{remaining_users.size} processes/threads are still using it."
+                      end
+                      false
                     end
-                    false
                   end
                 end
               end
@@ -562,8 +629,9 @@ module HybridPlatformsConductor
             # TODO: Add test case when control file is missing ad when it is stale
             # Get the list of existing process/thread ids using this control master
             existing_users = File.exist?(control_master_users_file) ? File.read(control_master_users_file).split("\n") : []
-            ssh_url = "#{@ssh_user}@hpc.#{node}"
-            control_path_file = control_master_file(connection_info_for(node).first, '22', @ssh_user)
+            ssh_url = "hpc.#{node}"
+            connection, connection_user, _gateway, _gateway_user = connection_info_for(node)
+            control_path_file = control_master_file(connection, '22', connection_user)
             if existing_users.empty?
               # Make sure there is no stale one.
               if File.exist?(control_path_file)
@@ -595,7 +663,7 @@ module HybridPlatformsConductor
         # * *port* (String): The port. Can be a string as ssh config uses wildchars.
         # * *user* (String): The user
         def control_master_file(host, port, user)
-          "#{@tmp_dir}/hpc_actions_executor_mux_#{host}_#{port}_#{user}"
+          "#{@tmp_dir}/hpc_ssh_mux_#{host}_#{port}_#{user}"
         end
 
         # Provide a bootstrapped ssh executable that includes an SSH config allowing access to nodes.
@@ -627,7 +695,7 @@ module HybridPlatformsConductor
                 nodes.sort.each do |node|
                   host_keys = @nodes_handler.get_host_keys_of(node)
                   if host_keys && !host_keys.empty?
-                    connection, _gateway, _gateway_user = connection_info_for(node)
+                    connection, _connection_user, _gateway, _gateway_user = connection_info_for(node)
                     host_keys.each do |host_key|
                       file.puts "#{connection} #{host_key}"
                     end
@@ -660,11 +728,13 @@ module HybridPlatformsConductor
         #
         # Parameters::
         # * *node* (String): The node to access
+        # * *no_exception* (Boolean): Should we skip exceptions in case of no connection possible? [default: false]
         # Result::
-        # * String: The real hostname or IP to be used to connect
+        # * String: The real hostname or IP to be used to connect, or nil if none and no_exception is true
+        # * String: The real user to be used to connect, or nil if none and no_exception is true
         # * String or nil: The gateway name to be used (should be defined by the gateways configurations), or nil if no gateway to be used.
         # * String or nil: The gateway user to be used, or nil if none.
-        def connection_info_for(node)
+        def connection_info_for(node, no_exception: false)
           connection =
             if @nodes_handler.get_host_ip_of(node)
               @nodes_handler.get_host_ip_of(node)
@@ -673,12 +743,18 @@ module HybridPlatformsConductor
             elsif @nodes_handler.get_hostname_of(node)
               @nodes_handler.get_hostname_of(node)
             else
-              raise NotConnectableError, "No connection possible to #{node}"
+              nil
             end
+          connection_user = @ssh_user
           gateway = @nodes_handler.get_gateway_of node
           gateway_user = @nodes_handler.get_gateway_user_of node
           gateway_user = @ssh_gateway_user if !gateway.nil? && gateway_user.nil?
-          [connection, gateway, gateway_user]
+          # In case we want to transform the connection info, do it here.
+          @nodes_handler.select_confs_for_node(node, @config.ssh_connection_transforms).each do |transform_info|
+            connection, connection_user, gateway, gateway_user = transform_info[:transform].call(node, connection, connection_user, gateway, gateway_user)
+          end
+          raise NotConnectableError, "No connection possible to #{node}" if connection.nil? && !no_exception
+          [connection, connection_user, gateway, gateway_user]
         end
 
         # Get the possible SSH aliases for a given node.

data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_deploy_and_idempotence.rb

@@ -54,17 +54,19 @@ module HybridPlatformsConductor
                 instance.stop
                 instance.with_running_instance(port: 22) do
 
-                  # ===== Deploy removes root access
-                  # Check that we can't connect with root
-                  ssh_ok = false
-                  begin
-                    Net::SSH.start(instance.ip, 'root', password: 'root_pwd', auth_methods: ['password'], verify_host_key: :never) do |ssh|
-                      ssh_ok = ssh.exec!('echo Works').strip == 'Works'
+                  unless @nodes_handler.get_root_access_allowed_of(@node) == 'true'
+                    # ===== Deploy removes root access
+                    # Check that we can't connect with root
+                    ssh_ok = false
+                    begin
+                      Net::SSH.start(instance.ip, 'root', password: 'root_pwd', auth_methods: ['password'], verify_host_key: :never) do |ssh|
+                        ssh_ok = ssh.exec!('echo Works').strip == 'Works'
+                      end
+                    rescue
                     end
-                  rescue
+                    assert_equal ssh_ok, false, 'Root can still connect on the image after deployment'
+                    # Even if we can connect using root, run the idempotence test
                   end
-                  assert_equal ssh_ok, false, 'Root can still connect on the image after deployment'
-                  # Even if we can connect using root, run the idempotence test
 
                   # ===== Idempotence
                   unless ssh_ok

data/lib/hybrid_platforms_conductor/provisioner.rb

@@ -76,6 +76,15 @@ module HybridPlatformsConductor
             # Make sure we update it.
             @nodes_handler.override_metadata_of @node, :host_ip, instance_ip
             @nodes_handler.invalidate_metadata_of @node, :host_keys
+            # Make sure the SSH transformations don't apply to this node
+            @config.ssh_connection_transforms.replace(@config.ssh_connection_transforms.map do |ssh_transform_info|
+              {
+                nodes_selectors_stack: ssh_transform_info[:nodes_selectors_stack].map do |nodes_selector|
+                  @nodes_handler.select_nodes(nodes_selector).select { |selected_node| selected_node != @node }
+                end,
+                transform: ssh_transform_info[:transform]
+              }
+            end)
           end
           wait_for_port!(port) if port
           yield

data/lib/hybrid_platforms_conductor/version.rb

@@ -1,5 +1,5 @@
 module HybridPlatformsConductor
 
-  VERSION = '32.5.0'
+  VERSION = '32.7.3'
 
 end