hybrid_platforms_conductor 32.3.6 → 32.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e16a6630a51ed9208fa4a3cecfce6604c65a39cb782d9efa426898ec980d5071
-  data.tar.gz: 33f38588a1b795e323f76e909ef301905cacf9d77c0a418ac0809713960659b6
+  metadata.gz: d6b6259b1f05bb082b4f2c6820f74b6c90dad15f5868a311337429b80d233500
+  data.tar.gz: c5f58e3ba104ba2365addea64ab08c43a4fe9797430c950bf01258eb7bfcdc1b
 SHA512:
-  metadata.gz: 16a3b130741673e1ce4bfee15a5a4d8c45c66e5712e130a746f2d6be1e9940a1052a2c9195cce748831ca1652cc52255002226b31c0494516250b5359074c592
-  data.tar.gz: 920f8a21967bc69ecaf6072eec422e119e989415ddcd26d9e42e2d38779a35ddcfe58f5819cd3165c12799cf297dc5530da9a8d80ea6ae8c6a81c662d62c8fda
+  metadata.gz: b079e89a17630ed994614bc24e76ba2154ae52fcaf3712feaaef9e823487d933d6b33b87b3aa253e8d11270d5765d061349c4a37f467e81d729d994ed1f7b72e
+  data.tar.gz: 4a89e197bed2c87dbc10d66d8a3f6240c68deb7c2799962be282634c3c109b653c2a66725baa5ca44e38fd0bf1ec992148720047668c1c7786bd494a8e3836e7

data/bin/nodes_to_deploy

@@ -72,11 +72,17 @@ unless ignore_deploy_info
         commit_id = node_deploy_info["commit_id_#{repo_idx}".to_sym]
         impacted_nodes = cache_impacted_nodes.dig(repo_name, commit_id)
         if impacted_nodes.nil?
-          impacted_nodes, _single_impacted_nodes, _impacted_services, _impact_global = nodes_handler.impacted_nodes_from_git_diff(
-            repo_name,
-            from_commit: commit_id,
-            to_commit: 'master'
-          )
+          begin
+            impacted_nodes, _single_impacted_nodes, _impacted_services, _impact_global = nodes_handler.impacted_nodes_from_git_diff(
+              repo_name,
+              from_commit: commit_id,
+              to_commit: 'master'
+            )
+          rescue HybridPlatformsConductor::NodesHandler::GitError
+            # Consider the node was deployed with a non-release branch commit (as it is missing)
+            # So we have to make sure we deploy it again
+            impacted_nodes = [node]
+          end
           cache_impacted_nodes[repo_name] = {} unless cache_impacted_nodes.key?(repo_name)
           cache_impacted_nodes[repo_name][commit_id] = impacted_nodes
         end

data/lib/hybrid_platforms_conductor/deployer.rb

@@ -18,8 +18,31 @@ module HybridPlatformsConductor
   # Gives ways to deploy on several nodes
   class Deployer
 
+    # Extend the Config DSL
+    module ConfigDSLExtension
+
+      # Integer: Timeout (in seconds) for packaging repositories
+      attr_reader :packaging_timeout_secs
+
+      # Mixin initializer
+      def init_deployer_config
+        @packaging_timeout_secs = 60
+      end
+
+      # Set the packaging timeout
+      #
+      # Parameters::
+      # * *packaging_timeout_secs* (Integer): The packaging timeout, in seconds
+      def packaging_timeout(packaging_timeout_secs)
+        @packaging_timeout_secs = packaging_timeout_secs
+      end
+
+    end
+
     include LoggerHelpers
 
+    Config.extend_config_dsl_with ConfigDSLExtension, :init_nodes_handler_config
+
     # Do we use why-run mode while deploying? [default = false]
     #   Boolean
     attr_accessor :use_why_run
@@ -135,9 +158,6 @@ module HybridPlatformsConductor
     # String: File used as a Futex for packaging
     PACKAGING_FUTEX_FILE = "#{Dir.tmpdir}/hpc_packaging"
 
-    # Integer: Timeout in seconds to get the packaging Futex
-    PACKAGING_FUTEX_TIMEOUT = 60
-
     # Deploy on a given list of nodes selectors.
     # The workflow is the following:
     # 1. Package the services to be deployed, considering the nodes, services and context (options, secrets, environment...)
@@ -176,7 +196,7 @@ module HybridPlatformsConductor
 
       # Package the deployment
       # Protect packaging by a Futex
-      Futex.new(PACKAGING_FUTEX_FILE, timeout: PACKAGING_FUTEX_TIMEOUT).open do
+      Futex.new(PACKAGING_FUTEX_FILE, timeout: @config.packaging_timeout_secs).open do
         section 'Packaging deployment' do
           @services_handler.package(
             services: services_to_deploy,
@@ -460,6 +480,7 @@ module HybridPlatformsConductor
       outputs = @actions_executor.execute_actions(
         Hash[services.map do |node, node_services|
           image_id = @nodes_handler.get_image_of(node)
+          sudo = (ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} ")
           # Install My_company corporate certificates if present
           certificate_actions =
             if @local_environment && ENV['hpc_certificates']
@@ -469,20 +490,20 @@ module HybridPlatformsConductor
                 when 'debian_9', 'debian_10'
                   [
                     {
-                      remote_bash: "#{ssh_user == 'root' ? '' : 'sudo '}apt update && #{ssh_user == 'root' ? '' : 'sudo '}apt install -y ca-certificates"
+                      remote_bash: "#{sudo}apt update && #{sudo}apt install -y ca-certificates"
                     },
                     {
                       scp: {
                         ENV['hpc_certificates'] => '/usr/local/share/ca-certificates',
                         :sudo => ssh_user != 'root'
                       },
-                      remote_bash: "#{ssh_user == 'root' ? '' : 'sudo '}update-ca-certificates"
+                      remote_bash: "#{sudo}update-ca-certificates"
                     }
                   ]
                 when 'centos_7'
                   [
                     {
-                      remote_bash: "#{ssh_user == 'root' ? '' : 'sudo '}yum install -y ca-certificates"
+                      remote_bash: "#{sudo}yum install -y ca-certificates"
                     },
                     {
                       scp: Hash[Dir.glob("#{ENV['hpc_certificates']}/*.crt").map do |cert_file|
@@ -492,8 +513,8 @@ module HybridPlatformsConductor
                         ]
                       end].merge(sudo: ssh_user != 'root'),
                       remote_bash: [
-                        "#{ssh_user == 'root' ? '' : 'sudo '}update-ca-trust enable",
-                        "#{ssh_user == 'root' ? '' : 'sudo '}update-ca-trust extract"
+                        "#{sudo}update-ca-trust enable",
+                        "#{sudo}update-ca-trust extract"
                       ]
                     }
                   ]
@@ -512,7 +533,7 @@ module HybridPlatformsConductor
               # Install the mutex lock and acquire it
               {
                 scp: { "#{__dir__}/mutex_dir" => '.' },
-                remote_bash: "while ! #{ssh_user == 'root' ? '' : 'sudo '}./mutex_dir lock /tmp/hybrid_platforms_conductor_deploy_lock \"$(ps -o ppid= -p $$)\"; do echo -e 'Another deployment is running on #{node}. Waiting for it to finish to continue...' ; sleep 5 ; done"
+                remote_bash: "while ! #{sudo}./mutex_dir lock /tmp/hybrid_platforms_conductor_deploy_lock \"$(ps -o ppid= -p $$)\"; do echo -e 'Another deployment is running on #{node}. Waiting for it to finish to continue...' ; sleep 5 ; done"
               }
             ] +
               certificate_actions +
@@ -528,7 +549,7 @@ module HybridPlatformsConductor
         Hash[services.keys.map do |node|
           [
             node,
-            { remote_bash: "#{ssh_user == 'root' ? '' : 'sudo '}./mutex_dir unlock /tmp/hybrid_platforms_conductor_deploy_lock" }
+            { remote_bash: "#{ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} "}./mutex_dir unlock /tmp/hybrid_platforms_conductor_deploy_lock" }
           ]
         end],
         timeout: 10,
@@ -575,7 +596,7 @@ module HybridPlatformsConductor
               [
                 node,
                 {
-                  remote_bash: "#{ssh_user == 'root' ? '' : 'sudo '}mkdir -p /var/log/deployments",
+                  remote_bash: "#{ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} "}mkdir -p /var/log/deployments",
                   scp: {
                     log_file => '/var/log/deployments',
                     :sudo => ssh_user != 'root',

data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb

@@ -10,13 +10,46 @@ module HybridPlatformsConductor
       # Connect to node using SSH
       class Ssh < HybridPlatformsConductor::Connector
 
+        # Exception raise when a node is not connectable using SSH
+        class NotConnectableError < RuntimeError
+        end
+
         module PlatformsDslSsh
 
+          # List of SSH connection transformations:
+          # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
+          # * *transform* (Proc): Code called to transform SSH connection info:
+          #   Parameters::
+          #   * *node* (String): Node for which we transform the SSH connection
+          #   * *connection* (String or nil): The connection host or IP, or nil if none
+          #   * *connection_user* (String): The connection user
+          #   * *gateway* (String or nil): The gateway name, or nil if none
+          #   * *gateway_user* (String or nil): The gateway user, or nil if none
+          #   Result::
+          #   * String: The transformed connection host or IP, or nil if none
+          #   * String: The transformed connection user
+          #   * String or nil: The transformed gateway name, or nil if none
+          #   * String or nil: The transformed gateway user, or nil if none
+          # Array< Hash<Symbol, Object> >
+          attr_reader :ssh_connection_transforms
+
           # Initialize the DSL
           def init_ssh
             # List of gateway configurations, per gateway config name
             # Hash<Symbol, String>
             @gateways = {}
+            @ssh_connection_transforms = []
+          end
+
+          # Define a transformation of SSH connection.
+          #
+          # Parameters::
+          # * *transform* (Proc): Code to be called to transform an SSH connection (see ssh_connection_transforms signature for details)
+          def transform_ssh_connection(&transform)
+            @ssh_connection_transforms << {
+              nodes_selectors_stack: current_nodes_selectors_stack,
+              transform: transform
+            }
           end
 
           # Register a new gateway configuration
@@ -268,7 +301,7 @@ module HybridPlatformsConductor
               #{File.basename(from)} | \
             #{ssh_exec} \
               #{ssh_url} \
-              \"#{sudo ? 'sudo ' : ''}tar \
+              \"#{sudo ? "#{@nodes_handler.sudo_on(@node)} " : ''}tar \
                 --extract \
                 --gunzip \
                 --file - \
@@ -291,7 +324,7 @@ module HybridPlatformsConductor
         # Result::
         # * String: The ssh URL connecting to the current node
         def ssh_url
-          "#{@ssh_user}@hpc.#{@node}"
+          "hpc.#{@node}"
         end
 
         # Get an SSH configuration content giving access to nodes of the platforms with the current configuration
@@ -314,14 +347,6 @@ module HybridPlatformsConductor
             # ENDPOINTS #
             #############
 
-            Host *
-              User #{@ssh_user}
-              # Default control socket path to be used when multiplexing SSH connections
-              ControlPath #{control_master_file('%h', '%p', '%r')}
-              #{open_ssh_major_version >= 7 ? 'PubkeyAcceptedKeyTypes +ssh-dss' : ''}
-              #{known_hosts_file.nil? ? '' : "UserKnownHostsFile #{known_hosts_file}"}
-              #{@ssh_strict_host_key_checking ? '' : 'StrictHostKeyChecking no'}
-
           EOS
 
           # Add each node
@@ -329,17 +354,37 @@ module HybridPlatformsConductor
           @nodes_handler.prefetch_metadata_of nodes, %i[private_ips hostname host_ip description]
           nodes.sort.each do |node|
             # Generate the conf for the node
-            connection, gateway, gateway_user = connection_info_for(node)
-            config_content << "# #{node} - #{connection} - #{@nodes_handler.get_description_of(node) || ''}\n"
-            config_content << "Host #{ssh_aliases_for(node).join(' ')}\n"
-            config_content << "  Hostname #{connection}\n"
-            config_content << "  ProxyCommand #{ssh_exec} -q -W %h:%p #{gateway_user}@#{gateway}\n" unless gateway.nil?
-            if @passwords.key?(node)
-              config_content << "  PreferredAuthentications password\n"
-              config_content << "  PubkeyAuthentication no\n"
+            connection, connection_user, gateway, gateway_user = connection_info_for(node, no_exception: true)
+            if connection.nil?
+              config_content << "# #{node} - Not connectable using SSH - #{@nodes_handler.get_description_of(node) || ''}\n"
+            else
+              config_content << "# #{node} - #{connection} - #{@nodes_handler.get_description_of(node) || ''}\n"
+              config_content << "Host #{ssh_aliases_for(node).join(' ')}\n"
+              config_content << "  Hostname #{connection}\n"
+              config_content << "  User \"#{connection_user}\"\n" if connection_user != @ssh_user
+              config_content << "  ProxyCommand #{ssh_exec} -q -W %h:%p #{gateway_user}@#{gateway}\n" unless gateway.nil?
+              if @passwords.key?(node)
+                config_content << "  PreferredAuthentications password\n"
+                config_content << "  PubkeyAuthentication no\n"
+              end
             end
             config_content << "\n"
           end
+          # Add global definitions at the end of the SSH config, as they might be overriden by previous ones, and first match wins.
+          config_content << <<~EOS
+            ###########
+            # GLOBALS #
+            ###########
+
+            Host *
+              User #{@ssh_user}
+              # Default control socket path to be used when multiplexing SSH connections
+              ControlPath #{control_master_file('%h', '%p', '%r')}
+              #{open_ssh_major_version >= 7 ? 'PubkeyAcceptedKeyTypes +ssh-dss' : ''}
+              #{known_hosts_file.nil? ? '' : "UserKnownHostsFile #{known_hosts_file}"}
+              #{@ssh_strict_host_key_checking ? '' : 'StrictHostKeyChecking no'}
+
+          EOS
           config_content
         end
 
@@ -442,7 +487,7 @@ module HybridPlatformsConductor
                   with_lock_on_control_master_for(node) do |current_users, user_id|
                     working_master = false
                     ssh_exec = ssh_exec_for(node)
-                    ssh_url = "#{@ssh_user}@hpc.#{node}"
+                    ssh_url = "hpc.#{node}"
                     if current_users.empty?
                       log_debug "[ ControlMaster - #{ssh_url} ] - Creating SSH ControlMaster..."
                       # Create the control master
@@ -510,7 +555,7 @@ module HybridPlatformsConductor
               user_locks_mutex.synchronize do
                 user_locks.each do |node, user_id|
                   with_lock_on_control_master_for(node, user_id: user_id) do |current_users, user_id|
-                    ssh_url = "#{@ssh_user}@hpc.#{node}"
+                    ssh_url = "hpc.#{node}"
                     log_warn "[ ControlMaster - #{ssh_url} ] - Current process/thread was not part of the ControlMaster users anymore whereas it should have been" unless current_users.include?(user_id)
                     remaining_users = current_users - [user_id]
                     if remaining_users.empty?
@@ -554,8 +599,9 @@ module HybridPlatformsConductor
             # TODO: Add test case when control file is missing ad when it is stale
             # Get the list of existing process/thread ids using this control master
             existing_users = File.exist?(control_master_users_file) ? File.read(control_master_users_file).split("\n") : []
-            ssh_url = "#{@ssh_user}@hpc.#{node}"
-            control_path_file = control_master_file(connection_info_for(node).first, '22', @ssh_user)
+            ssh_url = "hpc.#{node}"
+            connection, connection_user, _gateway, _gateway_user = connection_info_for(node)
+            control_path_file = control_master_file(connection, '22', connection_user)
             if existing_users.empty?
               # Make sure there is no stale one.
               if File.exist?(control_path_file)
@@ -587,7 +633,7 @@ module HybridPlatformsConductor
         # * *port* (String): The port. Can be a string as ssh config uses wildchars.
         # * *user* (String): The user
         def control_master_file(host, port, user)
-          "#{@tmp_dir}/hpc_actions_executor_mux_#{host}_#{port}_#{user}"
+          "#{@tmp_dir}/hpc_ssh_mux_#{host}_#{port}_#{user}"
         end
 
         # Provide a bootstrapped ssh executable that includes an SSH config allowing access to nodes.
@@ -619,7 +665,7 @@ module HybridPlatformsConductor
                 nodes.sort.each do |node|
                   host_keys = @nodes_handler.get_host_keys_of(node)
                   if host_keys && !host_keys.empty?
-                    connection, _gateway, _gateway_user = connection_info_for(node)
+                    connection, _connection_user, _gateway, _gateway_user = connection_info_for(node)
                     host_keys.each do |host_key|
                       file.puts "#{connection} #{host_key}"
                     end
@@ -652,11 +698,13 @@ module HybridPlatformsConductor
         #
         # Parameters::
         # * *node* (String): The node to access
+        # * *no_exception* (Boolean): Should we skip exceptions in case of no connection possible? [default: false]
         # Result::
-        # * String: The real hostname or IP to be used to connect
+        # * String: The real hostname or IP to be used to connect, or nil if none and no_exception is true
+        # * String: The real user to be used to connect, or nil if none and no_exception is true
         # * String or nil: The gateway name to be used (should be defined by the gateways configurations), or nil if no gateway to be used.
         # * String or nil: The gateway user to be used, or nil if none.
-        def connection_info_for(node)
+        def connection_info_for(node, no_exception: false)
           connection =
             if @nodes_handler.get_host_ip_of(node)
               @nodes_handler.get_host_ip_of(node)
@@ -665,12 +713,18 @@ module HybridPlatformsConductor
             elsif @nodes_handler.get_hostname_of(node)
               @nodes_handler.get_hostname_of(node)
             else
-              raise "No connection possible to #{node}"
+              nil
             end
+          connection_user = @ssh_user
           gateway = @nodes_handler.get_gateway_of node
           gateway_user = @nodes_handler.get_gateway_user_of node
           gateway_user = @ssh_gateway_user if !gateway.nil? && gateway_user.nil?
-          [connection, gateway, gateway_user]
+          # In case we want to transform the connection info, do it here.
+          @nodes_handler.select_confs_for_node(node, @config.ssh_connection_transforms).each do |transform_info|
+            connection, connection_user, gateway, gateway_user = transform_info[:transform].call(node, connection, connection_user, gateway, gateway_user)
+          end
+          raise NotConnectableError, "No connection possible to #{node}" if connection.nil? && !no_exception
+          [connection, connection_user, gateway, gateway_user]
         end
 
         # Get the possible SSH aliases for a given node.

data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/platform_handler_plugin.rb.sample

@@ -173,7 +173,7 @@ module HybridPlatformsConductor
         # * *local_environment* (Boolean): Are we deploying to a local environment?
         def package(services:, secrets:, local_environment:)
           # This method should take all actions to prepare the repository to be deployed on nodes later.
-          File.write('temporary_secrets_to_be_deployed.json', secrets)
+          File.write("#{@repository_path}/temporary_secrets_to_be_deployed.json", secrets.to_json)
           # Usually it is meant to package the deployment scripts.
           @cmd_runner.run_cmd "cd #{@repository_path} && ./scripts/package_in_debian_format.sh"
         end

data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb

@@ -404,7 +404,7 @@ module HybridPlatformsConductor
                 {
                   proxmox_test_info[:sync_node] => {
                     remote_bash: {
-                      commands: "#{@actions_executor.connector(:ssh).ssh_user == 'root' ? '' : 'sudo -E '}./proxmox/#{cmd}",
+                      commands: "#{@actions_executor.connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(proxmox_test_info[:sync_node])} -E "}./proxmox/#{cmd}",
                       env: {
                         'hpc_user_for_proxmox' => user,
                         'hpc_password_for_proxmox' => password,

data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_freshness.rb

@@ -15,7 +15,7 @@ module HybridPlatformsConductor
         def test_on_node
           now = Time.now
           {
-            'sudo ls -t /var/log/deployments' => proc do |stdout|
+            "#{@nodes_handler.sudo_on(@node)} ls -t /var/log/deployments" => proc do |stdout|
               if stdout.empty?
                 error 'Node has never been deployed using deploy (/var/log/deployments is empty)'
               elsif stdout.first =~ /No such file or directory/

data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb

@@ -17,7 +17,7 @@ module HybridPlatformsConductor
           Hash[
             @config.aggregate_files_rules(@nodes_handler, @node).map do |path, rule_info|
               [
-                "if sudo /bin/bash -c '[[ -d \"#{path}\" ]]' ; then echo 1 ; else echo 0 ; fi",
+                "if #{@nodes_handler.sudo_on(@node)} /bin/bash -c '[[ -d \"#{path}\" ]]' ; then echo 1 ; else echo 0 ; fi",
                 {
                   validator: proc do |stdout, stderr|
                     case stdout.last

data/lib/hybrid_platforms_conductor/hpc_plugins/test/hostname.rb

@@ -10,7 +10,7 @@ module HybridPlatformsConductor
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_node
           {
-            'sudo hostname -s' => proc do |stdout|
+            "#{@nodes_handler.sudo_on(@node)} hostname -s" => proc do |stdout|
               assert_equal stdout.first, @node, "Expected hostname to be #{@node}, but got #{stdout.first} instead."
             end
           }

data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb

@@ -10,7 +10,7 @@ module HybridPlatformsConductor
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_node
           {
-            'sudo hostname -I' => proc do |stdout|
+            "#{@nodes_handler.sudo_on(@node)} hostname -I" => proc do |stdout|
               if stdout.first.nil?
                 error 'No IP returned by "hostname -I"'
               else

data/lib/hybrid_platforms_conductor/hpc_plugins/test/local_users.rb

@@ -57,7 +57,7 @@ module HybridPlatformsConductor
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_node
           {
-            "sudo cat /etc/passwd" => proc do |stdout|
+            "#{@nodes_handler.sudo_on(@node)} cat /etc/passwd" => proc do |stdout|
               passwd_users = stdout.map { |passwd_line| passwd_line.split(':').first }
               missing_users = @nodes_handler.
                 select_confs_for_node(@node, @config.users_that_should_be_present).

data/lib/hybrid_platforms_conductor/hpc_plugins/test/mounts.rb

@@ -61,7 +61,7 @@ module HybridPlatformsConductor
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_node
           {
-            'sudo mount' => proc do |stdout|
+            "#{@nodes_handler.sudo_on(@node)} mount" => proc do |stdout|
               mounts_info = stdout.map do |line|
                 fields = line.split
                 {

data/lib/hybrid_platforms_conductor/hpc_plugins/test/orphan_files.rb

@@ -50,7 +50,7 @@ module HybridPlatformsConductor
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_node
           {
-            "sudo /usr/bin/find / \\( #{@nodes_handler.
+            "#{@nodes_handler.sudo_on(@node)} /usr/bin/find / \\( #{@nodes_handler.
               select_confs_for_node(@node, @config.ignored_orphan_files_paths).
               inject(DIRECTORIES_TO_ALWAYS_IGNORE) { |merged_paths, paths_to_ignore_info| merged_paths + paths_to_ignore_info[:ignored_paths] }.
               uniq.