hybrid_platforms_conductor 33.3.0 → 33.7.0

data/lib/hybrid_platforms_conductor/deployer.rb

@@ -106,10 +106,10 @@ module HybridPlatformsConductor
 
     end
 
-    include LoggerHelpers
-
     Config.extend_config_dsl_with ConfigDSLExtension, :init_deployer_config
 
+    include LoggerHelpers
+
     # Do we use why-run mode while deploying? [default = false]
     #   Boolean
     attr_accessor :use_why_run
@@ -544,15 +544,13 @@ module HybridPlatformsConductor
     # Result::
     # * Hash<String, [Integer or Symbol, String, String]>: Exit status code (or Symbol in case of error or dry run), standard output and error for each node.
     def deploy(services)
-      # Get the ssh user directly from the connector
-      ssh_user = @actions_executor.connector(:ssh).ssh_user
-
       # Deploy for real
       @nodes_handler.prefetch_metadata_of services.keys, :image
       outputs = @actions_executor.execute_actions(
         services.map do |node, node_services|
           image_id = @nodes_handler.get_image_of(node)
-          sudo = (ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} ")
+          need_sudo = !@actions_executor.privileged_access?(node)
+          sudo = @actions_executor.sudo_prefix(node)
           # Install corporate certificates if present
           certificate_actions =
             if @local_environment && ENV['hpc_certificates']
@@ -568,7 +566,7 @@ module HybridPlatformsConductor
                   {
                     scp: {
                       ENV['hpc_certificates'] => '/usr/local/share/ca-certificates',
-                      :sudo => ssh_user != 'root'
+                      :sudo => need_sudo
                     },
                     remote_bash: "#{sudo}update-ca-certificates"
                   }
@@ -584,7 +582,7 @@ module HybridPlatformsConductor
                         cert_file,
                         '/etc/pki/ca-trust/source/anchors'
                       ]
-                    end.to_h.merge(sudo: ssh_user != 'root'),
+                    end.to_h.merge(sudo: need_sudo),
                     remote_bash: [
                       "#{sudo}update-ca-trust enable",
                       "#{sudo}update-ca-trust extract"
@@ -619,7 +617,7 @@ module HybridPlatformsConductor
         services.keys.map do |node|
           [
             node,
-            { remote_bash: "#{ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} "}./mutex_dir unlock /tmp/hybrid_platforms_conductor_deploy_lock" }
+            { remote_bash: "#{@actions_executor.sudo_prefix(node)}./mutex_dir unlock /tmp/hybrid_platforms_conductor_deploy_lock" }
           ]
         end.to_h,
         timeout: 10,

data/lib/hybrid_platforms_conductor/github.rb

@@ -0,0 +1,39 @@
+require 'octokit'
+require 'hybrid_platforms_conductor/credentials'
+
+module HybridPlatformsConductor
+
+  # Mixin used to access Github API
+  module Github
+
+    include Credentials
+
+    # Iterate over each Github repository
+    #
+    # Parameters::
+    # * Proc: Code called for each Github repository:
+    #   * Parameters::
+    #     * *github* (Octokit::Client): The client instance accessing the Github API
+    #     * *repo_info* (Hash<Symbol, Object>): The repository info:
+    #       * *name* (String): Repository name.
+    #       * *slug* (String): Repository slug.
+    def for_each_github_repo
+      @config.known_github_repos.each do |repo_info|
+        Octokit.configure do |c|
+          c.api_endpoint = repo_info[:url]
+        end
+        with_credentials_for(:github, resource: repo_info[:url]) do |_github_user, github_token|
+          client = Octokit::Client.new(access_token: github_token&.to_unprotected)
+          (repo_info[:repos] == :all ? client.repositories(repo_info[:user]).map { |repo| repo[:name] } : repo_info[:repos]).each do |name|
+            yield client, {
+              name: name,
+              slug: "#{repo_info[:user]}/#{name}"
+            }
+          end
+        end
+      end
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/action/bash.rb

@@ -14,7 +14,7 @@ module HybridPlatformsConductor
         # [API] - @actions_executor is accessible
         #
         # Parameters::
-        # * *cmd* (String): The bash command to execute
+        # * *cmd* (String or SecretString): The bash command to execute
         def setup(cmd)
           @cmd = cmd
         end

data/lib/hybrid_platforms_conductor/hpc_plugins/action/remote_bash.rb

@@ -15,30 +15,30 @@ module HybridPlatformsConductor
         #
         # Parameters::
         # * *remote_bash* (Array or Object): List of commands (or single command) to be executed. Each command can be the following:
-        #   * String: Simple bash command.
+        #   * String or SecretString: Simple bash command.
         #   * Hash<Symbol, Object>: Information about the commands to execute. Can have the following properties:
-        #     * *commands* (Array<String> or String): List of bash commands to execute (can be a single one) [default: ''].
+        #     * *commands* (Array<String or SecretString> or String or SecretString): List of bash commands to execute (can be a single one) [default: ''].
         #     * *file* (String): Name of file from which commands should be taken [optional].
-        #     * *env* (Hash<String, String>): Environment variables to be set before executing those commands [default: {}].
+        #     * *env* (Hash<String, String or SecretString>): Environment variables to be set before executing those commands [default: {}].
         def setup(remote_bash)
           # Normalize the parameters.
           # Array< Hash<Symbol,Object> >: Simple array of info:
-          # * *commands* (Array<String>): List of bash commands to execute.
-          # * *env* (Hash<String, String>): Environment variables to be set before executing those commands.
+          # * *commands* (Array<String or SecretString>): List of bash commands to execute.
+          # * *env* (Hash<String, String or SecretString>): Environment variables to be set before executing those commands.
           @remote_bash = (remote_bash.is_a?(Array) ? remote_bash : [remote_bash]).map do |cmd_info|
-            if cmd_info.is_a?(String)
-              {
-                commands: [cmd_info],
-                env: {}
-              }
-            else
+            if cmd_info.is_a?(Hash)
               commands = []
-              commands.concat(cmd_info[:commands].is_a?(String) ? [cmd_info[:commands]] : cmd_info[:commands]) if cmd_info[:commands]
+              commands.concat(cmd_info[:commands].is_a?(Array) ? cmd_info[:commands] : [cmd_info[:commands]]) if cmd_info[:commands]
               commands << File.read(cmd_info[:file]) if cmd_info[:file]
               {
                 commands: commands,
                 env: cmd_info[:env] || {}
               }
+            else
+              {
+                commands: [cmd_info],
+                env: {}
+              }
             end
           end
         end
@@ -63,11 +63,21 @@ module HybridPlatformsConductor
         # [API] - @stderr_io can be used to log stderr messages
         # [API] - run_cmd(String) method can be used to execute a command. See CmdRunner#run_cmd to know about the result's signature.
         def execute
-          bash_str = @remote_bash.map do |cmd_info|
-            (cmd_info[:env].map { |var_name, var_value| "export #{var_name}='#{var_value}'" } + cmd_info[:commands]).join("\n")
-          end.join("\n")
-          log_debug "[#{@node}] - Execute remote Bash commands \"#{bash_str}\"..."
-          @connector.remote_bash bash_str
+          # The commands or ENV variables can contain secrets, so make sure to protect all strings from secrets leaking
+          bash_cmds = @remote_bash.map do |cmd_info|
+            cmd_info[:env].map do |var_name, var_value|
+              SecretString.new("export #{var_name}='#{var_value.to_unprotected}'", silenced_str: "export #{var_name}='#{var_value}'")
+            end + cmd_info[:commands]
+          end.flatten
+          begin
+            SecretString.protect(bash_cmds.map(&:to_unprotected).join("\n"), silenced_str: bash_cmds.join("\n")) do |bash_str|
+              log_debug "[#{@node}] - Execute remote Bash commands \"#{bash_str}\"..."
+              @connector.remote_bash bash_str
+            end
+          ensure
+            # Make sure we erase all secret strings
+            bash_cmds.each(&:erase)
+          end
         end
 
       end

data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_keys.rb

@@ -24,7 +24,7 @@ module HybridPlatformsConductor
         # * Hash<Symbol, Symbol or Array<Symbol> >: The list of necessary properties (or single one) that should be set, per property name (:others can also be used here)
         def property_dependencies
           {
-            host_keys: %i[hostname host_ip]
+            host_keys: %i[hostname host_ip ssh_port]
           }
         end
 
@@ -41,19 +41,20 @@ module HybridPlatformsConductor
         #     Nodes for which the property can't be fetched can be ommitted.
         def get_host_keys(_nodes, metadata)
           updated_metadata = {}
-          # Get the list of nodes, per hostname (just in case several nodes share the same hostname)
-          # Hash<String, Array<String> >
+          # Get the list of nodes, per [hostname, port] (just in case several nodes share the same hostname and port)
+          # Hash<[String, Integer], Array<String> >
           hostnames = Hash.new { |hash, key| hash[key] = [] }
           metadata.each do |node, node_metadata|
+            ssh_port = node_metadata[:ssh_port] || 22
             if node_metadata[:host_ip]
-              hostnames[node_metadata[:host_ip]] << node
+              hostnames[[node_metadata[:host_ip], ssh_port]] << node
             elsif node_metadata[:hostname]
-              hostnames[node_metadata[:hostname]] << node
+              hostnames[[node_metadata[:hostname], ssh_port]] << node
             end
           end
           unless hostnames.empty?
-            host_keys_for(*hostnames.keys).each do |hostname, ip|
-              hostnames[hostname].each do |node|
+            host_keys_for(*hostnames.keys).each do |host_id, ip|
+              hostnames[host_id].each do |node|
                 updated_metadata[node] = ip
               end
             end
@@ -71,7 +72,7 @@ module HybridPlatformsConductor
         # Discover the host keys associated to a list of hosts.
         #
         # Parameters::
-        # * *hosts* (Array<String>): The hosts to check for
+        # * *hosts* (Array<[String, Integer]>): The hosts to check for ([hostname, port])
         # Result::
         # * Hash<String, Array<String> >: The corresponding host keys, per host name
         def host_keys_for(*hosts)
@@ -82,9 +83,9 @@ module HybridPlatformsConductor
             parallel: true,
             nbr_threads_max: MAX_THREADS_SSH_KEY_SCAN,
             progress: log_debug? ? 'Gather host keys' : nil
-          ) do |host|
+          ) do |(host, ssh_port)|
             exit_status, stdout, _stderr = @cmd_runner.run_cmd(
-              "ssh-keyscan #{host}",
+              "ssh-keyscan -p #{ssh_port} #{host}",
               timeout: TIMEOUT_SSH_KEYSCAN,
               log_to_stdout: log_debug?,
               no_exception: true
@@ -97,9 +98,9 @@ module HybridPlatformsConductor
                   found_keys << "#{type} #{key}"
                 end
               end
-              results[host] = found_keys.sort unless found_keys.empty?
+              results[[host, ssh_port]] = found_keys.sort unless found_keys.empty?
             else
-              log_warn "Unable to get host key for #{host}. Ignoring it. Accessing #{host} might require manual acceptance of its host key."
+              log_warn "Unable to get host key for #{host} (port #{ssh_port}). Ignoring it. Accessing #{host} might require manual acceptance of its host key."
             end
           end
           results

data/lib/hybrid_platforms_conductor/hpc_plugins/connector/local.rb

@@ -35,9 +35,11 @@ module HybridPlatformsConductor
         # [API] - @stderr_io can be used to send stderr output
         #
         # Parameters::
-        # * *bash_cmds* (String): Bash commands to execute
+        # * *bash_cmds* (String or SecretString): Bash commands to execute. Use #to_unprotected to access the real content (otherwise secrets are obfuscated).
         def remote_bash(bash_cmds)
-          run_cmd "cd #{workspace_for(@node)} ; #{bash_cmds}", force_bash: true
+          SecretString.protect("cd #{workspace_for(@node)} ; #{bash_cmds.to_unprotected}", silenced_str: "cd #{workspace_for(@node)} ; #{bash_cmds}") do |cmd|
+            run_cmd cmd, force_bash: true
+          end
         end
 
         # Execute an interactive shell on the remote node
@@ -75,8 +77,8 @@ module HybridPlatformsConductor
         def remote_copy(from, to, sudo: false, owner: nil, group: nil)
           # If the destination is a relative path, prepend the workspace dir to it.
           to = "#{workspace_for(@node)}/#{to}" unless to.start_with?('/')
-          if sudo
-            run_cmd "#{@nodes_handler.sudo_on(@node)} cp -r \"#{from}\" \"#{to}\""
+          if sudo && !@actions_executor.privileged_access?(@node)
+            run_cmd "#{@actions_executor.sudo_prefix(@node)}cp -r \"#{from}\" \"#{to}\""
           else
             FileUtils.cp_r from, to unless @cmd_runner.dry_run
           end

data/lib/hybrid_platforms_conductor/hpc_plugins/connector/my_connector.rb.sample

@@ -104,7 +104,7 @@ module HybridPlatformsConductor
         # [API] - @stderr_io can be used to send stderr output
         #
         # Parameters::
-        # * *bash_cmds* (String): Bash commands to execute
+        # * *bash_cmds* (String or SecretString): Bash commands to execute. Use #to_unprotected to access the real content (otherwise secrets are obfuscated).
         def remote_bash(bash_cmds)
           MyConnectLib.connect_to(@nodes_handler.get_host_ip_of(@node)).run_bash(bash_cmds)
         end

data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb

@@ -236,31 +236,40 @@ module HybridPlatformsConductor
         # [API] - @stderr_io can be used to send stderr output
         #
         # Parameters::
-        # * *bash_cmds* (String): Bash commands to execute
+        # * *bash_cmds* (String or SecretString): Bash commands to execute. Use #to_unprotected to access the real content (otherwise secrets are obfuscated).
         def remote_bash(bash_cmds)
-          ssh_cmd =
+          SecretString.protect(
             if @nodes_handler.get_ssh_session_exec_of(@node) == false
               # When ExecSession is disabled we need to use stdin directly
-              "{ cat | #{ssh_exec} #{ssh_url} -T; } <<'HPC_EOF'\n#{bash_cmds}\nHPC_EOF"
+              "{ cat | #{ssh_exec} #{ssh_url} -T; } <<'HPC_EOF'\n#{bash_cmds.to_unprotected}\nHPC_EOF"
             else
-              "#{ssh_exec} #{ssh_url} /bin/bash <<'HPC_EOF'\n#{bash_cmds}\nHPC_EOF"
-            end
-          # Due to a limitation of Process.spawn, each individual argument is limited to 128KB of size.
-          # Therefore we need to make sure that if bash_cmds exceeds MAX_CMD_ARG_LENGTH bytes (considering EOF chars) then we use an intermediary shell script to store the commands.
-          if bash_cmds.size > MAX_CMD_ARG_LENGTH
-            # Write the commands in a file
-            temp_file = "#{Dir.tmpdir}/hpc_temp_cmds_#{Digest::MD5.hexdigest(bash_cmds)}.sh"
-            File.open(temp_file, 'w+') do |file|
-              file.write ssh_cmd
-              file.chmod 0o700
-            end
-            begin
-              run_cmd(temp_file)
-            ensure
-              File.unlink(temp_file)
+              "#{ssh_exec} #{ssh_url} /bin/bash <<'HPC_EOF'\n#{bash_cmds.to_unprotected}\nHPC_EOF"
+            end,
+            silenced_str:
+              if @nodes_handler.get_ssh_session_exec_of(@node) == false
+                # When ExecSession is disabled we need to use stdin directly
+                "{ cat | #{ssh_exec} #{ssh_url} -T; } <<'HPC_EOF'\n#{bash_cmds}\nHPC_EOF"
+              else
+                "#{ssh_exec} #{ssh_url} /bin/bash <<'HPC_EOF'\n#{bash_cmds}\nHPC_EOF"
+              end
+          ) do |ssh_cmd|
+            # Due to a limitation of Process.spawn, each individual argument is limited to 128KB of size.
+            # Therefore we need to make sure that if bash_cmds exceeds MAX_CMD_ARG_LENGTH bytes (considering EOF chars) then we use an intermediary shell script to store the commands.
+            if bash_cmds.to_unprotected.size > MAX_CMD_ARG_LENGTH
+              # Write the commands in a file
+              temp_file = "#{Dir.tmpdir}/hpc_temp_cmds_#{Digest::MD5.hexdigest(bash_cmds.to_unprotected)}.sh"
+              File.open(temp_file, 'w+') do |file|
+                file.write ssh_cmd.to_unprotected
+                file.chmod 0o700
+              end
+              begin
+                run_cmd(temp_file)
+              ensure
+                File.unlink(temp_file)
+              end
+            else
+              run_cmd ssh_cmd
             end
-          else
-            run_cmd ssh_cmd
           end
         end
 
@@ -301,13 +310,14 @@ module HybridPlatformsConductor
         # * *owner* (String or nil): Owner to be used when copying the files, or nil for current one [default: nil]
         # * *group* (String or nil): Group to be used when copying the files, or nil for current one [default: nil]
         def remote_copy(from, to, sudo: false, owner: nil, group: nil)
+          need_sudo = sudo && !@actions_executor.privileged_access?(@node)
           if @nodes_handler.get_ssh_session_exec_of(@node) == false
             # We don't have ExecSession, so don't use ssh, but scp instead.
-            if sudo
+            if need_sudo
               # We need to first copy the file in an accessible directory, and then sudo mv
               remote_bash('mkdir -p hpc_tmp_scp')
               run_cmd "scp -S #{ssh_exec} #{from} #{ssh_url}:./hpc_tmp_scp"
-              remote_bash("#{@nodes_handler.sudo_on(@node)} mv ./hpc_tmp_scp/#{File.basename(from)} #{to}")
+              remote_bash("#{@actions_executor.sudo_prefix(@node)}mv ./hpc_tmp_scp/#{File.basename(from)} #{to}")
             else
               run_cmd "scp -S #{ssh_exec} #{from} #{ssh_url}:#{to}"
             end
@@ -323,7 +333,7 @@ module HybridPlatformsConductor
                 #{File.basename(from)} | \
               #{ssh_exec} \
                 #{ssh_url} \
-                \"#{sudo ? "#{@nodes_handler.sudo_on(@node)} " : ''}tar \
+                \"#{need_sudo ? @actions_executor.sudo_prefix(@node) : ''}tar \
                   --extract \
                   --gunzip \
                   --file - \
@@ -374,16 +384,18 @@ module HybridPlatformsConductor
 
           # Add each node
           # Query for the metadata of all nodes at once
-          @nodes_handler.prefetch_metadata_of nodes, %i[private_ips hostname host_ip description]
+          @nodes_handler.prefetch_metadata_of nodes, %i[private_ips hostname host_ip description ssh_port]
           nodes.sort.each do |node|
             # Generate the conf for the node
             connection, connection_user, gateway, gateway_user = connection_info_for(node, no_exception: true)
             if connection.nil?
               config_content << "# #{node} - Not connectable using SSH - #{@nodes_handler.get_description_of(node) || ''}\n"
             else
+              ssh_port = @nodes_handler.get_ssh_port_of(node)
               config_content << "# #{node} - #{connection} - #{@nodes_handler.get_description_of(node) || ''}\n"
               config_content << "Host #{ssh_aliases_for(node).join(' ')}\n"
               config_content << "  Hostname #{connection}\n"
+              config_content << "  Port #{ssh_port}\n" unless ssh_port.nil?
               config_content << "  User \"#{connection_user}\"\n" if connection_user != @ssh_user
               config_content << "  ProxyCommand #{ssh_exec} -q -W %h:%p #{gateway_user}@#{gateway}\n" unless gateway.nil?
               if @passwords.key?(node)
@@ -648,7 +660,7 @@ module HybridPlatformsConductor
             existing_users = File.exist?(control_master_users_file) ? File.read(control_master_users_file).split("\n") : []
             ssh_url = "hpc.#{node}"
             connection, connection_user, _gateway, _gateway_user = connection_info_for(node)
-            control_path_file = control_master_file(connection, '22', connection_user)
+            control_path_file = control_master_file(connection, @nodes_handler.get_ssh_port_of(node) || 22, connection_user)
             if existing_users.empty?
               # Make sure there is no stale one.
               if File.exist?(control_path_file)

data/lib/hybrid_platforms_conductor/hpc_plugins/log/remote_fs.rb

@@ -29,10 +29,9 @@ module HybridPlatformsConductor
         # Result::
         # * Array< Hash<Symbol,Object> >: List of actions to be done
         def actions_to_save_logs(node, services, deployment_info, exit_status, stdout, stderr)
-          # Create a log file to be scp with all relevant info
-          ssh_user = @actions_executor.connector(:ssh).ssh_user
-          sudo_prefix = ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} "
-          log_file = "#{Dir.tmpdir}/hpc_deploy_logs/#{node}_#{Time.now.utc.strftime('%F_%H%M%S')}_#{ssh_user}"
+          # Create a log file to be scp-ed with all relevant info
+          sudo_prefix = @actions_executor.sudo_prefix(node)
+          log_file = "#{Dir.tmpdir}/hpc_deploy_logs/#{node}_#{Time.now.utc.strftime('%F_%H%M%S')}_#{@actions_executor.connector(:ssh).ssh_user}"
           [
             {
               ruby: proc do
@@ -56,7 +55,7 @@ module HybridPlatformsConductor
             {
               scp: {
                 log_file => '/var/log/deployments',
-                :sudo => ssh_user != 'root',
+                :sudo => !@actions_executor.privileged_access?(node),
                 :owner => 'root',
                 :group => 'root'
               }
@@ -85,7 +84,7 @@ module HybridPlatformsConductor
         # Result::
         # * Array< Hash<Symbol,Object> >: List of actions to be done
         def actions_to_read_logs(node)
-          sudo_prefix = @actions_executor.connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} "
+          sudo_prefix = @actions_executor.sudo_prefix(node)
           [
             { remote_bash: "#{sudo_prefix}cat /var/log/deployments/`#{sudo_prefix}ls -t /var/log/deployments/ | head -1`" }
           ]

data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb

@@ -263,7 +263,7 @@ module HybridPlatformsConductor
             raise "Missing file #{chef_versions_file} specifying the Chef Infra Client version to be deployed" unless File.exist?(chef_versions_file)
 
             required_chef_client_version = YAML.load_file(chef_versions_file)['client']
-            sudo = (@actions_executor.connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} -E ")
+            sudo = @actions_executor.sudo_prefix(node, forward_env: true)
             [
               {
                 # Install dependencies

data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/docker.rb

@@ -21,7 +21,7 @@ module HybridPlatformsConductor
             ::Docker.validate_version!
             docker_ok = true
           rescue
-            log_error "[ #{@node}/#{@environment} ] - Docker is not installed correctly. Please install it. Error: #{$ERROR_INFO}"
+            log_error "Docker is not installed correctly. Please install it. Error: #{$ERROR_INFO}"
           end
           docker_ok
         end