hybrid_platforms_conductor 33.2.2 → 33.5.0

data/lib/hybrid_platforms_conductor/core_extensions/bundler/without_bundled_env.rb

@@ -11,10 +11,27 @@ module Bundler
     # @return [Hash] Environment with all bundler-related variables removed
     def current_unbundled_env
       env = ENV.clone.to_hash
+      %w[
+        PATH
+        RUBYLIB
+        RUBYOPT
+      ].each do |env_name|
+        if original_env.key?(env_name)
+          env[env_name] = original_env[env_name]
+        else
+          env.delete(env_name)
+        end
+      end
 
       env['MANPATH'] = env['BUNDLER_ORIG_MANPATH'] if env.key?('BUNDLER_ORIG_MANPATH')
 
-      env.delete_if { |k, _| k[0, 7] == 'BUNDLE_' }
+      env.delete_if do |k, _|
+        %w[
+          GEM_
+          BUNDLE_
+          BUNDLER_
+        ].any? { |prefix| k.start_with?(prefix) }
+      end
 
       if env.key?('RUBYOPT')
         rubyopt = env['RUBYOPT'].split

data/lib/hybrid_platforms_conductor/credentials.rb

@@ -1,4 +1,5 @@
 require 'netrc'
+require 'secret_string'
 require 'uri'
 require 'hybrid_platforms_conductor/logger_helpers'
 
@@ -7,122 +8,146 @@ module HybridPlatformsConductor
   # Give a secured and harmonized way to access credentials for a given service.
   # It makes sure to remove passwords from memory for hardened security (this way if a vulnerability allows an attacker to dump the memory it won't get passwords).
   # It gets credentials from the following sources:
+  # * Configuration
   # * Environment variables
   # * Netrc file
-  class Credentials
+  module Credentials
 
-    include LoggerHelpers
+    # Extend the Config DSL
+    module ConfigDSLExtension
+
+      # List of credentials. Each info has the following properties:
+      # * *credential_id* (Symbol): Credential ID this rule applies to
+      # * *resource* (Regexp): Resource filtering for this rule
+      # * *provider* (Proc): The code providing the credentials:
+      #   * Parameters::
+      #     * *resource* (String or nil): The resource for which we want credentials, or nil if none
+      #     * *requester* (Proc): Code to be called to give credentials to:
+      #       * Parameters::
+      #         * *user* (String or nil): The user name, or nil if none
+      #         * *password* (String or nil): The password, or nil if none
+      attr_reader :credentials
+
+      # Mixin initializer
+      def init_credentials_config
+        @credentials = []
+      end
+
+      # Define a credentials provider
+      #
+      # Parameters::
+      # * *credential_id* (Symbol): Credential ID this rule applies to
+      # * *resource* (String or Regexp): Resource filtering for this rule [default: /.*/]
+      # * *provider* (Proc): The code providing the credentials:
+      #   * Parameters::
+      #     * *resource* (String or nil): The resource for which we want credentials, or nil if none
+      #     * *requester* (Proc): Code to be called to give credentials to:
+      #       * Parameters::
+      #         * *user* (String or nil): The user name, or nil if none
+      #         * *password* (String or nil): The password, or nil if none
+      def credentials_for(credential_id, resource: /.*/, &provider)
+        @credentials << {
+          credential_id: credential_id,
+          resource: resource.is_a?(String) ? /^#{Regexp.escape(resource)}$/ : resource,
+          provider: provider
+        }
+      end
+
+    end
+
+    Config.extend_config_dsl_with ConfigDSLExtension, :init_credentials_config
 
     # Get access to credentials and make sure they are wiped out from memory when client code ends.
     # To ensure password safety, never store the password in a scope beyond the client code's Proc.
     #
     # Parameters::
     # * *id* (Symbol): Credential ID
-    # * *logger* (Logger): Logger to be used
-    # * *logger_stderr* (Logger): Logger to be used for stderr
-    # * *url* (String or nil): The URL for which we want the credentials, or nil if not associated to a URL [default: nil]
+    # * *resource* (String or nil): The resource for which we want the credentials, or nil if not associated to a resource [default: nil]
     # * Proc: Client code called with credentials provided
     #   * Parameters::
     #     * *user* (String or nil): User name, or nil if none
-    #     * *password* (String or nil): Password, or nil if none.
-    #       !!! Never store this password in a scope broader than the client code itself !!!
-    def self.with_credentials_for(id, logger, logger_stderr, url: nil)
-      credentials = Credentials.new(id, url: url, logger: logger, logger_stderr: logger_stderr)
-      begin
-        yield credentials.user, credentials.password
-      ensure
-        credentials.clear_password
-      end
-    end
+    #     * *password* (SecretString or nil): Password, or nil if none.
+    #       !!! Never clone this password in a scope broader than the client code itself !!!
+    def with_credentials_for(id, resource: nil)
+      # Get the credentials provider
+      provider = nil
 
-    # Constructor
-    #
-    # Parameters::
-    # * *id* (Symbol): Credential ID
-    # * *url* (String or nil): The URL for which we want the credentials, or nil if not associated to a URL [default: nil]
-    # * *logger* (Logger): Logger to be used [default = Logger.new(STDOUT)]
-    # * *logger_stderr* (Logger): Logger to be used for stderr [default = Logger.new(STDERR)]
-    def initialize(id, url: nil, logger: Logger.new($stdout), logger_stderr: Logger.new($stderr))
-      init_loggers(logger, logger_stderr)
-      @id = id
-      @url = url
-      @user = nil
-      @password = nil
-      @retrieved = false
-    end
-
-    # Provide a helper to clear password from memory for security.
-    # To be used when the client knows it won't use the password anymore.
-    def clear_password
-      @password&.replace('gotyou!' * 100)
-      GC.start
-    end
-
-    # Get the associated user
-    #
-    # Result::
-    # * String or nil: The user name, or nil if none
-    def user
-      retrieve_credentials
-      @user
-    end
-
-    # Get the associated password
-    #
-    # Result::
-    # * String or nil: The password, or nil if none
-    def password
-      retrieve_credentials
-      @password
-    end
-
-    private
-
-    # Retrieve credentials in @user and @password.
-    # Do it only once.
-    # Make sure the retrieved credentials are not linked to other objects in memory, so that we can remove any other trace of secrets.
-    def retrieve_credentials
-      return if @retrieved
+      # Check configuration
+      # Take the last matching provider, this way we can define several providers for resources matched in a increasingly refined way.
+      @config.credentials.each do |credentials_info|
+        provider = credentials_info[:provider] if credentials_info[:credential_id] == id && (
+          (resource.nil? && credentials_info[:resource] == /.*/) || credentials_info[:resource] =~ resource
+        )
+      end
 
-      # Check environment variables
-      @user = ENV["hpc_user_for_#{@id}"].dup
-      @password = ENV["hpc_password_for_#{@id}"].dup
-      if @user.nil? || @user.empty? || @password.nil? || @password.empty?
-        log_debug "[ Credentials for #{@id} ] - Credentials not found from environment variables."
-        if @url.nil?
-          log_debug "[ Credentials for #{@id} ] - No URL associated to this credentials, so .netrc can't be used."
-        else
-          # Check Netrc
-          netrc = ::Netrc.read
-          begin
-            netrc_user, netrc_password = netrc[URI.parse(@url).host.downcase]
-            if netrc_user.nil?
-              log_debug "[ Credentials for #{@id} ] - No credentials retrieved from .netrc."
-              # TODO: Add more credentials source if needed here
-              log_warn "[ Credentials for #{@id} ] - Unable to get credentials for #{@id} (URL: #{@url})."
-            else
-              @user = netrc_user.dup
-              @password = netrc_password.dup
-              log_debug "[ Credentials for #{@id} ] - Credentials retrieved from .netrc using #{@url}."
-            end
-          ensure
-            # Make sure the password does not stay in Netrc memory
-            # Wipe out any memory trace that might contain passwords in clear
-            netrc.instance_variable_get(:@data).each do |data_line|
-              data_line.each do |data_string|
-                data_string.replace('GotYou!!!' * 100)
+      provider ||= proc do |requested_resource, requester|
+        # Check environment variables
+        user = ENV["hpc_user_for_#{id}"]
+        # Clone the password as we are going to treat it as a secret string that will be wiped out
+        password = ENV["hpc_password_for_#{id}"].dup
+        if user.nil? || user.empty? || password.nil? || password.empty?
+          log_debug "[ Credentials for #{id} ] - Credentials not found from environment variables."
+          if requested_resource.nil?
+            log_debug "[ Credentials for #{id} ] - No resource associated to this credentials, so .netrc can't be used."
+          else
+            # Check Netrc
+            netrc = ::Netrc.read
+            begin
+              netrc_user, netrc_password = netrc[
+                begin
+                  URI.parse(requested_resource).host.downcase
+                rescue URI::InvalidURIError
+                  requested_resource
+                end
+              ]
+              if netrc_user.nil?
+                log_debug "[ Credentials for #{id} ] - No credentials retrieved from .netrc."
+                # TODO: Add more credentials source if needed here
+                log_warn "[ Credentials for #{id} ] - Unable to get credentials for #{id} (Resource: #{requested_resource})."
+              else
+                # Clone in memory as we are going to wipe out ::Netrc's memory
+                user = netrc_user.dup
+                password = netrc_password.dup
+                log_debug "[ Credentials for #{id} ] - Credentials retrieved from .netrc using #{requested_resource}."
+              end
+            ensure
+              # Make sure the password does not stay in Netrc memory
+              # Wipe out any memory trace that might contain passwords in clear
+              netrc.instance_variable_get(:@data).each do |data_line|
+                data_line.each do |data_string|
+                  data_string.replace('GotYou!!!' * 100)
+                end
               end
             end
-            # We don this assignment on purpose so that GC can remove sensitive data later
-            # rubocop:disable Lint/UselessAssignment
-            netrc = nil
-            # rubocop:enable Lint/UselessAssignment
+          end
+        else
+          log_debug "[ Credentials for #{id} ] - Credentials retrieved from environment variables."
+        end
+        if password.nil?
+          requester.call user, password
+        else
+          SecretString.protect(password) do |secret_password|
+            requester.call user, secret_password
           end
         end
-      else
-        log_debug "[ Credentials for #{@id} ] - Credentials retrieved from environment variables."
       end
-      GC.start
+
+      requester_called = false
+      provider.call(
+        resource,
+        proc do |user, password|
+          requester_called = true
+          if password.is_a?(String)
+            SecretString.protect(password) do |secret_password|
+              yield user, secret_password
+            end
+          else
+            yield user, password
+          end
+        end
+      )
+
+      raise "Requester not called by the credentials provider for #{id} (resource: #{resource}) - Please check the credentials_for code in your configuration." unless requester_called
     end
 
   end

data/lib/hybrid_platforms_conductor/deployer.rb

@@ -9,6 +9,7 @@ require 'hybrid_platforms_conductor/logger_helpers'
 require 'hybrid_platforms_conductor/nodes_handler'
 require 'hybrid_platforms_conductor/services_handler'
 require 'hybrid_platforms_conductor/plugins'
+require 'hybrid_platforms_conductor/safe_merge'
 
 module HybridPlatformsConductor
 
@@ -18,6 +19,12 @@ module HybridPlatformsConductor
     # Extend the Config DSL
     module ConfigDSLExtension
 
+      # List of retriable errors. Each info has the following properties:
+      # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by those errors
+      # * *errors_on_stdout* (Array<String or Regexp>): List of errors match (as exact string match or using a regexp) to check against stdout
+      # * *errors_on_stderr* (Array<String or Regexp>): List of errors match (as exact string match or using a regexp) to check against stderr
+      attr_reader :retriable_errors
+
       # List of log plugins. Each info has the following properties:
       # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule.
       # * *log_plugins* (Array<Symbol>): List of log plugins to be used to store deployment logs.
@@ -36,6 +43,11 @@ module HybridPlatformsConductor
       # Mixin initializer
       def init_deployer_config
         @packaging_timeout_secs = 60
+        # List of retriable errors. Each info has the following properties:
+        # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by those errors
+        # * *errors_on_stdout* (Array<String or Regexp>): List of errors match (as exact string match or using a regexp) to check against stdout
+        # * *errors_on_stderr* (Array<String or Regexp>): List of errors match (as exact string match or using a regexp) to check against stderr
+        @retriable_errors = []
         @deployment_logs = []
         @secrets_readers = []
       end
@@ -48,6 +60,28 @@ module HybridPlatformsConductor
         @packaging_timeout_secs = packaging_timeout_secs
       end
 
+      # Mark some errors on stdout to be retriable during a deploy
+      #
+      # Parameters::
+      # * *errors* (String, Regexp or Array<String or Regexp>): Single (or list of) errors matching pattern (either as exact string match or using a regexp).
+      def retry_deploy_for_errors_on_stdout(errors)
+        @retriable_errors << {
+          errors_on_stdout: errors.is_a?(Array) ? errors : [errors],
+          nodes_selectors_stack: current_nodes_selectors_stack
+        }
+      end
+
+      # Mark some errors on stderr to be retriable during a deploy
+      #
+      # Parameters::
+      # * *errors* (String, Regexp or Array<String or Regexp>): Single (or list of) errors matching pattern (either as exact string match or using a regexp).
+      def retry_deploy_for_errors_on_stderr(errors)
+        @retriable_errors << {
+          errors_on_stderr: errors.is_a?(Array) ? errors : [errors],
+          nodes_selectors_stack: current_nodes_selectors_stack
+        }
+      end
+
       # Set the deployment log plugins to be used
       #
       # Parameters::
@@ -72,10 +106,10 @@ module HybridPlatformsConductor
 
     end
 
-    include LoggerHelpers
-
     Config.extend_config_dsl_with ConfigDSLExtension, :init_deployer_config
 
+    include LoggerHelpers
+
     # Do we use why-run mode while deploying? [default = false]
     #   Boolean
     attr_accessor :use_why_run
@@ -467,34 +501,7 @@ module HybridPlatformsConductor
 
     private
 
-    # Safe-merge 2 hashes.
-    # Safe-merging is done by:
-    # * Merging values that are hashes.
-    # * Reporting errors when values conflict.
-    # When values are conflicting, the initial hash won't modify those conflicting values and will stop the merge.
-    #
-    # Parameters::
-    # * *hash* (Hash): Hash to be modified merging hash_to_merge
-    # * *hash_to_merge* (Hash): Hash to be merged into hash
-    # Result::
-    # * nil or Array<Object>: nil in case of success, or the keys path leading to a conflicting value in case of error
-    def safe_merge(hash, hash_to_merge)
-      conflicting_path = nil
-      hash_to_merge.each do |key, value_to_merge|
-        if hash.key?(key)
-          if hash[key].is_a?(Hash) && value_to_merge.is_a?(Hash)
-            sub_conflicting_path = safe_merge(hash[key], value_to_merge)
-            conflicting_path = [key] + sub_conflicting_path unless sub_conflicting_path.nil?
-          elsif hash[key] != value_to_merge
-            conflicting_path = [key]
-          end
-        else
-          hash[key] = value_to_merge
-        end
-        break unless conflicting_path.nil?
-      end
-      conflicting_path
-    end
+    include SafeMerge
 
     # Get the list of retriable errors a node got from deployment logs.
     # Useful to know if an error is non-deterministic (due to external and temporary factors).

data/lib/hybrid_platforms_conductor/github.rb

@@ -0,0 +1,39 @@
+require 'octokit'
+require 'hybrid_platforms_conductor/credentials'
+
+module HybridPlatformsConductor
+
+  # Mixin used to access Github API
+  module Github
+
+    include Credentials
+
+    # Iterate over each Github repository
+    #
+    # Parameters::
+    # * Proc: Code called for each Github repository:
+    #   * Parameters::
+    #     * *github* (Octokit::Client): The client instance accessing the Github API
+    #     * *repo_info* (Hash<Symbol, Object>): The repository info:
+    #       * *name* (String): Repository name.
+    #       * *slug* (String): Repository slug.
+    def for_each_github_repo
+      @config.known_github_repos.each do |repo_info|
+        Octokit.configure do |c|
+          c.api_endpoint = repo_info[:url]
+        end
+        with_credentials_for(:github, resource: repo_info[:url]) do |_github_user, github_token|
+          client = Octokit::Client.new(access_token: github_token&.to_unprotected)
+          (repo_info[:repos] == :all ? client.repositories(repo_info[:user]).map { |repo| repo[:name] } : repo_info[:repos]).each do |name|
+            yield client, {
+              name: name,
+              slug: "#{repo_info[:user]}/#{name}"
+            }
+          end
+        end
+      end
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/action/bash.rb

@@ -14,7 +14,7 @@ module HybridPlatformsConductor
         # [API] - @actions_executor is accessible
         #
         # Parameters::
-        # * *cmd* (String): The bash command to execute
+        # * *cmd* (String or SecretString): The bash command to execute
         def setup(cmd)
           @cmd = cmd
         end

data/lib/hybrid_platforms_conductor/hpc_plugins/action/remote_bash.rb

@@ -15,30 +15,30 @@ module HybridPlatformsConductor
         #
         # Parameters::
         # * *remote_bash* (Array or Object): List of commands (or single command) to be executed. Each command can be the following:
-        #   * String: Simple bash command.
+        #   * String or SecretString: Simple bash command.
         #   * Hash<Symbol, Object>: Information about the commands to execute. Can have the following properties:
-        #     * *commands* (Array<String> or String): List of bash commands to execute (can be a single one) [default: ''].
+        #     * *commands* (Array<String or SecretString> or String or SecretString): List of bash commands to execute (can be a single one) [default: ''].
         #     * *file* (String): Name of file from which commands should be taken [optional].
-        #     * *env* (Hash<String, String>): Environment variables to be set before executing those commands [default: {}].
+        #     * *env* (Hash<String, String or SecretString>): Environment variables to be set before executing those commands [default: {}].
         def setup(remote_bash)
           # Normalize the parameters.
           # Array< Hash<Symbol,Object> >: Simple array of info:
-          # * *commands* (Array<String>): List of bash commands to execute.
-          # * *env* (Hash<String, String>): Environment variables to be set before executing those commands.
+          # * *commands* (Array<String or SecretString>): List of bash commands to execute.
+          # * *env* (Hash<String, String or SecretString>): Environment variables to be set before executing those commands.
           @remote_bash = (remote_bash.is_a?(Array) ? remote_bash : [remote_bash]).map do |cmd_info|
-            if cmd_info.is_a?(String)
-              {
-                commands: [cmd_info],
-                env: {}
-              }
-            else
+            if cmd_info.is_a?(Hash)
               commands = []
-              commands.concat(cmd_info[:commands].is_a?(String) ? [cmd_info[:commands]] : cmd_info[:commands]) if cmd_info[:commands]
+              commands.concat(cmd_info[:commands].is_a?(Array) ? cmd_info[:commands] : [cmd_info[:commands]]) if cmd_info[:commands]
               commands << File.read(cmd_info[:file]) if cmd_info[:file]
               {
                 commands: commands,
                 env: cmd_info[:env] || {}
               }
+            else
+              {
+                commands: [cmd_info],
+                env: {}
+              }
             end
           end
         end
@@ -63,11 +63,21 @@ module HybridPlatformsConductor
         # [API] - @stderr_io can be used to log stderr messages
         # [API] - run_cmd(String) method can be used to execute a command. See CmdRunner#run_cmd to know about the result's signature.
         def execute
-          bash_str = @remote_bash.map do |cmd_info|
-            (cmd_info[:env].map { |var_name, var_value| "export #{var_name}='#{var_value}'" } + cmd_info[:commands]).join("\n")
-          end.join("\n")
-          log_debug "[#{@node}] - Execute remote Bash commands \"#{bash_str}\"..."
-          @connector.remote_bash bash_str
+          # The commands or ENV variables can contain secrets, so make sure to protect all strings from secrets leaking
+          bash_cmds = @remote_bash.map do |cmd_info|
+            cmd_info[:env].map do |var_name, var_value|
+              SecretString.new("export #{var_name}='#{var_value.to_unprotected}'", silenced_str: "export #{var_name}='#{var_value}'")
+            end + cmd_info[:commands]
+          end.flatten
+          begin
+            SecretString.protect(bash_cmds.map(&:to_unprotected).join("\n"), silenced_str: bash_cmds.join("\n")) do |bash_str|
+              log_debug "[#{@node}] - Execute remote Bash commands \"#{bash_str}\"..."
+              @connector.remote_bash bash_str
+            end
+          ensure
+            # Make sure we erase all secret strings
+            bash_cmds.each(&:erase)
+          end
         end
 
       end