hybrid_platforms_conductor 32.17.0 → 33.0.2

data/lib/hybrid_platforms_conductor/credentials.rb

@@ -42,7 +42,7 @@ module HybridPlatformsConductor
     # * *url* (String or nil): The URL for which we want the credentials, or nil if not associated to a URL [default: nil]
     # * *logger* (Logger): Logger to be used [default = Logger.new(STDOUT)]
     # * *logger_stderr* (Logger): Logger to be used for stderr [default = Logger.new(STDERR)]
-    def initialize(id, url: nil, logger: Logger.new(STDOUT), logger_stderr: Logger.new(STDERR))
+    def initialize(id, url: nil, logger: Logger.new($stdout), logger_stderr: Logger.new($stderr))
       init_loggers(logger, logger_stderr)
       @id = id
       @url = url
@@ -54,7 +54,7 @@ module HybridPlatformsConductor
     # Provide a helper to clear password from memory for security.
     # To be used when the client knows it won't use the password anymore.
     def clear_password
-      @password.replace('gotyou!' * 100) unless @password.nil?
+      @password&.replace('gotyou!' * 100)
       GC.start
     end
 
@@ -82,46 +82,49 @@ module HybridPlatformsConductor
     # Do it only once.
     # Make sure the retrieved credentials are not linked to other objects in memory, so that we can remove any other trace of secrets.
     def retrieve_credentials
-      unless @retrieved
-        # Check environment variables
-        @user = ENV["hpc_user_for_#{@id}"].dup
-        @password = ENV["hpc_password_for_#{@id}"].dup
-        if @user.nil? || @user.empty? || @password.nil? || @password.empty?
-          log_debug "[ Credentials for #{@id} ] - Credentials not found from environment variables."
-          if @url.nil?
-            log_debug "[ Credentials for #{@id} ] - No URL associated to this credentials, so .netrc can't be used."
-          else
-            # Check Netrc
-            netrc = ::Netrc.read
-            begin
-              netrc_user, netrc_password = netrc[URI.parse(@url).host.downcase]
-              if netrc_user.nil?
-                log_debug "[ Credentials for #{@id} ] - No credentials retrieved from .netrc."
-                # TODO: Add more credentials source if needed here
-                log_warn "[ Credentials for #{@id} ] - Unable to get credentials for #{@id} (URL: #{@url})."
-              else
-                @user = netrc_user.dup
-                @password = netrc_password.dup
-                log_debug "[ Credentials for #{@id} ] - Credentials retrieved from .netrc using #{@url}."
-              end
-            ensure
-              # Make sure the password does not stay in Netrc memory
-              # Wipe out any memory trace that might contain passwords in clear
-              netrc.instance_variable_get(:@data).each do |data_line|
-                data_line.each do |data_string|
-                  data_string.replace('GotYou!!!' * 100)
-                end
+      return if @retrieved
+
+      # Check environment variables
+      @user = ENV["hpc_user_for_#{@id}"].dup
+      @password = ENV["hpc_password_for_#{@id}"].dup
+      if @user.nil? || @user.empty? || @password.nil? || @password.empty?
+        log_debug "[ Credentials for #{@id} ] - Credentials not found from environment variables."
+        if @url.nil?
+          log_debug "[ Credentials for #{@id} ] - No URL associated to this credentials, so .netrc can't be used."
+        else
+          # Check Netrc
+          netrc = ::Netrc.read
+          begin
+            netrc_user, netrc_password = netrc[URI.parse(@url).host.downcase]
+            if netrc_user.nil?
+              log_debug "[ Credentials for #{@id} ] - No credentials retrieved from .netrc."
+              # TODO: Add more credentials source if needed here
+              log_warn "[ Credentials for #{@id} ] - Unable to get credentials for #{@id} (URL: #{@url})."
+            else
+              @user = netrc_user.dup
+              @password = netrc_password.dup
+              log_debug "[ Credentials for #{@id} ] - Credentials retrieved from .netrc using #{@url}."
+            end
+          ensure
+            # Make sure the password does not stay in Netrc memory
+            # Wipe out any memory trace that might contain passwords in clear
+            netrc.instance_variable_get(:@data).each do |data_line|
+              data_line.each do |data_string|
+                data_string.replace('GotYou!!!' * 100)
               end
-              netrc = nil
             end
+            # We don this assignment on purpose so that GC can remove sensitive data later
+            # rubocop:disable Lint/UselessAssignment
+            netrc = nil
+            # rubocop:enable Lint/UselessAssignment
           end
-        else
-          log_debug "[ Credentials for #{@id} ] - Credentials retrieved from environment variables."
         end
-        GC.start
+      else
+        log_debug "[ Credentials for #{@id} ] - Credentials retrieved from environment variables."
       end
+      GC.start
     end
 
   end
 
-end
+end

data/lib/hybrid_platforms_conductor/current_dir_monitor.rb

@@ -1,5 +1,6 @@
 require 'monitor'
 
+# Decorate methods changing the process' current directory with a mutex to ensure they have an exclusive access
 module HybridPlatformsConductor
 
   # Implement a global monitor to protect accesses to the current directory.
@@ -7,7 +8,9 @@ module HybridPlatformsConductor
   module CurrentDirMonitor
 
     class << self
+
       attr_reader :monitor
+
     end
 
     @monitor = Monitor.new
@@ -24,7 +27,7 @@ module HybridPlatformsConductor
         result = nil
         CurrentDirMonitor.monitor.synchronize do
           # puts "TID #{Thread.current.object_id} from #{caller[2]} - Current dir monitor taken from #{Dir.pwd}"
-          result = self.send(original_method_name, *args, &block)
+          result = send(original_method_name, *args, &block)
           # puts "TID #{Thread.current.object_id} from #{caller[2]} - Current dir monitor released back to #{Dir.pwd}"
         end
         result

data/lib/hybrid_platforms_conductor/deployer.rb

@@ -3,7 +3,6 @@ require 'futex'
 require 'json'
 require 'securerandom'
 require 'time'
-require 'thread'
 require 'hybrid_platforms_conductor/actions_executor'
 require 'hybrid_platforms_conductor/cmd_runner'
 require 'hybrid_platforms_conductor/executable'
@@ -11,7 +10,6 @@ require 'hybrid_platforms_conductor/logger_helpers'
 require 'hybrid_platforms_conductor/nodes_handler'
 require 'hybrid_platforms_conductor/services_handler'
 require 'hybrid_platforms_conductor/plugins'
-require 'hybrid_platforms_conductor/thycotic'
 
 module HybridPlatformsConductor
 
@@ -21,12 +19,26 @@ module HybridPlatformsConductor
     # Extend the Config DSL
     module ConfigDSLExtension
 
+      # List of log plugins. Each info has the following properties:
+      # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule.
+      # * *log_plugins* (Array<Symbol>): List of log plugins to be used to store deployment logs.
+      # Array< Hash<Symbol, Object> >
+      attr_reader :deployment_logs
+
+      # List of secrets reader plugins. Each info has the following properties:
+      # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule.
+      # * *secrets_readers* (Array<Symbol>): List of log plugins to be used to store deployment logs.
+      # Array< Hash<Symbol, Object> >
+      attr_reader :secrets_readers
+
       # Integer: Timeout (in seconds) for packaging repositories
       attr_reader :packaging_timeout_secs
 
       # Mixin initializer
       def init_deployer_config
         @packaging_timeout_secs = 60
+        @deployment_logs = []
+        @secrets_readers = []
       end
 
       # Set the packaging timeout
@@ -37,6 +49,28 @@ module HybridPlatformsConductor
         @packaging_timeout_secs = packaging_timeout_secs
       end
 
+      # Set the deployment log plugins to be used
+      #
+      # Parameters::
+      # * *log_plugins* (Symbol or Array<Symbol>): The list of (or single) log plugins to be used
+      def send_logs_to(*log_plugins)
+        @deployment_logs << {
+          nodes_selectors_stack: current_nodes_selectors_stack,
+          log_plugins: log_plugins.flatten
+        }
+      end
+
+      # Set the secrets readers
+      #
+      # Parameters::
+      # * *secrets_readers* (Symbol or Array<Symbol>): The list of (or single) secrets readers plugins to be used
+      def read_secrets_from(*secrets_readers)
+        @secrets_readers << {
+          nodes_selectors_stack: current_nodes_selectors_stack,
+          secrets_readers: secrets_readers.flatten
+        }
+      end
+
     end
 
     include LoggerHelpers
@@ -55,10 +89,6 @@ module HybridPlatformsConductor
     #   Boolean
     attr_accessor :concurrent_execution
 
-    # The list of JSON secrets
-    #   Array<Hash>
-    attr_accessor :secrets
-
     # Are we deploying in a local environment?
     #   Boolean
     attr_accessor :local_environment
@@ -78,8 +108,8 @@ module HybridPlatformsConductor
     # * *actions_executor* (ActionsExecutor): Actions Executor to be used. [default: ActionsExecutor.new]
     # * *services_handler* (ServicesHandler): Services Handler to be used. [default: ServicesHandler.new]
     def initialize(
-      logger: Logger.new(STDOUT),
-      logger_stderr: Logger.new(STDERR),
+      logger: Logger.new($stdout),
+      logger_stderr: Logger.new($stderr),
       config: Config.new,
       cmd_runner: CmdRunner.new,
       nodes_handler: NodesHandler.new,
@@ -92,8 +122,36 @@ module HybridPlatformsConductor
       @nodes_handler = nodes_handler
       @actions_executor = actions_executor
       @services_handler = services_handler
-      @secrets = []
+      @override_secrets = nil
+      @secrets_readers = Plugins.new(
+        :secrets_reader,
+        logger: @logger,
+        logger_stderr: @logger_stderr,
+        init_plugin: proc do |plugin_class|
+          plugin_class.new(
+            logger: @logger,
+            logger_stderr: @logger_stderr,
+            config: @config,
+            cmd_runner: @cmd_runner,
+            nodes_handler: @nodes_handler
+          )
+        end
+      )
       @provisioners = Plugins.new(:provisioner, logger: @logger, logger_stderr: @logger_stderr)
+      @log_plugins = Plugins.new(
+        :log,
+        logger: @logger,
+        logger_stderr: @logger_stderr,
+        init_plugin: proc do |plugin_class|
+          plugin_class.new(
+            logger: @logger,
+            logger_stderr: @logger_stderr,
+            config: @config,
+            nodes_handler: @nodes_handler,
+            actions_executor: @actions_executor
+          )
+        end
+      )
       # Default values
       @use_why_run = false
       @timeout = nil
@@ -112,42 +170,32 @@ module HybridPlatformsConductor
     def options_parse(options_parser, parallel_switch: true, why_run_switch: false, timeout_options: true)
       options_parser.separator ''
       options_parser.separator 'Deployer options:'
-      options_parser.on(
-        '-e', '--secrets SECRETS_LOCATION',
-        'Specify a secrets location. Can be specified several times. Location can be:',
-        '* Local path to a JSON file',
-        '* URL of the form http[s]://<url>:<secret_id> to get a secret JSON file from a Thycotic Secret Server at the given URL.'
-      ) do |secrets_location|
-        @secrets << JSON.parse(
-          if secrets_location =~ /^(https?:\/\/.+):(\d+)$/
-            url = $1
-            secret_id = $2
-            secret = nil
-            Thycotic.with_thycotic(url, @logger, @logger_stderr) do |thycotic|
-              secret_file_item_id = thycotic.get_secret(secret_id).dig(:secret, :items, :secret_item, :id)
-              raise "Unable to fetch secret file ID #{secrets_location}" if secret_file_item_id.nil?
-              secret = thycotic.download_file_attachment_by_item_id(secret_id, secret_file_item_id)
-              raise "Unable to fetch secret file attachment from #{secrets_location}" if secret.nil?
-            end
-            secret
-          else
-            raise "Missing secret file: #{secrets_location}" unless File.exist?(secrets_location)
-            File.read(secrets_location)
-          end
-        )
+      if parallel_switch
+        options_parser.on('-p', '--parallel', 'Execute the commands in parallel (put the standard output in files <hybrid-platforms-dir>/run_logs/*.stdout)') do
+          @concurrent_execution = true
+        end
+      end
+      if timeout_options
+        options_parser.on('-t', '--timeout SECS', "Timeout in seconds to wait for each chef run. Only used in why-run mode. (defaults to #{@timeout.nil? ? 'no timeout' : @timeout})") do |nbr_secs|
+          @timeout = nbr_secs.to_i
+        end
+      end
+      if why_run_switch
+        options_parser.on('-W', '--why-run', 'Use the why-run mode to see what would be the result of the deploy instead of deploying it for real.') do
+          @use_why_run = true
+        end
       end
-      options_parser.on('-p', '--parallel', 'Execute the commands in parallel (put the standard output in files <hybrid-platforms-dir>/run_logs/*.stdout)') do
-        @concurrent_execution = true
-      end if parallel_switch
-      options_parser.on('-t', '--timeout SECS', "Timeout in seconds to wait for each chef run. Only used in why-run mode. (defaults to #{@timeout.nil? ? 'no timeout' : @timeout})") do |nbr_secs|
-        @timeout = nbr_secs.to_i
-      end if timeout_options
-      options_parser.on('-W', '--why-run', 'Use the why-run mode to see what would be the result of the deploy instead of deploying it for real.') do
-        @use_why_run = true
-      end if why_run_switch
       options_parser.on('--retries-on-error NBR', "Number of retries in case of non-deterministic errors (defaults to #{@nbr_retries_on_error})") do |nbr_retries|
         @nbr_retries_on_error = nbr_retries.to_i
       end
+      # Display options secrets readers might have
+      @secrets_readers.each do |secret_reader_name, secret_reader|
+        next unless secret_reader.respond_to?(:options_parse)
+
+        options_parser.separator ''
+        options_parser.separator "Secrets reader #{secret_reader_name} options:"
+        secret_reader.options_parse(options_parser)
+      end
     end
 
     # Validate that parsed parameters are valid
@@ -158,6 +206,16 @@ module HybridPlatformsConductor
     # String: File used as a Futex for packaging
     PACKAGING_FUTEX_FILE = "#{Dir.tmpdir}/hpc_packaging"
 
+    # Override the secrets with a given JSON.
+    # When using this method with a secrets Hash, further deployments will not query secrets readers, but will use those secrets directly.
+    # Useful to override secrets in test conditions when using dummy secrets for example.
+    #
+    # Parameters::
+    # * *secrets* (Hash or nil): Secrets to take into account in place of secrets readers, or nil to cancel a previous overriding and use secrets readers instead.
+    def override_secrets(secrets)
+      @override_secrets = secrets
+    end
+
     # Deploy on a given list of nodes selectors.
     # The workflow is the following:
     # 1. Package the services to be deployed, considering the nodes, services and context (options, secrets, environment...)
@@ -171,16 +229,26 @@ module HybridPlatformsConductor
     def deploy_on(*nodes_selectors)
       # Get the sorted list of services to be deployed, per node
       # Hash<String, Array<String> >
-      services_to_deploy = Hash[@nodes_handler.select_nodes(nodes_selectors.flatten).map do |node|
+      services_to_deploy = @nodes_handler.select_nodes(nodes_selectors.flatten).map do |node|
         [node, @nodes_handler.get_services_of(node)]
-      end]
+      end.to_h
 
       # Get the secrets to be deployed
       secrets = {}
-      @secrets.each do |secret_json|
-        secrets.merge!(secret_json) do |key, value1, value2|
-          raise "Secret #{key} has conflicting values between different secret JSON files." if value1 != value2
-          value1
+      if @override_secrets
+        secrets = @override_secrets
+      else
+        services_to_deploy.each do |node, services|
+          # If there is no config for secrets, just use cli
+          (@config.secrets_readers.empty? ? [{ secrets_readers: %i[cli] }] : @nodes_handler.select_confs_for_node(node, @config.secrets_readers)).inject([]) do |secrets_readers, secrets_readers_info|
+            secrets_readers + secrets_readers_info[:secrets_readers]
+          end.sort.uniq.each do |secrets_reader|
+            services.each do |service|
+              node_secrets = @secrets_readers[secrets_reader].secrets_for(node, service)
+              conflicting_path = safe_merge(secrets, node_secrets)
+              raise "Secret set at path #{conflicting_path.join('->')} by #{secrets_reader} for service #{service} on node #{node} has conflicting values (#{log_debug? ? "#{node_secrets.dig(*conflicting_path)} != #{secrets.dig(*conflicting_path)}" : 'set debug for value details'})." unless conflicting_path.nil?
+            end
+          end
         end
       end
 
@@ -188,7 +256,6 @@ module HybridPlatformsConductor
       unless @use_why_run
         reason_for_interdiction = @services_handler.deploy_allowed?(
           services: services_to_deploy,
-          secrets: secrets,
           local_environment: @local_environment
         )
         raise "Deployment not allowed: #{reason_for_interdiction}" unless reason_for_interdiction.nil?
@@ -226,51 +293,50 @@ module HybridPlatformsConductor
           remaining_nodes_to_deploy = services_to_deploy.keys
           while nbr_retries >= 0 && !remaining_nodes_to_deploy.empty?
             last_deploy_results = deploy(services_to_deploy.slice(*remaining_nodes_to_deploy))
-            if nbr_retries > 0
+            if nbr_retries.positive?
               # Check if we need to retry deployment on some nodes
               # Only parse the last deployment attempt logs
-              retriable_nodes = Hash[
-                remaining_nodes_to_deploy.
-                  map do |node|
-                    exit_status, stdout, stderr = last_deploy_results[node]
-                    if exit_status == 0
+              retriable_nodes = remaining_nodes_to_deploy.
+                map do |node|
+                  exit_status, stdout, stderr = last_deploy_results[node]
+                  if exit_status.zero?
+                    nil
+                  else
+                    retriable_errors = retriable_errors_from(node, exit_status, stdout, stderr)
+                    if retriable_errors.empty?
                       nil
                     else
-                      retriable_errors = retriable_errors_from(node, exit_status, stdout, stderr)
-                      if retriable_errors.empty?
-                        nil
-                      else
-                        # Log the issue in the stderr of the deployment
-                        stderr << "!!! #{retriable_errors.size} retriable errors detected in this deployment:\n#{retriable_errors.map { |error| "* #{error}" }.join("\n")}\n"
-                        [node, retriable_errors]
-                      end
+                      # Log the issue in the stderr of the deployment
+                      stderr << "!!! #{retriable_errors.size} retriable errors detected in this deployment:\n#{retriable_errors.map { |error| "* #{error}" }.join("\n")}\n"
+                      [node, retriable_errors]
                     end
-                  end.
-                  compact
-              ]
+                  end
+                end.
+                compact.
+                to_h
               unless retriable_nodes.empty?
-                log_warn <<~EOS.strip
+                log_warn <<~EO_LOG.strip
                   Retry deployment for #{retriable_nodes.size} nodes as they got non-deterministic errors (#{nbr_retries} retries remaining):
                   #{retriable_nodes.map { |node, retriable_errors| "  * #{node}:\n#{retriable_errors.map { |error| "    - #{error}" }.join("\n")}" }.join("\n")}
-                EOS
+                EO_LOG
               end
               remaining_nodes_to_deploy = retriable_nodes.keys
             end
             # Merge deployment results
-            results.merge!(last_deploy_results) do |node, (exit_status_1, stdout_1, stderr_1), (exit_status_2, stdout_2, stderr_2)|
+            results.merge!(last_deploy_results) do |_node, (exit_status_1, stdout_1, stderr_1), (exit_status_2, stdout_2, stderr_2)|
               [
                 exit_status_2,
-                <<~EOS,
+                <<~EO_STDOUT,
                   #{stdout_1}
                   Deployment exit status code: #{exit_status_1}
                   !!! Retry deployment due to non-deterministic error (#{nbr_retries} remaining attempts)...
                   #{stdout_2}
-                EOS
-                <<~EOS
+                EO_STDOUT
+                <<~EO_STDERR
                   #{stderr_1}
                   !!! Retry deployment due to non-deterministic error (#{nbr_retries} remaining attempts)...
                   #{stderr_2}
-                EOS
+                EO_STDERR
               ]
             end
             nbr_retries -= 1
@@ -325,7 +391,7 @@ module HybridPlatformsConductor
           sub_executable.config.sudo_procs.replace(sub_executable.config.sudo_procs.map do |sudo_proc_info|
             {
               nodes_selectors_stack: sudo_proc_info[:nodes_selectors_stack].map do |nodes_selector|
-                @nodes_handler.select_nodes(nodes_selector).select { |selected_node| selected_node != node }
+                @nodes_handler.select_nodes(nodes_selector).reject { |selected_node| selected_node == node }
               end,
               sudo_proc: sudo_proc_info[:sudo_proc]
             }
@@ -338,13 +404,13 @@ module HybridPlatformsConductor
           deployer.local_environment = true
           # Ignore secrets that might have been given: in Docker containers we always use dummy secrets
           dummy_secrets_file = "#{@config.hybrid_platforms_dir}/dummy_secrets.json"
-          deployer.secrets = File.exist?(dummy_secrets_file) ? [JSON.parse(File.read(dummy_secrets_file))] : []
+          deployer.override_secrets(File.exist?(dummy_secrets_file) ? JSON.parse(File.read(dummy_secrets_file)) : {})
           yield deployer, instance
         end
       rescue
         # Make sure Docker logs are being output to better investigate errors if we were not already outputing them in debug mode
         stdouts = sub_executable.stdouts_to_s
-        log_error "[ #{node}/#{environment} ] - Encountered unhandled exception #{$!}\n#{$!.backtrace.join("\n")}\n-----\n#{stdouts}" unless stdouts.nil?
+        log_error "[ #{node}/#{environment} ] - Encountered unhandled exception #{$ERROR_INFO}\n#{$ERROR_INFO.backtrace.join("\n")}\n-----\n#{stdouts}" unless stdouts.nil?
         raise
       end
     end
@@ -355,72 +421,31 @@ module HybridPlatformsConductor
     # * *nodes* (Array<String>): Nodes to get info from
     # Result::
     # * Hash<String, Hash<Symbol,Object>: The deployed info, per node name.
-    #   Properties are defined by the Deployer#save_logs method, and additionally to them the following properties can be set:
-    #   * *error* (String): Optional property set in case of error
+    #   * *error* (String): Error string in case deployment logs could not be retrieved. If set then further properties will be ignored. [optional]
+    #   * *services* (Array<String>): List of services deployed on the node
+    #   * *deployment_info* (Hash<Symbol,Object>): Deployment metadata
+    #   * *exit_status* (Integer or Symbol): Deployment exit status
+    #   * *stdout* (String): Deployment stdout
+    #   * *stderr* (String): Deployment stderr
     def deployment_info_from(*nodes)
+      nodes = nodes.flatten
       @actions_executor.max_threads = 64
-      Hash[@actions_executor.
-        execute_actions(
-          Hash[nodes.flatten.map do |node|
-            [
-              node,
-              { remote_bash: "cd /var/log/deployments && ls -t | head -1 | xargs sed '/===== STDOUT =====/q'" }
-            ]
-          end],
-          log_to_stdout: false,
-          concurrent: true,
-          timeout: 10,
-          progress_name: 'Getting deployment info'
-        ).
-        map do |node, (exit_status, stdout, stderr)|
-          # Expected format for stdout:
-          # Property1: Value1
-          # ...
-          # PropertyN: ValueN
-          # ===== STDOUT =====
-          # ...
-          deploy_info = {}
-          if exit_status.is_a?(Symbol)
-            deploy_info[:error] = "Error: #{exit_status}\n#{stderr}"
-          else
-            stdout_lines = stdout.split("\n")
-            if stdout_lines.first =~ /No such file or directory/
-              deploy_info[:error] = '/var/log/deployments missing'
-            else
-              stdout_lines.each do |line|
-                if line =~ /^([^:]+): (.+)$/
-                  key_str, value = $1, $2
-                  key = key_str.to_sym
-                  # Type-cast some values
-                  case key_str
-                  when 'date'
-                    # Date and time values
-                    # Thu Nov 23 18:43:01 UTC 2017
-                    deploy_info[key] = Time.parse(value)
-                  when 'debug'
-                    # Boolean values
-                    # Yes
-                    deploy_info[key] = (value == 'Yes')
-                  when /^diff_files_.+$/, 'services'
-                    # Array of strings
-                    # my_file.txt, other_file.txt
-                    deploy_info[key] = value.split(', ')
-                  else
-                    deploy_info[key] = value
-                  end
-                else
-                  deploy_info[:unknown_lines] = [] unless deploy_info.key?(:unknown_lines)
-                  deploy_info[:unknown_lines] << line
-                end
-              end
-            end
-          end
-          [
-            node,
-            deploy_info
-          ]
-        end
-      ]
+      read_actions_results = @actions_executor.execute_actions(
+        nodes.map do |node|
+          master_log_plugin = @log_plugins[log_plugins_for(node).first]
+          master_log_plugin.respond_to?(:actions_to_read_logs) ? [node, master_log_plugin.actions_to_read_logs(node)] : nil
+        end.compact.to_h,
+        log_to_stdout: false,
+        concurrent: true,
+        timeout: 10,
+        progress_name: 'Read deployment logs'
+      )
+      nodes.map do |node|
+        [
+          node,
+          @log_plugins[log_plugins_for(node).first].logs_for(node, *(read_actions_results[node] || [nil, nil, nil]))
+        ]
+      end.to_h
     end
 
     # Parse stdout and stderr of a given deploy run and get the list of tasks with their status
@@ -436,12 +461,41 @@ module HybridPlatformsConductor
     #     * *:changed*: The task has been changed
     #     * *:identical*: The task has not been changed
     #   * *diffs* (String): Differences, if any
-    def parse_deploy_output(node, stdout, stderr)
+    def parse_deploy_output(_node, stdout, stderr)
       @services_handler.parse_deploy_output(stdout, stderr).map { |deploy_info| deploy_info[:tasks] }.flatten
     end
 
     private
 
+    # Safe-merge 2 hashes.
+    # Safe-merging is done by:
+    # * Merging values that are hashes.
+    # * Reporting errors when values conflict.
+    # When values are conflicting, the initial hash won't modify those conflicting values and will stop the merge.
+    #
+    # Parameters::
+    # * *hash* (Hash): Hash to be modified merging hash_to_merge
+    # * *hash_to_merge* (Hash): Hash to be merged into hash
+    # Result::
+    # * nil or Array<Object>: nil in case of success, or the keys path leading to a conflicting value in case of error
+    def safe_merge(hash, hash_to_merge)
+      conflicting_path = nil
+      hash_to_merge.each do |key, value_to_merge|
+        if hash.key?(key)
+          if hash[key].is_a?(Hash) && value_to_merge.is_a?(Hash)
+            sub_conflicting_path = safe_merge(hash[key], value_to_merge)
+            conflicting_path = [key] + sub_conflicting_path unless sub_conflicting_path.nil?
+          elsif hash[key] != value_to_merge
+            conflicting_path = [key]
+          end
+        else
+          hash[key] = value_to_merge
+        end
+        break unless conflicting_path.nil?
+      end
+      conflicting_path
+    end
+
     # Get the list of retriable errors a node got from deployment logs.
     # Useful to know if an error is non-deterministic (due to external and temporary factors).
     #
@@ -452,7 +506,7 @@ module HybridPlatformsConductor
     # * *stderr* (String): Deployment stderr
     # Result::
     # * Array<String>: List of retriable errors that have been matched
-    def retriable_errors_from(node, exit_status, stdout, stderr)
+    def retriable_errors_from(node, _exit_status, stdout, stderr)
       # List of retriable errors for this node, as exact string match or regexps.
       # Array<String or Regexp>
       retriable_errors_on_stdout = []
@@ -483,59 +537,55 @@ module HybridPlatformsConductor
     # Result::
     # * Hash<String, [Integer or Symbol, String, String]>: Exit status code (or Symbol in case of error or dry run), standard output and error for each node.
     def deploy(services)
-      outputs = {}
-
       # Get the ssh user directly from the connector
       ssh_user = @actions_executor.connector(:ssh).ssh_user
 
       # Deploy for real
       @nodes_handler.prefetch_metadata_of services.keys, :image
       outputs = @actions_executor.execute_actions(
-        Hash[services.map do |node, node_services|
+        services.map do |node, node_services|
           image_id = @nodes_handler.get_image_of(node)
           sudo = (ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} ")
-          # Install My_company corporate certificates if present
+          # Install corporate certificates if present
           certificate_actions =
             if @local_environment && ENV['hpc_certificates']
-              if File.exist?(ENV['hpc_certificates'])
-                log_debug "Deploy certificates from #{ENV['hpc_certificates']}"
-                case image_id
-                when 'debian_9', 'debian_10'
-                  [
-                    {
-                      remote_bash: "#{sudo}apt update && #{sudo}apt install -y ca-certificates"
-                    },
-                    {
-                      scp: {
-                        ENV['hpc_certificates'] => '/usr/local/share/ca-certificates',
-                        :sudo => ssh_user != 'root'
-                      },
-                      remote_bash: "#{sudo}update-ca-certificates"
-                    }
-                  ]
-                when 'centos_7'
-                  [
-                    {
-                      remote_bash: "#{sudo}yum install -y ca-certificates"
+              raise "Missing path referenced by the hpc_certificates environment variable: #{ENV['hpc_certificates']}" unless File.exist?(ENV['hpc_certificates'])
+
+              log_debug "Deploy certificates from #{ENV['hpc_certificates']}"
+              case image_id
+              when 'debian_9', 'debian_10'
+                [
+                  {
+                    remote_bash: "#{sudo}apt update && #{sudo}apt install -y ca-certificates"
+                  },
+                  {
+                    scp: {
+                      ENV['hpc_certificates'] => '/usr/local/share/ca-certificates',
+                      :sudo => ssh_user != 'root'
                     },
-                    {
-                      scp: Hash[Dir.glob("#{ENV['hpc_certificates']}/*.crt").map do |cert_file|
-                        [
-                          cert_file,
-                          '/etc/pki/ca-trust/source/anchors'
-                        ]
-                      end].merge(sudo: ssh_user != 'root'),
-                      remote_bash: [
-                        "#{sudo}update-ca-trust enable",
-                        "#{sudo}update-ca-trust extract"
+                    remote_bash: "#{sudo}update-ca-certificates"
+                  }
+                ]
+              when 'centos_7'
+                [
+                  {
+                    remote_bash: "#{sudo}yum install -y ca-certificates"
+                  },
+                  {
+                    scp: Dir.glob("#{ENV['hpc_certificates']}/*.crt").map do |cert_file|
+                      [
+                        cert_file,
+                        '/etc/pki/ca-trust/source/anchors'
                       ]
-                    }
-                  ]
-                else
-                  raise "Unknown image ID for node #{node}: #{image_id}. Check metadata for this node."
-                end
+                    end.to_h.merge(sudo: ssh_user != 'root'),
+                    remote_bash: [
+                      "#{sudo}update-ca-trust enable",
+                      "#{sudo}update-ca-trust extract"
+                    ]
+                  }
+                ]
               else
-                raise "Missing path referenced by the hpc_certificates environment variable: #{ENV['hpc_certificates']}"
+                raise "Unknown image ID for node #{node}: #{image_id}. Check metadata for this node."
               end
             else
               []
@@ -552,19 +602,19 @@ module HybridPlatformsConductor
               certificate_actions +
               @services_handler.actions_to_deploy_on(node, node_services, @use_why_run)
           ]
-        end],
+        end.to_h,
         timeout: @timeout,
         concurrent: @concurrent_execution,
         log_to_stdout: !@concurrent_execution
       )
       # Free eventual locks
       @actions_executor.execute_actions(
-        Hash[services.keys.map do |node|
+        services.keys.map do |node|
           [
             node,
             { remote_bash: "#{ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} "}./mutex_dir unlock /tmp/hybrid_platforms_conductor_deploy_lock" }
           ]
-        end],
+        end.to_h,
         timeout: 10,
         concurrent: true,
         log_to_dir: nil
@@ -584,47 +634,48 @@ module HybridPlatformsConductor
     # * *services* (Hash<String, Array<String>>): List of services that have been deployed, per node
     def save_logs(logs, services)
       section "Saving deployment logs for #{logs.size} nodes" do
-        Dir.mktmpdir('hybrid_platforms_conductor-logs') do |tmp_dir|
-          ssh_user = @actions_executor.connector(:ssh).ssh_user
-          @actions_executor.execute_actions(
-            Hash[logs.map do |node, (exit_status, stdout, stderr)|
-              # Create a log file to be scp with all relevant info
-              now = Time.now.utc
-              log_file = "#{tmp_dir}/#{node}_#{now.strftime('%F_%H%M%S')}_#{ssh_user}"
-              services_info = @services_handler.log_info_for(node, services[node])
-              File.write(
-                log_file,
-                services_info.merge(
-                  date: now.strftime('%F %T'),
-                  user: ssh_user,
-                  debug: log_debug? ? 'Yes' : 'No',
-                  services: services[node].join(', '),
-                  exit_status: exit_status
-                ).map { |property, value| "#{property}: #{value}" }.join("\n") +
-                  "\n===== STDOUT =====\n" +
-                  (stdout || '') +
-                  "\n===== STDERR =====\n" +
-                  (stderr || '')
-              )
-              [
-                node,
-                {
-                  remote_bash: "#{ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} "}mkdir -p /var/log/deployments",
-                  scp: {
-                    log_file => '/var/log/deployments',
-                    :sudo => ssh_user != 'root',
-                    :owner => 'root',
-                    :group => 'root'
-                  }
-                }
-              ]
-            end],
-            timeout: 10,
-            concurrent: true,
-            log_to_dir: nil
-          )
-        end
+        ssh_user = @actions_executor.connector(:ssh).ssh_user
+        @actions_executor.execute_actions(
+          logs.map do |node, (exit_status, stdout, stderr)|
+            [
+              node,
+              log_plugins_for(node).
+                map do |log_plugin|
+                  @log_plugins[log_plugin].actions_to_save_logs(
+                    node,
+                    services[node],
+                    @services_handler.log_info_for(node, services[node]).merge(
+                      date: Time.now.utc.strftime('%F %T'),
+                      user: ssh_user
+                    ),
+                    exit_status,
+                    stdout,
+                    stderr
+                  )
+                end.
+                flatten(1)
+            ]
+          end.to_h,
+          timeout: 10,
+          concurrent: true,
+          log_to_dir: nil,
+          progress_name: 'Saving logs'
+        )
+      end
+    end
+
+    # Get the list of log plugins to be used for a given node
+    #
+    # Parameters::
+    # * *node* (String): The node for which log plugins are queried
+    # Result::
+    # * Array<Symbol>: The list of log plugins
+    def log_plugins_for(node)
+      node_log_plugins = @nodes_handler.select_confs_for_node(node, @config.deployment_logs).inject([]) do |log_plugins, deployment_logs_info|
+        log_plugins + deployment_logs_info[:log_plugins]
       end
+      node_log_plugins << :remote_fs if node_log_plugins.empty?
+      node_log_plugins
     end
 
   end