hybrid_platforms_conductor 32.10.0 → 32.13.0

data/examples/bare/Gemfile

@@ -0,0 +1,4 @@
+source 'http://rubygems.org'
+
+# Orchestrate all the platforms with Hybrid Platforms Conductor
+gem 'hybrid_platforms_conductor', path: '../..'

data/examples/bare/hpc_config.rb

@@ -0,0 +1,2 @@
+# Just a bare configuration.
+# It should work already like this.

data/examples/localhost/Gemfile

@@ -0,0 +1,4 @@
+source 'http://rubygems.org'
+
+# Orchestrate all the platforms with Hybrid Platforms Conductor
+gem 'hybrid_platforms_conductor', path: '../..'

data/examples/localhost/hpc_config.rb

@@ -0,0 +1,2 @@
+# This configuration file just defines 1 node being localhost
+yaml_inventory_platform path: '.'

data/examples/localhost/inventory.yaml

@@ -0,0 +1,4 @@
+---
+local:
+  metadata:
+    host_ip: 127.0.0.1

data/lib/hybrid_platforms_conductor/actions_executor.rb

@@ -5,6 +5,7 @@ require 'securerandom'
 require 'tmpdir'
 require 'hybrid_platforms_conductor/action'
 require 'hybrid_platforms_conductor/cmd_runner'
+require 'hybrid_platforms_conductor/config'
 require 'hybrid_platforms_conductor/connector'
 require 'hybrid_platforms_conductor/io_router'
 require 'hybrid_platforms_conductor/logger_helpers'

data/lib/hybrid_platforms_conductor/common_config_dsl/idempotence_tests.rb

@@ -11,13 +11,24 @@ module HybridPlatformsConductor
       # Array< Hash<Symbol, Object> >
       attr_reader :ignored_idempotence_tasks
 
-      # Initialize the DSL 
+      # List of ignored tasks info. Each info has the following properties:
+      # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
+      # * *ignored_tasks* (Hash<String, String>): List of task names for which we ignore divergence errors, with the corresponding descriptive reason for ignore.
+      # Array< Hash<Symbol, Object> >
+      attr_reader :ignored_divergent_tasks
+
+      # Initialize the DSL
       def init_idempotence_tests
         # List of ignored tasks info. Each info has the following properties:
         # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
         # * *ignored_tasks* (Hash<String, String>): List of task names for which we ignore idempotence errors, with the corresponding descriptive reason for ignore.
         # Array< Hash<Symbol, Object> >
         @ignored_idempotence_tasks = []
+        # List of ignored tasks info. Each info has the following properties:
+        # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule
+        # * *ignored_tasks* (Hash<String, String>): List of task names for which we ignore divergence errors, with the corresponding descriptive reason for ignore.
+        # Array< Hash<Symbol, Object> >
+        @ignored_divergent_tasks = []
       end
 
       # Ignore idempotence errors on a set of tasks
@@ -31,6 +42,17 @@ module HybridPlatformsConductor
         }
       end
 
+      # Ignore idempotence errors on a set of tasks
+      #
+      # Parameters::
+      # * *tasks_to_ignore* (Hash<String, String>): Set of tasks to ignore, along with the reason
+      def ignore_divergent_tasks(tasks_to_ignore)
+        @ignored_divergent_tasks << {
+          ignored_tasks: tasks_to_ignore,
+          nodes_selectors_stack: current_nodes_selectors_stack,
+        }
+      end
+
     end
 
   end

data/lib/hybrid_platforms_conductor/deployer.rb

@@ -41,7 +41,7 @@ module HybridPlatformsConductor
 
     include LoggerHelpers
 
-    Config.extend_config_dsl_with ConfigDSLExtension, :init_nodes_handler_config
+    Config.extend_config_dsl_with ConfigDSLExtension, :init_deployer_config
 
     # Do we use why-run mode while deploying? [default = false]
     #   Boolean
@@ -336,7 +336,8 @@ module HybridPlatformsConductor
           actions_executor.connector(:ssh).passwords[node] = 'root_pwd'
           deployer.local_environment = true
           # Ignore secrets that might have been given: in Docker containers we always use dummy secrets
-          deployer.secrets = [JSON.parse(File.read("#{@config.hybrid_platforms_dir}/dummy_secrets.json"))]
+          dummy_secrets_file = "#{@config.hybrid_platforms_dir}/dummy_secrets.json"
+          deployer.secrets = File.exist?(dummy_secrets_file) ? [JSON.parse(File.read(dummy_secrets_file))] : []
           yield deployer, instance
         end
       rescue

data/lib/hybrid_platforms_conductor/hpc_plugins/action/remote_bash.rb

@@ -14,16 +14,33 @@ module HybridPlatformsConductor
         # [API] - @actions_executor is accessible
         #
         # Parameters::
-        # * *remote_bash* (Array< Hash<Symbol, Object> or Array<String> or String>): List of bash actions to execute. Each action can have the following properties:
-        #   * *commands* (Array<String> or String): List of bash commands to execute (can be a single one). This is the default property also that allows to not use the Hash form for brevity.
-        #   * *file* (String): Name of file from which commands should be taken.
-        #   * *env* (Hash<String, String>): Environment variables to be set before executing those commands.
+        # * *remote_bash* (Array or Object): List of commands (or single command) to be executed. Each command can be the following:
+        #   * String: Simple bash command.
+        #   * Hash<Symbol, Object>: Information about the commands to execute. Can have the following properties:
+        #     * *commands* (Array<String> or String): List of bash commands to execute (can be a single one) [default: ''].
+        #     * *file* (String): Name of file from which commands should be taken [optional].
+        #     * *env* (Hash<String, String>): Environment variables to be set before executing those commands [default: {}].
         def setup(remote_bash)
-          @remote_bash = remote_bash
-          # Normalize the parameters
-          @remote_bash = [@remote_bash] if @remote_bash.is_a?(String)
-          @remote_bash = { commands: @remote_bash } if @remote_bash.is_a?(Array)
-          @remote_bash[:commands] = [@remote_bash[:commands]] if @remote_bash[:commands].is_a?(String)
+          # Normalize the parameters.
+          # Array< Hash<Symbol,Object> >: Simple array of info:
+          # * *commands* (Array<String>): List of bash commands to execute.
+          # * *env* (Hash<String, String>): Environment variables to be set before executing those commands.
+          @remote_bash = (remote_bash.is_a?(Array) ? remote_bash : [remote_bash]).map do |cmd_info|
+            if cmd_info.is_a?(String)
+              {
+                commands: [cmd_info],
+                env: {}
+              }
+            else
+              commands = []
+              commands.concat(cmd_info[:commands].is_a?(String) ? [cmd_info[:commands]] : cmd_info[:commands]) if cmd_info[:commands]
+              commands << File.read(cmd_info[:file]) if cmd_info[:file]
+              {
+                commands: commands,
+                env: cmd_info[:env] || {}
+              }
+            end
+          end
         end
 
         # Do we need a connector to execute this action on a node?
@@ -46,10 +63,9 @@ module HybridPlatformsConductor
         # [API] - @stderr_io can be used to log stderr messages
         # [API] - run_cmd(String) method can be used to execute a command. See CmdRunner#run_cmd to know about the result's signature.
         def execute
-          bash_commands = (@remote_bash[:env] || {}).map { |var_name, var_value| "export #{var_name}='#{var_value}'" }
-          bash_commands.concat(@remote_bash[:commands].clone) if @remote_bash.key?(:commands)
-          bash_commands << File.read(@remote_bash[:file]) if @remote_bash.key?(:file)
-          bash_str = bash_commands.join("\n")
+          bash_str = @remote_bash.map do |cmd_info|
+            (cmd_info[:env].map { |var_name, var_value| "export #{var_name}='#{var_value}'" } + cmd_info[:commands]).join("\n")
+          end.join("\n")
           log_debug "[#{@node}] - Execute remote Bash commands \"#{bash_str}\"..."
           @connector.remote_bash bash_str
         end

data/lib/hybrid_platforms_conductor/hpc_plugins/action/scp.rb

@@ -16,7 +16,7 @@ module HybridPlatformsConductor
         # Parameters::
         # * *mappings* (Hash<String or Symbol, Object>): Set of couples source => destination_dir to copy files or directories from the local file system to the remote file system.
         #   The following properties can also be used:
-        #   * *sudo* (Boolean): Do we use sudo to make the copy? [default: false]
+        #   * *sudo* (Boolean): Do we use sudo on the remote to make the copy? [default: false]
         #   * *owner* (String or nil): Owner to use for files, or nil to use current one [default: nil]
         #   * *group* (String or nil): Group to use for files, or nil to use current one [default: nil]
         def setup(mappings)

data/lib/hybrid_platforms_conductor/hpc_plugins/connector/local.rb

@@ -0,0 +1,98 @@
+require 'fileutils'
+require 'tmpdir'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Connector
+
+      # Connector executing remote commands on the local environment in dedicated workspaces (/tmp/hpc_local_workspaces)
+      class Local < HybridPlatformsConductor::Connector
+
+        # Select nodes where this connector can connect.
+        # [API] - This method is mandatory
+        # [API] - @cmd_runner can be used
+        # [API] - @nodes_handler can be used
+        #
+        # Parameters::
+        # * *nodes* (Array<String>): List of candidate nodes
+        # Result::
+        # * Array<String>: List of nodes we can connect to from the candidates
+        def connectable_nodes_from(nodes)
+          @nodes_handler.prefetch_metadata_of nodes, :local_node
+          nodes.select { |node| @nodes_handler.get_local_node_of(node) }
+        end
+
+        # Run bash commands on a given node.
+        # [API] - This method is mandatory
+        # [API] - If defined, then with_connection_to has been called before this method.
+        # [API] - @cmd_runner can be used
+        # [API] - @nodes_handler can be used
+        # [API] - @node can be used to access the node on which we execute the remote bash
+        # [API] - @timeout can be used to know when the action should fail
+        # [API] - @stdout_io can be used to send stdout output
+        # [API] - @stderr_io can be used to send stderr output
+        #
+        # Parameters::
+        # * *bash_cmds* (String): Bash commands to execute
+        def remote_bash(bash_cmds)
+          run_cmd "cd #{workspace_for(@node)} ; #{bash_cmds}"
+        end
+
+        # Execute an interactive shell on the remote node
+        # [API] - This method is mandatory
+        # [API] - If defined, then with_connection_to has been called before this method.
+        # [API] - @cmd_runner can be used
+        # [API] - @nodes_handler can be used
+        # [API] - @node can be used to access the node on which we execute the remote bash
+        # [API] - @timeout can be used to know when the action should fail
+        # [API] - @stdout_io can be used to send stdout output
+        # [API] - @stderr_io can be used to send stderr output
+        def remote_interactive
+          system "cd #{workspace_for(@node)} ; /bin/bash"
+        end
+
+        # Copy a file to the remote node in a directory
+        # [API] - This method is mandatory
+        # [API] - If defined, then with_connection_to has been called before this method.
+        # [API] - @cmd_runner can be used
+        # [API] - @nodes_handler can be used
+        # [API] - @node can be used to access the node on which we execute the remote bash
+        # [API] - @timeout can be used to know when the action should fail
+        # [API] - @stdout_io can be used to send stdout output
+        # [API] - @stderr_io can be used to send stderr output
+        #
+        # Parameters::
+        # * *from* (String): Local file to copy
+        # * *to* (String): Remote directory to copy to
+        # * *sudo* (Boolean): Do we use sudo to copy? [default: false]
+        # * *owner* (String or nil): Owner to be used when copying the files, or nil for current one [default: nil]
+        # * *group* (String or nil): Group to be used when copying the files, or nil for current one [default: nil]
+        def remote_copy(from, to, sudo: false, owner: nil, group: nil)
+          # If the destination is a relative path, prepend the workspace dir to it.
+          to = "#{workspace_for(@node)}/#{to}" unless to.start_with?('/')
+          FileUtils.cp_r from, to
+        end
+
+        private
+
+        # Create or reuse a dedicated workspace for a node
+        #
+        # Parameters::
+        # * *node* (String): Node for which we want a dedicated workspace
+        # Result::
+        # * String: Dedicated workspace path
+        def workspace_for(node)
+          workspace = "#{Dir.tmpdir}/hpc_local_workspaces/#{node}"
+          FileUtils.mkdir_p workspace
+          workspace
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/connector/my_connector.rb.sample

@@ -73,7 +73,7 @@ module HybridPlatformsConductor
         # Result::
         # * Array<String>: List of nodes we can connect to from the candidates
         def connectable_nodes_from(nodes)
-          nodes.select { |node| @nodes_handler.get_ip_of(node) =~ /^192\.168\..+$/ }
+          nodes.select { |node| @nodes_handler.get_host_ip_of(node) =~ /^192\.168\..+$/ }
         end
 
         # Prepare connections to a given set of nodes.
@@ -135,7 +135,7 @@ module HybridPlatformsConductor
         # Parameters::
         # * *from* (String): Local file to copy
         # * *to* (String): Remote directory to copy to
-        # * *sudo* (Boolean): Do we use sudo to copy? [default: false]
+        # * *sudo* (Boolean): Do we use sudo on the remote to copy? [default: false]
         # * *owner* (String or nil): Owner to be used when copying the files, or nil for current one [default: nil]
         # * *group* (String or nil): Group to be used when copying the files, or nil for current one [default: nil]
         def remote_copy(from, to, sudo: false, owner: nil, group: nil)

data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb

@@ -131,6 +131,10 @@ module HybridPlatformsConductor
           # Default values
           @ssh_user = ENV['hpc_ssh_user']
           @ssh_user = ENV['USER'] if @ssh_user.nil? || @ssh_user.empty?
+          if @ssh_user.nil? || @ssh_user.empty?
+            _exit_status, stdout = @cmd_runner.run_cmd 'whoami', log_to_stdout: log_debug?
+            @ssh_user = stdout.strip
+          end
           @ssh_use_control_master = true
           @ssh_strict_host_key_checking = true
           @passwords = {}
@@ -236,9 +240,9 @@ module HybridPlatformsConductor
           ssh_cmd =
             if @nodes_handler.get_ssh_session_exec_of(@node) == 'false'
               # When ExecSession is disabled we need to use stdin directly
-              "{ cat | #{ssh_exec} #{ssh_url} -T; } <<'EOF'\n#{bash_cmds}\nEOF"
+              "{ cat | #{ssh_exec} #{ssh_url} -T; } <<'HPC_EOF'\n#{bash_cmds}\nHPC_EOF"
             else
-              "#{ssh_exec} #{ssh_url} /bin/bash <<'EOF'\n#{bash_cmds}\nEOF"
+              "#{ssh_exec} #{ssh_url} /bin/bash <<'HPC_EOF'\n#{bash_cmds}\nHPC_EOF"
             end
           # Due to a limitation of Process.spawn, each individual argument is limited to 128KB of size.
           # Therefore we need to make sure that if bash_cmds exceeds MAX_CMD_ARG_LENGTH bytes (considering EOF chars) then we use an intermediary shell script to store the commands.
@@ -292,13 +296,20 @@ module HybridPlatformsConductor
         # Parameters::
         # * *from* (String): Local file to copy
         # * *to* (String): Remote directory to copy to
-        # * *sudo* (Boolean): Do we use sudo to copy? [default: false]
+        # * *sudo* (Boolean): Do we use sudo on the remote to copy? [default: false]
         # * *owner* (String or nil): Owner to be used when copying the files, or nil for current one [default: nil]
         # * *group* (String or nil): Group to be used when copying the files, or nil for current one [default: nil]
         def remote_copy(from, to, sudo: false, owner: nil, group: nil)
           if @nodes_handler.get_ssh_session_exec_of(@node) == 'false'
             # We don't have ExecSession, so don't use ssh, but scp instead.
-            run_cmd "scp -S #{ssh_exec} #{from} #{ssh_url}:#{to}"
+            if sudo
+              # We need to first copy the file in an accessible directory, and then sudo mv
+              remote_bash('mkdir -p hpc_tmp_scp')
+              run_cmd "scp -S #{ssh_exec} #{from} #{ssh_url}:./hpc_tmp_scp"
+              remote_bash("#{@nodes_handler.sudo_on(@node)} mv ./hpc_tmp_scp/#{File.basename(from)} #{to}")
+            else
+              run_cmd "scp -S #{ssh_exec} #{from} #{ssh_url}:#{to}"
+            end
           else
             run_cmd <<~EOS
               cd #{File.dirname(from)} && \

data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/platform_handler_plugin.rb.sample

@@ -69,7 +69,7 @@ module HybridPlatformsConductor
         end
 
         # Setup the platform, install dependencies...
-        # [API] - This method is mandatory.
+        # [API] - This method is optional.
         # [API] - @cmd_runner is accessible.
         def setup
           # This method is called by the setup executable.
@@ -163,7 +163,7 @@ module HybridPlatformsConductor
         end
 
         # Package the repository, ready to be deployed on artefacts or directly to a node.
-        # [API] - This method is mandatory.
+        # [API] - This method is optional.
         # [API] - @cmd_runner is accessible.
         # [API] - @actions_executor is accessible.
         #
@@ -212,7 +212,7 @@ module HybridPlatformsConductor
           # This method returns all the actions to execute to deploy on a node.
           # The use_why_run switch is on if the deployment should just be simulated.
           # Those actions (bash commands, scp of files, ruby code...) should be thread safe as they can be executed in parallel with other deployment actions for other nodes in case of a concurrent deployment on several nodes.
-          # The complete description of an action can be found in actions_executor.rb file, in the execute_actions_on method description.
+          # The complete description of an action can be found in the action plugins' documentation.
           [
             {
               scp: {
@@ -229,7 +229,7 @@ module HybridPlatformsConductor
         end
 
         # Prepare a why-run deployment so that a JSON file describing the nodes will be output in the run_logs.
-        # [API] - This method is mandatory.
+        # [API] - This method is optional.
         # [API] - @cmd_runner is accessible.
         # [API] - @actions_executor is accessible.
         # [API] - @deployer is accessible.
@@ -247,7 +247,7 @@ module HybridPlatformsConductor
         # Result::
         # * Array< Hash<Symbol,Object> >: List of task properties. The following properties should be returned, among free ones:
         #   * *name* (String): Task name
-        #   * *status* (Symbol): Task status. Should be on of:
+        #   * *status* (Symbol): Task status. Should be one of:
         #     * *:changed*: The task has been changed
         #     * *:identical*: The task has not been changed
         #   * *diffs* (String): Differences, if any

data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/yaml_inventory.rb

@@ -0,0 +1,140 @@
+require 'yaml'
+require 'hybrid_platforms_conductor/platform_handler'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module PlatformHandler
+
+      # Basic platform handler, reading inventory and metadata from simple Yaml files.
+      class YamlInventory < HybridPlatformsConductor::PlatformHandler
+
+        # Initialize a new instance of this platform handler.
+        # [API] - This method is optional.
+        # [API] - @cmd_runner is accessible.
+        def init
+          # This method is called when initializing a new instance of this platform handler, for a given repository.
+          inv_file = "#{@repository_path}/inventory.yaml"
+          @inventory = File.exist?(inv_file) ? YAML.load(File.read(inv_file)) : {}
+        end
+
+        # Get the list of known nodes.
+        # [API] - This method is mandatory.
+        #
+        # Result::
+        # * Array<String>: List of node names
+        def known_nodes
+          @inventory.keys
+        end
+
+        # Get the metadata of a given node.
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *node* (String): Node to read metadata from
+        # Result::
+        # * Hash<Symbol,Object>: The corresponding metadata
+        def metadata_for(node)
+          (@inventory[node]['metadata'] || {}).transform_keys(&:to_sym)
+        end
+
+        # Return the services for a given node
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *node* (String): node to read configuration from
+        # Result::
+        # * Array<String>: The corresponding services
+        def services_for(node)
+          @inventory[node]['services'] || []
+        end
+
+        # Get the list of services we can deploy
+        # [API] - This method is mandatory.
+        #
+        # Result::
+        # * Array<String>: The corresponding services
+        def deployable_services
+          Dir.glob("#{@repository_path}/service_*.rb").map { |file| File.basename(file).match(/^service_(.*)\.rb$/)[1] }
+        end
+
+        # Get the list of actions to perform to deploy on a given node.
+        # Those actions can be executed in parallel with other deployments on other nodes. They must be thread safe.
+        # [API] - This method is mandatory.
+        # [API] - @cmd_runner is accessible.
+        # [API] - @actions_executor is accessible.
+        #
+        # Parameters::
+        # * *node* (String): Node to deploy on
+        # * *service* (String): Service to be deployed
+        # * *use_why_run* (Boolean): Do we use a why-run mode? [default = true]
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of actions to be done
+        def actions_to_deploy_on(node, service, use_why_run: true)
+          # Load the check and deploy methods in a temporary class for encapsulation
+          service_file = "#{@repository_path}/service_#{service}.rb"
+          Class.new do
+
+            include LoggerHelpers
+
+            # Constructor
+            #
+            # Parameters::
+            # * *platform_handler* (PlatformHandler): PlatformHandler needing this service to be deployed
+            # * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
+            # * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
+            # * *config* (Config): Config to be used. [default: Config.new]
+            # * *nodes_handler* (NodesHandler): NodesHandler to be used [default: NodesHandler.new]
+            # * *cmd_runner* (CmdRunner): CmdRunner to be used [default: CmdRunner.new]
+            def initialize(
+              platform_handler,
+              logger: Logger.new(STDOUT),
+              logger_stderr: Logger.new(STDERR),
+              config: Config.new,
+              nodes_handler: NodesHandler.new,
+              cmd_runner: CmdRunner.new
+            )
+              init_loggers(logger, logger_stderr)
+              @platform_handler = platform_handler
+              @config = config
+              @nodes_handler = nodes_handler
+              @cmd_runner = cmd_runner
+            end
+
+            class_eval(File.read(service_file))
+
+          end.new(
+            self,
+            logger: @logger,
+            logger_stderr: @logger_stderr,
+            config: @config,
+            nodes_handler: @nodes_handler,
+            cmd_runner: @cmd_runner
+          ).send(use_why_run ? :check : :deploy, node)
+        end
+
+        # Parse stdout and stderr of a given deploy run and get the list of tasks with their status
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *stdout* (String): stdout to be parsed
+        # * *stderr* (String): stderr to be parsed
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of task properties. The following properties should be returned, among free ones:
+        #   * *name* (String): Task name
+        #   * *status* (Symbol): Task status. Should be one of:
+        #     * *:changed*: The task has been changed
+        #     * *:identical*: The task has not been changed
+        #   * *diffs* (String): Differences, if any
+        def parse_deploy_output(stdout, stderr)
+          []
+        end
+
+      end
+
+    end
+
+  end
+
+end