hybrid_platforms_conductor 32.3.6

data/lib/hybrid_platforms_conductor/executable.rb

@@ -0,0 +1,145 @@
+require 'optparse'
+require 'logger'
+require 'hybrid_platforms_conductor/config'
+require 'hybrid_platforms_conductor/platforms_handler'
+require 'hybrid_platforms_conductor/nodes_handler'
+require 'hybrid_platforms_conductor/actions_executor'
+require 'hybrid_platforms_conductor/cmd_runner'
+require 'hybrid_platforms_conductor/deployer'
+require 'hybrid_platforms_conductor/json_dumper'
+require 'hybrid_platforms_conductor/reports_handler'
+require 'hybrid_platforms_conductor/tests_runner'
+require 'hybrid_platforms_conductor/topographer'
+require 'hybrid_platforms_conductor/logger_helpers'
+require 'hybrid_platforms_conductor/current_dir_monitor'
+
+module HybridPlatformsConductor
+
+  # Give a common executable interface to all our executables
+  class Executable
+
+    include LoggerHelpers
+
+    # Give the list of selected nodes, if the option was offered. Check NodesHandler#select_nodes to know which kind of nodes description exist.
+    #   Array<Object>
+    attr_reader :selected_nodes
+
+    # Constructor
+    #
+    # Parameters::
+    # * *check_options* (Boolean): Do we offer check/why-run options? [default: true]
+    # * *nodes_selection_options* (Boolean): Do we offer nodes selection options? [default: true]
+    # * *parallel_options* (Boolean): Do we offer parallel options? [default: true]
+    # * *timeout_options* (Boolean): Do we offer timeout options? [default: true]
+    # * *deploy_options* (Boolean): Do we offer deploy options? [default: true]
+    # * *logger* (Logger): The stdout logger to be used [default: Logger.new(STDOUT, level: :info)]
+    # * *logger_stderr* (Logger): The stderr logger to be used [default: Logger.new(STDERR, level: :info)]
+    # * *opts_block* (Proc): Optional code called to register main options
+    #   * Parameters::
+    #     * *opts* (OptionsParser): The options parser to complete
+    def initialize(
+      check_options: true,
+      nodes_selection_options: true,
+      parallel_options: true,
+      timeout_options: true,
+      deploy_options: true,
+      logger: Logger.new(STDOUT, level: :info),
+      logger_stderr: Logger.new(STDERR, level: :info),
+      &opts_block
+    )
+      init_loggers(logger, logger_stderr)
+      @check_options = check_options
+      @nodes_selection_options = nodes_selection_options
+      @parallel_options = parallel_options
+      @timeout_options = timeout_options
+      @deploy_options = deploy_options
+      @opts_block = opts_block
+      # List of nodes description selected
+      @selected_nodes = []
+      # Possible Conductor components this executable can use
+      @instantiated_components = {}
+      # Initialize the loggers
+      # We set the debug format right now before calling the options parser, just in case some option parsing needs debugging (like plugins discovery)
+      self.log_level = :debug if ARGV.include?('--debug') || ARGV.include?('-d')
+    end
+
+    # Define all the dependency injection rules between the various APIs.
+    # Singleton accessors for each one of those components will be generated automatically based on these definitions.
+    {
+      config: [],
+      cmd_runner: [],
+      platforms_handler: %i[config cmd_runner],
+      nodes_handler: %i[config cmd_runner platforms_handler],
+      actions_executor: %i[config cmd_runner nodes_handler],
+      services_handler: %i[config cmd_runner platforms_handler nodes_handler actions_executor],
+      deployer: %i[config cmd_runner nodes_handler actions_executor services_handler],
+      json_dumper: %i[config nodes_handler deployer],
+      reports_handler: %i[config platforms_handler nodes_handler],
+      tests_runner: %i[config cmd_runner platforms_handler nodes_handler actions_executor deployer],
+      topographer: %i[nodes_handler json_dumper]
+    }.each do |component, dependencies|
+
+      # Has a singleton been instantiated for this component?
+      #
+      # Result::
+      # * Boolean: Has a singleton been instantiated for this component?
+      define_method("#{component}_instantiated?".to_sym) do
+        @instantiated_components.key?(component)
+      end
+
+      # Get a singleton for this component
+      #
+      # Result::
+      # * Object: The corresponding component
+      define_method(component) do
+        @instantiated_components[component] = HybridPlatformsConductor.const_get(component.to_s.split('_').collect(&:capitalize).join.to_sym).new(
+          logger: @logger,
+          logger_stderr: @logger_stderr,
+          **Hash[dependencies.map { |dependency| [dependency, send(dependency)] }]
+        ) unless @instantiated_components.key?(component)
+        @instantiated_components[component]
+      end
+
+    end
+
+    # Parse options for this executable.
+    # Use options for any Hybrid Platforms Conductor component that has been accessed through the above methods.
+    # Handle common options (like logging and help).
+    def parse_options!
+      OptionParser.new do |opts|
+        opts.banner = "Usage: #{$0} [options]"
+        opts.separator ''
+        opts.separator 'Main options:'
+        opts.on('-d', '--debug', 'Activate debug mode') do
+          self.log_level = :debug
+        end
+        opts.on('-h', '--help', 'Display help and exit') do
+          out opts
+          exit 0
+        end
+        @opts_block.call(opts) if @opts_block
+        nodes_handler.options_parse(opts) if nodes_handler_instantiated?
+        nodes_handler.options_parse_nodes_selectors(opts, @selected_nodes) if @nodes_selection_options
+        cmd_runner.options_parse(opts) if cmd_runner_instantiated?
+        actions_executor.options_parse(opts, parallel: @parallel_options) if actions_executor_instantiated?
+        deployer.options_parse(
+          opts,
+          parallel_switch: @parallel_options,
+          timeout_options: @timeout_options,
+          why_run_switch: @check_options
+        ) if deployer_instantiated? && @deploy_options
+        json_dumper.options_parse(opts) if json_dumper_instantiated?
+        reports_handler.options_parse(opts) if reports_handler_instantiated?
+        tests_runner.options_parse(opts) if tests_runner_instantiated?
+        topographer.options_parse(opts) if topographer_instantiated?
+      end.parse!
+      actions_executor.validate_params if actions_executor_instantiated?
+      deployer.validate_params if deployer_instantiated?
+      reports_handler.validate_params if reports_handler_instantiated?
+      topographer.validate_params if topographer_instantiated?
+      raise "Unknown options: #{ARGV.join(' ')}" unless ARGV.empty?
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/action/bash.rb

@@ -0,0 +1,44 @@
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Action
+
+      # Execute Bash commands locally
+      class Bash < HybridPlatformsConductor::Action
+
+        # Setup the action.
+        # This is called by the constructor itself, when an action is instantiated to be executed for a node.
+        # [API] - This method is optional
+        # [API] - @cmd_runner is accessible
+        # [API] - @actions_executor is accessible
+        #
+        # Parameters::
+        # * *cmd* (String): The bash command to execute
+        def setup(cmd)
+          @cmd = cmd
+        end
+
+        # Execute the action
+        # [API] - This method is mandatory
+        # [API] - @cmd_runner is accessible
+        # [API] - @actions_executor is accessible
+        # [API] - @action_info is accessible with the action details
+        # [API] - @node (String) can be used to know on which node the action is to be executed
+        # [API] - @connector (Connector or nil) can be used to access the node's connector if the action needs remote connection
+        # [API] - @timeout (Integer) should be used to make sure the action execution does not get past this number of seconds
+        # [API] - @stdout_io can be used to log stdout messages
+        # [API] - @stderr_io can be used to log stderr messages
+        # [API] - run_cmd(String) method can be used to execute a command. See CmdRunner#run_cmd to know about the result's signature.
+        def execute
+          log_debug "[#{@node}] - Execute local Bash commands \"#{@cmd}\"..."
+          run_cmd @cmd
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/action/interactive.rb

@@ -0,0 +1,44 @@
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Action
+
+      # Execute an interactive session on the remote node
+      class Interactive < HybridPlatformsConductor::Action
+
+        # Do we need a connector to execute this action on a node?
+        #
+        # Result::
+        # * Boolean: Do we need a connector to execute this action on a node?
+        def need_connector?
+          true
+        end
+
+        # Execute the action
+        # [API] - This method is mandatory
+        # [API] - @cmd_runner is accessible
+        # [API] - @actions_executor is accessible
+        # [API] - @action_info is accessible with the action details
+        # [API] - @node (String) can be used to know on which node the action is to be executed
+        # [API] - @connector (Connector or nil) can be used to access the node's connector if the action needs remote connection
+        # [API] - @timeout (Integer) should be used to make sure the action execution does not get past this number of seconds
+        # [API] - @stdout_io can be used to log stdout messages
+        # [API] - @stderr_io can be used to log stderr messages
+        # [API] - run_cmd(String) method can be used to execute a command. See CmdRunner#run_cmd to know about the result's signature.
+        def execute
+          log_debug "[#{@node}] - Run interactive remote session..."
+          if @cmd_runner.dry_run
+            log_debug "[#{@node}] - Won't execute interactive shell in dry_run mode."
+          else
+            @connector.remote_interactive
+          end
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/action/my_action.rb.sample

@@ -0,0 +1,79 @@
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Action
+
+      # Example of an action plugin
+      class MyAction < HybridPlatformsConductor::Action
+
+        # Are dependencies met before using this plugin?
+        # [API] - This method is optional
+        #
+        # Result::
+        # * Boolean: Are dependencies met before using this plugin?
+        def self.valid?
+          true
+        end
+
+        # Add a Mixin to the DSL parsing the platforms configuration file.
+        # This can be used by any plugin to add plugin-specific configuration getters and setters, accessible later from NodesHandler instances.
+        # An optional initializer can also be given.
+        # [API] - Those calls are optional
+        module MyDSLExtension
+
+          attr_accessor :my_property
+
+          # Initialize the DSL
+          def init_my_dsl_extension
+            @my_property = 42
+          end
+
+        end
+        self.extend_config_dsl_with MyDSLExtension, :init_my_dsl_extension
+
+        # Setup the action.
+        # This is called by the constructor itself, when an action is instantiated to be executed for a node.
+        # [API] - This method is optional
+        # [API] - @cmd_runner is accessible
+        # [API] - @actions_executor is accessible
+        #
+        # Parameters::
+        # * *action_info* (Object): The action info
+        def setup(action_info)
+          @action_info[:message] = 'Default message' unless @action_info.key?(:message)
+        end
+
+        # Do we need a connector to execute this action on a node?
+        # [API] - This method is optional
+        # [API] - @cmd_runner is accessible
+        # [API] - @actions_executor is accessible
+        #
+        # Result::
+        # * Boolean: Do we need a connector to execute this action on a node?
+        def need_connector?
+          true
+        end
+
+        # Execute the action
+        # [API] - This method is mandatory
+        # [API] - @cmd_runner is accessible
+        # [API] - @actions_executor is accessible
+        # [API] - @action_info is accessible with the action details
+        # [API] - @node (String) can be used to know on which node the action is to be executed
+        # [API] - @connector (Connector or nil) can be used to access the node's connector if the action needs remote connection
+        # [API] - @timeout (Integer) should be used to make sure the action execution does not get past this number of seconds
+        # [API] - @stdout_io can be used to log stdout messages
+        # [API] - @stderr_io can be used to log stderr messages
+        # [API] - run_cmd(String) method can be used to execute a command. See CmdRunner#run_cmd to know about the result's signature.
+        def execute
+          run_cmd "echo #{@action_info[:message]} on #{@node}"
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/action/remote_bash.rb

@@ -0,0 +1,63 @@
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Action
+
+      # Execute a bash command on the remote node
+      class RemoteBash < HybridPlatformsConductor::Action
+
+        # Setup the action.
+        # This is called by the constructor itself, when an action is instantiated to be executed for a node.
+        # [API] - This method is optional
+        # [API] - @cmd_runner is accessible
+        # [API] - @actions_executor is accessible
+        #
+        # Parameters::
+        # * *remote_bash* (Array< Hash<Symbol, Object> or Array<String> or String>): List of bash actions to execute. Each action can have the following properties:
+        #   * *commands* (Array<String> or String): List of bash commands to execute (can be a single one). This is the default property also that allows to not use the Hash form for brevity.
+        #   * *file* (String): Name of file from which commands should be taken.
+        #   * *env* (Hash<String, String>): Environment variables to be set before executing those commands.
+        def setup(remote_bash)
+          @remote_bash = remote_bash
+          # Normalize the parameters
+          @remote_bash = [@remote_bash] if @remote_bash.is_a?(String)
+          @remote_bash = { commands: @remote_bash } if @remote_bash.is_a?(Array)
+          @remote_bash[:commands] = [@remote_bash[:commands]] if @remote_bash[:commands].is_a?(String)
+        end
+
+        # Do we need a connector to execute this action on a node?
+        #
+        # Result::
+        # * Boolean: Do we need a connector to execute this action on a node?
+        def need_connector?
+          true
+        end
+
+        # Execute the action
+        # [API] - This method is mandatory
+        # [API] - @cmd_runner is accessible
+        # [API] - @actions_executor is accessible
+        # [API] - @action_info is accessible with the action details
+        # [API] - @node (String) can be used to know on which node the action is to be executed
+        # [API] - @connector (Connector or nil) can be used to access the node's connector if the action needs remote connection
+        # [API] - @timeout (Integer) should be used to make sure the action execution does not get past this number of seconds
+        # [API] - @stdout_io can be used to log stdout messages
+        # [API] - @stderr_io can be used to log stderr messages
+        # [API] - run_cmd(String) method can be used to execute a command. See CmdRunner#run_cmd to know about the result's signature.
+        def execute
+          bash_commands = (@remote_bash[:env] || {}).map { |var_name, var_value| "export #{var_name}='#{var_value}'" }
+          bash_commands.concat(@remote_bash[:commands].clone) if @remote_bash.key?(:commands)
+          bash_commands << File.read(@remote_bash[:file]) if @remote_bash.key?(:file)
+          bash_str = bash_commands.join("\n")
+          log_debug "[#{@node}] - Execute remote Bash commands \"#{bash_str}\"..."
+          @connector.remote_bash bash_str
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/action/ruby.rb

@@ -0,0 +1,69 @@
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Action
+
+      # Execute Ruby commands locally
+      class Ruby < HybridPlatformsConductor::Action
+
+        # Setup the action.
+        # This is called by the constructor itself, when an action is instantiated to be executed for a node.
+        # [API] - This method is optional
+        # [API] - @cmd_runner is accessible
+        # [API] - @actions_executor is accessible
+        #
+        # Parameters::
+        # * *info* (Hash<Symbol, Object>): Properties for the Ruby action:
+        #   * *code* (Proc): Ruby code to be executed.
+        #     This is the default property, and can be given directly without using a Hash.
+        #     * Parameters::
+        #       * *stdout* (IO): Stream in which stdout of this action should be written.
+        #       * *stderr* (IO): Stream in which stderr of this action should be written.
+        #       * *action* (Action): Action we can use to access other context-specific methods, such as run_cmd.
+        #       * *connector* (Connector or nil): The connector to the node, or nil if none.
+        #   * *need_remote* (Boolean): Do we need a remote connection to the node for this code to run? [default = false]
+        def setup(info)
+          info = { code: info } if info.is_a?(Proc)
+          @need_remote = info[:need_remote] || false
+          @code = info[:code]
+        end
+
+        # Do we need a connector to execute this action on a node?
+        #
+        # Result::
+        # * Boolean: Do we need a connector to execute this action on a node?
+        def need_connector?
+          @need_remote
+        end
+
+        # Execute the action
+        # [API] - This method is mandatory
+        # [API] - @cmd_runner is accessible
+        # [API] - @actions_executor is accessible
+        # [API] - @action_info is accessible with the action details
+        # [API] - @node (String) can be used to know on which node the action is to be executed
+        # [API] - @connector (Connector or nil) can be used to access the node's connector if the action needs remote connection
+        # [API] - @timeout (Integer) should be used to make sure the action execution does not get past this number of seconds
+        # [API] - @stdout_io can be used to log stdout messages
+        # [API] - @stderr_io can be used to log stderr messages
+        # [API] - run_cmd(String) method can be used to execute a command. See CmdRunner#run_cmd to know about the result's signature.
+        def execute
+          log_debug "[#{@node}] - Execute local Ruby code #{@code}..."
+          # TODO: Handle timeout without using Timeout which is harmful when dealing with SSH connections and multithread.
+          @code.call @stdout_io, @stderr_io, self, @connector
+        end
+
+        # Make the run_cmd method public for this action as it can be used by client procs
+        public :run_cmd
+
+        # Give access to the node so that action can use it
+        attr_reader :node
+
+      end
+
+    end
+
+  end
+
+end