hybrid_platforms_conductor 32.3.6

data/lib/hybrid_platforms_conductor/hpc_plugins/action/scp.rb

@@ -0,0 +1,61 @@
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Action
+
+      # Copy files and directories from the local host to the remote one
+      class Scp < HybridPlatformsConductor::Action
+
+        # Setup the action.
+        # This is called by the constructor itself, when an action is instantiated to be executed for a node.
+        # [API] - This method is optional
+        # [API] - @cmd_runner is accessible
+        # [API] - @actions_executor is accessible
+        #
+        # Parameters::
+        # * *mappings* (Hash<String or Symbol, Object>): Set of couples source => destination_dir to copy files or directories from the local file system to the remote file system.
+        #   The following properties can also be used:
+        #   * *sudo* (Boolean): Do we use sudo to make the copy? [default: false]
+        #   * *owner* (String or nil): Owner to use for files, or nil to use current one [default: nil]
+        #   * *group* (String or nil): Group to use for files, or nil to use current one [default: nil]
+        def setup(mappings)
+          @mappings = mappings
+          @sudo = @mappings.delete(:sudo) || false
+          @owner = @mappings.delete(:owner)
+          @group = @mappings.delete(:group)
+        end
+
+        # Do we need a connector to execute this action on a node?
+        #
+        # Result::
+        # * Boolean: Do we need a connector to execute this action on a node?
+        def need_connector?
+          true
+        end
+
+        # Execute the action
+        # [API] - This method is mandatory
+        # [API] - @cmd_runner is accessible
+        # [API] - @actions_executor is accessible
+        # [API] - @action_info is accessible with the action details
+        # [API] - @node (String) can be used to know on which node the action is to be executed
+        # [API] - @connector (Connector or nil) can be used to access the node's connector if the action needs remote connection
+        # [API] - @timeout (Integer) should be used to make sure the action execution does not get past this number of seconds
+        # [API] - @stdout_io can be used to log stdout messages
+        # [API] - @stderr_io can be used to log stderr messages
+        # [API] - run_cmd(String) method can be used to execute a command. See CmdRunner#run_cmd to know about the result's signature.
+        def execute
+          @mappings.each do |from, to|
+            log_debug "[#{@node}] - Copy to remote \"#{from}\" => \"#{to}\""
+            @connector.remote_copy from, to, sudo: @sudo, owner: @owner, group: @group
+          end
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/config.rb

@@ -0,0 +1,78 @@
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Cmdb
+
+      # CMDB getting metadata from the Config DSL
+      class Config < HybridPlatformsConductor::Cmdb
+
+        # Extend the Config DSL
+        module ConfigDSLExtension
+
+          # List of metadata to be set. Each info has the following properties:
+          # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule.
+          # * *metadata* (Hash<Symbol,Object>): Metadata to associate to the nodes.
+          # Array< Hash<Symbol, Object> >
+          attr_reader :nodes_metadata
+
+          # Mixin initializer
+          def init_cmdb_config
+            @nodes_metadata = []
+          end
+
+          # Set metadata associated to the nodes
+          #
+          # Parameters::
+          # * *metadata* (Hash<Symbol,Object>): Metadata to associate to the nodes.
+          def set_metadata(metadata)
+            @nodes_metadata << {
+              metadata: metadata,
+              nodes_selectors_stack: current_nodes_selectors_stack
+            }
+          end
+
+        end
+
+        extend_config_dsl_with ConfigDSLExtension, :init_cmdb_config
+
+        # get_* methods are automatically detected as possible metadata properties this plugin can fill.
+        # The property name filled by such method is given by the method's name: get_my_property will fill the :my_property metadata.
+        # The method get_others is used specifically to return properties whose name is unknown before fetching them.
+
+        # Get other properties for a given set of nodes.
+        # It's better to not use this method and prefer using methods naming the property being returned.
+        # As the nodes_handler can't know in advance which properties will be returned, it will call it every time there is a missing property.
+        # If this method always returns the same values, it would be clever to cache it here.
+        # [API] - This method is optional.
+        # [API] - @platforms_handler can be used.
+        # [API] - @nodes_handler can be used.
+        # [API] - @cmd_runner can be used.
+        #
+        # Parameters::
+        # * *nodes* (Array<String>): The nodes to lookup the property for.
+        # * *metadata* (Hash<String, Hash<Symbol,Object> >): Existing metadata for each node. Dependent properties should be present here.
+        # Result::
+        # * Hash<String, Hash<Symbol,Object> >: The corresponding properties, per required node.
+        #     Nodes for which the property can't be fetched can be ommitted.
+        def get_others(nodes, metadata)
+          # Keep metadata values in a cache, per node
+          @cached_metadata = {} unless defined?(@cached_metadata)
+          nodes.each do |node|
+            unless @cached_metadata.key?(node)
+              @cached_metadata[node] = @nodes_handler.
+                select_confs_for_node(node, @config.nodes_metadata).
+                map { |nodes_metadata_info| nodes_metadata_info[:metadata] }.
+                inject({}) { |merged_metadata, metadata| merged_metadata.merge(metadata) }
+            end
+          end
+          @cached_metadata.slice(*nodes)
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_ip.rb

@@ -0,0 +1,104 @@
+require 'hybrid_platforms_conductor/parallel_threads'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Cmdb
+
+      # Get the host IP
+      class HostIp < HybridPlatformsConductor::Cmdb
+
+        include ParallelThreads
+
+        # get_* methods are automatically detected as possible metadata properties this plugin can fill.
+        # The property name filled by such method is given by the method's name: get_my_property will fill the :my_property metadata.
+        # The method get_others is used specifically to return properties whose name is unknown before fetching them.
+
+        # Return possible dependencies between properties.
+        # A property can need another property to be set before.
+        # For example an IP would need first to have the hostname to be known in order to be looked up.
+        # [API] - This method is optional
+        #
+        # Result::
+        # * Hash<Symbol, Symbol or Array<Symbol> >: The list of necessary properties (or single one) that should be set, per property name (:others can also be used here)
+        def property_dependencies
+          {
+            host_ip: :hostname
+          }
+        end
+
+        # Get a specific property for a given set of nodes.
+        # [API] - @platforms_handler can be used.
+        # [API] - @nodes_handler can be used.
+        # [API] - @cmd_runner can be used.
+        #
+        # Parameters::
+        # * *nodes* (Array<String>): The nodes to lookup the property for.
+        # * *metadata* (Hash<String, Hash<Symbol,Object> >): Existing metadata for each node. Dependent properties should be present here.
+        # Result::
+        # * Hash<String, Object>: The corresponding property, per required node.
+        #     Nodes for which the property can't be fetched can be ommitted.
+        def get_host_ip(nodes, metadata)
+          updated_metadata = {}
+          # Get the list of nodes, per hostname (just in case several nodes share the same hostname)
+          # Hash<String, Array<String> >
+          hostnames = Hash.new { |hash, key| hash[key] = [] }
+          metadata.each do |node, node_metadata|
+            hostnames[node_metadata[:hostname]] << node if node_metadata[:hostname]
+          end
+          unless hostnames.empty?
+            ip_for(*hostnames.keys).each do |hostname, ip|
+              hostnames[hostname].each do |node|
+                updated_metadata[node] = ip
+              end
+            end
+          end
+          updated_metadata
+        end
+
+        private
+
+        # Timeout (in seconds) to use getent, per host
+        TIMEOUT_GETENT = 30
+        # Number of threads max to use for getent calls
+        MAX_THREADS_GETENT = 32
+
+        # Discover the real IPs associated to a list of hosts.
+        #
+        # Parameters::
+        # * *hosts* (Array<String>): The hosts to check for
+        # Result::
+        # * Hash<String, String or nil>: The corresponding IP (or nil if none), per host name
+        def ip_for(*hosts)
+          results = {}
+          log_debug "Get IPs of #{hosts.size} hosts..."
+          for_each_element_in(
+            hosts,
+            parallel: true,
+            nbr_threads_max: MAX_THREADS_GETENT,
+            progress: log_debug? ? 'Gather IPs' : nil
+          ) do |host|
+            _exit_status, stdout, _stderr = @cmd_runner.run_cmd(
+              "getent hosts #{host}",
+              timeout: TIMEOUT_GETENT,
+              log_to_stdout: log_debug?,
+              no_exception: true
+            )
+            ip = stdout.strip.split(/\s+/).first
+            if ip.nil?
+              log_warn "Host #{host} has no IP."
+            else
+              results[host] = ip
+            end
+          end
+          results
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/host_keys.rb

@@ -0,0 +1,114 @@
+require 'hybrid_platforms_conductor/parallel_threads'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Cmdb
+
+      # Get the needed host keys to nodes
+      class HostKeys < HybridPlatformsConductor::Cmdb
+
+        include ParallelThreads
+
+        # get_* methods are automatically detected as possible metadata properties this plugin can fill.
+        # The property name filled by such method is given by the method's name: get_my_property will fill the :my_property metadata.
+        # The method get_others is used specifically to return properties whose name is unknown before fetching them.
+
+        # Return possible dependencies between properties.
+        # A property can need another property to be set before.
+        # For example an IP would need first to have the hostname to be known in order to be looked up.
+        # [API] - This method is optional
+        #
+        # Result::
+        # * Hash<Symbol, Symbol or Array<Symbol> >: The list of necessary properties (or single one) that should be set, per property name (:others can also be used here)
+        def property_dependencies
+          {
+            host_keys: %i[hostname host_ip]
+          }
+        end
+
+        # Get a specific property for a given set of nodes.
+        # [API] - @platforms_handler can be used.
+        # [API] - @nodes_handler can be used.
+        # [API] - @cmd_runner can be used.
+        #
+        # Parameters::
+        # * *nodes* (Array<String>): The nodes to lookup the property for.
+        # * *metadata* (Hash<String, Hash<Symbol,Object> >): Existing metadata for each node. Dependent properties should be present here.
+        # Result::
+        # * Hash<String, Object>: The corresponding property, per required node.
+        #     Nodes for which the property can't be fetched can be ommitted.
+        def get_host_keys(nodes, metadata)
+          updated_metadata = {}
+          # Get the list of nodes, per hostname (just in case several nodes share the same hostname)
+          # Hash<String, Array<String> >
+          hostnames = Hash.new { |hash, key| hash[key] = [] }
+          metadata.each do |node, node_metadata|
+            if node_metadata[:host_ip]
+              hostnames[node_metadata[:host_ip]] << node
+            elsif node_metadata[:hostname]
+              hostnames[node_metadata[:hostname]] << node
+            end
+          end
+          unless hostnames.empty?
+            host_keys_for(*hostnames.keys).each do |hostname, ip|
+              hostnames[hostname].each do |node|
+                updated_metadata[node] = ip
+              end
+            end
+          end
+          updated_metadata
+        end
+
+        private
+
+        # Timeout (in seconds) to use ssh-keyscan, per host
+        TIMEOUT_SSH_KEYSCAN = 30
+        # Number of threads max to use for ssh-keyscan calls
+        MAX_THREADS_SSH_KEY_SCAN = 32
+
+        # Discover the host keys associated to a list of hosts.
+        #
+        # Parameters::
+        # * *hosts* (Array<String>): The hosts to check for
+        # Result::
+        # * Hash<String, Array<String> >: The corresponding host keys, per host name
+        def host_keys_for(*hosts)
+          results = {}
+          log_debug "Get host keys of #{hosts.size} hosts..."
+          for_each_element_in(
+            hosts,
+            parallel: true,
+            nbr_threads_max: MAX_THREADS_SSH_KEY_SCAN,
+            progress: log_debug? ? 'Gather host keys' : nil
+          ) do |host|
+            exit_status, stdout, _stderr = @cmd_runner.run_cmd(
+              "ssh-keyscan #{host}",
+              timeout: TIMEOUT_SSH_KEYSCAN,
+              log_to_stdout: log_debug?,
+              no_exception: true
+            )
+            if exit_status == 0
+              found_keys = []
+              stdout.split("\n").each do |line|
+                unless line =~ /^# .*$/
+                  _host, type, key = line.split(/\s+/)
+                  found_keys << "#{type} #{key}"
+                end
+              end
+              results[host] = found_keys.sort unless found_keys.empty?
+            else
+              log_warn "Unable to get host key for #{host}. Ignoring it. Accessing #{host} might require manual acceptance of its host key."
+            end
+          end
+          results
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/cmdb/my_cmdb.rb.sample

@@ -0,0 +1,129 @@
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Cmdb
+
+      # Example of a CMDB plugin
+      class MyCmdb < HybridPlatformsConductor::Cmdb
+
+        # Are dependencies met before using this plugin?
+        # [API] - This method is optional
+        #
+        # Result::
+        # * Boolean: Are dependencies met before using this plugin?
+        def self.valid?
+          begin
+            @cmd_runner.run_cmd 'command -v getent'
+          rescue
+            false
+          end
+          true
+        end
+
+        # Add a Mixin to the DSL parsing the platforms configuration file.
+        # This can be used by any plugin to add plugin-specific configuration getters and setters, accessible later from NodesHandler instances.
+        # An optional initializer can also be given.
+        # [API] - Those calls are optional
+        module MyDSLExtension
+
+          attr_accessor :my_property
+
+          # Initialize the DSL
+          def init_my_dsl_extension
+            @my_property = 42
+          end
+
+        end
+        self.extend_config_dsl_with MyDSLExtension, :init_my_dsl_extension
+
+        # get_* methods are automatically detected as possible metadata properties this plugin can fill.
+        # The property name filled by such method is given by the method's name: get_my_property will fill the :my_property metadata.
+        # The method get_others is used specifically to return properties whose name is unknown before fetching them.
+
+        # Return possible dependencies between properties.
+        # A property can need another property to be set before.
+        # For example an IP would need first to have the hostname to be known in order to be looked up.
+        # [API] - This method is optional
+        #
+        # Result::
+        # * Hash<Symbol, Symbol or Array<Symbol> >: The list of necessary properties (or single one) that should be set, per property name (:others can also be used here)
+        def property_dependencies
+          {
+            ip: :hostname
+          }
+        end
+
+        # Get a specific property for a given set of nodes.
+        # [API] - @platforms_handler can be used.
+        # [API] - @nodes_handler can be used.
+        # [API] - @cmd_runner can be used.
+        #
+        # Parameters::
+        # * *nodes* (Array<String>): The nodes to lookup the property for.
+        # * *metadata* (Hash<String, Hash<Symbol,Object> >): Existing metadata for each node. Dependent properties should be present here.
+        # Result::
+        # * Hash<String, Object>: The corresponding property, per required node.
+        #     Nodes for which the property can't be fetched can be ommitted.
+        def get_ip(nodes, metadata)
+          Hash[nodes.map do |node|
+            _exit_status, stdout = @cmd_runner.run_cmd "getent hosts #{metadata[node][:hostname]}"
+            [node, stdout.strip.split(/\s+/).first]
+          end]
+        end
+
+        # Get a specific property for a given set of nodes.
+        # [API] - @platforms_handler can be used.
+        # [API] - @nodes_handler can be used.
+        # [API] - @cmd_runner can be used.
+        #
+        # Parameters::
+        # * *nodes* (Array<String>): The nodes to lookup the property for.
+        # * *metadata* (Hash<String, Hash<Symbol,Object> >): Existing metadata for each node. Dependent properties should be present here.
+        # Result::
+        # * Hash<String, Object>: The corresponding property, per required node.
+        #     Nodes for which the property can't be fetched can be ommitted.
+        def get_description(nodes, metadata)
+          descriptions = {}
+          nodes.each do |node|
+            # Let's say the description of a node is stored in a text file of the node's name
+            desc_file = "/path/to/descriptions/#{node}.txt"
+            descriptions[node] = File.read(desc_file) if File.exist?(desc_file)
+          end
+          descriptions
+        end
+
+        # Get other properties for a given set of nodes.
+        # It's better to not use this method and prefer using methods naming the property being returned.
+        # As the nodes_handler can't know in advance which properties will be returned, it will call it every time there is a missing property.
+        # If this method always returns the same values, it would be clever to cache it here.
+        # [API] - This method is optional.
+        # [API] - @platforms_handler can be used.
+        # [API] - @nodes_handler can be used.
+        # [API] - @cmd_runner can be used.
+        #
+        # Parameters::
+        # * *nodes* (Array<String>): The nodes to lookup the property for.
+        # * *metadata* (Hash<String, Hash<Symbol,Object> >): Existing metadata for each node. Dependent properties should be present here.
+        # Result::
+        # * Hash<String, Hash<Symbol,Object> >: The corresponding properties, per required node.
+        #     Nodes for which the property can't be fetched can be ommitted.
+        def get_others(nodes, metadata)
+          Hash[nodes.map do |node|
+            [
+              node,
+              {
+                property1: 42,
+                property2: 666
+              }
+            ]
+          end]
+        end
+
+      end
+
+    end
+
+  end
+
+end