hybrid_platforms_conductor 32.16.2 → 32.18.0

data/docs/plugins/test/vulnerabilities.md

@@ -54,6 +54,7 @@ None
 | Metadata | Type | Usage
 | --- | --- | --- |
 | `image` | `String` | The name of the OS image to be used. The [configuration](../../config_dsl.md) should define the image and point it to a directory containing a `oval.json` that will contain definition of OVAL files to be checked for this OS(see above). |
+| `local_node` | `Boolean` | Skip this test for nodes having this metadata set to `true` |
 
 ## Used environment variables
 

data/lib/hybrid_platforms_conductor/actions_executor.rb

@@ -102,7 +102,14 @@ module HybridPlatformsConductor
     # * *progress_name* (String): Name to display on the progress bar [default: 'Executing actions']
     # Result::
     # * Hash<String, [Integer or Symbol, String, String]>: Exit status code (or Symbol in case of error or dry run), standard output and error for each node.
-    def execute_actions(actions_per_nodes, timeout: nil, concurrent: false, log_to_dir: "#{@config.hybrid_platforms_dir}/run_logs", log_to_stdout: true, progress_name: 'Executing actions')
+    def execute_actions(
+      actions_per_nodes,
+      timeout: nil,
+      concurrent: false,
+      log_to_dir: "#{@config.hybrid_platforms_dir}/run_logs",
+      log_to_stdout: true,
+      progress_name: 'Executing actions'
+    )
       # Keep a list of nodes that will need remote access
       nodes_needing_connectors = []
       # Compute the ordered list of actions per selected node

data/lib/hybrid_platforms_conductor/common_config_dsl/github.rb

@@ -0,0 +1,62 @@
+require 'octokit'
+require 'hybrid_platforms_conductor/credentials'
+
+module HybridPlatformsConductor
+
+  module CommonConfigDsl
+
+    module Github
+
+      # Initialize the DSL 
+      def init_github
+        # List of Github repositories definitions
+        # Array< Hash<Symbol, Object> >
+        # Each definition is just mapping the signature of #github_repos
+        @github_repos = []
+      end
+
+      # Register new Github repositories
+      #
+      # Parameters::
+      # * *url* (String): URL to the Github API [default: 'https://api.github.com']
+      # * *user* (String): User or organization name, storing repositories
+      # * *repos* (Array<String> or Symbol): List of repository names from this project, or :all for all [default: :all]
+      def github_repos(url: 'https://api.github.com', user:, repos: :all)
+        @github_repos << {
+          url: url,
+          user: user,
+          repos: repos
+        }
+      end
+
+      # Iterate over each Github repository
+      #
+      # Parameters::
+      # * Proc: Code called for each Github repository:
+      #   * Parameters::
+      #     * *github* (Octokit::Client): The client instance accessing the Github API
+      #     * *repo_info* (Hash<Symbol, Object>): The repository info:
+      #       * *name* (String): Repository name.
+      #       * *slug* (String): Repository slug.
+      def for_each_github_repo
+        @github_repos.each do |repo_info|
+          Octokit.configure do |c|
+            c.api_endpoint = repo_info[:url]
+          end
+          Credentials.with_credentials_for(:github, @logger, @logger_stderr, url: repo_info[:url]) do |_github_user, github_token|
+            client = Octokit::Client.new(access_token: github_token)
+            (repo_info[:repos] == :all ? client.repositories(repo_info[:user]).map { |repo| repo[:name] } : repo_info[:repos]).each do |name|
+              yield client, {
+                name: name,
+                slug: "#{repo_info[:user]}/#{name}"
+              }
+            end
+          end
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/deployer.rb

@@ -21,12 +21,19 @@ module HybridPlatformsConductor
     # Extend the Config DSL
     module ConfigDSLExtension
 
+      # List of log plugins. Each info has the following properties:
+      # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule.
+      # * *log_plugins* (Array<Symbol>): List of log plugins to be used to store deployment logs.
+      # Array< Hash<Symbol, Object> >
+      attr_reader :deployment_logs
+
       # Integer: Timeout (in seconds) for packaging repositories
       attr_reader :packaging_timeout_secs
 
       # Mixin initializer
       def init_deployer_config
         @packaging_timeout_secs = 60
+        @deployment_logs = []
       end
 
       # Set the packaging timeout
@@ -37,6 +44,17 @@ module HybridPlatformsConductor
         @packaging_timeout_secs = packaging_timeout_secs
       end
 
+      # Set the deployment log plugins to be used
+      #
+      # Parameters::
+      # * *log_plugins* (Symbol or Array<Symbol>): The list of (or single) log plugins to be used
+      def send_logs_to(*log_plugins)
+        @deployment_logs << {
+          nodes_selectors_stack: current_nodes_selectors_stack,
+          log_plugins: log_plugins.flatten
+        }
+      end
+
     end
 
     include LoggerHelpers
@@ -94,6 +112,20 @@ module HybridPlatformsConductor
       @services_handler = services_handler
       @secrets = []
       @provisioners = Plugins.new(:provisioner, logger: @logger, logger_stderr: @logger_stderr)
+      @log_plugins = Plugins.new(
+        :log,
+        logger: @logger,
+        logger_stderr: @logger_stderr,
+        init_plugin: proc do |plugin_class|
+          plugin_class.new(
+            logger: @logger,
+            logger_stderr: @logger_stderr,
+            config: @config,
+            nodes_handler: @nodes_handler,
+            actions_executor: @actions_executor
+          )
+        end
+      )
       # Default values
       @use_why_run = false
       @timeout = nil
@@ -319,7 +351,7 @@ module HybridPlatformsConductor
         )
         instance.with_running_instance(stop_on_exit: true, destroy_on_exit: !reuse_instance, port: 22) do
           # Test-provisioned nodes have SSH Session Exec capabilities and are not local
-          sub_executable.nodes_handler.override_metadata_of node, :ssh_session_exec, 'true'
+          sub_executable.nodes_handler.override_metadata_of node, :ssh_session_exec, true
           sub_executable.nodes_handler.override_metadata_of node, :local_node, false
           # Test-provisioned nodes use default sudo
           sub_executable.config.sudo_procs.replace(sub_executable.config.sudo_procs.map do |sudo_proc_info|
@@ -355,72 +387,31 @@ module HybridPlatformsConductor
     # * *nodes* (Array<String>): Nodes to get info from
     # Result::
     # * Hash<String, Hash<Symbol,Object>: The deployed info, per node name.
-    #   Properties are defined by the Deployer#save_logs method, and additionally to them the following properties can be set:
-    #   * *error* (String): Optional property set in case of error
+    #   * *error* (String): Error string in case deployment logs could not be retrieved. If set then further properties will be ignored. [optional]
+    #   * *services* (Array<String>): List of services deployed on the node
+    #   * *deployment_info* (Hash<Symbol,Object>): Deployment metadata
+    #   * *exit_status* (Integer or Symbol): Deployment exit status
+    #   * *stdout* (String): Deployment stdout
+    #   * *stderr* (String): Deployment stderr
     def deployment_info_from(*nodes)
+      nodes = nodes.flatten
       @actions_executor.max_threads = 64
-      Hash[@actions_executor.
-        execute_actions(
-          Hash[nodes.flatten.map do |node|
-            [
-              node,
-              { remote_bash: "cd /var/log/deployments && ls -t | head -1 | xargs sed '/===== STDOUT =====/q'" }
-            ]
-          end],
-          log_to_stdout: false,
-          concurrent: true,
-          timeout: 10,
-          progress_name: 'Getting deployment info'
-        ).
-        map do |node, (exit_status, stdout, stderr)|
-          # Expected format for stdout:
-          # Property1: Value1
-          # ...
-          # PropertyN: ValueN
-          # ===== STDOUT =====
-          # ...
-          deploy_info = {}
-          if exit_status.is_a?(Symbol)
-            deploy_info[:error] = "Error: #{exit_status}\n#{stderr}"
-          else
-            stdout_lines = stdout.split("\n")
-            if stdout_lines.first =~ /No such file or directory/
-              deploy_info[:error] = '/var/log/deployments missing'
-            else
-              stdout_lines.each do |line|
-                if line =~ /^([^:]+): (.+)$/
-                  key_str, value = $1, $2
-                  key = key_str.to_sym
-                  # Type-cast some values
-                  case key_str
-                  when 'date'
-                    # Date and time values
-                    # Thu Nov 23 18:43:01 UTC 2017
-                    deploy_info[key] = Time.parse(value)
-                  when 'debug'
-                    # Boolean values
-                    # Yes
-                    deploy_info[key] = (value == 'Yes')
-                  when /^diff_files_.+$/, 'services'
-                    # Array of strings
-                    # my_file.txt, other_file.txt
-                    deploy_info[key] = value.split(', ')
-                  else
-                    deploy_info[key] = value
-                  end
-                else
-                  deploy_info[:unknown_lines] = [] unless deploy_info.key?(:unknown_lines)
-                  deploy_info[:unknown_lines] << line
-                end
-              end
-            end
-          end
-          [
-            node,
-            deploy_info
-          ]
-        end
-      ]
+      read_actions_results = @actions_executor.execute_actions(
+        Hash[nodes.map do |node|
+          master_log_plugin = @log_plugins[log_plugins_for(node).first]
+          master_log_plugin.respond_to?(:actions_to_read_logs) ? [node, master_log_plugin.actions_to_read_logs(node)] : nil
+        end.compact],
+        log_to_stdout: false,
+        concurrent: true,
+        timeout: 10,
+        progress_name: 'Read deployment logs'
+      )
+      Hash[nodes.map do |node|
+        [
+          node,
+          @log_plugins[log_plugins_for(node).first].logs_for(node, *(read_actions_results[node] || [nil, nil, nil]))
+        ]
+      end]
     end
 
     # Parse stdout and stderr of a given deploy run and get the list of tasks with their status
@@ -584,47 +575,48 @@ module HybridPlatformsConductor
     # * *services* (Hash<String, Array<String>>): List of services that have been deployed, per node
     def save_logs(logs, services)
       section "Saving deployment logs for #{logs.size} nodes" do
-        Dir.mktmpdir('hybrid_platforms_conductor-logs') do |tmp_dir|
-          ssh_user = @actions_executor.connector(:ssh).ssh_user
-          @actions_executor.execute_actions(
-            Hash[logs.map do |node, (exit_status, stdout, stderr)|
-              # Create a log file to be scp with all relevant info
-              now = Time.now.utc
-              log_file = "#{tmp_dir}/#{node}_#{now.strftime('%F_%H%M%S')}_#{ssh_user}"
-              services_info = @services_handler.log_info_for(node, services[node])
-              File.write(
-                log_file,
-                services_info.merge(
-                  date: now.strftime('%F %T'),
-                  user: ssh_user,
-                  debug: log_debug? ? 'Yes' : 'No',
-                  services: services[node].join(', '),
-                  exit_status: exit_status
-                ).map { |property, value| "#{property}: #{value}" }.join("\n") +
-                  "\n===== STDOUT =====\n" +
-                  (stdout || '') +
-                  "\n===== STDERR =====\n" +
-                  (stderr || '')
-              )
-              [
-                node,
-                {
-                  remote_bash: "#{ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} "}mkdir -p /var/log/deployments",
-                  scp: {
-                    log_file => '/var/log/deployments',
-                    :sudo => ssh_user != 'root',
-                    :owner => 'root',
-                    :group => 'root'
-                  }
-                }
-              ]
-            end],
-            timeout: 10,
-            concurrent: true,
-            log_to_dir: nil
-          )
-        end
+        ssh_user = @actions_executor.connector(:ssh).ssh_user
+        @actions_executor.execute_actions(
+          Hash[logs.map do |node, (exit_status, stdout, stderr)|
+            [
+              node,
+              log_plugins_for(node).
+                map do |log_plugin|
+                  @log_plugins[log_plugin].actions_to_save_logs(
+                    node,
+                    services[node],
+                    @services_handler.log_info_for(node, services[node]).merge(
+                      date: Time.now.utc.strftime('%F %T'),
+                      user: ssh_user
+                    ),
+                    exit_status,
+                    stdout,
+                    stderr
+                  )
+                end.
+                flatten(1)
+            ]
+          end],
+          timeout: 10,
+          concurrent: true,
+          log_to_dir: nil,
+          progress_name: 'Saving logs'
+        )
+      end
+    end
+
+    # Get the list of log plugins to be used for a given node
+    #
+    # Parameters::
+    # * *node* (String): The node for which log plugins are queried
+    # Result::
+    # * Array<Symbol>: The list of log plugins
+    def log_plugins_for(node)
+      node_log_plugins = @nodes_handler.select_confs_for_node(node, @config.deployment_logs).inject([]) do |log_plugins, deployment_logs_info|
+        log_plugins + deployment_logs_info[:log_plugins]
       end
+      node_log_plugins << :remote_fs if node_log_plugins.empty?
+      node_log_plugins
     end
 
   end

data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb

@@ -238,7 +238,7 @@ module HybridPlatformsConductor
         # * *bash_cmds* (String): Bash commands to execute
         def remote_bash(bash_cmds)
           ssh_cmd =
-            if @nodes_handler.get_ssh_session_exec_of(@node) == 'false'
+            if @nodes_handler.get_ssh_session_exec_of(@node) == false
               # When ExecSession is disabled we need to use stdin directly
               "{ cat | #{ssh_exec} #{ssh_url} -T; } <<'HPC_EOF'\n#{bash_cmds}\nHPC_EOF"
             else
@@ -300,7 +300,7 @@ module HybridPlatformsConductor
         # * *owner* (String or nil): Owner to be used when copying the files, or nil for current one [default: nil]
         # * *group* (String or nil): Group to be used when copying the files, or nil for current one [default: nil]
         def remote_copy(from, to, sudo: false, owner: nil, group: nil)
-          if @nodes_handler.get_ssh_session_exec_of(@node) == 'false'
+          if @nodes_handler.get_ssh_session_exec_of(@node) == false
             # We don't have ExecSession, so don't use ssh, but scp instead.
             if sudo
               # We need to first copy the file in an accessible directory, and then sudo mv
@@ -513,7 +513,7 @@ module HybridPlatformsConductor
                     if current_users.empty?
                       log_debug "[ ControlMaster - #{ssh_url} ] - Creating SSH ControlMaster..."
                       exit_status = nil
-                      if @nodes_handler.get_ssh_session_exec_of(node) == 'false'
+                      if @nodes_handler.get_ssh_session_exec_of(node) == false
                         # Here we have to create a ControlMaster using an interactive session, as the SSH server prohibits ExecSession, and so command executions.
                         # We'll do that using another terminal spawned in the background.
                         if ENV['hpc_interactive'] == 'false'

data/lib/hybrid_platforms_conductor/hpc_plugins/log/my_log_plugin.rb.sample

@@ -0,0 +1,100 @@
+require 'hybrid_platforms_conductor/log'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Log
+
+      # Save logs on the remote node's file system
+      class RemoteFs < HybridPlatformsConductor::Log
+
+        # Get actions to save logs
+        # [API] - This method is mandatory.
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        # * *@actions_executor* (ActionsExecutor): Actions executor API.
+        #
+        # Parameters::
+        # * *node* (String): Node for which logs are being saved
+        # * *services* (Array<String>): The list of services that have been deployed on this node
+        # * *deployment_info* (Hash<Symbol,Object>): Additional information to attach to the logs
+        # * *exit_status* (Integer or Symbol): Exit status of the deployment
+        # * *stdout* (String): Deployment's stdout
+        # * *stderr* (String): Deployment's stderr
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of actions to be done
+        def actions_to_save_logs(node, services, deployment_info, exit_status, stdout, stderr)
+          # Return here all actions that are to be run to save those logs.
+          [
+            {
+              bash: <<~EOS
+                cat <<'EOLOGFILE' >>all_logs.txt
+                Deployment log on #{node}:
+                #{stdout}
+                EOLOGFILE
+              EOS
+            }
+          ]
+        end
+
+        # Get actions to read logs.
+        # If provided, this method can return some actions to be executed that will fetch logs from servers or remote nodes.
+        # By using this method to run actions instead of the synchronous method logs_from, such actions will be run in parallel which can greatly improve time-consuming operations when querying a lot of nodes.
+        # [API] - This method is optional.
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        # * *@actions_executor* (ActionsExecutor): Actions executor API.
+        #
+        # Parameters::
+        # * *node* (String): Node for which deployment logs are being read
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of actions to be done
+        def actions_to_read_logs(node)
+          [
+            { bash: 'cat all_logs.txt' }
+          ]
+        end
+
+        # Get deployment logs from a node.
+        # This method can use the result of actions previously run to read logs, as returned by the actions_to_read_logs method.
+        # [API] - This method is mandatory.
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        # * *@actions_executor* (ActionsExecutor): Actions executor API.
+        #
+        # Parameters::
+        # * *node* (String): The node we want deployment logs from
+        # * *exit_status* (Integer, Symbol or nil): Exit status of actions to read logs, or nil if no action was returned by actions_to_read_logs
+        # * *stdout* (String or nil): stdout of actions to read logs, or nil if no action was returned by actions_to_read_logs
+        # * *stderr* (String or nil): stderr of actions to read logs, or nil if no action was returned by actions_to_read_logs
+        # Result::
+        # * Hash<Symbol,Object>: Deployment log information:
+        #   * *error* (String): Error string in case deployment logs could not be retrieved. If set then further properties will be ignored. [optional]
+        #   * *services* (Array<String>): List of services deployed on the node
+        #   * *deployment_info* (Hash<Symbol,Object>): Deployment metadata
+        #   * *exit_status* (Integer or Symbol): Deployment exit status
+        #   * *stdout* (String): Deployment stdout
+        #   * *stderr* (String): Deployment stderr
+        def logs_for(node, exit_status, stdout, stderr)
+          # Here we should retrieve this info from somewhere.
+          # Based on this example, stdout is the value of the execution of the action 'cat all_logs.txt' as returned by actions_to_read_logs.
+          {
+            services: %w[unknown],
+            deployment_info: {},
+            exit_status: 0,
+            stdout: stdout,
+            stderr: ''
+          }
+        end
+
+      end
+
+    end
+
+  end
+
+end