hybrid_platforms_conductor 32.16.3 → 33.0.0

data/lib/hybrid_platforms_conductor/hpc_plugins/connector/ssh.rb

@@ -238,7 +238,7 @@ module HybridPlatformsConductor
         # * *bash_cmds* (String): Bash commands to execute
         def remote_bash(bash_cmds)
           ssh_cmd =
-            if @nodes_handler.get_ssh_session_exec_of(@node) == 'false'
+            if @nodes_handler.get_ssh_session_exec_of(@node) == false
               # When ExecSession is disabled we need to use stdin directly
               "{ cat | #{ssh_exec} #{ssh_url} -T; } <<'HPC_EOF'\n#{bash_cmds}\nHPC_EOF"
             else
@@ -300,7 +300,7 @@ module HybridPlatformsConductor
         # * *owner* (String or nil): Owner to be used when copying the files, or nil for current one [default: nil]
         # * *group* (String or nil): Group to be used when copying the files, or nil for current one [default: nil]
         def remote_copy(from, to, sudo: false, owner: nil, group: nil)
-          if @nodes_handler.get_ssh_session_exec_of(@node) == 'false'
+          if @nodes_handler.get_ssh_session_exec_of(@node) == false
             # We don't have ExecSession, so don't use ssh, but scp instead.
             if sudo
               # We need to first copy the file in an accessible directory, and then sudo mv
@@ -513,7 +513,7 @@ module HybridPlatformsConductor
                     if current_users.empty?
                       log_debug "[ ControlMaster - #{ssh_url} ] - Creating SSH ControlMaster..."
                       exit_status = nil
-                      if @nodes_handler.get_ssh_session_exec_of(node) == 'false'
+                      if @nodes_handler.get_ssh_session_exec_of(node) == false
                         # Here we have to create a ControlMaster using an interactive session, as the SSH server prohibits ExecSession, and so command executions.
                         # We'll do that using another terminal spawned in the background.
                         if ENV['hpc_interactive'] == 'false'

data/lib/hybrid_platforms_conductor/hpc_plugins/log/my_log_plugin.rb.sample

@@ -0,0 +1,100 @@
+require 'hybrid_platforms_conductor/log'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Log
+
+      # Save logs on the remote node's file system
+      class RemoteFs < HybridPlatformsConductor::Log
+
+        # Get actions to save logs
+        # [API] - This method is mandatory.
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        # * *@actions_executor* (ActionsExecutor): Actions executor API.
+        #
+        # Parameters::
+        # * *node* (String): Node for which logs are being saved
+        # * *services* (Array<String>): The list of services that have been deployed on this node
+        # * *deployment_info* (Hash<Symbol,Object>): Additional information to attach to the logs
+        # * *exit_status* (Integer or Symbol): Exit status of the deployment
+        # * *stdout* (String): Deployment's stdout
+        # * *stderr* (String): Deployment's stderr
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of actions to be done
+        def actions_to_save_logs(node, services, deployment_info, exit_status, stdout, stderr)
+          # Return here all actions that are to be run to save those logs.
+          [
+            {
+              bash: <<~EOS
+                cat <<'EOLOGFILE' >>all_logs.txt
+                Deployment log on #{node}:
+                #{stdout}
+                EOLOGFILE
+              EOS
+            }
+          ]
+        end
+
+        # Get actions to read logs.
+        # If provided, this method can return some actions to be executed that will fetch logs from servers or remote nodes.
+        # By using this method to run actions instead of the synchronous method logs_from, such actions will be run in parallel which can greatly improve time-consuming operations when querying a lot of nodes.
+        # [API] - This method is optional.
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        # * *@actions_executor* (ActionsExecutor): Actions executor API.
+        #
+        # Parameters::
+        # * *node* (String): Node for which deployment logs are being read
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of actions to be done
+        def actions_to_read_logs(node)
+          [
+            { bash: 'cat all_logs.txt' }
+          ]
+        end
+
+        # Get deployment logs from a node.
+        # This method can use the result of actions previously run to read logs, as returned by the actions_to_read_logs method.
+        # [API] - This method is mandatory.
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        # * *@actions_executor* (ActionsExecutor): Actions executor API.
+        #
+        # Parameters::
+        # * *node* (String): The node we want deployment logs from
+        # * *exit_status* (Integer, Symbol or nil): Exit status of actions to read logs, or nil if no action was returned by actions_to_read_logs
+        # * *stdout* (String or nil): stdout of actions to read logs, or nil if no action was returned by actions_to_read_logs
+        # * *stderr* (String or nil): stderr of actions to read logs, or nil if no action was returned by actions_to_read_logs
+        # Result::
+        # * Hash<Symbol,Object>: Deployment log information:
+        #   * *error* (String): Error string in case deployment logs could not be retrieved. If set then further properties will be ignored. [optional]
+        #   * *services* (Array<String>): List of services deployed on the node
+        #   * *deployment_info* (Hash<Symbol,Object>): Deployment metadata
+        #   * *exit_status* (Integer or Symbol): Deployment exit status
+        #   * *stdout* (String): Deployment stdout
+        #   * *stderr* (String): Deployment stderr
+        def logs_for(node, exit_status, stdout, stderr)
+          # Here we should retrieve this info from somewhere.
+          # Based on this example, stdout is the value of the execution of the action 'cat all_logs.txt' as returned by actions_to_read_logs.
+          {
+            services: %w[unknown],
+            deployment_info: {},
+            exit_status: 0,
+            stdout: stdout,
+            stderr: ''
+          }
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/log/remote_fs.rb

@@ -0,0 +1,179 @@
+require 'hybrid_platforms_conductor/log'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Log
+
+      # Save logs on the remote node's file system
+      class RemoteFs < HybridPlatformsConductor::Log
+
+        MARKER_STDOUT = '===== STDOUT ====='
+        MARKER_STDERR = '===== STDERR ====='
+
+        # Get actions to save logs
+        # [API] - This method is mandatory.
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        # * *@actions_executor* (ActionsExecutor): Actions executor API.
+        #
+        # Parameters::
+        # * *node* (String): Node for which logs are being saved
+        # * *services* (Array<String>): The list of services that have been deployed on this node
+        # * *deployment_info* (Hash<Symbol,Object>): Additional information to attach to the logs
+        # * *exit_status* (Integer or Symbol): Exit status of the deployment
+        # * *stdout* (String): Deployment's stdout
+        # * *stderr* (String): Deployment's stderr
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of actions to be done
+        def actions_to_save_logs(node, services, deployment_info, exit_status, stdout, stderr)
+          # Create a log file to be scp with all relevant info
+          ssh_user = @actions_executor.connector(:ssh).ssh_user
+          sudo_prefix = ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} "
+          log_file = "#{Dir.tmpdir}/hpc_deploy_logs/#{node}_#{Time.now.utc.strftime('%F_%H%M%S')}_#{ssh_user}"
+          [
+            {
+              ruby: proc do
+                FileUtils.mkdir_p File.dirname(log_file)
+                File.write(log_file, <<~EOS)
+                  #{
+                    deployment_info.merge(
+                      debug: log_debug? ? 'Yes' : 'No',
+                      services: services.join(', '),
+                      exit_status: exit_status
+                    ).map { |property, value| "#{property}: #{value}" }.join("\n")
+                  }
+                  #{MARKER_STDOUT}
+                  #{stdout}
+                  #{MARKER_STDERR}
+                  #{stderr}
+                EOS
+              end,
+              remote_bash: "#{sudo_prefix}mkdir -p /var/log/deployments && #{sudo_prefix}chmod 600 /var/log/deployments"
+            },
+            {
+              scp: {
+                log_file => '/var/log/deployments',
+                :sudo => ssh_user != 'root',
+                :owner => 'root',
+                :group => 'root'
+              }
+            },
+            {
+              remote_bash: "#{sudo_prefix}chmod 600 /var/log/deployments/#{File.basename(log_file)}",
+              # Remove temporary files storing logs for security
+              ruby: proc do
+                File.unlink(log_file)
+              end
+            }
+          ]
+        end
+
+        # Get actions to read logs.
+        # If provided, this method can return some actions to be executed that will fetch logs from servers or remote nodes.
+        # By using this method to run actions instead of the synchronous method logs_from, such actions will be run in parallel which can greatly improve time-consuming operations when querying a lot of nodes.
+        # [API] - This method is optional.
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        # * *@actions_executor* (ActionsExecutor): Actions executor API.
+        #
+        # Parameters::
+        # * *node* (String): Node for which deployment logs are being read
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of actions to be done
+        def actions_to_read_logs(node)
+          sudo_prefix = @actions_executor.connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} "
+          [
+            { remote_bash: "#{sudo_prefix}cat /var/log/deployments/`#{sudo_prefix}ls -t /var/log/deployments/ | head -1`" }
+          ]
+        end
+
+        # Get deployment logs from a node.
+        # This method can use the result of actions previously run to read logs, as returned by the actions_to_read_logs method.
+        # [API] - This method is mandatory.
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        # * *@actions_executor* (ActionsExecutor): Actions executor API.
+        #
+        # Parameters::
+        # * *node* (String): The node we want deployment logs from
+        # * *exit_status* (Integer, Symbol or nil): Exit status of actions to read logs, or nil if no action was returned by actions_to_read_logs
+        # * *stdout* (String or nil): stdout of actions to read logs, or nil if no action was returned by actions_to_read_logs
+        # * *stderr* (String or nil): stderr of actions to read logs, or nil if no action was returned by actions_to_read_logs
+        # Result::
+        # * Hash<Symbol,Object>: Deployment log information:
+        #   * *error* (String): Error string in case deployment logs could not be retrieved. If set then further properties will be ignored. [optional]
+        #   * *services* (Array<String>): List of services deployed on the node
+        #   * *deployment_info* (Hash<Symbol,Object>): Deployment metadata
+        #   * *exit_status* (Integer or Symbol): Deployment exit status
+        #   * *stdout* (String): Deployment stdout
+        #   * *stderr* (String): Deployment stderr
+        def logs_for(node, exit_status, stdout, stderr)
+          # Expected format for stdout:
+          # Property1: Value1
+          # ...
+          # PropertyN: ValueN
+          # ===== STDOUT =====
+          # ...
+          # ===== STDERR =====
+          # ...
+          if exit_status.is_a?(Symbol)
+            { error: "Error: #{exit_status}\n#{stderr}" }
+          else
+            stdout_lines = stdout.split("\n")
+            if stdout_lines.first =~ /No such file or directory/
+              { error: '/var/log/deployments missing' }
+            else
+              stdout_idx = stdout_lines.index(MARKER_STDOUT)
+              stderr_idx = stdout_lines.index(MARKER_STDERR)
+              deploy_info = {}
+              stdout_lines[0..stdout_idx - 1].each do |line|
+                if line =~ /^([^:]+): (.+)$/
+                  key_str, value = $1, $2
+                  key = key_str.to_sym
+                  # Type-cast some values
+                  case key_str
+                  when 'date'
+                    # Date and time values
+                    # Thu Nov 23 18:43:01 UTC 2017
+                    deploy_info[key] = Time.parse("#{value} UTC")
+                  when 'debug'
+                    # Boolean values
+                    # Yes
+                    deploy_info[key] = (value == 'Yes')
+                  when /^diff_files_.+$/, 'services'
+                    # Array of strings
+                    # my_file.txt, other_file.txt
+                    deploy_info[key] = value.split(', ')
+                  else
+                    deploy_info[key] = value
+                  end
+                else
+                  deploy_info[:unknown_lines] = [] unless deploy_info.key?(:unknown_lines)
+                  deploy_info[:unknown_lines] << line
+                end
+              end
+              services = deploy_info.delete(:services)
+              exit_status = deploy_info.delete(:exit_status)
+              {
+                services: services,
+                deployment_info: deploy_info,
+                exit_status: exit_status =~ /^\d+$/ ? Integer(exit_status) : exit_status.to_sym,
+                stdout: stdout_lines[stdout_idx + 1..stderr_idx - 1].join("\n"),
+                stderr: stdout_lines[stderr_idx + 1..-1].join("\n")
+              }
+            end
+          end
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/cli.rb

@@ -0,0 +1,75 @@
+require 'hybrid_platforms_conductor/secrets_reader'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module SecretsReader
+
+      # Get secrets from the command-line
+      class Cli < HybridPlatformsConductor::SecretsReader
+
+        # Constructor
+        #
+        # Parameters::
+        # * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
+        # * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
+        # * *config* (Config): Config to be used. [default: Config.new]
+        # * *cmd_runner* (CmdRunner): CmdRunner to be used. [default: CmdRunner.new]
+        # * *nodes_handler* (NodesHandler): Nodes handler to be used. [default: NodesHandler.new]
+        def initialize(
+          logger: Logger.new(STDOUT),
+          logger_stderr: Logger.new(STDERR),
+          config: Config.new,
+          cmd_runner: CmdRunner.new,
+          nodes_handler: NodesHandler.new
+        )
+          super
+          @secrets_files = []
+        end
+
+        # Complete an option parser with options meant to control this secrets reader
+        # [API] - This method is optional
+        #
+        # Parameters::
+        # * *options_parser* (OptionParser): The option parser to complete
+        def options_parse(options_parser)
+          options_parser.on('-e', '--secrets JSON_FILE', 'Specify a secrets location from a local JSON file. Can be specified several times.') do |file|
+            @secrets_files << file
+          end
+        end
+
+        # Return secrets for a given service to be deployed on a node.
+        # [API] - This method is mandatory
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@cmd_runner* (CmdRunner): Command Runner API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        #
+        # Parameters::
+        # * *node* (String): Node to be deployed
+        # * *service* (String): Service to be deployed
+        # Result::
+        # * Hash: The secrets
+        def secrets_for(node, service)
+          # As we are dealing with global secrets, cache the reading for performance between nodes and services.
+          unless defined?(@secrets)
+            @secrets = {}
+            @secrets_files.each do |secrets_file|
+              raise "Missing secrets file: #{secrets_file}" unless File.exist?(secrets_file)
+              @secrets.merge!(JSON.parse(File.read(secrets_file))) do |key, value1, value2|
+                raise "Secret #{key} has conflicting values between different secret JSON files." if value1 != value2
+                value1
+              end
+            end
+          end
+          @secrets
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/my_secrets_reader_plugin.rb.sample

@@ -0,0 +1,46 @@
+require 'hybrid_platforms_conductor/secrets_reader'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module SecretsReader
+
+      # Read secrets from a secrets source
+      class MySecretsReaderPlugin < HybridPlatformsConductor::SecretsReader
+
+        # Complete an option parser with options meant to control this secrets reader
+        # [API] - This method is optional
+        #
+        # Parameters::
+        # * *options_parser* (OptionParser): The option parser to complete
+        def options_parse(options_parser)
+          @key_file = nil
+          options_parser.on('--key-file FILE', 'Key file decrypting a secret vault.') do |file|
+            @key_file = file
+          end
+        end
+
+        # Return secrets for a given service to be deployed on a node.
+        # [API] - This method is mandatory
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@cmd_runner* (CmdRunner): Command Runner API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        #
+        # Parameters::
+        # * *node* (String): Node to be deployed
+        # * *service* (String): Service to be deployed
+        # Result::
+        # * Hash: The secrets
+        def secrets_for(node, service)
+          JSON.parse(Vault.decrypt("/path/to/#{node}_#{service}.vault", key: @key_file))
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/thycotic.rb

@@ -0,0 +1,87 @@
+require 'hybrid_platforms_conductor/secrets_reader'
+require 'hybrid_platforms_conductor/thycotic'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module SecretsReader
+
+      # Get secrets from a Thycotic secrets server
+      class Thycotic < HybridPlatformsConductor::SecretsReader
+
+        # Extend the Config DSL
+        module ConfigDSLExtension
+
+          # List of defined Thycotic secrets. Each info has the following properties:
+          # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule.
+          # * *thycotic_url* (String): Thycotic URL.
+          # * *secret_id* (Integer): Thycotic secret ID.
+          # Array< Hash<Symbol, Object> >
+          attr_reader :thycotic_secrets
+
+          # Mixin initializer
+          def init_thycotic_config
+            @thycotic_secrets = []
+          end
+
+          # Set a Thycotic secret server configuration
+          #
+          # Parameters::
+          # * *thycotic_url* (String): The Thycotic server URL.
+          # * *secret_id* (Integer): The Thycotic secret ID containing the secrets file to be used as secrets.
+          def secrets_from_thycotic(thycotic_url:, secret_id:)
+            @thycotic_secrets << {
+              nodes_selectors_stack: current_nodes_selectors_stack,
+              thycotic_url: thycotic_url,
+              secret_id: secret_id
+            }
+          end
+
+        end
+
+        Config.extend_config_dsl_with ConfigDSLExtension, :init_thycotic_config
+
+        # Return secrets for a given service to be deployed on a node.
+        # [API] - This method is mandatory
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@cmd_runner* (CmdRunner): Command Runner API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        #
+        # Parameters::
+        # * *node* (String): Node to be deployed
+        # * *service* (String): Service to be deployed
+        # Result::
+        # * Hash: The secrets
+        def secrets_for(node, service)
+          secrets = {}
+          # As we are dealing with global secrets, cache the reading for performance between nodes and services.
+          # Keep secrets cache grouped by URL/ID
+          @secrets = {} unless defined?(@secrets)
+          @nodes_handler.select_confs_for_node(node, @config.thycotic_secrets).each do |thycotic_secrets_info|
+            server_id = "#{thycotic_secrets_info[:thycotic_url]}:#{thycotic_secrets_info[:secret_id]}"
+            unless @secrets.key?(server_id)
+              HybridPlatformsConductor::Thycotic.with_thycotic(thycotic_secrets_info[:thycotic_url], @logger, @logger_stderr) do |thycotic|
+                secret_file_item_id = thycotic.get_secret(thycotic_secrets_info[:secret_id]).dig(:secret, :items, :secret_item, :id)
+                raise "Unable to fetch secret file ID #{thycotic_secrets_info[:secret_id]} from #{thycotic_secrets_info[:thycotic_url]}" if secret_file_item_id.nil?
+                secret = thycotic.download_file_attachment_by_item_id(thycotic_secrets_info[:secret_id], secret_file_item_id)
+                raise "Unable to fetch secret file attachment from secret ID #{thycotic_secrets_info[:secret_id]} from #{thycotic_secrets_info[:thycotic_url]}" if secret.nil?
+                @secrets[server_id] = JSON.parse(secret)
+              end
+            end
+            secrets.merge!(@secrets[server_id]) do |key, value1, value2|
+              raise "Thycotic secret #{key} served by #{thycotic_secrets_info[:thycotic_url]} from secret ID #{thycotic_secrets_info[:secret_id]} has conflicting values between different secrets." if value1 != value2
+              value1
+            end
+          end
+          secrets
+        end
+
+      end
+
+    end
+
+  end
+
+end