hybrid_platforms_conductor 32.16.2 → 32.18.0

data/lib/hybrid_platforms_conductor/hpc_plugins/log/remote_fs.rb

@@ -0,0 +1,179 @@
+require 'hybrid_platforms_conductor/log'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Log
+
+      # Save logs on the remote node's file system
+      class RemoteFs < HybridPlatformsConductor::Log
+
+        MARKER_STDOUT = '===== STDOUT ====='
+        MARKER_STDERR = '===== STDERR ====='
+
+        # Get actions to save logs
+        # [API] - This method is mandatory.
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        # * *@actions_executor* (ActionsExecutor): Actions executor API.
+        #
+        # Parameters::
+        # * *node* (String): Node for which logs are being saved
+        # * *services* (Array<String>): The list of services that have been deployed on this node
+        # * *deployment_info* (Hash<Symbol,Object>): Additional information to attach to the logs
+        # * *exit_status* (Integer or Symbol): Exit status of the deployment
+        # * *stdout* (String): Deployment's stdout
+        # * *stderr* (String): Deployment's stderr
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of actions to be done
+        def actions_to_save_logs(node, services, deployment_info, exit_status, stdout, stderr)
+          # Create a log file to be scp with all relevant info
+          ssh_user = @actions_executor.connector(:ssh).ssh_user
+          sudo_prefix = ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} "
+          log_file = "#{Dir.tmpdir}/hpc_deploy_logs/#{node}_#{Time.now.utc.strftime('%F_%H%M%S')}_#{ssh_user}"
+          [
+            {
+              ruby: proc do
+                FileUtils.mkdir_p File.dirname(log_file)
+                File.write(log_file, <<~EOS)
+                  #{
+                    deployment_info.merge(
+                      debug: log_debug? ? 'Yes' : 'No',
+                      services: services.join(', '),
+                      exit_status: exit_status
+                    ).map { |property, value| "#{property}: #{value}" }.join("\n")
+                  }
+                  #{MARKER_STDOUT}
+                  #{stdout}
+                  #{MARKER_STDERR}
+                  #{stderr}
+                EOS
+              end,
+              remote_bash: "#{sudo_prefix}mkdir -p /var/log/deployments && #{sudo_prefix}chmod 600 /var/log/deployments"
+            },
+            {
+              scp: {
+                log_file => '/var/log/deployments',
+                :sudo => ssh_user != 'root',
+                :owner => 'root',
+                :group => 'root'
+              }
+            },
+            {
+              remote_bash: "#{sudo_prefix}chmod 600 /var/log/deployments/#{File.basename(log_file)}",
+              # Remove temporary files storing logs for security
+              ruby: proc do
+                File.unlink(log_file)
+              end
+            }
+          ]
+        end
+
+        # Get actions to read logs.
+        # If provided, this method can return some actions to be executed that will fetch logs from servers or remote nodes.
+        # By using this method to run actions instead of the synchronous method logs_from, such actions will be run in parallel which can greatly improve time-consuming operations when querying a lot of nodes.
+        # [API] - This method is optional.
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        # * *@actions_executor* (ActionsExecutor): Actions executor API.
+        #
+        # Parameters::
+        # * *node* (String): Node for which deployment logs are being read
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of actions to be done
+        def actions_to_read_logs(node)
+          sudo_prefix = @actions_executor.connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(node)} "
+          [
+            { remote_bash: "#{sudo_prefix}cat /var/log/deployments/`#{sudo_prefix}ls -t /var/log/deployments/ | head -1`" }
+          ]
+        end
+
+        # Get deployment logs from a node.
+        # This method can use the result of actions previously run to read logs, as returned by the actions_to_read_logs method.
+        # [API] - This method is mandatory.
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        # * *@actions_executor* (ActionsExecutor): Actions executor API.
+        #
+        # Parameters::
+        # * *node* (String): The node we want deployment logs from
+        # * *exit_status* (Integer, Symbol or nil): Exit status of actions to read logs, or nil if no action was returned by actions_to_read_logs
+        # * *stdout* (String or nil): stdout of actions to read logs, or nil if no action was returned by actions_to_read_logs
+        # * *stderr* (String or nil): stderr of actions to read logs, or nil if no action was returned by actions_to_read_logs
+        # Result::
+        # * Hash<Symbol,Object>: Deployment log information:
+        #   * *error* (String): Error string in case deployment logs could not be retrieved. If set then further properties will be ignored. [optional]
+        #   * *services* (Array<String>): List of services deployed on the node
+        #   * *deployment_info* (Hash<Symbol,Object>): Deployment metadata
+        #   * *exit_status* (Integer or Symbol): Deployment exit status
+        #   * *stdout* (String): Deployment stdout
+        #   * *stderr* (String): Deployment stderr
+        def logs_for(node, exit_status, stdout, stderr)
+          # Expected format for stdout:
+          # Property1: Value1
+          # ...
+          # PropertyN: ValueN
+          # ===== STDOUT =====
+          # ...
+          # ===== STDERR =====
+          # ...
+          if exit_status.is_a?(Symbol)
+            { error: "Error: #{exit_status}\n#{stderr}" }
+          else
+            stdout_lines = stdout.split("\n")
+            if stdout_lines.first =~ /No such file or directory/
+              { error: '/var/log/deployments missing' }
+            else
+              stdout_idx = stdout_lines.index(MARKER_STDOUT)
+              stderr_idx = stdout_lines.index(MARKER_STDERR)
+              deploy_info = {}
+              stdout_lines[0..stdout_idx - 1].each do |line|
+                if line =~ /^([^:]+): (.+)$/
+                  key_str, value = $1, $2
+                  key = key_str.to_sym
+                  # Type-cast some values
+                  case key_str
+                  when 'date'
+                    # Date and time values
+                    # Thu Nov 23 18:43:01 UTC 2017
+                    deploy_info[key] = Time.parse("#{value} UTC")
+                  when 'debug'
+                    # Boolean values
+                    # Yes
+                    deploy_info[key] = (value == 'Yes')
+                  when /^diff_files_.+$/, 'services'
+                    # Array of strings
+                    # my_file.txt, other_file.txt
+                    deploy_info[key] = value.split(', ')
+                  else
+                    deploy_info[key] = value
+                  end
+                else
+                  deploy_info[:unknown_lines] = [] unless deploy_info.key?(:unknown_lines)
+                  deploy_info[:unknown_lines] << line
+                end
+              end
+              services = deploy_info.delete(:services)
+              exit_status = deploy_info.delete(:exit_status)
+              {
+                services: services,
+                deployment_info: deploy_info,
+                exit_status: exit_status =~ /^\d+$/ ? Integer(exit_status) : exit_status.to_sym,
+                stdout: stdout_lines[stdout_idx + 1..stderr_idx - 1].join("\n"),
+                stderr: stdout_lines[stderr_idx + 1..-1].join("\n")
+              }
+            end
+          end
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/serverless_chef.rb

@@ -175,11 +175,11 @@ module HybridPlatformsConductor
                 end
                 lock_file = "#{File.dirname(policy_file)}/#{File.basename(policy_file, '.rb')}.lock.json"
                 # If the policy lock file does not exist, generate it
-                @cmd_runner.run_cmd "cd #{@repository_path} && /opt/chef-workstation/bin/chef install #{policy_file}" unless File.exist?("#{@repository_path}/#{lock_file}")
+                @cmd_runner.run_cmd "cd #{@repository_path} && /opt/chef-workstation/bin/chef install #{policy_file} --chef-license accept" unless File.exist?("#{@repository_path}/#{lock_file}")
                 extra_cp_data_bags = File.exist?("#{@repository_path}/data_bags") ? " && cp -ar data_bags/ #{package_dir}/" : ''
                 @cmd_runner.run_cmd "cd #{@repository_path} && \
                   #{@cmd_runner.root? ? '' : 'sudo '}rm -rf #{package_dir} && \
-                  /opt/chef-workstation/bin/chef export #{policy_file} #{package_dir}#{extra_cp_data_bags}"
+                  /opt/chef-workstation/bin/chef export #{policy_file} #{package_dir} --chef-license accept#{extra_cp_data_bags}"
               end
               unless @cmd_runner.dry_run
                 # Create secrets file

data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_deploy_and_idempotence.rb

@@ -54,7 +54,7 @@ module HybridPlatformsConductor
                 instance.stop
                 instance.with_running_instance(port: 22) do
 
-                  unless @nodes_handler.get_root_access_allowed_of(@node) == 'true'
+                  unless @nodes_handler.get_root_access_allowed_of(@node)
                     # ===== Deploy removes root access
                     # Check that we can't connect with root
                     ssh_ok = false

data/lib/hybrid_platforms_conductor/hpc_plugins/test/connection.rb

@@ -1,3 +1,5 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
+
 module HybridPlatformsConductor
 
   module HpcPlugins
@@ -5,7 +7,7 @@ module HybridPlatformsConductor
     module Test
 
       # Test that the connection works by simply outputing something
-      class Connection < HybridPlatformsConductor::Test
+      class Connection < TestOnlyRemoteNode
 
         TEST_CONNECTION_STRING = 'Test connection - ok'
 

data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_freshness.rb

@@ -12,27 +12,14 @@ module HybridPlatformsConductor
         MAX_ACCEPTABLE_REFRESH_PERIOD_SECS = 3 * 31 * 24 * 60 * 60 # 3 months
 
         # Check my_test_plugin.rb.sample documentation for signature details.
-        def test_on_node
+        def test_for_node
           now = Time.now
-          {
-            "#{@nodes_handler.sudo_on(@node)} ls -t /var/log/deployments" => proc do |stdout|
-              if stdout.empty?
-                error 'Node has never been deployed using deploy (/var/log/deployments is empty)'
-              elsif stdout.first =~ /No such file or directory/
-                error 'Node has never been deployed using deploy (/var/log/deployments does not exist)'
-              else
-                # Expecting following file names
-                # node-name_2017-12-01_093418_user-name
-                file_match = stdout.first.match(/^#{Regexp.escape(@node)}_(\d{4}-\d{2}-\d{2})_.+$/)
-                if file_match.nil?
-                  error "Invalid chef deployment log file found: #{stdout.first}"
-                else
-                  last_deploy_time = Time.parse(file_match[1])
-                  error "Last deployment has been done on #{last_deploy_time.strftime('%F')}. Should refresh it." if now - last_deploy_time > MAX_ACCEPTABLE_REFRESH_PERIOD_SECS
-                end
-              end
-            end
-          }
+          deploy_info = @deployer.deployment_info_from(@node)[@node]
+          if deploy_info.key?(:error)
+            error "Error while getting deployment info: #{deploy_info[:error]}"
+          elsif Time.now.utc - deploy_info[:deployment_info][:date] > MAX_ACCEPTABLE_REFRESH_PERIOD_SECS
+            error "Last deployment has been done on #{deploy_info[:deployment_info][:date].strftime('%F')}. Should refresh it."
+          end
         end
 
       end

data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_removes_root_access.rb

@@ -12,7 +12,7 @@ module HybridPlatformsConductor
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_for_node
-          unless @nodes_handler.get_root_access_allowed_of(@node) == 'true'
+          unless @nodes_handler.get_root_access_allowed_of(@node)
             @deployer.with_test_provisioned_instance(@config.tests_provisioner_id, @node, environment: 'deploy_removes_root_access', reuse_instance: log_debug?) do |deployer, instance|
               # Check that we can connect with root
               ssh_ok = false

data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb

@@ -1,3 +1,4 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
 require 'hybrid_platforms_conductor/common_config_dsl/file_system_tests'
 
 module HybridPlatformsConductor
@@ -7,7 +8,7 @@ module HybridPlatformsConductor
     module Test
 
       # Perform various tests on a node's file system
-      class FileSystem < HybridPlatformsConductor::Test
+      class FileSystem < TestOnlyRemoteNode
 
         self.extend_config_dsl_with CommonConfigDsl::FileSystemTests, :init_file_system_tests
 

data/lib/hybrid_platforms_conductor/hpc_plugins/test/github_ci.rb

@@ -0,0 +1,32 @@
+require 'hybrid_platforms_conductor/common_config_dsl/github'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Test
+
+      # Check that all repositories have a successful Github CI
+      class GithubCi < HybridPlatformsConductor::Test
+
+        self.extend_config_dsl_with CommonConfigDsl::Github, :init_github
+
+        # Check my_test_plugin.rb.sample documentation for signature details.
+        def test
+          @config.for_each_github_repo do |client, repo_info|
+            log_debug "Checking CI for Github repository #{repo_info[:slug]}"
+            last_status = client.repository_workflow_runs(repo_info[:slug])[:workflow_runs].
+              select { |run| run[:head_branch] == 'master' }.
+              sort_by { |run| run[:created_at] }.
+              last[:conclusion]
+            error "Last workflow status for repository #{repo_info[:slug]} is #{last_status}" unless last_status == 'success'
+          end
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/test/hostname.rb

@@ -1,3 +1,5 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
+
 module HybridPlatformsConductor
 
   module HpcPlugins
@@ -5,7 +7,7 @@ module HybridPlatformsConductor
     module Test
 
       # Test that the hostname is correct
-      class Hostname < HybridPlatformsConductor::Test
+      class Hostname < TestOnlyRemoteNode
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_node

data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb

@@ -1,3 +1,5 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
+
 module HybridPlatformsConductor
 
   module HpcPlugins
@@ -5,7 +7,7 @@ module HybridPlatformsConductor
     module Test
 
       # Test that the private IP address is correct
-      class Ip < HybridPlatformsConductor::Test
+      class Ip < TestOnlyRemoteNode
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_node

data/lib/hybrid_platforms_conductor/hpc_plugins/test/local_users.rb

@@ -1,3 +1,5 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
+
 module HybridPlatformsConductor
 
   module HpcPlugins
@@ -5,7 +7,7 @@ module HybridPlatformsConductor
     module Test
 
       # Test that the node's local users
-      class LocalUsers < HybridPlatformsConductor::Test
+      class LocalUsers < TestOnlyRemoteNode
 
         # Config DSL extension for this test plugin
         module ConfigDslExtension

data/lib/hybrid_platforms_conductor/hpc_plugins/test/mounts.rb

@@ -1,3 +1,5 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
+
 module HybridPlatformsConductor
 
   module HpcPlugins
@@ -5,7 +7,7 @@ module HybridPlatformsConductor
     module Test
 
       # Various tests on mounts
-      class Mounts < HybridPlatformsConductor::Test
+      class Mounts < TestOnlyRemoteNode
 
         # Config DSL extension for this test plugin
         module ConfigDslExtension

data/lib/hybrid_platforms_conductor/hpc_plugins/test/orphan_files.rb

@@ -1,3 +1,5 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
+
 module HybridPlatformsConductor
 
   module HpcPlugins
@@ -5,7 +7,7 @@ module HybridPlatformsConductor
     module Test
 
       # Test that the node has no orphan files
-      class OrphanFiles < HybridPlatformsConductor::Test
+      class OrphanFiles < TestOnlyRemoteNode
 
         # Config DSL extension for this test plugin
         module ConfigDslExtension

data/lib/hybrid_platforms_conductor/hpc_plugins/test/ports.rb

@@ -1,3 +1,5 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
+
 module HybridPlatformsConductor
 
   module HpcPlugins
@@ -5,7 +7,7 @@ module HybridPlatformsConductor
     module Test
 
       # Various tests on ports
-      class Ports < HybridPlatformsConductor::Test
+      class Ports < TestOnlyRemoteNode
 
         # Config DSL extension for this test plugin
         module ConfigDslExtension

data/lib/hybrid_platforms_conductor/hpc_plugins/test/spectre.rb

@@ -1,3 +1,5 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
+
 module HybridPlatformsConductor
 
   module HpcPlugins
@@ -5,7 +7,7 @@ module HybridPlatformsConductor
     module Test
 
       # Test that the vulnerabilities Spectre and Meltdown are patched
-      class Spectre < HybridPlatformsConductor::Test
+      class Spectre < TestOnlyRemoteNode
 
         VULNERABILITIES_TO_CHECK = {
           'CVE-2017-5753' => 'Spectre Variant 1',

data/lib/hybrid_platforms_conductor/hpc_plugins/test/vulnerabilities.rb

@@ -1,4 +1,5 @@
 require 'nokogiri'
+require 'hybrid_platforms_conductor/test_only_remote_node'
 
 module HybridPlatformsConductor
 
@@ -17,7 +18,7 @@ module HybridPlatformsConductor
       #     Each final OVAL URL can be directly an XML file, either raw or compressed with .gz or .bz2.
       #     This is useful to follow repository links, such as jFrog or web servers serving common file systems structure storing several versions of the OVAL file.
       # * *reported_severities* (Array<String> or nil): List of severities to report, if any (use Unknown when the severity is not known), or nil for all [default: nil]
-      class Vulnerabilities < HybridPlatformsConductor::Test
+      class Vulnerabilities < TestOnlyRemoteNode
 
         # Known compression methods, per file extension, and their corresponding uncompress bash script
         KNOWN_COMPRESSIONS = {

data/lib/hybrid_platforms_conductor/log.rb

@@ -0,0 +1,31 @@
+require 'hybrid_platforms_conductor/logger_helpers'
+require 'hybrid_platforms_conductor/plugin'
+
+module HybridPlatformsConductor
+
+  # Ancestor of all log plugins
+  class Log < Plugin
+
+    # Constructor
+    #
+    # Parameters::
+    # * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
+    # * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
+    # * *config* (Config): Config to be used. [default: Config.new]
+    # * *nodes_handler* (NodesHandler): Nodes handler to be used. [default: NodesHandler.new]
+    # * *actions_executor* (ActionsExecutor): Actions executor to be used. [default: ActionsExecutor.new]
+    def initialize(
+      logger: Logger.new(STDOUT),
+      logger_stderr: Logger.new(STDERR),
+      config: Config.new,
+      nodes_handler: NodesHandler.new,
+      actions_executor: ActionsExecutor.new
+    )
+      super(logger: logger, logger_stderr: logger_stderr, config: config)
+      @nodes_handler = nodes_handler
+      @actions_executor = actions_executor
+    end
+
+  end
+
+end