hybrid_platforms_conductor 32.16.3 → 33.0.0

data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_deploy_and_idempotence.rb

@@ -54,7 +54,7 @@ module HybridPlatformsConductor
                 instance.stop
                 instance.with_running_instance(port: 22) do
 
-                  unless @nodes_handler.get_root_access_allowed_of(@node) == 'true'
+                  unless @nodes_handler.get_root_access_allowed_of(@node)
                     # ===== Deploy removes root access
                     # Check that we can't connect with root
                     ssh_ok = false

data/lib/hybrid_platforms_conductor/hpc_plugins/test/connection.rb

@@ -1,3 +1,5 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
+
 module HybridPlatformsConductor
 
   module HpcPlugins
@@ -5,7 +7,7 @@ module HybridPlatformsConductor
     module Test
 
       # Test that the connection works by simply outputing something
-      class Connection < HybridPlatformsConductor::Test
+      class Connection < TestOnlyRemoteNode
 
         TEST_CONNECTION_STRING = 'Test connection - ok'
 

data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_freshness.rb

@@ -12,27 +12,14 @@ module HybridPlatformsConductor
         MAX_ACCEPTABLE_REFRESH_PERIOD_SECS = 3 * 31 * 24 * 60 * 60 # 3 months
 
         # Check my_test_plugin.rb.sample documentation for signature details.
-        def test_on_node
+        def test_for_node
           now = Time.now
-          {
-            "#{@nodes_handler.sudo_on(@node)} ls -t /var/log/deployments" => proc do |stdout|
-              if stdout.empty?
-                error 'Node has never been deployed using deploy (/var/log/deployments is empty)'
-              elsif stdout.first =~ /No such file or directory/
-                error 'Node has never been deployed using deploy (/var/log/deployments does not exist)'
-              else
-                # Expecting following file names
-                # node-name_2017-12-01_093418_user-name
-                file_match = stdout.first.match(/^#{Regexp.escape(@node)}_(\d{4}-\d{2}-\d{2})_.+$/)
-                if file_match.nil?
-                  error "Invalid chef deployment log file found: #{stdout.first}"
-                else
-                  last_deploy_time = Time.parse(file_match[1])
-                  error "Last deployment has been done on #{last_deploy_time.strftime('%F')}. Should refresh it." if now - last_deploy_time > MAX_ACCEPTABLE_REFRESH_PERIOD_SECS
-                end
-              end
-            end
-          }
+          deploy_info = @deployer.deployment_info_from(@node)[@node]
+          if deploy_info.key?(:error)
+            error "Error while getting deployment info: #{deploy_info[:error]}"
+          elsif Time.now.utc - deploy_info[:deployment_info][:date] > MAX_ACCEPTABLE_REFRESH_PERIOD_SECS
+            error "Last deployment has been done on #{deploy_info[:deployment_info][:date].strftime('%F')}. Should refresh it."
+          end
         end
 
       end

data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_removes_root_access.rb

@@ -12,7 +12,7 @@ module HybridPlatformsConductor
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_for_node
-          unless @nodes_handler.get_root_access_allowed_of(@node) == 'true'
+          unless @nodes_handler.get_root_access_allowed_of(@node)
             @deployer.with_test_provisioned_instance(@config.tests_provisioner_id, @node, environment: 'deploy_removes_root_access', reuse_instance: log_debug?) do |deployer, instance|
               # Check that we can connect with root
               ssh_ok = false

data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb

@@ -1,3 +1,4 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
 require 'hybrid_platforms_conductor/common_config_dsl/file_system_tests'
 
 module HybridPlatformsConductor
@@ -7,7 +8,7 @@ module HybridPlatformsConductor
     module Test
 
       # Perform various tests on a node's file system
-      class FileSystem < HybridPlatformsConductor::Test
+      class FileSystem < TestOnlyRemoteNode
 
         self.extend_config_dsl_with CommonConfigDsl::FileSystemTests, :init_file_system_tests
 

data/lib/hybrid_platforms_conductor/hpc_plugins/test/github_ci.rb

@@ -0,0 +1,32 @@
+require 'hybrid_platforms_conductor/common_config_dsl/github'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Test
+
+      # Check that all repositories have a successful Github CI
+      class GithubCi < HybridPlatformsConductor::Test
+
+        self.extend_config_dsl_with CommonConfigDsl::Github, :init_github
+
+        # Check my_test_plugin.rb.sample documentation for signature details.
+        def test
+          @config.for_each_github_repo do |client, repo_info|
+            log_debug "Checking CI for Github repository #{repo_info[:slug]}"
+            last_status = client.repository_workflow_runs(repo_info[:slug])[:workflow_runs].
+              select { |run| run[:head_branch] == 'master' }.
+              sort_by { |run| run[:created_at] }.
+              last[:conclusion]
+            error "Last workflow status for repository #{repo_info[:slug]} is #{last_status}" unless last_status == 'success'
+          end
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/test/hostname.rb

@@ -1,3 +1,5 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
+
 module HybridPlatformsConductor
 
   module HpcPlugins
@@ -5,7 +7,7 @@ module HybridPlatformsConductor
     module Test
 
       # Test that the hostname is correct
-      class Hostname < HybridPlatformsConductor::Test
+      class Hostname < TestOnlyRemoteNode
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_node

data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb

@@ -1,3 +1,5 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
+
 module HybridPlatformsConductor
 
   module HpcPlugins
@@ -5,7 +7,7 @@ module HybridPlatformsConductor
     module Test
 
       # Test that the private IP address is correct
-      class Ip < HybridPlatformsConductor::Test
+      class Ip < TestOnlyRemoteNode
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_node

data/lib/hybrid_platforms_conductor/hpc_plugins/test/local_users.rb

@@ -1,3 +1,5 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
+
 module HybridPlatformsConductor
 
   module HpcPlugins
@@ -5,7 +7,7 @@ module HybridPlatformsConductor
     module Test
 
       # Test that the node's local users
-      class LocalUsers < HybridPlatformsConductor::Test
+      class LocalUsers < TestOnlyRemoteNode
 
         # Config DSL extension for this test plugin
         module ConfigDslExtension

data/lib/hybrid_platforms_conductor/hpc_plugins/test/mounts.rb

@@ -1,3 +1,5 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
+
 module HybridPlatformsConductor
 
   module HpcPlugins
@@ -5,7 +7,7 @@ module HybridPlatformsConductor
     module Test
 
       # Various tests on mounts
-      class Mounts < HybridPlatformsConductor::Test
+      class Mounts < TestOnlyRemoteNode
 
         # Config DSL extension for this test plugin
         module ConfigDslExtension

data/lib/hybrid_platforms_conductor/hpc_plugins/test/orphan_files.rb

@@ -1,3 +1,5 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
+
 module HybridPlatformsConductor
 
   module HpcPlugins
@@ -5,7 +7,7 @@ module HybridPlatformsConductor
     module Test
 
       # Test that the node has no orphan files
-      class OrphanFiles < HybridPlatformsConductor::Test
+      class OrphanFiles < TestOnlyRemoteNode
 
         # Config DSL extension for this test plugin
         module ConfigDslExtension

data/lib/hybrid_platforms_conductor/hpc_plugins/test/ports.rb

@@ -1,3 +1,5 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
+
 module HybridPlatformsConductor
 
   module HpcPlugins
@@ -5,7 +7,7 @@ module HybridPlatformsConductor
     module Test
 
       # Various tests on ports
-      class Ports < HybridPlatformsConductor::Test
+      class Ports < TestOnlyRemoteNode
 
         # Config DSL extension for this test plugin
         module ConfigDslExtension

data/lib/hybrid_platforms_conductor/hpc_plugins/test/spectre.rb

@@ -1,3 +1,5 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
+
 module HybridPlatformsConductor
 
   module HpcPlugins
@@ -5,7 +7,7 @@ module HybridPlatformsConductor
     module Test
 
       # Test that the vulnerabilities Spectre and Meltdown are patched
-      class Spectre < HybridPlatformsConductor::Test
+      class Spectre < TestOnlyRemoteNode
 
         VULNERABILITIES_TO_CHECK = {
           'CVE-2017-5753' => 'Spectre Variant 1',

data/lib/hybrid_platforms_conductor/hpc_plugins/test/vulnerabilities.rb

@@ -1,4 +1,5 @@
 require 'nokogiri'
+require 'hybrid_platforms_conductor/test_only_remote_node'
 
 module HybridPlatformsConductor
 
@@ -17,7 +18,7 @@ module HybridPlatformsConductor
       #     Each final OVAL URL can be directly an XML file, either raw or compressed with .gz or .bz2.
       #     This is useful to follow repository links, such as jFrog or web servers serving common file systems structure storing several versions of the OVAL file.
       # * *reported_severities* (Array<String> or nil): List of severities to report, if any (use Unknown when the severity is not known), or nil for all [default: nil]
-      class Vulnerabilities < HybridPlatformsConductor::Test
+      class Vulnerabilities < TestOnlyRemoteNode
 
         # Known compression methods, per file extension, and their corresponding uncompress bash script
         KNOWN_COMPRESSIONS = {

data/lib/hybrid_platforms_conductor/log.rb

@@ -0,0 +1,31 @@
+require 'hybrid_platforms_conductor/logger_helpers'
+require 'hybrid_platforms_conductor/plugin'
+
+module HybridPlatformsConductor
+
+  # Ancestor of all log plugins
+  class Log < Plugin
+
+    # Constructor
+    #
+    # Parameters::
+    # * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
+    # * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
+    # * *config* (Config): Config to be used. [default: Config.new]
+    # * *nodes_handler* (NodesHandler): Nodes handler to be used. [default: NodesHandler.new]
+    # * *actions_executor* (ActionsExecutor): Actions executor to be used. [default: ActionsExecutor.new]
+    def initialize(
+      logger: Logger.new(STDOUT),
+      logger_stderr: Logger.new(STDERR),
+      config: Config.new,
+      nodes_handler: NodesHandler.new,
+      actions_executor: ActionsExecutor.new
+    )
+      super(logger: logger, logger_stderr: logger_stderr, config: config)
+      @nodes_handler = nodes_handler
+      @actions_executor = actions_executor
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/plugins.rb

@@ -38,6 +38,7 @@ module HybridPlatformsConductor
       []
       each
       empty?
+      find
       key?
       keys
       select

data/lib/hybrid_platforms_conductor/secrets_reader.rb

@@ -0,0 +1,31 @@
+require 'hybrid_platforms_conductor/logger_helpers'
+require 'hybrid_platforms_conductor/plugin'
+
+module HybridPlatformsConductor
+
+  # Ancestor of all secrets reader plugins
+  class SecretsReader < Plugin
+
+    # Constructor
+    #
+    # Parameters::
+    # * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
+    # * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
+    # * *config* (Config): Config to be used. [default: Config.new]
+    # * *cmd_runner* (CmdRunner): CmdRunner to be used. [default: CmdRunner.new]
+    # * *nodes_handler* (NodesHandler): Nodes handler to be used. [default: NodesHandler.new]
+    def initialize(
+      logger: Logger.new(STDOUT),
+      logger_stderr: Logger.new(STDERR),
+      config: Config.new,
+      cmd_runner: CmdRunner.new,
+      nodes_handler: NodesHandler.new
+    )
+      super(logger: logger, logger_stderr: logger_stderr, config: config)
+      @cmd_runner = cmd_runner
+      @nodes_handler = nodes_handler
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/test_only_remote_node.rb

@@ -0,0 +1,18 @@
+module HybridPlatformsConductor
+
+  # Ancestor for all tests that should be run just on remote nodes
+  class TestOnlyRemoteNode < Test
+
+    # Limit the list of nodes for these tests.
+    #
+    # Result::
+    # * Array<String or Regex> or nil: List of nodes allowed for this test, or nil for all. Regular expressions matching node names can also be used.
+    def self.only_on_nodes
+      # Just 1 node per service and platform
+      Test.nodes_handler.prefetch_metadata_of Test.nodes_handler.known_nodes, :local_node
+      Test.nodes_handler.known_nodes.select { |node| !Test.nodes_handler.get_local_node_of(node) }
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/version.rb

@@ -1,5 +1,5 @@
 module HybridPlatformsConductor
 
-  VERSION = '32.16.3'
+  VERSION = '33.0.0'
 
 end

data/spec/hybrid_platforms_conductor_test.rb

@@ -4,7 +4,9 @@ require 'hybrid_platforms_conductor/config'
 require 'hybrid_platforms_conductor/platforms_handler'
 require 'hybrid_platforms_conductor/actions_executor'
 require 'hybrid_platforms_conductor/cmd_runner'
+require 'hybrid_platforms_conductor/credentials'
 require 'hybrid_platforms_conductor/deployer'
+require 'hybrid_platforms_conductor/log'
 require 'hybrid_platforms_conductor/nodes_handler'
 require 'hybrid_platforms_conductor/platform_handler'
 require 'hybrid_platforms_conductor/provisioner'
@@ -43,6 +45,8 @@ require 'hybrid_platforms_conductor_test/platform_handler_plugins/test2'
 require 'hybrid_platforms_conductor_test/report_plugin'
 require 'hybrid_platforms_conductor_test/test_action'
 require 'hybrid_platforms_conductor_test/test_connector'
+require 'hybrid_platforms_conductor_test/test_log_plugin'
+require 'hybrid_platforms_conductor_test/test_log_no_read_plugin'
 require 'hybrid_platforms_conductor_test/test_plugins/global'
 require 'hybrid_platforms_conductor_test/test_plugins/node'
 require 'hybrid_platforms_conductor_test/test_plugins/node_check'
@@ -50,6 +54,7 @@ require 'hybrid_platforms_conductor_test/test_plugins/node_ssh'
 require 'hybrid_platforms_conductor_test/test_plugins/platform'
 require 'hybrid_platforms_conductor_test/test_plugins/several_checks'
 require 'hybrid_platforms_conductor_test/test_provisioner'
+require 'hybrid_platforms_conductor_test/test_secrets_reader_plugin'
 require 'hybrid_platforms_conductor_test/tests_report_plugin'
 
 module HybridPlatformsConductorTest
@@ -128,16 +133,32 @@ module HybridPlatformsConductorTest
         HybridPlatformsConductorTest::TestPlugins::NodeCheck.only_on_platform_types = nil
         HybridPlatformsConductorTest::TestPlugins::NodeCheck.only_on_nodes = nil
         HybridPlatformsConductorTest::TestPlugins::SeveralChecks.runs = []
+        HybridPlatformsConductorTest::TestLogPlugin.calls = []
+        HybridPlatformsConductorTest::TestLogNoReadPlugin.calls = []
+        HybridPlatformsConductorTest::TestSecretsReaderPlugin.calls = []
+        HybridPlatformsConductorTest::TestSecretsReaderPlugin.deployer = nil
+        HybridPlatformsConductorTest::TestSecretsReaderPlugin.mocked_secrets = {}
         FileUtils.rm_rf './run_logs'
         FileUtils.rm_rf './testadmin.key.pub'
         FileUtils.rm_rf '/tmp/hpc_ssh'
+        register_plugins(
+          :log,
+          {
+            test_log: HybridPlatformsConductorTest::TestLogPlugin,
+            test_log_no_read: HybridPlatformsConductorTest::TestLogNoReadPlugin
+          },
+          replace: false
+        )
         # Make sure CMDB plugin classes loaded by test framework are not added automatically
-        register_plugins(:cmdb, {
-          config: HybridPlatformsConductor::HpcPlugins::Cmdb::Config,
-          host_ip: HybridPlatformsConductor::HpcPlugins::Cmdb::HostIp,
-          host_keys: HybridPlatformsConductor::HpcPlugins::Cmdb::HostKeys,
-          platform_handlers: HybridPlatformsConductor::HpcPlugins::Cmdb::PlatformHandlers
-        })
+        register_plugins(
+          :cmdb,
+          {
+            config: HybridPlatformsConductor::HpcPlugins::Cmdb::Config,
+            host_ip: HybridPlatformsConductor::HpcPlugins::Cmdb::HostIp,
+            host_keys: HybridPlatformsConductor::HpcPlugins::Cmdb::HostKeys,
+            platform_handlers: HybridPlatformsConductor::HpcPlugins::Cmdb::PlatformHandlers
+          }
+        )
       end
     end
 

data/spec/hybrid_platforms_conductor_test/api/actions_executor/connectors/ssh/connections_spec.rb

@@ -29,7 +29,7 @@ describe HybridPlatformsConductor::ActionsExecutor do
       end
 
       it 'creates an SSH master to 1 node not having Session Exec capabilities' do
-        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42', ssh_session_exec: 'false' } } }) do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42', ssh_session_exec: false } } }) do
           with_cmd_runner_mocked(
             [
               ['which env', proc { [0, "/usr/bin/env\n", ''] }],
@@ -45,7 +45,7 @@ describe HybridPlatformsConductor::ActionsExecutor do
       end
 
       it 'can\'t create an SSH master to 1 node not having Session Exec capabilities when hpc_interactive is false' do
-        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42', ssh_session_exec: 'false' } } }) do
+        with_test_platform(nodes: { 'node' => { meta: { host_ip: '192.168.42.42', ssh_session_exec: false } } }) do
           ENV['hpc_interactive'] = 'false'
           with_cmd_runner_mocked(
             [
@@ -69,7 +69,7 @@ describe HybridPlatformsConductor::ActionsExecutor do
       it 'fails without creating exception when creating an SSH master to 1 node not having Session Exec capabilities when hpc_interactive is false and we use no_exception' do
         with_test_platform(nodes: {
           'node1' => { meta: { host_ip: '192.168.42.1' } },
-          'node2' => { meta: { host_ip: '192.168.42.2', ssh_session_exec: 'false' } },
+          'node2' => { meta: { host_ip: '192.168.42.2', ssh_session_exec: false } },
           'node3' => { meta: { host_ip: '192.168.42.3' } }
         }) do
           ENV['hpc_interactive'] = 'false'

data/spec/hybrid_platforms_conductor_test/api/deployer/config_dsl_spec.rb

@@ -3,10 +3,52 @@ describe HybridPlatformsConductor::Deployer do
   context 'checking deployer specific config DSL' do
 
     it 'declares a packaging timeout' do
-      with_repository do |repository|
-        with_platforms('packaging_timeout 666') do
-          expect(test_config.packaging_timeout_secs).to eq 666
-        end
+      with_platforms('packaging_timeout 666') do
+        expect(test_config.packaging_timeout_secs).to eq 666
+      end
+    end
+
+    it 'declares log plugins to be used' do
+      with_test_platforms(
+        { nodes: { 'node1' => {}, 'node2' => {} } },
+        false,
+        <<~EOS
+          send_logs_to %i[log_plugin_1 log_plugin_2]
+          for_nodes('node2') { send_logs_to :log_plugin_3 }
+        EOS
+      ) do
+        expect(test_config.deployment_logs).to eq [
+          {
+            nodes_selectors_stack: [],
+            log_plugins: %i[log_plugin_1 log_plugin_2]
+          },
+          {
+            nodes_selectors_stack: %w[node2],
+            log_plugins: %i[log_plugin_3]
+          }
+        ]
+      end
+    end
+
+    it 'declares secrets readers plugins to be used' do
+      with_test_platforms(
+        { nodes: { 'node1' => {}, 'node2' => {} } },
+        false,
+        <<~EOS
+          read_secrets_from %i[secrets_reader_plugin_1 secrets_reader_plugin_2]
+          for_nodes('node2') { read_secrets_from :secrets_reader_plugin_3 }
+        EOS
+      ) do
+        expect(test_config.secrets_readers).to eq [
+          {
+            nodes_selectors_stack: [],
+            secrets_readers: %i[secrets_reader_plugin_1 secrets_reader_plugin_2]
+          },
+          {
+            nodes_selectors_stack: %w[node2],
+            secrets_readers: %i[secrets_reader_plugin_3]
+          }
+        ]
       end
     end