hybrid_platforms_conductor 32.17.0 → 33.0.2

data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/my_secrets_reader_plugin.rb.sample

@@ -0,0 +1,46 @@
+require 'hybrid_platforms_conductor/secrets_reader'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module SecretsReader
+
+      # Read secrets from a secrets source
+      class MySecretsReaderPlugin < HybridPlatformsConductor::SecretsReader
+
+        # Complete an option parser with options meant to control this secrets reader
+        # [API] - This method is optional
+        #
+        # Parameters::
+        # * *options_parser* (OptionParser): The option parser to complete
+        def options_parse(options_parser)
+          @key_file = nil
+          options_parser.on('--key-file FILE', 'Key file decrypting a secret vault.') do |file|
+            @key_file = file
+          end
+        end
+
+        # Return secrets for a given service to be deployed on a node.
+        # [API] - This method is mandatory
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@cmd_runner* (CmdRunner): Command Runner API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        #
+        # Parameters::
+        # * *node* (String): Node to be deployed
+        # * *service* (String): Service to be deployed
+        # Result::
+        # * Hash: The secrets
+        def secrets_for(node, service)
+          JSON.parse(Vault.decrypt("/path/to/#{node}_#{service}.vault", key: @key_file))
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/secrets_reader/thycotic.rb

@@ -0,0 +1,90 @@
+require 'hybrid_platforms_conductor/secrets_reader'
+require 'hybrid_platforms_conductor/thycotic'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module SecretsReader
+
+      # Get secrets from a Thycotic secrets server
+      class Thycotic < HybridPlatformsConductor::SecretsReader
+
+        # Extend the Config DSL
+        module ConfigDSLExtension
+
+          # List of defined Thycotic secrets. Each info has the following properties:
+          # * *nodes_selectors_stack* (Array<Object>): Stack of nodes selectors impacted by this rule.
+          # * *thycotic_url* (String): Thycotic URL.
+          # * *secret_id* (Integer): Thycotic secret ID.
+          # Array< Hash<Symbol, Object> >
+          attr_reader :thycotic_secrets
+
+          # Mixin initializer
+          def init_thycotic_config
+            @thycotic_secrets = []
+          end
+
+          # Set a Thycotic secret server configuration
+          #
+          # Parameters::
+          # * *thycotic_url* (String): The Thycotic server URL.
+          # * *secret_id* (Integer): The Thycotic secret ID containing the secrets file to be used as secrets.
+          def secrets_from_thycotic(thycotic_url:, secret_id:)
+            @thycotic_secrets << {
+              nodes_selectors_stack: current_nodes_selectors_stack,
+              thycotic_url: thycotic_url,
+              secret_id: secret_id
+            }
+          end
+
+        end
+
+        Config.extend_config_dsl_with ConfigDSLExtension, :init_thycotic_config
+
+        # Return secrets for a given service to be deployed on a node.
+        # [API] - This method is mandatory
+        # [API] - The following API components are accessible:
+        # * *@config* (Config): Main configuration API.
+        # * *@cmd_runner* (CmdRunner): Command Runner API.
+        # * *@nodes_handler* (NodesHandler): Nodes handler API.
+        #
+        # Parameters::
+        # * *node* (String): Node to be deployed
+        # * *service* (String): Service to be deployed
+        # Result::
+        # * Hash: The secrets
+        def secrets_for(node, _service)
+          secrets = {}
+          # As we are dealing with global secrets, cache the reading for performance between nodes and services.
+          # Keep secrets cache grouped by URL/ID
+          @secrets = {} unless defined?(@secrets)
+          @nodes_handler.select_confs_for_node(node, @config.thycotic_secrets).each do |thycotic_secrets_info|
+            server_id = "#{thycotic_secrets_info[:thycotic_url]}:#{thycotic_secrets_info[:secret_id]}"
+            unless @secrets.key?(server_id)
+              HybridPlatformsConductor::Thycotic.with_thycotic(thycotic_secrets_info[:thycotic_url], @logger, @logger_stderr) do |thycotic|
+                secret_file_item_id = thycotic.get_secret(thycotic_secrets_info[:secret_id]).dig(:secret, :items, :secret_item, :id)
+                raise "Unable to fetch secret file ID #{thycotic_secrets_info[:secret_id]} from #{thycotic_secrets_info[:thycotic_url]}" if secret_file_item_id.nil?
+
+                secret = thycotic.download_file_attachment_by_item_id(thycotic_secrets_info[:secret_id], secret_file_item_id)
+                raise "Unable to fetch secret file attachment from secret ID #{thycotic_secrets_info[:secret_id]} from #{thycotic_secrets_info[:thycotic_url]}" if secret.nil?
+
+                @secrets[server_id] = JSON.parse(secret)
+              end
+            end
+            secrets.merge!(@secrets[server_id]) do |key, value_1, value_2|
+              raise "Thycotic secret #{key} served by #{thycotic_secrets_info[:thycotic_url]} from secret ID #{thycotic_secrets_info[:secret_id]} has conflicting values between different secrets." if value_1 != value_2
+
+              value_1
+            end
+          end
+          secrets
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/test/bitbucket_conf.rb

@@ -10,7 +10,7 @@ module HybridPlatformsConductor
       # Check that all repositories in Bitbucket have a consistent dev workflow.
       class BitbucketConf < HybridPlatformsConductor::Test
 
-        self.extend_config_dsl_with CommonConfigDsl::Bitbucket, :init_bitbucket
+        extend_config_dsl_with CommonConfigDsl::Bitbucket, :init_bitbucket
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test
@@ -107,12 +107,12 @@ module HybridPlatformsConductor
             mandatory_default_reviewers = repo_info.dig(*%i[checks pr_settings mandatory_default_reviewers])
             if mandatory_default_reviewers
               reviewers_found = default_reviewers.any? do |condition_info|
-                reviewers = condition_info.dig('reviewers')
+                reviewers = condition_info['reviewers']
                 condition_info.dig('sourceRefMatcher', 'id') == 'ANY_REF_MATCHER_ID' &&
                   condition_info.dig('targetRefMatcher', 'id') == 'ANY_REF_MATCHER_ID' &&
                   !reviewers.nil? &&
                   (mandatory_default_reviewers - reviewers.map { |reviewer_info| reviewer_info['name'] }).empty? &&
-                  (required_approvers.nil? || condition_info.dig('requiredApprovals') == required_approvers)
+                  (required_approvers.nil? || condition_info['requiredApprovals'] == required_approvers)
               end
               assert_equal(
                 reviewers_found,

data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_deploy_and_idempotence.rb

@@ -18,7 +18,7 @@ module HybridPlatformsConductor
       # Don't forget to add the testadmin private key in your SSH agent if you run this test locally.
       class CheckDeployAndIdempotence < TestByService
 
-        self.extend_config_dsl_with CommonConfigDsl::IdempotenceTests, :init_idempotence_tests
+        extend_config_dsl_with CommonConfigDsl::IdempotenceTests, :init_idempotence_tests
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_for_node
@@ -30,6 +30,7 @@ module HybridPlatformsConductor
                 ssh_ok = ssh.exec!('echo Works').strip == 'Works'
               end
             rescue
+              nil
             end
             assert_equal ssh_ok, true, 'Root does not have access from the empty image'
 
@@ -46,7 +47,7 @@ module HybridPlatformsConductor
               deployer.nbr_retries_on_error = 3
               exit_status, _stdout, _stderr = deployer.deploy_on(@node)[@node]
               assert_equal exit_status, 0, "Deploy from scratch returned error code #{exit_status}", log_debug? ? nil : deployer.stdouts_to_s
-              if exit_status == 0
+              if exit_status.zero?
                 # As it's possible sshd has to be restarted because of a change in its conf, restart the container.
                 # Otherwise you'll get the following error upon reconnection:
                 #   System is booting up. See pam_nologin(8)
@@ -63,6 +64,7 @@ module HybridPlatformsConductor
                         ssh_ok = ssh.exec!('echo Works').strip == 'Works'
                       end
                     rescue
+                      nil
                     end
                     assert_equal ssh_ok, false, 'Root can still connect on the image after deployment'
                     # Even if we can connect using root, run the idempotence test

data/lib/hybrid_platforms_conductor/hpc_plugins/test/connection.rb

@@ -1,3 +1,5 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
+
 module HybridPlatformsConductor
 
   module HpcPlugins
@@ -5,7 +7,7 @@ module HybridPlatformsConductor
     module Test
 
       # Test that the connection works by simply outputing something
-      class Connection < HybridPlatformsConductor::Test
+      class Connection < TestOnlyRemoteNode
 
         TEST_CONNECTION_STRING = 'Test connection - ok'
 

data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_freshness.rb

@@ -12,27 +12,13 @@ module HybridPlatformsConductor
         MAX_ACCEPTABLE_REFRESH_PERIOD_SECS = 3 * 31 * 24 * 60 * 60 # 3 months
 
         # Check my_test_plugin.rb.sample documentation for signature details.
-        def test_on_node
-          now = Time.now
-          {
-            "#{@nodes_handler.sudo_on(@node)} ls -t /var/log/deployments" => proc do |stdout|
-              if stdout.empty?
-                error 'Node has never been deployed using deploy (/var/log/deployments is empty)'
-              elsif stdout.first =~ /No such file or directory/
-                error 'Node has never been deployed using deploy (/var/log/deployments does not exist)'
-              else
-                # Expecting following file names
-                # node-name_2017-12-01_093418_user-name
-                file_match = stdout.first.match(/^#{Regexp.escape(@node)}_(\d{4}-\d{2}-\d{2})_.+$/)
-                if file_match.nil?
-                  error "Invalid chef deployment log file found: #{stdout.first}"
-                else
-                  last_deploy_time = Time.parse(file_match[1])
-                  error "Last deployment has been done on #{last_deploy_time.strftime('%F')}. Should refresh it." if now - last_deploy_time > MAX_ACCEPTABLE_REFRESH_PERIOD_SECS
-                end
-              end
-            end
-          }
+        def test_for_node
+          deploy_info = @deployer.deployment_info_from(@node)[@node]
+          if deploy_info.key?(:error)
+            error "Error while getting deployment info: #{deploy_info[:error]}"
+          elsif Time.now.utc - deploy_info[:deployment_info][:date] > MAX_ACCEPTABLE_REFRESH_PERIOD_SECS
+            error "Last deployment has been done on #{deploy_info[:deployment_info][:date].strftime('%F')}. Should refresh it."
+          end
         end
 
       end

data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_removes_root_access.rb

@@ -12,32 +12,34 @@ module HybridPlatformsConductor
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_for_node
-          unless @nodes_handler.get_root_access_allowed_of(@node)
-            @deployer.with_test_provisioned_instance(@config.tests_provisioner_id, @node, environment: 'deploy_removes_root_access', reuse_instance: log_debug?) do |deployer, instance|
-              # Check that we can connect with root
+          return if @nodes_handler.get_root_access_allowed_of(@node)
+
+          @deployer.with_test_provisioned_instance(@config.tests_provisioner_id, @node, environment: 'deploy_removes_root_access', reuse_instance: log_debug?) do |deployer, instance|
+            # Check that we can connect with root
+            ssh_ok = false
+            begin
+              Net::SSH.start(instance.ip, 'root', password: 'root_pwd', auth_methods: ['password'], verify_host_key: :never) do |ssh|
+                ssh_ok = ssh.exec!('echo Works').strip == 'Works'
+              end
+            rescue
+              nil
+            end
+            assert_equal ssh_ok, true, 'Root does not have access from the empty image'
+            if ssh_ok
+              deployer.nbr_retries_on_error = 3
+              deployer.deploy_on @node
+              # As sshd is certainly being restarted, start and stop the container to reload it.
+              deployer.restart @node
+              # Check that we can't connect with root
               ssh_ok = false
               begin
                 Net::SSH.start(instance.ip, 'root', password: 'root_pwd', auth_methods: ['password'], verify_host_key: :never) do |ssh|
                   ssh_ok = ssh.exec!('echo Works').strip == 'Works'
                 end
               rescue
+                nil
               end
-              assert_equal ssh_ok, true, 'Root does not have access from the empty image'
-              if ssh_ok
-                deployer.nbr_retries_on_error = 3
-                deployer.deploy_on @node
-                # As sshd is certainly being restarted, start and stop the container to reload it.
-                deployer.restart @node
-                # Check that we can't connect with root
-                ssh_ok = false
-                begin
-                  Net::SSH.start(instance.ip, 'root', password: 'root_pwd', auth_methods: ['password'], verify_host_key: :never) do |ssh|
-                    ssh_ok = ssh.exec!('echo Works').strip == 'Works'
-                  end
-                rescue
-                end
-                assert_equal ssh_ok, false, 'Root can still connect on the image after deployment'
-              end
+              assert_equal ssh_ok, false, 'Root can still connect on the image after deployment'
             end
           end
         end

data/lib/hybrid_platforms_conductor/hpc_plugins/test/divergence.rb

@@ -10,10 +10,10 @@ module HybridPlatformsConductor
       # Test that the node has not diverged since last deployment
       class Divergence < HybridPlatformsConductor::Test
 
-        self.extend_config_dsl_with CommonConfigDsl::IdempotenceTests, :init_idempotence_tests
+        extend_config_dsl_with CommonConfigDsl::IdempotenceTests, :init_idempotence_tests
 
         # Check my_test_plugin.rb.sample documentation for signature details.
-        def test_on_check_node(stdout, stderr, exit_status)
+        def test_on_check_node(stdout, stderr, _exit_status)
           # Check that the output of the check-node returns no changes.
           ignored_tasks = @nodes_handler.select_confs_for_node(@node, @config.ignored_divergent_tasks).inject({}) do |merged_ignored_tasks, conf|
             merged_ignored_tasks.merge(conf[:ignored_tasks])

data/lib/hybrid_platforms_conductor/hpc_plugins/test/executables.rb

@@ -27,7 +27,7 @@ module HybridPlatformsConductor
           ).known_platforms.first
           unless example_platform.nil?
             tests.concat [
-              "#{CmdRunner.executables_prefix}get_impacted_nodes --platform #{example_platform.name} --show-commands",
+              "#{CmdRunner.executables_prefix}get_impacted_nodes --platform #{example_platform.name} --show-commands"
             ]
             example_node = example_platform.known_nodes.first
             unless example_node.nil?
@@ -42,7 +42,7 @@ module HybridPlatformsConductor
               ]
             end
           end
-        	tests.sort.each do |cmd|
+          tests.sort.each do |cmd|
             log_debug "Testing #{cmd}"
             exit_status, stdout, _stderr = @cmd_runner.run_cmd "#{cmd} 2>&1", no_exception: true, log_to_stdout: log_debug?
             assert_equal(exit_status, 0, "Command #{cmd} returned code #{exit_status}:\n#{stdout}")

data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system.rb

@@ -1,3 +1,4 @@
+require 'hybrid_platforms_conductor/test_only_remote_node'
 require 'hybrid_platforms_conductor/common_config_dsl/file_system_tests'
 
 module HybridPlatformsConductor
@@ -7,33 +8,31 @@ module HybridPlatformsConductor
     module Test
 
       # Perform various tests on a node's file system
-      class FileSystem < HybridPlatformsConductor::Test
+      class FileSystem < TestOnlyRemoteNode
 
-        self.extend_config_dsl_with CommonConfigDsl::FileSystemTests, :init_file_system_tests
+        extend_config_dsl_with CommonConfigDsl::FileSystemTests, :init_file_system_tests
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_node
           # Flatten the paths rules so that we can spot inconsistencies in configuration
-          Hash[
-            @config.aggregate_files_rules(@nodes_handler, @node).map do |path, rule_info|
-              [
-                "if #{@nodes_handler.sudo_on(@node)} /bin/bash -c '[[ -d \"#{path}\" ]]' ; then echo 1 ; else echo 0 ; fi",
-                {
-                  validator: proc do |stdout, stderr|
-                    case stdout.last
-                    when '1'
-                      error "Path found that should be absent: #{path}" if rule_info[:state] == :absent
-                    when '0'
-                      error "Path not found that should be present: #{path}" if rule_info[:state] == :present
-                    else
-                      error "Could not check for existence of path #{path}", "----- STDOUT:\n#{stdout.join("\n")}----- STDERR:\n#{stderr.join("\n")}"
-                    end
-                  end,
-                  timeout: 2
-                }
-              ]
-            end
-          ]
+          @config.aggregate_files_rules(@nodes_handler, @node).map do |path, rule_info|
+            [
+              "if #{@nodes_handler.sudo_on(@node)} /bin/bash -c '[[ -d \"#{path}\" ]]' ; then echo 1 ; else echo 0 ; fi",
+              {
+                validator: proc do |stdout, stderr|
+                  case stdout.last
+                  when '1'
+                    error "Path found that should be absent: #{path}" if rule_info[:state] == :absent
+                  when '0'
+                    error "Path not found that should be present: #{path}" if rule_info[:state] == :present
+                  else
+                    error "Could not check for existence of path #{path}", "----- STDOUT:\n#{stdout.join("\n")}----- STDERR:\n#{stderr.join("\n")}"
+                  end
+                end,
+                timeout: 2
+              }
+            ]
+          end.to_h
         end
 
       end

data/lib/hybrid_platforms_conductor/hpc_plugins/test/file_system_hdfs.rb

@@ -9,31 +9,29 @@ module HybridPlatformsConductor
       # Perform various tests on a HDFS's file system
       class FileSystemHdfs < HybridPlatformsConductor::Test
 
-        self.extend_config_dsl_with CommonConfigDsl::FileSystemTests, :init_file_system_tests
+        extend_config_dsl_with CommonConfigDsl::FileSystemTests, :init_file_system_tests
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_node
           # Flatten the paths rules so that we can spot inconsistencies in configuration
-          Hash[
-            @config.aggregate_files_rules(@nodes_handler, @node, file_system_type: :hdfs).map do |path, rule_info|
-              [
-                "if sudo#{rule_info[:context][:sudo_user] ? " -u #{rule_info[:context][:sudo_user]}" : ''} hdfs dfs -ls \"#{path}\" ; then echo 1 ; else echo 0 ; fi",
-                {
-                  validator: proc do |stdout, stderr|
-                    case stdout.last
-                    when '1'
-                      error "HDFS path found that should be absent: #{path}" if rule_info[:state] == :absent
-                    when '0'
-                      error "HDFS path not found that should be present: #{path}" if rule_info[:state] == :present
-                    else
-                      error "Could not check for existence of HDFS path #{path}", "----- STDOUT:\n#{stdout.join("\n")}----- STDERR:\n#{stderr.join("\n")}"
-                    end
-                  end,
-                  timeout: 5
-                }
-              ]
-            end
-          ]
+          @config.aggregate_files_rules(@nodes_handler, @node, file_system_type: :hdfs).map do |path, rule_info|
+            [
+              "if sudo#{rule_info[:context][:sudo_user] ? " -u #{rule_info[:context][:sudo_user]}" : ''} hdfs dfs -ls \"#{path}\" ; then echo 1 ; else echo 0 ; fi",
+              {
+                validator: proc do |stdout, stderr|
+                  case stdout.last
+                  when '1'
+                    error "HDFS path found that should be absent: #{path}" if rule_info[:state] == :absent
+                  when '0'
+                    error "HDFS path not found that should be present: #{path}" if rule_info[:state] == :present
+                  else
+                    error "Could not check for existence of HDFS path #{path}", "----- STDOUT:\n#{stdout.join("\n")}----- STDERR:\n#{stderr.join("\n")}"
+                  end
+                end,
+                timeout: 5
+              }
+            ]
+          end.to_h
         end
 
       end

data/lib/hybrid_platforms_conductor/hpc_plugins/test/github_ci.rb

@@ -9,7 +9,7 @@ module HybridPlatformsConductor
       # Check that all repositories have a successful Github CI
       class GithubCi < HybridPlatformsConductor::Test
 
-        self.extend_config_dsl_with CommonConfigDsl::Github, :init_github
+        extend_config_dsl_with CommonConfigDsl::Github, :init_github
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test
@@ -17,8 +17,7 @@ module HybridPlatformsConductor
             log_debug "Checking CI for Github repository #{repo_info[:slug]}"
             last_status = client.repository_workflow_runs(repo_info[:slug])[:workflow_runs].
               select { |run| run[:head_branch] == 'master' }.
-              sort_by { |run| run[:created_at] }.
-              last[:conclusion]
+              max_by { |run| run[:created_at] }[:conclusion]
             error "Last workflow status for repository #{repo_info[:slug]} is #{last_status}" unless last_status == 'success'
           end
         end