hybrid_platforms_conductor 32.11.2 → 32.13.3

data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/platform_handler_plugin.rb.sample

@@ -69,7 +69,7 @@ module HybridPlatformsConductor
         end
 
         # Setup the platform, install dependencies...
-        # [API] - This method is mandatory.
+        # [API] - This method is optional.
         # [API] - @cmd_runner is accessible.
         def setup
           # This method is called by the setup executable.
@@ -163,7 +163,7 @@ module HybridPlatformsConductor
         end
 
         # Package the repository, ready to be deployed on artefacts or directly to a node.
-        # [API] - This method is mandatory.
+        # [API] - This method is optional.
         # [API] - @cmd_runner is accessible.
         # [API] - @actions_executor is accessible.
         #
@@ -212,7 +212,7 @@ module HybridPlatformsConductor
           # This method returns all the actions to execute to deploy on a node.
           # The use_why_run switch is on if the deployment should just be simulated.
           # Those actions (bash commands, scp of files, ruby code...) should be thread safe as they can be executed in parallel with other deployment actions for other nodes in case of a concurrent deployment on several nodes.
-          # The complete description of an action can be found in actions_executor.rb file, in the execute_actions_on method description.
+          # The complete description of an action can be found in the action plugins' documentation.
           [
             {
               scp: {
@@ -229,7 +229,7 @@ module HybridPlatformsConductor
         end
 
         # Prepare a why-run deployment so that a JSON file describing the nodes will be output in the run_logs.
-        # [API] - This method is mandatory.
+        # [API] - This method is optional.
         # [API] - @cmd_runner is accessible.
         # [API] - @actions_executor is accessible.
         # [API] - @deployer is accessible.
@@ -247,7 +247,7 @@ module HybridPlatformsConductor
         # Result::
         # * Array< Hash<Symbol,Object> >: List of task properties. The following properties should be returned, among free ones:
         #   * *name* (String): Task name
-        #   * *status* (Symbol): Task status. Should be on of:
+        #   * *status* (Symbol): Task status. Should be one of:
         #     * *:changed*: The task has been changed
         #     * *:identical*: The task has not been changed
         #   * *diffs* (String): Differences, if any

data/lib/hybrid_platforms_conductor/hpc_plugins/platform_handler/yaml_inventory.rb

@@ -0,0 +1,140 @@
+require 'yaml'
+require 'hybrid_platforms_conductor/platform_handler'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module PlatformHandler
+
+      # Basic platform handler, reading inventory and metadata from simple Yaml files.
+      class YamlInventory < HybridPlatformsConductor::PlatformHandler
+
+        # Initialize a new instance of this platform handler.
+        # [API] - This method is optional.
+        # [API] - @cmd_runner is accessible.
+        def init
+          # This method is called when initializing a new instance of this platform handler, for a given repository.
+          inv_file = "#{@repository_path}/inventory.yaml"
+          @inventory = File.exist?(inv_file) ? YAML.load(File.read(inv_file)) : {}
+        end
+
+        # Get the list of known nodes.
+        # [API] - This method is mandatory.
+        #
+        # Result::
+        # * Array<String>: List of node names
+        def known_nodes
+          @inventory.keys
+        end
+
+        # Get the metadata of a given node.
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *node* (String): Node to read metadata from
+        # Result::
+        # * Hash<Symbol,Object>: The corresponding metadata
+        def metadata_for(node)
+          (@inventory[node]['metadata'] || {}).transform_keys(&:to_sym)
+        end
+
+        # Return the services for a given node
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *node* (String): node to read configuration from
+        # Result::
+        # * Array<String>: The corresponding services
+        def services_for(node)
+          @inventory[node]['services'] || []
+        end
+
+        # Get the list of services we can deploy
+        # [API] - This method is mandatory.
+        #
+        # Result::
+        # * Array<String>: The corresponding services
+        def deployable_services
+          Dir.glob("#{@repository_path}/service_*.rb").map { |file| File.basename(file).match(/^service_(.*)\.rb$/)[1] }
+        end
+
+        # Get the list of actions to perform to deploy on a given node.
+        # Those actions can be executed in parallel with other deployments on other nodes. They must be thread safe.
+        # [API] - This method is mandatory.
+        # [API] - @cmd_runner is accessible.
+        # [API] - @actions_executor is accessible.
+        #
+        # Parameters::
+        # * *node* (String): Node to deploy on
+        # * *service* (String): Service to be deployed
+        # * *use_why_run* (Boolean): Do we use a why-run mode? [default = true]
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of actions to be done
+        def actions_to_deploy_on(node, service, use_why_run: true)
+          # Load the check and deploy methods in a temporary class for encapsulation
+          service_file = "#{@repository_path}/service_#{service}.rb"
+          Class.new do
+
+            include LoggerHelpers
+
+            # Constructor
+            #
+            # Parameters::
+            # * *platform_handler* (PlatformHandler): PlatformHandler needing this service to be deployed
+            # * *logger* (Logger): Logger to be used [default: Logger.new(STDOUT)]
+            # * *logger_stderr* (Logger): Logger to be used for stderr [default: Logger.new(STDERR)]
+            # * *config* (Config): Config to be used. [default: Config.new]
+            # * *nodes_handler* (NodesHandler): NodesHandler to be used [default: NodesHandler.new]
+            # * *cmd_runner* (CmdRunner): CmdRunner to be used [default: CmdRunner.new]
+            def initialize(
+              platform_handler,
+              logger: Logger.new(STDOUT),
+              logger_stderr: Logger.new(STDERR),
+              config: Config.new,
+              nodes_handler: NodesHandler.new,
+              cmd_runner: CmdRunner.new
+            )
+              init_loggers(logger, logger_stderr)
+              @platform_handler = platform_handler
+              @config = config
+              @nodes_handler = nodes_handler
+              @cmd_runner = cmd_runner
+            end
+
+            class_eval(File.read(service_file))
+
+          end.new(
+            self,
+            logger: @logger,
+            logger_stderr: @logger_stderr,
+            config: @config,
+            nodes_handler: @nodes_handler,
+            cmd_runner: @cmd_runner
+          ).send(use_why_run ? :check : :deploy, node)
+        end
+
+        # Parse stdout and stderr of a given deploy run and get the list of tasks with their status
+        # [API] - This method is mandatory.
+        #
+        # Parameters::
+        # * *stdout* (String): stdout to be parsed
+        # * *stderr* (String): stderr to be parsed
+        # Result::
+        # * Array< Hash<Symbol,Object> >: List of task properties. The following properties should be returned, among free ones:
+        #   * *name* (String): Task name
+        #   * *status* (Symbol): Task status. Should be one of:
+        #     * *:changed*: The task has been changed
+        #     * *:identical*: The task has not been changed
+        #   * *diffs* (String): Differences, if any
+        def parse_deploy_output(stdout, stderr)
+          []
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/provisioner/proxmox.rb

@@ -15,7 +15,7 @@ module HybridPlatformsConductor
 
         include LoggerHelpers
 
-        attr_accessor *%i[logger logger_stderr]
+        attr_accessor(*%i[logger logger_stderr])
 
         def check_response(response)
           msg = "Response from Proxmox API: #{response} - #{response.net_http_res.message}"
@@ -154,7 +154,7 @@ module HybridPlatformsConductor
                     hostname = "-#{Digest::MD5.hexdigest(hostname)[0..7]}.hpc-test.com"
                     hostname = "#{@node}.#{@environment}"[0..MAX_PROXMOX_HOSTNAME_SIZE - hostname.size - 1] + hostname
                   end
-                  @lxc_details = request_lxc_creation_for(                      {
+                  @lxc_details = request_lxc_creation_for({
                     ostemplate: pve_template,
                     hostname: hostname.gsub('_', '-'),
                     cores: min_resources_to_deploy[:cpus],
@@ -540,7 +540,10 @@ module HybridPlatformsConductor
         #   * *api_wait_between_retries_secs* (Integer): Number of seconds to wait between API retries
         #   * *sync_node* (String): Node to be used to synchronize Proxmox resources acquisition
         #   * *test_config* (Hash<Symbol,Object>): The test configuration. Check ProxmoxWaiter#initialize (config_file structure) method to get details.
-        #   * *vm_config* (Hash<Symbol,Object>): Extra configuration of a created container. Check #request_lxc_creation_for results to get details.
+        #   * *vm_config* (Hash<Symbol,Object>): Extra configuration of a created container:
+        #     * *vm_dns_servers* (Array<String>): List of DNS servers
+        #     * *vm_search_domain* (String): Default search domain
+        #     * *vm_gateway* (String): Gateway hostname or IP
         #   * *default_timeout* (Integer): The default timeout tobe applied when starting/stopping containers [default: 3600].
         def proxmox_test_info
           @config.proxmox_servers.first

data/lib/hybrid_platforms_conductor/hpc_plugins/test/bitbucket_conf.rb

@@ -60,7 +60,7 @@ module HybridPlatformsConductor
               end
             end
             # Merge checks
-            required_approvers = repo_info.dig *%i[checks pr_settings required_approvers]
+            required_approvers = repo_info.dig(*%i[checks pr_settings required_approvers])
             if required_approvers
               assert_equal(
                 settings_pr.dig('com.atlassian.bitbucket.server.bitbucket-bundled-hooks:requiredApprovers', 'enable'),
@@ -73,7 +73,7 @@ module HybridPlatformsConductor
                 "[#{repo_id}] - Number of required approvers should be #{required_approvers}"
               )
             end
-            required_builds = repo_info.dig *%i[checks pr_settings required_builds]
+            required_builds = repo_info.dig(*%i[checks pr_settings required_builds])
             if required_builds
               assert_equal(
                 settings_pr.dig('com.atlassian.bitbucket.server.bitbucket-build:requiredBuilds', 'enable'),
@@ -87,7 +87,7 @@ module HybridPlatformsConductor
               )
             end
             # Default merge strategy
-            default_merge_strategy = repo_info.dig *%i[checks pr_settings default_merge_strategy]
+            default_merge_strategy = repo_info.dig(*%i[checks pr_settings default_merge_strategy])
             if default_merge_strategy
               assert_equal(
                 settings_pr.dig('mergeConfig', 'defaultStrategy', 'id'),
@@ -104,7 +104,7 @@ module HybridPlatformsConductor
               )
             end
             # Default reviewers should include our team from any branch to any branch
-            mandatory_default_reviewers = repo_info.dig *%i[checks pr_settings mandatory_default_reviewers]
+            mandatory_default_reviewers = repo_info.dig(*%i[checks pr_settings mandatory_default_reviewers])
             if mandatory_default_reviewers
               reviewers_found = default_reviewers.any? do |condition_info|
                 reviewers = condition_info.dig('reviewers')

data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_freshness.rb

@@ -22,7 +22,7 @@ module HybridPlatformsConductor
                 error 'Node has never been deployed using deploy (/var/log/deployments does not exist)'
               else
                 # Expecting following file names
-                # 2017-12-01_093418_a_usernme
+                # node-name_2017-12-01_093418_user-name
                 file_match = stdout.first.match(/^#{Regexp.escape(@node)}_(\d{4}-\d{2}-\d{2})_.+$/)
                 if file_match.nil?
                   error "Invalid chef deployment log file found: #{stdout.first}"

data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_removes_root_access.rb

@@ -12,22 +12,9 @@ module HybridPlatformsConductor
 
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_for_node
-          @deployer.with_test_provisioned_instance(@config.tests_provisioner_id, @node, environment: 'deploy_removes_root_access', reuse_instance: log_debug?) do |deployer, instance|
-            # Check that we can connect with root
-            ssh_ok = false
-            begin
-              Net::SSH.start(instance.ip, 'root', password: 'root_pwd', auth_methods: ['password'], verify_host_key: :never) do |ssh|
-                ssh_ok = ssh.exec!('echo Works').strip == 'Works'
-              end
-            rescue
-            end
-            assert_equal ssh_ok, true, 'Root does not have access from the empty image'
-            if ssh_ok
-              deployer.nbr_retries_on_error = 3
-              deployer.deploy_on @node
-              # As sshd is certainly being restarted, start and stop the container to reload it.
-              deployer.restart @node
-              # Check that we can't connect with root
+          unless @nodes_handler.get_root_access_allowed_of(@node) == 'true'
+            @deployer.with_test_provisioned_instance(@config.tests_provisioner_id, @node, environment: 'deploy_removes_root_access', reuse_instance: log_debug?) do |deployer, instance|
+              # Check that we can connect with root
               ssh_ok = false
               begin
                 Net::SSH.start(instance.ip, 'root', password: 'root_pwd', auth_methods: ['password'], verify_host_key: :never) do |ssh|
@@ -35,7 +22,22 @@ module HybridPlatformsConductor
                 end
               rescue
               end
-              assert_equal ssh_ok, false, 'Root can still connect on the image after deployment'
+              assert_equal ssh_ok, true, 'Root does not have access from the empty image'
+              if ssh_ok
+                deployer.nbr_retries_on_error = 3
+                deployer.deploy_on @node
+                # As sshd is certainly being restarted, start and stop the container to reload it.
+                deployer.restart @node
+                # Check that we can't connect with root
+                ssh_ok = false
+                begin
+                  Net::SSH.start(instance.ip, 'root', password: 'root_pwd', auth_methods: ['password'], verify_host_key: :never) do |ssh|
+                    ssh_ok = ssh.exec!('echo Works').strip == 'Works'
+                  end
+                rescue
+                end
+                assert_equal ssh_ok, false, 'Root can still connect on the image after deployment'
+              end
             end
           end
         end

data/lib/hybrid_platforms_conductor/hpc_plugins/test/divergence.rb

@@ -1,4 +1,5 @@
 require 'json'
+require 'hybrid_platforms_conductor/common_config_dsl/idempotence_tests'
 
 module HybridPlatformsConductor
 
@@ -9,6 +10,8 @@ module HybridPlatformsConductor
       # Test that the node has not diverged since last deployment
       class Divergence < HybridPlatformsConductor::Test
 
+        self.extend_config_dsl_with CommonConfigDsl::IdempotenceTests, :init_idempotence_tests
+
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_check_node(stdout, stderr, exit_status)
           # Check that the output of the check-node returns no changes.

data/lib/hybrid_platforms_conductor/hpc_plugins/test/hostname.rb

@@ -10,7 +10,8 @@ module HybridPlatformsConductor
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_node
           {
-            "#{@nodes_handler.sudo_on(@node)} hostname -s" => proc do |stdout|
+            # TODO: Access the user correctly when the user notion will be moved out of the ssh connector
+            "#{@deployer.instance_variable_get(:@actions_executor).connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(@node)} "}hostname -s" => proc do |stdout|
               assert_equal stdout.first, @node, "Expected hostname to be #{@node}, but got #{stdout.first} instead."
             end
           }

data/lib/hybrid_platforms_conductor/hpc_plugins/test/ip.rb

@@ -10,7 +10,8 @@ module HybridPlatformsConductor
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_node
           {
-            "#{@nodes_handler.sudo_on(@node)} hostname -I" => proc do |stdout|
+            # TODO: Access the user correctly when the user notion will be moved out of the ssh connector
+            "#{@deployer.instance_variable_get(:@actions_executor).connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(@node)} "}hostname -I" => proc do |stdout|
               if stdout.first.nil?
                 error 'No IP returned by "hostname -I"'
               else

data/lib/hybrid_platforms_conductor/hpc_plugins/test/local_users.rb

@@ -57,7 +57,8 @@ module HybridPlatformsConductor
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_node
           {
-            "#{@nodes_handler.sudo_on(@node)} cat /etc/passwd" => proc do |stdout|
+            # TODO: Access the user correctly when the user notion will be moved out of the ssh connector
+            "#{@deployer.instance_variable_get(:@actions_executor).connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(@node)} "}cat /etc/passwd" => proc do |stdout|
               passwd_users = stdout.map { |passwd_line| passwd_line.split(':').first }
               missing_users = @nodes_handler.
                 select_confs_for_node(@node, @config.users_that_should_be_present).

data/lib/hybrid_platforms_conductor/hpc_plugins/test/mounts.rb

@@ -32,7 +32,7 @@ module HybridPlatformsConductor
           #
           # Parameters::
           # * *mount_rules* (Hash<String or Regexp, String or Regexp>):
-          #     List of (or single) sets of { source => destination } mounts that should not be present.
+          #     Set of { source => destination } mounts that should not be present.
           #     Each source or destination can be a string for exact match, or a regexp to match a pattern on the mounts done on the node.
           def check_mounts_do_not_include(mount_rules)
             @mount_rules_that_should_be_absent << {
@@ -45,7 +45,7 @@ module HybridPlatformsConductor
           #
           # Parameters::
           # * *mount_rules* (Hash<String or Regexp, String or Regexp>):
-          #     List of (or single) sets of { source => destination } mounts that should be present.
+          #     Set of { source => destination } mounts that should be present.
           #     Each source or destination can be a string for exact match, or a regexp to match a pattern on the mounts done on the node.
           def check_mounts_do_include(mount_rules)
             @mount_rules_that_should_be_present << {
@@ -61,7 +61,8 @@ module HybridPlatformsConductor
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_node
           {
-            "#{@nodes_handler.sudo_on(@node)} mount" => proc do |stdout|
+            # TODO: Access the user correctly when the user notion will be moved out of the ssh connector
+            "#{@deployer.instance_variable_get(:@actions_executor).connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(@node)} "}mount" => proc do |stdout|
               mounts_info = stdout.map do |line|
                 fields = line.split
                 {

data/lib/hybrid_platforms_conductor/hpc_plugins/test/orphan_files.rb

@@ -50,7 +50,8 @@ module HybridPlatformsConductor
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_node
           {
-            "#{@nodes_handler.sudo_on(@node)} /usr/bin/find / \\( #{@nodes_handler.
+            # TODO: Access the user correctly when the user notion will be moved out of the ssh connector
+            "#{@deployer.instance_variable_get(:@actions_executor).connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(@node)} "}/usr/bin/find / \\( #{@nodes_handler.
               select_confs_for_node(@node, @config.ignored_orphan_files_paths).
               inject(DIRECTORIES_TO_ALWAYS_IGNORE) { |merged_paths, paths_to_ignore_info| merged_paths + paths_to_ignore_info[:ignored_paths] }.
               uniq.

data/lib/hybrid_platforms_conductor/hpc_plugins/test/spectre.rb

@@ -16,7 +16,7 @@ module HybridPlatformsConductor
         # Check my_test_plugin.rb.sample documentation for signature details.
         def test_on_node
           spectre_cmd = <<~EOS
-            #{@nodes_handler.sudo_on(@node)} /bin/bash <<'EOAction'
+            #{@deployer.instance_variable_get(:@actions_executor).connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(@node)} "}/bin/bash <<'EOAction'
             #{File.read("#{__dir__}/spectre-meltdown-checker.sh")}
             EOAction
           EOS

data/lib/hybrid_platforms_conductor/hpc_plugins/test/vulnerabilities.rb

@@ -54,7 +54,8 @@ module HybridPlatformsConductor
                   current_url
                 end
               )
-              sudo = @nodes_handler.sudo_on(@node)
+              # TODO: Access the user correctly when the user notion will be moved out of the ssh connector
+              sudo = @deployer.instance_variable_get(:@actions_executor).connector(:ssh).ssh_user == 'root' ? '' : "#{@nodes_handler.sudo_on(@node)} "
               Hash[urls.map do |url|
                 # 1. Get the OVAL file on the node to be tested (uncompress it if needed)
                 # 2. Make sure oscap is installed
@@ -75,9 +76,9 @@ module HybridPlatformsConductor
                   #{
                     case image
                     when :centos_7
-                      "#{sudo} yum install -y wget openscap-scanner #{packages_to_install.join(' ')}"
+                      "#{sudo}yum install -y wget openscap-scanner #{packages_to_install.join(' ')}"
                     when :debian_9
-                      "#{sudo} apt install -y wget libopenscap8 #{packages_to_install.join(' ')}"
+                      "#{sudo}apt install -y wget libopenscap8 #{packages_to_install.join(' ')}"
                     when :debian_10
                       # On Debian 10 we have to compile it from sources, as the packaged official version has core dumps.
                       # cf https://www.mail-archive.com/debian-bugs-dist@lists.debian.org/msg1688223.html
@@ -87,13 +88,13 @@ module HybridPlatformsConductor
                           rm -rf openscap
                           git clone --recurse-submodules https://github.com/OpenSCAP/openscap.git
                           cd openscap
-                          #{sudo} apt install -y cmake libdbus-1-dev libdbus-glib-1-dev libcurl4-openssl-dev libgcrypt20-dev libselinux1-dev libxslt1-dev libgconf2-dev libacl1-dev libblkid-dev libcap-dev libxml2-dev libldap2-dev libpcre3-dev python-dev swig libxml-parser-perl libxml-xpath-perl libperl-dev libbz2-dev librpm-dev g++ libapt-pkg-dev libyaml-dev
+                          #{sudo}apt install -y cmake libdbus-1-dev libdbus-glib-1-dev libcurl4-openssl-dev libgcrypt20-dev libselinux1-dev libxslt1-dev libgconf2-dev libacl1-dev libblkid-dev libcap-dev libxml2-dev libldap2-dev libpcre3-dev python-dev swig libxml-parser-perl libxml-xpath-perl libperl-dev libbz2-dev librpm-dev g++ libapt-pkg-dev libyaml-dev
                           cd build
                           cmake ../
                           make
-                          #{sudo} make install
+                          #{sudo}make install
                         fi
-                        #{sudo} apt install -y wget #{packages_to_install.join(' ')}
+                        #{sudo}apt install -y wget #{packages_to_install.join(' ')}
                       EOS2
                     else
                       raise "Non supported image: #{image}. Please adapt this test's code."
@@ -104,7 +105,7 @@ module HybridPlatformsConductor
                   cd hpc_vulnerabilities_test
                   wget -N #{url}
                   #{uncompress_cmds.join("\n")}
-                  #{sudo} oscap oval eval --skip-valid --results "#{local_oval_file}.results.xml" "#{local_oval_file}"
+                  #{sudo}oscap oval eval --skip-valid --results "#{local_oval_file}.results.xml" "#{local_oval_file}"
                   echo "===== RESULTS ====="
                   cat "#{local_oval_file}.results.xml"
                   cd ..