hybrid_platforms_conductor 32.3.6

data/lib/hybrid_platforms_conductor/hpc_plugins/report/templates/confluence_inventory.html.erb

@@ -0,0 +1,33 @@
+<ac:structured-macro ac:name="toc" ac:schema-version="1"/>
+<table>
+  <tbody>
+    <tr>
+      <th>Node</th>
+      <th>Platform</th>
+      <th>Host name</th>
+      <th>IP</th>
+      <th>Physical?</th>
+      <th>OS</th>
+      <th>Description</th>
+      <th>Services</th>
+    </tr>
+    <% @nodes_handler.prefetch_metadata_of @nodes, %i[hostname host_ip physical image description services] %>
+    <% @nodes.sort.each do |node| %>
+      <tr style="vertical-align: middle;">
+        <td><strong><%= node %></strong></td>
+        <td><%= @platforms_handler.known_platforms.find { |platform| platform.known_nodes.include?(node) }&.name %></td>
+        <td><%= @nodes_handler.get_hostname_of(node) %></td>
+        <td><%= @nodes_handler.get_host_ip_of(node) %></td>
+        <td><%= @nodes_handler.get_physical_of(node) ? 'Yes' : 'No' %></td>
+        <td><%= @nodes_handler.get_image_of node %></td>
+        <td><%= @nodes_handler.get_description_of node %></td>
+        <td><%= (@nodes_handler.get_services_of(node) || []).sort.join(', ') %></td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>
+<p>
+  <br/>
+  Generated at <%= Time.now.utc.strftime('%F %T') %> UTC by <a href="https://www.site.my_company.net/git/projects/PROJECTrepos/hybrid_platforms_conductor/browse">Hybrid Platforms Conductor</a>.
+  <br/>
+</p>

data/lib/hybrid_platforms_conductor/hpc_plugins/test/bitbucket_conf.rb

@@ -0,0 +1,137 @@
+require 'git'
+require 'hybrid_platforms_conductor/common_config_dsl/bitbucket'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Test
+
+      # Check that all repositories in Bitbucket have a consistent dev workflow.
+      class BitbucketConf < HybridPlatformsConductor::Test
+
+        self.extend_config_dsl_with CommonConfigDsl::Bitbucket, :init_bitbucket
+
+        # Check my_test_plugin.rb.sample documentation for signature details.
+        def test
+          @config.for_each_bitbucket_repo do |bitbucket, repo_info|
+            # Test repo_info
+            repo_id = "#{repo_info[:project]}/#{repo_info[:name]}"
+            settings_pr = bitbucket.settings_pr(repo_info[:project], repo_info[:name])
+            default_reviewers = bitbucket.default_reviewers(repo_info[:project], repo_info[:name])
+            # There can be a post Webhook to Jenkins CI
+            # TODO: Find a way to query the Post Webhook plugin API
+            # Branch permissions
+            if repo_info[:checks][:branch_permissions]
+              branch_permissions = bitbucket.branch_permissions(repo_info[:project], repo_info[:name])
+              repo_info[:checks][:branch_permissions].each do |branch_permissions_to_check|
+                exempted_users = branch_permissions_to_check[:exempted_users] || []
+                exempted_groups = branch_permissions_to_check[:exempted_groups] || []
+                exempted_keys = branch_permissions_to_check[:exempted_keys] || []
+                message = "[#{repo_id}] - Branch permissions for #{branch_permissions_to_check[:branch]} should prohibit \"#{
+                  case branch_permissions_to_check[:type]
+                  when 'fast-forward-only'
+                    'Rewriting history'
+                  when 'no-deletes'
+                    'Deletion'
+                  when 'pull-request-only'
+                    'Changes without a pull request'
+                  else
+                    log_warn "Unknown branch permission type #{branch_permissions_to_check[:type]} - Please adapt this test plugin's code."
+                    branch_permissions_to_check[:type]
+                  end
+                }\""
+                exceptions = []
+                exceptions << "users #{exempted_users.join(', ')}" unless exempted_users.empty?
+                exceptions << "groups #{exempted_groups.join(', ')}" unless exempted_groups.empty?
+                exceptions << "keys #{exempted_keys.join(', ')}" unless exempted_keys.empty?
+                message << " except for #{exceptions.join(' and ')}" unless exceptions.empty?
+                assert_equal(
+                  branch_permissions['values'].any? do |branch_permission_info|
+                    branch_permission_info['type'] == branch_permissions_to_check[:type] &&
+                      branch_permission_info['matcher']['id'] == "refs/heads/#{branch_permissions_to_check[:branch]}" &&
+                      branch_permission_info['users'].map { |user_info| user_info['name'] }.sort == exempted_users.sort &&
+                      branch_permission_info['groups'].sort == exempted_groups.sort &&
+                      branch_permission_info['accessKeys'].sort == exempted_keys.sort
+                  end,
+                  true,
+                  message
+                )
+              end
+            end
+            # Merge checks
+            required_approvers = repo_info.dig *%i[checks pr_settings required_approvers]
+            if required_approvers
+              assert_equal(
+                settings_pr.dig('com.atlassian.bitbucket.server.bitbucket-bundled-hooks:requiredApprovers', 'enable'),
+                true,
+                "[#{repo_id}] - Required approvers should be enabled"
+              )
+              assert_equal(
+                settings_pr.dig('com.atlassian.bitbucket.server.bitbucket-bundled-hooks:requiredApprovers', 'count'),
+                required_approvers,
+                "[#{repo_id}] - Number of required approvers should be #{required_approvers}"
+              )
+            end
+            required_builds = repo_info.dig *%i[checks pr_settings required_builds]
+            if required_builds
+              assert_equal(
+                settings_pr.dig('com.atlassian.bitbucket.server.bitbucket-build:requiredBuilds', 'enable'),
+                true,
+                "[#{repo_id}] - Required builds should be enabled"
+              )
+              assert_equal(
+                settings_pr.dig('com.atlassian.bitbucket.server.bitbucket-build:requiredBuilds', 'count'),
+                required_builds,
+                "[#{repo_id}] - Number of required builds should be #{required_builds}"
+              )
+            end
+            # Default merge strategy
+            default_merge_strategy = repo_info.dig *%i[checks pr_settings default_merge_strategy]
+            if default_merge_strategy
+              assert_equal(
+                settings_pr.dig('mergeConfig', 'defaultStrategy', 'id'),
+                default_merge_strategy,
+                "[#{repo_id}] - Default merge strategy should be #{
+                  case default_merge_strategy
+                  when 'rebase-no-ff'
+                    'Rebase + Merge --no-ff'
+                  else
+                    log_warn "Unknown merge strategy #{default_merge_strategy} - Please adapt this test plugin's code."
+                    default_merge_strategy
+                  end
+                }"
+              )
+            end
+            # Default reviewers should include our team from any branch to any branch
+            mandatory_default_reviewers = repo_info.dig *%i[checks pr_settings mandatory_default_reviewers]
+            if mandatory_default_reviewers
+              reviewers_found = default_reviewers.any? do |condition_info|
+                reviewers = condition_info.dig('reviewers')
+                condition_info.dig('sourceRefMatcher', 'id') == 'ANY_REF_MATCHER_ID' &&
+                  condition_info.dig('targetRefMatcher', 'id') == 'ANY_REF_MATCHER_ID' &&
+                  !reviewers.nil? &&
+                  (mandatory_default_reviewers - reviewers.map { |reviewer_info| reviewer_info['name'] }).empty? &&
+                  (required_approvers.nil? || condition_info.dig('requiredApprovals') == required_approvers)
+              end
+              assert_equal(
+                reviewers_found,
+                true,
+                "[#{repo_id}] - Missing mandatory reviewers among #{mandatory_default_reviewers.join(', ')}#{required_approvers.nil? ? '' : " with a minimum of #{required_approvers} approvals"} from any branch to any branch"
+              )
+            end
+            # Make sure the repository has master being tagged correctly
+            log_debug "Check that master branch of #{repo_info[:url]} has a semantic tag"
+            refs_info = Git.ls_remote(repo_info[:url])
+            master_sha = refs_info['branches']['master'][:sha]
+            error "[#{repo_id}] - No semantic tag found on master branch (#{repo_info[:url]} ref #{master_sha})" unless refs_info['tags'].any? { |tag_name, tag_info| tag_info[:sha] == master_sha && tag_name =~ /^v\d+\.\d+\.\d+$/ }
+          end
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/test/can_be_checked.rb

@@ -0,0 +1,21 @@
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Test
+
+      # Test that the node can be checked without error
+      class CanBeChecked < HybridPlatformsConductor::Test
+
+        # Check my_test_plugin.rb.sample documentation for signature details.
+        def test_on_check_node(stdout, stderr, exit_status)
+          assert_equal exit_status, 0, "Check-node run returned error #{exit_status}#{log_debug? ? ":\n===== STDOUT =====\n#{stdout}===== STDERR =====#{stderr}" : ''}"
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_deploy_and_idempotence.rb

@@ -0,0 +1,112 @@
+require 'json'
+require 'hybrid_platforms_conductor/common_config_dsl/idempotence_tests'
+require 'hybrid_platforms_conductor/test_by_service'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Test
+
+      # Test that combines the following tests in just 1 test to avoid spawning several Docker containers:
+      # * check_from_scratch
+      # * deploy_from_scratch
+      # * deploy_removes_root_access
+      # * idempotence
+      # Especially useful if your tests run in an environment having limited Docker resources.
+      # This tests uses the testadmin user access once deployed.
+      # Don't forget to add the testadmin private key in your SSH agent if you run this test locally.
+      class CheckDeployAndIdempotence < TestByService
+
+        self.extend_config_dsl_with CommonConfigDsl::IdempotenceTests, :init_idempotence_tests
+
+        # Check my_test_plugin.rb.sample documentation for signature details.
+        def test_for_node
+          @deployer.with_test_provisioned_instance(@config.tests_provisioner_id, @node, environment: 'check_deploy_and_idempotence', reuse_instance: log_debug?) do |deployer, instance|
+            # Check that we can connect with root
+            ssh_ok = false
+            begin
+              Net::SSH.start(instance.ip, 'root', password: 'root_pwd', auth_methods: ['password'], verify_host_key: :never) do |ssh|
+                ssh_ok = ssh.exec!('echo Works').strip == 'Works'
+              end
+            rescue
+            end
+            assert_equal ssh_ok, true, 'Root does not have access from the empty image'
+
+            if ssh_ok
+
+              # ===== Check from scratch
+              deployer.use_why_run = true
+              exit_status, _stdout, _stderr = deployer.deploy_on(@node)[@node]
+              assert_equal exit_status, 0, "Check-node from scratch returned error code #{exit_status}", log_debug? ? nil : deployer.stdouts_to_s
+              # Even if the check has failed, we try to deploy it
+
+              # ===== Deploy from scratch
+              deployer.use_why_run = false
+              deployer.nbr_retries_on_error = 3
+              exit_status, _stdout, _stderr = deployer.deploy_on(@node)[@node]
+              assert_equal exit_status, 0, "Deploy from scratch returned error code #{exit_status}", log_debug? ? nil : deployer.stdouts_to_s
+              if exit_status == 0
+                # As it's possible sshd has to be restarted because of a change in its conf, restart the container.
+                # Otherwise you'll get the following error upon reconnection:
+                #   System is booting up. See pam_nologin(8)
+                #   Authentication failed.
+                instance.stop
+                instance.with_running_instance(port: 22) do
+
+                  # ===== Deploy removes root access
+                  # Check that we can't connect with root
+                  ssh_ok = false
+                  begin
+                    Net::SSH.start(instance.ip, 'root', password: 'root_pwd', auth_methods: ['password'], verify_host_key: :never) do |ssh|
+                      ssh_ok = ssh.exec!('echo Works').strip == 'Works'
+                    end
+                  rescue
+                  end
+                  assert_equal ssh_ok, false, 'Root can still connect on the image after deployment'
+                  # Even if we can connect using root, run the idempotence test
+
+                  # ===== Idempotence
+                  unless ssh_ok
+                    # Now that the node has been deployed, use the a_testadmin user for the check-node (as root has no more access)
+                    deployer.instance_variable_get(:@actions_executor).connector(:ssh).ssh_user = 'a_testadmin'
+                    deployer.instance_variable_get(:@actions_executor).connector(:ssh).passwords.delete(@node)
+                  end
+                  deployer.use_why_run = true
+                  deployer.nbr_retries_on_error = 0
+                  # For the idempotence testing activate log debugs, so that in case of failures we have full details
+                  deployer.log_level = :debug
+                  exit_status, stdout, stderr = deployer.deploy_on(@node)[@node]
+                  assert_equal exit_status, 0, "Check-node after deployment returned error code #{exit_status}", log_debug? ? nil : deployer.stdouts_to_s
+                  # Check that the output of the check-node returns no changes.
+                  ignored_tasks = @nodes_handler.select_confs_for_node(@node, @config.ignored_idempotence_tasks).inject({}) do |merged_ignored_tasks, conf|
+                    merged_ignored_tasks.merge(conf[:ignored_tasks])
+                  end
+                  @deployer.parse_deploy_output(@node, stdout, stderr).each do |task_info|
+                    if task_info[:status] == :changed
+                      if ignored_tasks.key?(task_info[:name])
+                        # It was expected that this task is not idempotent
+                        log_debug "Task #{task_info[:name]} was expected to not be idempotent. Reason: #{ignored_tasks[task_info[:name]]}"
+                      else
+                        extra_details = task_info.slice(*(task_info.keys - %i[name status diffs]))
+                        error_details = []
+                        error_details << "----- Changes:\n#{task_info[:diffs].strip}\n-----" if task_info[:diffs]
+                        error_details << "----- Additional details:\n#{JSON.pretty_generate(extra_details)}\n-----" unless extra_details.empty?
+                        error "Task #{task_info[:name]} is not idempotent", error_details.empty? ? nil : error_details.join("\n")
+                      end
+                    end
+                  end
+
+                end
+              end
+            end
+          end
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/test/check_from_scratch.rb

@@ -0,0 +1,35 @@
+require 'hybrid_platforms_conductor/test_by_service'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Test
+
+      # Test that check-node returns no error on an empty image
+      class CheckFromScratch < TestByService
+
+        # Check my_test_plugin.rb.sample documentation for signature details.
+        def test_for_node
+          @deployer.with_test_provisioned_instance(@config.tests_provisioner_id, @node, environment: 'check_from_scratch', reuse_instance: log_debug?) do |deployer|
+            deployer.use_why_run = true
+            result = deployer.deploy_on(@node)
+            assert_equal result.size, 1, "Wrong number of nodes being tested: #{result.size}"
+            tested_node, (exit_status, _stdout, _stderr) = result.first
+            if exit_status.is_a?(Symbol)
+              # In debug mode, the logger is the normal one, already outputting the error. No need to get it back from the logs.
+              error "Check-node could not run because of error: #{exit_status}.", log_debug? ? nil : deployer.stdouts_to_s
+            else
+              assert_equal tested_node, @node, "Wrong node being tested: #{tested_node} should be #{@node}"
+              assert_equal exit_status, 0, "Check-node returned error code #{exit_status}"
+            end
+          end
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/test/connection.rb

@@ -0,0 +1,28 @@
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Test
+
+      # Test that the connection works by simply outputing something
+      class Connection < HybridPlatformsConductor::Test
+
+        TEST_CONNECTION_STRING = 'Test connection - ok'
+
+        # Check my_test_plugin.rb.sample documentation for signature details.
+        def test_on_node
+          node_connection_string = "#{TEST_CONNECTION_STRING} for #{@node}"
+          {
+            "echo '#{node_connection_string}'" => proc do |stdout|
+              assert_equal stdout.first, node_connection_string, 'Connection failed'
+            end
+          }
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_freshness.rb

@@ -0,0 +1,44 @@
+require 'time'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Test
+
+      # Test that the last deployment was done recently
+      class DeployFreshness < HybridPlatformsConductor::Test
+
+        MAX_ACCEPTABLE_REFRESH_PERIOD_SECS = 3 * 31 * 24 * 60 * 60 # 3 months
+
+        # Check my_test_plugin.rb.sample documentation for signature details.
+        def test_on_node
+          now = Time.now
+          {
+            'sudo ls -t /var/log/deployments' => proc do |stdout|
+              if stdout.empty?
+                error 'Node has never been deployed using deploy (/var/log/deployments is empty)'
+              elsif stdout.first =~ /No such file or directory/
+                error 'Node has never been deployed using deploy (/var/log/deployments does not exist)'
+              else
+                # Expecting following file names
+                # 2017-12-01_093418_a_usernme
+                file_match = stdout.first.match(/^#{Regexp.escape(@node)}_(\d{4}-\d{2}-\d{2})_.+$/)
+                if file_match.nil?
+                  error "Invalid chef deployment log file found: #{stdout.first}"
+                else
+                  last_deploy_time = Time.parse(file_match[1])
+                  error "Last deployment has been done on #{last_deploy_time.strftime('%F')}. Should refresh it." if now - last_deploy_time > MAX_ACCEPTABLE_REFRESH_PERIOD_SECS
+                end
+              end
+            end
+          }
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/hybrid_platforms_conductor/hpc_plugins/test/deploy_from_scratch.rb

@@ -0,0 +1,36 @@
+require 'hybrid_platforms_conductor/test_by_service'
+
+module HybridPlatformsConductor
+
+  module HpcPlugins
+
+    module Test
+
+      # Test that deploy returns no error on an empty image
+      class DeployFromScratch < TestByService
+
+        # Check my_test_plugin.rb.sample documentation for signature details.
+        def test_for_node
+          @deployer.with_test_provisioned_instance(@config.tests_provisioner_id, @node, environment: 'deploy_from_scratch', reuse_instance: log_debug?) do |deployer|
+            deployer.nbr_retries_on_error = 3
+            deployer.log_level = :debug
+            result = deployer.deploy_on(@node)
+            assert_equal result.size, 1, "Wrong number of nodes being tested: #{result.size}"
+            tested_node, (exit_status, _stdout, _stderr) = result.first
+            if exit_status.is_a?(Symbol)
+              # In debug mode, the logger is the normal one, already outputting the error. No need to get it back from the logs.
+              error "Deploy could not run because of error: #{exit_status}.", log_debug? ? nil : deployer.stdouts_to_s
+            else
+              assert_equal tested_node, @node, "Wrong node being deployed: #{tested_node} should be #{@node}"
+              assert_equal exit_status, 0, "Deploy returned error code #{exit_status}"
+            end
+          end
+        end
+
+      end
+
+    end
+
+  end
+
+end