gitlab-qa 6.21.4 → 7.0.1

data/lib/gitlab/qa/runtime/env.rb

@@ -10,6 +10,7 @@ module Gitlab
         # These variables should be listed in /docs/what_tests_can_be_run.md#supported-gitlab-environment-variables
         # unless they're defined elsewhere (e.g.: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html)
         ENV_VARIABLES = {
+          'QA_IMAGE' => :qa_image,
           'QA_REMOTE_GRID' => :remote_grid,
           'QA_REMOTE_GRID_USERNAME' => :remote_grid_username,
           'QA_REMOTE_GRID_ACCESS_KEY' => :remote_grid_access_key,
@@ -64,6 +65,7 @@ module Gitlab
           'KNAPSACK_TEST_FILE_PATTERN' => :knapsack_test_file_pattern,
           'KNAPSACK_TEST_DIR' => :knapsack_test_dir,
           'CI' => :ci,
+          'CI_JOB_NAME' => :ci_job_name,
           'CI_JOB_URL' => :ci_job_url,
           'CI_RUNNER_ID' => :ci_runner_id,
           'CI_SERVER_HOST' => :ci_server_host,

data/lib/gitlab/qa/runtime/omnibus_configuration.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/object/blank'
+
+module Gitlab
+  module QA
+    module Runtime
+      class OmnibusConfiguration
+        def initialize
+          @config = ["# Generated by GitLab QA Omnibus Configurator at #{Time.now.strftime('%Y-%m-%d %H:%M:%S')}"]
+        end
+
+        def to_s
+          sanitize!.join("\n")
+        end
+
+        def configuration
+          raise NotImplementedError
+        end
+
+        # Before hook for any additional configuration
+        # This would usually be a container that needs to be running
+        # @return Any instance of [Gitlab::QA::Component::Base]
+        def prepare; end
+
+        # Commands to execute before GitLab boots
+        def exec_commands
+          []
+        end
+
+        # Ensures no duplicate entries
+        # @raise RuntimeError if competing configurations exist
+        # rubocop:disable Metrics/AbcSize
+        def sanitize!
+          sanitized = @config.map do |config|
+            next config if config.start_with?('#') || config.match(/\w+\(/) # allow for comments and method invocations
+
+            # sometimes "=" is part of a Hash. Only split based on the first "="
+            k, v = config.split("=", 2)
+            # make sure each config is well-formed
+            # e.g., gitlab_rails['packages_enabled'] = true
+            #   NOT gitlab_rails['packages_enabled']=true
+
+            v.nil? ? k.strip : "#{k.strip} = #{v.strip.tr('"', "'")}".strip
+          end.uniq
+
+          errors = []
+
+          # check for duplicates
+          duplicate_keys = []
+          duplicates = sanitized.reject do |n|
+            key = n.split('=').first
+            duplicate_keys << key unless duplicate_keys.include?(key)
+          end
+
+          duplicates.each { |duplicate| errors << "Duplicate entry found: `#{duplicate}`" }
+
+          raise "Errors exist within the Omnibus Configuration!\n#{errors.join(',')}" if errors.any?
+
+          @config = sanitized
+        end
+        # rubocop:enable Metrics/AbcSize
+
+        def <<(config)
+          @config << config.strip unless config.strip.empty?
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/runtime/omnibus_configurations/default.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Runtime
+      module OmnibusConfigurations
+        # Default Configuration for Omnibus
+        # All runs will include this configuration
+        class Default < Runtime::OmnibusConfiguration
+          def configuration
+            <<~OMNIBUS
+              gitlab_rails['gitlab_default_theme'] = 10 # Light Red Theme
+              gitlab_rails['gitlab_disable_animations'] = true # Disable animations
+              gitlab_rails['application_settings_cache_seconds'] = 0 # Settings cache expiry
+            OMNIBUS
+          end
+
+          def self.configuration
+            new.configuration
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/runtime/omnibus_configurations/object_storage.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Runtime
+      module OmnibusConfigurations
+        class ObjectStorage < Default
+          TYPES = %w[artifacts external_diffs lfs uploads packages dependency_proxy].freeze
+
+          def configuration
+            TYPES.each_with_object(+'') do |object_type, omnibus|
+              omnibus << <<~OMNIBUS
+                gitlab_rails['#{object_type}_enabled'] = true
+                gitlab_rails['#{object_type}_storage_path'] = '/var/opt/gitlab/gitlab-rails/shared/#{object_type}'
+                gitlab_rails['#{object_type}_object_store_enabled'] = true
+                gitlab_rails['#{object_type}_object_store_remote_directory'] = '#{object_type}-bucket'
+                gitlab_rails['#{object_type}_object_store_background_upload'] = false
+                gitlab_rails['#{object_type}_object_store_direct_upload'] = true
+                gitlab_rails['#{object_type}_object_store_proxy_download'] = true
+                gitlab_rails['#{object_type}_object_store_connection'] = #{minio.to_config}
+              OMNIBUS
+            end
+          end
+
+          def prepare
+            minio.network = 'test'
+
+            TYPES.each do |bucket_name|
+              minio.add_bucket("#{bucket_name}-bucket")
+            end
+
+            minio
+          end
+
+          def exec_commands
+            QA::Scenario::CLICommands.git_lfs_install_commands
+          end
+
+          private
+
+          def minio
+            @minio ||= Component::Minio.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/runtime/omnibus_configurations/packages.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Runtime
+      module OmnibusConfigurations
+        class Packages < Default
+          def configuration
+            <<~OMNIBUS
+              gitlab_rails['packages_enabled'] = true
+            OMNIBUS
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/scenario/cli_commands.rb

@@ -4,11 +4,11 @@ module Gitlab
       module CLICommands
         GIT_LFS_VERSION = '2.8.0'.freeze
 
-        def git_lfs_install_commands
-          @git_lfs_install_commands ||= [
+        def self.git_lfs_install_commands
+          [
             "cd /tmp ; curl -qsL https://github.com/git-lfs/git-lfs/releases/download/v#{GIT_LFS_VERSION}/git-lfs-linux-amd64-v#{GIT_LFS_VERSION}.tar.gz | tar xzvf -",
             'cp /tmp/git-lfs /usr/local/bin'
-          ]
+          ].freeze
         end
       end
     end

data/lib/gitlab/qa/scenario/test/instance/relative_url.rb

@@ -10,9 +10,7 @@ module Gitlab
                 gitlab.network = 'test'
                 gitlab.relative_path = '/relative'
 
-                gitlab.omnibus_config = <<~OMNIBUS
-                  external_url '#{gitlab.address}';
-                OMNIBUS
+                gitlab.omnibus_configuration << "external_url '#{gitlab.address}'"
 
                 gitlab.instance do
                   Component::Specs.perform do |specs|

data/lib/gitlab/qa/scenario/test/instance/repository_storage.rb

@@ -9,7 +9,7 @@ module Gitlab
                 gitlab.release = QA::Release.new(release)
                 gitlab.name = 'gitlab'
                 gitlab.network = 'test'
-                gitlab.omnibus_config = <<~OMNIBUS
+                gitlab.omnibus_configuration << <<~OMNIBUS
                   git_data_dirs({
                     'default' => {
                       'path' => '/var/opt/gitlab/git-data/repositories/default'

data/lib/gitlab/qa/scenario/test/integration/actioncable.rb

@@ -9,9 +9,7 @@ module Gitlab
                 gitlab.release = QA::Release.new(release)
                 gitlab.name = 'gitlab-actioncable'
                 gitlab.network = 'test'
-                gitlab.omnibus_config = <<~OMNIBUS
-                  actioncable['enable'] = true;
-                OMNIBUS
+                gitlab.omnibus_configuration << "actioncable['enable'] = true"
 
                 gitlab.instance do
                   puts "Running actioncable specs!"

data/lib/gitlab/qa/scenario/test/integration/geo.rb

@@ -4,7 +4,6 @@ module Gitlab
       module Test
         module Integration
           class Geo < Scenario::Template
-            include Scenario::CLICommands
             ##
             # rubocop:disable Lint/MissingCopEnableDirective
             # rubocop:disable Metrics/AbcSize
@@ -20,7 +19,7 @@ module Gitlab
                 primary.release = release
                 primary.name = 'gitlab-primary'
                 primary.network = 'geo'
-                primary.omnibus_config = <<~OMNIBUS
+                primary.omnibus_configuration << <<~OMNIBUS
                   gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc';
                   geo_primary_role['enable'] = true;
                   gitlab_rails['db_password'] = 'mypass';
@@ -36,14 +35,14 @@ module Gitlab
                   sidekiq['concurrency'] = 2;
                   puma['worker_processes'] = 2;
                 OMNIBUS
-                primary.exec_commands = fast_ssh_key_lookup_commands + git_lfs_install_commands
+                primary.exec_commands = fast_ssh_key_lookup_commands + QA::Scenario::CLICommands.git_lfs_install_commands
 
                 primary.instance do
                   Component::Gitlab.perform do |secondary|
                     secondary.release = release
                     secondary.name = 'gitlab-secondary'
                     secondary.network = 'geo'
-                    secondary.omnibus_config = <<~OMNIBUS
+                    secondary.omnibus_configuration << <<~OMNIBUS
                       geo_secondary['db_fdw'] = true;
                       geo_secondary_role['enable'] = true;
                       gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc';
@@ -58,7 +57,7 @@ module Gitlab
                       sidekiq['concurrency'] = 2;
                       puma['worker_processes'] = 2;
                     OMNIBUS
-                    secondary.exec_commands = fast_ssh_key_lookup_commands + git_lfs_install_commands
+                    secondary.exec_commands = fast_ssh_key_lookup_commands + QA::Scenario::CLICommands.git_lfs_install_commands
 
                     secondary.act do
                       # TODO, we do not wait for secondary to start because of

data/lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb

@@ -39,7 +39,7 @@ module Gitlab
                 praefect.network = @network
                 praefect.skip_availability_check = true
 
-                praefect.omnibus_config = praefect_omnibus_configuration
+                praefect.omnibus_configuration << praefect_omnibus_configuration
 
                 praefect.instance(skip_teardown: true)
               end
@@ -49,7 +49,7 @@ module Gitlab
                 gitlab.name = gitlab_name
                 gitlab.network = @network
 
-                gitlab.omnibus_config = gitlab_omnibus_configuration
+                gitlab.omnibus_configuration << gitlab_omnibus_configuration
                 gitlab.instance do
                   puts "Running Gitaly Cluster specs!"
 
@@ -131,8 +131,7 @@ module Gitlab
 
             def gitaly_omnibus_configuration
               <<~OMNIBUS
-                #{disable_other_services}
-                prometheus['enable'] = true;
+                #{disable_other_services.sub(/(prometheus\['enable'\]) = false/, '\1 = true')}
                 prometheus_monitoring['enable'] = false;
                 gitaly['enable'] = true;
                 gitaly['listen_addr'] = '0.0.0.0:8075';
@@ -197,7 +196,7 @@ module Gitlab
                 gitaly.name = name
                 gitaly.network = @network
                 gitaly.skip_availability_check = true
-                gitaly.omnibus_config = gitaly_omnibus_configuration
+                gitaly.omnibus_configuration << gitaly_omnibus_configuration
                 gitaly.instance(skip_teardown: true)
               end
             end

data/lib/gitlab/qa/scenario/test/integration/group_saml.rb

@@ -17,7 +17,7 @@ module Gitlab
             end
 
             def configure(gitlab, saml)
-              gitlab.omnibus_config = <<~OMNIBUS
+              gitlab.omnibus_configuration << <<~OMNIBUS
                 gitlab_rails['omniauth_enabled'] = true;
                 gitlab_rails['omniauth_providers'] = [{ name: 'group_saml' }];
               OMNIBUS

data/lib/gitlab/qa/scenario/test/integration/instance_saml.rb

@@ -17,7 +17,7 @@ module Gitlab
               saml.set_assertion_consumer_service("#{gitlab.address}/users/auth/saml/callback")
               saml.set_simple_saml_hostname
 
-              gitlab.omnibus_config = <<~OMNIBUS
+              gitlab.omnibus_configuration << <<~OMNIBUS
                 gitlab_rails['omniauth_enabled'] = true;
                 gitlab_rails['omniauth_allow_single_sign_on'] = ['saml'];
                 gitlab_rails['omniauth_block_auto_created_users'] = false;

data/lib/gitlab/qa/scenario/test/integration/kubernetes.rb

@@ -21,7 +21,7 @@ module Gitlab
                     tunnel_registry.gitlab_hostname = gitlab.hostname
                     tunnel_registry.network = 'test'
 
-                    gitlab.omnibus_config = <<~OMNIBUS
+                    gitlab.omnibus_configuration << <<~OMNIBUS
                       external_url '#{tunnel_gitlab.url}';
                       nginx['listen_port'] = 80;
                       nginx['listen_https'] = false;

data/lib/gitlab/qa/scenario/test/integration/ldap.rb

@@ -27,7 +27,7 @@ module Gitlab
             end
 
             def ldap_servers_omnibus_config
-              config = YAML.safe_load <<~CFG
+              YAML.safe_load <<~CFG
                 main:
                   label: LDAP
                   host: #{ldap_hostname}
@@ -44,9 +44,6 @@ module Gitlab
                   external_groups: ''
                   sync_ssh_keys: false
               CFG
-
-              # Quotes get eaten up when the string is set in the environment
-              config.to_s.gsub("\"", "\\\"")
             end
 
             def ldap_hostname

data/lib/gitlab/qa/scenario/test/integration/ldap_no_server.rb

@@ -12,7 +12,7 @@ module Gitlab
             end
 
             def configure_omnibus(gitlab)
-              gitlab.omnibus_config = <<~OMNIBUS
+              gitlab.omnibus_configuration << <<~OMNIBUS
                     gitlab_rails['ldap_enabled'] = true;
                     gitlab_rails['ldap_servers'] = #{ldap_servers_omnibus_config};
                     gitlab_rails['ldap_sync_worker_cron'] = '* * * * *';

data/lib/gitlab/qa/scenario/test/integration/ldap_no_tls.rb

@@ -15,7 +15,7 @@ module Gitlab
             end
 
             def configure_omnibus(gitlab)
-              gitlab.omnibus_config = <<~OMNIBUS
+              gitlab.omnibus_configuration << <<~OMNIBUS
                     gitlab_rails['ldap_enabled'] = true;
                     gitlab_rails['ldap_servers'] = #{ldap_servers_omnibus_config};
                     gitlab_rails['ldap_sync_worker_cron'] = '* * * * *';

data/lib/gitlab/qa/scenario/test/integration/ldap_tls.rb

@@ -15,8 +15,7 @@ module Gitlab
             end
 
             def configure_omnibus(gitlab)
-              gitlab.set_accept_insecure_certs
-              gitlab.omnibus_config = <<~OMNIBUS
+              gitlab.omnibus_configuration << <<~OMNIBUS
                     gitlab_rails['ldap_enabled'] = true;
                     gitlab_rails['ldap_servers'] = #{ldap_servers_omnibus_config};
                     letsencrypt['enable'] = false;

data/lib/gitlab/qa/scenario/test/integration/mattermost.rb

@@ -13,7 +13,7 @@ module Gitlab
                 mattermost_external_url = "http://#{mattermost_hostname}"
 
                 gitlab.add_network_alias(mattermost_hostname)
-                gitlab.omnibus_config = <<~OMNIBUS
+                gitlab.omnibus_configuration << <<~OMNIBUS
                   mattermost_external_url '#{mattermost_external_url}';
                 OMNIBUS
 

data/lib/gitlab/qa/scenario/test/integration/mtls.rb

@@ -21,9 +21,8 @@ module Gitlab
                 gitaly.network = @network
                 gitaly.skip_availability_check = true
 
-                gitaly.set_accept_insecure_certs
-                gitaly.omnibus_config = gitaly_omnibus
-                gitaly.mtls
+                gitaly.omnibus_configuration << gitaly_omnibus
+                gitaly.gitaly_tls
 
                 gitaly.instance do
                   Component::Gitlab.perform do |gitlab|
@@ -31,10 +30,8 @@ module Gitlab
                     gitlab.name = @gitlab_name
                     gitlab.network = @network
 
-                    gitlab.set_accept_insecure_certs
-                    gitlab.omnibus_config = gitlab_omnibus
+                    gitlab.omnibus_configuration << gitlab_omnibus
                     gitlab.tls = true
-                    gitlab.set_trusted_certificates
 
                     gitlab.instance do
                       puts "Running mTLS specs!"
@@ -50,7 +47,6 @@ module Gitlab
                         specs.network = gitlab.network
                         specs.args = [gitlab.address, *rspec_args]
                         specs.env = @env
-                        specs.mtls
                       end
                     end
                   end

data/lib/gitlab/qa/scenario/test/integration/smtp.rb

@@ -14,7 +14,7 @@ module Gitlab
             attr_reader :gitlab_name, :spec_suite
 
             def configure_omnibus(gitlab, mail_hog)
-              gitlab.omnibus_config = <<~OMNIBUS
+              gitlab.omnibus_configuration << <<~OMNIBUS
                     gitlab_rails['smtp_enable'] = true;
                     gitlab_rails['smtp_address'] = '#{mail_hog.hostname}';
                     gitlab_rails['smtp_port'] = 1025;