gitlab-qa 6.23.0 → 7.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be38f6268c3a91ecdeee390c040f658e07194eb1e3d5f99694d50c68540a5461
-  data.tar.gz: bb9fd66f54e18792a9d5a8bb445fefe7cc63d824ef445b61b60f2c7ef378422f
+  metadata.gz: ce83d37d177260e577ff1ffbc8e606eab4bc449953d8fc9de669404371433014
+  data.tar.gz: ef2a8ba114e9799c4d8b4cc71f3c098ac708b79e8573486305b9802d6bdf1826
 SHA512:
-  metadata.gz: 54f0d06a51b27c5e4ea0c1352ca01f634d8e7bc00475345c87650beaf783e7f4dcc4cc53828d54e75b01ada167f002c1a51451147afac400a27ae9132008ee30
-  data.tar.gz: a651ecb24eb49d6e3997c5c6abfef1048900f760d81374e35b9104b05b25210f94c0ecf4de65e5e79c710ca7552153da7aab36e765b5e1acd9c829cf6feb36c1
+  metadata.gz: 22d88835fee7158e9e5829e5d5e415abfa85d157148514c21eb904eecf69a8e8c4a49dc6e3aaae0e35824fc8f3a2be2d0c983f2f8057c915bcd0332a2fbea41f
+  data.tar.gz: 6e8007813c6258741ba516b011b8a5fc7efa1ee1ada5c385580477a18b02c35ac8e232370a80395fd3259f98058abaaffe11941c7ec9f8108a19dd36aafa4df3

data/.gitlab-ci.yml

@@ -18,7 +18,8 @@ default:
       - vendor/ruby
   before_script:
     - bundle version
-    - bundle install --jobs=$(nproc) --path=vendor --retry=3 --quiet && bundle check
+    - bundle config path vendor
+    - bundle install --jobs=$(nproc) --retry=3 --quiet && bundle check
     - if [ -n "$TRIGGERED_USER" ] && [ -n "$TRIGGER_SOURCE" ]; then
         echo "Pipeline triggered by $TRIGGERED_USER at $TRIGGER_SOURCE";
       fi

data/docs/configuring_omnibus.md

@@ -0,0 +1,208 @@
+# Configuring Omnibus
+
+Omnibus will, by default, contain the configuration specified in `Gitlab::QA::Runtime::OmnibusConfigurations::Default`.
+
+Omnibus can be configured from two places:
+
+1. Within a custom `Scenario::Template`.
+1. Represented as a `Gitlab::QA::Runtime::OmnibusConfiguration` class.
+
+## Adding an Omnibus Configurator
+
+All configurators are held within `Gitlab::QA::Runtime::OmnibusConfigurations` and represented as separate classes.
+
+Notes:
+
+- If it is required that more than one GitLab instance is configured, 
+  you may skip adding an Omnibus Configurator.  In this case, it should be handled by a separate `Scenario::Template`.
+  An example of this would be a Geo Primary and Secondary.
+- All classes should be a type of `Runtime::OmnibusConfiguration`.
+
+### Add the Configurator Class
+
+Create a new file in `lib/gitlab/qa/runtime/omnibus_configurations` called `registry.rb`.
+
+```ruby
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Runtime
+      module OmnibusConfigurations
+        class Registry < Default
+          def configuration
+            <<~OMNIBUS
+              gitlab_rails['registry_enabled'] = true
+            OMNIBUS
+          end
+        end
+      end
+    end
+  end
+end
+```
+
+Notes:
+
+- Refrain from adding unnecessary statement terminations (`;`).
+
+### Prepare Sidecar Container
+
+If the tests require an additional container to be spun up adjacent to GitLab, you may override the `prepare` method.
+
+```ruby
+#=> lib/gitlab/qa/runtime/omnibus_configurations/registry.rb
+
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Runtime
+      module OmnibusConfigurations
+        class Registry < Default
+          def configuration
+            <<~OMNIBUS
+              gitlab_rails['registry_enabled'] = true
+            OMNIBUS
+          end
+
+          def prepare
+            Component::Sidecar.new
+          end
+        end
+      end
+    end
+  end
+end
+```
+
+Notes:
+
+- `prepare` **must** return an instance of `Component::Base`.
+- The sidecar container will be prepared before the test runs, and will run the tests after it is running.
+- Extending the class from `Default` is only a nicety that adds a singleton method called `Registry.configuration` and returns the configuration.
+  It is recommended to extend from `Default`, but you may also extend from any other Omnibus Configuration class, including `Runtime::OmnibusConfiguration`.
+
+## Running tests with Omnibus Configured
+
+All Omnibus Configurators can be called by passing arguments into the `gitlab-qa` executable.
+
+```shell
+exe/gitlab-qa Test::Instance::Image EE --omnibus-config registry
+```
+
+Notes:
+
+- `--omnibus-config registry` must match the name of the Omnibus Configurator Class name (`Runtime::OmnibusConfigurations::Registry`), but lowercase.
+- If the Configurator Class name contains several words, the argument will be named the same, but snake cased. E.g. `--omnibus-config some_class` matches `SomeClass`, `--omnibus-config some_other_class` matches `SomeOtherClass`.
+- The Omnibus GitLab Instance will have the configuration from `Default` and `Registry` (in that order) put into `/etc/gitlab/gitlab.rb` and GitLab QA will proceed to run the tests.
+- If a specified Omnibus Configuration does not exist, GitLab QA will raise an error and fail immediately.
+
+## Further reading
+
+### Multiple Configurators
+
+Multiple Configurators may be specified and the order will be preserved in which the arguments were passed.
+
+E.g., given the arguments:
+
+```
+exe/gitlab-qa Test::Instance::Image EE --omnibus-config packages,registry
+# or
+exe/gitlab-qa Test::Instance::Image EE --omnibus-config packages --omnibus-config registry
+```
+
+Omnibus will be configured in the order they appear.
+
+```ruby
+# /etc/gitlab/gitlab.rb
+#=> Runtime::OmnibusConfiguration::Default#configuration
+#=> Runtime::OmnibusConfiguration::Packages#configuration
+#=> Runtime::OmnibusConfiguration::Registry#configuration
+```
+
+The order is also preserved for Sidecar containers.  If the `Packages` and `Registry` Configurators each prepare a sidecar container, they will be spun up in order from first to last.
+
+### Adding one-off configurations
+
+#### From a new Scenario::Template
+
+If it is required to create a new `Scenario::Template`, you may add new Configurations to the respective GitLab Instances by invoking `omnibus_configuration#<<`
+
+```ruby
+# Geo example
+
+Component::Gitlab.perform do |primary|
+  primary.omnibus_configuration << <<~OMNIBUS
+    geo_primary_role['enable'] = true
+  OMNIBUS
+  
+  primary.instance do
+    Component::Gitlab.perform do |secondary|
+      secondary.omnibus_configuration << <<~OMNIBUS
+        geo_secondary_role['enable'] = true
+      OMNIBUS
+    end
+  end
+end
+```
+
+Notes:
+
+- The `primary` instance will be configured using the `Runtime::OmnibusConfigurations::Default` configuration, *then* `geo_primary_role['enable'] = true` will be affixed afterwards.
+- The `secondary` instance will be configured using the `Runtime::OmnibusConfigurations::Default` configuration, *then* `geo_secondary_role['enable'] = true` will be affixed afterwards.
+
+#### From Component::Gitlab
+
+Any additional one-off configurations needing to be added may be directly appended to `@omnibus_configuration` as such:
+
+```ruby
+disable_animations = true
+
+@omnibus_configuration << "gitlab_rails['gitlab_disable_animations'] = true" if disable_animations
+```
+
+This will add the specified configuration **after** what has already been specified beforehand (Configurators or Default configurations).
+
+Note:
+
+- If there is no issue appending this configuration to the **rest** of the GitLab Instances that might be spun up, you can add this
+  to the global Omnibus Configuration. E.g., use `Runtime::Scenario.omnibus_configuration << ...` instead of `@omnibus_configuration << ...`
+
+#### Difference between Runtime::Scenario.omnibus_configuration and Gitlab#omnibus_configuration
+
+Generally, while running GitLab QA, only one GitLab instance is necessary. `Runtime::Scenario.omnibus_configuration` contains all of the global
+Omnibus configurations required for this one environment.  This also contains the configuration for any other GitLab instance.
+
+When multiple GitLab Instances are required, `@omnibus_configuration#<<` is preferred over `Runtime::Scenario.omnibus_configuration#<<` since the First Instance might require
+one Omnibus configuration that might be unneccesary or Invalid for the Second Instance.
+
+#### Load order and precedence
+
+1. `Runtime::OmnibusConfigurations::Default`
+1. `Runtime::OmnibusConfigurations::[A,B,...]` where `A` and `B` are Configurators specified through the positional arguments `--a --b`
+1. Custom written `Scenario::Template` (such as `Test::Integation::Geo`)
+1. `lib/gitlab/qa/component/gitlab.rb`
+
+From top to bottom, configurations will be loaded and any configurations that are superceded, will take precedence over the one before it, and so on.
+
+### Executing arbitrary shell commands within the GitLab Instance
+
+Sometimes it's necessary to execute arbitrary commands within the GitLab instance before the tests start.
+
+You may specify these commands by overriding the `exec_commands` method within the Configurator.
+
+```ruby
+class Registry < Default
+  def configuration
+    # configuration
+  end
+  
+  def exec_commands
+    [
+      'cp /etc/gitlab/gitlab.rb /etc/gitlab/gitlab.bak.rb',
+      'rm /etc/gitlab/gitlab.bak.rb'
+    ]
+  end
+end
+```

data/gitlab-qa.gemspec

@@ -31,5 +31,5 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'gitlab', '~> 4.16.1'
   spec.add_runtime_dependency 'http', '4.3.0'
   spec.add_runtime_dependency 'nokogiri', '~> 1.10'
-  spec.add_runtime_dependency 'table_print', '1.5.6'
+  spec.add_runtime_dependency 'table_print', '1.5.7'
 end

data/lib/gitlab/qa.rb

@@ -8,6 +8,13 @@ module Gitlab
       autoload :Env, 'gitlab/qa/runtime/env'
       autoload :Scenario, 'gitlab/qa/runtime/scenario'
       autoload :TokenFinder, 'gitlab/qa/runtime/token_finder'
+      autoload :OmnibusConfiguration, 'gitlab/qa/runtime/omnibus_configuration'
+
+      module OmnibusConfigurations
+        autoload :Default, 'gitlab/qa/runtime/omnibus_configurations/default'
+        autoload :Packages, 'gitlab/qa/runtime/omnibus_configurations/packages'
+        autoload :ObjectStorage, 'gitlab/qa/runtime/omnibus_configurations/object_storage'
+      end
     end
 
     module Scenario

data/lib/gitlab/qa/component/gitlab.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'securerandom'
 require 'net/http'
 require 'uri'
@@ -10,39 +12,36 @@ module Gitlab
       class Gitlab < Base
         extend Forwardable
 
-        attr_reader :release
-        attr_accessor :tls, :disable_animations, :skip_availability_check, :runner_network
+        attr_reader :release, :omnibus_configuration
+        attr_accessor :tls, :skip_availability_check, :runner_network
         attr_writer :name, :relative_path
 
         def_delegators :release, :tag, :image, :edition
 
-        AUTHORITY_CERTIFICATES_PATH = File.expand_path('../../../../tls_certificates/authority'.freeze, __dir__)
-        GITLAB_CERTIFICATES_PATH = File.expand_path('../../../../tls_certificates/gitlab'.freeze, __dir__)
-        GITALY_CERTIFICATES_PATH = File.expand_path('../../../../tls_certificates/gitaly'.freeze, __dir__)
+        AUTHORITY_CERTIFICATES_PATH = File.expand_path('../../../../tls_certificates/authority', __dir__)
+        GITLAB_CERTIFICATES_PATH = File.expand_path('../../../../tls_certificates/gitlab', __dir__)
+        GITALY_CERTIFICATES_PATH = File.expand_path('../../../../tls_certificates/gitaly', __dir__)
 
-        SSL_PATH = '/etc/gitlab/ssl'.freeze
-        TRUSTED_PATH = '/etc/gitlab/trusted-certs'.freeze
+        SSL_PATH = '/etc/gitlab/ssl'
+        TRUSTED_PATH = '/etc/gitlab/trusted-certs'
 
         def initialize
           super
 
-          @disable_animations = true
           @skip_availability_check = false
 
           @volumes[GITLAB_CERTIFICATES_PATH] = SSL_PATH
           @volumes[AUTHORITY_CERTIFICATES_PATH] = TRUSTED_PATH
 
-          self.release = 'CE'
-        end
+          @omnibus_configuration ||= OmnibusConfiguration.new
 
-        def omnibus_config=(config)
-          @environment['GITLAB_OMNIBUS_CONFIG'] = config.tr("\n", ' ')
+          self.release = 'CE'
         end
 
         def set_formless_login_token
           return if Runtime::Env.gitlab_qa_formless_login_token.to_s.strip.empty?
 
-          @environment['GITLAB_OMNIBUS_CONFIG'] = "gitlab_rails['env'] = { 'GITLAB_QA_FORMLESS_LOGIN_TOKEN' => '#{Runtime::Env.gitlab_qa_formless_login_token}' }; #{@environment['GITLAB_OMNIBUS_CONFIG'] || ''}"
+          @omnibus_configuration << "gitlab_rails['env'] = { 'GITLAB_QA_FORMLESS_LOGIN_TOKEN' => '#{Runtime::Env.gitlab_qa_formless_login_token}' }"
         end
 
         def elastic_url=(url)
@@ -95,17 +94,7 @@ module Gitlab
         end
 
         def prepare_gitlab_omnibus_config
-          setup_disable_animations if disable_animations
           set_formless_login_token
-          setup_application_settings_cache_expiry
-        end
-
-        def setup_disable_animations
-          @environment['GITLAB_OMNIBUS_CONFIG'] = "gitlab_rails['gitlab_disable_animations'] = true; #{@environment['GITLAB_OMNIBUS_CONFIG'] || ''}"
-        end
-
-        def setup_application_settings_cache_expiry
-          @environment['GITLAB_OMNIBUS_CONFIG'] = "gitlab_rails['application_settings_cache_seconds'] = 0; #{@environment['GITLAB_OMNIBUS_CONFIG'] || ''}"
         end
 
         def start # rubocop:disable Metrics/AbcSize
@@ -135,6 +124,8 @@ module Gitlab
         end
 
         def reconfigure
+          setup_omnibus
+
           @docker.attach(name) do |line, wait|
             puts line
             # TODO, workaround which allows to detach from the container
@@ -170,6 +161,12 @@ module Gitlab
           raise 'Please configure an instance first!' unless [name, release, network].all?
         end
 
+        def setup_omnibus
+          @docker.write_files(name) do |f|
+            f.write('/etc/gitlab/gitlab.rb', @omnibus_configuration.to_s)
+          end
+        end
+
         class Availability
           def initialize(name, relative_path: '', scheme: 'http', protocol_port: 80)
             @docker = Docker::Engine.new
@@ -209,6 +206,20 @@ module Gitlab
             @uri.scheme == 'https' ? { use_ssl: true, verify_mode: OpenSSL::SSL::VERIFY_NONE } : {}
           end
         end
+
+        class OmnibusConfiguration
+          def initialize
+            @config = Runtime::Scenario.attributes[:omnibus_configuration] || []
+          end
+
+          def <<(configuration)
+            @config << configuration
+          end
+
+          def to_s
+            @config.to_s
+          end
+        end
       end
     end
   end

data/lib/gitlab/qa/component/minio.rb

@@ -1,5 +1,6 @@
 require 'securerandom'
 require 'fileutils'
+require 'yaml'
 
 # This component sets up the Minio (https://hub.docker.com/r/minio/minio)
 # image with the proper configuration for GitLab users to use object storage.
@@ -23,18 +24,12 @@ module Gitlab
           @buckets = []
         end
 
-        def instance
-          raise 'Please provide a block!' unless block_given?
-
-          super
-        end
-
         def add_bucket(name)
           @buckets << name
         end
 
         def to_config
-          config = YAML.safe_load <<~CFG
+          YAML.safe_load <<~CFG
             provider: AWS
             aws_access_key_id: #{AWS_ACCESS_KEY}
             aws_secret_access_key: #{AWS_SECRET_KEY}
@@ -43,9 +38,6 @@ module Gitlab
             endpoint: http://#{hostname}:#{port}
             path_style: true
           CFG
-
-          # Quotes get eaten up when the string is set in the environment
-          config.to_s.gsub('"', '\\"')
         end
 
         private

data/lib/gitlab/qa/docker/engine.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module QA
     module Docker
@@ -36,10 +38,33 @@ module Gitlab
           false
         end
 
+        # Write to file(s) in the Docker container specified by @param name
+        # @param name The name of the Docker Container
+        # @example
+        #   engine.write_files('gitlab-abc123') do |files|
+        #     files.append('/etc/hosts', '127.0.0.1 localhost')
+        #     files.write('/opt/other', <<~TEXT
+        #       This is content
+        #       That goes within /opt/other
+        #     TEXT)
+        def write_files(name)
+          exec(name, yield(
+            Class.new do
+              def self.write(file, contents)
+                %(echo "#{contents}" > #{file};)
+              end
+
+              def self.append(file, contents)
+                %(echo "#{contents}" >> #{file};)
+              end
+            end
+          ))
+        end
+
         def exec(name, command)
           cmd = ['exec']
           cmd << '--privileged' if privileged_command?(command)
-          Docker::Command.execute("#{cmd.join(' ')} #{name} bash -c '#{command}'")
+          Docker::Command.execute(%(#{cmd.join(' ')} #{name} bash -c "#{command.gsub('"', '\\"')}"))
         end
 
         def read_file(image, tag, path, &block)

data/lib/gitlab/qa/runner.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 require 'optparse'
+require 'active_support/inflector'
 
 module Gitlab
   module QA
@@ -8,6 +11,11 @@ module Gitlab
         Runtime::Scenario.define(:teardown, true)
         Runtime::Scenario.define(:run_tests, true)
         Runtime::Scenario.define(:qa_image, Runtime::Env.qa_image) if Runtime::Env.qa_image
+        Runtime::Scenario.define(:omnibus_configuration, Runtime::OmnibusConfiguration.new)
+
+        # Omnibus Configurators specified by flags
+        @active_configurators = []
+        @omnibus_configurations = %w[default] # always load default configuration
 
         @options = OptionParser.new do |opts|
           opts.banner = 'Usage: gitlab-qa [options] Scenario URL [[--] path] [rspec_options]'
@@ -31,6 +39,12 @@ module Gitlab
             exit
           end
 
+          opts.on('--omnibus-config config1[,config2,...]', 'Use Omnibus Configuration package') do |configuration|
+            configuration.split(',').map do |config|
+              @omnibus_configurations << config
+            end
+          end
+
           opts.on_tail('-h', '--help', 'Show the usage') do
             puts opts
             exit
@@ -44,21 +58,54 @@ module Gitlab
         end
 
         args.reject! { |arg| gitlab_qa_options.include?(arg) }
-
         if args.size >= 1
-          Scenario
-            .const_get(args.shift)
-            .perform(*args)
+          load_omnibus_configurations
+
+          begin
+            @active_configurators.compact.each do |configurator|
+              configurator.instance(skip_teardown: true)
+            end
+
+            Scenario
+              .const_get(args.shift)
+              .perform(*args)
+          ensure
+            @active_configurators.compact.each(&:teardown)
+          end
         else
           puts @options
           exit 1
         end
       end
-      # rubocop:enable Metrics/AbcSize
 
       def self.gitlab_qa_options
         @gitlab_qa_options ||= @options.top.list.map(&:long).flatten
       end
+
+      def self.load_omnibus_configurations
+        # OmnibusConfiguration::Test       => --test
+        # OmnibusConfiguration::HelloThere => --hello_there
+        @omnibus_configurations.uniq.each do |config|
+          Runtime::OmnibusConfigurations.const_get(config.camelize).new.tap do |configurator|
+            @active_configurators << configurator.prepare
+
+            # */etc/gitlab/gitlab.rb*
+            # -----------------------
+            # # Runtime::OmnibusConfiguration::Packages
+            # gitlab_rails['packages_enabled'] = true
+            Runtime::Scenario.omnibus_configuration << "# #{configurator.class.name}"
+
+            # Load the configuration
+            configurator.configuration.split("\n").each { |c| Runtime::Scenario.omnibus_configuration << c }
+          end
+        rescue NameError
+          raise <<~ERROR
+                Invalid Omnibus Configuration `#{config}`.
+                Possible configurations: #{Runtime::OmnibusConfigurations.constants.map { |c| c.to_s.underscore }.join(',')}"
+          ERROR
+        end
+      end
     end
+    # rubocop:enable Metrics/AbcSize
   end
 end

data/lib/gitlab/qa/runtime/env.rb

@@ -65,6 +65,7 @@ module Gitlab
           'KNAPSACK_TEST_FILE_PATTERN' => :knapsack_test_file_pattern,
           'KNAPSACK_TEST_DIR' => :knapsack_test_dir,
           'CI' => :ci,
+          'CI_JOB_NAME' => :ci_job_name,
           'CI_JOB_URL' => :ci_job_url,
           'CI_RUNNER_ID' => :ci_runner_id,
           'CI_SERVER_HOST' => :ci_server_host,

data/lib/gitlab/qa/runtime/omnibus_configuration.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/object/blank'
+
+module Gitlab
+  module QA
+    module Runtime
+      class OmnibusConfiguration
+        def initialize
+          @config = ["# Generated by GitLab QA Omnibus Configurator at #{Time.now.strftime('%Y-%m-%d %H:%M:%S')}"]
+        end
+
+        def to_s
+          sanitize!.join("\n")
+        end
+
+        def configuration
+          raise NotImplementedError
+        end
+
+        # Before hook for any additional configuration
+        # This would usually be a container that needs to be running
+        # @return Any instance of [Gitlab::QA::Component::Base]
+        def prepare; end
+
+        # Commands to execute before GitLab boots
+        def exec_commands
+          []
+        end
+
+        # Ensures no duplicate entries
+        # @raise RuntimeError if competing configurations exist
+        # rubocop:disable Metrics/AbcSize
+        def sanitize!
+          sanitized = @config.map do |config|
+            next config if config.start_with?('#') # allow for comment
+
+            # sometimes "=" is part of a Hash. Only split based on the first "="
+            k, v = config.split("=", 2)
+            # make sure each config is well-formed
+            # e.g., gitlab_rails['packages_enabled'] = true
+            #   NOT gitlab_rails['packages_enabled']=true
+            "#{k.strip} = #{v.strip.tr('"', "'")}".strip
+          end.uniq
+
+          errors = []
+
+          # check for duplicates
+          duplicate_keys = []
+          duplicates = sanitized.reject do |n|
+            key = n.split('=').first
+            duplicate_keys << key unless duplicate_keys.include?(key)
+          end
+
+          duplicates.each { |duplicate| errors << "Duplicate entry found: `#{duplicate}`" }
+
+          raise "Errors exist within the Omnibus Configuration!\n#{errors.join(',')}" if errors.any?
+
+          @config = sanitized
+        end
+        # rubocop:enable Metrics/AbcSize
+
+        def <<(config)
+          @config << config.strip unless config.strip.empty?
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/runtime/omnibus_configurations/default.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Runtime
+      module OmnibusConfigurations
+        # Default Configuration for Omnibus
+        # All runs will include this configuration
+        class Default < Runtime::OmnibusConfiguration
+          def configuration
+            <<~OMNIBUS
+              gitlab_rails['gitlab_default_theme'] = 10 # Light Red Theme
+              gitlab_rails['gitlab_disable_animations'] = true # Disable animations
+              gitlab_rails['application_settings_cache_seconds'] = 0 # Settings cache expiry
+            OMNIBUS
+          end
+
+          def self.configuration
+            new.configuration
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/runtime/omnibus_configurations/object_storage.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Runtime
+      module OmnibusConfigurations
+        class ObjectStorage < Default
+          TYPES = %w[artifacts external_diffs lfs uploads packages dependency_proxy].freeze
+
+          def configuration
+            TYPES.each_with_object(+'') do |object_type, omnibus|
+              omnibus << <<~OMNIBUS
+                gitlab_rails['#{object_type}_enabled'] = true
+                gitlab_rails['#{object_type}_storage_path'] = '/var/opt/gitlab/gitlab-rails/shared/#{object_type}'
+                gitlab_rails['#{object_type}_object_store_enabled'] = true
+                gitlab_rails['#{object_type}_object_store_remote_directory'] = '#{object_type}-bucket'
+                gitlab_rails['#{object_type}_object_store_background_upload'] = false
+                gitlab_rails['#{object_type}_object_store_direct_upload'] = true
+                gitlab_rails['#{object_type}_object_store_proxy_download'] = true
+                gitlab_rails['#{object_type}_object_store_connection'] = #{minio.to_config}
+              OMNIBUS
+            end
+          end
+
+          def prepare
+            minio.network = 'test'
+
+            TYPES.each do |bucket_name|
+              minio.add_bucket("#{bucket_name}-bucket")
+            end
+
+            minio
+          end
+
+          def exec_commands
+            QA::Scenario::CLICommands.git_lfs_install_commands
+          end
+
+          private
+
+          def minio
+            @minio ||= Component::Minio.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/runtime/omnibus_configurations/packages.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Runtime
+      module OmnibusConfigurations
+        class Packages < Default
+          def configuration
+            <<~OMNIBUS
+              gitlab_rails['packages_enabled'] = true
+            OMNIBUS
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/scenario/cli_commands.rb

@@ -4,11 +4,11 @@ module Gitlab
       module CLICommands
         GIT_LFS_VERSION = '2.8.0'.freeze
 
-        def git_lfs_install_commands
-          @git_lfs_install_commands ||= [
+        def self.git_lfs_install_commands
+          [
             "cd /tmp ; curl -qsL https://github.com/git-lfs/git-lfs/releases/download/v#{GIT_LFS_VERSION}/git-lfs-linux-amd64-v#{GIT_LFS_VERSION}.tar.gz | tar xzvf -",
             'cp /tmp/git-lfs /usr/local/bin'
-          ]
+          ].freeze
         end
       end
     end

data/lib/gitlab/qa/scenario/test/instance/relative_url.rb

@@ -10,9 +10,7 @@ module Gitlab
                 gitlab.network = 'test'
                 gitlab.relative_path = '/relative'
 
-                gitlab.omnibus_config = <<~OMNIBUS
-                  external_url '#{gitlab.address}';
-                OMNIBUS
+                gitlab.omnibus_configuration << "external_url '#{gitlab.address}'"
 
                 gitlab.instance do
                   Component::Specs.perform do |specs|

data/lib/gitlab/qa/scenario/test/instance/repository_storage.rb

@@ -9,7 +9,7 @@ module Gitlab
                 gitlab.release = QA::Release.new(release)
                 gitlab.name = 'gitlab'
                 gitlab.network = 'test'
-                gitlab.omnibus_config = <<~OMNIBUS
+                gitlab.omnibus_configuration << <<~OMNIBUS
                   git_data_dirs({
                     'default' => {
                       'path' => '/var/opt/gitlab/git-data/repositories/default'

data/lib/gitlab/qa/scenario/test/integration/actioncable.rb

@@ -9,9 +9,7 @@ module Gitlab
                 gitlab.release = QA::Release.new(release)
                 gitlab.name = 'gitlab-actioncable'
                 gitlab.network = 'test'
-                gitlab.omnibus_config = <<~OMNIBUS
-                  actioncable['enable'] = true;
-                OMNIBUS
+                gitlab.omnibus_configuration << "actioncable['enable'] = true"
 
                 gitlab.instance do
                   puts "Running actioncable specs!"

data/lib/gitlab/qa/scenario/test/integration/geo.rb

@@ -4,7 +4,6 @@ module Gitlab
       module Test
         module Integration
           class Geo < Scenario::Template
-            include Scenario::CLICommands
             ##
             # rubocop:disable Lint/MissingCopEnableDirective
             # rubocop:disable Metrics/AbcSize
@@ -20,7 +19,7 @@ module Gitlab
                 primary.release = release
                 primary.name = 'gitlab-primary'
                 primary.network = 'geo'
-                primary.omnibus_config = <<~OMNIBUS
+                primary.omnibus_configuration << <<~OMNIBUS
                   gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc';
                   geo_primary_role['enable'] = true;
                   gitlab_rails['db_password'] = 'mypass';
@@ -36,14 +35,14 @@ module Gitlab
                   sidekiq['concurrency'] = 2;
                   puma['worker_processes'] = 2;
                 OMNIBUS
-                primary.exec_commands = fast_ssh_key_lookup_commands + git_lfs_install_commands
+                primary.exec_commands = fast_ssh_key_lookup_commands + QA::Scenario::CLICommands.git_lfs_install_commands
 
                 primary.instance do
                   Component::Gitlab.perform do |secondary|
                     secondary.release = release
                     secondary.name = 'gitlab-secondary'
                     secondary.network = 'geo'
-                    secondary.omnibus_config = <<~OMNIBUS
+                    secondary.omnibus_configuration << <<~OMNIBUS
                       geo_secondary['db_fdw'] = true;
                       geo_secondary_role['enable'] = true;
                       gitlab_rails['db_key_base'] = '4dd58204865eb41bca93bd38131d51cc';
@@ -58,7 +57,7 @@ module Gitlab
                       sidekiq['concurrency'] = 2;
                       puma['worker_processes'] = 2;
                     OMNIBUS
-                    secondary.exec_commands = fast_ssh_key_lookup_commands + git_lfs_install_commands
+                    secondary.exec_commands = fast_ssh_key_lookup_commands + QA::Scenario::CLICommands.git_lfs_install_commands
 
                     secondary.act do
                       # TODO, we do not wait for secondary to start because of

data/lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb

@@ -39,7 +39,7 @@ module Gitlab
                 praefect.network = @network
                 praefect.skip_availability_check = true
 
-                praefect.omnibus_config = praefect_omnibus_configuration
+                praefect.omnibus_configuration << praefect_omnibus_configuration
 
                 praefect.instance(skip_teardown: true)
               end
@@ -49,7 +49,7 @@ module Gitlab
                 gitlab.name = gitlab_name
                 gitlab.network = @network
 
-                gitlab.omnibus_config = gitlab_omnibus_configuration
+                gitlab.omnibus_configuration << gitlab_omnibus_configuration
                 gitlab.instance do
                   puts "Running Gitaly Cluster specs!"
 
@@ -197,7 +197,7 @@ module Gitlab
                 gitaly.name = name
                 gitaly.network = @network
                 gitaly.skip_availability_check = true
-                gitaly.omnibus_config = gitaly_omnibus_configuration
+                gitaly.omnibus_configuration << gitaly_omnibus_configuration
                 gitaly.instance(skip_teardown: true)
               end
             end

data/lib/gitlab/qa/scenario/test/integration/group_saml.rb

@@ -17,7 +17,7 @@ module Gitlab
             end
 
             def configure(gitlab, saml)
-              gitlab.omnibus_config = <<~OMNIBUS
+              gitlab.omnibus_configuration << <<~OMNIBUS
                 gitlab_rails['omniauth_enabled'] = true;
                 gitlab_rails['omniauth_providers'] = [{ name: 'group_saml' }];
               OMNIBUS

data/lib/gitlab/qa/scenario/test/integration/instance_saml.rb

@@ -17,7 +17,7 @@ module Gitlab
               saml.set_assertion_consumer_service("#{gitlab.address}/users/auth/saml/callback")
               saml.set_simple_saml_hostname
 
-              gitlab.omnibus_config = <<~OMNIBUS
+              gitlab.omnibus_configuration << <<~OMNIBUS
                 gitlab_rails['omniauth_enabled'] = true;
                 gitlab_rails['omniauth_allow_single_sign_on'] = ['saml'];
                 gitlab_rails['omniauth_block_auto_created_users'] = false;

data/lib/gitlab/qa/scenario/test/integration/kubernetes.rb

@@ -21,7 +21,7 @@ module Gitlab
                     tunnel_registry.gitlab_hostname = gitlab.hostname
                     tunnel_registry.network = 'test'
 
-                    gitlab.omnibus_config = <<~OMNIBUS
+                    gitlab.omnibus_configuration << <<~OMNIBUS
                       external_url '#{tunnel_gitlab.url}';
                       nginx['listen_port'] = 80;
                       nginx['listen_https'] = false;

data/lib/gitlab/qa/scenario/test/integration/ldap_no_server.rb

@@ -12,7 +12,7 @@ module Gitlab
             end
 
             def configure_omnibus(gitlab)
-              gitlab.omnibus_config = <<~OMNIBUS
+              gitlab.omnibus_configuration << <<~OMNIBUS
                     gitlab_rails['ldap_enabled'] = true;
                     gitlab_rails['ldap_servers'] = #{ldap_servers_omnibus_config};
                     gitlab_rails['ldap_sync_worker_cron'] = '* * * * *';

data/lib/gitlab/qa/scenario/test/integration/ldap_no_tls.rb

@@ -15,7 +15,7 @@ module Gitlab
             end
 
             def configure_omnibus(gitlab)
-              gitlab.omnibus_config = <<~OMNIBUS
+              gitlab.omnibus_configuration << <<~OMNIBUS
                     gitlab_rails['ldap_enabled'] = true;
                     gitlab_rails['ldap_servers'] = #{ldap_servers_omnibus_config};
                     gitlab_rails['ldap_sync_worker_cron'] = '* * * * *';

data/lib/gitlab/qa/scenario/test/integration/ldap_tls.rb

@@ -15,7 +15,7 @@ module Gitlab
             end
 
             def configure_omnibus(gitlab)
-              gitlab.omnibus_config = <<~OMNIBUS
+              gitlab.omnibus_configuration << <<~OMNIBUS
                     gitlab_rails['ldap_enabled'] = true;
                     gitlab_rails['ldap_servers'] = #{ldap_servers_omnibus_config};
                     letsencrypt['enable'] = false;

data/lib/gitlab/qa/scenario/test/integration/mattermost.rb

@@ -13,7 +13,7 @@ module Gitlab
                 mattermost_external_url = "http://#{mattermost_hostname}"
 
                 gitlab.add_network_alias(mattermost_hostname)
-                gitlab.omnibus_config = <<~OMNIBUS
+                gitlab.omnibus_configuration << <<~OMNIBUS
                   mattermost_external_url '#{mattermost_external_url}';
                 OMNIBUS
 

data/lib/gitlab/qa/scenario/test/integration/mtls.rb

@@ -21,7 +21,7 @@ module Gitlab
                 gitaly.network = @network
                 gitaly.skip_availability_check = true
 
-                gitaly.omnibus_config = gitaly_omnibus
+                gitaly.omnibus_configuration << gitaly_omnibus
                 gitaly.gitaly_tls
 
                 gitaly.instance do
@@ -30,7 +30,7 @@ module Gitlab
                     gitlab.name = @gitlab_name
                     gitlab.network = @network
 
-                    gitlab.omnibus_config = gitlab_omnibus
+                    gitlab.omnibus_configuration << gitlab_omnibus
                     gitlab.tls = true
 
                     gitlab.instance do

data/lib/gitlab/qa/scenario/test/integration/smtp.rb

@@ -14,7 +14,7 @@ module Gitlab
             attr_reader :gitlab_name, :spec_suite
 
             def configure_omnibus(gitlab, mail_hog)
-              gitlab.omnibus_config = <<~OMNIBUS
+              gitlab.omnibus_configuration << <<~OMNIBUS
                     gitlab_rails['smtp_enable'] = true;
                     gitlab_rails['smtp_address'] = '#{mail_hog.hostname}';
                     gitlab_rails['smtp_port'] = 1025;

data/lib/gitlab/qa/scenario/test/integration/ssh_tunnel.rb

@@ -22,7 +22,7 @@ module Gitlab
                     tunnel_registry.gitlab_hostname = gitlab.hostname
                     tunnel_registry.network = 'test'
 
-                    gitlab.omnibus_config = <<~OMNIBUS
+                    gitlab.omnibus_configuration << <<~OMNIBUS
                       external_url '#{tunnel_gitlab.url}';
                       nginx['listen_port'] = 80;
                       nginx['listen_https'] = false;

data/lib/gitlab/qa/version.rb

@@ -1,5 +1,5 @@
 module Gitlab
   module QA
-    VERSION = '6.23.0'.freeze
+    VERSION = '7.0.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 6.23.0
+  version: 7.0.0
 platform: ruby
 authors:
 - Grzegorz Bizon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-03-16 00:00:00.000000000 Z
+date: 2021-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -184,14 +184,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.5.6
+        version: 1.5.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.5.6
+        version: 1.5.7
 description: 
 email:
 - grzesiek.bizon@gmail.com
@@ -221,6 +221,7 @@ files:
 - bin/slack
 - docs/README.md
 - docs/architecture.md
+- docs/configuring_omnibus.md
 - docs/how_it_works.md
 - docs/release_process.md
 - docs/run_qa_against_gdk.md
@@ -273,6 +274,10 @@ files:
 - lib/gitlab/qa/reporter.rb
 - lib/gitlab/qa/runner.rb
 - lib/gitlab/qa/runtime/env.rb
+- lib/gitlab/qa/runtime/omnibus_configuration.rb
+- lib/gitlab/qa/runtime/omnibus_configurations/default.rb
+- lib/gitlab/qa/runtime/omnibus_configurations/object_storage.rb
+- lib/gitlab/qa/runtime/omnibus_configurations/packages.rb
 - lib/gitlab/qa/runtime/scenario.rb
 - lib/gitlab/qa/runtime/token_finder.rb
 - lib/gitlab/qa/scenario/actable.rb
@@ -305,8 +310,6 @@ files:
 - lib/gitlab/qa/scenario/test/integration/ldap_tls.rb
 - lib/gitlab/qa/scenario/test/integration/mattermost.rb
 - lib/gitlab/qa/scenario/test/integration/mtls.rb
-- lib/gitlab/qa/scenario/test/integration/object_storage.rb
-- lib/gitlab/qa/scenario/test/integration/packages.rb
 - lib/gitlab/qa/scenario/test/integration/praefect.rb
 - lib/gitlab/qa/scenario/test/integration/saml.rb
 - lib/gitlab/qa/scenario/test/integration/smtp.rb

data/lib/gitlab/qa/scenario/test/integration/object_storage.rb

@@ -1,64 +0,0 @@
-require 'yaml'
-
-module Gitlab
-  module QA
-    module Scenario
-      module Test
-        module Integration
-          class ObjectStorage < Scenario::Template
-            include Scenario::CLICommands
-
-            TYPES = %w[artifacts external_diffs lfs uploads packages dependency_proxy].freeze
-
-            def perform(release, *rspec_args)
-              Component::Gitlab.perform do |gitlab|
-                gitlab.release = release
-                gitlab.name = 'gitlab-object-storage'
-                gitlab.network = 'test'
-
-                Component::Minio.perform do |minio|
-                  minio.network = 'test'
-
-                  TYPES.each do |bucket_name|
-                    minio.add_bucket("#{bucket_name}-bucket")
-                  end
-
-                  gitlab.omnibus_config = object_storage_config(minio)
-                  gitlab.exec_commands = git_lfs_install_commands
-
-                  minio.instance do
-                    gitlab.instance do
-                      puts 'Running object storage specs!'
-
-                      Component::Specs.perform do |specs|
-                        specs.suite = 'Test::Integration::ObjectStorage'
-                        specs.release = gitlab.release
-                        specs.network = gitlab.network
-                        specs.args = [gitlab.address, *rspec_args]
-                      end
-                    end
-                  end
-                end
-              end
-            end
-
-            def object_storage_config(minio)
-              TYPES.map do |object_type|
-                <<~OMNIBUS
-                  gitlab_rails['#{object_type}_enabled'] = true;
-                  gitlab_rails['#{object_type}_storage_path'] = '/var/opt/gitlab/gitlab-rails/shared/#{object_type}';
-                  gitlab_rails['#{object_type}_object_store_enabled'] = true;
-                  gitlab_rails['#{object_type}_object_store_remote_directory'] = '#{object_type}-bucket';
-                  gitlab_rails['#{object_type}_object_store_background_upload'] = false;
-                  gitlab_rails['#{object_type}_object_store_direct_upload'] = true;
-                  gitlab_rails['#{object_type}_object_store_proxy_download'] = true;
-                  gitlab_rails['#{object_type}_object_store_connection'] = #{minio.to_config};
-                OMNIBUS
-              end.join("\n")
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/gitlab/qa/scenario/test/integration/packages.rb

@@ -1,36 +0,0 @@
-module Gitlab
-  module QA
-    module Scenario
-      module Test
-        module Integration
-          class Packages < Scenario::Template
-            def perform(release, *rspec_args)
-              Component::Gitlab.perform do |gitlab|
-                gitlab.release = QA::Release.new(release)
-                gitlab.name = 'gitlab-packages'
-                gitlab.network = 'test'
-                gitlab.omnibus_config = <<~OMNIBUS
-                  gitlab_rails['packages_enabled'] = true;
-                OMNIBUS
-
-                gitlab.instance do
-                  puts "Running packages specs!"
-
-                  rspec_args << "--" unless rspec_args.include?('--')
-                  rspec_args << %w[--tag packages]
-
-                  Component::Specs.perform do |specs|
-                    specs.suite = 'Test::Instance::All'
-                    specs.release = gitlab.release
-                    specs.network = gitlab.network
-                    specs.args = [gitlab.address, *rspec_args]
-                  end
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end