gitlab-qa 6.21.3 → 7.0.0

data/lib/gitlab/qa/component/minio.rb

@@ -1,5 +1,6 @@
 require 'securerandom'
 require 'fileutils'
+require 'yaml'
 
 # This component sets up the Minio (https://hub.docker.com/r/minio/minio)
 # image with the proper configuration for GitLab users to use object storage.
@@ -23,18 +24,12 @@ module Gitlab
           @buckets = []
         end
 
-        def instance
-          raise 'Please provide a block!' unless block_given?
-
-          super
-        end
-
         def add_bucket(name)
           @buckets << name
         end
 
         def to_config
-          config = YAML.safe_load <<~CFG
+          YAML.safe_load <<~CFG
             provider: AWS
             aws_access_key_id: #{AWS_ACCESS_KEY}
             aws_secret_access_key: #{AWS_SECRET_KEY}
@@ -43,9 +38,6 @@ module Gitlab
             endpoint: http://#{hostname}:#{port}
             path_style: true
           CFG
-
-          # Quotes get eaten up when the string is set in the environment
-          config.to_s.gsub('"', '\\"')
         end
 
         private

data/lib/gitlab/qa/component/specs.rb

@@ -10,9 +10,6 @@ module Gitlab
       class Specs < Scenario::Template
         attr_accessor :suite, :release, :network, :args, :volumes, :env, :runner_network
 
-        TRUSTED_CERTIFICATES_PATH = File.expand_path('../../../../tls_certificates/gitaly/trusted-certs'.freeze, __dir__)
-        TRUSTED_PATH = '/etc/ssl/certs'.freeze
-
         def initialize
           @docker = Docker::Engine.new
           @volumes = {}
@@ -26,7 +23,7 @@ module Gitlab
 
           @docker.login(**release.login_params) if release.login_params
 
-          @docker.pull(release.qa_image, release.qa_tag) unless Runtime::Env.skip_pull?
+          @docker.pull(qa_image) unless Runtime::Env.skip_pull?
 
           puts "Running test suite `#{suite}` for #{release.project_name}"
 
@@ -46,7 +43,7 @@ module Gitlab
           feature_flag_sets << [] unless feature_flag_sets.any?
 
           feature_flag_sets.each do |feature_flag_set|
-            @docker.run(release.qa_image, release.qa_tag, suite, *args_with_flags(args, feature_flag_set)) do |command|
+            @docker.run(qa_image, nil, suite, *args_with_flags(args, feature_flag_set)) do |command|
               command << "-t --rm --net=#{network || 'bridge'}"
 
               env.merge(Runtime::Env.variables).each do |key, value|
@@ -65,10 +62,6 @@ module Gitlab
           end
         end
 
-        def mtls
-          @volumes[TRUSTED_CERTIFICATES_PATH] = TRUSTED_PATH
-        end
-
         private
 
         def args_with_flags(args, feature_flag_set)
@@ -83,6 +76,14 @@ module Gitlab
         def skip_tests?
           Runtime::Scenario.attributes.include?(:run_tests) && !Runtime::Scenario.run_tests
         end
+
+        def qa_image
+          if Runtime::Scenario.attributes.include?(:qa_image)
+            Runtime::Scenario.qa_image
+          else
+            "#{release.qa_image}:#{release.qa_tag}"
+          end
+        end
       end
     end
   end

data/lib/gitlab/qa/docker/engine.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Gitlab
   module QA
     module Docker
@@ -13,15 +15,15 @@ module Gitlab
           Docker::Command.execute(%(login --username "#{username}" --password "#{password}" #{registry}), mask_secrets: password)
         end
 
-        def pull(image, tag)
-          Docker::Command.execute("pull #{image}:#{tag}")
+        def pull(image, tag = nil)
+          Docker::Command.execute("pull #{full_image_name(image, tag)}")
         end
 
         def run(image, tag, *args)
           Docker::Command.new('run').tap do |command|
             yield command if block_given?
 
-            command << "#{image}:#{tag}"
+            command << full_image_name(image, tag)
             command << args if args.any?
 
             command.execute!
@@ -36,14 +38,37 @@ module Gitlab
           false
         end
 
+        # Write to file(s) in the Docker container specified by @param name
+        # @param name The name of the Docker Container
+        # @example
+        #   engine.write_files('gitlab-abc123') do |files|
+        #     files.append('/etc/hosts', '127.0.0.1 localhost')
+        #     files.write('/opt/other', <<~TEXT
+        #       This is content
+        #       That goes within /opt/other
+        #     TEXT)
+        def write_files(name)
+          exec(name, yield(
+            Class.new do
+              def self.write(file, contents)
+                %(echo "#{contents}" > #{file};)
+              end
+
+              def self.append(file, contents)
+                %(echo "#{contents}" >> #{file};)
+              end
+            end
+          ))
+        end
+
         def exec(name, command)
           cmd = ['exec']
           cmd << '--privileged' if privileged_command?(command)
-          Docker::Command.execute("#{cmd.join(' ')} #{name} bash -c '#{command}'")
+          Docker::Command.execute(%(#{cmd.join(' ')} #{name} bash -c "#{command.gsub('"', '\\"')}"))
         end
 
         def read_file(image, tag, path, &block)
-          cat_file = "run --rm --entrypoint /bin/cat #{image}:#{tag} #{path}"
+          cat_file = "run --rm --entrypoint /bin/cat #{full_image_name(image, tag)} #{path}"
           Docker::Command.execute(cat_file, &block)
         end
 
@@ -94,6 +119,12 @@ module Gitlab
         def ps(name = nil)
           Docker::Command.execute(['ps', name].compact.join(' '))
         end
+
+        private
+
+        def full_image_name(image, tag)
+          [image, tag].compact.join(':')
+        end
       end
     end
   end

data/lib/gitlab/qa/report/test_result.rb

@@ -101,10 +101,14 @@ module Gitlab
               {
                 'message' => "#{exception['class']}: #{exception['message']}",
                 'message_lines' => exception['message_lines'],
-                'stacktrace' => "#{exception['message_lines'].join("\n")}\n#{exception['backtrace'].slice(0..spec_file_first_index).join("\n")}"
+                'stacktrace' => "#{format_message_lines(exception['message_lines'])}\n#{exception['backtrace'].slice(0..spec_file_first_index).join("\n")}"
               }
             end
           end
+
+          def format_message_lines(message_lines)
+            message_lines.is_a?(Array) ? message_lines.join("\n") : message_lines
+          end
           # rubocop:enable Metrics/AbcSize
         end
 

data/lib/gitlab/qa/runner.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 require 'optparse'
+require 'active_support/inflector'
 
 module Gitlab
   module QA
@@ -7,6 +10,12 @@ module Gitlab
       def self.run(args)
         Runtime::Scenario.define(:teardown, true)
         Runtime::Scenario.define(:run_tests, true)
+        Runtime::Scenario.define(:qa_image, Runtime::Env.qa_image) if Runtime::Env.qa_image
+        Runtime::Scenario.define(:omnibus_configuration, Runtime::OmnibusConfiguration.new)
+
+        # Omnibus Configurators specified by flags
+        @active_configurators = []
+        @omnibus_configurations = %w[default] # always load default configuration
 
         @options = OptionParser.new do |opts|
           opts.banner = 'Usage: gitlab-qa [options] Scenario URL [[--] path] [rspec_options]'
@@ -20,12 +29,22 @@ module Gitlab
             Runtime::Scenario.define(:teardown, false)
           end
 
+          opts.on('--qa-image QA_IMAGE', String, 'Specifies a QA image to be used instead of inferring it from the GitLab image. See Gitlab::QA::Release#qa_image') do |value|
+            Runtime::Scenario.define(:qa_image, value)
+          end
+
           opts.on_tail('-v', '--version', 'Show the version') do
             require 'gitlab/qa/version'
             puts "#{$PROGRAM_NAME} : #{VERSION}"
             exit
           end
 
+          opts.on('--omnibus-config config1[,config2,...]', 'Use Omnibus Configuration package') do |configuration|
+            configuration.split(',').map do |config|
+              @omnibus_configurations << config
+            end
+          end
+
           opts.on_tail('-h', '--help', 'Show the usage') do
             puts opts
             exit
@@ -39,21 +58,54 @@ module Gitlab
         end
 
         args.reject! { |arg| gitlab_qa_options.include?(arg) }
-
         if args.size >= 1
-          Scenario
-            .const_get(args.shift)
-            .perform(*args)
+          load_omnibus_configurations
+
+          begin
+            @active_configurators.compact.each do |configurator|
+              configurator.instance(skip_teardown: true)
+            end
+
+            Scenario
+              .const_get(args.shift)
+              .perform(*args)
+          ensure
+            @active_configurators.compact.each(&:teardown)
+          end
         else
           puts @options
           exit 1
         end
       end
-      # rubocop:enable Metrics/AbcSize
 
       def self.gitlab_qa_options
         @gitlab_qa_options ||= @options.top.list.map(&:long).flatten
       end
+
+      def self.load_omnibus_configurations
+        # OmnibusConfiguration::Test       => --test
+        # OmnibusConfiguration::HelloThere => --hello_there
+        @omnibus_configurations.uniq.each do |config|
+          Runtime::OmnibusConfigurations.const_get(config.camelize).new.tap do |configurator|
+            @active_configurators << configurator.prepare
+
+            # */etc/gitlab/gitlab.rb*
+            # -----------------------
+            # # Runtime::OmnibusConfiguration::Packages
+            # gitlab_rails['packages_enabled'] = true
+            Runtime::Scenario.omnibus_configuration << "# #{configurator.class.name}"
+
+            # Load the configuration
+            configurator.configuration.split("\n").each { |c| Runtime::Scenario.omnibus_configuration << c }
+          end
+        rescue NameError
+          raise <<~ERROR
+                Invalid Omnibus Configuration `#{config}`.
+                Possible configurations: #{Runtime::OmnibusConfigurations.constants.map { |c| c.to_s.underscore }.join(',')}"
+          ERROR
+        end
+      end
     end
+    # rubocop:enable Metrics/AbcSize
   end
 end

data/lib/gitlab/qa/runtime/env.rb

@@ -10,6 +10,7 @@ module Gitlab
         # These variables should be listed in /docs/what_tests_can_be_run.md#supported-gitlab-environment-variables
         # unless they're defined elsewhere (e.g.: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html)
         ENV_VARIABLES = {
+          'QA_IMAGE' => :qa_image,
           'QA_REMOTE_GRID' => :remote_grid,
           'QA_REMOTE_GRID_USERNAME' => :remote_grid_username,
           'QA_REMOTE_GRID_ACCESS_KEY' => :remote_grid_access_key,
@@ -64,6 +65,7 @@ module Gitlab
           'KNAPSACK_TEST_FILE_PATTERN' => :knapsack_test_file_pattern,
           'KNAPSACK_TEST_DIR' => :knapsack_test_dir,
           'CI' => :ci,
+          'CI_JOB_NAME' => :ci_job_name,
           'CI_JOB_URL' => :ci_job_url,
           'CI_RUNNER_ID' => :ci_runner_id,
           'CI_SERVER_HOST' => :ci_server_host,
@@ -106,6 +108,10 @@ module Gitlab
           ENV['QA_DEFAULT_BRANCH'] || 'master'
         end
 
+        def gitlab_availability_timeout
+          ENV.fetch('GITLAB_QA_AVAILABILITY_TIMEOUT', 360).to_i
+        end
+
         def gitlab_username
           ENV['GITLAB_USERNAME'] || 'gitlab-qa'
         end

data/lib/gitlab/qa/runtime/omnibus_configuration.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/object/blank'
+
+module Gitlab
+  module QA
+    module Runtime
+      class OmnibusConfiguration
+        def initialize
+          @config = ["# Generated by GitLab QA Omnibus Configurator at #{Time.now.strftime('%Y-%m-%d %H:%M:%S')}"]
+        end
+
+        def to_s
+          sanitize!.join("\n")
+        end
+
+        def configuration
+          raise NotImplementedError
+        end
+
+        # Before hook for any additional configuration
+        # This would usually be a container that needs to be running
+        # @return Any instance of [Gitlab::QA::Component::Base]
+        def prepare; end
+
+        # Commands to execute before GitLab boots
+        def exec_commands
+          []
+        end
+
+        # Ensures no duplicate entries
+        # @raise RuntimeError if competing configurations exist
+        # rubocop:disable Metrics/AbcSize
+        def sanitize!
+          sanitized = @config.map do |config|
+            next config if config.start_with?('#') # allow for comment
+
+            # sometimes "=" is part of a Hash. Only split based on the first "="
+            k, v = config.split("=", 2)
+            # make sure each config is well-formed
+            # e.g., gitlab_rails['packages_enabled'] = true
+            #   NOT gitlab_rails['packages_enabled']=true
+            "#{k.strip} = #{v.strip.tr('"', "'")}".strip
+          end.uniq
+
+          errors = []
+
+          # check for duplicates
+          duplicate_keys = []
+          duplicates = sanitized.reject do |n|
+            key = n.split('=').first
+            duplicate_keys << key unless duplicate_keys.include?(key)
+          end
+
+          duplicates.each { |duplicate| errors << "Duplicate entry found: `#{duplicate}`" }
+
+          raise "Errors exist within the Omnibus Configuration!\n#{errors.join(',')}" if errors.any?
+
+          @config = sanitized
+        end
+        # rubocop:enable Metrics/AbcSize
+
+        def <<(config)
+          @config << config.strip unless config.strip.empty?
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/runtime/omnibus_configurations/default.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Runtime
+      module OmnibusConfigurations
+        # Default Configuration for Omnibus
+        # All runs will include this configuration
+        class Default < Runtime::OmnibusConfiguration
+          def configuration
+            <<~OMNIBUS
+              gitlab_rails['gitlab_default_theme'] = 10 # Light Red Theme
+              gitlab_rails['gitlab_disable_animations'] = true # Disable animations
+              gitlab_rails['application_settings_cache_seconds'] = 0 # Settings cache expiry
+            OMNIBUS
+          end
+
+          def self.configuration
+            new.configuration
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/runtime/omnibus_configurations/object_storage.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Runtime
+      module OmnibusConfigurations
+        class ObjectStorage < Default
+          TYPES = %w[artifacts external_diffs lfs uploads packages dependency_proxy].freeze
+
+          def configuration
+            TYPES.each_with_object(+'') do |object_type, omnibus|
+              omnibus << <<~OMNIBUS
+                gitlab_rails['#{object_type}_enabled'] = true
+                gitlab_rails['#{object_type}_storage_path'] = '/var/opt/gitlab/gitlab-rails/shared/#{object_type}'
+                gitlab_rails['#{object_type}_object_store_enabled'] = true
+                gitlab_rails['#{object_type}_object_store_remote_directory'] = '#{object_type}-bucket'
+                gitlab_rails['#{object_type}_object_store_background_upload'] = false
+                gitlab_rails['#{object_type}_object_store_direct_upload'] = true
+                gitlab_rails['#{object_type}_object_store_proxy_download'] = true
+                gitlab_rails['#{object_type}_object_store_connection'] = #{minio.to_config}
+              OMNIBUS
+            end
+          end
+
+          def prepare
+            minio.network = 'test'
+
+            TYPES.each do |bucket_name|
+              minio.add_bucket("#{bucket_name}-bucket")
+            end
+
+            minio
+          end
+
+          def exec_commands
+            QA::Scenario::CLICommands.git_lfs_install_commands
+          end
+
+          private
+
+          def minio
+            @minio ||= Component::Minio.new
+          end
+        end
+      end
+    end
+  end
+end