gitlab-qa 5.14.1 → 6.0.0

data/lib/gitlab/qa/docker/command.rb

@@ -4,8 +4,9 @@ module Gitlab
       class Command
         attr_reader :args
 
-        def initialize(cmd = nil)
+        def initialize(cmd = nil, mask_secrets: nil)
           @args = Array(cmd)
+          @mask_secrets = Array(mask_secrets)
         end
 
         def <<(*args)
@@ -28,6 +29,17 @@ module Gitlab
           "docker #{@args.join(' ')}"
         end
 
+        # Returns a masked string form of a Command
+        #
+        # @example
+        #   Command.new('a docker command', mask_secrets: 'command').mask_secrets #=> 'a docker *****'
+        #   Command.new('a docker command', mask_secrets: %w[docker command]).mask_secrets #=> 'a ***** *****'
+        #
+        # @return [String] The masked command string
+        def mask_secrets
+          @mask_secrets.each_with_object(to_s) { |secret, s| s.gsub!(secret, '*****') }
+        end
+
         def ==(other)
           to_s == other.to_s
         end
@@ -36,8 +48,8 @@ module Gitlab
           Docker::Shellout.new(self).execute!(&block)
         end
 
-        def self.execute(cmd, &block)
-          new(cmd).execute!(&block)
+        def self.execute(cmd, mask_secrets: nil, &block)
+          new(cmd, mask_secrets: mask_secrets).execute!(&block)
         end
       end
     end

data/lib/gitlab/qa/docker/engine.rb

@@ -10,7 +10,7 @@ module Gitlab
         end
 
         def login(username:, password:, registry:)
-          Docker::Command.execute(%(login --username "#{username}" --password "#{password}" #{registry}))
+          Docker::Command.execute(%(login --username "#{username}" --password "#{password}" #{registry}), mask_secrets: password)
         end
 
         def pull(image, tag)
@@ -82,6 +82,10 @@ module Gitlab
         def running?(name)
           Docker::Command.execute("ps -f name=#{name}").include?(name)
         end
+
+        def ps(name = nil)
+          Docker::Command.execute(['ps', name].compact.join(' '))
+        end
       end
     end
   end

data/lib/gitlab/qa/docker/shellout.rb

@@ -10,7 +10,7 @@ module Gitlab
           @command = command
           @output = []
 
-          puts "Docker shell command: `#{@command}`"
+          puts "Docker shell command: `#{@command.mask_secrets}`"
         end
 
         def execute!
@@ -28,7 +28,7 @@ module Gitlab
             end
 
             if wait.value.exited? && wait.value.exitstatus.nonzero?
-              raise StatusError, "Docker command `#{@command}` failed!"
+              raise StatusError, "Docker command `#{@command.mask_secrets}` failed!"
             end
           end
 

data/lib/gitlab/qa/release.rb

@@ -136,6 +136,8 @@ module Gitlab
       end
 
       def login_params
+        return if Runtime::Env.skip_pull?
+
         if dev_gitlab_org?
           Runtime::Env.require_qa_dev_access_token!
 
@@ -145,14 +147,15 @@ module Gitlab
             registry: DEV_REGISTRY
           }
         elsif omnibus_mirror?
-          username, password = if Runtime::Env.ci_job_token
+          username, password = if Runtime::Env.ci_job_token && Runtime::Env.ci_pipeline_source == 'pipeline'
                                  ['gitlab-ci-token', Runtime::Env.ci_job_token]
+                               elsif Runtime::Env.qa_container_registry_access_token
+                                 [Runtime::Env.gitlab_username, Runtime::Env.qa_container_registry_access_token]
                                else
                                  Runtime::Env.require_qa_access_token!
 
                                  [Runtime::Env.gitlab_username, Runtime::Env.qa_access_token]
                                end
-
           {
             username: username,
             password: password,
@@ -173,6 +176,10 @@ module Gitlab
         canonical? || release.match?(CUSTOM_GITLAB_IMAGE_REGEX)
       end
 
+      def api_project_name
+        project_name.gsub('ce', 'foss').gsub('-ee', '')
+      end
+
       private
 
       def canonical?

data/lib/gitlab/qa/report/json_test_results.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Gitlab
+  module QA
+    module Report
+      class JsonTestResults
+        include Enumerable
+
+        def initialize(file)
+          @testcases = JSON.parse(File.read(file))['examples'].map { |test| TestResult.from_json(test) }
+        end
+
+        def each(&block)
+          @testcases.each(&block)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/report/junit_test_results.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+
+module Gitlab
+  module QA
+    module Report
+      class JUnitTestResults
+        include Enumerable
+
+        def initialize(file)
+          @testcases = Nokogiri::XML(File.read(file)).xpath('//testcase').map { |test| TestResult.from_junit(test) }
+        end
+
+        def each(&block)
+          @testcases.each(&block)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/report/results_in_issues.rb

@@ -30,11 +30,12 @@ module Gitlab
         RETRY_BACK_OFF_DELAY = 60
         MAX_RETRY_ATTEMPTS = 3
 
-        def initialize(token:, input_files:, project: nil)
+        def initialize(token:, input_files:, project: nil, input_format: :junit)
           @token = token
           @files = Array(input_files)
           @project = project
           @retry_backoff = 0
+          @input_format = input_format.to_sym
         end
 
         def invoke!
@@ -46,7 +47,14 @@ module Gitlab
 
           Dir.glob(files).each do |file|
             puts "Reporting tests in #{file}"
-            Nokogiri::XML(File.open(file)).xpath('//testcase').each do |test|
+            case input_format
+            when :json
+              test_results = Report::JsonTestResults.new(file)
+            when :junit
+              test_results = Report::JUnitTestResults.new(file)
+            end
+
+            test_results.each do |test|
               report_test(test)
             end
           end
@@ -54,7 +62,7 @@ module Gitlab
 
         private
 
-        attr_reader :files, :token, :project
+        attr_reader :files, :token, :project, :input_format
 
         def validate_input!
           assert_project!
@@ -99,9 +107,9 @@ module Gitlab
         end
 
         def report_test(test)
-          return if test.search('skipped').any?
+          return if test.skipped
 
-          puts "Reporting test: #{test['file']} | #{test['name']}"
+          puts "Reporting test: #{test.file} | #{test.name}"
 
           issue = find_issue(test)
           if issue
@@ -124,7 +132,7 @@ module Gitlab
             Gitlab.create_issue(
               project,
               title_from_test(test),
-              { description: "### Full description\n\n#{search_safe(test['name'])}\n\n### File path\n\n#{test['file']}", labels: 'status::automated' }
+              { description: "### Full description\n\n#{search_safe(test.name)}\n\n### File path\n\n#{test.file}", labels: 'status::automated' }
             )
           end
         end
@@ -135,18 +143,18 @@ module Gitlab
               .auto_paginate
               .select { |issue| issue.state == 'opened' && issue.title.strip == title_from_test(test) }
 
-            warn(%(Too many issues found with the file path "#{test['file']}" and name "#{test['name']}")) if issues.many?
+            warn(%(Too many issues found with the file path "#{test.file}" and name "#{test.name}")) if issues.many?
 
             issues.first
           end
         end
 
         def search_term(test)
-          %("#{test['file']}" "#{search_safe(test['name'])}")
+          %("#{test.file}" "#{search_safe(test.name)}")
         end
 
         def title_from_test(test)
-          title = "#{partial_file_path(test['file'])} | #{search_safe(test['name'])}".strip
+          title = "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
 
           return title unless title.length > MAX_TITLE_LENGTH
 
@@ -162,7 +170,7 @@ module Gitlab
         end
 
         def note_status(issue, test)
-          return if failures(test).empty?
+          return if test.failures.empty?
 
           note = note_content(test)
 
@@ -176,7 +184,7 @@ module Gitlab
         end
 
         def note_content(test)
-          errors = failures(test).each_with_object([]) do |failure, text|
+          errors = test.failures.each_with_object([]) do |failure, text|
             text << <<~TEXT
               Error:
               ```
@@ -185,7 +193,7 @@ module Gitlab
 
               Stacktrace:
               ```
-              #{failure.content}
+              #{failure['stacktrace']}
               ```
             TEXT
           end.join("\n\n")
@@ -231,7 +239,7 @@ module Gitlab
         def update_labels(issue, test)
           labels = issue.labels
           labels.delete_if { |label| label.start_with?("#{pipeline}::") }
-          labels << (failures(test).empty? ? "#{pipeline}::passed" : "#{pipeline}::failed")
+          labels << (test.failures.empty? ? "#{pipeline}::passed" : "#{pipeline}::failed")
           labels << "Enterprise Edition" if ee_test?(test)
           quarantine_job? ? labels << "quarantine" : labels.delete("quarantine")
 
@@ -242,11 +250,7 @@ module Gitlab
         # rubocop:enable Metrics/AbcSize
 
         def ee_test?(test)
-          test['file'] =~ %r{features/ee/(api|browser_ui)}
-        end
-
-        def failures(test)
-          test.search('failure')
+          test.file =~ %r{features/ee/(api|browser_ui)}
         end
 
         def pipeline

data/lib/gitlab/qa/report/test_result.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Report
+      class TestResult
+        attr_accessor :name, :file, :skipped, :failures
+
+        def self.from_json(test)
+          new.tap do |test_result|
+            test_result.name = test['full_description']
+            test_result.file = test['file_path']
+            test_result.skipped = test['status'] == 'pending'
+            test_result.failures = failures_from_json_exceptions(test)
+          end
+        end
+
+        def self.from_junit(test)
+          new.tap do |test_result|
+            test_result.name = test['name']
+            test_result.file = test['file']
+            test_result.skipped = test.search('skipped').any?
+            test_result.failures = failures_from_junit_exceptions(test)
+          end
+        end
+
+        def self.failures_from_json_exceptions(test)
+          return [] unless test.key?('exceptions')
+
+          test['exceptions'].map do |exception|
+            {
+              'message' => "#{exception['class']}: #{exception['message']}",
+              'stacktrace' => exception['backtrace'].join('\n')
+            }
+          end
+        end
+        private_class_method :failures_from_json_exceptions
+
+        def self.failures_from_junit_exceptions(test)
+          failures = test.search('failure')
+          return [] if failures.empty?
+
+          failures.map do |exception|
+            {
+              'message' => "#{exception['type']}: #{exception['message']}",
+              'stacktrace' => exception.content
+            }
+          end
+        end
+        private_class_method :failures_from_junit_exceptions
+      end
+    end
+  end
+end

data/lib/gitlab/qa/runner.rb

@@ -4,22 +4,20 @@ module Gitlab
   module QA
     # rubocop:disable Metrics/AbcSize
     class Runner
-      # These options are implemented in the QA framework (i.e., in the CE/EE codebase)
-      # They're included here so that gitlab-qa treats them as valid options
-      PASS_THROUGH_OPTS = [
-        ['--address URL', 'Address of the instance to test'],
-        ['--enable-feature FEATURE_FLAG', 'Enable a feature before running tests'],
-        ['--mattermost-address URL', 'Address of the Mattermost server'],
-        ['--parallel', 'Execute tests in parallel'],
-        ['--loop', 'Execute tests in a loop']
-      ].freeze
-
       def self.run(args)
-        options = OptionParser.new do |opts|
+        Runtime::Scenario.define(:teardown, true)
+        Runtime::Scenario.define(:run_tests, true)
+
+        @options = OptionParser.new do |opts|
           opts.banner = 'Usage: gitlab-qa [options] Scenario URL [[--] path] [rspec_options]'
 
-          PASS_THROUGH_OPTS.each do |opt|
-            opts.on(*opt)
+          opts.on('--no-teardown', 'Skip teardown of containers after the scenario completes.') do
+            Runtime::Scenario.define(:teardown, false)
+          end
+
+          opts.on('--no-tests', 'Orchestrates the docker containers but does not run the tests. Implies --no-teardown') do
+            Runtime::Scenario.define(:run_tests, false)
+            Runtime::Scenario.define(:teardown, false)
           end
 
           opts.on_tail('-v', '--version', 'Show the version') do
@@ -33,19 +31,29 @@ module Gitlab
             exit
           end
 
-          opts.parse(args)
+          begin
+            opts.parse(args)
+          rescue OptionParser::InvalidOption
+            # Ignore invalid options and options that are passed through to the tests
+          end
         end
 
+        args.reject! { |arg| gitlab_qa_options.include?(arg) }
+
         if args.size >= 1
           Scenario
             .const_get(args.shift)
             .perform(*args)
         else
-          puts options
+          puts @options
           exit 1
         end
       end
       # rubocop:enable Metrics/AbcSize
+
+      def self.gitlab_qa_options
+        @gitlab_qa_options ||= @options.top.list.map(&:long).flatten
+      end
     end
   end
 end

data/lib/gitlab/qa/runtime/env.rb

@@ -6,6 +6,8 @@ module Gitlab
       module Env
         extend self
 
+        # Variables that are used in tests and are passed through to the docker container that executes the tests.
+        # These variables should be listed in /docs/what_tests_can_be_run.md#supported-gitlab-environment-variables
         ENV_VARIABLES = {
           'QA_REMOTE_GRID' => :remote_grid,
           'QA_REMOTE_GRID_USERNAME' => :remote_grid_username,
@@ -38,6 +40,7 @@ module Gitlab
           'SIGNUP_DISABLED' => :signup_disabled,
           'QA_ADDITIONAL_REPOSITORY_STORAGE' => :qa_additional_repository_storage,
           'QA_PRAEFECT_REPOSITORY_STORAGE' => :qa_praefect_repository_storage,
+          'QA_GITALY_NON_CLUSTER_STORAGE' => :qa_gitaly_non_cluster_storage,
           'QA_COOKIES' => :qa_cookie,
           'QA_DEBUG' => :qa_debug,
           'QA_LOG_PATH' => :qa_log_path,
@@ -112,6 +115,10 @@ module Gitlab
           ENV['CI_JOB_URL']
         end
 
+        def ci_pipeline_source
+          ENV['CI_PIPELINE_SOURCE']
+        end
+
         def ci_project_name
           ENV['CI_PROJECT_NAME']
         end
@@ -148,6 +155,10 @@ module Gitlab
           ENV['GITLAB_QA_DEV_ACCESS_TOKEN']
         end
 
+        def qa_container_registry_access_token
+          ENV['GITLAB_QA_CONTAINER_REGISTRY_ACCESS_TOKEN']
+        end
+
         def host_artifacts_dir
           @host_artifacts_dir ||= File.join(ENV['QA_ARTIFACTS_DIR'] || '/tmp/gitlab-qa', Runtime::Env.run_id)
         end

data/lib/gitlab/qa/scenario/test/instance/geo.rb

@@ -8,9 +8,6 @@ module Gitlab
 
           class Geo < Scenario::Template
             def perform(release, primary_address, secondary_address, *rspec_args)
-              # Geo requires an EE license
-              Runtime::Env.require_license!
-
               Component::Specs.perform do |specs|
                 specs.suite = 'QA::EE::Scenario::Test::Geo'
                 specs.release = QA::Release.new(release)