gitlab-qa 6.1.3 → 6.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52046fa3ef635b05a1b01e3c4d4ec0b3b0cad5abdbdcc52b2d54e07f6b0e19fe
-  data.tar.gz: 6107e5c74d71e9ef3d9c423168676ce877b1745de1e8ba7d7570a2ac001b9c57
+  metadata.gz: 97ec064795e04d091b6d04bff27d6c163635a1a08d1e9b28efedf17111f2b1a6
+  data.tar.gz: 8318fb6919671c0af482d9dda37f751d15d60732b6654eca890fd87118f9f8d3
 SHA512:
-  metadata.gz: d3161f3b19e78dffc3baa9baf1b104d925eaf5d3cd091ab28b58d9718ffc3fc5432d53b58b8b2559c0a46a78ac2d852ee28f99a8d5117f882e0cbf8a1e4b5e7c
-  data.tar.gz: 89cad7516fedeb377c8720a00a49553778d6cd6f0075471fd6f44c53bfec6714dbf3a510f5bd3239ae1082f5cca006a911d2758744d885b38e4bfc025ea747f8
+  metadata.gz: dcc298d8f343ee158a4e1229748a96853555082f20de96ed3b5851e413fdc47a832eebd9d0f27f8aba959aacdcbd28c0409beea9abbf8ca7a8feeb18cc066171
+  data.tar.gz: 8bdf4528158b4f5c5f09bb87f4604dc3eda02c73d98a064714375f24f0e245c636fefb5a6e5b2f7c93db8d46053b5ebeb7eb3821fd17ae8c04eb459f8c0154fd

data/.gitlab-ci.yml

@@ -229,44 +229,6 @@ ee:instance-quarantine:
   variables:
     QA_RSPEC_TAGS: "--tag quarantine --tag ~orchestrated"
 
-ce:docker:
-  extends:
-    - .test
-    - .high-capacity
-    - .ce-qa
-    - .rspec-report-opts
-  variables:
-    QA_RSPEC_TAGS: "--tag docker"
-
-ce:docker-quarantine:
-  extends:
-    - .test
-    - .high-capacity
-    - .ce-qa
-    - .quarantine
-    - .rspec-report-opts
-  variables:
-    QA_RSPEC_TAGS: "--tag docker --tag quarantine"
-
-ee:docker:
-  extends:
-    - .test
-    - .high-capacity
-    - .ee-qa
-    - .rspec-report-opts
-  variables:
-    QA_RSPEC_TAGS: "--tag docker"
-
-ee:docker-quarantine:
-  extends:
-    - .test
-    - .high-capacity
-    - .ee-qa
-    - .quarantine
-    - .rspec-report-opts
-  variables:
-    QA_RSPEC_TAGS: "--tag docker --tag quarantine"
-
 ce:relative_url:
   extends:
     - .test

data/.gitlab/merge_request_templates/Release.md

@@ -32,4 +32,4 @@ with the latest commit from https://gitlab.com/gitlab-org/gitlab-qa/commits/mast
 - Checklist after merging:
   - [ ] [Create a tag for the new release version](docs/release_process.md#how-to).
 
-/label ~Quality ~backstage
+/label ~Quality ~"feature::maintenance"

data/docs/what_tests_can_be_run.md

@@ -83,6 +83,7 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `JIRA_ADMIN_USERNAME` |-  | Username for authenticating with Jira server as admin. | No|
 | `JIRA_ADMIN_PASSWORD` |-  | Password for authenticating with Jira server as admin. | No|
 | `CACHE_NAMESPACE_NAME` | `true` | Cache namespace name for groups. | No|
+| `DEPLOY_VERSION` |- | The version of GitLab being tested against. | No|
 
 ## [Supported Remote Grid environment variables](./running_against_remote_grid.md)
 

data/lib/gitlab/qa.rb

@@ -97,9 +97,12 @@ module Gitlab
     end
 
     module Report
+      autoload :GitlabIssueClient, 'gitlab/qa/report/gitlab_issue_client'
+      autoload :BaseTestResults, 'gitlab/qa/report/base_test_results'
       autoload :JsonTestResults, 'gitlab/qa/report/json_test_results'
       autoload :JUnitTestResults, 'gitlab/qa/report/junit_test_results'
       autoload :PrepareStageReports, 'gitlab/qa/report/prepare_stage_reports'
+      autoload :ReportAsIssue, 'gitlab/qa/report/report_as_issue'
       autoload :ResultsInIssues, 'gitlab/qa/report/results_in_issues'
       autoload :SummaryTable, 'gitlab/qa/report/summary_table'
       autoload :TestResult, 'gitlab/qa/report/test_result'

data/lib/gitlab/qa/component/gitlab.rb

@@ -128,10 +128,10 @@ module Gitlab
           end
         end
 
-        def wait
+        def wait_until_ready
           return if skip_availability_check
 
-          if Availability.new(name, relative_path: relative_path, scheme: scheme, protocol_port: port.to_i).check(180)
+          if Availability.new(name, relative_path: relative_path, scheme: scheme, protocol_port: port.to_i).check(300)
             sleep 12 # TODO, handle that better
             puts ' -> GitLab is available.'
           else

data/lib/gitlab/qa/component/postgresql.rb

@@ -30,7 +30,8 @@ module Gitlab
           begin
             run_psql 'template1'
           rescue StandardError
-            retry if Time.now - start < 30
+            sleep 5
+            retry if Time.now - start < 60
             raise
           end
         end

data/lib/gitlab/qa/component/specs.rb

@@ -23,6 +23,8 @@ module Gitlab
 
           @docker.login(**release.login_params) if release.login_params
 
+          @docker.pull(release.qa_image, release.qa_tag) unless Runtime::Env.skip_pull?
+
           puts "Running test suite `#{suite}` for #{release.project_name}"
 
           name = "#{release.project_name}-qa-#{SecureRandom.hex(4)}"

data/lib/gitlab/qa/report/base_test_results.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Report
+      class BaseTestResults
+        include Enumerable
+
+        def initialize(path)
+          @results = parse(path)
+          @testcases = process
+        end
+
+        def each(&block)
+          testcases.each(&block)
+        end
+
+        def write(path)
+          raise NotImplementedError
+        end
+
+        private
+
+        attr_reader :results, :testcases
+
+        def parse(path)
+          raise NotImplementedError
+        end
+
+        def process
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/report/gitlab_issue_client.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require 'gitlab'
+
+module Gitlab
+  # Monkey patch the Gitlab client to use the correct API path and add required methods
+  class Client
+    def team_member(project, id)
+      get("/projects/#{url_encode(project)}/members/all/#{id}")
+    end
+
+    def issue_discussions(project, issue_id, options = {})
+      get("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions", query: options)
+    end
+
+    def add_note_to_issue_discussion_as_thread(project, issue_id, discussion_id, options = {})
+      post("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions/#{discussion_id}/notes", query: options)
+    end
+  end
+
+  module QA
+    module Report
+      # The GitLab client is used for API access: https://github.com/NARKOZ/gitlab
+      class GitlabIssueClient
+        MAINTAINER_ACCESS_LEVEL = 40
+        RETRY_BACK_OFF_DELAY = 60
+        MAX_RETRY_ATTEMPTS = 3
+
+        def initialize(token:, project:)
+          @token = token
+          @project = project
+          @retry_backoff = 0
+
+          configure_gitlab_client
+        end
+
+        def assert_user_permission!
+          handle_gitlab_client_exceptions do
+            user = Gitlab.user
+            member = Gitlab.team_member(project, user.id)
+
+            abort_not_permitted if member.access_level < MAINTAINER_ACCESS_LEVEL
+          end
+        rescue Gitlab::Error::NotFound
+          abort_not_permitted
+        end
+
+        def find_issues(iid:, options: {}, &select)
+          select ||= :itself
+
+          handle_gitlab_client_exceptions do
+            return [Gitlab.issue(project, iid)] if iid
+
+            Gitlab.issues(project, options)
+              .auto_paginate
+              .select(&select)
+          end
+        end
+
+        def create_issue(title:, description:, labels:)
+          puts "Creating issue..."
+
+          handle_gitlab_client_exceptions do
+            Gitlab.create_issue(
+              project,
+              title,
+              { description: description, labels: labels }
+            )
+          end
+        end
+
+        def edit_issue(iid:, options: {})
+          handle_gitlab_client_exceptions do
+            Gitlab.edit_issue(project, iid, options)
+          end
+        end
+
+        def handle_gitlab_client_exceptions
+          yield
+        rescue Gitlab::Error::NotFound
+          # This error could be raised in assert_user_permission!
+          # If so, we want it to terminate at that point
+          raise
+        rescue SystemCallError, OpenSSL::SSL::SSLError, Net::OpenTimeout, Net::ReadTimeout, Gitlab::Error::InternalServerError, Gitlab::Error::Parsing => e
+          @retry_backoff += RETRY_BACK_OFF_DELAY
+
+          raise if @retry_backoff > RETRY_BACK_OFF_DELAY * MAX_RETRY_ATTEMPTS
+
+          warn_exception(e)
+          warn("Sleeping for #{@retry_backoff} seconds before retrying...")
+          sleep @retry_backoff
+
+          retry
+        rescue StandardError => e
+          pipeline = QA::Runtime::Env.pipeline_from_project_name
+          channel = pipeline == "canary" ? "qa-production" : "qa-#{pipeline}"
+          error_msg = warn_exception(e)
+          slack_options = {
+            channel: channel,
+            icon_emoji: ':ci_failing:',
+            message: <<~MSG
+              An unexpected error occurred while reporting test results in issues.
+              The error occurred in job: #{QA::Runtime::Env.ci_job_url}
+              `#{error_msg}`
+            MSG
+          }
+          puts "Posting Slack message to channel: #{channel}"
+
+          Gitlab::QA::Slack::PostToSlack.new(**slack_options).invoke!
+        end
+
+        private
+
+        attr_reader :token, :project
+
+        def configure_gitlab_client
+          Gitlab.configure do |config|
+            config.endpoint = Runtime::Env.gitlab_api_base
+            config.private_token = token
+          end
+        end
+
+        def abort_not_permitted
+          abort "You must have at least Maintainer access to the project to use this feature."
+        end
+
+        def warn_exception(error)
+          error_msg = "#{error.class.name} #{error.message}"
+          warn(error_msg)
+          error_msg
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/report/json_test_results.rb

@@ -5,15 +5,23 @@ require 'json'
 module Gitlab
   module QA
     module Report
-      class JsonTestResults
-        include Enumerable
+      class JsonTestResults < BaseTestResults
+        def write(path)
+          json = results.merge('examples' => testcases.map(&:report))
 
-        def initialize(file)
-          @testcases = JSON.parse(File.read(file))['examples'].map { |test| TestResult.from_json(test) }
+          File.write(path, JSON.pretty_generate(json))
         end
 
-        def each(&block)
-          @testcases.each(&block)
+        private
+
+        def parse(path)
+          JSON.parse(File.read(path))
+        end
+
+        def process
+          results['examples'].map do |test|
+            TestResult.from_json(test)
+          end
         end
       end
     end

data/lib/gitlab/qa/report/junit_test_results.rb

@@ -5,15 +5,21 @@ require 'nokogiri'
 module Gitlab
   module QA
     module Report
-      class JUnitTestResults
-        include Enumerable
+      class JUnitTestResults < BaseTestResults
+        def write(path)
+          # Ignore it for now
+        end
+
+        private
 
-        def initialize(file)
-          @testcases = Nokogiri::XML(File.read(file)).xpath('//testcase').map { |test| TestResult.from_junit(test) }
+        def parse(path)
+          Nokogiri::XML.parse(File.read(path))
         end
 
-        def each(&block)
-          @testcases.each(&block)
+        def process
+          results.xpath('//testcase').map do |test|
+            TestResult.from_junit(test)
+          end
         end
       end
     end

data/lib/gitlab/qa/report/report_as_issue.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Report
+      class ReportAsIssue
+        def initialize(token:, input_files:, project: nil)
+          @gitlab = GitlabIssueClient.new(token: token, project: project)
+          @files = Array(input_files)
+          @project = project
+        end
+
+        def invoke!
+          validate_input!
+
+          run!
+        end
+
+        private
+
+        attr_reader :gitlab, :files, :project
+
+        def run!
+          raise NotImplementedError
+        end
+
+        def validate_input!
+          assert_project!
+          assert_input_files!(files)
+          gitlab.assert_user_permission!
+        end
+
+        def assert_project!
+          return if project
+
+          abort "Please provide a valid project ID or path with the `-p/--project` option!"
+        end
+
+        def assert_input_files!(files)
+          return if Dir.glob(files).any?
+
+          abort "Please provide valid JUnit report files. No files were found matching `#{files.join(',')}`"
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/report/results_in_issues.rb

@@ -1,58 +1,29 @@
 # frozen_string_literal: true
 
 require 'nokogiri'
-require 'gitlab'
 require 'active_support/core_ext/enumerable'
 
 module Gitlab
-  # Monkey patch the Gitlab client to use the correct API path and add required methods
-  class Client
-    def team_member(project, id)
-      get("/projects/#{url_encode(project)}/members/all/#{id}")
-    end
-
-    def issue_discussions(project, issue_id, options = {})
-      get("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions", query: options)
-    end
-
-    def add_note_to_issue_discussion_as_thread(project, issue_id, discussion_id, options = {})
-      post("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions/#{discussion_id}/notes", query: options)
-    end
-  end
-
   module QA
     module Report
       # Uses the API to create or update GitLab issues with the results of tests from RSpec report files.
-      # The GitLab client is used for API access: https://github.com/NARKOZ/gitlab
-      class ResultsInIssues
-        MAINTAINER_ACCESS_LEVEL = 40
+      class ResultsInIssues < ReportAsIssue
         MAX_TITLE_LENGTH = 255
-        RETRY_BACK_OFF_DELAY = 60
-        MAX_RETRY_ATTEMPTS = 3
-
-        def initialize(token:, input_files:, project: nil)
-          @token = token
-          @files = Array(input_files)
-          @project = project
-          @retry_backoff = 0
-        end
-
-        def invoke!
-          configure_gitlab_client
 
-          validate_input!
+        private
 
+        def run!
           puts "Reporting test results in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
 
-          Dir.glob(files).each do |file|
-            puts "Reporting tests in #{file}"
-            extension = File.extname(file)
+          Dir.glob(files).each do |path|
+            puts "Reporting tests in #{path}"
+            extension = File.extname(path)
 
             case extension
             when '.json'
-              test_results = Report::JsonTestResults.new(file)
+              test_results = Report::JsonTestResults.new(path)
             when '.xml'
-              test_results = Report::JUnitTestResults.new(file)
+              test_results = Report::JUnitTestResults.new(path)
             else
               raise "Unknown extension #{extension}"
             end
@@ -60,52 +31,8 @@ module Gitlab
             test_results.each do |test|
               report_test(test)
             end
-          end
-        end
-
-        private
-
-        attr_reader :files, :token, :project
-
-        def validate_input!
-          assert_project!
-          assert_input_files!(files)
-          assert_user_permission!
-        end
-
-        def assert_project!
-          return if project
-
-          abort "Please provide a valid project ID or path with the `-p/--project` option!"
-        end
-
-        def assert_input_files!(files)
-          return if Dir.glob(files).any?
-
-          abort "Please provide valid JUnit report files. No files were found matching `#{files.join(',')}`"
-        end
-
-        def assert_user_permission!
-          handle_gitlab_client_exceptions do
-            user = Gitlab.user
-            member = Gitlab.team_member(project, user.id)
 
-            abort_not_permitted if member.access_level < MAINTAINER_ACCESS_LEVEL
-          end
-        rescue Gitlab::Error::NotFound
-          abort_not_permitted
-        end
-
-        def abort_not_permitted
-          abort "You must have at least Maintainer access to the project to use this feature."
-        end
-
-        def configure_gitlab_client
-          handle_gitlab_client_exceptions do
-            Gitlab.configure do |config|
-              config.endpoint = Runtime::Env.gitlab_api_base
-              config.private_token = token
-            end
+            test_results.write(path)
           end
         end
 
@@ -115,6 +42,7 @@ module Gitlab
           puts "Reporting test: #{test.file} | #{test.name}"
 
           issue = find_issue(test)
+
           if issue
             puts "Found existing issue: #{issue.web_url}"
           else
@@ -122,42 +50,38 @@ module Gitlab
             puts "Created new issue: #{issue.web_url}"
           end
 
+          test.testcase ||= issue.web_url
+
           update_labels(issue, test)
           note_status(issue, test)
 
           puts "Issue updated"
         end
 
-        def create_issue(test)
-          puts "Creating issue..."
-
-          handle_gitlab_client_exceptions do
-            Gitlab.create_issue(
-              project,
-              title_from_test(test),
-              { description: "### Full description\n\n#{search_safe(test.name)}\n\n### File path\n\n#{test.file}", labels: 'status::automated' }
-            )
-          end
-        end
-
-        # rubocop:disable Metrics/AbcSize
         def find_issue(test)
-          handle_gitlab_client_exceptions do
-            return Gitlab.issue(project, id_from_testcase_url(test.testcase)) if test.testcase
+          title = title_from_test(test)
+          issues =
+            gitlab.find_issues(
+              iid: iid_from_testcase_url(test.testcase),
+              options: { search: search_term(test) }) do |issue|
+                issue.state == 'opened' && issue.title.strip == title
+              end
 
-            issues = Gitlab.issues(project, { search: search_term(test) })
-              .auto_paginate
-              .select { |issue| issue.state == 'opened' && issue.title.strip == title_from_test(test) }
+          warn(%(Too many issues found with the file path "#{test.file}" and name "#{test.name}")) if issues.many?
 
-            warn(%(Too many issues found with the file path "#{test.file}" and name "#{test.name}")) if issues.many?
+          issues.first
+        end
 
-            issues.first
-          end
+        def create_issue(test)
+          gitlab.create_issue(
+            title: title_from_test(test),
+            description: "### Full description\n\n#{search_safe(test.name)}\n\n### File path\n\n#{test.file}",
+            labels: 'status::automated'
+          )
         end
-        # rubocop:enable Metrics/AbcSize
 
-        def id_from_testcase_url(url)
-          url.split('/').last.to_i
+        def iid_from_testcase_url(url)
+          url && url.split('/').last.to_i
         end
 
         def search_term(test)
@@ -185,7 +109,7 @@ module Gitlab
 
           note = note_content(test)
 
-          handle_gitlab_client_exceptions do
+          gitlab.handle_gitlab_client_exceptions do
             Gitlab.issue_discussions(project, issue.iid, order_by: 'created_at', sort: 'asc').each do |discussion|
               return add_note_to_discussion(issue.iid, discussion.id) if new_note_matches_discussion?(note, discussion)
             end
@@ -241,12 +165,11 @@ module Gitlab
         end
 
         def add_note_to_discussion(issue_iid, discussion_id)
-          handle_gitlab_client_exceptions do
+          gitlab.handle_gitlab_client_exceptions do
             Gitlab.add_note_to_issue_discussion_as_thread(project, issue_iid, discussion_id, body: failure_summary)
           end
         end
 
-        # rubocop:disable Metrics/AbcSize
         def update_labels(issue, test)
           labels = issue.labels
           labels.delete_if { |label| label.start_with?("#{pipeline}::") }
@@ -254,11 +177,8 @@ module Gitlab
           labels << "Enterprise Edition" if ee_test?(test)
           quarantine_job? ? labels << "quarantine" : labels.delete("quarantine")
 
-          handle_gitlab_client_exceptions do
-            Gitlab.edit_issue(project, issue.iid, labels: labels)
-          end
+          gitlab.edit_issue(iid: issue.iid, options: { labels: labels })
         end
-        # rubocop:enable Metrics/AbcSize
 
         def ee_test?(test)
           test.file =~ %r{features/ee/(api|browser_ui)}
@@ -279,46 +199,6 @@ module Gitlab
 
           Runtime::Env.pipeline_from_project_name
         end
-
-        def handle_gitlab_client_exceptions
-          yield
-        rescue Gitlab::Error::NotFound
-          # This error could be raised in assert_user_permission!
-          # If so, we want it to terminate at that point
-          raise
-        rescue SystemCallError, OpenSSL::SSL::SSLError, Net::OpenTimeout, Net::ReadTimeout, Gitlab::Error::InternalServerError, Gitlab::Error::Parsing => e
-          @retry_backoff += RETRY_BACK_OFF_DELAY
-
-          raise if @retry_backoff > RETRY_BACK_OFF_DELAY * MAX_RETRY_ATTEMPTS
-
-          warn_exception(e)
-          warn("Sleeping for #{@retry_backoff} seconds before retrying...")
-          sleep @retry_backoff
-
-          retry
-        rescue StandardError => e
-          pipeline = QA::Runtime::Env.pipeline_from_project_name
-          channel = pipeline == "canary" ? "qa-production" : "qa-#{pipeline}"
-          error_msg = warn_exception(e)
-          slack_options = {
-            channel: channel,
-            icon_emoji: ':ci_failing:',
-            message: <<~MSG
-              An unexpected error occurred while reporting test results in issues.
-              The error occurred in job: #{QA::Runtime::Env.ci_job_url}
-              `#{error_msg}`
-            MSG
-          }
-          puts "Posting Slack message to channel: #{channel}"
-
-          Gitlab::QA::Slack::PostToSlack.new(**slack_options).invoke!
-        end
-
-        def warn_exception(error)
-          error_msg = "#{error.class.name} #{error.message}"
-          warn(error_msg)
-          error_msg
-        end
       end
     end
   end

data/lib/gitlab/qa/report/test_result.rb

@@ -4,60 +4,114 @@ module Gitlab
   module QA
     module Report
       class TestResult
-        attr_accessor :name, :file, :skipped, :failures, :testcase
-
-        def self.from_json(test)
-          new.tap do |test_result|
-            test_result.name = test['full_description']
-            test_result.file = test['file_path']
-            test_result.skipped = test['status'] == 'pending'
-            test_result.failures = failures_from_json_exceptions(test)
-            test_result.testcase = test['testcase']
-          end
+        def self.from_json(report)
+          JsonTestResult.new(report)
         end
 
-        def self.from_junit(test)
-          new.tap do |test_result|
-            test_result.name = test['name']
-            test_result.file = test['file']
-            test_result.skipped = test.search('skipped').any?
-            test_result.failures = failures_from_junit_exceptions(test)
-          end
+        def self.from_junit(report)
+          JUnitTestResult.new(report)
         end
 
-        def self.failures_from_json_exceptions(test)
-          return [] unless test.key?('exceptions')
+        attr_accessor :report, :failures
 
-          test['exceptions'].map do |exception|
-            spec_file_first_index = exception['backtrace'].rindex do |line|
-              line.include?(File.basename(test['file_path']))
-            end
+        def initialize(report)
+          self.report = report
+          self.failures = failures_from_exceptions
+        end
 
-            {
-              'message' => "#{exception['class']}: #{exception['message']}",
-              'stacktrace' => "#{exception['message_lines'].join("\n")}\n#{exception['backtrace'].slice(0..spec_file_first_index).join("\n")}"
-            }
-          end
+        def name
+          raise NotImplementedError
         end
-        private_class_method :failures_from_json_exceptions
 
-        def self.failures_from_junit_exceptions(test)
-          failures = test.search('failure')
-          return [] if failures.empty?
+        def file
+          raise NotImplementedError
+        end
+
+        def skipped
+          raise NotImplementedError
+        end
+
+        private
+
+        def failures_from_exceptions
+          raise NotImplementedError
+        end
+
+        class JsonTestResult < TestResult
+          def name
+            report['full_description']
+          end
+
+          def file
+            report['file_path']
+          end
+
+          def skipped
+            report['status'] == 'pending'
+          end
+
+          def testcase
+            report['testcase']
+          end
 
-          failures.map do |exception|
-            trace = exception.content.split("\n").map(&:strip)
-            spec_file_first_index = trace.rindex do |line|
-              line.include?(File.basename(test['file']))
+          def testcase=(new_testcase)
+            report['testcase'] = new_testcase
+          end
+
+          private
+
+          # rubocop:disable Metrics/AbcSize
+          def failures_from_exceptions
+            return [] unless report.key?('exceptions')
+
+            report['exceptions'].map do |exception|
+              spec_file_first_index = exception['backtrace'].rindex do |line|
+                line.include?(File.basename(report['file_path']))
+              end
+
+              {
+                'message' => "#{exception['class']}: #{exception['message']}",
+                'stacktrace' => "#{exception['message_lines'].join("\n")}\n#{exception['backtrace'].slice(0..spec_file_first_index).join("\n")}"
+              }
             end
+          end
+          # rubocop:enable Metrics/AbcSize
+        end
+
+        class JUnitTestResult < TestResult
+          def name
+            report['name']
+          end
 
-            {
-              'message' => "#{exception['type']}: #{exception['message']}",
-              'stacktrace' => trace.slice(0..spec_file_first_index).join("\n")
-            }
+          def file
+            report['file']
+          end
+
+          def skipped
+            report.search('skipped').any?
+          end
+
+          attr_accessor :testcase # Ignore it for now
+
+          private
+
+          def failures_from_exceptions
+            failures = report.search('failure')
+            return [] if failures.empty?
+
+            failures.map do |exception|
+              trace = exception.content.split("\n").map(&:strip)
+              spec_file_first_index = trace.rindex do |line|
+                line.include?(File.basename(report['file']))
+              end
+
+              {
+                'message' => "#{exception['type']}: #{exception['message']}",
+                'stacktrace' => trace.slice(0..spec_file_first_index).join("\n")
+              }
+            end
           end
         end
-        private_class_method :failures_from_junit_exceptions
       end
     end
   end

data/lib/gitlab/qa/reporter.rb

@@ -60,34 +60,19 @@ module Gitlab
         end
 
         if args.any?
-          if report_options[:prepare_stage_reports]
-            report_options.delete(:prepare_stage_reports)
+          if report_options.delete(:prepare_stage_reports)
             Gitlab::QA::Report::PrepareStageReports.new(**report_options).invoke!
 
-            exit
-          end
-
-          if report_options[:report_in_issues]
-            report_options.delete(:report_in_issues)
+          elsif report_options.delete(:report_in_issues)
             report_options[:token] = Runtime::TokenFinder.find_token!(report_options[:token])
             Gitlab::QA::Report::ResultsInIssues.new(**report_options).invoke!
 
-            exit
-          end
-
-          if slack_options[:post_to_slack]
-            slack_options.delete(:post_to_slack)
+          elsif slack_options.delete(:post_to_slack)
             Gitlab::QA::Slack::PostToSlack.new(**slack_options).invoke!
 
-            exit
-          end
-
-          if report_options[:update_screenshot_path]
-            report_options.delete(:update_screenshot_path)
-
+          elsif report_options.delete(:update_screenshot_path)
             Gitlab::QA::Report::UpdateScreenshotPath.new(**report_options).invoke!
 
-            exit
           end
         else
           puts options

data/lib/gitlab/qa/runtime/env.rb

@@ -62,6 +62,8 @@ module Gitlab
           'KNAPSACK_TEST_FILE_PATTERN' => :knapsack_test_file_pattern,
           'KNAPSACK_TEST_DIR' => :knapsack_test_dir,
           'CI' => :ci,
+          'CI_JOB_ID' => :ci_job_id,
+          'CI_JOB_URL' => :ci_job_url,
           'CI_RUNNER_ID' => :ci_runner_id,
           'CI_SERVER_HOST' => :ci_server_host,
           'CI_SERVER_PERSONAL_ACCESS_TOKEN' => :ci_server_personal_access_token,
@@ -81,11 +83,16 @@ module Gitlab
           'JIRA_HOSTNAME' => :jira_hostname,
           'JIRA_ADMIN_USERNAME' => :jira_admin_username,
           'JIRA_ADMIN_PASSWORD' => :jira_admin_password,
-          'CACHE_NAMESPACE_NAME' => :cache_namespace_name
+          'CACHE_NAMESPACE_NAME' => :cache_namespace_name,
+          'DEPLOY_VERSION' => :deploy_version
         }.freeze
 
-        ENV_VARIABLES.each_value do |accessor|
-          send(:attr_accessor, accessor) # rubocop:disable GitlabSecurity/PublicSend
+        ENV_VARIABLES.each do |env_name, method_name|
+          attr_writer(method_name)
+
+          define_method(method_name) do
+            ENV[env_name] || instance_variable_get("@#{method_name}")
+          end
         end
 
         def gitlab_username
@@ -112,10 +119,6 @@ module Gitlab
           ENV['CI_JOB_TOKEN']
         end
 
-        def ci_job_url
-          ENV['CI_JOB_URL']
-        end
-
         def ci_pipeline_source
           ENV['CI_PIPELINE_SOURCE']
         end
@@ -124,30 +127,14 @@ module Gitlab
           ENV['CI_PROJECT_NAME']
         end
 
-        def ci_slack_webhook_url
-          ENV['CI_SLACK_WEBHOOK_URL']
-        end
-
         def pipeline_from_project_name
           ci_project_name.to_s.start_with?('gitlab-qa') ? 'master' : ci_project_name
         end
 
-        def slack_qa_channel
-          ENV['SLACK_QA_CHANNEL']
-        end
-
-        def slack_icon_emoji
-          ENV['SLACK_ICON_EMOJI']
-        end
-
         def run_id
           @run_id ||= "gitlab-qa-run-#{Time.now.strftime('%Y-%m-%d-%H-%M-%S')}-#{SecureRandom.hex(4)}"
         end
 
-        def qa_access_token
-          ENV['GITLAB_QA_ACCESS_TOKEN']
-        end
-
         def dev_access_token_variable
           env_value_if_defined('GITLAB_QA_DEV_ACCESS_TOKEN')
         end
@@ -227,10 +214,6 @@ module Gitlab
           enabled?(ENV['QA_SKIP_PULL'], default: false)
         end
 
-        def gitlab_qa_formless_login_token
-          env_value_if_defined('GITLAB_QA_FORMLESS_LOGIN_TOKEN')
-        end
-
         private
 
         def enabled?(value, default: true)

data/lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb

@@ -15,6 +15,8 @@ module Gitlab
               @database = 'postgres'
               @spec_suite = 'Test::Instance::All'
               @network = 'test'
+              @env = {}
+              @tag = 'gitaly_cluster'
             end
 
             # rubocop:disable Metrics/AbcSize
@@ -51,14 +53,17 @@ module Gitlab
                 gitlab.instance do
                   puts "Running Gitaly Cluster specs!"
 
-                  rspec_args << "--" unless rspec_args.include?('--')
-                  rspec_args << %w[--tag gitaly_cluster]
+                  if @tag
+                    rspec_args << "--" unless rspec_args.include?('--')
+                    rspec_args << "--tag" << @tag
+                  end
 
                   Component::Specs.perform do |specs|
                     specs.suite = spec_suite
                     specs.release = gitlab.release
                     specs.network = gitlab.network
                     specs.args = [gitlab.address, *rspec_args]
+                    specs.env = @env
                   end
                 end
               end

data/lib/gitlab/qa/scenario/test/integration/praefect.rb

@@ -3,105 +3,62 @@ module Gitlab
     module Scenario
       module Test
         module Integration
-          class Praefect < Scenario::Template
-            # rubocop:disable Metrics/AbcSize
-            def perform(release, *rspec_args)
-              Docker::Volumes.new.with_temporary_volumes do |volumes|
-                # Create the Praefect database before enabling Praefect
-                Component::Gitlab.perform do |gitlab|
-                  gitlab.release = QA::Release.new(release)
-                  gitlab.name = 'gitlab'
-                  gitlab.network = 'test'
-                  gitlab.volumes = volumes
-                  gitlab.exec_commands = [
-                    'gitlab-psql -d template1 -c "CREATE DATABASE praefect_production OWNER gitlab"',
-                    'mkdir -p /var/opt/gitlab/git-data/repositories/praefect',
-                    'chown -R git:root /var/opt/gitlab/git-data/repositories'
-                  ]
+          class Praefect < GitalyCluster
+            attr_reader :gitlab_name, :spec_suite
 
-                  gitlab.act do
-                    prepare
-                    start
-                    reconfigure
-                    process_exec_commands
-                    wait
-                    teardown!
-                  end
-                end
+            def initialize
+              super
 
-                # Restart GitLab with Praefect enabled and then run the tests
-                Component::Gitlab.perform do |gitlab|
-                  gitlab.release = QA::Release.new(release)
-                  gitlab.name = 'gitlab'
-                  gitlab.network = 'test'
-                  gitlab.volumes = volumes
-                  gitlab.omnibus_config = omnibus_config_with_praefect
-
-                  gitlab.act do
-                    prepare_gitlab_omnibus_config
-                    start
-                    reconfigure
-                    wait
-
-                    puts "Running Praefect specs!"
-
-                    Component::Specs.perform do |specs|
-                      specs.suite = 'Test::Instance::All'
-                      specs.release = gitlab.release
-                      specs.network = gitlab.network
-                      specs.args = [gitlab.address, *rspec_args]
-                      specs.env = { QA_PRAEFECT_REPOSITORY_STORAGE: 'default' }
-                    end
-
-                    teardown
-                  end
-                end
-              end
+              @tag = nil
+              @env = { QA_PRAEFECT_REPOSITORY_STORAGE: 'default' }
             end
-            # rubocop:enable Metrics/AbcSize
-
-            private
 
-            def omnibus_config_with_praefect
+            def gitlab_omnibus_configuration
               <<~OMNIBUS
-                gitaly['enable'] = true;
+                external_url 'http://#{@gitlab_name}.#{@network}';
+
+                git_data_dirs({
+                  'default' => {
+                    'gitaly_address' => 'tcp://#{@praefect_node_name}.#{@network}:2305',
+                    'gitaly_token' => 'PRAEFECT_EXTERNAL_TOKEN'
+                  },
+                  'gitaly' => {
+                    'gitaly_address' => 'tcp://#{@gitlab_name}.#{@network}:8075',
+                    'path' => '/var/opt/gitlab/git-data'
+                  }
+                });
                 gitaly['listen_addr'] = '0.0.0.0:8075';
                 gitaly['auth_token'] = 'secret-token';
                 gitaly['storage'] = [
-                  {
-                    'name' => 'praefect-gitaly-0',
-                    'path' => '/var/opt/gitlab/git-data/repositories/praefect'
-                  },
                   {
                     'name' => 'gitaly',
-                    'path' => '/var/opt/gitlab/git-data/repositories/gitaly'
+                    'path' => '/var/opt/gitlab/git-data/repositories'
                   }
                 ];
-                praefect['enable'] = true;
-                praefect['listen_addr'] = '0.0.0.0:2305';
-                praefect['auth_token'] = 'secret-token';
-                praefect['virtual_storages'] = {
-                  'default' => {
-                    'praefect-gitaly-0' => {
-                      'address' => 'tcp://localhost:8075',
-                      'token'   => 'secret-token',
-                      'primary' => true
-                    }
-                  }
-                };
-                praefect['database_host'] = '/var/opt/gitlab/postgresql';
-                praefect['database_user'] = 'gitlab';
-                praefect['database_dbname'] = 'praefect_production';
-                praefect['postgres_queue_enabled'] = true;
                 gitlab_rails['gitaly_token'] = 'secret-token';
-                git_data_dirs({
-                  'default' => {
-                    'gitaly_address' => 'tcp://localhost:2305'
+                gitlab_shell['secret_token'] = 'GITLAB_SHELL_SECRET_TOKEN';
+                prometheus['scrape_configs'] = [
+                  {
+                    'job_name' => 'praefect',
+                    'static_configs' => [
+                      'targets' => [
+                        '#{@praefect_node_name}.#{@network}:9652'
+                      ]
+                    ]
                   },
-                  'gitaly' => {
-                    'path' => '/var/opt/gitlab/git-data/repositories/gitaly'
+                  {
+                    'job_name' => 'praefect-gitaly',
+                    'static_configs' => [
+                      'targets' => [
+                        '#{@primary_node_name}.#{@network}:9236',
+                        '#{@secondary_node_name}.#{@network}:9236',
+                        '#{@tertiary_node_name}.#{@network}:9236'
+                      ]
+                    ]
                   }
-                });
+                ];
+                grafana['disable_login_form'] = false;
+                grafana['admin_password'] = 'GRAFANA_ADMIN_PASSWORD';
               OMNIBUS
             end
           end

data/lib/gitlab/qa/scenario/test/omnibus/image.rb

@@ -15,7 +15,7 @@ module Gitlab
                   reconfigure
 
                   restart
-                  wait
+                  wait_until_ready
                   teardown
                 end
               end

data/lib/gitlab/qa/version.rb

@@ -1,5 +1,5 @@
 module Gitlab
   module QA
-    VERSION = '6.1.3'.freeze
+    VERSION = '6.5.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 6.1.3
+  version: 6.5.0
 platform: ruby
 authors:
 - Grzegorz Bizon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-10 00:00:00.000000000 Z
+date: 2020-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -257,9 +257,12 @@ files:
 - lib/gitlab/qa/docker/shellout.rb
 - lib/gitlab/qa/docker/volumes.rb
 - lib/gitlab/qa/release.rb
+- lib/gitlab/qa/report/base_test_results.rb
+- lib/gitlab/qa/report/gitlab_issue_client.rb
 - lib/gitlab/qa/report/json_test_results.rb
 - lib/gitlab/qa/report/junit_test_results.rb
 - lib/gitlab/qa/report/prepare_stage_reports.rb
+- lib/gitlab/qa/report/report_as_issue.rb
 - lib/gitlab/qa/report/results_in_issues.rb
 - lib/gitlab/qa/report/summary_table.rb
 - lib/gitlab/qa/report/test_result.rb