gitlab-qa 6.1.2 → 6.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6f02b51e013ff64f6e9080771f02e439efcf9093cedba40ca625fd10a932436e
-  data.tar.gz: 7089a36078ed197e4abb318f78d67d8f04289803c5bbfb6598fcd0761d071e6b
+  metadata.gz: 156602d47187662c86a6cdf07d6e737cdf8083c1098a411dc612ffbdaeb0f5fc
+  data.tar.gz: f0678a3a1587fbe094ecb3804b2d4a3e6b040ee27e2c7dc766c9b9a1f3495341
 SHA512:
-  metadata.gz: 4906ac6a4ddacb5061c870a2e7e484741c60b488a8efb5051187d768c7a9a990d73b4bb5ba47ff9cf48980a63197da25890c428b27aaddb5caf69e06931ced0e
-  data.tar.gz: 8dd810d48fcbbd70f858ff4fd511a9c79f29131f13968b2e990483a79e927cd6f7841fcbf47dd6ea63933ea09b7319ff630b1d337085e4a4a731eb606a9c8cc8
+  metadata.gz: 4def52124e2d59910e64c29699b55faabd46e29bca5030938ecd93aec69b638fb855a6943ffaad91ede0c98d012b47fedbb6f340cbf74188cb1cf1386d2d136b
+  data.tar.gz: 92fca187c1845867f066921c3ff7718de149043526c5f85f6108db4053554b993e85c8f382c084cd49a20f1198c0456c89ea067182d9d0ade2a43801a2e7b382

data/.gitlab-ci.yml

@@ -94,7 +94,7 @@ release:
     - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
     - exit $test_run_exit_code
 
 .ce-qa:
@@ -229,44 +229,6 @@ ee:instance-quarantine:
   variables:
     QA_RSPEC_TAGS: "--tag quarantine --tag ~orchestrated"
 
-ce:docker:
-  extends:
-    - .test
-    - .high-capacity
-    - .ce-qa
-    - .rspec-report-opts
-  variables:
-    QA_RSPEC_TAGS: "--tag docker"
-
-ce:docker-quarantine:
-  extends:
-    - .test
-    - .high-capacity
-    - .ce-qa
-    - .quarantine
-    - .rspec-report-opts
-  variables:
-    QA_RSPEC_TAGS: "--tag docker --tag quarantine"
-
-ee:docker:
-  extends:
-    - .test
-    - .high-capacity
-    - .ee-qa
-    - .rspec-report-opts
-  variables:
-    QA_RSPEC_TAGS: "--tag docker"
-
-ee:docker-quarantine:
-  extends:
-    - .test
-    - .high-capacity
-    - .ee-qa
-    - .quarantine
-    - .rspec-report-opts
-  variables:
-    QA_RSPEC_TAGS: "--tag docker --tag quarantine"
-
 ce:relative_url:
   extends:
     - .test
@@ -370,7 +332,7 @@ ce:update:
   script:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -384,7 +346,7 @@ ce:update-quarantine:
   script:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -397,7 +359,7 @@ ee:update:
   script:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -411,7 +373,7 @@ ee:update-quarantine:
   script:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -676,6 +638,7 @@ ce:kubernetes:
     - .high-capacity
     - .ce-qa
     - .rspec-report-opts
+  allow_failure: true
   variables:
     QA_SCENARIO: "Test::Integration::Kubernetes"
 
@@ -695,6 +658,7 @@ ee:kubernetes:
     - .high-capacity
     - .ee-qa
     - .rspec-report-opts
+  allow_failure: true
   variables:
     QA_SCENARIO: "Test::Integration::Kubernetes"
 

data/docs/what_tests_can_be_run.md

@@ -83,6 +83,7 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `JIRA_ADMIN_USERNAME` |-  | Username for authenticating with Jira server as admin. | No|
 | `JIRA_ADMIN_PASSWORD` |-  | Password for authenticating with Jira server as admin. | No|
 | `CACHE_NAMESPACE_NAME` | `true` | Cache namespace name for groups. | No|
+| `DEPLOY_VERSION` |- | The version of GitLab being tested against. | No|
 
 ## [Supported Remote Grid environment variables](./running_against_remote_grid.md)
 

data/lib/gitlab/qa.rb

@@ -97,9 +97,12 @@ module Gitlab
     end
 
     module Report
+      autoload :GitlabIssueClient, 'gitlab/qa/report/gitlab_issue_client'
+      autoload :BaseTestResults, 'gitlab/qa/report/base_test_results'
       autoload :JsonTestResults, 'gitlab/qa/report/json_test_results'
       autoload :JUnitTestResults, 'gitlab/qa/report/junit_test_results'
       autoload :PrepareStageReports, 'gitlab/qa/report/prepare_stage_reports'
+      autoload :ReportAsIssue, 'gitlab/qa/report/report_as_issue'
       autoload :ResultsInIssues, 'gitlab/qa/report/results_in_issues'
       autoload :SummaryTable, 'gitlab/qa/report/summary_table'
       autoload :TestResult, 'gitlab/qa/report/test_result'

data/lib/gitlab/qa/component/gitlab.rb

@@ -128,10 +128,10 @@ module Gitlab
           end
         end
 
-        def wait
+        def wait_until_ready
           return if skip_availability_check
 
-          if Availability.new(name, relative_path: relative_path, scheme: scheme, protocol_port: port.to_i).check(180)
+          if Availability.new(name, relative_path: relative_path, scheme: scheme, protocol_port: port.to_i).check(300)
             sleep 12 # TODO, handle that better
             puts ' -> GitLab is available.'
           else

data/lib/gitlab/qa/component/specs.rb

@@ -23,6 +23,8 @@ module Gitlab
 
           @docker.login(**release.login_params) if release.login_params
 
+          @docker.pull(release.qa_image, release.qa_tag) unless Runtime::Env.skip_pull?
+
           puts "Running test suite `#{suite}` for #{release.project_name}"
 
           name = "#{release.project_name}-qa-#{SecureRandom.hex(4)}"

data/lib/gitlab/qa/report/base_test_results.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Report
+      class BaseTestResults
+        include Enumerable
+
+        def initialize(path)
+          @results = parse(path)
+          @testcases = process
+        end
+
+        def each(&block)
+          testcases.each(&block)
+        end
+
+        def write(path)
+          raise NotImplementedError
+        end
+
+        private
+
+        attr_reader :results, :testcases
+
+        def parse(path)
+          raise NotImplementedError
+        end
+
+        def process
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/report/gitlab_issue_client.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require 'gitlab'
+
+module Gitlab
+  # Monkey patch the Gitlab client to use the correct API path and add required methods
+  class Client
+    def team_member(project, id)
+      get("/projects/#{url_encode(project)}/members/all/#{id}")
+    end
+
+    def issue_discussions(project, issue_id, options = {})
+      get("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions", query: options)
+    end
+
+    def add_note_to_issue_discussion_as_thread(project, issue_id, discussion_id, options = {})
+      post("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions/#{discussion_id}/notes", query: options)
+    end
+  end
+
+  module QA
+    module Report
+      # The GitLab client is used for API access: https://github.com/NARKOZ/gitlab
+      class GitlabIssueClient
+        MAINTAINER_ACCESS_LEVEL = 40
+        RETRY_BACK_OFF_DELAY = 60
+        MAX_RETRY_ATTEMPTS = 3
+
+        def initialize(token:, project:)
+          @token = token
+          @project = project
+          @retry_backoff = 0
+
+          configure_gitlab_client
+        end
+
+        def assert_user_permission!
+          handle_gitlab_client_exceptions do
+            user = Gitlab.user
+            member = Gitlab.team_member(project, user.id)
+
+            abort_not_permitted if member.access_level < MAINTAINER_ACCESS_LEVEL
+          end
+        rescue Gitlab::Error::NotFound
+          abort_not_permitted
+        end
+
+        def find_issues(iid:, options: {}, &select)
+          select ||= :itself
+
+          handle_gitlab_client_exceptions do
+            return [Gitlab.issue(project, iid)] if iid
+
+            Gitlab.issues(project, options)
+              .auto_paginate
+              .select(&select)
+          end
+        end
+
+        def create_issue(title:, description:, labels:)
+          puts "Creating issue..."
+
+          handle_gitlab_client_exceptions do
+            Gitlab.create_issue(
+              project,
+              title,
+              { description: description, labels: labels }
+            )
+          end
+        end
+
+        def edit_issue(iid:, options: {})
+          handle_gitlab_client_exceptions do
+            Gitlab.edit_issue(project, iid, options)
+          end
+        end
+
+        def handle_gitlab_client_exceptions
+          yield
+        rescue Gitlab::Error::NotFound
+          # This error could be raised in assert_user_permission!
+          # If so, we want it to terminate at that point
+          raise
+        rescue SystemCallError, OpenSSL::SSL::SSLError, Net::OpenTimeout, Net::ReadTimeout, Gitlab::Error::InternalServerError, Gitlab::Error::Parsing => e
+          @retry_backoff += RETRY_BACK_OFF_DELAY
+
+          raise if @retry_backoff > RETRY_BACK_OFF_DELAY * MAX_RETRY_ATTEMPTS
+
+          warn_exception(e)
+          warn("Sleeping for #{@retry_backoff} seconds before retrying...")
+          sleep @retry_backoff
+
+          retry
+        rescue StandardError => e
+          pipeline = QA::Runtime::Env.pipeline_from_project_name
+          channel = pipeline == "canary" ? "qa-production" : "qa-#{pipeline}"
+          error_msg = warn_exception(e)
+          slack_options = {
+            channel: channel,
+            icon_emoji: ':ci_failing:',
+            message: <<~MSG
+              An unexpected error occurred while reporting test results in issues.
+              The error occurred in job: #{QA::Runtime::Env.ci_job_url}
+              `#{error_msg}`
+            MSG
+          }
+          puts "Posting Slack message to channel: #{channel}"
+
+          Gitlab::QA::Slack::PostToSlack.new(**slack_options).invoke!
+        end
+
+        private
+
+        attr_reader :token, :project
+
+        def configure_gitlab_client
+          Gitlab.configure do |config|
+            config.endpoint = Runtime::Env.gitlab_api_base
+            config.private_token = token
+          end
+        end
+
+        def abort_not_permitted
+          abort "You must have at least Maintainer access to the project to use this feature."
+        end
+
+        def warn_exception(error)
+          error_msg = "#{error.class.name} #{error.message}"
+          warn(error_msg)
+          error_msg
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/report/json_test_results.rb

@@ -5,15 +5,23 @@ require 'json'
 module Gitlab
   module QA
     module Report
-      class JsonTestResults
-        include Enumerable
+      class JsonTestResults < BaseTestResults
+        def write(path)
+          json = results.merge('examples' => testcases.map(&:report))
 
-        def initialize(file)
-          @testcases = JSON.parse(File.read(file))['examples'].map { |test| TestResult.from_json(test) }
+          File.write(path, JSON.pretty_generate(json))
         end
 
-        def each(&block)
-          @testcases.each(&block)
+        private
+
+        def parse(path)
+          JSON.parse(File.read(path))
+        end
+
+        def process
+          results['examples'].map do |test|
+            TestResult.from_json(test)
+          end
         end
       end
     end

data/lib/gitlab/qa/report/junit_test_results.rb

@@ -5,15 +5,21 @@ require 'nokogiri'
 module Gitlab
   module QA
     module Report
-      class JUnitTestResults
-        include Enumerable
+      class JUnitTestResults < BaseTestResults
+        def write(path)
+          # Ignore it for now
+        end
+
+        private
 
-        def initialize(file)
-          @testcases = Nokogiri::XML(File.read(file)).xpath('//testcase').map { |test| TestResult.from_junit(test) }
+        def parse(path)
+          Nokogiri::XML.parse(File.read(path))
         end
 
-        def each(&block)
-          @testcases.each(&block)
+        def process
+          results.xpath('//testcase').map do |test|
+            TestResult.from_junit(test)
+          end
         end
       end
     end

data/lib/gitlab/qa/report/report_as_issue.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Report
+      class ReportAsIssue
+        def initialize(token:, input_files:, project: nil)
+          @gitlab = GitlabIssueClient.new(token: token, project: project)
+          @files = Array(input_files)
+          @project = project
+        end
+
+        def invoke!
+          validate_input!
+
+          run!
+        end
+
+        private
+
+        attr_reader :gitlab, :files, :project
+
+        def run!
+          raise NotImplementedError
+        end
+
+        def validate_input!
+          assert_project!
+          assert_input_files!(files)
+          gitlab.assert_user_permission!
+        end
+
+        def assert_project!
+          return if project
+
+          abort "Please provide a valid project ID or path with the `-p/--project` option!"
+        end
+
+        def assert_input_files!(files)
+          return if Dir.glob(files).any?
+
+          abort "Please provide valid JUnit report files. No files were found matching `#{files.join(',')}`"
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/report/results_in_issues.rb

@@ -1,58 +1,29 @@
 # frozen_string_literal: true
 
 require 'nokogiri'
-require 'gitlab'
 require 'active_support/core_ext/enumerable'
 
 module Gitlab
-  # Monkey patch the Gitlab client to use the correct API path and add required methods
-  class Client
-    def team_member(project, id)
-      get("/projects/#{url_encode(project)}/members/all/#{id}")
-    end
-
-    def issue_discussions(project, issue_id, options = {})
-      get("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions", query: options)
-    end
-
-    def add_note_to_issue_discussion_as_thread(project, issue_id, discussion_id, options = {})
-      post("/projects/#{url_encode(project)}/issues/#{issue_id}/discussions/#{discussion_id}/notes", query: options)
-    end
-  end
-
   module QA
     module Report
       # Uses the API to create or update GitLab issues with the results of tests from RSpec report files.
-      # The GitLab client is used for API access: https://github.com/NARKOZ/gitlab
-      class ResultsInIssues
-        MAINTAINER_ACCESS_LEVEL = 40
+      class ResultsInIssues < ReportAsIssue
         MAX_TITLE_LENGTH = 255
-        RETRY_BACK_OFF_DELAY = 60
-        MAX_RETRY_ATTEMPTS = 3
-
-        def initialize(token:, input_files:, project: nil)
-          @token = token
-          @files = Array(input_files)
-          @project = project
-          @retry_backoff = 0
-        end
-
-        def invoke!
-          configure_gitlab_client
 
-          validate_input!
+        private
 
+        def run!
           puts "Reporting test results in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
 
-          Dir.glob(files).each do |file|
-            puts "Reporting tests in #{file}"
-            extension = File.extname(file)
+          Dir.glob(files).each do |path|
+            puts "Reporting tests in #{path}"
+            extension = File.extname(path)
 
             case extension
             when '.json'
-              test_results = Report::JsonTestResults.new(file)
+              test_results = Report::JsonTestResults.new(path)
             when '.xml'
-              test_results = Report::JUnitTestResults.new(file)
+              test_results = Report::JUnitTestResults.new(path)
             else
               raise "Unknown extension #{extension}"
             end
@@ -60,52 +31,8 @@ module Gitlab
             test_results.each do |test|
               report_test(test)
             end
-          end
-        end
-
-        private
-
-        attr_reader :files, :token, :project
-
-        def validate_input!
-          assert_project!
-          assert_input_files!(files)
-          assert_user_permission!
-        end
-
-        def assert_project!
-          return if project
-
-          abort "Please provide a valid project ID or path with the `-p/--project` option!"
-        end
-
-        def assert_input_files!(files)
-          return if Dir.glob(files).any?
-
-          abort "Please provide valid JUnit report files. No files were found matching `#{files.join(',')}`"
-        end
-
-        def assert_user_permission!
-          handle_gitlab_client_exceptions do
-            user = Gitlab.user
-            member = Gitlab.team_member(project, user.id)
 
-            abort_not_permitted if member.access_level < MAINTAINER_ACCESS_LEVEL
-          end
-        rescue Gitlab::Error::NotFound
-          abort_not_permitted
-        end
-
-        def abort_not_permitted
-          abort "You must have at least Maintainer access to the project to use this feature."
-        end
-
-        def configure_gitlab_client
-          handle_gitlab_client_exceptions do
-            Gitlab.configure do |config|
-              config.endpoint = Runtime::Env.gitlab_api_base
-              config.private_token = token
-            end
+            test_results.write(path)
           end
         end
 
@@ -115,6 +42,7 @@ module Gitlab
           puts "Reporting test: #{test.file} | #{test.name}"
 
           issue = find_issue(test)
+
           if issue
             puts "Found existing issue: #{issue.web_url}"
           else
@@ -122,42 +50,38 @@ module Gitlab
             puts "Created new issue: #{issue.web_url}"
           end
 
+          test.testcase ||= issue.web_url
+
           update_labels(issue, test)
           note_status(issue, test)
 
           puts "Issue updated"
         end
 
-        def create_issue(test)
-          puts "Creating issue..."
-
-          handle_gitlab_client_exceptions do
-            Gitlab.create_issue(
-              project,
-              title_from_test(test),
-              { description: "### Full description\n\n#{search_safe(test.name)}\n\n### File path\n\n#{test.file}", labels: 'status::automated' }
-            )
-          end
-        end
-
-        # rubocop:disable Metrics/AbcSize
         def find_issue(test)
-          handle_gitlab_client_exceptions do
-            return Gitlab.issue(project, id_from_testcase_url(test.testcase)) if test.testcase
+          title = title_from_test(test)
+          issues =
+            gitlab.find_issues(
+              iid: iid_from_testcase_url(test.testcase),
+              options: { search: search_term(test) }) do |issue|
+                issue.state == 'opened' && issue.title.strip == title
+              end
 
-            issues = Gitlab.issues(project, { search: search_term(test) })
-              .auto_paginate
-              .select { |issue| issue.state == 'opened' && issue.title.strip == title_from_test(test) }
+          warn(%(Too many issues found with the file path "#{test.file}" and name "#{test.name}")) if issues.many?
 
-            warn(%(Too many issues found with the file path "#{test.file}" and name "#{test.name}")) if issues.many?
+          issues.first
+        end
 
-            issues.first
-          end
+        def create_issue(test)
+          gitlab.create_issue(
+            title: title_from_test(test),
+            description: "### Full description\n\n#{search_safe(test.name)}\n\n### File path\n\n#{test.file}",
+            labels: 'status::automated'
+          )
         end
-        # rubocop:enable Metrics/AbcSize
 
-        def id_from_testcase_url(url)
-          url.split('/').last.to_i
+        def iid_from_testcase_url(url)
+          url && url.split('/').last.to_i
         end
 
         def search_term(test)
@@ -185,7 +109,7 @@ module Gitlab
 
           note = note_content(test)
 
-          handle_gitlab_client_exceptions do
+          gitlab.handle_gitlab_client_exceptions do
             Gitlab.issue_discussions(project, issue.iid, order_by: 'created_at', sort: 'asc').each do |discussion|
               return add_note_to_discussion(issue.iid, discussion.id) if new_note_matches_discussion?(note, discussion)
             end
@@ -241,12 +165,11 @@ module Gitlab
         end
 
         def add_note_to_discussion(issue_iid, discussion_id)
-          handle_gitlab_client_exceptions do
+          gitlab.handle_gitlab_client_exceptions do
             Gitlab.add_note_to_issue_discussion_as_thread(project, issue_iid, discussion_id, body: failure_summary)
           end
         end
 
-        # rubocop:disable Metrics/AbcSize
         def update_labels(issue, test)
           labels = issue.labels
           labels.delete_if { |label| label.start_with?("#{pipeline}::") }
@@ -254,11 +177,8 @@ module Gitlab
           labels << "Enterprise Edition" if ee_test?(test)
           quarantine_job? ? labels << "quarantine" : labels.delete("quarantine")
 
-          handle_gitlab_client_exceptions do
-            Gitlab.edit_issue(project, issue.iid, labels: labels)
-          end
+          gitlab.edit_issue(iid: issue.iid, options: { labels: labels })
         end
-        # rubocop:enable Metrics/AbcSize
 
         def ee_test?(test)
           test.file =~ %r{features/ee/(api|browser_ui)}
@@ -279,46 +199,6 @@ module Gitlab
 
           Runtime::Env.pipeline_from_project_name
         end
-
-        def handle_gitlab_client_exceptions
-          yield
-        rescue Gitlab::Error::NotFound
-          # This error could be raised in assert_user_permission!
-          # If so, we want it to terminate at that point
-          raise
-        rescue SystemCallError, OpenSSL::SSL::SSLError, Net::ReadTimeout, Gitlab::Error::Parsing => e
-          @retry_backoff += RETRY_BACK_OFF_DELAY
-
-          raise if @retry_backoff > RETRY_BACK_OFF_DELAY * MAX_RETRY_ATTEMPTS
-
-          warn_exception(e)
-          warn("Sleeping for #{@retry_backoff} seconds before retrying...")
-          sleep @retry_backoff
-
-          retry
-        rescue StandardError => e
-          pipeline = QA::Runtime::Env.pipeline_from_project_name
-          channel = pipeline == "canary" ? "qa-production" : "qa-#{pipeline}"
-          error_msg = warn_exception(e)
-          slack_options = {
-            channel: channel,
-            icon_emoji: ':ci_failing:',
-            message: <<~MSG
-              An unexpected error occurred while reporting test results in issues.
-              The error occurred in job: #{QA::Runtime::Env.ci_job_url}
-              `#{error_msg}`
-            MSG
-          }
-          puts "Posting Slack message to channel: #{channel}"
-
-          Gitlab::QA::Slack::PostToSlack.new(**slack_options).invoke!
-        end
-
-        def warn_exception(error)
-          error_msg = "#{error.class.name} #{error.message}"
-          warn(error_msg)
-          error_msg
-        end
       end
     end
   end

data/lib/gitlab/qa/report/test_result.rb

@@ -4,60 +4,114 @@ module Gitlab
   module QA
     module Report
       class TestResult
-        attr_accessor :name, :file, :skipped, :failures, :testcase
-
-        def self.from_json(test)
-          new.tap do |test_result|
-            test_result.name = test['full_description']
-            test_result.file = test['file_path']
-            test_result.skipped = test['status'] == 'pending'
-            test_result.failures = failures_from_json_exceptions(test)
-            test_result.testcase = test['testcase']
-          end
+        def self.from_json(report)
+          JsonTestResult.new(report)
         end
 
-        def self.from_junit(test)
-          new.tap do |test_result|
-            test_result.name = test['name']
-            test_result.file = test['file']
-            test_result.skipped = test.search('skipped').any?
-            test_result.failures = failures_from_junit_exceptions(test)
-          end
+        def self.from_junit(report)
+          JUnitTestResult.new(report)
         end
 
-        def self.failures_from_json_exceptions(test)
-          return [] unless test.key?('exceptions')
+        attr_accessor :report, :failures
 
-          test['exceptions'].map do |exception|
-            spec_file_first_index = exception['backtrace'].rindex do |line|
-              line.include?(File.basename(test['file_path']))
-            end
+        def initialize(report)
+          self.report = report
+          self.failures = failures_from_exceptions
+        end
 
-            {
-              'message' => "#{exception['class']}: #{exception['message']}",
-              'stacktrace' => exception['backtrace'].slice(0..spec_file_first_index).join("\n")
-            }
-          end
+        def name
+          raise NotImplementedError
         end
-        private_class_method :failures_from_json_exceptions
 
-        def self.failures_from_junit_exceptions(test)
-          failures = test.search('failure')
-          return [] if failures.empty?
+        def file
+          raise NotImplementedError
+        end
+
+        def skipped
+          raise NotImplementedError
+        end
+
+        private
+
+        def failures_from_exceptions
+          raise NotImplementedError
+        end
+
+        class JsonTestResult < TestResult
+          def name
+            report['full_description']
+          end
+
+          def file
+            report['file_path']
+          end
+
+          def skipped
+            report['status'] == 'pending'
+          end
+
+          def testcase
+            report['testcase']
+          end
 
-          failures.map do |exception|
-            trace = exception.content.split("\n").map(&:strip)
-            spec_file_first_index = trace.rindex do |line|
-              line.include?(File.basename(test['file']))
+          def testcase=(new_testcase)
+            report['testcase'] = new_testcase
+          end
+
+          private
+
+          # rubocop:disable Metrics/AbcSize
+          def failures_from_exceptions
+            return [] unless report.key?('exceptions')
+
+            report['exceptions'].map do |exception|
+              spec_file_first_index = exception['backtrace'].rindex do |line|
+                line.include?(File.basename(report['file_path']))
+              end
+
+              {
+                'message' => "#{exception['class']}: #{exception['message']}",
+                'stacktrace' => "#{exception['message_lines'].join("\n")}\n#{exception['backtrace'].slice(0..spec_file_first_index).join("\n")}"
+              }
             end
+          end
+          # rubocop:enable Metrics/AbcSize
+        end
+
+        class JUnitTestResult < TestResult
+          def name
+            report['name']
+          end
 
-            {
-              'message' => "#{exception['type']}: #{exception['message']}",
-              'stacktrace' => trace.slice(0..spec_file_first_index).join("\n")
-            }
+          def file
+            report['file']
+          end
+
+          def skipped
+            report.search('skipped').any?
+          end
+
+          attr_accessor :testcase # Ignore it for now
+
+          private
+
+          def failures_from_exceptions
+            failures = report.search('failure')
+            return [] if failures.empty?
+
+            failures.map do |exception|
+              trace = exception.content.split("\n").map(&:strip)
+              spec_file_first_index = trace.rindex do |line|
+                line.include?(File.basename(report['file']))
+              end
+
+              {
+                'message' => "#{exception['type']}: #{exception['message']}",
+                'stacktrace' => trace.slice(0..spec_file_first_index).join("\n")
+              }
+            end
           end
         end
-        private_class_method :failures_from_junit_exceptions
       end
     end
   end

data/lib/gitlab/qa/reporter.rb

@@ -60,34 +60,19 @@ module Gitlab
         end
 
         if args.any?
-          if report_options[:prepare_stage_reports]
-            report_options.delete(:prepare_stage_reports)
+          if report_options.delete(:prepare_stage_reports)
             Gitlab::QA::Report::PrepareStageReports.new(**report_options).invoke!
 
-            exit
-          end
-
-          if report_options[:report_in_issues]
-            report_options.delete(:report_in_issues)
+          elsif report_options.delete(:report_in_issues)
             report_options[:token] = Runtime::TokenFinder.find_token!(report_options[:token])
             Gitlab::QA::Report::ResultsInIssues.new(**report_options).invoke!
 
-            exit
-          end
-
-          if slack_options[:post_to_slack]
-            slack_options.delete(:post_to_slack)
+          elsif slack_options.delete(:post_to_slack)
             Gitlab::QA::Slack::PostToSlack.new(**slack_options).invoke!
 
-            exit
-          end
-
-          if report_options[:update_screenshot_path]
-            report_options.delete(:update_screenshot_path)
-
+          elsif report_options.delete(:update_screenshot_path)
             Gitlab::QA::Report::UpdateScreenshotPath.new(**report_options).invoke!
 
-            exit
           end
         else
           puts options

data/lib/gitlab/qa/runtime/env.rb

@@ -62,6 +62,8 @@ module Gitlab
           'KNAPSACK_TEST_FILE_PATTERN' => :knapsack_test_file_pattern,
           'KNAPSACK_TEST_DIR' => :knapsack_test_dir,
           'CI' => :ci,
+          'CI_JOB_ID' => :ci_job_id,
+          'CI_JOB_URL' => :ci_job_url,
           'CI_RUNNER_ID' => :ci_runner_id,
           'CI_SERVER_HOST' => :ci_server_host,
           'CI_SERVER_PERSONAL_ACCESS_TOKEN' => :ci_server_personal_access_token,
@@ -81,11 +83,16 @@ module Gitlab
           'JIRA_HOSTNAME' => :jira_hostname,
           'JIRA_ADMIN_USERNAME' => :jira_admin_username,
           'JIRA_ADMIN_PASSWORD' => :jira_admin_password,
-          'CACHE_NAMESPACE_NAME' => :cache_namespace_name
+          'CACHE_NAMESPACE_NAME' => :cache_namespace_name,
+          'DEPLOY_VERSION' => :deploy_version
         }.freeze
 
-        ENV_VARIABLES.each_value do |accessor|
-          send(:attr_accessor, accessor) # rubocop:disable GitlabSecurity/PublicSend
+        ENV_VARIABLES.each do |env_name, method_name|
+          attr_writer(method_name)
+
+          define_method(method_name) do
+            ENV[env_name] || instance_variable_get("@#{method_name}")
+          end
         end
 
         def gitlab_username
@@ -112,10 +119,6 @@ module Gitlab
           ENV['CI_JOB_TOKEN']
         end
 
-        def ci_job_url
-          ENV['CI_JOB_URL']
-        end
-
         def ci_pipeline_source
           ENV['CI_PIPELINE_SOURCE']
         end
@@ -124,30 +127,14 @@ module Gitlab
           ENV['CI_PROJECT_NAME']
         end
 
-        def ci_slack_webhook_url
-          ENV['CI_SLACK_WEBHOOK_URL']
-        end
-
         def pipeline_from_project_name
           ci_project_name.to_s.start_with?('gitlab-qa') ? 'master' : ci_project_name
         end
 
-        def slack_qa_channel
-          ENV['SLACK_QA_CHANNEL']
-        end
-
-        def slack_icon_emoji
-          ENV['SLACK_ICON_EMOJI']
-        end
-
         def run_id
           @run_id ||= "gitlab-qa-run-#{Time.now.strftime('%Y-%m-%d-%H-%M-%S')}-#{SecureRandom.hex(4)}"
         end
 
-        def qa_access_token
-          ENV['GITLAB_QA_ACCESS_TOKEN']
-        end
-
         def dev_access_token_variable
           env_value_if_defined('GITLAB_QA_DEV_ACCESS_TOKEN')
         end
@@ -227,10 +214,6 @@ module Gitlab
           enabled?(ENV['QA_SKIP_PULL'], default: false)
         end
 
-        def gitlab_qa_formless_login_token
-          env_value_if_defined('GITLAB_QA_FORMLESS_LOGIN_TOKEN')
-        end
-
         private
 
         def enabled?(value, default: true)

data/lib/gitlab/qa/scenario/test/integration/praefect.rb

@@ -24,7 +24,7 @@ module Gitlab
                     start
                     reconfigure
                     process_exec_commands
-                    wait
+                    wait_until_ready
                     teardown!
                   end
                 end
@@ -41,7 +41,7 @@ module Gitlab
                     prepare_gitlab_omnibus_config
                     start
                     reconfigure
-                    wait
+                    wait_until_ready
 
                     puts "Running Praefect specs!"
 

data/lib/gitlab/qa/scenario/test/omnibus/image.rb

@@ -15,7 +15,7 @@ module Gitlab
                   reconfigure
 
                   restart
-                  wait
+                  wait_until_ready
                   teardown
                 end
               end

data/lib/gitlab/qa/version.rb

@@ -1,5 +1,5 @@
 module Gitlab
   module QA
-    VERSION = '6.1.2'.freeze
+    VERSION = '6.4.1'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 6.1.2
+  version: 6.4.1
 platform: ruby
 authors:
 - Grzegorz Bizon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-03 00:00:00.000000000 Z
+date: 2020-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -257,9 +257,12 @@ files:
 - lib/gitlab/qa/docker/shellout.rb
 - lib/gitlab/qa/docker/volumes.rb
 - lib/gitlab/qa/release.rb
+- lib/gitlab/qa/report/base_test_results.rb
+- lib/gitlab/qa/report/gitlab_issue_client.rb
 - lib/gitlab/qa/report/json_test_results.rb
 - lib/gitlab/qa/report/junit_test_results.rb
 - lib/gitlab/qa/report/prepare_stage_reports.rb
+- lib/gitlab/qa/report/report_as_issue.rb
 - lib/gitlab/qa/report/results_in_issues.rb
 - lib/gitlab/qa/report/summary_table.rb
 - lib/gitlab/qa/report/test_result.rb