gitlab-qa 6.10.0 → 6.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e7b24a0d671a29568447a0db95b57a06c43ed750d220dc13a0ad8216b7ba6297
-  data.tar.gz: 9e03938be0b67f4f9a522703156bfc1b191ac0b8b97661c5eb74b82bfd91c84f
+  metadata.gz: 79cd3084cf93dc40c0935800fd13991f2b8e057fd08aca2b91cffddc776675c3
+  data.tar.gz: 0ad9aecd55123681543e84b7ff2150ec8d4a0f004374dd43cd7ced64a89b11f9
 SHA512:
-  metadata.gz: 21ea402dd74ab453e9ac303303f50925a32a82e58b7e4ff0aa57721d4e8301717febc4da56b1b5b3c7141ad7f0e613b56fc7d47b7a25256b83f07ec53980db33
-  data.tar.gz: e31d991f0f78bfc510dbaafb3a844126d95c2e46655b2d655c256dcdfda673ca8038085c33deb65caa00728effb63e75020e3f6c217ddc697e87ceaded1270ba
+  metadata.gz: 29b7032d6ad65269e2aa27d03bea3bff2805bd160550c69fefd08475bb60ad5160c2417422e58b2b29d1ef4c2c31d3ebc6661cb0cbb151522dedb6dda97e64d7
+  data.tar.gz: 16d5d8ae03e4542c7518e1466723eee32e927a523890c3d56c565df2cabd363a180116a2d4db5d2eefd323dadb355f1471c7d7da9a0fcff61eb13f607a174b95

data/.gitlab-ci.yml

@@ -49,9 +49,9 @@ variables:
   QA_CAN_TEST_GIT_PROTOCOL_V2: "true"
   QA_CAN_TEST_PRAEFECT: "false"
   QA_TESTCASES_REPORTING_PROJECT: "gitlab-org/quality/testcases"
-  QA_FAILURES_REPORTING_PROJECT: "gitlab-org/gitlab"
+  QA_FAILURES_REPORTING_PROJECT: "rymai/gitlab-qa-issues"
   # The --dry-run or --max-diff-ratio option can be set to modify the behavior of `exe/gitlab-qa-report --relate-failure-issue` without releasing a new gem version.
-  QA_FAILURES_REPORTER_OPTIONS: "--dry-run"
+  QA_FAILURES_REPORTER_OPTIONS: ""
   QA_TESTCASE_SESSIONS_PROJECT: "gitlab-org/quality/testcase-sessions"
 
 .check-base:
@@ -95,8 +95,8 @@ release:
     reports:
       junit: gitlab-qa-run-*/**/rspec-*.xml
   script:
-    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
-    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
+    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
+    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
@@ -175,8 +175,8 @@ ee:sanity-framework:
 # The custom jobs are for manually running specific/alternative tests in MRs, so we don't report them in issues
 ce:custom-parallel:
   script:
-    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
-    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
+    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
+    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
   extends:
     - .test
     - .high-capacity
@@ -187,8 +187,8 @@ ce:custom-parallel:
 
 ee:custom-parallel:
   script:
-    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
-    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
+    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
+    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
   extends:
     - .test
     - .high-capacity

data/docs/running_specific_orchestrated_tests.md

@@ -4,7 +4,7 @@
 
 The [maven repository spec](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/specs/features/ee/browser_ui/5_package/maven_repository_spec.rb) creates a Maven artifact and links it to a GitLab project. The artifact is created within a [Maven docker image](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/service/docker_run/maven.rb#L8).
 
-Using `gitlab-qa` to run these tests reduces the liklihood of network errors between the maven container and GitLab instance.
+Using `gitlab-qa` to run these tests reduces the likelihood of network errors between the maven container and GitLab instance.
 
 To run this with `gitlab-qa` you can use the `Test::Instance::Image` that is needed for your test. For example:
 

data/docs/what_tests_can_be_run.md

@@ -84,6 +84,7 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `JIRA_ADMIN_PASSWORD` |-  | Password for authenticating with Jira server as admin. | No|
 | `CACHE_NAMESPACE_NAME` | `true` | Cache namespace name for groups. | No|
 | `DEPLOY_VERSION` |- | The version of GitLab being tested against. | No|
+| `GITLAB_QA_USER_AGENT` |- | The browser user-agent to use instead of the default Chrome user-agent. | No|
 
 ## [Supported Remote Grid environment variables](./running_against_remote_grid.md)
 

data/lib/gitlab/qa/report/generate_test_session.rb

@@ -6,6 +6,7 @@ module Gitlab
       class GenerateTestSession < ReportAsIssue
         private
 
+        # rubocop:disable Metrics/AbcSize
         def run!
           puts "Generating test results in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
 
@@ -18,11 +19,12 @@ module Gitlab
           issue = gitlab.create_issue(
             title: "Test session report | #{Runtime::Env.deploy_environment}",
             description: generate_description(tests),
-            labels: ['Quality', 'QA', 'triage report']
+            labels: ['Quality', 'QA', 'triage report', deploy_environment_label]
           )
 
           File.write('REPORT_ISSUE_URL', issue.web_url)
         end
+        # rubocop:enable Metrics/AbcSize
 
         def generate_description(tests)
           <<~MARKDOWN
@@ -32,6 +34,8 @@ module Gitlab
           * Pipeline: [#{Runtime::Env.ci_pipeline_id}](#{Runtime::Env.ci_pipeline_url})
           #{generate_summary(tests: tests)}
 
+          #{generate_failed_jobs_listing}
+
           #{generate_stages_listing(tests)}
 
           ## Release QA issue
@@ -55,6 +59,33 @@ module Gitlab
           MARKDOWN
         end
 
+        def generate_failed_jobs_listing
+          failed_jobs = []
+
+          client = Gitlab.client(
+            endpoint: Runtime::Env.ci_api_v4_url,
+            private_token: Runtime::Env.gitlab_ci_api_token)
+
+          gitlab.handle_gitlab_client_exceptions do
+            failed_jobs = client.pipeline_jobs(
+              Runtime::Env.ci_project_id,
+              Runtime::Env.ci_pipeline_id,
+              scope: 'failed')
+          end
+
+          listings = failed_jobs.map do |job|
+            allowed_to_fail = ' (allowed to fail)' if job.allow_failure
+
+            "* [#{job.name}](#{job.web_url})#{allowed_to_fail}"
+          end.join("\n")
+
+          <<~MARKDOWN.chomp if failed_jobs.any?
+          ## Failed jobs
+
+          #{listings}
+          MARKDOWN
+        end
+
         def generate_stages_listing(tests)
           generate_tests_by_stage(tests).map do |stage, tests_for_stage|
             tests_by_status = tests_for_stage.group_by(&:status)
@@ -185,7 +216,7 @@ module Gitlab
               tests_with_same_testcase.select(&:failure_issue)
 
             if tests_having_failure_issue.any?
-              items = tests_having_failure_issue.map do |test|
+              items = tests_having_failure_issue.uniq(&:failure_issue).map do |test|
                 "<li>[ ] [failure issue](#{test.failure_issue})</li>"
               end.join(' ')
 

data/lib/gitlab/qa/report/gitlab_issue_client.rb

@@ -109,6 +109,9 @@ module Gitlab
           pipeline = QA::Runtime::Env.pipeline_from_project_name
           channel = pipeline == "canary" ? "qa-production" : "qa-#{pipeline}"
           error_msg = warn_exception(e)
+
+          return unless QA::Runtime::Env.ci_commit_ref_name == 'master'
+
           slack_options = {
             channel: channel,
             icon_emoji: ':ci_failing:',

data/lib/gitlab/qa/report/relate_failure_issue.rb

@@ -10,7 +10,7 @@ module Gitlab
       # Uses the API to create or update GitLab issues with the results of tests from RSpec report files.
       class RelateFailureIssue < ReportAsIssue
         DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.05
-        STACKTRACE_REGEX = %r{### Stack trace\s*(```)\s*.*(Failure/Error: .+)(\1)}m.freeze
+        STACKTRACE_REGEX = %r{### Stack trace\s*(```)\s*.*(Failure/Error:.+)(\1)}m.freeze
         NEW_ISSUE_LABELS = Set.new(%w[QA Quality test failure::investigating priority::2]).freeze
 
         MultipleIssuesFound = Class.new(StandardError)
@@ -47,6 +47,7 @@ module Gitlab
             issue = find_or_create_issue(test)
             return unless issue
 
+            update_labels(issue, test)
             post_failed_job_note(issue, test)
             puts "  => Marked #{issue.web_url} as related to #{test.testcase}."
           rescue MultipleIssuesFound => e
@@ -81,17 +82,20 @@ module Gitlab
           # Search with the `search` param returns 500 errors, so we filter by ~QA and then filter further in Ruby
           failure_issues(test).each_with_object({}) do |issue, memo|
             relevant_issue_stacktrace = find_issue_stacktrace(issue)
-            unless relevant_issue_stacktrace
-              puts "  => [DEBUG] Found issue #{issue.web_url} but stacktrace doesn't match."
-              next
-            end
+            next unless relevant_issue_stacktrace
 
             distance = ld.call(first_test_failure_stacktrace, relevant_issue_stacktrace)
             diff_ratio = (distance.to_f / first_test_failure_stacktrace.size).round(3)
             if diff_ratio <= max_diff_ratio
-              memo[issue] = diff_ratio
+              puts "  => [DEBUG] Issue #{issue} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
+              # The `Gitlab::ObjectifiedHash` class overrides `#hash` which is used by `Hash#[]=` to compute the hash key.
+              # This leads to a `TypeError Exception: no implicit conversion of Hash into Integer` error, so we convert the object to a hash before using it as a Hash key.
+              # See:
+              # - https://gitlab.com/gitlab-org/gitlab-qa/-/merge_requests/587#note_453336995
+              # - https://github.com/NARKOZ/gitlab/commit/cbdbd1e32623f018a8fae39932a8e3bc4d929abb?_pjax=%23js-repo-pjax-container#r44484494
+              memo[issue.to_h] = diff_ratio
             else
-              puts "  => [DEBUG] Found issue #{issue.web_url} but stacktrace is too different (#{(diff_ratio * 100).round(2)}%)."
+              puts "  => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%)."
             end
           end
         end
@@ -102,7 +106,7 @@ module Gitlab
           if issue_stacktrace_match
             issue_stacktrace_match[2].gsub(/^#.*$/, '').strip
           else
-            puts "\n  => Stacktrace couldn't be found for #{issue.web_url}:\n\n#{issue.description}\n\n----------------------------------\n"
+            puts "  => [DEBUG] Stacktrace couldn't be found for #{issue.web_url}:\n\n#{issue.description}\n\n----------------------------------\n"
           end
         end
 
@@ -117,24 +121,35 @@ module Gitlab
             raise(MultipleIssuesFound, %(Too many issues found for test '#{test.name}' (`#{test.file}`)!))
           end
 
+          # Re-instantiate a `Gitlab::ObjectifiedHash` object after having converted it to a hash in #find_relevant_failure_issues above.
+          best_matching_issue = Gitlab::ObjectifiedHash.new(best_matching_issue)
+
           test.failure_issue ||= best_matching_issue.web_url
 
           [best_matching_issue, smaller_diff_ratio]
         end
 
         def new_issue_description(test)
-          super + "\n\n### Stack trace\n\n```\n#{test.failures.first['message_lines'].join("\n")}\n```"
+          super + [
+            "\n\n### Stack trace",
+            "```\n#{test.failures.first['message_lines'].join("\n")}\n```",
+            "First happened in #{test.ci_job_url}."
+          ].join("\n\n")
         end
 
         def new_issue_labels(test)
           NEW_ISSUE_LABELS + up_to_date_labels(test: test)
         end
 
+        def up_to_date_labels(test:, issue: nil)
+          super << deploy_environment_label
+        end
+
         def post_failed_job_note(issue, test)
           gitlab.create_issue_note(iid: issue.iid, note: "/relate #{test.testcase}")
         end
 
-        def title_from_test(test)
+        def new_issue_title(test)
           "Failure in #{super}"
         end
       end

data/lib/gitlab/qa/report/report_as_issue.rb

@@ -28,6 +28,10 @@ module Gitlab
           raise NotImplementedError
         end
 
+        def new_issue_title(test)
+          "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
+        end
+
         def new_issue_description(test)
           "### Full description\n\n#{search_safe(test.name)}\n\n### File path\n\n#{test.file}"
         end
@@ -100,6 +104,23 @@ module Gitlab
           labels
         end
 
+        def deploy_environment_label
+          environment = Runtime::Env.deploy_environment
+
+          case environment
+          when 'production'
+            'found:gitlab.com'
+          when 'canary', 'staging'
+            "found:#{environment}.gitlab.com"
+          when 'preprod'
+            'found:pre.gitlab.com'
+          when 'staging-orchestrated', 'nightly', 'master'
+            "found:#{environment}"
+          else
+            raise "No `found:*` label for the `#{environment}` environment!"
+          end
+        end
+
         def ee_test?(test)
           test.file =~ %r{features/ee/(api|browser_ui)}
         end
@@ -113,7 +134,7 @@ module Gitlab
         end
 
         def title_from_test(test)
-          title = "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
+          title = new_issue_title(test)
 
           return title unless title.length > MAX_TITLE_LENGTH
 

data/lib/gitlab/qa/report/results_in_issues.rb

@@ -25,8 +25,6 @@ module Gitlab
         end
 
         def report_test(test)
-          return if test.skipped
-
           puts "Reporting test: #{test.file} | #{test.name}"
 
           issue = find_issue(test)
@@ -34,16 +32,19 @@ module Gitlab
           if issue
             puts "Found existing issue: #{issue.web_url}"
           else
+            # Don't create new issues for skipped tests
+            return if test.skipped
+
             issue = create_issue(test)
             puts "Created new issue: #{issue.web_url}"
           end
 
           test.testcase ||= issue.web_url
 
-          update_labels(issue, test)
+          labels_updated = update_labels(issue, test)
           note_posted = note_status(issue, test)
 
-          if note_posted
+          if labels_updated || note_posted
             puts "Issue updated."
           else
             puts "Test passed, no update needed."
@@ -83,6 +84,7 @@ module Gitlab
         end
 
         def note_status(issue, test)
+          return false if test.skipped
           return false if test.failures.empty?
 
           note = note_content(test)

data/lib/gitlab/qa/runtime/env.rb

@@ -85,7 +85,8 @@ module Gitlab
           'JIRA_ADMIN_USERNAME' => :jira_admin_username,
           'JIRA_ADMIN_PASSWORD' => :jira_admin_password,
           'CACHE_NAMESPACE_NAME' => :cache_namespace_name,
-          'DEPLOY_VERSION' => :deploy_version
+          'DEPLOY_VERSION' => :deploy_version,
+          'GITLAB_QA_USER_AGENT' => :gitlab_qa_user_agent
         }.freeze
 
         ENV_VARIABLES.each do |env_name, method_name|
@@ -115,6 +116,14 @@ module Gitlab
           ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
         end
 
+        def gitlab_ci_api_token
+          ENV['GITLAB_CI_API_TOKEN']
+        end
+
+        def ci_api_v4_url
+          ENV['CI_API_V4_URL']
+        end
+
         def ci_job_name
           ENV['CI_JOB_NAME']
         end
@@ -135,8 +144,12 @@ module Gitlab
           ENV['CI_PIPELINE_ID']
         end
 
-        def ci_project_name
-          ENV['CI_PROJECT_NAME']
+        def ci_project_id
+          ENV['CI_PROJECT_ID']
+        end
+
+        def ci_commit_ref_name
+          ENV['CI_COMMIT_REF_NAME']
         end
 
         def pipeline_from_project_name

data/lib/gitlab/qa/version.rb

@@ -1,5 +1,5 @@
 module Gitlab
   module QA
-    VERSION = '6.10.0'.freeze
+    VERSION = '6.14.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 6.10.0
+  version: 6.14.0
 platform: ruby
 authors:
 - Grzegorz Bizon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-17 00:00:00.000000000 Z
+date: 2020-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control