gitlab-qa 6.8.0 → 6.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: daf050b7d871f8eefdc1fa68b052608c11ee186b927fe89accfceca15291ed45
-  data.tar.gz: 4510e98db8e41c11a1585070a39cd19769384917cff551195942955974dcd78d
+  metadata.gz: a7db5ca406c7143f6b4b8ac2d5061080f52157dbe77f23459fdf15919cc24209
+  data.tar.gz: b0dca304397f7c373ca94b41936f8d04db818b797dc9a09e05e31accfd8a9cfd
 SHA512:
-  metadata.gz: 3a3a6dc22aac1a77fe46f1379a43b489d08cc771b38ea629f9ff3a8f24fd9886f767ed3b3ca8aa18bfeabadc9019aefe2933261c969167e6c8e95f7f75e2c6a1
-  data.tar.gz: c612e3b42691a28864bcf703fdf89cbbf57f75817341f867d1e8486a25adcbfcf262f3506157841d734813e699e628620dd524af22b4f8da17c689a96c6116cf
+  metadata.gz: 25678e888d7a338ef9f54269927e16ab6d5f60bc4b37175185bda64174689d9aecf40ac35c7d31be15e6fb1305527604f4667f09aa6c89f42ba4e02c7187a68d
+  data.tar.gz: ff1c2808b0030677ba8420444e4cb303b0d58d6751511bef90a8bcbea98df6a98e1efea83945d7b6488f076eb8f978519198b4938dc9b99503bd97c0d2b9ea57

data/.gitlab-ci.yml

@@ -49,9 +49,9 @@ variables:
   QA_CAN_TEST_GIT_PROTOCOL_V2: "true"
   QA_CAN_TEST_PRAEFECT: "false"
   QA_TESTCASES_REPORTING_PROJECT: "gitlab-org/quality/testcases"
-  QA_FAILURES_REPORTING_PROJECT: "gitlab-org/gitlab"
+  QA_FAILURES_REPORTING_PROJECT: "rymai/gitlab-qa-issues"
   # The --dry-run or --max-diff-ratio option can be set to modify the behavior of `exe/gitlab-qa-report --relate-failure-issue` without releasing a new gem version.
-  QA_FAILURES_REPORTER_OPTIONS: "--dry-run"
+  QA_FAILURES_REPORTER_OPTIONS: ""
   QA_TESTCASE_SESSIONS_PROJECT: "gitlab-org/quality/testcase-sessions"
 
 .check-base:
@@ -95,8 +95,8 @@ release:
     reports:
       junit: gitlab-qa-run-*/**/rspec-*.xml
   script:
-    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
-    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
+    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
+    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
@@ -175,8 +175,8 @@ ee:sanity-framework:
 # The custom jobs are for manually running specific/alternative tests in MRs, so we don't report them in issues
 ce:custom-parallel:
   script:
-    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
-    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
+    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
+    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
   extends:
     - .test
     - .high-capacity
@@ -187,8 +187,8 @@ ce:custom-parallel:
 
 ee:custom-parallel:
   script:
-    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
-    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
+    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
+    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
   extends:
     - .test
     - .high-capacity

data/docs/running_specific_orchestrated_tests.md

@@ -4,7 +4,7 @@
 
 The [maven repository spec](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/specs/features/ee/browser_ui/5_package/maven_repository_spec.rb) creates a Maven artifact and links it to a GitLab project. The artifact is created within a [Maven docker image](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/service/docker_run/maven.rb#L8).
 
-Using `gitlab-qa` to run these tests reduces the liklihood of network errors between the maven container and GitLab instance.
+Using `gitlab-qa` to run these tests reduces the likelihood of network errors between the maven container and GitLab instance.
 
 To run this with `gitlab-qa` you can use the `Test::Instance::Image` that is needed for your test. For example:
 

data/docs/what_tests_can_be_run.md

@@ -84,6 +84,7 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `JIRA_ADMIN_PASSWORD` |-  | Password for authenticating with Jira server as admin. | No|
 | `CACHE_NAMESPACE_NAME` | `true` | Cache namespace name for groups. | No|
 | `DEPLOY_VERSION` |- | The version of GitLab being tested against. | No|
+| `GITLAB_QA_USER_AGENT` |- | The browser user-agent to use instead of the default Chrome user-agent. | No|
 
 ## [Supported Remote Grid environment variables](./running_against_remote_grid.md)
 

data/gitlab-qa.gemspec

@@ -27,8 +27,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rubocop', '~> 0.82.0'
   spec.add_development_dependency 'rubocop-rspec', '~> 1.36'
   spec.add_development_dependency 'webmock', '3.7.0'
-  spec.add_runtime_dependency 'activesupport', '~>  6.0.2'
-  spec.add_runtime_dependency 'gitlab', '~>  4.11.0'
+  spec.add_runtime_dependency 'activesupport', '~> 6.0.2'
+  spec.add_runtime_dependency 'gitlab', '~> 4.16.1'
   spec.add_runtime_dependency 'http', '4.3.0'
   spec.add_runtime_dependency 'nokogiri', '~> 1.10'
   spec.add_runtime_dependency 'table_print', '1.5.6'

data/lib/gitlab/qa/component/base.rb

@@ -53,6 +53,7 @@ module Gitlab
 
         def prepare
           prepare_docker_image
+          prepare_docker_container
           prepare_network
         end
 
@@ -70,6 +71,12 @@ module Gitlab
           docker.network_create(network)
         end
 
+        def prepare_docker_container
+          return unless docker.container_exists?(name)
+
+          docker.remove(name)
+        end
+
         def start # rubocop:disable Metrics/AbcSize
           docker.run(image, tag) do |command|
             command << "-d"

data/lib/gitlab/qa/docker/engine.rb

@@ -63,6 +63,14 @@ module Gitlab
           Docker::Command.execute("rm -f #{name}")
         end
 
+        def container_exists?(name)
+          Docker::Command.execute("container inspect #{name}")
+        rescue Docker::Shellout::StatusError
+          false
+        else
+          true
+        end
+
         def network_exists?(name)
           Docker::Command.execute("network inspect #{name}")
         rescue Docker::Shellout::StatusError

data/lib/gitlab/qa/report/generate_test_session.rb

@@ -161,11 +161,11 @@ module Gitlab
         end
 
         def generate_test_job(tests_with_same_testcase)
-          tests_with_same_testcase.map(&:ci_job_url).uniq.map do |ci_job_url|
-            ci_job_id = ci_job_url[/\d+\z/]
+          tests_with_same_testcase.map do |test|
+            ci_job_id = test.ci_job_url[/\d+\z/]
 
-            "[#{ci_job_id}](#{ci_job_url})"
-          end.join(', ')
+            "[#{ci_job_id}](#{test.ci_job_url})#{' ~"quarantine"' if test.quarantine?}"
+          end.uniq.join(', ')
         end
 
         def generate_test_status(tests_with_same_testcase)

data/lib/gitlab/qa/report/gitlab_issue_client.rb

@@ -59,7 +59,7 @@ module Gitlab
 
         def find_issue_discussions(iid:)
           handle_gitlab_client_exceptions do
-            Gitlab.issue_discussions(project, iid, order_by: 'created_at', sort: 'asc')
+            Gitlab.issue_discussions(project, iid, order_by: 'created_at', sort: 'asc').auto_paginate
           end
         end
 
@@ -109,6 +109,9 @@ module Gitlab
           pipeline = QA::Runtime::Env.pipeline_from_project_name
           channel = pipeline == "canary" ? "qa-production" : "qa-#{pipeline}"
           error_msg = warn_exception(e)
+
+          return unless QA::Runtime::Env.ci_commit_ref_name == 'master'
+
           slack_options = {
             channel: channel,
             icon_emoji: ':ci_failing:',

data/lib/gitlab/qa/report/relate_failure_issue.rb

@@ -10,7 +10,7 @@ module Gitlab
       # Uses the API to create or update GitLab issues with the results of tests from RSpec report files.
       class RelateFailureIssue < ReportAsIssue
         DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.05
-        STACKTRACE_REGEX = %r{### Stack trace\s*(```)\s*.*(Failure/Error: .+)(\1)}m.freeze
+        STACKTRACE_REGEX = %r{### Stack trace\s*(```)\s*.*(Failure/Error:.+)(\1)}m.freeze
         NEW_ISSUE_LABELS = Set.new(%w[QA Quality test failure::investigating priority::2]).freeze
 
         MultipleIssuesFound = Class.new(StandardError)
@@ -47,6 +47,7 @@ module Gitlab
             issue = find_or_create_issue(test)
             return unless issue
 
+            update_labels(issue, test)
             post_failed_job_note(issue, test)
             puts "  => Marked #{issue.web_url} as related to #{test.testcase}."
           rescue MultipleIssuesFound => e
@@ -81,17 +82,15 @@ module Gitlab
           # Search with the `search` param returns 500 errors, so we filter by ~QA and then filter further in Ruby
           failure_issues(test).each_with_object({}) do |issue, memo|
             relevant_issue_stacktrace = find_issue_stacktrace(issue)
-            unless relevant_issue_stacktrace
-              puts "  => [DEBUG] Found issue #{issue.web_url} but stacktrace doesn't match."
-              next
-            end
+            next unless relevant_issue_stacktrace
 
             distance = ld.call(first_test_failure_stacktrace, relevant_issue_stacktrace)
             diff_ratio = (distance.to_f / first_test_failure_stacktrace.size).round(3)
             if diff_ratio <= max_diff_ratio
+              puts "  => [DEBUG] Issue #{issue} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
               memo[issue] = diff_ratio
             else
-              puts "  => [DEBUG] Found issue #{issue.web_url} but stacktrace is too different (#{(diff_ratio * 100).round(2)}%)."
+              puts "  => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%)."
             end
           end
         end
@@ -102,7 +101,7 @@ module Gitlab
           if issue_stacktrace_match
             issue_stacktrace_match[2].gsub(/^#.*$/, '').strip
           else
-            puts "\n  => Stacktrace couldn't be found for #{issue.web_url}:\n\n#{issue.description}\n\n----------------------------------\n"
+            puts "  => [DEBUG] Stacktrace couldn't be found for #{issue.web_url}:\n\n#{issue.description}\n\n----------------------------------\n"
           end
         end
 
@@ -123,16 +122,45 @@ module Gitlab
         end
 
         def new_issue_description(test)
-          super + "\n\n### Stack trace\n\n```\n#{test.failures.first['message_lines'].join("\n")}\n```"
+          super + [
+            "\n\n### Stack trace",
+            "```\n#{test.failures.first['message_lines'].join("\n")}\n```",
+            "First happened in #{test.ci_job_url}."
+          ].join("\n\n")
+        end
+
+        def deploy_environment_label
+          environment = Runtime::Env.deploy_environment
+
+          case environment
+          when 'production'
+            'found:gitlab.com'
+          when 'canary', 'staging'
+            "found:#{environment}.gitlab.com"
+          when 'preprod'
+            'found:pre.gitlab.com'
+          when 'staging-orchestrated', 'nightly', 'master'
+            "found:#{environment}"
+          else
+            raise "No `found:*` label for the `#{environment}` environment!"
+          end
         end
 
         def new_issue_labels(test)
           NEW_ISSUE_LABELS + up_to_date_labels(test: test)
         end
 
+        def up_to_date_labels(test:, issue: nil)
+          super << deploy_environment_label
+        end
+
         def post_failed_job_note(issue, test)
           gitlab.create_issue_note(iid: issue.iid, note: "/relate #{test.testcase}")
         end
+
+        def new_issue_title(test)
+          "Failure in #{super}"
+        end
       end
     end
   end

data/lib/gitlab/qa/report/report_as_issue.rb

@@ -28,6 +28,10 @@ module Gitlab
           raise NotImplementedError
         end
 
+        def new_issue_title(test)
+          "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
+        end
+
         def new_issue_description(test)
           "### Full description\n\n#{search_safe(test.name)}\n\n### File path\n\n#{test.file}"
         end
@@ -76,7 +80,7 @@ module Gitlab
           gitlab.create_issue(
             title: title_from_test(test),
             description: new_issue_description(test),
-            labels: new_issue_labels(test)
+            labels: new_issue_labels(test).to_a
           )
         end
 
@@ -113,7 +117,7 @@ module Gitlab
         end
 
         def title_from_test(test)
-          title = "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
+          title = new_issue_title(test)
 
           return title unless title.length > MAX_TITLE_LENGTH
 

data/lib/gitlab/qa/report/results_in_issues.rb

@@ -25,8 +25,6 @@ module Gitlab
         end
 
         def report_test(test)
-          return if test.skipped
-
           puts "Reporting test: #{test.file} | #{test.name}"
 
           issue = find_issue(test)
@@ -34,16 +32,23 @@ module Gitlab
           if issue
             puts "Found existing issue: #{issue.web_url}"
           else
+            # Don't create new issues for skipped tests
+            return if test.skipped
+
             issue = create_issue(test)
             puts "Created new issue: #{issue.web_url}"
           end
 
           test.testcase ||= issue.web_url
 
-          update_labels(issue, test)
-          note_status(issue, test)
+          labels_updated = update_labels(issue, test)
+          note_posted = note_status(issue, test)
 
-          puts "Issue updated"
+          if labels_updated || note_posted
+            puts "Issue updated."
+          else
+            puts "Test passed, no update needed."
+          end
         end
 
         def find_issue(test)
@@ -79,7 +84,8 @@ module Gitlab
         end
 
         def note_status(issue, test)
-          return if test.failures.empty?
+          return false if test.skipped
+          return false if test.failures.empty?
 
           note = note_content(test)
 
@@ -88,6 +94,8 @@ module Gitlab
           end
 
           gitlab.create_issue_note(iid: issue.iid, note: note)
+
+          true
         end
 
         def note_content(test)
@@ -125,11 +133,7 @@ module Gitlab
         end
 
         def error_and_stack_trace(text)
-          result = text.strip[/Error:(.*)/m, 1].to_s
-
-          warn "Could not find `Error:` in text: #{text}" if result.empty?
-
-          result
+          text.strip[/Error:(.*)/m, 1].to_s
         end
       end
     end

data/lib/gitlab/qa/report/test_result.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'active_support/core_ext/object/blank'
+
 module Gitlab
   module QA
     module Report
@@ -78,6 +80,13 @@ module Gitlab
             report['failure_issue'] = new_failure_issue
           end
 
+          def quarantine?
+            # The value for 'quarantine' could be nil, a hash, a string,
+            # or true (if the test just has the :quarantine tag)
+            # But any non-nil or false value should means the test is in quarantine
+            report['quarantine'].present?
+          end
+
           private
 
           # rubocop:disable Metrics/AbcSize

data/lib/gitlab/qa/runtime/env.rb

@@ -8,6 +8,7 @@ module Gitlab
 
         # Variables that are used in tests and are passed through to the docker container that executes the tests.
         # These variables should be listed in /docs/what_tests_can_be_run.md#supported-gitlab-environment-variables
+        # unless they're defined elsewhere (e.g.: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html)
         ENV_VARIABLES = {
           'QA_REMOTE_GRID' => :remote_grid,
           'QA_REMOTE_GRID_USERNAME' => :remote_grid_username,
@@ -68,6 +69,7 @@ module Gitlab
           'CI_SERVER_PERSONAL_ACCESS_TOKEN' => :ci_server_personal_access_token,
           'CI_NODE_INDEX' => :ci_node_index,
           'CI_NODE_TOTAL' => :ci_node_total,
+          'CI_PROJECT_NAME' => :ci_project_name,
           'GITLAB_CI' => :gitlab_ci,
           'QA_SKIP_PULL' => :qa_skip_pull,
           'ELASTIC_URL' => :elastic_url,
@@ -83,7 +85,8 @@ module Gitlab
           'JIRA_ADMIN_USERNAME' => :jira_admin_username,
           'JIRA_ADMIN_PASSWORD' => :jira_admin_password,
           'CACHE_NAMESPACE_NAME' => :cache_namespace_name,
-          'DEPLOY_VERSION' => :deploy_version
+          'DEPLOY_VERSION' => :deploy_version,
+          'GITLAB_QA_USER_AGENT' => :gitlab_qa_user_agent
         }.freeze
 
         ENV_VARIABLES.each do |env_name, method_name|
@@ -137,6 +140,10 @@ module Gitlab
           ENV['CI_PROJECT_NAME']
         end
 
+        def ci_commit_ref_name
+          ENV['CI_COMMIT_REF_NAME']
+        end
+
         def pipeline_from_project_name
           if ci_project_name.to_s.start_with?('gitlab-qa')
             if ENV['TOP_UPSTREAM_SOURCE_JOB'].to_s.start_with?('https://ops.gitlab.net')

data/lib/gitlab/qa/version.rb

@@ -1,5 +1,5 @@
 module Gitlab
   module QA
-    VERSION = '6.8.0'.freeze
+    VERSION = '6.12.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 6.8.0
+  version: 6.12.0
 platform: ruby
 authors:
 - Grzegorz Bizon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-10 00:00:00.000000000 Z
+date: 2020-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -142,14 +142,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.11.0
+        version: 4.16.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.11.0
+        version: 4.16.1
 - !ruby/object:Gem::Dependency
   name: http
   requirement: !ruby/object:Gem::Requirement