gitlab-qa 6.11.0 → 6.15.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb6d9e91936be3f2ffdcee43a8693171f691714e086de83065c3ef44580f095e
-  data.tar.gz: a154de6d77c7ffa54426282f0bee792ed13380a5bdb95534fc6bdbc50791c82c
+  metadata.gz: 7423be5876c10c8dbace27a5fa18a3b0590a8bc49198f16b67f14d6b7435f9cc
+  data.tar.gz: 774eb7f437c5be8d533ac76e913883725d7b824eedeb3e4225f9200cb740cc80
 SHA512:
-  metadata.gz: b01ee4edc97797c0db5eeacc19dd427d7da082dcea78f7175921b17b73a128c529244aebf7129f2ff901da16496805cd9e6364fa6587db54676d10fcbc1f009d
-  data.tar.gz: d54a08f9c1e02c2bf16f214c3a468cf619438b4e9620cb052c0106c4a336c586218b41d3e050d5e39980d2e148f4ea77631e08b3cbe6afb25f34a1838e0281d6
+  metadata.gz: b680735a2f62976139e848396ae6a5b5bf23d27e25163d03003933a33be7d91e32644b20a5c2a00687a09821b97ccda0212c18f048789ed2b52ecac0eb23c22b
+  data.tar.gz: 5489e42c1191a98364f9f3e30a4b9768f0105e2b149ec731f6142334ac10336751e108116736ec214a190accf898dfed9d5c205b44dd9bed8869076eee857db7

data/.gitlab-ci.yml

@@ -1,8 +1,8 @@
 stages:
   - check
-  - release
   - test
   - report
+  - deploy
   - notify
 
 default:
@@ -35,13 +35,6 @@ workflow:
     # For triggers from GitLab MR pipelines (and pipelines from other projects), create a pipeline
     - if: '$CI_PIPELINE_SOURCE == "pipeline"'
 
-.default-rules:
-  rules:
-    - if: '$CI_COMMIT_TAG || $RELEASE'
-      when: never
-    - if: '$RELEASE == null && $CI_JOB_NAME =~ /staging/'
-      when: manual
-
 variables:
   DOCKER_DRIVER: overlay2
   DOCKER_HOST: tcp://docker:2375
@@ -65,21 +58,6 @@ rubocop:
 rspec:
   extends: .check-base
 
-release:
-  stage: release
-  rules:
-    - if: '$CI_COMMIT_TAG'
-  script:
-    - gem update --system
-    - ruby --version
-    - gem env version
-    - gem build gitlab-qa.gemspec
-    - gem push gitlab-qa*.gem
-  artifacts:
-    paths:
-      - gitlab-qa*.gem
-    expire_in: 30 days
-
 .test:
   stage: test
   services:
@@ -95,8 +73,8 @@ release:
     reports:
       junit: gitlab-qa-run-*/**/rspec-*.xml
   script:
-    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
-    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
+    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
+    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
@@ -107,6 +85,9 @@ release:
   variables:
     DEFAULT_RELEASE: "CE"
   rules:
+    # Don't run E2E jobs on release pipelines
+    - changes: ["lib/**/version.rb"]
+      when: never
     - if: '$CI_COMMIT_TAG || $RELEASE =~ /gitlab-ee/'
       when: never
     - if: '$RELEASE == null && $CI_JOB_NAME =~ /quarantine|custom/'
@@ -121,6 +102,9 @@ release:
   variables:
     DEFAULT_RELEASE: "EE"
   rules:
+    # Don't run E2E jobs on release pipelines
+    - changes: ["lib/**/version.rb"]
+      when: never
     - if: '$CI_COMMIT_TAG || $RELEASE =~ /gitlab-ce/'
       when: never
     - if: '$RELEASE == null && $CI_JOB_NAME =~ /quarantine|custom/'
@@ -132,7 +116,14 @@ release:
     - if: '$RELEASE == null || $RELEASE =~ /gitlab-ee/ || $CI_MERGE_REQUEST_ID || $CI_COMMIT_REF_NAME == "master"'
 
 .only-qa:
-  extends: .default-rules
+  rules:
+    # Don't run E2E jobs on release pipelines
+    - changes: ["lib/**/version.rb"]
+      when: never
+    - if: '$CI_COMMIT_TAG || $RELEASE'
+      when: never
+    - if: '$RELEASE == null && $CI_JOB_NAME =~ /staging/'
+      when: manual
 
 .high-capacity:
   tags:
@@ -175,8 +166,8 @@ ee:sanity-framework:
 # The custom jobs are for manually running specific/alternative tests in MRs, so we don't report them in issues
 ce:custom-parallel:
   script:
-    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
-    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
+    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
+    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
   extends:
     - .test
     - .high-capacity
@@ -187,8 +178,8 @@ ce:custom-parallel:
 
 ee:custom-parallel:
   script:
-    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
-    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
+    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
+    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
   extends:
     - .test
     - .high-capacity
@@ -1012,3 +1003,7 @@ notify_slack:
     - echo "CI_PIPELINE_URL is $CI_PIPELINE_URL"
     - echo "TOP_UPSTREAM_SOURCE_JOB is $TOP_UPSTREAM_SOURCE_JOB"
     - 'bin/slack $NOTIFY_CHANNEL "☠️ QA against $RELEASE failed! ☠️ See the test session report: $(cat REPORT_ISSUE_URL), and pipeline: $CI_PIPELINE_URL (triggered from $TOP_UPSTREAM_SOURCE_JOB)" ci_failing'
+
+include:
+  - project: 'gitlab-org/quality/pipeline-common'
+    file: '/ci/gem-release.yml'

data/.gitlab/merge_request_templates/Release.md

@@ -30,6 +30,6 @@ with the latest commit from https://gitlab.com/gitlab-org/gitlab-qa/commits/mast
   - [ ] Release notes are accurate.
 
 - Checklist after merging:
-  - [ ] [Create a tag for the new release version](docs/release_process.md#how-to).
+  - [ ] [Update the release notes for the newly created tag](docs/release_process.md#how-to).
 
 /label ~Quality ~"feature::maintenance"

data/docs/release_process.md

@@ -24,12 +24,13 @@ when we make a change - no matter the size of the change.
   - If not, update [`lib/gitlab/qa/version.rb`] to an appropriate [semantic version](https://semver.org) in a new merge request using the [release template](https://gitlab.com/gitlab-org/gitlab-qa/blob/master/.gitlab/merge_request_templates/Release.md)
     and title the MR like `"Bump version to <version>"`.
 - Merge the merge request.
-- Create a new tag via the UI (https://gitlab.com/gitlab-org/gitlab-qa/-/tags/new).
-  * **Tag name**: The same version found in [`lib/gitlab/qa/version.rb`], prefixed with `v`, e.g. if the version is `4.7.1`, the tag would be `v4.7.1`.
-  * **Message**: Set it to "Version x.y.z", e.g. if the version is `4.7.1`, the message would be "Version 4.7.1".
+- The new version should automatically be tagged and pushed to Rubygems by the `gem-release` job in the merge commit pipeline.
+- Update the release notes for the newly created tag (https://gitlab.com/gitlab-org/gitlab-qa/-/tags):
   * **Release notes**: Copy the release notes from the merge request.
-  * Click *Create Tag*.
-  
-GitLab will then start a pipeline for this new tag, and the `release` job will build and push the new version of `gitlab-qa` to RubyGems.
+
+Note: The `gem-release` job uses:
+
+- the `GITLAB_API_TOKEN` environment variable to authenticate against GitLab.com's API in order to create the tag
+- the `GEM_HOST_API_KEY` environment variable to authenticate against Rubygems.org's API in order to release the gem
 
 [`lib/gitlab/qa/version.rb`]: https://gitlab.com/gitlab-org/gitlab-qa/blob/master/lib/gitlab/qa/version.rb#L3

data/docs/what_tests_can_be_run.md

@@ -84,6 +84,7 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `JIRA_ADMIN_PASSWORD` |-  | Password for authenticating with Jira server as admin. | No|
 | `CACHE_NAMESPACE_NAME` | `true` | Cache namespace name for groups. | No|
 | `DEPLOY_VERSION` |- | The version of GitLab being tested against. | No|
+| `GITLAB_QA_USER_AGENT` |- | The browser user-agent to use instead of the default Chrome user-agent. | No|
 
 ## [Supported Remote Grid environment variables](./running_against_remote_grid.md)
 

data/lib/gitlab/qa/report/generate_test_session.rb

@@ -6,6 +6,7 @@ module Gitlab
       class GenerateTestSession < ReportAsIssue
         private
 
+        # rubocop:disable Metrics/AbcSize
         def run!
           puts "Generating test results in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
 
@@ -16,22 +17,26 @@ module Gitlab
           end
 
           issue = gitlab.create_issue(
-            title: "Test session report | #{Runtime::Env.deploy_environment}",
+            title: "Test session report | #{pipeline}",
             description: generate_description(tests),
-            labels: ['Quality', 'QA', 'triage report']
+            labels: ['Quality', 'QA', 'triage report', pipeline_name_label]
           )
 
           File.write('REPORT_ISSUE_URL', issue.web_url)
         end
+        # rubocop:enable Metrics/AbcSize
 
         def generate_description(tests)
           <<~MARKDOWN
           ## Session summary
 
           * Deploy version: #{Runtime::Env.deploy_version}
-          * Pipeline: [#{Runtime::Env.ci_pipeline_id}](#{Runtime::Env.ci_pipeline_url})
+          * Deploy environment: #{Runtime::Env.deploy_environment}
+          * Pipeline: #{Runtime::Env.pipeline_from_project_name} [#{Runtime::Env.ci_pipeline_id}](#{Runtime::Env.ci_pipeline_url})
           #{generate_summary(tests: tests)}
 
+          #{generate_failed_jobs_listing}
+
           #{generate_stages_listing(tests)}
 
           ## Release QA issue
@@ -55,6 +60,33 @@ module Gitlab
           MARKDOWN
         end
 
+        def generate_failed_jobs_listing
+          failed_jobs = []
+
+          client = Gitlab.client(
+            endpoint: Runtime::Env.ci_api_v4_url,
+            private_token: Runtime::Env.gitlab_ci_api_token)
+
+          gitlab.handle_gitlab_client_exceptions do
+            failed_jobs = client.pipeline_jobs(
+              Runtime::Env.ci_project_id,
+              Runtime::Env.ci_pipeline_id,
+              scope: 'failed')
+          end
+
+          listings = failed_jobs.map do |job|
+            allowed_to_fail = ' (allowed to fail)' if job.allow_failure
+
+            "* [#{job.name}](#{job.web_url})#{allowed_to_fail}"
+          end.join("\n")
+
+          <<~MARKDOWN.chomp if failed_jobs.any?
+          ## Failed jobs
+
+          #{listings}
+          MARKDOWN
+        end
+
         def generate_stages_listing(tests)
           generate_tests_by_stage(tests).map do |stage, tests_for_stage|
             tests_by_status = tests_for_stage.group_by(&:status)
@@ -185,7 +217,7 @@ module Gitlab
               tests_with_same_testcase.select(&:failure_issue)
 
             if tests_having_failure_issue.any?
-              items = tests_having_failure_issue.map do |test|
+              items = tests_having_failure_issue.uniq(&:failure_issue).map do |test|
                 "<li>[ ] [failure issue](#{test.failure_issue})</li>"
               end.join(' ')
 

data/lib/gitlab/qa/report/relate_failure_issue.rb

@@ -88,7 +88,12 @@ module Gitlab
             diff_ratio = (distance.to_f / first_test_failure_stacktrace.size).round(3)
             if diff_ratio <= max_diff_ratio
               puts "  => [DEBUG] Issue #{issue} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
-              memo[issue] = diff_ratio
+              # The `Gitlab::ObjectifiedHash` class overrides `#hash` which is used by `Hash#[]=` to compute the hash key.
+              # This leads to a `TypeError Exception: no implicit conversion of Hash into Integer` error, so we convert the object to a hash before using it as a Hash key.
+              # See:
+              # - https://gitlab.com/gitlab-org/gitlab-qa/-/merge_requests/587#note_453336995
+              # - https://github.com/NARKOZ/gitlab/commit/cbdbd1e32623f018a8fae39932a8e3bc4d929abb?_pjax=%23js-repo-pjax-container#r44484494
+              memo[issue.to_h] = diff_ratio
             else
               puts "  => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%)."
             end
@@ -116,6 +121,9 @@ module Gitlab
             raise(MultipleIssuesFound, %(Too many issues found for test '#{test.name}' (`#{test.file}`)!))
           end
 
+          # Re-instantiate a `Gitlab::ObjectifiedHash` object after having converted it to a hash in #find_relevant_failure_issues above.
+          best_matching_issue = Gitlab::ObjectifiedHash.new(best_matching_issue)
+
           test.failure_issue ||= best_matching_issue.web_url
 
           [best_matching_issue, smaller_diff_ratio]
@@ -129,29 +137,12 @@ module Gitlab
           ].join("\n\n")
         end
 
-        def deploy_environment_label
-          environment = Runtime::Env.deploy_environment
-
-          case environment
-          when 'production'
-            'found:gitlab.com'
-          when 'canary', 'staging'
-            "found:#{environment}.gitlab.com"
-          when 'preprod'
-            'found:pre.gitlab.com'
-          when 'staging-orchestrated', 'nightly', 'master'
-            "found:#{environment}"
-          else
-            raise "No `found:*` label for the `#{environment}` environment!"
-          end
-        end
-
         def new_issue_labels(test)
           NEW_ISSUE_LABELS + up_to_date_labels(test: test)
         end
 
         def up_to_date_labels(test:, issue: nil)
-          super << deploy_environment_label
+          super << pipeline_name_label
         end
 
         def post_failed_job_note(issue, test)

data/lib/gitlab/qa/report/report_as_issue.rb

@@ -104,6 +104,21 @@ module Gitlab
           labels
         end
 
+        def pipeline_name_label
+          case pipeline
+          when 'production'
+            'found:gitlab.com'
+          when 'canary', 'staging'
+            "found:#{pipeline}.gitlab.com"
+          when 'preprod'
+            'found:pre.gitlab.com'
+          when 'staging-orchestrated', 'nightly', 'master'
+            "found:#{pipeline}"
+          else
+            raise "No `found:*` label for the `#{pipeline}` pipeline!"
+          end
+        end
+
         def ee_test?(test)
           test.file =~ %r{features/ee/(api|browser_ui)}
         end

data/lib/gitlab/qa/runtime/env.rb

@@ -85,7 +85,8 @@ module Gitlab
           'JIRA_ADMIN_USERNAME' => :jira_admin_username,
           'JIRA_ADMIN_PASSWORD' => :jira_admin_password,
           'CACHE_NAMESPACE_NAME' => :cache_namespace_name,
-          'DEPLOY_VERSION' => :deploy_version
+          'DEPLOY_VERSION' => :deploy_version,
+          'GITLAB_QA_USER_AGENT' => :gitlab_qa_user_agent
         }.freeze
 
         ENV_VARIABLES.each do |env_name, method_name|
@@ -115,6 +116,14 @@ module Gitlab
           ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
         end
 
+        def gitlab_ci_api_token
+          ENV['GITLAB_CI_API_TOKEN']
+        end
+
+        def ci_api_v4_url
+          ENV['CI_API_V4_URL']
+        end
+
         def ci_job_name
           ENV['CI_JOB_NAME']
         end
@@ -135,8 +144,8 @@ module Gitlab
           ENV['CI_PIPELINE_ID']
         end
 
-        def ci_project_name
-          ENV['CI_PROJECT_NAME']
+        def ci_project_id
+          ENV['CI_PROJECT_ID']
         end
 
         def ci_commit_ref_name

data/lib/gitlab/qa/version.rb

@@ -1,5 +1,5 @@
 module Gitlab
   module QA
-    VERSION = '6.11.0'.freeze
+    VERSION = '6.15.1'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 6.11.0
+  version: 6.15.1
 platform: ruby
 authors:
 - Grzegorz Bizon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-20 00:00:00.000000000 Z
+date: 2020-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control