gitlab-qa 6.14.0 → 6.15.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 79cd3084cf93dc40c0935800fd13991f2b8e057fd08aca2b91cffddc776675c3
-  data.tar.gz: 0ad9aecd55123681543e84b7ff2150ec8d4a0f004374dd43cd7ced64a89b11f9
+  metadata.gz: 40d1c050d428a575441e8608a95d08de4e683fcce3e761407b3af5d528ed5fb0
+  data.tar.gz: f94798ad1e5e0f7dec06421bca8dabd7bf6cb9fef05afe8842072bb1734507b7
 SHA512:
-  metadata.gz: 29b7032d6ad65269e2aa27d03bea3bff2805bd160550c69fefd08475bb60ad5160c2417422e58b2b29d1ef4c2c31d3ebc6661cb0cbb151522dedb6dda97e64d7
-  data.tar.gz: 16d5d8ae03e4542c7518e1466723eee32e927a523890c3d56c565df2cabd363a180116a2d4db5d2eefd323dadb355f1471c7d7da9a0fcff61eb13f607a174b95
+  metadata.gz: f040e24b9bea5771f89f18088b31d99910827922dec8e9cea7b2e8e59a932aa6b7ecc7e93024c4af026cbe4da287af83703fb758dca7395899e833e8c8018410
+  data.tar.gz: 13213f41ff01e4429b66c64cbe25347bab733785f76417c94696732cd3f0c6502ea8a845c2e43af7c452225bdcf6ea7c66ae7df739c2d1d3d2253d3da838dbab

data/.gitlab-ci.yml

@@ -1,8 +1,8 @@
 stages:
   - check
-  - release
   - test
   - report
+  - deploy
   - notify
 
 default:
@@ -35,13 +35,6 @@ workflow:
     # For triggers from GitLab MR pipelines (and pipelines from other projects), create a pipeline
     - if: '$CI_PIPELINE_SOURCE == "pipeline"'
 
-.default-rules:
-  rules:
-    - if: '$CI_COMMIT_TAG || $RELEASE'
-      when: never
-    - if: '$RELEASE == null && $CI_JOB_NAME =~ /staging/'
-      when: manual
-
 variables:
   DOCKER_DRIVER: overlay2
   DOCKER_HOST: tcp://docker:2375
@@ -65,21 +58,6 @@ rubocop:
 rspec:
   extends: .check-base
 
-release:
-  stage: release
-  rules:
-    - if: '$CI_COMMIT_TAG'
-  script:
-    - gem update --system
-    - ruby --version
-    - gem env version
-    - gem build gitlab-qa.gemspec
-    - gem push gitlab-qa*.gem
-  artifacts:
-    paths:
-      - gitlab-qa*.gem
-    expire_in: 30 days
-
 .test:
   stage: test
   services:
@@ -107,6 +85,10 @@ release:
   variables:
     DEFAULT_RELEASE: "CE"
   rules:
+    # Don't run E2E jobs on release pipelines
+    - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab-qa" && $RELEASE == null'
+      changes: ["lib/**/version.rb"]
+      when: never
     - if: '$CI_COMMIT_TAG || $RELEASE =~ /gitlab-ee/'
       when: never
     - if: '$RELEASE == null && $CI_JOB_NAME =~ /quarantine|custom/'
@@ -121,6 +103,10 @@ release:
   variables:
     DEFAULT_RELEASE: "EE"
   rules:
+    # Don't run E2E jobs on release pipelines
+    - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab-qa" && $RELEASE == null'
+      changes: ["lib/**/version.rb"]
+      when: never
     - if: '$CI_COMMIT_TAG || $RELEASE =~ /gitlab-ce/'
       when: never
     - if: '$RELEASE == null && $CI_JOB_NAME =~ /quarantine|custom/'
@@ -132,7 +118,15 @@ release:
     - if: '$RELEASE == null || $RELEASE =~ /gitlab-ee/ || $CI_MERGE_REQUEST_ID || $CI_COMMIT_REF_NAME == "master"'
 
 .only-qa:
-  extends: .default-rules
+  rules:
+    # Don't run E2E jobs on release pipelines
+    - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab-qa" && $RELEASE == null'
+      changes: ["lib/**/version.rb"]
+      when: never
+    - if: '$CI_COMMIT_TAG || $RELEASE'
+      when: never
+    - if: '$RELEASE == null && $CI_JOB_NAME =~ /staging/'
+      when: manual
 
 .high-capacity:
   tags:
@@ -474,7 +468,6 @@ ee:mattermost-quarantine:
 ee:geo:
   extends:
     - .test
-    - .high-capacity
     - .ee-qa
     - .rspec-report-opts
   variables:
@@ -483,7 +476,6 @@ ee:geo:
 ee:geo-quarantine:
   extends:
     - .test
-    - .high-capacity
     - .ee-qa
     - .quarantine
     - .rspec-report-opts
@@ -1012,3 +1004,7 @@ notify_slack:
     - echo "CI_PIPELINE_URL is $CI_PIPELINE_URL"
     - echo "TOP_UPSTREAM_SOURCE_JOB is $TOP_UPSTREAM_SOURCE_JOB"
     - 'bin/slack $NOTIFY_CHANNEL "☠️ QA against $RELEASE failed! ☠️ See the test session report: $(cat REPORT_ISSUE_URL), and pipeline: $CI_PIPELINE_URL (triggered from $TOP_UPSTREAM_SOURCE_JOB)" ci_failing'
+
+include:
+  - project: 'gitlab-org/quality/pipeline-common'
+    file: '/ci/gem-release.yml'

data/.gitlab/merge_request_templates/Release.md

@@ -30,6 +30,6 @@ with the latest commit from https://gitlab.com/gitlab-org/gitlab-qa/commits/mast
   - [ ] Release notes are accurate.
 
 - Checklist after merging:
-  - [ ] [Create a tag for the new release version](docs/release_process.md#how-to).
+  - [ ] [Update the release notes for the newly created tag](docs/release_process.md#how-to).
 
 /label ~Quality ~"feature::maintenance"

data/docs/release_process.md

@@ -24,12 +24,13 @@ when we make a change - no matter the size of the change.
   - If not, update [`lib/gitlab/qa/version.rb`] to an appropriate [semantic version](https://semver.org) in a new merge request using the [release template](https://gitlab.com/gitlab-org/gitlab-qa/blob/master/.gitlab/merge_request_templates/Release.md)
     and title the MR like `"Bump version to <version>"`.
 - Merge the merge request.
-- Create a new tag via the UI (https://gitlab.com/gitlab-org/gitlab-qa/-/tags/new).
-  * **Tag name**: The same version found in [`lib/gitlab/qa/version.rb`], prefixed with `v`, e.g. if the version is `4.7.1`, the tag would be `v4.7.1`.
-  * **Message**: Set it to "Version x.y.z", e.g. if the version is `4.7.1`, the message would be "Version 4.7.1".
+- The new version should automatically be tagged and pushed to Rubygems by the `gem-release` job in the merge commit pipeline.
+- Update the release notes for the newly created tag (https://gitlab.com/gitlab-org/gitlab-qa/-/tags):
   * **Release notes**: Copy the release notes from the merge request.
-  * Click *Create Tag*.
-  
-GitLab will then start a pipeline for this new tag, and the `release` job will build and push the new version of `gitlab-qa` to RubyGems.
+
+Note: The `gem-release` job uses:
+
+- the `GITLAB_API_TOKEN` environment variable to authenticate against GitLab.com's API in order to create the tag
+- the `GEM_HOST_API_KEY` environment variable to authenticate against Rubygems.org's API in order to release the gem
 
 [`lib/gitlab/qa/version.rb`]: https://gitlab.com/gitlab-org/gitlab-qa/blob/master/lib/gitlab/qa/version.rb#L3

data/lib/gitlab/qa/report/generate_test_session.rb

@@ -17,9 +17,9 @@ module Gitlab
           end
 
           issue = gitlab.create_issue(
-            title: "Test session report | #{Runtime::Env.deploy_environment}",
+            title: "Test session report | #{pipeline}",
             description: generate_description(tests),
-            labels: ['Quality', 'QA', 'triage report', deploy_environment_label]
+            labels: ['Quality', 'QA', 'triage report', pipeline_name_label]
           )
 
           File.write('REPORT_ISSUE_URL', issue.web_url)
@@ -27,20 +27,19 @@ module Gitlab
         # rubocop:enable Metrics/AbcSize
 
         def generate_description(tests)
-          <<~MARKDOWN
+          <<~MARKDOWN.rstrip
           ## Session summary
 
           * Deploy version: #{Runtime::Env.deploy_version}
-          * Pipeline: [#{Runtime::Env.ci_pipeline_id}](#{Runtime::Env.ci_pipeline_url})
+          * Deploy environment: #{Runtime::Env.deploy_environment}
+          * Pipeline: #{Runtime::Env.pipeline_from_project_name} [#{Runtime::Env.ci_pipeline_id}](#{Runtime::Env.ci_pipeline_url})
           #{generate_summary(tests: tests)}
 
           #{generate_failed_jobs_listing}
 
           #{generate_stages_listing(tests)}
 
-          ## Release QA issue
-
-          * #{Runtime::Env.qa_issue_url}
+          #{generate_qa_issue_relation}
           MARKDOWN
         end
 
@@ -228,6 +227,18 @@ module Gitlab
             '-'
           end
         end
+
+        def generate_qa_issue_relation
+          return unless Runtime::Env.qa_issue_url
+
+          <<~MARKDOWN.chomp
+          ## Release QA issue
+
+          * #{Runtime::Env.qa_issue_url}
+
+          /relate #{Runtime::Env.qa_issue_url}
+          MARKDOWN
+        end
       end
     end
   end

data/lib/gitlab/qa/report/prepare_stage_reports.rb

@@ -34,7 +34,10 @@ module Gitlab
               #  /qa/specs/features/ee/api/<stage>
               #  /qa/specs/features/ee/browser_ui/<stage>
               # For now we assume the Stage is whatever follows api/ or browser_ui/
-              stage = strip_number_prefix(test['file'].match(%r{(api|browser_ui)/([a-z0-9_]+)}i)[2])
+              test_path_match = test['file'].match(%r{(api|browser_ui)/([a-z0-9_]+)}i)
+              next unless test_path_match
+
+              stage = strip_number_prefix(test_path_match[2])
               testcases[stage] = [] unless testcases.key?(stage)
               testcases[stage] << test
             end

data/lib/gitlab/qa/report/relate_failure_issue.rb

@@ -10,7 +10,7 @@ module Gitlab
       # Uses the API to create or update GitLab issues with the results of tests from RSpec report files.
       class RelateFailureIssue < ReportAsIssue
         DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.05
-        STACKTRACE_REGEX = %r{### Stack trace\s*(```)\s*.*(Failure/Error:.+)(\1)}m.freeze
+        STACKTRACE_REGEX = %r{### Stack trace\s*(```)\s*(?!.*Failure/Error:)?(.+)(\1)}m.freeze
         NEW_ISSUE_LABELS = Set.new(%w[QA Quality test failure::investigating priority::2]).freeze
 
         MultipleIssuesFound = Class.new(StandardError)
@@ -85,7 +85,8 @@ module Gitlab
             next unless relevant_issue_stacktrace
 
             distance = ld.call(first_test_failure_stacktrace, relevant_issue_stacktrace)
-            diff_ratio = (distance.to_f / first_test_failure_stacktrace.size).round(3)
+            diff_ratio = distance.zero? ? 0.0 : (distance.to_f / first_test_failure_stacktrace.size).round(3)
+
             if diff_ratio <= max_diff_ratio
               puts "  => [DEBUG] Issue #{issue} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
               # The `Gitlab::ObjectifiedHash` class overrides `#hash` which is used by `Hash#[]=` to compute the hash key.
@@ -142,7 +143,7 @@ module Gitlab
         end
 
         def up_to_date_labels(test:, issue: nil)
-          super << deploy_environment_label
+          super << pipeline_name_label
         end
 
         def post_failed_job_note(issue, test)

data/lib/gitlab/qa/report/report_as_issue.rb

@@ -104,20 +104,18 @@ module Gitlab
           labels
         end
 
-        def deploy_environment_label
-          environment = Runtime::Env.deploy_environment
-
-          case environment
+        def pipeline_name_label
+          case pipeline
           when 'production'
             'found:gitlab.com'
           when 'canary', 'staging'
-            "found:#{environment}.gitlab.com"
+            "found:#{pipeline}.gitlab.com"
           when 'preprod'
             'found:pre.gitlab.com'
           when 'staging-orchestrated', 'nightly', 'master'
-            "found:#{environment}"
+            "found:#{pipeline}"
           else
-            raise "No `found:*` label for the `#{environment}` environment!"
+            raise "No `found:*` label for the `#{pipeline}` pipeline!"
           end
         end
 

data/lib/gitlab/qa/version.rb

@@ -1,5 +1,5 @@
 module Gitlab
   module QA
-    VERSION = '6.14.0'.freeze
+    VERSION = '6.15.4'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 6.14.0
+  version: 6.15.4
 platform: ruby
 authors:
 - Grzegorz Bizon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-11-30 00:00:00.000000000 Z
+date: 2020-12-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control