gitlab-qa 6.17.0 → 6.17.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fc3ae23427dadce6649bb30dbc58dada153af79f60efb265edd5f7439facc56f
-  data.tar.gz: 9308ba475631df203f08503929787c1015fb4313b455d30a8fee2175016b1429
+  metadata.gz: 9f01ee39d7d17bf54f28ad2453823912d17ef0ef93a3eef47aca9b60519074c3
+  data.tar.gz: 994940086f1f623e5b31592747b8cd3d8fe32b383575c16cd4127fe9860cb1a2
 SHA512:
-  metadata.gz: 89c9c13ae10ff6e8f0076fb02d365cb45ede7320901bc5b4e4ca25ae935b9af26bc7da4d48bdab172ea6c3b74092032e5c9c5f04704ee7f98b38b67a594f81e5
-  data.tar.gz: 246f02d2389dd6db83ee1aac5e7dbc3e95ebfb12f9524f11181decf68ec5c9906adaa88531da62f66c2037c800de86084e4741193c3eaced1229ce135610fb1b
+  metadata.gz: 5648961ce9819c2b0e7262dea58fa90b04c0e998cbf0de8a183fd47e0c8c9ab0a26a075f38756c4bee8ffb20c87eabe1d7fe627145e8a11adad075a02dafe86f
+  data.tar.gz: 1cb3b8715444d2e2837fdde66069810bb27ef4fad76e0375011d107621e4afec48c2820110ee0a306bb0e89654af4d08325968027e9d8dca9aa209ef4a7ec359

data/docs/what_tests_can_be_run.md

@@ -1,4 +1,3 @@
-
 # What tests can be run?
 [[_TOC_]]
 ## The two types of QA tests
@@ -142,6 +141,30 @@ To run EE tests, the `EE_LICENSE` environment variable needs to be set:
 
 `$ export EE_LICENSE=$(cat /path/to/GitLab.gitlab_license)`
 
+## Running a specific test (or set of tests)
+
+In most of the scenarios listed below, if you don't want to run all the tests
+it's possible to specify one or more tests. The framework uses RSpec, so tests can be
+specified as you would when using RSpec.
+
+For example, the following would run `create_merge_request_spec.rb`:
+
+```shell
+$ gitlab-qa Test::Instance::Image EE -- qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb
+```
+
+While the following would run all Create UI tests:
+
+```shell
+$ gitlab-qa Test::Instance::Image EE -- qa/specs/features/browser_ui/3_create
+```
+
+And the following would run all Create API tests as well as UI tests:
+
+```shell
+$ gitlab-qa Test::Instance::Image EE -- qa/specs/features/browser_ui/3_create qa/specs/features/api/3_create
+```
+
 ## Examples
 
 ### `Test::Instance::Image CE|EE|<full image address>`

data/lib/gitlab/qa/report/generate_test_session.rb

@@ -22,6 +22,11 @@ module Gitlab
             labels: ['Quality', 'QA', 'triage report', pipeline_name_label]
           )
 
+          # Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/295493
+          gitlab.create_issue_note(
+            iid: issue.iid,
+            note: "/relate #{Runtime::Env.qa_issue_url}")
+
           File.write('REPORT_ISSUE_URL', issue.web_url)
         end
         # rubocop:enable Metrics/AbcSize

data/lib/gitlab/qa/report/relate_failure_issue.rb

@@ -10,7 +10,8 @@ module Gitlab
       # Uses the API to create or update GitLab issues with the results of tests from RSpec report files.
       class RelateFailureIssue < ReportAsIssue
         DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.05
-        STACKTRACE_REGEX = %r{### Stack trace\s*(```)\s*(?!.*Failure/Error:)?(.+)(\1)}m.freeze
+        FAILURE_STACKTRACE_REGEX = %r{((.*Failure\/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m.freeze
+        ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)/m.freeze
         NEW_ISSUE_LABELS = Set.new(%w[QA Quality test failure::investigating priority::2]).freeze
 
         MultipleIssuesFound = Class.new(StandardError)
@@ -77,7 +78,8 @@ module Gitlab
 
         def find_relevant_failure_issues(test) # rubocop:disable Metrics/AbcSize
           ld = Class.new.extend(Gem::Text).method(:levenshtein_distance)
-          first_test_failure_stacktrace = test.failures.first['message_lines'].join("\n")
+          full_stacktrace = test.failures.first['message_lines'].join("\n")
+          first_test_failure_stacktrace = sanitize_stacktrace(full_stacktrace, FAILURE_STACKTRACE_REGEX) || full_stacktrace
 
           # Search with the `search` param returns 500 errors, so we filter by ~QA and then filter further in Ruby
           failure_issues(test).each_with_object({}) do |issue, memo|
@@ -88,7 +90,7 @@ module Gitlab
             diff_ratio = distance.zero? ? 0.0 : (distance.to_f / first_test_failure_stacktrace.size).round(3)
 
             if diff_ratio <= max_diff_ratio
-              puts "  => [DEBUG] Issue #{issue} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
+              puts "  => [DEBUG] Issue #{issue.web_url} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
               # The `Gitlab::ObjectifiedHash` class overrides `#hash` which is used by `Hash#[]=` to compute the hash key.
               # This leads to a `TypeError Exception: no implicit conversion of Hash into Integer` error, so we convert the object to a hash before using it as a Hash key.
               # See:
@@ -96,18 +98,27 @@ module Gitlab
               # - https://github.com/NARKOZ/gitlab/commit/cbdbd1e32623f018a8fae39932a8e3bc4d929abb?_pjax=%23js-repo-pjax-container#r44484494
               memo[issue.to_h] = diff_ratio
             else
-              puts "  => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%)."
+              puts "  => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%).\n"
+              puts "  => [DEBUG] Issue stacktrace:\n----------------\n#{relevant_issue_stacktrace}\n----------------\n"
+              puts "  => [DEBUG] Failure stacktrace:\n----------------\n#{first_test_failure_stacktrace}\n----------------\n"
             end
           end
         end
 
         def find_issue_stacktrace(issue)
-          issue_stacktrace_match = issue.description.match(STACKTRACE_REGEX)
+          issue_stacktrace = sanitize_stacktrace(issue.description, ISSUE_STACKTRACE_REGEX)
+          return issue_stacktrace if issue_stacktrace
 
-          if issue_stacktrace_match
-            issue_stacktrace_match[2].gsub(/^#.*$/, '').strip
+          puts "  => [DEBUG] Stacktrace couldn't be found for #{issue.web_url}!"
+        end
+
+        def sanitize_stacktrace(stacktrace, regex)
+          stacktrace_match = stacktrace.match(regex)
+
+          if stacktrace_match
+            stacktrace_match[:stacktrace].gsub(/^\s*#.*$/, '').gsub(/^[[:space:]]+/, '').strip
           else
-            puts "  => [DEBUG] Stacktrace couldn't be found for #{issue.web_url}:\n\n#{issue.description}\n\n----------------------------------\n"
+            puts "  => [DEBUG] Stacktrace doesn't match the expected regex (#{regex}):\n----------------\n#{stacktrace}\n----------------\n"
           end
         end
 

data/lib/gitlab/qa/version.rb

@@ -1,5 +1,5 @@
 module Gitlab
   module QA
-    VERSION = '6.17.0'.freeze
+    VERSION = '6.17.1'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 6.17.0
+  version: 6.17.1
 platform: ruby
 authors:
 - Grzegorz Bizon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-12-22 00:00:00.000000000 Z
+date: 2021-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control