gitlab-qa 6.17.0 → 6.17.1
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9f01ee39d7d17bf54f28ad2453823912d17ef0ef93a3eef47aca9b60519074c3
|
4
|
+
data.tar.gz: 994940086f1f623e5b31592747b8cd3d8fe32b383575c16cd4127fe9860cb1a2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5648961ce9819c2b0e7262dea58fa90b04c0e998cbf0de8a183fd47e0c8c9ab0a26a075f38756c4bee8ffb20c87eabe1d7fe627145e8a11adad075a02dafe86f
|
7
|
+
data.tar.gz: 1cb3b8715444d2e2837fdde66069810bb27ef4fad76e0375011d107621e4afec48c2820110ee0a306bb0e89654af4d08325968027e9d8dca9aa209ef4a7ec359
|
@@ -1,4 +1,3 @@
|
|
1
|
-
|
2
1
|
# What tests can be run?
|
3
2
|
[[_TOC_]]
|
4
3
|
## The two types of QA tests
|
@@ -142,6 +141,30 @@ To run EE tests, the `EE_LICENSE` environment variable needs to be set:
|
|
142
141
|
|
143
142
|
`$ export EE_LICENSE=$(cat /path/to/GitLab.gitlab_license)`
|
144
143
|
|
144
|
+
## Running a specific test (or set of tests)
|
145
|
+
|
146
|
+
In most of the scenarios listed below, if you don't want to run all the tests
|
147
|
+
it's possible to specify one or more tests. The framework uses RSpec, so tests can be
|
148
|
+
specified as you would when using RSpec.
|
149
|
+
|
150
|
+
For example, the following would run `create_merge_request_spec.rb`:
|
151
|
+
|
152
|
+
```shell
|
153
|
+
$ gitlab-qa Test::Instance::Image EE -- qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb
|
154
|
+
```
|
155
|
+
|
156
|
+
While the following would run all Create UI tests:
|
157
|
+
|
158
|
+
```shell
|
159
|
+
$ gitlab-qa Test::Instance::Image EE -- qa/specs/features/browser_ui/3_create
|
160
|
+
```
|
161
|
+
|
162
|
+
And the following would run all Create API tests as well as UI tests:
|
163
|
+
|
164
|
+
```shell
|
165
|
+
$ gitlab-qa Test::Instance::Image EE -- qa/specs/features/browser_ui/3_create qa/specs/features/api/3_create
|
166
|
+
```
|
167
|
+
|
145
168
|
## Examples
|
146
169
|
|
147
170
|
### `Test::Instance::Image CE|EE|<full image address>`
|
@@ -22,6 +22,11 @@ module Gitlab
|
|
22
22
|
labels: ['Quality', 'QA', 'triage report', pipeline_name_label]
|
23
23
|
)
|
24
24
|
|
25
|
+
# Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/295493
|
26
|
+
gitlab.create_issue_note(
|
27
|
+
iid: issue.iid,
|
28
|
+
note: "/relate #{Runtime::Env.qa_issue_url}")
|
29
|
+
|
25
30
|
File.write('REPORT_ISSUE_URL', issue.web_url)
|
26
31
|
end
|
27
32
|
# rubocop:enable Metrics/AbcSize
|
@@ -10,7 +10,8 @@ module Gitlab
|
|
10
10
|
# Uses the API to create or update GitLab issues with the results of tests from RSpec report files.
|
11
11
|
class RelateFailureIssue < ReportAsIssue
|
12
12
|
DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.05
|
13
|
-
|
13
|
+
FAILURE_STACKTRACE_REGEX = %r{((.*Failure\/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m.freeze
|
14
|
+
ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)/m.freeze
|
14
15
|
NEW_ISSUE_LABELS = Set.new(%w[QA Quality test failure::investigating priority::2]).freeze
|
15
16
|
|
16
17
|
MultipleIssuesFound = Class.new(StandardError)
|
@@ -77,7 +78,8 @@ module Gitlab
|
|
77
78
|
|
78
79
|
def find_relevant_failure_issues(test) # rubocop:disable Metrics/AbcSize
|
79
80
|
ld = Class.new.extend(Gem::Text).method(:levenshtein_distance)
|
80
|
-
|
81
|
+
full_stacktrace = test.failures.first['message_lines'].join("\n")
|
82
|
+
first_test_failure_stacktrace = sanitize_stacktrace(full_stacktrace, FAILURE_STACKTRACE_REGEX) || full_stacktrace
|
81
83
|
|
82
84
|
# Search with the `search` param returns 500 errors, so we filter by ~QA and then filter further in Ruby
|
83
85
|
failure_issues(test).each_with_object({}) do |issue, memo|
|
@@ -88,7 +90,7 @@ module Gitlab
|
|
88
90
|
diff_ratio = distance.zero? ? 0.0 : (distance.to_f / first_test_failure_stacktrace.size).round(3)
|
89
91
|
|
90
92
|
if diff_ratio <= max_diff_ratio
|
91
|
-
puts " => [DEBUG] Issue #{issue} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
|
93
|
+
puts " => [DEBUG] Issue #{issue.web_url} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
|
92
94
|
# The `Gitlab::ObjectifiedHash` class overrides `#hash` which is used by `Hash#[]=` to compute the hash key.
|
93
95
|
# This leads to a `TypeError Exception: no implicit conversion of Hash into Integer` error, so we convert the object to a hash before using it as a Hash key.
|
94
96
|
# See:
|
@@ -96,18 +98,27 @@ module Gitlab
|
|
96
98
|
# - https://github.com/NARKOZ/gitlab/commit/cbdbd1e32623f018a8fae39932a8e3bc4d929abb?_pjax=%23js-repo-pjax-container#r44484494
|
97
99
|
memo[issue.to_h] = diff_ratio
|
98
100
|
else
|
99
|
-
puts " => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%)
|
101
|
+
puts " => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%).\n"
|
102
|
+
puts " => [DEBUG] Issue stacktrace:\n----------------\n#{relevant_issue_stacktrace}\n----------------\n"
|
103
|
+
puts " => [DEBUG] Failure stacktrace:\n----------------\n#{first_test_failure_stacktrace}\n----------------\n"
|
100
104
|
end
|
101
105
|
end
|
102
106
|
end
|
103
107
|
|
104
108
|
def find_issue_stacktrace(issue)
|
105
|
-
|
109
|
+
issue_stacktrace = sanitize_stacktrace(issue.description, ISSUE_STACKTRACE_REGEX)
|
110
|
+
return issue_stacktrace if issue_stacktrace
|
106
111
|
|
107
|
-
|
108
|
-
|
112
|
+
puts " => [DEBUG] Stacktrace couldn't be found for #{issue.web_url}!"
|
113
|
+
end
|
114
|
+
|
115
|
+
def sanitize_stacktrace(stacktrace, regex)
|
116
|
+
stacktrace_match = stacktrace.match(regex)
|
117
|
+
|
118
|
+
if stacktrace_match
|
119
|
+
stacktrace_match[:stacktrace].gsub(/^\s*#.*$/, '').gsub(/^[[:space:]]+/, '').strip
|
109
120
|
else
|
110
|
-
puts " => [DEBUG] Stacktrace
|
121
|
+
puts " => [DEBUG] Stacktrace doesn't match the expected regex (#{regex}):\n----------------\n#{stacktrace}\n----------------\n"
|
111
122
|
end
|
112
123
|
end
|
113
124
|
|
data/lib/gitlab/qa/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-qa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 6.17.
|
4
|
+
version: 6.17.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Grzegorz Bizon
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-01-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: climate_control
|