gitlab-qa 6.15.2 → 6.17.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +47 -34
- data/docs/what_tests_can_be_run.md +24 -1
- data/lib/gitlab/qa.rb +1 -0
- data/lib/gitlab/qa/report/generate_test_session.rb +19 -4
- data/lib/gitlab/qa/report/gitlab_issue_client.rb +1 -1
- data/lib/gitlab/qa/report/relate_failure_issue.rb +21 -9
- data/lib/gitlab/qa/report/report_as_issue.rb +5 -5
- data/lib/gitlab/qa/runtime/env.rb +29 -1
- data/lib/gitlab/qa/scenario/test/integration/ssh_tunnel.rb +59 -0
- data/lib/gitlab/qa/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9f01ee39d7d17bf54f28ad2453823912d17ef0ef93a3eef47aca9b60519074c3
|
4
|
+
data.tar.gz: 994940086f1f623e5b31592747b8cd3d8fe32b383575c16cd4127fe9860cb1a2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5648961ce9819c2b0e7262dea58fa90b04c0e998cbf0de8a183fd47e0c8c9ab0a26a075f38756c4bee8ffb20c87eabe1d7fe627145e8a11adad075a02dafe86f
|
7
|
+
data.tar.gz: 1cb3b8715444d2e2837fdde66069810bb27ef4fad76e0375011d107621e4afec48c2820110ee0a306bb0e89654af4d08325968027e9d8dca9aa209ef4a7ec359
|
data/.gitlab-ci.yml
CHANGED
@@ -28,7 +28,7 @@ workflow:
|
|
28
28
|
rules:
|
29
29
|
# For merge requests, create a pipeline.
|
30
30
|
- if: '$CI_MERGE_REQUEST_IID'
|
31
|
-
# For
|
31
|
+
# For the default branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
|
32
32
|
- if: '$CI_COMMIT_BRANCH == "master"'
|
33
33
|
# For tags, create a pipeline.
|
34
34
|
- if: '$CI_COMMIT_TAG'
|
@@ -42,10 +42,11 @@ variables:
|
|
42
42
|
QA_CAN_TEST_GIT_PROTOCOL_V2: "true"
|
43
43
|
QA_CAN_TEST_PRAEFECT: "false"
|
44
44
|
QA_TESTCASES_REPORTING_PROJECT: "gitlab-org/quality/testcases"
|
45
|
-
QA_FAILURES_REPORTING_PROJECT: "rymai/gitlab-qa-issues"
|
46
|
-
# The --dry-run or --max-diff-ratio option can be set to modify the behavior of `exe/gitlab-qa-report --relate-failure-issue` without releasing a new gem version.
|
47
|
-
QA_FAILURES_REPORTER_OPTIONS: ""
|
48
45
|
QA_TESTCASE_SESSIONS_PROJECT: "gitlab-org/quality/testcase-sessions"
|
46
|
+
# QA_DEFAULT_BRANCH is the default branch name of the instance under test.
|
47
|
+
QA_DEFAULT_BRANCH: "master"
|
48
|
+
# TOP_UPSTREAM_DEFAULT_BRANCH is the default branch name of the original project that triggered a pipeline in this project.
|
49
|
+
TOP_UPSTREAM_DEFAULT_BRANCH: "master"
|
49
50
|
|
50
51
|
.check-base:
|
51
52
|
stage: check
|
@@ -77,8 +78,7 @@ rspec:
|
|
77
78
|
- exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
78
79
|
- exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
|
79
80
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
80
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" ==
|
81
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
81
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
82
82
|
- exit $test_run_exit_code
|
83
83
|
|
84
84
|
.ce-qa:
|
@@ -332,8 +332,7 @@ ce:update:
|
|
332
332
|
script:
|
333
333
|
- exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
334
334
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
335
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" ==
|
336
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
335
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
337
336
|
- exit $test_run_exit_code
|
338
337
|
extends:
|
339
338
|
- .test
|
@@ -347,8 +346,7 @@ ce:update-quarantine:
|
|
347
346
|
script:
|
348
347
|
- exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
349
348
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
350
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" ==
|
351
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
349
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
352
350
|
- exit $test_run_exit_code
|
353
351
|
extends:
|
354
352
|
- .test
|
@@ -361,8 +359,7 @@ ee:update:
|
|
361
359
|
script:
|
362
360
|
- exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
363
361
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
364
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" ==
|
365
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
362
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
366
363
|
- exit $test_run_exit_code
|
367
364
|
extends:
|
368
365
|
- .test
|
@@ -376,8 +373,7 @@ ee:update-quarantine:
|
|
376
373
|
script:
|
377
374
|
- exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
378
375
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
379
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" ==
|
380
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
376
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
381
377
|
- exit $test_run_exit_code
|
382
378
|
extends:
|
383
379
|
- .test
|
@@ -465,24 +461,23 @@ ee:mattermost-quarantine:
|
|
465
461
|
variables:
|
466
462
|
QA_SCENARIO: "Test::Integration::Mattermost"
|
467
463
|
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
ee:geo-quarantine:
|
478
|
-
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
|
483
|
-
|
484
|
-
|
485
|
-
QA_SCENARIO: "Test::Integration::Geo"
|
464
|
+
# Disabling geo jobs temporarily due to https://gitlab.com/gitlab-org/gitlab/-/issues/273063
|
465
|
+
# ee:geo:
|
466
|
+
# extends:
|
467
|
+
# - .test
|
468
|
+
# - .ee-qa
|
469
|
+
# - .rspec-report-opts
|
470
|
+
# variables:
|
471
|
+
# QA_SCENARIO: "Test::Integration::Geo"
|
472
|
+
|
473
|
+
# ee:geo-quarantine:
|
474
|
+
# extends:
|
475
|
+
# - .test
|
476
|
+
# - .ee-qa
|
477
|
+
# - .quarantine
|
478
|
+
# - .rspec-report-opts
|
479
|
+
# variables:
|
480
|
+
# QA_SCENARIO: "Test::Integration::Geo"
|
486
481
|
|
487
482
|
ce:ldap_no_tls:
|
488
483
|
extends:
|
@@ -948,7 +943,7 @@ staging:
|
|
948
943
|
generate_test_session:
|
949
944
|
stage: report
|
950
945
|
rules:
|
951
|
-
- if: '$TOP_UPSTREAM_SOURCE_JOB && $TOP_UPSTREAM_SOURCE_REF ==
|
946
|
+
- if: '$TOP_UPSTREAM_SOURCE_JOB && $TOP_UPSTREAM_SOURCE_REF == $TOP_UPSTREAM_DEFAULT_BRANCH'
|
952
947
|
when: always
|
953
948
|
- if: '$TOP_UPSTREAM_SOURCE_JOB =~ /\Ahttps:\/\/ops.gitlab.net\//'
|
954
949
|
when: always
|
@@ -961,6 +956,24 @@ generate_test_session:
|
|
961
956
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
962
957
|
- exe/gitlab-qa-report --generate-test-session "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASE_SESSIONS_PROJECT"
|
963
958
|
|
959
|
+
relate_test_failures:
|
960
|
+
stage: report
|
961
|
+
rules:
|
962
|
+
- if: '$DISABLE_RELATING_FAILURE_ISSUES'
|
963
|
+
when: never
|
964
|
+
- if: '$TOP_UPSTREAM_SOURCE_JOB && $TOP_UPSTREAM_SOURCE_REF == $TOP_UPSTREAM_DEFAULT_BRANCH'
|
965
|
+
when: always
|
966
|
+
- if: '$TOP_UPSTREAM_SOURCE_JOB =~ /\Ahttps:\/\/ops.gitlab.net\//'
|
967
|
+
when: always
|
968
|
+
variables:
|
969
|
+
QA_FAILURES_REPORTING_PROJECT: "gitlab-org/gitlab"
|
970
|
+
QA_FAILURES_MAX_DIFF_RATIO: "0.05"
|
971
|
+
# The --dry-run can be set to modify the behavior of `exe/gitlab-qa-report --relate-failure-issue` without releasing a new gem version.
|
972
|
+
QA_FAILURES_REPORTER_OPTIONS: ""
|
973
|
+
script:
|
974
|
+
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
975
|
+
- exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" --max-diff-ratio "$QA_FAILURES_MAX_DIFF_RATIO" $QA_FAILURES_REPORTER_OPTIONS
|
976
|
+
|
964
977
|
.notify_upstream_commit:
|
965
978
|
stage: notify
|
966
979
|
image: ruby:2.6
|
@@ -997,7 +1010,7 @@ notify_slack:
|
|
997
1010
|
rules:
|
998
1011
|
- if: '$TOP_UPSTREAM_SOURCE_JOB && $NOTIFY_CHANNEL'
|
999
1012
|
when: on_failure
|
1000
|
-
- if: '$TOP_UPSTREAM_SOURCE_JOB && $TOP_UPSTREAM_SOURCE_REF ==
|
1013
|
+
- if: '$TOP_UPSTREAM_SOURCE_JOB && $TOP_UPSTREAM_SOURCE_REF == $TOP_UPSTREAM_DEFAULT_BRANCH'
|
1001
1014
|
when: on_failure
|
1002
1015
|
script:
|
1003
1016
|
- export RELEASE=${TOP_UPSTREAM_SOURCE_REF:-$RELEASE}
|
@@ -1,4 +1,3 @@
|
|
1
|
-
|
2
1
|
# What tests can be run?
|
3
2
|
[[_TOC_]]
|
4
3
|
## The two types of QA tests
|
@@ -142,6 +141,30 @@ To run EE tests, the `EE_LICENSE` environment variable needs to be set:
|
|
142
141
|
|
143
142
|
`$ export EE_LICENSE=$(cat /path/to/GitLab.gitlab_license)`
|
144
143
|
|
144
|
+
## Running a specific test (or set of tests)
|
145
|
+
|
146
|
+
In most of the scenarios listed below, if you don't want to run all the tests
|
147
|
+
it's possible to specify one or more tests. The framework uses RSpec, so tests can be
|
148
|
+
specified as you would when using RSpec.
|
149
|
+
|
150
|
+
For example, the following would run `create_merge_request_spec.rb`:
|
151
|
+
|
152
|
+
```shell
|
153
|
+
$ gitlab-qa Test::Instance::Image EE -- qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb
|
154
|
+
```
|
155
|
+
|
156
|
+
While the following would run all Create UI tests:
|
157
|
+
|
158
|
+
```shell
|
159
|
+
$ gitlab-qa Test::Instance::Image EE -- qa/specs/features/browser_ui/3_create
|
160
|
+
```
|
161
|
+
|
162
|
+
And the following would run all Create API tests as well as UI tests:
|
163
|
+
|
164
|
+
```shell
|
165
|
+
$ gitlab-qa Test::Instance::Image EE -- qa/specs/features/browser_ui/3_create qa/specs/features/api/3_create
|
166
|
+
```
|
167
|
+
|
145
168
|
## Examples
|
146
169
|
|
147
170
|
### `Test::Instance::Image CE|EE|<full image address>`
|
data/lib/gitlab/qa.rb
CHANGED
@@ -57,6 +57,7 @@ module Gitlab
|
|
57
57
|
autoload :SMTP, 'gitlab/qa/scenario/test/integration/smtp'
|
58
58
|
autoload :GitalyCluster, 'gitlab/qa/scenario/test/integration/gitaly_cluster'
|
59
59
|
autoload :Jira, 'gitlab/qa/scenario/test/integration/jira'
|
60
|
+
autoload :SSHTunnel, 'gitlab/qa/scenario/test/integration/ssh_tunnel'
|
60
61
|
end
|
61
62
|
|
62
63
|
module Sanity
|
@@ -22,12 +22,17 @@ module Gitlab
|
|
22
22
|
labels: ['Quality', 'QA', 'triage report', pipeline_name_label]
|
23
23
|
)
|
24
24
|
|
25
|
+
# Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/295493
|
26
|
+
gitlab.create_issue_note(
|
27
|
+
iid: issue.iid,
|
28
|
+
note: "/relate #{Runtime::Env.qa_issue_url}")
|
29
|
+
|
25
30
|
File.write('REPORT_ISSUE_URL', issue.web_url)
|
26
31
|
end
|
27
32
|
# rubocop:enable Metrics/AbcSize
|
28
33
|
|
29
34
|
def generate_description(tests)
|
30
|
-
<<~MARKDOWN
|
35
|
+
<<~MARKDOWN.rstrip
|
31
36
|
## Session summary
|
32
37
|
|
33
38
|
* Deploy version: #{Runtime::Env.deploy_version}
|
@@ -39,9 +44,7 @@ module Gitlab
|
|
39
44
|
|
40
45
|
#{generate_stages_listing(tests)}
|
41
46
|
|
42
|
-
|
43
|
-
|
44
|
-
* #{Runtime::Env.qa_issue_url}
|
47
|
+
#{generate_qa_issue_relation}
|
45
48
|
MARKDOWN
|
46
49
|
end
|
47
50
|
|
@@ -229,6 +232,18 @@ module Gitlab
|
|
229
232
|
'-'
|
230
233
|
end
|
231
234
|
end
|
235
|
+
|
236
|
+
def generate_qa_issue_relation
|
237
|
+
return unless Runtime::Env.qa_issue_url
|
238
|
+
|
239
|
+
<<~MARKDOWN.chomp
|
240
|
+
## Release QA issue
|
241
|
+
|
242
|
+
* #{Runtime::Env.qa_issue_url}
|
243
|
+
|
244
|
+
/relate #{Runtime::Env.qa_issue_url}
|
245
|
+
MARKDOWN
|
246
|
+
end
|
232
247
|
end
|
233
248
|
end
|
234
249
|
end
|
@@ -110,7 +110,7 @@ module Gitlab
|
|
110
110
|
channel = pipeline == "canary" ? "qa-production" : "qa-#{pipeline}"
|
111
111
|
error_msg = warn_exception(e)
|
112
112
|
|
113
|
-
return unless QA::Runtime::Env.ci_commit_ref_name ==
|
113
|
+
return unless QA::Runtime::Env.ci_commit_ref_name == QA::Runtime::Env.default_branch
|
114
114
|
|
115
115
|
slack_options = {
|
116
116
|
channel: channel,
|
@@ -10,7 +10,8 @@ module Gitlab
|
|
10
10
|
# Uses the API to create or update GitLab issues with the results of tests from RSpec report files.
|
11
11
|
class RelateFailureIssue < ReportAsIssue
|
12
12
|
DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.05
|
13
|
-
|
13
|
+
FAILURE_STACKTRACE_REGEX = %r{((.*Failure\/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m.freeze
|
14
|
+
ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)/m.freeze
|
14
15
|
NEW_ISSUE_LABELS = Set.new(%w[QA Quality test failure::investigating priority::2]).freeze
|
15
16
|
|
16
17
|
MultipleIssuesFound = Class.new(StandardError)
|
@@ -77,7 +78,8 @@ module Gitlab
|
|
77
78
|
|
78
79
|
def find_relevant_failure_issues(test) # rubocop:disable Metrics/AbcSize
|
79
80
|
ld = Class.new.extend(Gem::Text).method(:levenshtein_distance)
|
80
|
-
|
81
|
+
full_stacktrace = test.failures.first['message_lines'].join("\n")
|
82
|
+
first_test_failure_stacktrace = sanitize_stacktrace(full_stacktrace, FAILURE_STACKTRACE_REGEX) || full_stacktrace
|
81
83
|
|
82
84
|
# Search with the `search` param returns 500 errors, so we filter by ~QA and then filter further in Ruby
|
83
85
|
failure_issues(test).each_with_object({}) do |issue, memo|
|
@@ -85,9 +87,10 @@ module Gitlab
|
|
85
87
|
next unless relevant_issue_stacktrace
|
86
88
|
|
87
89
|
distance = ld.call(first_test_failure_stacktrace, relevant_issue_stacktrace)
|
88
|
-
diff_ratio = (distance.to_f / first_test_failure_stacktrace.size).round(3)
|
90
|
+
diff_ratio = distance.zero? ? 0.0 : (distance.to_f / first_test_failure_stacktrace.size).round(3)
|
91
|
+
|
89
92
|
if diff_ratio <= max_diff_ratio
|
90
|
-
puts " => [DEBUG] Issue #{issue} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
|
93
|
+
puts " => [DEBUG] Issue #{issue.web_url} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
|
91
94
|
# The `Gitlab::ObjectifiedHash` class overrides `#hash` which is used by `Hash#[]=` to compute the hash key.
|
92
95
|
# This leads to a `TypeError Exception: no implicit conversion of Hash into Integer` error, so we convert the object to a hash before using it as a Hash key.
|
93
96
|
# See:
|
@@ -95,18 +98,27 @@ module Gitlab
|
|
95
98
|
# - https://github.com/NARKOZ/gitlab/commit/cbdbd1e32623f018a8fae39932a8e3bc4d929abb?_pjax=%23js-repo-pjax-container#r44484494
|
96
99
|
memo[issue.to_h] = diff_ratio
|
97
100
|
else
|
98
|
-
puts " => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%)
|
101
|
+
puts " => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%).\n"
|
102
|
+
puts " => [DEBUG] Issue stacktrace:\n----------------\n#{relevant_issue_stacktrace}\n----------------\n"
|
103
|
+
puts " => [DEBUG] Failure stacktrace:\n----------------\n#{first_test_failure_stacktrace}\n----------------\n"
|
99
104
|
end
|
100
105
|
end
|
101
106
|
end
|
102
107
|
|
103
108
|
def find_issue_stacktrace(issue)
|
104
|
-
|
109
|
+
issue_stacktrace = sanitize_stacktrace(issue.description, ISSUE_STACKTRACE_REGEX)
|
110
|
+
return issue_stacktrace if issue_stacktrace
|
111
|
+
|
112
|
+
puts " => [DEBUG] Stacktrace couldn't be found for #{issue.web_url}!"
|
113
|
+
end
|
114
|
+
|
115
|
+
def sanitize_stacktrace(stacktrace, regex)
|
116
|
+
stacktrace_match = stacktrace.match(regex)
|
105
117
|
|
106
|
-
if
|
107
|
-
|
118
|
+
if stacktrace_match
|
119
|
+
stacktrace_match[:stacktrace].gsub(/^\s*#.*$/, '').gsub(/^[[:space:]]+/, '').strip
|
108
120
|
else
|
109
|
-
puts " => [DEBUG] Stacktrace
|
121
|
+
puts " => [DEBUG] Stacktrace doesn't match the expected regex (#{regex}):\n----------------\n#{stacktrace}\n----------------\n"
|
110
122
|
end
|
111
123
|
end
|
112
124
|
|
@@ -112,7 +112,7 @@ module Gitlab
|
|
112
112
|
"found:#{pipeline}.gitlab.com"
|
113
113
|
when 'preprod'
|
114
114
|
'found:pre.gitlab.com'
|
115
|
-
when 'staging-orchestrated', 'nightly',
|
115
|
+
when 'staging-orchestrated', 'nightly', QA::Runtime::Env.default_branch
|
116
116
|
"found:#{pipeline}"
|
117
117
|
else
|
118
118
|
raise "No `found:*` label for the `#{pipeline}` pipeline!"
|
@@ -147,14 +147,14 @@ module Gitlab
|
|
147
147
|
# Gets the name of the pipeline the test was run in, to be used as the key of a scoped label
|
148
148
|
#
|
149
149
|
# Tests can be run in several pipelines:
|
150
|
-
# gitlab-qa, nightly,
|
150
|
+
# gitlab-qa, nightly, staging, canary, production, preprod, MRs, and the default branch (master/main)
|
151
151
|
#
|
152
152
|
# Some of those run in their own project, so CI_PROJECT_NAME is the name we need. Those are:
|
153
153
|
# nightly, staging, canary, production, and preprod
|
154
154
|
#
|
155
|
-
# MR, master, and gitlab-qa tests run in gitlab-qa, but we only want to report tests run on
|
156
|
-
# because the other pipelines will be monitored by the author of the MR that triggered them.
|
157
|
-
# So we assume that we're reporting a master pipeline if the project name is 'gitlab-qa'.
|
155
|
+
# MR, master/main, and gitlab-qa tests run in gitlab-qa, but we only want to report tests run on
|
156
|
+
# master/main because the other pipelines will be monitored by the author of the MR that triggered them.
|
157
|
+
# So we assume that we're reporting a master/main pipeline if the project name is 'gitlab-qa'.
|
158
158
|
|
159
159
|
@pipeline ||= Runtime::Env.pipeline_from_project_name
|
160
160
|
end
|
@@ -44,6 +44,7 @@ module Gitlab
|
|
44
44
|
'QA_GITALY_NON_CLUSTER_STORAGE' => :qa_gitaly_non_cluster_storage,
|
45
45
|
'QA_COOKIES' => :qa_cookie,
|
46
46
|
'QA_DEBUG' => :qa_debug,
|
47
|
+
'QA_DEFAULT_BRANCH' => :qa_default_branch,
|
47
48
|
'QA_LOG_PATH' => :qa_log_path,
|
48
49
|
'QA_CAN_TEST_ADMIN_FEATURES' => :qa_can_test_admin_features,
|
49
50
|
'QA_CAN_TEST_GIT_PROTOCOL_V2' => :qa_can_test_git_protocol_v2,
|
@@ -100,6 +101,10 @@ module Gitlab
|
|
100
101
|
end
|
101
102
|
end
|
102
103
|
|
104
|
+
def default_branch
|
105
|
+
ENV['QA_DEFAULT_BRANCH'] || 'master'
|
106
|
+
end
|
107
|
+
|
103
108
|
def gitlab_username
|
104
109
|
ENV['GITLAB_USERNAME'] || 'gitlab-qa'
|
105
110
|
end
|
@@ -157,7 +162,7 @@ module Gitlab
|
|
157
162
|
if ENV['TOP_UPSTREAM_SOURCE_JOB'].to_s.start_with?('https://ops.gitlab.net')
|
158
163
|
'staging-orchestrated'
|
159
164
|
else
|
160
|
-
|
165
|
+
QA::Runtime::Env.default_branch
|
161
166
|
end
|
162
167
|
else
|
163
168
|
ci_project_name
|
@@ -176,6 +181,10 @@ module Gitlab
|
|
176
181
|
ENV['GITLAB_QA_DEV_ACCESS_TOKEN']
|
177
182
|
end
|
178
183
|
|
184
|
+
def initial_root_password
|
185
|
+
ENV['GITLAB_INITIAL_ROOT_PASSWORD']
|
186
|
+
end
|
187
|
+
|
179
188
|
def qa_container_registry_access_token
|
180
189
|
ENV['GITLAB_QA_CONTAINER_REGISTRY_ACCESS_TOKEN']
|
181
190
|
end
|
@@ -251,6 +260,25 @@ module Gitlab
|
|
251
260
|
end
|
252
261
|
end
|
253
262
|
|
263
|
+
def require_initial_password!
|
264
|
+
return unless ENV['GITLAB_INITIAL_ROOT_PASSWORD'].to_s.strip.empty?
|
265
|
+
|
266
|
+
raise ArgumentError, "Environment variable GITLAB_INITIAL_ROOT_PASSWORD must be provided to set an initial root password."
|
267
|
+
end
|
268
|
+
|
269
|
+
def require_ssh_tunnel!
|
270
|
+
%w[TUNNEL_SSH_PRIVATE_KEY TUNNEL_SERVER_HOSTNAME TUNNEL_SSH_USER].each do |env_key|
|
271
|
+
raise ArgumentError, "Environment variable #{env_key} must be set to run tunnel specs" unless ENV.key?(env_key)
|
272
|
+
end
|
273
|
+
end
|
274
|
+
|
275
|
+
def override_default_password!
|
276
|
+
require_initial_password!
|
277
|
+
|
278
|
+
ENV['GITLAB_PASSWORD'] = initial_root_password
|
279
|
+
ENV['GITLAB_ADMIN_PASSWORD'] = initial_root_password
|
280
|
+
end
|
281
|
+
|
254
282
|
def skip_pull?
|
255
283
|
enabled?(ENV['QA_SKIP_PULL'], default: false)
|
256
284
|
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module QA
|
5
|
+
module Scenario
|
6
|
+
module Test
|
7
|
+
module Integration
|
8
|
+
class SSHTunnel < Scenario::Template
|
9
|
+
# rubocop:disable Metrics/AbcSize
|
10
|
+
def perform(release, *rspec_args)
|
11
|
+
Runtime::Env.override_default_password!
|
12
|
+
Runtime::Env.require_ssh_tunnel!
|
13
|
+
|
14
|
+
Component::Gitlab.perform do |gitlab|
|
15
|
+
gitlab.release = release
|
16
|
+
gitlab.network = 'test'
|
17
|
+
|
18
|
+
Component::InternetTunnel.perform do |tunnel_gitlab|
|
19
|
+
Component::InternetTunnel.perform do |tunnel_registry|
|
20
|
+
tunnel_gitlab.gitlab_hostname = gitlab.hostname
|
21
|
+
tunnel_gitlab.network = 'test'
|
22
|
+
tunnel_registry.gitlab_hostname = gitlab.hostname
|
23
|
+
tunnel_registry.network = 'test'
|
24
|
+
|
25
|
+
gitlab.omnibus_config = <<~OMNIBUS
|
26
|
+
external_url '#{tunnel_gitlab.url}';
|
27
|
+
nginx['listen_port'] = 80;
|
28
|
+
nginx['listen_https'] = false;
|
29
|
+
# documentation: https://docs.gitlab.com/omnibus/settings/database.html#seed-the-database-fresh-installs-only
|
30
|
+
gitlab_rails['initial_root_password'] = '#{Runtime::Env.require_initial_password}'
|
31
|
+
|
32
|
+
registry_external_url '#{tunnel_registry.url}';
|
33
|
+
registry_nginx['listen_port'] = 80;
|
34
|
+
registry_nginx['listen_https'] = false;
|
35
|
+
OMNIBUS
|
36
|
+
|
37
|
+
tunnel_gitlab.instance do
|
38
|
+
tunnel_registry.instance do
|
39
|
+
gitlab.instance do
|
40
|
+
Component::Specs.perform do |specs|
|
41
|
+
specs.suite = 'Test::Integration::SSHTunnel'
|
42
|
+
specs.release = gitlab.release
|
43
|
+
specs.network = gitlab.network
|
44
|
+
specs.args = [tunnel_gitlab.url, *rspec_args]
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
# rubocop:enable Metrics/AbcSize
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
data/lib/gitlab/qa/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-qa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 6.
|
4
|
+
version: 6.17.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Grzegorz Bizon
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-01-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: climate_control
|
@@ -310,6 +310,7 @@ files:
|
|
310
310
|
- lib/gitlab/qa/scenario/test/integration/praefect.rb
|
311
311
|
- lib/gitlab/qa/scenario/test/integration/saml.rb
|
312
312
|
- lib/gitlab/qa/scenario/test/integration/smtp.rb
|
313
|
+
- lib/gitlab/qa/scenario/test/integration/ssh_tunnel.rb
|
313
314
|
- lib/gitlab/qa/scenario/test/omnibus/image.rb
|
314
315
|
- lib/gitlab/qa/scenario/test/omnibus/update.rb
|
315
316
|
- lib/gitlab/qa/scenario/test/omnibus/upgrade.rb
|