gitlab-qa 6.15.4 → 6.18.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +87 -32
- data/docs/what_tests_can_be_run.md +24 -1
- data/lib/gitlab/qa.rb +1 -0
- data/lib/gitlab/qa/report/generate_test_session.rb +5 -0
- data/lib/gitlab/qa/report/gitlab_issue_client.rb +1 -1
- data/lib/gitlab/qa/report/relate_failure_issue.rb +19 -8
- data/lib/gitlab/qa/report/report_as_issue.rb +5 -5
- data/lib/gitlab/qa/runtime/env.rb +29 -1
- data/lib/gitlab/qa/scenario/test/integration/ssh_tunnel.rb +59 -0
- data/lib/gitlab/qa/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: '094c38faab90bbf9f69be789508850032c958e86f2766a4f36d73554ca4ba915'
|
4
|
+
data.tar.gz: b706f623bb39c14beadc570ed64ab6d7e20e3302b855f2b4c3fb4ae79a43b576
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 576e64251a075a66d5afc7d75285695c24dcd4b30cf859abb27d8c7a4bb658f0ce1e4e764f84ca885e9b306e7ebc963b52b7c4d962059249b5f2e5ae3fa8da6a
|
7
|
+
data.tar.gz: 328d0b8c2efd8565944c5c5ea94f22cb819427b6db1f75c75b21970fb77bd2175a23a1f3d7bd57d546ff9ee86e7f042e215198228a372c972593560959aefc94
|
data/.gitlab-ci.yml
CHANGED
@@ -28,7 +28,7 @@ workflow:
|
|
28
28
|
rules:
|
29
29
|
# For merge requests, create a pipeline.
|
30
30
|
- if: '$CI_MERGE_REQUEST_IID'
|
31
|
-
# For
|
31
|
+
# For the default branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
|
32
32
|
- if: '$CI_COMMIT_BRANCH == "master"'
|
33
33
|
# For tags, create a pipeline.
|
34
34
|
- if: '$CI_COMMIT_TAG'
|
@@ -42,10 +42,11 @@ variables:
|
|
42
42
|
QA_CAN_TEST_GIT_PROTOCOL_V2: "true"
|
43
43
|
QA_CAN_TEST_PRAEFECT: "false"
|
44
44
|
QA_TESTCASES_REPORTING_PROJECT: "gitlab-org/quality/testcases"
|
45
|
-
QA_FAILURES_REPORTING_PROJECT: "rymai/gitlab-qa-issues"
|
46
|
-
# The --dry-run or --max-diff-ratio option can be set to modify the behavior of `exe/gitlab-qa-report --relate-failure-issue` without releasing a new gem version.
|
47
|
-
QA_FAILURES_REPORTER_OPTIONS: ""
|
48
45
|
QA_TESTCASE_SESSIONS_PROJECT: "gitlab-org/quality/testcase-sessions"
|
46
|
+
# QA_DEFAULT_BRANCH is the default branch name of the instance under test.
|
47
|
+
QA_DEFAULT_BRANCH: "master"
|
48
|
+
# TOP_UPSTREAM_DEFAULT_BRANCH is the default branch name of the original project that triggered a pipeline in this project.
|
49
|
+
TOP_UPSTREAM_DEFAULT_BRANCH: "master"
|
49
50
|
|
50
51
|
.check-base:
|
51
52
|
stage: check
|
@@ -77,8 +78,7 @@ rspec:
|
|
77
78
|
- exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
78
79
|
- exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
|
79
80
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
80
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" ==
|
81
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
81
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
82
82
|
- exit $test_run_exit_code
|
83
83
|
|
84
84
|
.ce-qa:
|
@@ -332,8 +332,7 @@ ce:update:
|
|
332
332
|
script:
|
333
333
|
- exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
334
334
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
335
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" ==
|
336
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
335
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
337
336
|
- exit $test_run_exit_code
|
338
337
|
extends:
|
339
338
|
- .test
|
@@ -347,8 +346,7 @@ ce:update-quarantine:
|
|
347
346
|
script:
|
348
347
|
- exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
349
348
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
350
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" ==
|
351
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
349
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
352
350
|
- exit $test_run_exit_code
|
353
351
|
extends:
|
354
352
|
- .test
|
@@ -361,8 +359,7 @@ ee:update:
|
|
361
359
|
script:
|
362
360
|
- exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
363
361
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
364
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" ==
|
365
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
362
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
366
363
|
- exit $test_run_exit_code
|
367
364
|
extends:
|
368
365
|
- .test
|
@@ -376,8 +373,7 @@ ee:update-quarantine:
|
|
376
373
|
script:
|
377
374
|
- exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
378
375
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
379
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" ==
|
380
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" $QA_FAILURES_REPORTER_OPTIONS || true; fi
|
376
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
381
377
|
- exit $test_run_exit_code
|
382
378
|
extends:
|
383
379
|
- .test
|
@@ -465,22 +461,23 @@ ee:mattermost-quarantine:
|
|
465
461
|
variables:
|
466
462
|
QA_SCENARIO: "Test::Integration::Mattermost"
|
467
463
|
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
|
480
|
-
|
481
|
-
|
482
|
-
|
483
|
-
|
464
|
+
# Disabling geo jobs temporarily due to https://gitlab.com/gitlab-org/gitlab/-/issues/273063
|
465
|
+
# ee:geo:
|
466
|
+
# extends:
|
467
|
+
# - .test
|
468
|
+
# - .ee-qa
|
469
|
+
# - .rspec-report-opts
|
470
|
+
# variables:
|
471
|
+
# QA_SCENARIO: "Test::Integration::Geo"
|
472
|
+
|
473
|
+
# ee:geo-quarantine:
|
474
|
+
# extends:
|
475
|
+
# - .test
|
476
|
+
# - .ee-qa
|
477
|
+
# - .quarantine
|
478
|
+
# - .rspec-report-opts
|
479
|
+
# variables:
|
480
|
+
# QA_SCENARIO: "Test::Integration::Geo"
|
484
481
|
|
485
482
|
ce:ldap_no_tls:
|
486
483
|
extends:
|
@@ -674,6 +671,46 @@ ee:kubernetes-quarantine:
|
|
674
671
|
variables:
|
675
672
|
QA_SCENARIO: "Test::Integration::Kubernetes"
|
676
673
|
|
674
|
+
ce:ssh_tunnel:
|
675
|
+
extends:
|
676
|
+
- .test
|
677
|
+
- .high-capacity
|
678
|
+
- .ce-qa
|
679
|
+
- .rspec-report-opts
|
680
|
+
allow_failure: true
|
681
|
+
variables:
|
682
|
+
QA_SCENARIO: "Test::Integration::SSHTunnel"
|
683
|
+
|
684
|
+
ce:ssh_tunnel-quarantine:
|
685
|
+
extends:
|
686
|
+
- .test
|
687
|
+
- .high-capacity
|
688
|
+
- .ce-qa
|
689
|
+
- .quarantine
|
690
|
+
- .rspec-report-opts
|
691
|
+
variables:
|
692
|
+
QA_SCENARIO: "Test::Integration::SSHTunnel"
|
693
|
+
|
694
|
+
ee:ssh_tunnel:
|
695
|
+
extends:
|
696
|
+
- .test
|
697
|
+
- .high-capacity
|
698
|
+
- .ee-qa
|
699
|
+
- .rspec-report-opts
|
700
|
+
allow_failure: true
|
701
|
+
variables:
|
702
|
+
QA_SCENARIO: "Test::Integration::SSHTunnel"
|
703
|
+
|
704
|
+
ee:ssh_tunnel-quarantine:
|
705
|
+
extends:
|
706
|
+
- .test
|
707
|
+
- .high-capacity
|
708
|
+
- .ee-qa
|
709
|
+
- .quarantine
|
710
|
+
- .rspec-report-opts
|
711
|
+
variables:
|
712
|
+
QA_SCENARIO: "Test::Integration::SSHTunnel"
|
713
|
+
|
677
714
|
ce:object_storage:
|
678
715
|
extends:
|
679
716
|
- .test
|
@@ -946,7 +983,7 @@ staging:
|
|
946
983
|
generate_test_session:
|
947
984
|
stage: report
|
948
985
|
rules:
|
949
|
-
- if: '$TOP_UPSTREAM_SOURCE_JOB && $TOP_UPSTREAM_SOURCE_REF ==
|
986
|
+
- if: '$TOP_UPSTREAM_SOURCE_JOB && $TOP_UPSTREAM_SOURCE_REF == $TOP_UPSTREAM_DEFAULT_BRANCH'
|
950
987
|
when: always
|
951
988
|
- if: '$TOP_UPSTREAM_SOURCE_JOB =~ /\Ahttps:\/\/ops.gitlab.net\//'
|
952
989
|
when: always
|
@@ -959,6 +996,24 @@ generate_test_session:
|
|
959
996
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
960
997
|
- exe/gitlab-qa-report --generate-test-session "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASE_SESSIONS_PROJECT"
|
961
998
|
|
999
|
+
relate_test_failures:
|
1000
|
+
stage: report
|
1001
|
+
rules:
|
1002
|
+
- if: '$DISABLE_RELATING_FAILURE_ISSUES'
|
1003
|
+
when: never
|
1004
|
+
- if: '$TOP_UPSTREAM_SOURCE_JOB && $TOP_UPSTREAM_SOURCE_REF == $TOP_UPSTREAM_DEFAULT_BRANCH'
|
1005
|
+
when: always
|
1006
|
+
- if: '$TOP_UPSTREAM_SOURCE_JOB =~ /\Ahttps:\/\/ops.gitlab.net\//'
|
1007
|
+
when: always
|
1008
|
+
variables:
|
1009
|
+
QA_FAILURES_REPORTING_PROJECT: "gitlab-org/gitlab"
|
1010
|
+
QA_FAILURES_MAX_DIFF_RATIO: "0.05"
|
1011
|
+
# The --dry-run can be set to modify the behavior of `exe/gitlab-qa-report --relate-failure-issue` without releasing a new gem version.
|
1012
|
+
QA_FAILURES_REPORTER_OPTIONS: ""
|
1013
|
+
script:
|
1014
|
+
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
1015
|
+
- exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" --max-diff-ratio "$QA_FAILURES_MAX_DIFF_RATIO" $QA_FAILURES_REPORTER_OPTIONS
|
1016
|
+
|
962
1017
|
.notify_upstream_commit:
|
963
1018
|
stage: notify
|
964
1019
|
image: ruby:2.6
|
@@ -995,7 +1050,7 @@ notify_slack:
|
|
995
1050
|
rules:
|
996
1051
|
- if: '$TOP_UPSTREAM_SOURCE_JOB && $NOTIFY_CHANNEL'
|
997
1052
|
when: on_failure
|
998
|
-
- if: '$TOP_UPSTREAM_SOURCE_JOB && $TOP_UPSTREAM_SOURCE_REF ==
|
1053
|
+
- if: '$TOP_UPSTREAM_SOURCE_JOB && $TOP_UPSTREAM_SOURCE_REF == $TOP_UPSTREAM_DEFAULT_BRANCH'
|
999
1054
|
when: on_failure
|
1000
1055
|
script:
|
1001
1056
|
- export RELEASE=${TOP_UPSTREAM_SOURCE_REF:-$RELEASE}
|
@@ -1,4 +1,3 @@
|
|
1
|
-
|
2
1
|
# What tests can be run?
|
3
2
|
[[_TOC_]]
|
4
3
|
## The two types of QA tests
|
@@ -142,6 +141,30 @@ To run EE tests, the `EE_LICENSE` environment variable needs to be set:
|
|
142
141
|
|
143
142
|
`$ export EE_LICENSE=$(cat /path/to/GitLab.gitlab_license)`
|
144
143
|
|
144
|
+
## Running a specific test (or set of tests)
|
145
|
+
|
146
|
+
In most of the scenarios listed below, if you don't want to run all the tests
|
147
|
+
it's possible to specify one or more tests. The framework uses RSpec, so tests can be
|
148
|
+
specified as you would when using RSpec.
|
149
|
+
|
150
|
+
For example, the following would run `create_merge_request_spec.rb`:
|
151
|
+
|
152
|
+
```shell
|
153
|
+
$ gitlab-qa Test::Instance::Image EE -- qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb
|
154
|
+
```
|
155
|
+
|
156
|
+
While the following would run all Create UI tests:
|
157
|
+
|
158
|
+
```shell
|
159
|
+
$ gitlab-qa Test::Instance::Image EE -- qa/specs/features/browser_ui/3_create
|
160
|
+
```
|
161
|
+
|
162
|
+
And the following would run all Create API tests as well as UI tests:
|
163
|
+
|
164
|
+
```shell
|
165
|
+
$ gitlab-qa Test::Instance::Image EE -- qa/specs/features/browser_ui/3_create qa/specs/features/api/3_create
|
166
|
+
```
|
167
|
+
|
145
168
|
## Examples
|
146
169
|
|
147
170
|
### `Test::Instance::Image CE|EE|<full image address>`
|
data/lib/gitlab/qa.rb
CHANGED
@@ -57,6 +57,7 @@ module Gitlab
|
|
57
57
|
autoload :SMTP, 'gitlab/qa/scenario/test/integration/smtp'
|
58
58
|
autoload :GitalyCluster, 'gitlab/qa/scenario/test/integration/gitaly_cluster'
|
59
59
|
autoload :Jira, 'gitlab/qa/scenario/test/integration/jira'
|
60
|
+
autoload :SSHTunnel, 'gitlab/qa/scenario/test/integration/ssh_tunnel'
|
60
61
|
end
|
61
62
|
|
62
63
|
module Sanity
|
@@ -22,6 +22,11 @@ module Gitlab
|
|
22
22
|
labels: ['Quality', 'QA', 'triage report', pipeline_name_label]
|
23
23
|
)
|
24
24
|
|
25
|
+
# Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/295493
|
26
|
+
gitlab.create_issue_note(
|
27
|
+
iid: issue.iid,
|
28
|
+
note: "/relate #{Runtime::Env.qa_issue_url}")
|
29
|
+
|
25
30
|
File.write('REPORT_ISSUE_URL', issue.web_url)
|
26
31
|
end
|
27
32
|
# rubocop:enable Metrics/AbcSize
|
@@ -110,7 +110,7 @@ module Gitlab
|
|
110
110
|
channel = pipeline == "canary" ? "qa-production" : "qa-#{pipeline}"
|
111
111
|
error_msg = warn_exception(e)
|
112
112
|
|
113
|
-
return unless QA::Runtime::Env.ci_commit_ref_name ==
|
113
|
+
return unless QA::Runtime::Env.ci_commit_ref_name == QA::Runtime::Env.default_branch
|
114
114
|
|
115
115
|
slack_options = {
|
116
116
|
channel: channel,
|
@@ -10,7 +10,8 @@ module Gitlab
|
|
10
10
|
# Uses the API to create or update GitLab issues with the results of tests from RSpec report files.
|
11
11
|
class RelateFailureIssue < ReportAsIssue
|
12
12
|
DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.05
|
13
|
-
|
13
|
+
FAILURE_STACKTRACE_REGEX = %r{((.*Failure\/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m.freeze
|
14
|
+
ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)/m.freeze
|
14
15
|
NEW_ISSUE_LABELS = Set.new(%w[QA Quality test failure::investigating priority::2]).freeze
|
15
16
|
|
16
17
|
MultipleIssuesFound = Class.new(StandardError)
|
@@ -77,7 +78,8 @@ module Gitlab
|
|
77
78
|
|
78
79
|
def find_relevant_failure_issues(test) # rubocop:disable Metrics/AbcSize
|
79
80
|
ld = Class.new.extend(Gem::Text).method(:levenshtein_distance)
|
80
|
-
|
81
|
+
full_stacktrace = test.failures.first['message_lines'].join("\n")
|
82
|
+
first_test_failure_stacktrace = sanitize_stacktrace(full_stacktrace, FAILURE_STACKTRACE_REGEX) || full_stacktrace
|
81
83
|
|
82
84
|
# Search with the `search` param returns 500 errors, so we filter by ~QA and then filter further in Ruby
|
83
85
|
failure_issues(test).each_with_object({}) do |issue, memo|
|
@@ -88,7 +90,7 @@ module Gitlab
|
|
88
90
|
diff_ratio = distance.zero? ? 0.0 : (distance.to_f / first_test_failure_stacktrace.size).round(3)
|
89
91
|
|
90
92
|
if diff_ratio <= max_diff_ratio
|
91
|
-
puts " => [DEBUG] Issue #{issue} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
|
93
|
+
puts " => [DEBUG] Issue #{issue.web_url} has an acceptable diff ratio of #{(diff_ratio * 100).round(2)}%."
|
92
94
|
# The `Gitlab::ObjectifiedHash` class overrides `#hash` which is used by `Hash#[]=` to compute the hash key.
|
93
95
|
# This leads to a `TypeError Exception: no implicit conversion of Hash into Integer` error, so we convert the object to a hash before using it as a Hash key.
|
94
96
|
# See:
|
@@ -96,18 +98,27 @@ module Gitlab
|
|
96
98
|
# - https://github.com/NARKOZ/gitlab/commit/cbdbd1e32623f018a8fae39932a8e3bc4d929abb?_pjax=%23js-repo-pjax-container#r44484494
|
97
99
|
memo[issue.to_h] = diff_ratio
|
98
100
|
else
|
99
|
-
puts " => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%)
|
101
|
+
puts " => [DEBUG] Found issue #{issue.web_url} but stacktraces are too different (#{(diff_ratio * 100).round(2)}%).\n"
|
102
|
+
puts " => [DEBUG] Issue stacktrace:\n----------------\n#{relevant_issue_stacktrace}\n----------------\n"
|
103
|
+
puts " => [DEBUG] Failure stacktrace:\n----------------\n#{first_test_failure_stacktrace}\n----------------\n"
|
100
104
|
end
|
101
105
|
end
|
102
106
|
end
|
103
107
|
|
104
108
|
def find_issue_stacktrace(issue)
|
105
|
-
|
109
|
+
issue_stacktrace = sanitize_stacktrace(issue.description, ISSUE_STACKTRACE_REGEX)
|
110
|
+
return issue_stacktrace if issue_stacktrace
|
106
111
|
|
107
|
-
|
108
|
-
|
112
|
+
puts " => [DEBUG] Stacktrace couldn't be found for #{issue.web_url}!"
|
113
|
+
end
|
114
|
+
|
115
|
+
def sanitize_stacktrace(stacktrace, regex)
|
116
|
+
stacktrace_match = stacktrace.match(regex)
|
117
|
+
|
118
|
+
if stacktrace_match
|
119
|
+
stacktrace_match[:stacktrace].gsub(/^\s*#.*$/, '').gsub(/^[[:space:]]+/, '').strip
|
109
120
|
else
|
110
|
-
puts " => [DEBUG] Stacktrace
|
121
|
+
puts " => [DEBUG] Stacktrace doesn't match the expected regex (#{regex}):\n----------------\n#{stacktrace}\n----------------\n"
|
111
122
|
end
|
112
123
|
end
|
113
124
|
|
@@ -112,7 +112,7 @@ module Gitlab
|
|
112
112
|
"found:#{pipeline}.gitlab.com"
|
113
113
|
when 'preprod'
|
114
114
|
'found:pre.gitlab.com'
|
115
|
-
when 'staging-orchestrated', 'nightly',
|
115
|
+
when 'staging-orchestrated', 'nightly', QA::Runtime::Env.default_branch
|
116
116
|
"found:#{pipeline}"
|
117
117
|
else
|
118
118
|
raise "No `found:*` label for the `#{pipeline}` pipeline!"
|
@@ -147,14 +147,14 @@ module Gitlab
|
|
147
147
|
# Gets the name of the pipeline the test was run in, to be used as the key of a scoped label
|
148
148
|
#
|
149
149
|
# Tests can be run in several pipelines:
|
150
|
-
# gitlab-qa, nightly,
|
150
|
+
# gitlab-qa, nightly, staging, canary, production, preprod, MRs, and the default branch (master/main)
|
151
151
|
#
|
152
152
|
# Some of those run in their own project, so CI_PROJECT_NAME is the name we need. Those are:
|
153
153
|
# nightly, staging, canary, production, and preprod
|
154
154
|
#
|
155
|
-
# MR, master, and gitlab-qa tests run in gitlab-qa, but we only want to report tests run on
|
156
|
-
# because the other pipelines will be monitored by the author of the MR that triggered them.
|
157
|
-
# So we assume that we're reporting a master pipeline if the project name is 'gitlab-qa'.
|
155
|
+
# MR, master/main, and gitlab-qa tests run in gitlab-qa, but we only want to report tests run on
|
156
|
+
# master/main because the other pipelines will be monitored by the author of the MR that triggered them.
|
157
|
+
# So we assume that we're reporting a master/main pipeline if the project name is 'gitlab-qa'.
|
158
158
|
|
159
159
|
@pipeline ||= Runtime::Env.pipeline_from_project_name
|
160
160
|
end
|
@@ -44,6 +44,7 @@ module Gitlab
|
|
44
44
|
'QA_GITALY_NON_CLUSTER_STORAGE' => :qa_gitaly_non_cluster_storage,
|
45
45
|
'QA_COOKIES' => :qa_cookie,
|
46
46
|
'QA_DEBUG' => :qa_debug,
|
47
|
+
'QA_DEFAULT_BRANCH' => :qa_default_branch,
|
47
48
|
'QA_LOG_PATH' => :qa_log_path,
|
48
49
|
'QA_CAN_TEST_ADMIN_FEATURES' => :qa_can_test_admin_features,
|
49
50
|
'QA_CAN_TEST_GIT_PROTOCOL_V2' => :qa_can_test_git_protocol_v2,
|
@@ -100,6 +101,10 @@ module Gitlab
|
|
100
101
|
end
|
101
102
|
end
|
102
103
|
|
104
|
+
def default_branch
|
105
|
+
ENV['QA_DEFAULT_BRANCH'] || 'master'
|
106
|
+
end
|
107
|
+
|
103
108
|
def gitlab_username
|
104
109
|
ENV['GITLAB_USERNAME'] || 'gitlab-qa'
|
105
110
|
end
|
@@ -157,7 +162,7 @@ module Gitlab
|
|
157
162
|
if ENV['TOP_UPSTREAM_SOURCE_JOB'].to_s.start_with?('https://ops.gitlab.net')
|
158
163
|
'staging-orchestrated'
|
159
164
|
else
|
160
|
-
|
165
|
+
QA::Runtime::Env.default_branch
|
161
166
|
end
|
162
167
|
else
|
163
168
|
ci_project_name
|
@@ -176,6 +181,10 @@ module Gitlab
|
|
176
181
|
ENV['GITLAB_QA_DEV_ACCESS_TOKEN']
|
177
182
|
end
|
178
183
|
|
184
|
+
def initial_root_password
|
185
|
+
ENV['GITLAB_INITIAL_ROOT_PASSWORD']
|
186
|
+
end
|
187
|
+
|
179
188
|
def qa_container_registry_access_token
|
180
189
|
ENV['GITLAB_QA_CONTAINER_REGISTRY_ACCESS_TOKEN']
|
181
190
|
end
|
@@ -251,6 +260,25 @@ module Gitlab
|
|
251
260
|
end
|
252
261
|
end
|
253
262
|
|
263
|
+
def require_initial_password!
|
264
|
+
return unless ENV['GITLAB_INITIAL_ROOT_PASSWORD'].to_s.strip.empty?
|
265
|
+
|
266
|
+
raise ArgumentError, "Environment variable GITLAB_INITIAL_ROOT_PASSWORD must be provided to set an initial root password."
|
267
|
+
end
|
268
|
+
|
269
|
+
def require_ssh_tunnel!
|
270
|
+
%w[TUNNEL_SSH_PRIVATE_KEY TUNNEL_SERVER_HOSTNAME TUNNEL_SSH_USER].each do |env_key|
|
271
|
+
raise ArgumentError, "Environment variable #{env_key} must be set to run tunnel specs" unless ENV.key?(env_key)
|
272
|
+
end
|
273
|
+
end
|
274
|
+
|
275
|
+
def override_default_password!
|
276
|
+
require_initial_password!
|
277
|
+
|
278
|
+
ENV['GITLAB_PASSWORD'] = initial_root_password
|
279
|
+
ENV['GITLAB_ADMIN_PASSWORD'] = initial_root_password
|
280
|
+
end
|
281
|
+
|
254
282
|
def skip_pull?
|
255
283
|
enabled?(ENV['QA_SKIP_PULL'], default: false)
|
256
284
|
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Gitlab
|
4
|
+
module QA
|
5
|
+
module Scenario
|
6
|
+
module Test
|
7
|
+
module Integration
|
8
|
+
class SSHTunnel < Scenario::Template
|
9
|
+
# rubocop:disable Metrics/AbcSize
|
10
|
+
def perform(release, *rspec_args)
|
11
|
+
Runtime::Env.override_default_password!
|
12
|
+
Runtime::Env.require_ssh_tunnel!
|
13
|
+
|
14
|
+
Component::Gitlab.perform do |gitlab|
|
15
|
+
gitlab.release = release
|
16
|
+
gitlab.network = 'test'
|
17
|
+
|
18
|
+
Component::InternetTunnel.perform do |tunnel_gitlab|
|
19
|
+
Component::InternetTunnel.perform do |tunnel_registry|
|
20
|
+
tunnel_gitlab.gitlab_hostname = gitlab.hostname
|
21
|
+
tunnel_gitlab.network = 'test'
|
22
|
+
tunnel_registry.gitlab_hostname = gitlab.hostname
|
23
|
+
tunnel_registry.network = 'test'
|
24
|
+
|
25
|
+
gitlab.omnibus_config = <<~OMNIBUS
|
26
|
+
external_url '#{tunnel_gitlab.url}';
|
27
|
+
nginx['listen_port'] = 80;
|
28
|
+
nginx['listen_https'] = false;
|
29
|
+
# documentation: https://docs.gitlab.com/omnibus/settings/database.html#seed-the-database-fresh-installs-only
|
30
|
+
gitlab_rails['initial_root_password'] = '#{Runtime::Env.require_initial_password}'
|
31
|
+
|
32
|
+
registry_external_url '#{tunnel_registry.url}';
|
33
|
+
registry_nginx['listen_port'] = 80;
|
34
|
+
registry_nginx['listen_https'] = false;
|
35
|
+
OMNIBUS
|
36
|
+
|
37
|
+
tunnel_gitlab.instance do
|
38
|
+
tunnel_registry.instance do
|
39
|
+
gitlab.instance do
|
40
|
+
Component::Specs.perform do |specs|
|
41
|
+
specs.suite = 'Test::Integration::SSHTunnel'
|
42
|
+
specs.release = gitlab.release
|
43
|
+
specs.network = gitlab.network
|
44
|
+
specs.args = [tunnel_gitlab.url, *rspec_args]
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
# rubocop:enable Metrics/AbcSize
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
data/lib/gitlab/qa/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-qa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 6.
|
4
|
+
version: 6.18.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Grzegorz Bizon
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-01-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: climate_control
|
@@ -310,6 +310,7 @@ files:
|
|
310
310
|
- lib/gitlab/qa/scenario/test/integration/praefect.rb
|
311
311
|
- lib/gitlab/qa/scenario/test/integration/saml.rb
|
312
312
|
- lib/gitlab/qa/scenario/test/integration/smtp.rb
|
313
|
+
- lib/gitlab/qa/scenario/test/integration/ssh_tunnel.rb
|
313
314
|
- lib/gitlab/qa/scenario/test/omnibus/image.rb
|
314
315
|
- lib/gitlab/qa/scenario/test/omnibus/update.rb
|
315
316
|
- lib/gitlab/qa/scenario/test/omnibus/upgrade.rb
|