gitlab-qa 7.8.2 → 7.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +73 -39
- data/docs/what_tests_can_be_run.md +15 -1
- data/lib/gitlab/qa/component/specs.rb +4 -5
- data/lib/gitlab/qa/report/results_in_issues.rb +51 -35
- data/lib/gitlab/qa/runner.rb +1 -1
- data/lib/gitlab/qa/runtime/env.rb +16 -3
- data/lib/gitlab/qa/runtime/omnibus_configurations/object_storage_gcs.rb +39 -0
- data/lib/gitlab/qa/version.rb +1 -1
- data/lib/gitlab/qa.rb +1 -0
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 24eecf2dd51ae68e0329202fc5af2cea9b8117f8951833773f8a41a446937468
|
4
|
+
data.tar.gz: 0b09b0122282d99f8af09cd075b4c00cdc0ef89b5d8f7b60087143c7038464be
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: db871aedf8de37c94394776c01f7e93ab08970e85b70ec0e1f67ee7cacf66f4b3b0a21bf32d43e5e92e41906beb0b553bd40ea1d754b0fb721e137060825b460
|
7
|
+
data.tar.gz: 3510bb706ff671ce5b818e5ed4d0457e627c96fadd2e909799aa56947dcaf678ebdc6a2b3a643e45783e0bcbd9b9c8bccff27e36315f37ae45d5c3d5f4119604
|
data/.gitlab-ci.yml
CHANGED
@@ -78,8 +78,31 @@ rspec:
|
|
78
78
|
reports:
|
79
79
|
junit: gitlab-qa-run-*/**/rspec-*.xml
|
80
80
|
script:
|
81
|
-
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $
|
82
|
-
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $
|
81
|
+
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
82
|
+
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
83
|
+
- bundle exec exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
|
84
|
+
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
85
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
86
|
+
- exit $test_run_exit_code
|
87
|
+
|
88
|
+
# For jobs that shouldn't report results in issues, e.g., manually run custom jobs
|
89
|
+
.no-issue-report-script:
|
90
|
+
script:
|
91
|
+
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
92
|
+
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
|
93
|
+
|
94
|
+
# For jobs that provide additional GITLAB_QA_OPTIONS, e.g., jobs that include --omnibus-config
|
95
|
+
.combined-gitlab-qa-options-script:
|
96
|
+
script:
|
97
|
+
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS_COMBINED -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
98
|
+
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS_COMBINED -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
|
99
|
+
|
100
|
+
# The Test::Omnibus::Update scenarios require the release to be specified twice, which can't be done dynamically using the `variables` parameter
|
101
|
+
# So instead we can use this script with two release variables
|
102
|
+
.update-scenario-script:
|
103
|
+
script:
|
104
|
+
- 'echo "Running: bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=$DEFAULT_RELEASE} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
105
|
+
- bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=$DEFAULT_RELEASE} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
83
106
|
- bundle exec exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
|
84
107
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
85
108
|
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
@@ -158,6 +181,7 @@ rspec:
|
|
158
181
|
ce:sanity-framework:
|
159
182
|
variables:
|
160
183
|
QA_GENERATE_ALLURE_REPORT: "false"
|
184
|
+
QA_EXPORT_TEST_METRICS: "false"
|
161
185
|
script:
|
162
186
|
- ./bin/expect_exit_code_and_text "bundle exec exe/gitlab-qa Test::Instance::Image ${RELEASE:=CE} -- --tag framework" 1 "2 examples, 1 failure"
|
163
187
|
extends:
|
@@ -168,6 +192,7 @@ ce:sanity-framework:
|
|
168
192
|
ee:sanity-framework:
|
169
193
|
variables:
|
170
194
|
QA_GENERATE_ALLURE_REPORT: "false"
|
195
|
+
QA_EXPORT_TEST_METRICS: "false"
|
171
196
|
script:
|
172
197
|
- ./bin/expect_exit_code_and_text "bundle exec exe/gitlab-qa Test::Instance::Image ${RELEASE:=EE} -- --tag framework" 1 "2 examples, 1 failure"
|
173
198
|
extends:
|
@@ -175,28 +200,23 @@ ee:sanity-framework:
|
|
175
200
|
- .high-capacity
|
176
201
|
- .ee-qa
|
177
202
|
|
178
|
-
# The custom jobs are for manually running specific/alternative tests in MRs, so we don't report them in issues
|
179
203
|
ce:custom-parallel:
|
180
|
-
script:
|
181
|
-
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
182
|
-
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
|
183
204
|
extends:
|
184
205
|
- .test
|
185
206
|
- .high-capacity
|
186
207
|
- .ce-qa
|
187
208
|
- .rspec-report-opts
|
209
|
+
- .no-issue-report-script
|
188
210
|
allow_failure: true
|
189
211
|
parallel: 10
|
190
212
|
|
191
213
|
ee:custom-parallel:
|
192
|
-
script:
|
193
|
-
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
194
|
-
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
|
195
214
|
extends:
|
196
215
|
- .test
|
197
216
|
- .high-capacity
|
198
217
|
- .ee-qa
|
199
218
|
- .rspec-report-opts
|
219
|
+
- .no-issue-report-script
|
200
220
|
allow_failure: true
|
201
221
|
parallel: 10
|
202
222
|
|
@@ -335,61 +355,46 @@ ee:image:
|
|
335
355
|
- .test
|
336
356
|
- .ee-qa
|
337
357
|
|
338
|
-
# The Test::Omnibus::Update scenarios require the release to be specified twice, which can't be done dynamically using the `variables` parameter
|
339
|
-
# So instead we include the script here again, with two release variables
|
340
358
|
ce:update:
|
341
|
-
script:
|
342
|
-
- bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
343
|
-
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
344
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
345
|
-
- exit $test_run_exit_code
|
346
359
|
extends:
|
347
360
|
- .test
|
348
361
|
- .high-capacity
|
349
362
|
- .ce-qa
|
350
363
|
- .rspec-report-opts
|
351
364
|
- .knapsack-variables
|
365
|
+
- .update-scenario-script
|
352
366
|
parallel: 5
|
353
367
|
|
354
368
|
ce:update-quarantine:
|
355
|
-
script:
|
356
|
-
- bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
357
|
-
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
358
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
359
|
-
- exit $test_run_exit_code
|
360
369
|
extends:
|
361
370
|
- .test
|
362
371
|
- .high-capacity
|
363
372
|
- .ce-qa
|
364
373
|
- .quarantine
|
365
374
|
- .rspec-report-opts
|
375
|
+
- .update-scenario-script
|
376
|
+
variables:
|
377
|
+
QA_RSPEC_TAGS: "--tag quarantine --tag ~orchestrated"
|
366
378
|
|
367
379
|
ee:update:
|
368
|
-
script:
|
369
|
-
- bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
370
|
-
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
371
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
372
|
-
- exit $test_run_exit_code
|
373
380
|
extends:
|
374
381
|
- .test
|
375
382
|
- .high-capacity
|
376
383
|
- .ee-qa
|
377
384
|
- .rspec-report-opts
|
378
385
|
- .knapsack-variables
|
386
|
+
- .update-scenario-script
|
379
387
|
parallel: 10
|
380
388
|
|
381
389
|
ee:update-quarantine:
|
382
|
-
script:
|
383
|
-
- bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
384
|
-
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
385
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
386
|
-
- exit $test_run_exit_code
|
387
390
|
extends:
|
388
391
|
- .test
|
389
392
|
- .high-capacity
|
390
393
|
- .ee-qa
|
391
394
|
- .quarantine
|
392
395
|
- .rspec-report-opts
|
396
|
+
variables:
|
397
|
+
QA_RSPEC_TAGS: "--tag quarantine --tag ~orchestrated"
|
393
398
|
|
394
399
|
# The Test::Omnibus::Upgrade scenario isn't run on master (because it always uses the latest CE/EE image) so we don't report the test results in issues
|
395
400
|
ce:upgrade:
|
@@ -646,8 +651,9 @@ ce:object_storage:
|
|
646
651
|
- .high-capacity
|
647
652
|
- .ce-qa
|
648
653
|
- .rspec-report-opts
|
654
|
+
- .combined-gitlab-qa-options-script
|
649
655
|
variables:
|
650
|
-
|
656
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage"
|
651
657
|
|
652
658
|
ce:object_storage-quarantine:
|
653
659
|
extends:
|
@@ -656,8 +662,9 @@ ce:object_storage-quarantine:
|
|
656
662
|
- .ce-qa
|
657
663
|
- .quarantine
|
658
664
|
- .rspec-report-opts
|
665
|
+
- .combined-gitlab-qa-options-script
|
659
666
|
variables:
|
660
|
-
|
667
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage"
|
661
668
|
|
662
669
|
ee:object_storage_aws:
|
663
670
|
extends:
|
@@ -665,11 +672,34 @@ ee:object_storage_aws:
|
|
665
672
|
- .high-capacity
|
666
673
|
- .ee-qa
|
667
674
|
- .rspec-report-opts
|
675
|
+
- .combined-gitlab-qa-options-script
|
668
676
|
variables:
|
669
|
-
|
677
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage_aws"
|
670
678
|
QA_RSPEC_TAGS: "--tag object_storage"
|
671
679
|
|
672
680
|
ee:object_storage_aws-quarantine:
|
681
|
+
extends:
|
682
|
+
- .test
|
683
|
+
- .high-capacity
|
684
|
+
- .ee-qa
|
685
|
+
- .quarantine
|
686
|
+
- .rspec-report-opts
|
687
|
+
- .combined-gitlab-qa-options-script
|
688
|
+
variables:
|
689
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage_aws"
|
690
|
+
QA_RSPEC_TAGS: "--tag quarantine --tag object_storage"
|
691
|
+
|
692
|
+
ee:object_storage_gcs:
|
693
|
+
extends:
|
694
|
+
- .test
|
695
|
+
- .high-capacity
|
696
|
+
- .ee-qa
|
697
|
+
- .rspec-report-opts
|
698
|
+
variables:
|
699
|
+
GITLAB_QA_OPTS: "--omnibus-config object_storage_gcs"
|
700
|
+
QA_RSPEC_TAGS: "--tag object_storage"
|
701
|
+
|
702
|
+
ee:object_storage_gcs-quarantine:
|
673
703
|
extends:
|
674
704
|
- .test
|
675
705
|
- .high-capacity
|
@@ -677,7 +707,7 @@ ee:object_storage_aws-quarantine:
|
|
677
707
|
- .quarantine
|
678
708
|
- .rspec-report-opts
|
679
709
|
variables:
|
680
|
-
GITLAB_QA_OPTS: "--omnibus-config
|
710
|
+
GITLAB_QA_OPTS: "--omnibus-config object_storage_gcs"
|
681
711
|
QA_RSPEC_TAGS: "--tag quarantine --tag object_storage"
|
682
712
|
|
683
713
|
ee:object_storage:
|
@@ -686,8 +716,9 @@ ee:object_storage:
|
|
686
716
|
- .high-capacity
|
687
717
|
- .ee-qa
|
688
718
|
- .rspec-report-opts
|
719
|
+
- .combined-gitlab-qa-options-script
|
689
720
|
variables:
|
690
|
-
|
721
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage"
|
691
722
|
QA_RSPEC_TAGS: "--tag object_storage"
|
692
723
|
|
693
724
|
ee:object_storage-quarantine:
|
@@ -697,8 +728,9 @@ ee:object_storage-quarantine:
|
|
697
728
|
- .ee-qa
|
698
729
|
- .quarantine
|
699
730
|
- .rspec-report-opts
|
731
|
+
- .combined-gitlab-qa-options-script
|
700
732
|
variables:
|
701
|
-
|
733
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage"
|
702
734
|
QA_RSPEC_TAGS: "--tag quarantine --tag object_storage"
|
703
735
|
|
704
736
|
ee:packages:
|
@@ -707,8 +739,9 @@ ee:packages:
|
|
707
739
|
- .high-capacity
|
708
740
|
- .ee-qa
|
709
741
|
- .rspec-report-opts
|
742
|
+
- .combined-gitlab-qa-options-script
|
710
743
|
variables:
|
711
|
-
|
744
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config packages"
|
712
745
|
QA_RSPEC_TAGS: "--tag packages"
|
713
746
|
|
714
747
|
ee:packages-quarantine:
|
@@ -718,8 +751,9 @@ ee:packages-quarantine:
|
|
718
751
|
- .ee-qa
|
719
752
|
- .quarantine
|
720
753
|
- .rspec-report-opts
|
754
|
+
- .combined-gitlab-qa-options-script
|
721
755
|
variables:
|
722
|
-
|
756
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config packages"
|
723
757
|
QA_RSPEC_TAGS: "--tag quarantine --tag packages"
|
724
758
|
|
725
759
|
ce:registry:
|
@@ -115,9 +115,23 @@ This will first disable `feature_flag_name` flag and run the tests and then enab
|
|
115
115
|
|
116
116
|
You can pass any number of feature flag settings. The tests will run once for each setting.
|
117
117
|
|
118
|
-
See the [QA framework documentation](https://gitlab.com/gitlab-org/gitlab
|
118
|
+
See the [QA framework documentation](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/README.md#running-tests-with-a-feature-flag-enabled-or-disabled)
|
119
119
|
for information on running the tests with different feature flag settings from the QA framework.
|
120
120
|
|
121
|
+
## Running tests with multiple feature flags set
|
122
|
+
|
123
|
+
The options above allow you to enable or disable a single feature flag at a time. However, if you want to set more than
|
124
|
+
one feature flag at the same time you'll need to use `--set-feature-flags` instead.
|
125
|
+
|
126
|
+
The desired state must be set individually for each feature flag in a comma-separated list. For example to disable a feature flag
|
127
|
+
named `feature-one` and enable another named `feature-two`, use the following parameters:
|
128
|
+
|
129
|
+
```shell
|
130
|
+
--set-feature-flags feature-one=disable,feature-two=enable
|
131
|
+
```
|
132
|
+
|
133
|
+
Those parameters will instruct GitLab QA to set both feature flags before running the suite of tests.
|
134
|
+
|
121
135
|
## Specifying the GitLab version
|
122
136
|
|
123
137
|
In each of the examples below, it is possible to test a specific version of GitLab
|
@@ -31,15 +31,14 @@ module Gitlab
|
|
31
31
|
|
32
32
|
feature_flag_sets = []
|
33
33
|
|
34
|
-
# When `args` includes `[..., "--disable-feature", "a", "--enable-feature", "b", ...]`
|
35
|
-
# `feature_flag_sets` will be set to `[["--disable-feature", "a"], ["--enable-feature", "b"]]`
|
36
|
-
# This will result in tests running
|
34
|
+
# When `args` includes `[..., "--disable-feature", "a", "--enable-feature", "b", "--set-feature-flags", "c=enable", ...]`
|
35
|
+
# `feature_flag_sets` will be set to `[["--disable-feature", "a"], ["--enable-feature", "b"], ["--set-feature-flags", "c=enable"]]`
|
36
|
+
# This will result in tests running three times, once with each feature flag option.
|
37
37
|
while (index = args&.index { |x| x =~ /--.*-feature/ })
|
38
38
|
feature_flag_sets << args.slice!(index, 2)
|
39
39
|
end
|
40
40
|
|
41
|
-
# When `args` do not have
|
42
|
-
# add [] so that test is run exactly once.
|
41
|
+
# When `args` do not have any feature flag options, we add [] so that test is run exactly once.
|
43
42
|
feature_flag_sets << [] unless feature_flag_sets.any?
|
44
43
|
|
45
44
|
feature_flag_sets.each do |feature_flag_set|
|
@@ -19,6 +19,8 @@ module Gitlab
|
|
19
19
|
puts "Reporting tests in #{test_results.path}"
|
20
20
|
|
21
21
|
test_results.each do |test|
|
22
|
+
puts "Reporting test: #{test.file} | #{test.name}\n"
|
23
|
+
|
22
24
|
report_test(test) unless test.skipped
|
23
25
|
end
|
24
26
|
|
@@ -27,19 +29,18 @@ module Gitlab
|
|
27
29
|
end
|
28
30
|
|
29
31
|
def report_test(test)
|
30
|
-
puts "Reporting test: #{test.file} | #{test.name}"
|
31
|
-
|
32
32
|
testcase = find_testcase(test) || create_testcase(test)
|
33
33
|
test.testcase ||= testcase.web_url.sub('/issues/', '/quality/test_cases/')
|
34
34
|
|
35
|
-
issue =
|
35
|
+
issue = find_linked_results_issue_by_iid(testcase, test)
|
36
36
|
|
37
|
-
|
37
|
+
if issue
|
38
|
+
issue = update_issue_title(issue, test, 'issue') if issue.title.strip != title_from_test(test)
|
39
|
+
else
|
38
40
|
puts "No valid issue link found"
|
39
|
-
issue =
|
41
|
+
issue = find_or_create_results_issue(test)
|
40
42
|
|
41
43
|
add_issue_to_testcase(testcase, issue)
|
42
|
-
puts "Added issue #{issue.web_url} to testcase #{testcase.web_url}"
|
43
44
|
end
|
44
45
|
|
45
46
|
update_labels(testcase, test)
|
@@ -47,18 +48,47 @@ module Gitlab
|
|
47
48
|
end
|
48
49
|
|
49
50
|
def find_testcase(test)
|
50
|
-
|
51
|
+
testcase = find_testcase_by_iid(test)
|
52
|
+
|
53
|
+
if testcase
|
54
|
+
testcase = update_issue_title(testcase, test, 'test_case') if testcase.title.strip != title_from_test(test)
|
55
|
+
else
|
56
|
+
testcase = find_issue(test, 'test_case')
|
57
|
+
end
|
58
|
+
|
59
|
+
testcase
|
60
|
+
end
|
61
|
+
|
62
|
+
def find_testcase_by_iid(test)
|
63
|
+
iid = testcase_iid_from_url(test.testcase)
|
51
64
|
|
52
|
-
|
65
|
+
return unless iid
|
66
|
+
|
67
|
+
find_issue_by_iid(iid, 'test_case')
|
68
|
+
end
|
69
|
+
|
70
|
+
def find_linked_results_issue_by_iid(testcase, test)
|
71
|
+
iid = issue_iid_from_testcase(testcase)
|
53
72
|
|
54
|
-
|
55
|
-
|
56
|
-
|
73
|
+
return unless iid
|
74
|
+
|
75
|
+
find_issue_by_iid(iid, 'issue')
|
76
|
+
end
|
77
|
+
|
78
|
+
def find_issue_by_iid(iid, issue_type)
|
79
|
+
issues = gitlab.find_issues(iid: iid) do |issue|
|
80
|
+
issue.state == 'opened' && issue.issue_type == issue_type
|
57
81
|
end
|
58
82
|
|
59
|
-
warn(%(
|
83
|
+
warn(%(#{issue_type} iid "#{iid}" not valid)) if issues.empty?
|
60
84
|
|
61
|
-
|
85
|
+
issues.first
|
86
|
+
end
|
87
|
+
|
88
|
+
def update_issue_title(issue, test, issue_type)
|
89
|
+
warn(%(#{issue_type} title needs to be updated from '#{issue.title.strip}' to '#{title_from_test(test)}'))
|
90
|
+
|
91
|
+
gitlab.edit_issue(iid: issue.iid, options: { title: title_from_test(test) })
|
62
92
|
end
|
63
93
|
|
64
94
|
def create_testcase(test)
|
@@ -73,7 +103,7 @@ module Gitlab
|
|
73
103
|
)
|
74
104
|
end
|
75
105
|
|
76
|
-
def
|
106
|
+
def testcase_iid_from_url(url)
|
77
107
|
return warn(%(\nPlease update #{url} to test case url")) if url&.include?('/-/issues/')
|
78
108
|
|
79
109
|
url && url.split('/').last.to_i
|
@@ -93,7 +123,7 @@ module Gitlab
|
|
93
123
|
issue_iid&.to_i
|
94
124
|
end
|
95
125
|
|
96
|
-
def
|
126
|
+
def find_or_create_results_issue(test)
|
97
127
|
issue = find_issue(test, 'issue')
|
98
128
|
|
99
129
|
if issue
|
@@ -106,22 +136,12 @@ module Gitlab
|
|
106
136
|
issue
|
107
137
|
end
|
108
138
|
|
109
|
-
def find_issue_by_iid(testcase, test)
|
110
|
-
iid = issue_iid_from_testcase(testcase)
|
111
|
-
|
112
|
-
return unless iid
|
113
|
-
|
114
|
-
issues = search_issues(test: test, issue_type: 'issue', iid: iid)
|
115
|
-
|
116
|
-
warn(%(Issue iid "#{iid}" not valid)) if issues.empty?
|
117
|
-
|
118
|
-
issues.first
|
119
|
-
end
|
120
|
-
|
121
139
|
def find_issue(test, issue_type)
|
122
|
-
issues =
|
140
|
+
issues = gitlab.find_issues(options: { search: search_term(test) }) do |issue|
|
141
|
+
issue.state == 'opened' && issue.issue_type == issue_type && issue.title.strip == title_from_test(test)
|
142
|
+
end
|
123
143
|
|
124
|
-
warn(%(Too many
|
144
|
+
warn(%(Too many #{issue_type}s found with the file path "#{test.file}" and name "#{test.name}")) if issues.many?
|
125
145
|
|
126
146
|
issues.first
|
127
147
|
end
|
@@ -130,6 +150,8 @@ module Gitlab
|
|
130
150
|
results_section = testcase.description.include?(RESULTS_SECTION_TEMPLATE) ? '' : RESULTS_SECTION_TEMPLATE
|
131
151
|
|
132
152
|
gitlab.edit_issue(iid: testcase.iid, options: { description: (testcase.description + results_section + "\n\n#{issue.web_url}") })
|
153
|
+
|
154
|
+
puts "Added issue #{issue.web_url} to testcase #{testcase.web_url}"
|
133
155
|
end
|
134
156
|
|
135
157
|
def update_issue(issue, test)
|
@@ -157,12 +179,6 @@ module Gitlab
|
|
157
179
|
labels << (test.failures.empty? ? "#{pipeline}::passed" : "#{pipeline}::failed")
|
158
180
|
end
|
159
181
|
|
160
|
-
def search_issues(test:, issue_type:, iid: nil)
|
161
|
-
gitlab.find_issues(iid: iid, options: { search: search_term(test) }) do |issue|
|
162
|
-
issue.state == 'opened' && issue.issue_type == issue_type && issue.title.strip == title_from_test(test)
|
163
|
-
end
|
164
|
-
end
|
165
|
-
|
166
182
|
def search_term(test)
|
167
183
|
%("#{partial_file_path(test.file)}" "#{search_safe(test.name)}")
|
168
184
|
end
|
data/lib/gitlab/qa/runner.rb
CHANGED
@@ -18,7 +18,7 @@ module Gitlab
|
|
18
18
|
@omnibus_configurations = %w[default] # always load default configuration
|
19
19
|
|
20
20
|
@options = OptionParser.new do |opts|
|
21
|
-
opts.banner = 'Usage: gitlab-qa
|
21
|
+
opts.banner = 'Usage: gitlab-qa Scenario URL [options] [[--] path] [rspec_options]'
|
22
22
|
|
23
23
|
opts.on('--no-teardown', 'Skip teardown of containers after the scenario completes.') do
|
24
24
|
Runtime::Scenario.define(:teardown, false)
|
@@ -35,6 +35,7 @@ module Gitlab
|
|
35
35
|
'QA_INFLUXDB_URL' => :qa_influxdb_url,
|
36
36
|
'QA_INFLUXDB_TOKEN' => :qa_influxdb_token,
|
37
37
|
'QA_RUN_TYPE' => :qa_run_type,
|
38
|
+
'QA_SKIP_PULL' => :qa_skip_pull,
|
38
39
|
'GITLAB_API_BASE' => :api_base,
|
39
40
|
'GITLAB_ADMIN_USERNAME' => :admin_username,
|
40
41
|
'GITLAB_ADMIN_PASSWORD' => :admin_password,
|
@@ -78,13 +79,14 @@ module Gitlab
|
|
78
79
|
'CI_NODE_INDEX' => :ci_node_index,
|
79
80
|
'CI_NODE_TOTAL' => :ci_node_total,
|
80
81
|
'CI_PROJECT_NAME' => :ci_project_name,
|
82
|
+
'CI_SLACK_WEBHOOK_URL' => :ci_slack_webhook_url,
|
83
|
+
'CI_PIPELINE_CREATED_AT' => :ci_pipeline_created_at,
|
84
|
+
'CI_MERGE_REQUEST_IID' => :ci_merge_request_iid,
|
81
85
|
'GITLAB_CI' => :gitlab_ci,
|
82
|
-
'QA_SKIP_PULL' => :qa_skip_pull,
|
83
86
|
'ELASTIC_URL' => :elastic_url,
|
84
87
|
'GITLAB_QA_LOOP_RUNNER_MINUTES' => :gitlab_qa_loop_runner_minutes,
|
85
88
|
'MAILHOG_HOSTNAME' => :mailhog_hostname,
|
86
89
|
'SLACK_QA_CHANNEL' => :slack_qa_channel,
|
87
|
-
'CI_SLACK_WEBHOOK_URL' => :ci_slack_webhook_url,
|
88
90
|
'SLACK_ICON_EMOJI' => :slack_icon_emoji,
|
89
91
|
'GITLAB_QA_FORMLESS_LOGIN_TOKEN' => :gitlab_qa_formless_login_token,
|
90
92
|
'GEO_MAX_FILE_REPLICATION_TIME' => :geo_max_file_replication_time,
|
@@ -101,7 +103,12 @@ module Gitlab
|
|
101
103
|
'AWS_S3_REGION' => :aws_s3_region,
|
102
104
|
'AWS_S3_KEY_ID' => :aws_s3_key_id,
|
103
105
|
'AWS_S3_ACCESS_KEY' => :aws_s3_access_key,
|
104
|
-
'AWS_S3_BUCKET_NAME' => :aws_s3_bucket_name
|
106
|
+
'AWS_S3_BUCKET_NAME' => :aws_s3_bucket_name,
|
107
|
+
'TOP_UPSTREAM_MERGE_REQUEST_IID' => :top_upstream_merge_request_iid,
|
108
|
+
'GOOGLE_PROJECT' => :google_project,
|
109
|
+
'GOOGLE_CLIENT_EMAIL' => :google_client_email,
|
110
|
+
'GOOGLE_JSON_KEY' => :google_json_key,
|
111
|
+
'GCS_BUCKET_NAME' => :gcs_bucket_name
|
105
112
|
}.freeze
|
106
113
|
|
107
114
|
ENV_VARIABLES.each do |env_name, method_name|
|
@@ -296,6 +303,12 @@ module Gitlab
|
|
296
303
|
end
|
297
304
|
end
|
298
305
|
|
306
|
+
def require_gcs_environment!
|
307
|
+
%w[GOOGLE_PROJECT GOOGLE_CLIENT_EMAIL GOOGLE_JSON_KEY GCS_BUCKET_NAME].each do |env_key|
|
308
|
+
raise ArgumentError, "Environment variable #{env_key} must be set to run GCS object storage specs" unless ENV.key?(env_key)
|
309
|
+
end
|
310
|
+
end
|
311
|
+
|
299
312
|
def require_initial_password!
|
300
313
|
return unless ENV['GITLAB_INITIAL_ROOT_PASSWORD'].to_s.strip.empty?
|
301
314
|
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'tempfile'
|
3
|
+
|
4
|
+
module Gitlab
|
5
|
+
module QA
|
6
|
+
module Runtime
|
7
|
+
module OmnibusConfigurations
|
8
|
+
class ObjectStorageGcs < Default
|
9
|
+
def configuration
|
10
|
+
Runtime::Env.require_gcs_environment!
|
11
|
+
|
12
|
+
json_key = setup_json_key
|
13
|
+
|
14
|
+
<<~OMNIBUS
|
15
|
+
gitlab_rails['object_store']['connection'] = { 'provider' => 'Google', 'google_project' => '#{Runtime::Env.google_project}', 'google_client_email' => '#{Runtime::Env.google_client_email}', 'google_json_key_location' => '#{json_key.path}' }
|
16
|
+
|
17
|
+
gitlab_rails['object_store']['objects']['artifacts']['bucket'] = '#{Runtime::Env.gcs_bucket_name}'
|
18
|
+
gitlab_rails['object_store']['objects']['external_diffs']['bucket'] = '#{Runtime::Env.gcs_bucket_name}'
|
19
|
+
gitlab_rails['object_store']['objects']['lfs']['bucket'] = '#{Runtime::Env.gcs_bucket_name}'
|
20
|
+
gitlab_rails['object_store']['objects']['uploads']['bucket'] = '#{Runtime::Env.gcs_bucket_name}'
|
21
|
+
gitlab_rails['object_store']['objects']['packages']['bucket'] = '#{Runtime::Env.gcs_bucket_name}'
|
22
|
+
gitlab_rails['object_store']['objects']['dependency_proxy']['bucket'] = '#{Runtime::Env.gcs_bucket_name}'
|
23
|
+
gitlab_rails['object_store']['objects']['pages']['bucket'] = '#{Runtime::Env.gcs_bucket_name}'
|
24
|
+
gitlab_rails['object_store']['objects']['terraform_state']['bucket'] = '#{Runtime::Env.gcs_bucket_name}'
|
25
|
+
OMNIBUS
|
26
|
+
end
|
27
|
+
|
28
|
+
def setup_json_key
|
29
|
+
Tempfile.open('gcs-json-key', ENV['CI_PROJECT_DIR']) do |file|
|
30
|
+
file.write(ENV.fetch('GOOGLE_JSON_KEY'))
|
31
|
+
|
32
|
+
file
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/gitlab/qa/version.rb
CHANGED
data/lib/gitlab/qa.rb
CHANGED
@@ -15,6 +15,7 @@ module Gitlab
|
|
15
15
|
autoload :Packages, 'gitlab/qa/runtime/omnibus_configurations/packages'
|
16
16
|
autoload :ObjectStorage, 'gitlab/qa/runtime/omnibus_configurations/object_storage'
|
17
17
|
autoload :ObjectStorageAws, 'gitlab/qa/runtime/omnibus_configurations/object_storage_aws'
|
18
|
+
autoload :ObjectStorageGcs, 'gitlab/qa/runtime/omnibus_configurations/object_storage_gcs'
|
18
19
|
autoload :LicenseMode, 'gitlab/qa/runtime/omnibus_configurations/license_mode'
|
19
20
|
end
|
20
21
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-qa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 7.
|
4
|
+
version: 7.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- GitLab Quality
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-09-
|
11
|
+
date: 2021-09-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: climate_control
|
@@ -294,6 +294,7 @@ files:
|
|
294
294
|
- lib/gitlab/qa/runtime/omnibus_configurations/license_mode.rb
|
295
295
|
- lib/gitlab/qa/runtime/omnibus_configurations/object_storage.rb
|
296
296
|
- lib/gitlab/qa/runtime/omnibus_configurations/object_storage_aws.rb
|
297
|
+
- lib/gitlab/qa/runtime/omnibus_configurations/object_storage_gcs.rb
|
297
298
|
- lib/gitlab/qa/runtime/omnibus_configurations/packages.rb
|
298
299
|
- lib/gitlab/qa/runtime/scenario.rb
|
299
300
|
- lib/gitlab/qa/runtime/token_finder.rb
|