gitlab-qa 7.8.5 → 7.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +50 -39
- data/docs/what_tests_can_be_run.md +15 -1
- data/lib/gitlab/qa/component/specs.rb +4 -5
- data/lib/gitlab/qa/runner.rb +1 -1
- data/lib/gitlab/qa/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 24eecf2dd51ae68e0329202fc5af2cea9b8117f8951833773f8a41a446937468
|
4
|
+
data.tar.gz: 0b09b0122282d99f8af09cd075b4c00cdc0ef89b5d8f7b60087143c7038464be
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: db871aedf8de37c94394776c01f7e93ab08970e85b70ec0e1f67ee7cacf66f4b3b0a21bf32d43e5e92e41906beb0b553bd40ea1d754b0fb721e137060825b460
|
7
|
+
data.tar.gz: 3510bb706ff671ce5b818e5ed4d0457e627c96fadd2e909799aa56947dcaf678ebdc6a2b3a643e45783e0bcbd9b9c8bccff27e36315f37ae45d5c3d5f4119604
|
data/.gitlab-ci.yml
CHANGED
@@ -78,8 +78,31 @@ rspec:
|
|
78
78
|
reports:
|
79
79
|
junit: gitlab-qa-run-*/**/rspec-*.xml
|
80
80
|
script:
|
81
|
-
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $
|
82
|
-
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $
|
81
|
+
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
82
|
+
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
83
|
+
- bundle exec exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
|
84
|
+
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
85
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
86
|
+
- exit $test_run_exit_code
|
87
|
+
|
88
|
+
# For jobs that shouldn't report results in issues, e.g., manually run custom jobs
|
89
|
+
.no-issue-report-script:
|
90
|
+
script:
|
91
|
+
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
92
|
+
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
|
93
|
+
|
94
|
+
# For jobs that provide additional GITLAB_QA_OPTIONS, e.g., jobs that include --omnibus-config
|
95
|
+
.combined-gitlab-qa-options-script:
|
96
|
+
script:
|
97
|
+
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS_COMBINED -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
98
|
+
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS_COMBINED -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
|
99
|
+
|
100
|
+
# The Test::Omnibus::Update scenarios require the release to be specified twice, which can't be done dynamically using the `variables` parameter
|
101
|
+
# So instead we can use this script with two release variables
|
102
|
+
.update-scenario-script:
|
103
|
+
script:
|
104
|
+
- 'echo "Running: bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=$DEFAULT_RELEASE} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
105
|
+
- bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=$DEFAULT_RELEASE} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
83
106
|
- bundle exec exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
|
84
107
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
85
108
|
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
@@ -177,28 +200,23 @@ ee:sanity-framework:
|
|
177
200
|
- .high-capacity
|
178
201
|
- .ee-qa
|
179
202
|
|
180
|
-
# The custom jobs are for manually running specific/alternative tests in MRs, so we don't report them in issues
|
181
203
|
ce:custom-parallel:
|
182
|
-
script:
|
183
|
-
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
184
|
-
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
|
185
204
|
extends:
|
186
205
|
- .test
|
187
206
|
- .high-capacity
|
188
207
|
- .ce-qa
|
189
208
|
- .rspec-report-opts
|
209
|
+
- .no-issue-report-script
|
190
210
|
allow_failure: true
|
191
211
|
parallel: 10
|
192
212
|
|
193
213
|
ee:custom-parallel:
|
194
|
-
script:
|
195
|
-
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
196
|
-
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
|
197
214
|
extends:
|
198
215
|
- .test
|
199
216
|
- .high-capacity
|
200
217
|
- .ee-qa
|
201
218
|
- .rspec-report-opts
|
219
|
+
- .no-issue-report-script
|
202
220
|
allow_failure: true
|
203
221
|
parallel: 10
|
204
222
|
|
@@ -337,61 +355,46 @@ ee:image:
|
|
337
355
|
- .test
|
338
356
|
- .ee-qa
|
339
357
|
|
340
|
-
# The Test::Omnibus::Update scenarios require the release to be specified twice, which can't be done dynamically using the `variables` parameter
|
341
|
-
# So instead we include the script here again, with two release variables
|
342
358
|
ce:update:
|
343
|
-
script:
|
344
|
-
- bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
345
|
-
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
346
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
347
|
-
- exit $test_run_exit_code
|
348
359
|
extends:
|
349
360
|
- .test
|
350
361
|
- .high-capacity
|
351
362
|
- .ce-qa
|
352
363
|
- .rspec-report-opts
|
353
364
|
- .knapsack-variables
|
365
|
+
- .update-scenario-script
|
354
366
|
parallel: 5
|
355
367
|
|
356
368
|
ce:update-quarantine:
|
357
|
-
script:
|
358
|
-
- bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
359
|
-
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
360
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
361
|
-
- exit $test_run_exit_code
|
362
369
|
extends:
|
363
370
|
- .test
|
364
371
|
- .high-capacity
|
365
372
|
- .ce-qa
|
366
373
|
- .quarantine
|
367
374
|
- .rspec-report-opts
|
375
|
+
- .update-scenario-script
|
376
|
+
variables:
|
377
|
+
QA_RSPEC_TAGS: "--tag quarantine --tag ~orchestrated"
|
368
378
|
|
369
379
|
ee:update:
|
370
|
-
script:
|
371
|
-
- bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
372
|
-
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
373
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
374
|
-
- exit $test_run_exit_code
|
375
380
|
extends:
|
376
381
|
- .test
|
377
382
|
- .high-capacity
|
378
383
|
- .ee-qa
|
379
384
|
- .rspec-report-opts
|
380
385
|
- .knapsack-variables
|
386
|
+
- .update-scenario-script
|
381
387
|
parallel: 10
|
382
388
|
|
383
389
|
ee:update-quarantine:
|
384
|
-
script:
|
385
|
-
- bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
386
|
-
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
387
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
388
|
-
- exit $test_run_exit_code
|
389
390
|
extends:
|
390
391
|
- .test
|
391
392
|
- .high-capacity
|
392
393
|
- .ee-qa
|
393
394
|
- .quarantine
|
394
395
|
- .rspec-report-opts
|
396
|
+
variables:
|
397
|
+
QA_RSPEC_TAGS: "--tag quarantine --tag ~orchestrated"
|
395
398
|
|
396
399
|
# The Test::Omnibus::Upgrade scenario isn't run on master (because it always uses the latest CE/EE image) so we don't report the test results in issues
|
397
400
|
ce:upgrade:
|
@@ -648,8 +651,9 @@ ce:object_storage:
|
|
648
651
|
- .high-capacity
|
649
652
|
- .ce-qa
|
650
653
|
- .rspec-report-opts
|
654
|
+
- .combined-gitlab-qa-options-script
|
651
655
|
variables:
|
652
|
-
|
656
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage"
|
653
657
|
|
654
658
|
ce:object_storage-quarantine:
|
655
659
|
extends:
|
@@ -658,8 +662,9 @@ ce:object_storage-quarantine:
|
|
658
662
|
- .ce-qa
|
659
663
|
- .quarantine
|
660
664
|
- .rspec-report-opts
|
665
|
+
- .combined-gitlab-qa-options-script
|
661
666
|
variables:
|
662
|
-
|
667
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage"
|
663
668
|
|
664
669
|
ee:object_storage_aws:
|
665
670
|
extends:
|
@@ -667,8 +672,9 @@ ee:object_storage_aws:
|
|
667
672
|
- .high-capacity
|
668
673
|
- .ee-qa
|
669
674
|
- .rspec-report-opts
|
675
|
+
- .combined-gitlab-qa-options-script
|
670
676
|
variables:
|
671
|
-
|
677
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage_aws"
|
672
678
|
QA_RSPEC_TAGS: "--tag object_storage"
|
673
679
|
|
674
680
|
ee:object_storage_aws-quarantine:
|
@@ -678,8 +684,9 @@ ee:object_storage_aws-quarantine:
|
|
678
684
|
- .ee-qa
|
679
685
|
- .quarantine
|
680
686
|
- .rspec-report-opts
|
687
|
+
- .combined-gitlab-qa-options-script
|
681
688
|
variables:
|
682
|
-
|
689
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage_aws"
|
683
690
|
QA_RSPEC_TAGS: "--tag quarantine --tag object_storage"
|
684
691
|
|
685
692
|
ee:object_storage_gcs:
|
@@ -709,8 +716,9 @@ ee:object_storage:
|
|
709
716
|
- .high-capacity
|
710
717
|
- .ee-qa
|
711
718
|
- .rspec-report-opts
|
719
|
+
- .combined-gitlab-qa-options-script
|
712
720
|
variables:
|
713
|
-
|
721
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage"
|
714
722
|
QA_RSPEC_TAGS: "--tag object_storage"
|
715
723
|
|
716
724
|
ee:object_storage-quarantine:
|
@@ -720,8 +728,9 @@ ee:object_storage-quarantine:
|
|
720
728
|
- .ee-qa
|
721
729
|
- .quarantine
|
722
730
|
- .rspec-report-opts
|
731
|
+
- .combined-gitlab-qa-options-script
|
723
732
|
variables:
|
724
|
-
|
733
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage"
|
725
734
|
QA_RSPEC_TAGS: "--tag quarantine --tag object_storage"
|
726
735
|
|
727
736
|
ee:packages:
|
@@ -730,8 +739,9 @@ ee:packages:
|
|
730
739
|
- .high-capacity
|
731
740
|
- .ee-qa
|
732
741
|
- .rspec-report-opts
|
742
|
+
- .combined-gitlab-qa-options-script
|
733
743
|
variables:
|
734
|
-
|
744
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config packages"
|
735
745
|
QA_RSPEC_TAGS: "--tag packages"
|
736
746
|
|
737
747
|
ee:packages-quarantine:
|
@@ -741,8 +751,9 @@ ee:packages-quarantine:
|
|
741
751
|
- .ee-qa
|
742
752
|
- .quarantine
|
743
753
|
- .rspec-report-opts
|
754
|
+
- .combined-gitlab-qa-options-script
|
744
755
|
variables:
|
745
|
-
|
756
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config packages"
|
746
757
|
QA_RSPEC_TAGS: "--tag quarantine --tag packages"
|
747
758
|
|
748
759
|
ce:registry:
|
@@ -115,9 +115,23 @@ This will first disable `feature_flag_name` flag and run the tests and then enab
|
|
115
115
|
|
116
116
|
You can pass any number of feature flag settings. The tests will run once for each setting.
|
117
117
|
|
118
|
-
See the [QA framework documentation](https://gitlab.com/gitlab-org/gitlab
|
118
|
+
See the [QA framework documentation](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/README.md#running-tests-with-a-feature-flag-enabled-or-disabled)
|
119
119
|
for information on running the tests with different feature flag settings from the QA framework.
|
120
120
|
|
121
|
+
## Running tests with multiple feature flags set
|
122
|
+
|
123
|
+
The options above allow you to enable or disable a single feature flag at a time. However, if you want to set more than
|
124
|
+
one feature flag at the same time you'll need to use `--set-feature-flags` instead.
|
125
|
+
|
126
|
+
The desired state must be set individually for each feature flag in a comma-separated list. For example to disable a feature flag
|
127
|
+
named `feature-one` and enable another named `feature-two`, use the following parameters:
|
128
|
+
|
129
|
+
```shell
|
130
|
+
--set-feature-flags feature-one=disable,feature-two=enable
|
131
|
+
```
|
132
|
+
|
133
|
+
Those parameters will instruct GitLab QA to set both feature flags before running the suite of tests.
|
134
|
+
|
121
135
|
## Specifying the GitLab version
|
122
136
|
|
123
137
|
In each of the examples below, it is possible to test a specific version of GitLab
|
@@ -31,15 +31,14 @@ module Gitlab
|
|
31
31
|
|
32
32
|
feature_flag_sets = []
|
33
33
|
|
34
|
-
# When `args` includes `[..., "--disable-feature", "a", "--enable-feature", "b", ...]`
|
35
|
-
# `feature_flag_sets` will be set to `[["--disable-feature", "a"], ["--enable-feature", "b"]]`
|
36
|
-
# This will result in tests running
|
34
|
+
# When `args` includes `[..., "--disable-feature", "a", "--enable-feature", "b", "--set-feature-flags", "c=enable", ...]`
|
35
|
+
# `feature_flag_sets` will be set to `[["--disable-feature", "a"], ["--enable-feature", "b"], ["--set-feature-flags", "c=enable"]]`
|
36
|
+
# This will result in tests running three times, once with each feature flag option.
|
37
37
|
while (index = args&.index { |x| x =~ /--.*-feature/ })
|
38
38
|
feature_flag_sets << args.slice!(index, 2)
|
39
39
|
end
|
40
40
|
|
41
|
-
# When `args` do not have
|
42
|
-
# add [] so that test is run exactly once.
|
41
|
+
# When `args` do not have any feature flag options, we add [] so that test is run exactly once.
|
43
42
|
feature_flag_sets << [] unless feature_flag_sets.any?
|
44
43
|
|
45
44
|
feature_flag_sets.each do |feature_flag_set|
|
data/lib/gitlab/qa/runner.rb
CHANGED
@@ -18,7 +18,7 @@ module Gitlab
|
|
18
18
|
@omnibus_configurations = %w[default] # always load default configuration
|
19
19
|
|
20
20
|
@options = OptionParser.new do |opts|
|
21
|
-
opts.banner = 'Usage: gitlab-qa
|
21
|
+
opts.banner = 'Usage: gitlab-qa Scenario URL [options] [[--] path] [rspec_options]'
|
22
22
|
|
23
23
|
opts.on('--no-teardown', 'Skip teardown of containers after the scenario completes.') do
|
24
24
|
Runtime::Scenario.define(:teardown, false)
|
data/lib/gitlab/qa/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-qa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 7.
|
4
|
+
version: 7.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- GitLab Quality
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-09-
|
11
|
+
date: 2021-09-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: climate_control
|