gitlab-qa 7.8.5 → 7.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +50 -39
- data/docs/what_tests_can_be_run.md +15 -1
- data/lib/gitlab/qa/component/specs.rb +4 -5
- data/lib/gitlab/qa/runner.rb +1 -1
- data/lib/gitlab/qa/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 24eecf2dd51ae68e0329202fc5af2cea9b8117f8951833773f8a41a446937468
|
4
|
+
data.tar.gz: 0b09b0122282d99f8af09cd075b4c00cdc0ef89b5d8f7b60087143c7038464be
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: db871aedf8de37c94394776c01f7e93ab08970e85b70ec0e1f67ee7cacf66f4b3b0a21bf32d43e5e92e41906beb0b553bd40ea1d754b0fb721e137060825b460
|
7
|
+
data.tar.gz: 3510bb706ff671ce5b818e5ed4d0457e627c96fadd2e909799aa56947dcaf678ebdc6a2b3a643e45783e0bcbd9b9c8bccff27e36315f37ae45d5c3d5f4119604
|
data/.gitlab-ci.yml
CHANGED
@@ -78,8 +78,31 @@ rspec:
|
|
78
78
|
reports:
|
79
79
|
junit: gitlab-qa-run-*/**/rspec-*.xml
|
80
80
|
script:
|
81
|
-
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $
|
82
|
-
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $
|
81
|
+
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
82
|
+
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
83
|
+
- bundle exec exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
|
84
|
+
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
85
|
+
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
86
|
+
- exit $test_run_exit_code
|
87
|
+
|
88
|
+
# For jobs that shouldn't report results in issues, e.g., manually run custom jobs
|
89
|
+
.no-issue-report-script:
|
90
|
+
script:
|
91
|
+
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
92
|
+
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
|
93
|
+
|
94
|
+
# For jobs that provide additional GITLAB_QA_OPTIONS, e.g., jobs that include --omnibus-config
|
95
|
+
.combined-gitlab-qa-options-script:
|
96
|
+
script:
|
97
|
+
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS_COMBINED -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
98
|
+
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS_COMBINED -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
|
99
|
+
|
100
|
+
# The Test::Omnibus::Update scenarios require the release to be specified twice, which can't be done dynamically using the `variables` parameter
|
101
|
+
# So instead we can use this script with two release variables
|
102
|
+
.update-scenario-script:
|
103
|
+
script:
|
104
|
+
- 'echo "Running: bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=$DEFAULT_RELEASE} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
105
|
+
- bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=$DEFAULT_RELEASE} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
83
106
|
- bundle exec exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
|
84
107
|
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
85
108
|
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
@@ -177,28 +200,23 @@ ee:sanity-framework:
|
|
177
200
|
- .high-capacity
|
178
201
|
- .ee-qa
|
179
202
|
|
180
|
-
# The custom jobs are for manually running specific/alternative tests in MRs, so we don't report them in issues
|
181
203
|
ce:custom-parallel:
|
182
|
-
script:
|
183
|
-
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
184
|
-
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
|
185
204
|
extends:
|
186
205
|
- .test
|
187
206
|
- .high-capacity
|
188
207
|
- .ce-qa
|
189
208
|
- .rspec-report-opts
|
209
|
+
- .no-issue-report-script
|
190
210
|
allow_failure: true
|
191
211
|
parallel: 10
|
192
212
|
|
193
213
|
ee:custom-parallel:
|
194
|
-
script:
|
195
|
-
- 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
|
196
|
-
- bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
|
197
214
|
extends:
|
198
215
|
- .test
|
199
216
|
- .high-capacity
|
200
217
|
- .ee-qa
|
201
218
|
- .rspec-report-opts
|
219
|
+
- .no-issue-report-script
|
202
220
|
allow_failure: true
|
203
221
|
parallel: 10
|
204
222
|
|
@@ -337,61 +355,46 @@ ee:image:
|
|
337
355
|
- .test
|
338
356
|
- .ee-qa
|
339
357
|
|
340
|
-
# The Test::Omnibus::Update scenarios require the release to be specified twice, which can't be done dynamically using the `variables` parameter
|
341
|
-
# So instead we include the script here again, with two release variables
|
342
358
|
ce:update:
|
343
|
-
script:
|
344
|
-
- bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
345
|
-
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
346
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
347
|
-
- exit $test_run_exit_code
|
348
359
|
extends:
|
349
360
|
- .test
|
350
361
|
- .high-capacity
|
351
362
|
- .ce-qa
|
352
363
|
- .rspec-report-opts
|
353
364
|
- .knapsack-variables
|
365
|
+
- .update-scenario-script
|
354
366
|
parallel: 5
|
355
367
|
|
356
368
|
ce:update-quarantine:
|
357
|
-
script:
|
358
|
-
- bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
359
|
-
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
360
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
361
|
-
- exit $test_run_exit_code
|
362
369
|
extends:
|
363
370
|
- .test
|
364
371
|
- .high-capacity
|
365
372
|
- .ce-qa
|
366
373
|
- .quarantine
|
367
374
|
- .rspec-report-opts
|
375
|
+
- .update-scenario-script
|
376
|
+
variables:
|
377
|
+
QA_RSPEC_TAGS: "--tag quarantine --tag ~orchestrated"
|
368
378
|
|
369
379
|
ee:update:
|
370
|
-
script:
|
371
|
-
- bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
372
|
-
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
373
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
374
|
-
- exit $test_run_exit_code
|
375
380
|
extends:
|
376
381
|
- .test
|
377
382
|
- .high-capacity
|
378
383
|
- .ee-qa
|
379
384
|
- .rspec-report-opts
|
380
385
|
- .knapsack-variables
|
386
|
+
- .update-scenario-script
|
381
387
|
parallel: 10
|
382
388
|
|
383
389
|
ee:update-quarantine:
|
384
|
-
script:
|
385
|
-
- bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
|
386
|
-
- export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
|
387
|
-
- if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
|
388
|
-
- exit $test_run_exit_code
|
389
390
|
extends:
|
390
391
|
- .test
|
391
392
|
- .high-capacity
|
392
393
|
- .ee-qa
|
393
394
|
- .quarantine
|
394
395
|
- .rspec-report-opts
|
396
|
+
variables:
|
397
|
+
QA_RSPEC_TAGS: "--tag quarantine --tag ~orchestrated"
|
395
398
|
|
396
399
|
# The Test::Omnibus::Upgrade scenario isn't run on master (because it always uses the latest CE/EE image) so we don't report the test results in issues
|
397
400
|
ce:upgrade:
|
@@ -648,8 +651,9 @@ ce:object_storage:
|
|
648
651
|
- .high-capacity
|
649
652
|
- .ce-qa
|
650
653
|
- .rspec-report-opts
|
654
|
+
- .combined-gitlab-qa-options-script
|
651
655
|
variables:
|
652
|
-
|
656
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage"
|
653
657
|
|
654
658
|
ce:object_storage-quarantine:
|
655
659
|
extends:
|
@@ -658,8 +662,9 @@ ce:object_storage-quarantine:
|
|
658
662
|
- .ce-qa
|
659
663
|
- .quarantine
|
660
664
|
- .rspec-report-opts
|
665
|
+
- .combined-gitlab-qa-options-script
|
661
666
|
variables:
|
662
|
-
|
667
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage"
|
663
668
|
|
664
669
|
ee:object_storage_aws:
|
665
670
|
extends:
|
@@ -667,8 +672,9 @@ ee:object_storage_aws:
|
|
667
672
|
- .high-capacity
|
668
673
|
- .ee-qa
|
669
674
|
- .rspec-report-opts
|
675
|
+
- .combined-gitlab-qa-options-script
|
670
676
|
variables:
|
671
|
-
|
677
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage_aws"
|
672
678
|
QA_RSPEC_TAGS: "--tag object_storage"
|
673
679
|
|
674
680
|
ee:object_storage_aws-quarantine:
|
@@ -678,8 +684,9 @@ ee:object_storage_aws-quarantine:
|
|
678
684
|
- .ee-qa
|
679
685
|
- .quarantine
|
680
686
|
- .rspec-report-opts
|
687
|
+
- .combined-gitlab-qa-options-script
|
681
688
|
variables:
|
682
|
-
|
689
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage_aws"
|
683
690
|
QA_RSPEC_TAGS: "--tag quarantine --tag object_storage"
|
684
691
|
|
685
692
|
ee:object_storage_gcs:
|
@@ -709,8 +716,9 @@ ee:object_storage:
|
|
709
716
|
- .high-capacity
|
710
717
|
- .ee-qa
|
711
718
|
- .rspec-report-opts
|
719
|
+
- .combined-gitlab-qa-options-script
|
712
720
|
variables:
|
713
|
-
|
721
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage"
|
714
722
|
QA_RSPEC_TAGS: "--tag object_storage"
|
715
723
|
|
716
724
|
ee:object_storage-quarantine:
|
@@ -720,8 +728,9 @@ ee:object_storage-quarantine:
|
|
720
728
|
- .ee-qa
|
721
729
|
- .quarantine
|
722
730
|
- .rspec-report-opts
|
731
|
+
- .combined-gitlab-qa-options-script
|
723
732
|
variables:
|
724
|
-
|
733
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage"
|
725
734
|
QA_RSPEC_TAGS: "--tag quarantine --tag object_storage"
|
726
735
|
|
727
736
|
ee:packages:
|
@@ -730,8 +739,9 @@ ee:packages:
|
|
730
739
|
- .high-capacity
|
731
740
|
- .ee-qa
|
732
741
|
- .rspec-report-opts
|
742
|
+
- .combined-gitlab-qa-options-script
|
733
743
|
variables:
|
734
|
-
|
744
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config packages"
|
735
745
|
QA_RSPEC_TAGS: "--tag packages"
|
736
746
|
|
737
747
|
ee:packages-quarantine:
|
@@ -741,8 +751,9 @@ ee:packages-quarantine:
|
|
741
751
|
- .ee-qa
|
742
752
|
- .quarantine
|
743
753
|
- .rspec-report-opts
|
754
|
+
- .combined-gitlab-qa-options-script
|
744
755
|
variables:
|
745
|
-
|
756
|
+
GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config packages"
|
746
757
|
QA_RSPEC_TAGS: "--tag quarantine --tag packages"
|
747
758
|
|
748
759
|
ce:registry:
|
@@ -115,9 +115,23 @@ This will first disable `feature_flag_name` flag and run the tests and then enab
|
|
115
115
|
|
116
116
|
You can pass any number of feature flag settings. The tests will run once for each setting.
|
117
117
|
|
118
|
-
See the [QA framework documentation](https://gitlab.com/gitlab-org/gitlab
|
118
|
+
See the [QA framework documentation](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/README.md#running-tests-with-a-feature-flag-enabled-or-disabled)
|
119
119
|
for information on running the tests with different feature flag settings from the QA framework.
|
120
120
|
|
121
|
+
## Running tests with multiple feature flags set
|
122
|
+
|
123
|
+
The options above allow you to enable or disable a single feature flag at a time. However, if you want to set more than
|
124
|
+
one feature flag at the same time you'll need to use `--set-feature-flags` instead.
|
125
|
+
|
126
|
+
The desired state must be set individually for each feature flag in a comma-separated list. For example to disable a feature flag
|
127
|
+
named `feature-one` and enable another named `feature-two`, use the following parameters:
|
128
|
+
|
129
|
+
```shell
|
130
|
+
--set-feature-flags feature-one=disable,feature-two=enable
|
131
|
+
```
|
132
|
+
|
133
|
+
Those parameters will instruct GitLab QA to set both feature flags before running the suite of tests.
|
134
|
+
|
121
135
|
## Specifying the GitLab version
|
122
136
|
|
123
137
|
In each of the examples below, it is possible to test a specific version of GitLab
|
@@ -31,15 +31,14 @@ module Gitlab
|
|
31
31
|
|
32
32
|
feature_flag_sets = []
|
33
33
|
|
34
|
-
# When `args` includes `[..., "--disable-feature", "a", "--enable-feature", "b", ...]`
|
35
|
-
# `feature_flag_sets` will be set to `[["--disable-feature", "a"], ["--enable-feature", "b"]]`
|
36
|
-
# This will result in tests running
|
34
|
+
# When `args` includes `[..., "--disable-feature", "a", "--enable-feature", "b", "--set-feature-flags", "c=enable", ...]`
|
35
|
+
# `feature_flag_sets` will be set to `[["--disable-feature", "a"], ["--enable-feature", "b"], ["--set-feature-flags", "c=enable"]]`
|
36
|
+
# This will result in tests running three times, once with each feature flag option.
|
37
37
|
while (index = args&.index { |x| x =~ /--.*-feature/ })
|
38
38
|
feature_flag_sets << args.slice!(index, 2)
|
39
39
|
end
|
40
40
|
|
41
|
-
# When `args` do not have
|
42
|
-
# add [] so that test is run exactly once.
|
41
|
+
# When `args` do not have any feature flag options, we add [] so that test is run exactly once.
|
43
42
|
feature_flag_sets << [] unless feature_flag_sets.any?
|
44
43
|
|
45
44
|
feature_flag_sets.each do |feature_flag_set|
|
data/lib/gitlab/qa/runner.rb
CHANGED
@@ -18,7 +18,7 @@ module Gitlab
|
|
18
18
|
@omnibus_configurations = %w[default] # always load default configuration
|
19
19
|
|
20
20
|
@options = OptionParser.new do |opts|
|
21
|
-
opts.banner = 'Usage: gitlab-qa
|
21
|
+
opts.banner = 'Usage: gitlab-qa Scenario URL [options] [[--] path] [rspec_options]'
|
22
22
|
|
23
23
|
opts.on('--no-teardown', 'Skip teardown of containers after the scenario completes.') do
|
24
24
|
Runtime::Scenario.define(:teardown, false)
|
data/lib/gitlab/qa/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab-qa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 7.
|
4
|
+
version: 7.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- GitLab Quality
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-09-
|
11
|
+
date: 2021-09-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: climate_control
|