gitlab-qa 7.8.5 → 7.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 75ab48c7d1d2eba818e59d14fdf90a781faf6af477c5993f2ebe27296be785bd
-  data.tar.gz: bc2614ee2f780463fc21360b2e28885227bfe251e8dd5592657feb6b605fc264
+  metadata.gz: 24eecf2dd51ae68e0329202fc5af2cea9b8117f8951833773f8a41a446937468
+  data.tar.gz: 0b09b0122282d99f8af09cd075b4c00cdc0ef89b5d8f7b60087143c7038464be
 SHA512:
-  metadata.gz: 16de17dbf4a4c463fcc185e2432297b597a1d6704f02073010be159dc9994739b20d4733038721011fd1051918589c0cd93d3922d663cc3e3c4dd81545e2e4b0
-  data.tar.gz: 3208f2280f9d00c45e051e1bc2894dfbd9df989e7430931d5375054a79499c67c07f95b66dcee6a066ed9019c613cf6d976fa97ef35596c9e2459c43363c7023
+  metadata.gz: db871aedf8de37c94394776c01f7e93ab08970e85b70ec0e1f67ee7cacf66f4b3b0a21bf32d43e5e92e41906beb0b553bd40ea1d754b0fb721e137060825b460
+  data.tar.gz: 3510bb706ff671ce5b818e5ed4d0457e627c96fadd2e909799aa56947dcaf678ebdc6a2b3a643e45783e0bcbd9b9c8bccff27e36315f37ae45d5c3d5f4119604

data/.gitlab-ci.yml

@@ -78,8 +78,31 @@ rspec:
     reports:
       junit: gitlab-qa-run-*/**/rspec-*.xml
   script:
-    - 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
-    - bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
+    - 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
+    - bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
+    - bundle exec exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
+    - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
+    - exit $test_run_exit_code
+
+# For jobs that shouldn't report results in issues, e.g., manually run custom jobs
+.no-issue-report-script:
+  script:
+    - 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
+    - bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
+
+# For jobs that provide additional GITLAB_QA_OPTIONS, e.g., jobs that include --omnibus-config
+.combined-gitlab-qa-options-script:
+  script:
+    - 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS_COMBINED -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
+    - bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS_COMBINED -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
+
+# The Test::Omnibus::Update scenarios require the release to be specified twice, which can't be done dynamically using the `variables` parameter
+# So instead we can use this script with two release variables
+.update-scenario-script:
+  script:
+    - 'echo "Running: bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=$DEFAULT_RELEASE} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
+    - bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=$DEFAULT_RELEASE} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - bundle exec exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
@@ -177,28 +200,23 @@ ee:sanity-framework:
     - .high-capacity
     - .ee-qa
 
-# The custom jobs are for manually running specific/alternative tests in MRs, so we don't report them in issues
 ce:custom-parallel:
-  script:
-    - 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
-    - bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
   extends:
     - .test
     - .high-capacity
     - .ce-qa
     - .rspec-report-opts
+    - .no-issue-report-script
   allow_failure: true
   parallel: 10
 
 ee:custom-parallel:
-  script:
-    - 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
-    - bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
   extends:
     - .test
     - .high-capacity
     - .ee-qa
     - .rspec-report-opts
+    - .no-issue-report-script
   allow_failure: true
   parallel: 10
 
@@ -337,61 +355,46 @@ ee:image:
     - .test
     - .ee-qa
 
-# The Test::Omnibus::Update scenarios require the release to be specified twice, which can't be done dynamically using the `variables` parameter
-# So instead we include the script here again, with two release variables
 ce:update:
-  script:
-    - bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
-    - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
-    - exit $test_run_exit_code
   extends:
     - .test
     - .high-capacity
     - .ce-qa
     - .rspec-report-opts
     - .knapsack-variables
+    - .update-scenario-script
   parallel: 5
 
 ce:update-quarantine:
-  script:
-    - bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
-    - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
-    - exit $test_run_exit_code
   extends:
     - .test
     - .high-capacity
     - .ce-qa
     - .quarantine
     - .rspec-report-opts
+    - .update-scenario-script
+  variables:
+    QA_RSPEC_TAGS: "--tag quarantine --tag ~orchestrated"
 
 ee:update:
-  script:
-    - bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
-    - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
-    - exit $test_run_exit_code
   extends:
     - .test
     - .high-capacity
     - .ee-qa
     - .rspec-report-opts
     - .knapsack-variables
+    - .update-scenario-script
   parallel: 10
 
 ee:update-quarantine:
-  script:
-    - bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
-    - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
-    - exit $test_run_exit_code
   extends:
     - .test
     - .high-capacity
     - .ee-qa
     - .quarantine
     - .rspec-report-opts
+  variables:
+    QA_RSPEC_TAGS: "--tag quarantine --tag ~orchestrated"
 
 # The Test::Omnibus::Upgrade scenario isn't run on master (because it always uses the latest CE/EE image) so we don't report the test results in issues
 ce:upgrade:
@@ -648,8 +651,9 @@ ce:object_storage:
     - .high-capacity
     - .ce-qa
     - .rspec-report-opts
+    - .combined-gitlab-qa-options-script
   variables:
-    GITLAB_QA_OPTS: "--omnibus-config object_storage"
+    GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage"
 
 ce:object_storage-quarantine:
   extends:
@@ -658,8 +662,9 @@ ce:object_storage-quarantine:
     - .ce-qa
     - .quarantine
     - .rspec-report-opts
+    - .combined-gitlab-qa-options-script
   variables:
-    GITLAB_QA_OPTS: "--omnibus-config object_storage"
+    GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage"
 
 ee:object_storage_aws:
   extends:
@@ -667,8 +672,9 @@ ee:object_storage_aws:
     - .high-capacity
     - .ee-qa
     - .rspec-report-opts
+    - .combined-gitlab-qa-options-script
   variables:
-    GITLAB_QA_OPTS: "--omnibus-config object_storage_aws"
+    GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage_aws"
     QA_RSPEC_TAGS: "--tag object_storage"
 
 ee:object_storage_aws-quarantine:
@@ -678,8 +684,9 @@ ee:object_storage_aws-quarantine:
     - .ee-qa
     - .quarantine
     - .rspec-report-opts
+    - .combined-gitlab-qa-options-script
   variables:
-    GITLAB_QA_OPTS: "--omnibus-config object_storage_aws"
+    GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage_aws"
     QA_RSPEC_TAGS: "--tag quarantine --tag object_storage"
 
 ee:object_storage_gcs:
@@ -709,8 +716,9 @@ ee:object_storage:
     - .high-capacity
     - .ee-qa
     - .rspec-report-opts
+    - .combined-gitlab-qa-options-script
   variables:
-    GITLAB_QA_OPTS: "--omnibus-config object_storage"
+    GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage"
     QA_RSPEC_TAGS: "--tag object_storage"
 
 ee:object_storage-quarantine:
@@ -720,8 +728,9 @@ ee:object_storage-quarantine:
     - .ee-qa
     - .quarantine
     - .rspec-report-opts
+    - .combined-gitlab-qa-options-script
   variables:
-    GITLAB_QA_OPTS: "--omnibus-config object_storage"
+    GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config object_storage"
     QA_RSPEC_TAGS: "--tag quarantine --tag object_storage"
 
 ee:packages:
@@ -730,8 +739,9 @@ ee:packages:
     - .high-capacity
     - .ee-qa
     - .rspec-report-opts
+    - .combined-gitlab-qa-options-script
   variables:
-    GITLAB_QA_OPTS: "--omnibus-config packages"
+    GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config packages"
     QA_RSPEC_TAGS: "--tag packages"
 
 ee:packages-quarantine:
@@ -741,8 +751,9 @@ ee:packages-quarantine:
     - .ee-qa
     - .quarantine
     - .rspec-report-opts
+    - .combined-gitlab-qa-options-script
   variables:
-    GITLAB_QA_OPTS: "--omnibus-config packages"
+    GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --omnibus-config packages"
     QA_RSPEC_TAGS: "--tag quarantine --tag packages"
 
 ce:registry:

data/docs/what_tests_can_be_run.md

@@ -115,9 +115,23 @@ This will first disable `feature_flag_name` flag and run the tests and then enab
 
 You can pass any number of feature flag settings. The tests will run once for each setting.
 
-See the [QA framework documentation](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/qa/README.md#running-tests-with-a-feature-flag-enabled)
+See the [QA framework documentation](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/README.md#running-tests-with-a-feature-flag-enabled-or-disabled)
 for information on running the tests with different feature flag settings from the QA framework.
 
+## Running tests with multiple feature flags set
+
+The options above allow you to enable or disable a single feature flag at a time. However, if you want to set more than
+one feature flag at the same time you'll need to use `--set-feature-flags` instead.
+
+The desired state must be set individually for each feature flag in a comma-separated list. For example to disable a feature flag
+named `feature-one` and enable another named `feature-two`, use the following parameters:
+
+```shell
+--set-feature-flags feature-one=disable,feature-two=enable
+```
+
+Those parameters will instruct GitLab QA to set both feature flags before running the suite of tests.
+
 ## Specifying the GitLab version
 
 In each of the examples below, it is possible to test a specific version of GitLab

data/lib/gitlab/qa/component/specs.rb

@@ -31,15 +31,14 @@ module Gitlab
 
           feature_flag_sets = []
 
-          # When `args` includes `[..., "--disable-feature", "a", "--enable-feature", "b", ...]`
-          # `feature_flag_sets` will be set to `[["--disable-feature", "a"], ["--enable-feature", "b"]]`
-          # This will result in tests running twice, once with each feature.
+          # When `args` includes `[..., "--disable-feature", "a", "--enable-feature", "b", "--set-feature-flags", "c=enable", ...]`
+          # `feature_flag_sets` will be set to `[["--disable-feature", "a"], ["--enable-feature", "b"], ["--set-feature-flags", "c=enable"]]`
+          # This will result in tests running three times, once with each feature flag option.
           while (index = args&.index { |x| x =~ /--.*-feature/ })
             feature_flag_sets << args.slice!(index, 2)
           end
 
-          # When `args` do not have either "--disable-feature" or "--enable-feature", we
-          # add [] so that test is run exactly once.
+          # When `args` do not have any feature flag options, we add [] so that test is run exactly once.
           feature_flag_sets << [] unless feature_flag_sets.any?
 
           feature_flag_sets.each do |feature_flag_set|

data/lib/gitlab/qa/runner.rb

@@ -18,7 +18,7 @@ module Gitlab
         @omnibus_configurations = %w[default] # always load default configuration
 
         @options = OptionParser.new do |opts|
-          opts.banner = 'Usage: gitlab-qa [options] Scenario URL [[--] path] [rspec_options]'
+          opts.banner = 'Usage: gitlab-qa Scenario URL [options] [[--] path] [rspec_options]'
 
           opts.on('--no-teardown', 'Skip teardown of containers after the scenario completes.') do
             Runtime::Scenario.define(:teardown, false)

data/lib/gitlab/qa/version.rb

@@ -1,5 +1,5 @@
 module Gitlab
   module QA
-    VERSION = '7.8.5'.freeze
+    VERSION = '7.9.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 7.8.5
+  version: 7.9.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-09-16 00:00:00.000000000 Z
+date: 2021-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control