gitlab-qa 5.13.7 → 5.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 80567a3f14ce823c574bf448dd57339aa9ac95d39514b2c18ec4f8d9094c5e95
-  data.tar.gz: 4aa694b95e7bc667a58d63d7ac4c2606dbbd3d9a5819cd49e160dfae60b919cc
+  metadata.gz: a9c4251f7eaa756abc42d5cb76b0b318ec3b14004331d32a92e125135055ecef
+  data.tar.gz: 04bb9549728a2533cfb34b2e5ade00260511fda1d6e4d052769d9682b73b497b
 SHA512:
-  metadata.gz: 1a9f28e68a1717e93f7df4dda6de613185b3175417092517d56338d35af67679eed4a8d074e6ee5b6efd1f9d425b761aec3d9a8803d9d993341a9a28b1abde4c
-  data.tar.gz: da4ca897e44bd8c398c42f431370b73bb482fdb032b059053b759eb0c97646500057961b8a378a789b39cf5ab093e00a4a8577c36fbb6ef9353ce9116b24f98a
+  metadata.gz: dfb8a2afb17ac4d47e0c770c8da39a3290d6ac58d6306f3a930d50b08180660c85995ff248b1f50bdd6f8d9fe26b29f8680016b03ff07f85d5600e1bd5c7be90
+  data.tar.gz: a84cd2cef18c8f229826b47236edd8749345de2bfb51be1705109d623b0fefec7b09088f46244df206f85665420fb22dbdc3d8a250e617a8ab45876e95fc99d0

data/.gitlab-ci.yml

@@ -31,6 +31,8 @@ workflow:
     - if: '$CI_COMMIT_BRANCH == "master"'
     # For tags, create a pipeline.
     - if: '$CI_COMMIT_TAG'
+    # For triggers from GitLab MR pipelines (and pipelines from other projects), create a pipeline
+    - if: '$CI_PIPELINE_SOURCE == "pipeline"'
 
 .default-rules:
   rules:
@@ -830,6 +832,44 @@ ee:praefect-quarantine:
     QA_CAN_TEST_PRAEFECT: "true"
     QA_RSPEC_TAGS: "--tag quarantine --tag ~orchestrated"
 
+ce:gitaly-cluster:
+  extends:
+    - .test
+    - .high-capacity
+    - .ce-qa
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::GitalyCluster"
+
+ce:gitaly-cluster-quarantine:
+  extends:
+    - .test
+    - .high-capacity
+    - .ce-qa
+    - .quarantine
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::GitalyCluster"
+
+ee:gitaly-cluster:
+  extends:
+    - .test
+    - .high-capacity
+    - .ee-qa
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::GitalyCluster"
+
+ee:gitaly-cluster-quarantine:
+  extends:
+    - .test
+    - .high-capacity
+    - .ee-qa
+    - .quarantine
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::GitalyCluster"
+
 ce:smtp:
   extends:
     - .test
@@ -891,7 +931,8 @@ ee:jira-quarantine:
 staging:
   script:
     - unset EE_LICENSE
-    - exe/gitlab-qa Test::Instance::Staging
+    - 'echo "Running: exe/gitlab-qa Test::Instance::Staging ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS"'
+    - exe/gitlab-qa Test::Instance::Staging ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS
   extends:
     - .test
     - .high-capacity

data/README.md

@@ -121,11 +121,58 @@ All the scenarios you can run are described in the
 
 Note: The GitLab QA tool requires that [Docker](https://docs.docker.com/install/) is installed.
 
-### How to add new scenarios
-
-Scenarios (test cases) and scripts to run them are located in the
-[CE](https://gitlab.com/gitlab-org/gitlab-ce/tree/master/qa) and
-[EE](https://gitlab.com/gitlab-org/gitlab-ee/tree/master/qa)
+### Command-line options
+
+In addition to the [arguments you can use to specify the scenario and
+tests to run](/docs/what_tests_can_be_run.md), you can use the
+following options to control the tool's behavior.
+
+**Note:** These are `gitlab-qa` options so if you specify RSpec
+options as well, including test file paths, be sure to add these
+options before the `--` that indicates that subsequent arguments are
+intended for RSpec.
+
+| Option | Description |
+| ------ | ----------- |
+| `--no-teardown` | Skip teardown of containers after the scenario completes |
+| `--no-tests` | Orchestrates the docker containers but does not run the tests. Implies `--no-teardown` |
+
+For example, the following command would start an EE GitLab Docker
+container and would leave the instance running, but would not run the
+tests:
+
+```plaintext
+$ gitlab-qa Test::Instance::Image EE --no-tests
+```
+
+GitLab QA will have automatically run the `docker ps` command to show
+the port that container is running on, for example:
+
+```plaintext
+...
+Skipping tests.
+The orchestrated docker containers have not been removed.
+Docker shell command: `docker ps`
+CONTAINER ID  IMAGE                     ... PORTS
+fdeffd791b69  gitlab/gitlab-ee:nightly      22/tcp, 443/tcp, 0.0.0.0:32768->80/tcp
+```
+
+You could then run tests against that instance in a similar way to
+[running tests against GDK](/docs/run_qa_against_gdk.md). This can be
+useful if you want to run and debug a specific test, for example:
+
+```plaintext
+# From /path/to/gdk/gitlab/qa
+$ bundle exec bin/qa Test::Instance::All http://localhost:32768 -- qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb
+```
+
+### How to add new tests
+
+Please see the [Beginner's guide to writing end-to-end tests](https://docs.gitlab.com/ee/development/testing_guide/end_to_end/beginners_guide.html).
+
+Test cases and scripts to run them are located in the
+[GitLab FOSS](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/qa) and
+[GitLab](https://gitlab.com/gitlab-org/gitlab/tree/master/qa)
 repositories under the `qa/` directory, so please also check the documentation
 there.
 

data/docs/what_tests_can_be_run.md

@@ -1,5 +1,6 @@
-# What tests can be run?
 
+# What tests can be run?
+[[_TOC_]]
 ## The two types of QA tests
 
 First of all, the first thing to choose is whether you want to run orchestrated
@@ -28,6 +29,8 @@ For more details on the internals, please read the
 
 ## Supported GitLab environment variables
 
+All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa/runtime/env.rb`](https://gitlab.com/gitlab-org/gitlab-qa/-/blob/master/lib/gitlab/qa/runtime/env.rb).
+
 |        Variable       | Default   | Description           | Required |
 |-----------------------|-----------|-----------------------|----------|
 | `GITLAB_USERNAME`       | `root`  | Username to use when signing into GitLab. | Yes|
@@ -53,7 +56,8 @@ For more details on the internals, please read the
 | `GITLAB_SANDBOX_NAME`   | `gitlab-qa-sandbox` | The sandbox group name the test suite is going to use. | No|
 | `GITLAB_QA_ACCESS_TOKEN`|-  | A valid personal access token with the `api` scope. This is used for API access during tests, and is used in the version that staging is currently running. An existing token that is valid on [`Test::Instance::Staging`](#testinstancestaging) scenario to retrieve the staging can be found in the shared 1Password vault. |No|
 | `GITLAB_QA_ADMIN_ACCESS_TOKEN` |-  | A valid personal access token with the `api` scope from a user with admin access. Used for API access as an admin during tests. | No|
-| `EE_LICESEN` |-  | Enterprise Edition license. | No|
+| `GITLAB_QA_CONTAINER_REGISTRY_ACCESS_TOKEN` | - | A valid personal access token with the `read_registry` scope. Used to [access the container registry on `registry.gitlab.com` when tests run in a CI job that _is not_ triggered via another pipeline](https://gitlab.com/gitlab-org/gitlab-qa/-/blob/364addb83e7b136ff0f9d8719ca9553d290aa9ab/lib/gitlab/qa/release.rb#L152). For example, if you manually run a [new Staging pipeline](https://ops.gitlab.net/gitlab-org/quality/staging/-/pipelines/new), this token will be used. | No |
+| `EE_LICENSE` |-  | Enterprise Edition license. | No|
 | `QA_ARTIFACTS_DIR` |`/tmp/gitlab-qa`| Path to a directory where artifacts (logs and screenshots) for failing tests will be saved. | No|
 | `DOCKER_HOST` |`http://localhost`| Docker host to run tests against. | No|
 | `CHROME_HEADLESS` |-  | When running locally, set to `false` to allow Chrome tests to be visible - watch your tests being run. | No|
@@ -83,8 +87,27 @@ For more details on the internals, please read the
 
 ## Running tests with a feature flag enabled
 
-It is possible to enable a feature flag before running tests. See the [QA
-framework documentation](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/qa/README.md#running-tests-with-a-feature-flag-enabled) for details.
+It is possible to enable or disable a feature flag before running tests.
+To test a Gitlab image with a feature flag enabled, run this command:
+```
+$ gitlab-qa Test::Instance::Image gitlab/gitlab-ee:12.4.0-ee.0 --enable-feature feature_flag_name
+```
+
+To run a test with feature flag disabled, run this command:
+```
+$ gitlab-qa Test::Instance::Image gitlab/gitlab-ee:12.4.0-ee.0 --disable-feature feature_flag_name
+```
+
+You an also test a GitLab image multiple times with different feature flag settings:
+```
+$ gitlab-qa Test::Instance::Image gitlab/gitlab-ee:12.4.0-ee.0 --disable-feature feature_flag_name --enable-feature feature_flag_name
+```
+This will first disable `feature_flag_name` flag and run the tests and then enable it and run the tests again.
+
+You can pass any number of feature flag settings. The tests will run once for each setting.
+
+See the [QA framework documentation](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/qa/README.md#running-tests-with-a-feature-flag-enabled)
+for information on running the tests with different feature flag settings from the QA framework.
 
 ## Specifying the GitLab version
 
@@ -545,7 +568,7 @@ in the GitLab CE project).
 
 - `GITLAB_QA_DEV_ACCESS_TOKEN`: A valid personal access token for the
   `gitlab-qa-bot` on `dev.gitlab.org` with the `registry` scope.
-  This is used to pull the QA Docker from the Omnibus GitLab `dev` Container Registry.
+  This is used to pull the QA Docker image from the Omnibus GitLab `dev` Container Registry.
   If the variable isn't present, the QA image from Docker Hub will be used.
   This can be found in the shared 1Password vault.
 
@@ -560,6 +583,41 @@ $ export GITLAB_PASSWORD="$GITLAB_QA_PASSWORD"
 $ gitlab-qa Test::Instance::Staging
 ```
 
+### `Test::Instance::StagingGeo`
+
+This scenario tests that the Geo staging deployment (with [`staging.gitlab.com`](https://staging.gitlab.com) as the primary site and [`geo.staging.gitlab.com`](https://geo.staging.gitlab.com) as the secondary site) works as expected by running tests tagged `:geo` against it. This is done by spinning up a GitLab QA (`gitlab/gitlab-qa`) container and running the `QA::EE::Scenario::Test::Geo` scenario. Note that the Geo setup steps in the `QA::EE::Scenario::Test::Geo` scenario are skipped when testing a live Geo deployment.
+
+**Required user properties:**
+
+- The user must provide OAuth authorization on the secondary site before running Geo tests. This can be done via the authorization modal that appears after logging into the secondary node for the first time.
+
+- Some Geo tests require the user to have Admin access level (for example, the Geo Nodes API tests)
+
+**Required environment variables:**
+
+- `GITLAB_QA_ACCESS_TOKEN`: A valid personal access token with the `api` scope.
+  This is used to retrieve the version that staging is currently running.
+  This can be found in the shared 1Password vault.
+
+**Optional environment variables:**
+
+- `GITLAB_QA_DEV_ACCESS_TOKEN`: A valid personal access token for the
+  `gitlab-qa-bot` on `dev.gitlab.org` with the `registry` scope.
+  This is used to pull the QA Docker image from the Omnibus GitLab `dev` Container Registry.
+  If the variable isn't present, the QA image from Docker Hub will be used.
+  This can be found in the shared 1Password vault.
+
+Example:
+
+```
+$ export GITLAB_QA_ACCESS_TOKEN=your_api_access_token
+$ export GITLAB_QA_DEV_ACCESS_TOKEN=your_dev_registry_access_token
+$ export GITLAB_USERNAME="gitlab-qa"
+$ export GITLAB_PASSWORD="$GITLAB_QA_PASSWORD"
+
+$ gitlab-qa Test::Instance::StagingGeo
+```
+
 ### `Test::Instance::Production`
 
 This scenario functions the same as `Test::Instance::Staging`
@@ -618,6 +676,40 @@ $ export QA_ADDITIONAL_REPOSITORY_STORAGE=secondary
 $ gitlab-qa Test::Instance::RepositoryStorage
 ```
 
+### `Test::Instance::Airgapped`
+
+This scenario will run tests from the test suite against an airgapped instance.
+The airgapped instance is set up by using `iptables` in the GitLab container to block network traffic other than testable ports, and by using runners
+in a shared internal network.
+
+Example:
+
+```
+# For EE
+$ export EE_LICENSE=$(cat /path/to/gitlab_license)
+
+$ gitlab-qa Test::Instance::Airgapped EE -- --tag smoke
+```
+
+### `Test::Instance::Geo EE|<full image address>:nightly|latest|any_tag http://geo-primary.gitlab http://geo-secondary.gitlab`
+
+This scenario will run tests tagged `:geo` against a live Geo deployment, by spinning up a GitLab QA (`gitlab/gitlab-qa`)
+container and running the `QA::EE::Scenario::Test::Geo` scenario. Note that the Geo setup steps in the `QA::EE::Scenario::Test::Geo` scenario are skipped when testing a live Geo deployment. The URLs for the primary site and secondary site of the live Geo deployment must be provided as command line arguments.
+
+**Required user properties:**
+
+- The user must provide OAuth authorization on the secondary site before running Geo tests. This can be done via the authorization modal that appears after signing into the secondary node for the first time.
+
+- Some Geo tests require the user to have Admin access level (for example, the Geo Nodes API tests)
+
+Example:
+
+```
+$ export GITLAB_USERNAME="gitlab-qa"
+$ export GITLAB_PASSWORD="$GITLAB_QA_PASSWORD"
+
+$ gitlab-qa Test::Instance::Geo EE https://primary.gitlab.com https://secondary.gitlab.com
+```
 ----
 
 [Back to README.md](../README.md)

data/lib/gitlab/qa.rb

@@ -29,6 +29,7 @@ module Gitlab
           autoload :Release, 'gitlab/qa/scenario/test/instance/release'
           autoload :Geo, 'gitlab/qa/scenario/test/instance/geo'
           autoload :StagingGeo, 'gitlab/qa/scenario/test/instance/staging_geo'
+          autoload :Airgapped, 'gitlab/qa/scenario/test/instance/airgapped'
         end
 
         module Omnibus
@@ -53,7 +54,7 @@ module Gitlab
           autoload :Praefect, 'gitlab/qa/scenario/test/integration/praefect'
           autoload :Elasticsearch, 'gitlab/qa/scenario/test/integration/elasticsearch'
           autoload :SMTP, 'gitlab/qa/scenario/test/integration/smtp'
-          autoload :GitalyHA, 'gitlab/qa/scenario/test/integration/gitaly_ha'
+          autoload :GitalyCluster, 'gitlab/qa/scenario/test/integration/gitaly_cluster'
           autoload :Jira, 'gitlab/qa/scenario/test/integration/jira'
         end
 

data/lib/gitlab/qa/component/base.rb

@@ -5,7 +5,7 @@ module Gitlab
         include Scenario::Actable
 
         attr_reader :docker
-        attr_accessor :volumes, :network, :environment
+        attr_accessor :volumes, :network, :environment, :runner_network
         attr_writer :name, :exec_commands
 
         def initialize
@@ -42,6 +42,16 @@ module Gitlab
         end
 
         def instance
+          instance_no_teardown do
+            yield self if block_given?
+          end
+        ensure
+          teardown
+        end
+
+        alias_method :launch_and_teardown_instance, :instance
+
+        def instance_no_teardown
           prepare
           start
           reconfigure
@@ -49,24 +59,22 @@ module Gitlab
           process_exec_commands
 
           yield self if block_given?
-        ensure
-          teardown
         end
 
-        alias_method :launch_and_teardown_instance, :instance
-
         def prepare
           prepare_docker_image
           prepare_network
         end
 
         def prepare_docker_image
-          return if Runtime::Env.skip_pull?
-
           pull
         end
 
         def prepare_network
+          if runner_network && !docker.network_exists?(runner_network)
+            docker.network_create("--driver=bridge --internal #{runner_network}")
+          end
+
           return if docker.network_exists?(network)
 
           docker.network_create(network)
@@ -102,6 +110,17 @@ module Gitlab
         end
 
         def teardown
+          unless teardown?
+            puts "The orchestrated docker containers have not been removed."
+            docker.ps
+
+            return
+          end
+
+          teardown!
+        end
+
+        def teardown!
           assert_name!
 
           return unless docker.running?(name)
@@ -111,6 +130,8 @@ module Gitlab
         end
 
         def pull
+          return if Runtime::Env.skip_pull?
+
           docker.pull(image, tag)
         end
 
@@ -125,6 +146,10 @@ module Gitlab
         def assert_name!
           raise 'Invalid instance name!' unless name
         end
+
+        def teardown?
+          !Runtime::Scenario.attributes.include?(:teardown) || Runtime::Scenario.teardown
+        end
       end
     end
   end

data/lib/gitlab/qa/component/gitlab.rb

@@ -11,7 +11,7 @@ module Gitlab
         extend Forwardable
 
         attr_reader :release
-        attr_accessor :tls, :disable_animations, :skip_availability_check
+        attr_accessor :tls, :disable_animations, :skip_availability_check, :runner_network
         attr_writer :name, :relative_path
 
         def_delegators :release, :tag, :image, :edition
@@ -116,6 +116,7 @@ module Gitlab
               command << "--network-alias #{network_alias}"
             end
           end
+          Docker::Command.execute("network connect --alias #{name}.#{network} --alias #{name}.#{runner_network} #{runner_network} #{name}") if runner_network
         end
 
         def reconfigure

data/lib/gitlab/qa/component/specs.rb

@@ -8,7 +8,7 @@ module Gitlab
       # the `qa/` directory located in GitLab CE / EE repositories.
       #
       class Specs < Scenario::Template
-        attr_accessor :suite, :release, :network, :args, :volumes, :env
+        attr_accessor :suite, :release, :network, :args, :volumes, :env, :runner_network
 
         def initialize
           @docker = Docker::Engine.new
@@ -17,6 +17,8 @@ module Gitlab
         end
 
         def perform # rubocop:disable Metrics/AbcSize
+          return puts "Skipping tests." if skip_tests?
+
           raise ArgumentError unless [suite, release].all?
 
           @docker.login(**release.login_params) if release.login_params
@@ -25,23 +27,53 @@ module Gitlab
 
           name = "#{release.project_name}-qa-#{SecureRandom.hex(4)}"
 
-          @docker.run(release.qa_image, release.qa_tag, suite, *args) do |command|
-            command << "-t --rm --net=#{network || 'bridge'}"
+          feature_flag_sets = []
 
-            env.merge(Runtime::Env.variables).each do |key, value|
-              command.env(key, value)
-            end
+          # When `args` includes `[..., "--disable-feature", "a", "--enable-feature", "b", ...]`
+          # `feature_flag_sets` will be set to `[["--disable-feature", "a"], ["--enable-feature", "b"]]`
+          # This will result in tests running twice, once with each feature.
+          while (index = args&.index { |x| x =~ /--.*-feature/ })
+            feature_flag_sets << args.slice!(index, 2)
+          end
 
-            command.volume('/var/run/docker.sock', '/var/run/docker.sock')
-            command.volume(File.join(Runtime::Env.host_artifacts_dir, name), File.join(Docker::Volumes::QA_CONTAINER_WORKDIR, 'tmp'))
+          # When `args` do not have either "--disable-feature" or "--enable-feature", we
+          # add [] so that test is run exactly once.
+          feature_flag_sets << [] unless feature_flag_sets.any?
 
-            @volumes.to_h.each do |to, from|
-              command.volume(to, from)
-            end
+          feature_flag_sets.each do |feature_flag_set|
+            @docker.run(release.qa_image, release.qa_tag, suite, *args_with_flags(args, feature_flag_set)) do |command|
+              command << "-t --rm --net=#{network || 'bridge'}"
 
-            command.name(name)
+              env.merge(Runtime::Env.variables).each do |key, value|
+                command.env(key, value)
+              end
+
+              command.volume('/var/run/docker.sock', '/var/run/docker.sock')
+              command.volume(File.join(Runtime::Env.host_artifacts_dir, name), File.join(Docker::Volumes::QA_CONTAINER_WORKDIR, 'tmp'))
+
+              @volumes.to_h.each do |to, from|
+                command.volume(to, from)
+              end
+
+              command.name(name)
+            end
           end
         end
+
+        private
+
+        def args_with_flags(args, feature_flag_set)
+          return args if feature_flag_set.empty?
+
+          puts "Running with feature flag: #{feature_flag_set.join(' ')}"
+
+          args_with_f = args.dup
+          args_with_f.insert(1, *feature_flag_set)
+        end
+
+        def skip_tests?
+          Runtime::Scenario.attributes.include?(:run_tests) && !Runtime::Scenario.run_tests
+        end
       end
     end
   end