gitlab-qa 5.13.6 → 5.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 742350ecaafcf21ca9ff0fc50547cbb6cd28b2cf7a3fc090f5cb6ab542da3154
-  data.tar.gz: e65a23e2620961c5751eca131cf0dfa4b639f2915488b888975e6a0313c8baee
+  metadata.gz: bc79855be61afd8dfaef793cb1a6520bde1f07f6c44a18b9fdd605e26ac947c4
+  data.tar.gz: d326f64befb861020097fd55b9772b54d7014509e87c5c13c71e619c11a21a65
 SHA512:
-  metadata.gz: e42ac20b6a9dd4881c50d5018956241543395dae6c2aac148c1db87e77daa0ac677950a876ffa730e9d8f991c3850640775213ca6a0c8ebefa4a45dd2ab811c2
-  data.tar.gz: b7f993786ae4ac8fdc7b15d4d936cad11066e2f92731b30f696e8615a9454657f40d67e3cccb9074b6464755ab2d37a2e74569e92b66476328ee8215ea4543e3
+  metadata.gz: e497734ed2d4d9d58c2dcdf051eb3a4a6ad02adcfc7954ae54ac83ee912b47ff0d6f8478ea6e272a8a93feb14cdda8d55632fb0f92e97379596a12b67ee25f57
+  data.tar.gz: fc927589ca591708a7b61341acf69d48125812642ce9ca87f0183cf11bcf14c55d6bfa8bb04fe1cf3c4685e1d4da0781bab6af15a942c0675ae149fac8b20704

data/.gitlab-ci.yml

@@ -31,6 +31,8 @@ workflow:
     - if: '$CI_COMMIT_BRANCH == "master"'
     # For tags, create a pipeline.
     - if: '$CI_COMMIT_TAG'
+    # For triggers from GitLab MR pipelines (and pipelines from other projects), create a pipeline
+    - if: '$CI_PIPELINE_SOURCE == "pipeline"'
 
 .default-rules:
   rules:
@@ -830,6 +832,44 @@ ee:praefect-quarantine:
     QA_CAN_TEST_PRAEFECT: "true"
     QA_RSPEC_TAGS: "--tag quarantine --tag ~orchestrated"
 
+ce:gitaly-cluster:
+  extends:
+    - .test
+    - .high-capacity
+    - .ce-qa
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::GitalyCluster"
+
+ce:gitaly-cluster-quarantine:
+  extends:
+    - .test
+    - .high-capacity
+    - .ce-qa
+    - .quarantine
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::GitalyCluster"
+
+ee:gitaly-cluster:
+  extends:
+    - .test
+    - .high-capacity
+    - .ee-qa
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::GitalyCluster"
+
+ee:gitaly-cluster-quarantine:
+  extends:
+    - .test
+    - .high-capacity
+    - .ee-qa
+    - .quarantine
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::GitalyCluster"
+
 ce:smtp:
   extends:
     - .test
@@ -891,7 +931,8 @@ ee:jira-quarantine:
 staging:
   script:
     - unset EE_LICENSE
-    - exe/gitlab-qa Test::Instance::Staging
+    - 'echo "Running: exe/gitlab-qa Test::Instance::Staging ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS"'
+    - exe/gitlab-qa Test::Instance::Staging ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS
   extends:
     - .test
     - .high-capacity

data/README.md

@@ -121,11 +121,58 @@ All the scenarios you can run are described in the
 
 Note: The GitLab QA tool requires that [Docker](https://docs.docker.com/install/) is installed.
 
-### How to add new scenarios
-
-Scenarios (test cases) and scripts to run them are located in the
-[CE](https://gitlab.com/gitlab-org/gitlab-ce/tree/master/qa) and
-[EE](https://gitlab.com/gitlab-org/gitlab-ee/tree/master/qa)
+### Command-line options
+
+In addition to the [arguments you can use to specify the scenario and
+tests to run](/docs/what_tests_can_be_run.md), you can use the
+following options to control the tool's behavior.
+
+**Note:** These are `gitlab-qa` options so if you specify RSpec
+options as well, including test file paths, be sure to add these
+options before the `--` that indicates that subsequent arguments are
+intended for RSpec.
+
+| Option | Description |
+| ------ | ----------- |
+| `--no-teardown` | Skip teardown of containers after the scenario completes |
+| `--no-tests` | Orchestrates the docker containers but does not run the tests. Implies `--no-teardown` |
+
+For example, the following command would start an EE GitLab Docker
+container and would leave the instance running, but would not run the
+tests:
+
+```plaintext
+$ gitlab-qa Test::Instance::Image EE --no-tests
+```
+
+GitLab QA will have automatically run the `docker ps` command to show
+the port that container is running on, for example:
+
+```plaintext
+...
+Skipping tests.
+The orchestrated docker containers have not been removed.
+Docker shell command: `docker ps`
+CONTAINER ID  IMAGE                     ... PORTS
+fdeffd791b69  gitlab/gitlab-ee:nightly      22/tcp, 443/tcp, 0.0.0.0:32768->80/tcp
+```
+
+You could then run tests against that instance in a similar way to
+[running tests against GDK](/docs/run_qa_against_gdk.md). This can be
+useful if you want to run and debug a specific test, for example:
+
+```plaintext
+# From /path/to/gdk/gitlab/qa
+$ bundle exec bin/qa Test::Instance::All http://localhost:32768 -- qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb
+```
+
+### How to add new tests
+
+Please see the [Beginner's guide to writing end-to-end tests](https://docs.gitlab.com/ee/development/testing_guide/end_to_end/beginners_guide.html).
+
+Test cases and scripts to run them are located in the
+[GitLab FOSS](https://gitlab.com/gitlab-org/gitlab-foss/tree/master/qa) and
+[GitLab](https://gitlab.com/gitlab-org/gitlab/tree/master/qa)
 repositories under the `qa/` directory, so please also check the documentation
 there.
 

data/docs/run_qa_against_gdk.md

@@ -42,6 +42,9 @@ make a few changes to your `gdk/gitlab/config/gitlab.yml` file.
   $ exe/gitlab-qa Test::Instance::Any gitlab/gitlab-ce:your-custom-tag http://192.168.0.12:3000 -- qa/specs/features/browser_ui/1_manage/login/log_in_spec.rb
   ```
   
+**Note:** The hostname of the URL provided to `gitlab-qa` must match the hostname configured for GDK.
+If they do not match, a test will be signed out when it visits a page directly because the hostname of the URL visited will be different from the hostname that was used when signing in.
+  
 ### Running EE tests
 
 When running EE tests you'll need to have a license available. GitLab engineers can [request a license](https://about.gitlab.com/handbook/developer-onboarding/#working-on-gitlab-ee).

data/docs/what_tests_can_be_run.md

@@ -57,19 +57,19 @@ For more details on the internals, please read the
 | `QA_ARTIFACTS_DIR` |`/tmp/gitlab-qa`| Path to a directory where artifacts (logs and screenshots) for failing tests will be saved. | No|
 | `DOCKER_HOST` |`http://localhost`| Docker host to run tests against. | No|
 | `CHROME_HEADLESS` |-  | When running locally, set to `false` to allow Chrome tests to be visible - watch your tests being run. | No|
-| `QA_ADDITIONAL_REPOSITORY_STORAGE` |-  | The name of additional, non-default storage to be used with tests tagged `repository_storage`, run via the `Test::Instance::RepositoryStorage` scenario. | No|
-| `QA_PRAEFECT_REPOSITORY_STORAGE` |-  | The name of repository storage using Praefect. | No|
+| `QA_ADDITIONAL_REPOSITORY_STORAGE` |-  | The name of additional, non-default storage to be used with tests tagged `repository_storage`, run via the `Test::Instance::RepositoryStorage` scenario.  Note: Admin access is required to change repository storage. | No|
+| `QA_PRAEFECT_REPOSITORY_STORAGE` |-  | The name of repository storage using Praefect. Note: Admin access is required to change repository storage. | No|
 | `QA_COOKIES` |-  | Optionally set to "cookie1=value;cookie2=value" in order to add a cookie to every request. This can be used to set the canary cookie by setting it to "gitlab_canary=true". | No|
 | `QA_DEBUG` |-  | Set to `true` to verbosely log page object actions. Note: if enabled be aware that sensitive data might be logged. If an input element has a QA selector with `password` in the name, data entered into the input element will be masked. If the element doesn't have `password` in its name it won't be masked. | No|
 | `QA_LOG_PATH` |-  | Path to output debug logging to. If not set logging will be output to STDOUT. | No|
-| `QA_CAN_TEST_GIT_PROTOCOL_V2` |-  | Set to `false` to skip tests that require Git protocol v2 if your environment doesn't support it. | No|
-| `QA_CAN_TEST_ADMIN_FEATURES` |-  | Set to `false` to skip tests that require admin access. | No|
-| `QA_CAN_TEST_PRAEFECT` |-  | Set to `false` to skip tests that require Praefect to be running. | No|
+| `QA_CAN_TEST_GIT_PROTOCOL_V2` | `true` | Set to `false` to skip tests that require Git protocol v2 if your environment doesn't support it. | No|
+| `QA_CAN_TEST_ADMIN_FEATURES` | `true` | Set to `false` to skip tests that require admin access. | No|
+| `QA_CAN_TEST_PRAEFECT` | `true` | Set to `false` to skip tests that require Praefect to be running. | No|
 | `QA_DISABLE_RSPEC_RETRY` |-  | Set to `true` to turn off retrying tests on failure. | No|
 | `QA_SIMULATE_SLOW_CONNECTION` |-  | Set to `true` to configure Chrome's network settings to simulate a slow connection. | No|
 | `QA_SLOW_CONNECTION_LATENCY_MS` | `2000` | The additional latency (in ms) of the simulated slow connection. | No|
 | `QA_SLOW_CONNECTION_THROUGHPUT_KBPS` | `32` | The maximum throughput (in kbps) of the simulated slow connection. | No|
-| `QA_SKIP_PULL` |-  | Set to `true` to skip pulling docker images (e.g., to use one you built locally). | No|
+| `QA_SKIP_PULL` | `false` | Set to `true` to skip pulling docker images (e.g., to use one you built locally). | No|
 | `GITHUB_USERNAME` |-  | Username for authenticating with GitHub. | No|
 | `GITHUB_PASSWORD` |-  | Password for authenticating with GitHub. | No|
 | `GITLAB_QA_LOOP_RUNNER_MINUTES` | `1` | Minutes to run and repeat a spec while using the '--loop' option; default value is 1 minute. | No|
@@ -618,6 +618,21 @@ $ export QA_ADDITIONAL_REPOSITORY_STORAGE=secondary
 $ gitlab-qa Test::Instance::RepositoryStorage
 ```
 
+### `Test::Instance::Airgapped`
+
+This scenario will run tests from the test suite against an airgapped instance.
+The airgapped instance is set up by using `iptables` in the GitLab container to block network traffic other than testable ports, and by using runners
+in a shared internal network.
+
+Example:
+
+```
+# For EE
+$ export EE_LICENSE=$(cat /path/to/gitlab_license)
+
+$ gitlab-qa Test::Instance::Airgapped EE -- --tag smoke
+```
+
 ----
 
 [Back to README.md](../README.md)

data/lib/gitlab/qa.rb

@@ -6,6 +6,7 @@ module Gitlab
 
     module Runtime
       autoload :Env, 'gitlab/qa/runtime/env'
+      autoload :Scenario, 'gitlab/qa/runtime/scenario'
       autoload :TokenFinder, 'gitlab/qa/runtime/token_finder'
     end
 
@@ -28,6 +29,7 @@ module Gitlab
           autoload :Release, 'gitlab/qa/scenario/test/instance/release'
           autoload :Geo, 'gitlab/qa/scenario/test/instance/geo'
           autoload :StagingGeo, 'gitlab/qa/scenario/test/instance/staging_geo'
+          autoload :Airgapped, 'gitlab/qa/scenario/test/instance/airgapped'
         end
 
         module Omnibus
@@ -52,7 +54,7 @@ module Gitlab
           autoload :Praefect, 'gitlab/qa/scenario/test/integration/praefect'
           autoload :Elasticsearch, 'gitlab/qa/scenario/test/integration/elasticsearch'
           autoload :SMTP, 'gitlab/qa/scenario/test/integration/smtp'
-          autoload :GitalyHA, 'gitlab/qa/scenario/test/integration/gitaly_ha'
+          autoload :GitalyCluster, 'gitlab/qa/scenario/test/integration/gitaly_cluster'
           autoload :Jira, 'gitlab/qa/scenario/test/integration/jira'
         end
 
@@ -63,6 +65,7 @@ module Gitlab
     end
 
     module Component
+      autoload :Base, 'gitlab/qa/component/base'
       autoload :Gitlab, 'gitlab/qa/component/gitlab'
       autoload :InternetTunnel, 'gitlab/qa/component/internet_tunnel'
       autoload :LDAP, 'gitlab/qa/component/ldap'

data/lib/gitlab/qa/component/base.rb

@@ -0,0 +1,156 @@
+module Gitlab
+  module QA
+    module Component
+      class Base
+        include Scenario::Actable
+
+        attr_reader :docker
+        attr_accessor :volumes, :network, :environment, :runner_network
+        attr_writer :name, :exec_commands
+
+        def initialize
+          @docker = Docker::Engine.new
+          @environment = {}
+          @volumes = {}
+          @network_aliases = []
+
+          self.exec_commands = []
+        end
+
+        def add_network_alias(name)
+          @network_aliases.push(name)
+        end
+
+        def name
+          raise NotImplementedError, "#{self.class.name} must specify a default name"
+        end
+
+        def hostname
+          "#{name}.#{network}"
+        end
+
+        def image
+          return self.class.const_get('DOCKER_IMAGE') if self.class.const_defined?('DOCKER_IMAGE')
+
+          raise NotImplementedError, "#{self.class.name} must specify a docker image as DOCKER_IMAGE"
+        end
+
+        def tag
+          return self.class.const_get('DOCKER_IMAGE_TAG') if self.class.const_defined?('DOCKER_IMAGE_TAG')
+
+          raise NotImplementedError, "#{self.class.name} must specify a docker image tag as DOCKER_IMAGE_TAG"
+        end
+
+        def instance
+          instance_no_teardown do
+            yield self if block_given?
+          end
+        ensure
+          teardown
+        end
+
+        alias_method :launch_and_teardown_instance, :instance
+
+        def instance_no_teardown
+          prepare
+          start
+          reconfigure
+          wait_until_ready
+          process_exec_commands
+
+          yield self if block_given?
+        end
+
+        def prepare
+          prepare_docker_image
+          prepare_network
+        end
+
+        def prepare_docker_image
+          pull
+        end
+
+        def prepare_network
+          if runner_network && !docker.network_exists?(runner_network)
+            docker.network_create("--driver=bridge --internal #{runner_network}")
+          end
+
+          return if docker.network_exists?(network)
+
+          docker.network_create(network)
+        end
+
+        def start # rubocop:disable Metrics/AbcSize
+          docker.run(image, tag) do |command|
+            command << "-d"
+            command << "--name #{name}"
+            command << "--net #{network}"
+            command << "--hostname #{hostname}"
+
+            @volumes.to_h.each do |to, from|
+              command.volume(to, from, 'Z')
+            end
+
+            command.volume(File.join(Runtime::Env.host_artifacts_dir, name, 'logs'), '/var/log/gitlab', 'Z')
+
+            @environment.to_h.each do |key, value|
+              command.env(key, value)
+            end
+
+            @network_aliases.to_a.each do |network_alias|
+              command << "--network-alias #{network_alias}"
+            end
+          end
+        end
+
+        def restart
+          assert_name!
+
+          docker.restart(name)
+        end
+
+        def teardown
+          unless teardown?
+            puts "The orchestrated docker containers have not been removed."
+            docker.ps
+
+            return
+          end
+
+          teardown!
+        end
+
+        def teardown!
+          assert_name!
+
+          return unless docker.running?(name)
+
+          docker.stop(name)
+          docker.remove(name)
+        end
+
+        def pull
+          return if Runtime::Env.skip_pull?
+
+          docker.pull(image, tag)
+        end
+
+        def process_exec_commands
+          exec_commands.each { |command| docker.exec(name, command) }
+        end
+
+        private
+
+        attr_reader :exec_commands, :wait_until_ready, :reconfigure
+
+        def assert_name!
+          raise 'Invalid instance name!' unless name
+        end
+
+        def teardown?
+          !Runtime::Scenario.attributes.include?(:teardown) || Runtime::Scenario.teardown
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/component/elasticsearch.rb

@@ -1,41 +1,19 @@
 module Gitlab
   module QA
     module Component
-      class Elasticsearch
-        include Scenario::Actable
-
-        ELASTIC_IMAGE = 'docker.elastic.co/elasticsearch/elasticsearch'.freeze
-
-        attr_reader :docker
-        attr_accessor :environment, :network
-        attr_writer :name
-
-        def initialize
-          @docker = Docker::Engine.new
-          @environment = {}
-        end
+      class Elasticsearch < Base
+        DOCKER_IMAGE = 'docker.elastic.co/elasticsearch/elasticsearch'.freeze
 
         def name
           @name ||= "elastic68"
         end
 
-        def instance
-          prepare
-          start
-          yield self
-        ensure
-          teardown
-        end
-
-        def prepare
-          @docker.pull(ELASTIC_IMAGE, Runtime::Env.elastic_version)
-          return if @docker.network_exists?(network)
-
-          @docker.network_create(network)
+        def tag
+          Runtime::Env.elastic_version
         end
 
         def start
-          @docker.run(ELASTIC_IMAGE, Runtime::Env.elastic_version) do |command|
+          @docker.run(image, tag) do |command|
             command << "-d"
             command << "--name #{name}"
             command << "--net #{network}"
@@ -45,11 +23,6 @@ module Gitlab
             command.env("discovery.type", "single-node")
           end
         end
-
-        def teardown
-          @docker.stop(name)
-          @docker.remove(name)
-        end
       end
     end
   end

data/lib/gitlab/qa/component/gitlab.rb

@@ -7,13 +7,12 @@ require 'openssl'
 module Gitlab
   module QA
     module Component
-      class Gitlab
+      class Gitlab < Base
         extend Forwardable
-        include Scenario::Actable
 
-        attr_reader :release, :docker
-        attr_accessor :volumes, :network, :environment, :tls, :disable_animations
-        attr_writer :name, :relative_path, :exec_commands, :skip_check
+        attr_reader :release
+        attr_accessor :tls, :disable_animations, :skip_availability_check, :runner_network
+        attr_writer :name, :relative_path
 
         def_delegators :release, :tag, :image, :edition
 
@@ -21,17 +20,14 @@ module Gitlab
         SSL_PATH = '/etc/gitlab/ssl'.freeze
 
         def initialize
-          @docker = Docker::Engine.new
-          @environment = {}
-          @volumes = {}
-          @network_aliases = []
+          super
+
           @disable_animations = true
-          @skip_check = false
+          @skip_availability_check = false
 
           @volumes[CERTIFICATES_PATH] = SSL_PATH
 
           self.release = 'CE'
-          self.exec_commands = []
         end
 
         def omnibus_config=(config)
@@ -48,10 +44,6 @@ module Gitlab
           @environment['ELASTIC_URL'] = url
         end
 
-        def add_network_alias(name)
-          @network_aliases.push(name)
-        end
-
         def release=(release)
           @release = QA::Release.new(release)
         end
@@ -72,10 +64,6 @@ module Gitlab
           tls ? '443' : '80'
         end
 
-        def hostname
-          "#{name}.#{network}"
-        end
-
         def relative_path
           @relative_path ||= ''
         end
@@ -84,24 +72,16 @@ module Gitlab
           Runtime::Env.accept_insecure_certs = 'true'
         end
 
-        def instance
-          prepare
-          start
-          reconfigure
-          wait unless @skip_check
-          process_exec_commands
+        def prepare
+          prepare_gitlab_omnibus_config
 
-          yield self if block_given?
-        ensure
-          teardown
+          super
         end
 
-        alias_method :launch_and_teardown_instance, :instance
+        def pull
+          docker.login(**release.login_params) if release.login_params
 
-        def prepare
-          prepare_gitlab_omnibus_config
-          prepare_docker_image
-          prepare_network
+          super
         end
 
         def prepare_gitlab_omnibus_config
@@ -109,18 +89,6 @@ module Gitlab
           set_formless_login_token
         end
 
-        def prepare_docker_image
-          return if Runtime::Env.skip_pull?
-
-          @docker.pull(image, tag)
-        end
-
-        def prepare_network
-          return if @docker.network_exists?(network)
-
-          @docker.network_create(network)
-        end
-
         def setup_disable_animations
           @environment['GITLAB_OMNIBUS_CONFIG'] = "gitlab_rails['gitlab_disable_animations'] = true; #{@environment['GITLAB_OMNIBUS_CONFIG'] || ''}"
         end
@@ -128,28 +96,6 @@ module Gitlab
         def start # rubocop:disable Metrics/AbcSize
           ensure_configured!
 
-          if release.dev_gitlab_org?
-            Docker::Command.execute(
-              [
-                'login',
-                '--username gitlab-qa-bot',
-                %(--password "#{Runtime::Env.dev_access_token_variable}"),
-                QA::Release::DEV_REGISTRY
-              ]
-            )
-          elsif release.omnibus_mirror?
-            bot_token = ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
-
-            Docker::Command.execute(
-              [
-                'login',
-                '--username gitlab-bot',
-                %(--password "#{bot_token}"),
-                QA::Release::COM_REGISTRY
-              ]
-            )
-          end
-
           docker.run(image, tag) do |command|
             command << "-d -p #{port}"
             command << "--name #{name}"
@@ -170,6 +116,7 @@ module Gitlab
               command << "--network-alias #{network_alias}"
             end
           end
+          Docker::Command.execute("network connect --alias #{name}.#{network} --alias #{name}.#{runner_network} #{runner_network} #{name}") if runner_network
         end
 
         def reconfigure
@@ -181,18 +128,9 @@ module Gitlab
           end
         end
 
-        def restart
-          @docker.restart(name)
-        end
-
-        def teardown
-          raise 'Invalid instance name!' unless name
-
-          @docker.stop(name)
-          @docker.remove(name)
-        end
-
         def wait
+          return if skip_availability_check
+
           if Availability.new(name, relative_path: relative_path, scheme: scheme, protocol_port: port.to_i).check(180)
             sleep 12 # TODO, handle that better
             puts ' -> GitLab is available.'
@@ -201,10 +139,6 @@ module Gitlab
           end
         end
 
-        def pull
-          @docker.pull(@release.image, @release.tag)
-        end
-
         def sha_version
           json = @docker.read_file(
             @release.image, @release.tag,
@@ -215,14 +149,8 @@ module Gitlab
           manifest['software']['gitlab-rails']['locked_version']
         end
 
-        def process_exec_commands
-          exec_commands.each { |command| @docker.exec(name, command) }
-        end
-
         private
 
-        attr_reader :exec_commands
-
         def ensure_configured!
           raise 'Please configure an instance first!' unless [name, release, network].all?
         end