gitlab-qa 7.21.0 → 7.23.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7df136e02dc7640dd456d9e6f0f089cfdf105c7e8d4983ee41e487081eadb3eb
-  data.tar.gz: 9ab31d33024c1a2e6826213028e4a1f1c3ab69c8ece4d230be79a4f3b1f15108
+  metadata.gz: 2f7604899a14fd7faaae2ee398b5b0769df8dde4badfd2123fa27993b1a0cbda
+  data.tar.gz: c3fea270e1583201349b1c73c10aab49a5db9c969714ab842ab21436155c8d96
 SHA512:
-  metadata.gz: 5420d4ffea08d120903f51fa9d590b20cfee1686ecdff384ac2ff931ba1ead170409a412e3c45cb96f6310359540d245624bf05fce654f106030550d68ff8928
-  data.tar.gz: 459ce3e23b1113bcb52b2413d6cae58dc9946976e7615833fd5fbe9d0b979d0894c964fc6565cb2a0e13e9953369c4073b6a5ec00510b24edec3cd4b2ad00a61
+  metadata.gz: 404e72b88f829f186a99bf2302fa92161b5267494b6ac3e91209c19882da58f348814394a5c23f02edfa3462715bcc4b4b5e136d2bd13c1e5bfa6c475f164c6c
+  data.tar.gz: f043de46cddd1b120fbd972224e4eef339b3b22acfd25e28e972e9407c93d38c65234d4c67049d517366b5bbf2b54177da5cc336615cdb1ee4af572c9ba9467c

data/.gitlab-ci.yml

@@ -95,7 +95,11 @@ rspec:
 .combined-gitlab-qa-options-script:
   script:
     - 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS_COMBINED -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
-    - bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS_COMBINED -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
+    - bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTIONS_COMBINED -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
+    - bundle exec exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
+    - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-results "gitlab-qa-run-*/**/rspec-*.json" --test-case-project "$QA_TESTCASES_REPORTING_PROJECT" --results-issue-project "$QA_TEST_RESULTS_ISSUES_PROJECT" || true; fi
+    - exit $test_run_exit_code
 
 # The Test::Omnibus::Update scenarios require the release to be specified twice, which can't be done dynamically using the `variables` parameter
 # So instead we can use this script with two release variables
@@ -1186,6 +1190,21 @@ ee:jira-quarantine:
   variables:
     QA_SCENARIO: "Test::Integration::Jira"
 
+ce:integrations:
+  extends:
+    - .rules:ce-never-when-triggered-by-feature-flag-definition-change
+    - .test
+    - .high-capacity
+    - .ce-variables
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::Integrations"
+
+ce:integrations-quarantine:
+  extends:
+    - ce:integrations
+    - .quarantine
+
 ce:large-setup:
   extends:
     - .rules:ce-never-when-triggered-by-feature-flag-definition-change

data/README.md

@@ -88,7 +88,6 @@ of tests.
 - [How it works](docs/how_it_works.md)
 - [Release process](docs/release_process.md)
 - [Run QA tests against your GDK setup](docs/run_qa_against_gdk.md)
-- [Running specific orchestrated tests](docs/running_specific_orchestrated_tests.md)
 - [Trainings](docs/trainings.md)
 - [Waits](docs/waits.md)
 - [What tests can be run?](docs/what_tests_can_be_run.md)

data/docs/what_tests_can_be_run.md

@@ -82,6 +82,7 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `QA_INFLUXDB_URL` |- | Influxdb url for test metrics reporting | No|
 | `QA_INFLUXDB_TOKEN` |- | Influxdb token for test metrics reporting | No|
 | `QA_RUN_TYPE` |- | QA run type like `staging-full`, `canary`, `production` etc. Used in test metrics reporting | No|
+| `QA_VALIDATE_RESOURCE_REUSE` | `false` | Set to `true` to [validate resource reuse](https://docs.gitlab.com/ee/development/testing_guide/end_to_end/resources.html#reusable-resources) after a test suite | No |
 | `GITHUB_USERNAME` |-  | Username for authenticating with GitHub. | No|
 | `GITHUB_PASSWORD` |-  | Password for authenticating with GitHub. | No|
 | `GITLAB_QA_LOOP_RUNNER_MINUTES` | `1` | Minutes to run and repeat a spec while using the '--loop' option; default value is 1 minute. | No|
@@ -95,6 +96,7 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `GITLAB_QA_USER_AGENT` |- | The browser user-agent to use instead of the default Chrome user-agent. | No|
 | `GEO_FAILOVER` | `false` | Set to `true` when a Geo secondary site has been promoted to a Geo primary site. | No|
 | `GITLAB_INITIAL_ROOT_PASSWORD` | `5iveL!fe` | Initial root password for Omnibus installations | No|
+| `COLORIZED_LOGS` | `false` | Colors GitLab QA and test logs to improve readability | No|
 
 ## [Supported Remote Grid environment variables](./running_against_remote_grid.md)
 
@@ -664,6 +666,20 @@ Example:
 $ gitlab-qa Test::Integration::Actioncable CE
 ```
 
+### `Test::Integration::Integrations CE|<full image address>`
+
+This scenario is intended to test the different integrations that a GitLab instance can offer, such as WebHooks to an external service, Jenkins, etc.
+
+To run tests against the GitLab container, a GitLab QA (`gitlab/gitlab-qa`)
+container is spun up and tests are run from it by running the
+`Test::Integration::Integrations` scenario.
+
+Example:
+
+```
+$ gitlab-qa Test::Integration::Integrations CE
+```
+
 ### `Test::Instance::Any CE|EE|<full image address>:nightly|latest|any_tag http://your.instance.gitlab`
 
 This tests that a live GitLab instance works as expected by running tests
@@ -877,12 +893,39 @@ $ gitlab-qa Test::Instance::Airgapped EE -- --tag smoke
 
 ### `Test::Instance::Image CE|EE|<full image address> --omnibus-config object_storage`
 
-This orchestrated scenario runs tests from the test suite against a GitLab instance which has object storage configured instead of using local storage. The omnibus configuration `object_storage` ([configurator](configuring_omnibus.md)), uses Minio and `object_storage_aws` uses an AWS S3 account with credentials configured in the pipeline as CI Variables (available on 1Password). Requires:
+This orchestrated scenario runs tests from the test suite against a GitLab instance which has object storage configured instead of using local storage. The omnibus configuration `object_storage` ([configurator](configuring_omnibus.md)), uses Minio and `object_storage_aws` uses an AWS S3 account with credentials configured in the pipeline as CI Variables. To use Google Cloud Storage pass `--omnibus-config object_storage_gcp`. According to the storage selected it requires:
+
+| Scenario              | Variable            | Description                          |
+|-----------------------|---------------------|--------------------------------------|
+| AWS S3 Object Storage | AWS_S3_REGION       | AWS region where bucket is created   |
+| AWS S3 Object Storage | AWS_S3_KEY_ID       | AWS credentials                      |
+| AWS S3 Object Storage | AWS_S3_ACCESS_KEY   | AWS credentials                      |
+| AWS S3 Object Storage | AWS_S3_BUCKET_NAME  | Name of the bucket set in AWS        |
+| GCP Object Storage    | GCS_BUCKET_NAME     | Name of the bucket set in AWS        |
+| GCP Object Storage    | GOOGLE_JSON_KEY     | JSON key credential                  |
+| GCP Object Storage    | GOOGLE_CLIENT_EMAIL | Email address of the service account |
+| GCP Object Storage    | GOOGLE_PROJECT      | GCP project name                     |
+
+These variables are available at 1Password QA Vault.
+
+### `Test::Integration::RegistryTLS EE`
+
+It requires GitLab's TLS certificate found on the [`tls_certificates`](https://gitlab.com/gitlab-org/gitlab-qa/-/blob/master/tls_certificates/gitlab/gitlab.test.crt) folder
+```
+$ export GITLAB_TLS_CERTIFICATE=$(cat /path/to/certificate.crt)
+```
+
+To run a scenario with an insecure registry enabled use `Test::Integration::Registry EE` instead (it does not require the above certificate).
+
+### Test::Integration::RegistryTLS EE --omnibus-config object_storage
+
+This scenario is a composition of two orchestrated scenarios. It tests the container registry (TLS-enabled) integrated with an object storage backend.
+
+ An example would be to pass the option `--omnibus-config object_storage_aws` and the registry will be pulling and pushing images using AWS Cloud Storage as a storage backend.
 
-* `AWS_S3_REGION` - Region where the bucket is created. To optimize latency, minimize costs, or address regulatory requirements, choose any AWS Region that is geographically close to you.
-* `AWS_S3_KEY_ID` - Key ID for the user accessing the bucket.
-* `AWS_S3_ACCESS_KEY` - Access Key for the user accessing the bucket.
-* `AWS_S3_BUCKET_NAME` - The name of the bucket where objects are stored. After a bucket is created, the name of that bucket cannot be used by another AWS account in any AWS Region until the bucket is deleted.
+ ```
+ GITLAB_TLS_CERTIFICATE=$(cat /path/to/certificate.crt) gitlab-qa Test::Integration::RegistryTLS EE --omnibus-config object_storage_aws
+ ```
 
 ### `Test::Instance::Geo EE|<full image address>:nightly|latest|any_tag http://geo-primary.gitlab http://geo-secondary.gitlab`
 

data/gitlab-qa.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'climate_control', '~> 0.2'
+  spec.add_development_dependency 'climate_control', '~> 1.0.1'
   spec.add_development_dependency 'gitlab-styles', '~> 6.2.1'
   spec.add_development_dependency 'pry', '~> 0.11'
   spec.add_development_dependency 'rake', '~> 13.0'

data/lib/gitlab/qa/component/minio.rb

@@ -9,7 +9,7 @@ module Gitlab
     module Component
       class Minio < Base
         DOCKER_IMAGE = 'minio/minio'.freeze
-        DOCKER_IMAGE_TAG = 'RELEASE.2021-09-15T04-54-25Z'.freeze
+        DOCKER_IMAGE_TAG = 'latest'.freeze
         # These are secrets used in a local Minio container, they're not used for any online S3 server.
         AWS_ACCESS_KEY = 'AKIAIOSFODNN7EXAMPLE'.freeze
         AWS_SECRET_KEY = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'.freeze

data/lib/gitlab/qa/component/telegraf.rb

@@ -157,8 +157,8 @@ module Gitlab
               gather_services = false
               container_names = []
               source_tag = false
-              container_name_include = ["gitlab-{ce,ee}-*", "gitlab-{ce,ee}-qa-*"]
-              container_name_exclude = []
+              container_name_include = []
+              container_name_exclude = ["#{name}"]
               timeout = "5s"
               perdevice = false
               perdevice_include = []

data/lib/gitlab/qa/report/generate_test_session.rb

@@ -25,7 +25,7 @@ module Gitlab
           end
 
           issue = gitlab.create_issue(
-            title: "#{Time.now.strftime('%Y-%m-%d')} Test session report | #{pipeline}",
+            title: "#{Time.now.strftime('%Y-%m-%d')} Test session report | #{Runtime::Env.qa_run_type}",
             description: generate_description(tests),
             labels: ['Quality', 'QA', 'triage report', pipeline_name_label]
           )
@@ -55,6 +55,8 @@ module Gitlab
           #{generate_stages_listing(tests)}
 
           #{generate_qa_issue_relation}
+
+          #{generate_link_to_dashboard}
           MARKDOWN
         end
 
@@ -257,6 +259,16 @@ module Gitlab
           /relate #{Runtime::Env.qa_issue_url}
           MARKDOWN
         end
+
+        def generate_link_to_dashboard
+          return unless Runtime::Env.qa_run_type
+
+          <<~MARKDOWN.chomp
+          ## Link to Grafana dashboard for run-type of #{Runtime::Env.qa_run_type}
+
+          * https://dashboards.quality.gitlab.net/d/kuNYMgDnz/test-run-metrics?orgId=1&refresh=1m&var-run_type=#{Runtime::Env.qa_run_type}
+          MARKDOWN
+        end
       end
     end
   end

data/lib/gitlab/qa/report/results_in_testcases.rb

@@ -70,8 +70,8 @@ module Gitlab
           <<~MKDOWN.strip
             ### Executions
 
-            package-and-qa:
-            <img src="https://dashboards.quality.gitlab.net/render/d-solo/cW0UMgv7k/spec-health?orgId=1&var-run_type=package-and-qa&var-name=#{formatted_title}&panelId=4&width=1000&height=500" />
+            All Environments:
+            <img src="https://dashboards.quality.gitlab.net/render/d-solo/cW0UMgv7k/spec-health?orgId=1&var-run_type=All&var-name=#{formatted_title}&panelId=4&width=1000&height=500" />
           MKDOWN
         end
 

data/lib/gitlab/qa/runtime/env.rb

@@ -42,6 +42,7 @@ module Gitlab
           'QA_INFLUXDB_TOKEN' => :qa_influxdb_token,
           'QA_RUN_TYPE' => :qa_run_type,
           'QA_SKIP_PULL' => :qa_skip_pull,
+          'QA_VALIDATE_RESOURCE_REUSE' => :qa_validate_resource_reuse,
           'GITLAB_API_BASE' => :api_base,
           'GITLAB_ADMIN_USERNAME' => :admin_username,
           'GITLAB_ADMIN_PASSWORD' => :admin_password,
@@ -306,6 +307,23 @@ module Gitlab
           enabled?(env_var_value_if_defined('QA_EXPORT_TEST_METRICS'), default: true)
         end
 
+        def qa_run_type
+          return env_var_value_if_defined('QA_RUN_TYPE') if env_var_value_valid?('QA_RUN_TYPE')
+
+          live_envs = %w[staging staging-canary staging-ref canary preprod production]
+          return unless live_envs.include?(ci_project_name)
+
+          test_subset = if env_var_value_if_defined('NO_ADMIN') == 'true'
+                          'sanity-no-admin'
+                        elsif env_var_value_if_defined('SMOKE_ONLY') == 'true'
+                          'sanity'
+                        else
+                          'full'
+                        end
+
+          "#{ci_project_name}-#{test_subset}"
+        end
+
         private
 
         def enabled?(value, default: true)

data/lib/gitlab/qa/scenario/test/integration/integrations.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Scenario
+      module Test
+        module Integration
+          class Integrations < Scenario::Template
+            def perform(release, *rspec_args)
+              Component::Gitlab.perform do |gitlab|
+                gitlab.release = QA::Release.new(release)
+                gitlab.network = 'test'
+                gitlab.name = 'gitlab-integrations'
+                gitlab.instance do
+                  Component::Specs.perform do |specs|
+                    rspec_args << '--' unless rspec_args.include?('--')
+                    rspec_args << %w[--tag integrations]
+                    specs.suite = 'Test::Instance::All'
+                    specs.hostname = "qa-e2e-specs.#{gitlab.network}"
+                    specs.release = gitlab.release
+                    specs.network = gitlab.network
+                    specs.args = [gitlab.address, *rspec_args]
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   module QA
-    VERSION = '7.21.0'
+    VERSION = '7.23.0'
   end
 end

data/lib/gitlab/qa.rb

@@ -74,6 +74,7 @@ module Gitlab
           autoload :Registry, 'gitlab/qa/scenario/test/integration/registry'
           autoload :RegistryTLS, 'gitlab/qa/scenario/test/integration/registry_tls'
           autoload :ServicePingDisabled, 'gitlab/qa/scenario/test/integration/service_ping_disabled'
+          autoload :Integrations, 'gitlab/qa/scenario/test/integration/integrations'
         end
 
         module Sanity

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 7.21.0
+  version: 7.23.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-01-28 00:00:00.000000000 Z
+date: 2022-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: 1.0.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: gitlab-styles
   requirement: !ruby/object:Gem::Requirement
@@ -228,7 +228,6 @@ files:
 - docs/release_process.md
 - docs/run_qa_against_gdk.md
 - docs/running_against_remote_grid.md
-- docs/running_specific_orchestrated_tests.md
 - docs/trainings.md
 - docs/waits.md
 - docs/what_tests_can_be_run.md
@@ -316,6 +315,7 @@ files:
 - lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb
 - lib/gitlab/qa/scenario/test/integration/group_saml.rb
 - lib/gitlab/qa/scenario/test/integration/instance_saml.rb
+- lib/gitlab/qa/scenario/test/integration/integrations.rb
 - lib/gitlab/qa/scenario/test/integration/jira.rb
 - lib/gitlab/qa/scenario/test/integration/kubernetes.rb
 - lib/gitlab/qa/scenario/test/integration/ldap.rb

data/docs/running_specific_orchestrated_tests.md

@@ -1,11 +0,0 @@
-# Running specific orchestrated tests
-
-## Maven artifact spec
-
-The [maven repository spec](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/specs/features/ee/browser_ui/5_package/maven_repository_spec.rb) creates a Maven artifact and links it to a GitLab project. The artifact is created within a [Maven docker image](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/service/docker_run/maven.rb#L8).
-
-Using `gitlab-qa` to run these tests reduces the likelihood of network errors between the maven container and GitLab instance.
-
-To run this with `gitlab-qa` you can use the `Test::Instance::Image` that is needed for your test. For example:
-
-`gitlab-qa Test::Instance::Image registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:2cb9a85d2beadd51b926eaddb05005403bee0013 -- --tag orchestrated qa/specs/features/ee/browser_ui/5_package/maven_repository_spec.rb` - runs the test against a specific Omnibus GitLab image that was built through `package-and-qa`.