gitlab-qa 8.7.0 → 8.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a59adb2954b85b2adb1eddb1f0fbc568c01fa9d2d466a3a6c16fa4c5bb12e3b
-  data.tar.gz: c362d1e370e23f891e9a23a785ee88e48431fd466fd2aa1b38d2596f68ceeabf
+  metadata.gz: e89c4046779c498fad14fac05a4cc8ab68deb9d4ccb62e8eabaaace6af8be17d
+  data.tar.gz: a0c72ba4f7278b82f59678133578e11f23cb7e39449a82c21ec7beb967a524cc
 SHA512:
-  metadata.gz: 4de67f7a4bc50f6a28503ea6bfaf988f1ca18db321dba31006eac49de6cf5159c981862909ed57cef6f61126fa1d429843a912f59445ebab270e104dd5f0f382
-  data.tar.gz: 956a7115f74b6b563289feb1e75cfcb0d4901c2b1bef1fd8480508c7197c2e0934f0dd812a3a3ced498a430ccad3eeabb49751d2253cc02955e34db358cd83d9
+  metadata.gz: 7847ac077692217f68224ca9fdd31bcdadb833aa4aca0ab16113caf1efe6e1013689bca536d2b4c2b5fb3cd5d6e77a58f36b31e2b7c62d00f49abc66c2d0e683
+  data.tar.gz: 97e98ce591bb8a7cc2a804b1d1326605df8f68e0561cff27e235ddc48420be385794d8dd02ce635adafabd443b1e3f8ab9f4e4d194949b0fb91aedc06e243f32

data/.gitlab/ci/jobs/chaos.gitlab-ci.yml

@@ -0,0 +1,22 @@
+ee:chaos:
+  extends:
+    - .rules:ee-never-when-triggered-by-feature-flag-definition-change
+    - .test
+    - .high-capacity
+    - .ee-variables
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::Chaos"
+    QA_RSPEC_TAGS: "--tag chaos"
+
+ee:chaos-quarantine:
+  extends:
+    - .rules:ee-never-when-triggered-by-feature-flag-definition-change
+    - .test
+    - .high-capacity
+    - .ee-variables
+    - .quarantine
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::Chaos"
+    QA_RSPEC_TAGS: "--tag quarantine --tag chaos"

data/.gitlab/ci/jobs/sanity_framework.gitlab-ci.yml

@@ -4,7 +4,7 @@ ce:sanity-framework:
     QA_GENERATE_ALLURE_REPORT: "false"
     QA_EXPORT_TEST_METRICS: "false"
   script:
-    - ./bin/expect_exit_code_and_text "bundle exec exe/gitlab-qa Test::Instance::Image ${RELEASE:=CE} -- --tag framework" 1 "2 examples, 1 failure"
+    - ./bin/expect_exit_code_and_text "bundle exec exe/gitlab-qa Test::Instance::Image ${RELEASE:=CE} -- --tag framework" 1 "1 failure"
   extends:
     - .rules:ce-never-when-triggered-by-feature-flag-definition-change
     - .test
@@ -16,7 +16,7 @@ ee:sanity-framework:
     QA_GENERATE_ALLURE_REPORT: "false"
     QA_EXPORT_TEST_METRICS: "false"
   script:
-    - ./bin/expect_exit_code_and_text "bundle exec exe/gitlab-qa Test::Instance::Image ${RELEASE:=EE} -- --tag framework" 1 "2 examples, 1 failure"
+    - ./bin/expect_exit_code_and_text "bundle exec exe/gitlab-qa Test::Instance::Image ${RELEASE:=EE} -- --tag framework" 1 "1 failure"
   extends:
     - .rules:ee-never-when-triggered-by-feature-flag-definition-change
     - .test

data/Gemfile.lock

@@ -1,14 +1,15 @@
 PATH
   remote: .
   specs:
-    gitlab-qa (8.7.0)
+    gitlab-qa (8.8.0)
       activesupport (~> 6.1)
       gitlab (~> 4.18.0)
       http (~> 5.0)
       nokogiri (~> 1.10)
-      rainbow (~> 3.0.0)
+      rainbow (>= 3, < 4)
       table_print (= 1.5.7)
-      zeitwerk (~> 2.4)
+      toxiproxy (~> 2.0.2)
+      zeitwerk (>= 2, < 3)
 
 GEM
   remote: https://rubygems.org/
@@ -216,6 +217,7 @@ GEM
       unicode-display_width (>= 1.1.1, < 3)
     thor (1.2.1)
     tilt (2.0.11)
+    toxiproxy (2.0.2)
     tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
     unf (0.1.4)

data/bin/expect_exit_code_and_text

@@ -26,7 +26,7 @@ else
     if [ "$?" -eq "0" ]; then
       echo "'$SCRIPT' outputted '$EXPECTED_TEXT', as expected!"
     else
-      echo "'$SCRIPT' was expected to output '$EXPECTED_TEXT', but did not!"
+      echo "'$SCRIPT' was expected to match output '$EXPECTED_TEXT', but did not!"
       exit 1
     fi
   else

data/docs/what_tests_can_be_run.md

@@ -1035,6 +1035,33 @@ $ export GITLAB_PASSWORD="$GITLAB_QA_PASSWORD"
 
 $ gitlab-qa Test::Instance::Geo EE https://primary.gitlab.com https://secondary.gitlab.com
 ```
+
+### `Test::Instance::Chaos`
+
+This scenario will run a limited number of tests that are tagged with `:chaos`.
+
+These tests are designed to verify that our systems can gracefully handle scenarios which may occur if 
+networking or connectivity type issues occur in various Gitlab components. 
+They make use of [toxiproxy(https://github.com/Shopify/toxiproxy)] to act as a proxy allowing us to introduce 
+connectivity issues, via the [toxiproxy ruby client](https://github.com/Shopify/toxiproxy-ruby) in an E2E spec. 
+
+Sample Test:
+
+```ruby
+  context 'when a gitaly node is experiencing high latency' do
+    it 'can create a project' do
+      Toxiproxy[:gitaly1].toxic(:latency, latency: 30000).apply do
+        expect(create_a_project).to be true
+      end
+    end
+  end
+```
+
+Example:
+
+```
+$ gitlab-qa Test::Instance::Chaos
+```
 ----
 
 [Back to README.md](../README.md)

data/gitlab-qa.gemspec

@@ -32,7 +32,8 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'gitlab', '~> 4.18.0'
   spec.add_runtime_dependency 'http', '~> 5.0'
   spec.add_runtime_dependency 'nokogiri', '~> 1.10'
-  spec.add_runtime_dependency 'rainbow', '~> 3.0.0'
+  spec.add_runtime_dependency 'rainbow', '>= 3', '< 4'
   spec.add_runtime_dependency 'table_print', '1.5.7'
-  spec.add_runtime_dependency 'zeitwerk', '~> 2.4'
+  spec.add_runtime_dependency 'toxiproxy', '~> 2.0.2'
+  spec.add_runtime_dependency 'zeitwerk', '>= 2', '< 3'
 end

data/lib/gitlab/qa/component/chaos.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+require 'toxiproxy'
+
+module Gitlab
+  module QA
+    module Component
+      class Chaos < Base
+        DOCKER_IMAGE = 'ghcr.io/shopify/toxiproxy'
+        DOCKER_IMAGE_TAG = '2.5.0'
+
+        def initialize
+          super
+          @network = "test"
+        end
+
+        def name
+          @name ||= "chaos"
+        end
+
+        def start
+          prepare
+          docker.run(image: image, tag: tag) do |command|
+            command << '-d '
+            command << "--name #{name}"
+            command << "--network #{@network}"
+            command << "--publish 8474:8474"
+          end
+
+          Toxiproxy.host = "http://localhost:8474"
+
+          begin
+            attempts ||= 1
+            Toxiproxy.populate([
+                                 {
+                                   name: "postgres",
+                                   listen: "[::]:5432",
+                                   upstream: "postgres.test:5432",
+                                   "enabled": true
+                                 },
+                                 {
+                                   name: "praefect",
+                                   listen: "[::]:2305",
+                                   upstream: "praefect.test:2305",
+                                   "enabled": true
+                                 },
+                                 {
+                                   name: "gitaly1",
+                                   listen: "[::]:8076",
+                                   upstream: "gitaly1.test:8076",
+                                   "enabled": true
+                                 },
+                                 {
+                                   name: "gitaly2",
+                                   listen: "[::]:8077",
+                                   upstream: "gitaly2.test:8077",
+                                   "enabled": true
+                                 },
+                                 {
+                                   name: "gitaly3",
+                                   listen: "[::]:8078",
+                                   upstream: "gitaly3.test:8078",
+                                   "enabled": true
+                                 }
+                               ])
+          rescue StandardError => e
+            Runtime::Logger.warn e.message
+
+            # This should resolve within seconds, but give some leeway for the toxiproxy container to start
+            if (attempts += 1) < 10
+              Runtime::Logger.info "Waiting #{attempts} seconds before retrying setting toxiproxy config"
+              sleep attempts
+              retry
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/scenario/test/integration/chaos.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Scenario
+      module Test
+        module Integration
+          class Chaos < Praefect
+            def initialize
+              super
+
+              @praefect_addr = "chaos.test"
+              @database_addr = "chaos.test"
+              @primary_node_addr = "chaos.test"
+              @primary_node_port = 8076
+              @secondary_node_addr = "chaos.test"
+              @secondary_node_port = 8077
+              @tertiary_node_addr = "chaos.test"
+              @tertiary_node_port = 8078
+            end
+
+            def perform(release, *rspec_args)
+              @chaos_node = Component::Chaos.new.tap(&:start)
+
+              run_gitaly_cluster(release, rspec_args)
+            ensure
+              @chaos_node&.teardown
+              @praefect_node&.teardown
+              @sql_node&.teardown
+              @gitaly_primary_node&.teardown
+              @gitaly_secondary_node&.teardown
+              @gitaly_tertiary_node&.teardown
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb

@@ -8,13 +8,29 @@ module Gitlab
           class GitalyCluster < Scenario::Template
             attr_reader :gitlab_name, :spec_suite
 
-            def initialize
+            def initialize # rubocop:disable Metrics/AbcSize
               @gitlab_name = 'gitlab-gitaly-cluster'
+
               @primary_node_name = 'gitaly1'
+              @primary_node_addr = "#{@primary_node_name}.#{@network}"
+              @primary_node_port = 8075
+
               @secondary_node_name = 'gitaly2'
+              @secondary_node_addr = "#{@secondary_node_name}.#{@network}"
+              @secondary_node_port = 8075
+
               @tertiary_node_name = 'gitaly3'
+              @tertiary_node_addr = "#{@tertiary_node_name}.#{@network}"
+              @tertiary_node_port = 8075
+
               @praefect_node_name = 'praefect'
+              @praefect_port = 2305
+              @praefect_addr = "#{@praefect_node_name}.#{@network}"
+
               @database = 'postgres'
+              @database_port = 5432
+              @database_addr = "#{@database}.#{@network}"
+
               @spec_suite = 'Test::Instance::All'
               @network = 'test'
               @env = {}
@@ -22,6 +38,16 @@ module Gitlab
             end
 
             def perform(release, *rspec_args)
+              run_gitaly_cluster(release, rspec_args)
+            ensure
+              @praefect_node&.teardown
+              @sql_node&.teardown
+              @gitaly_primary_node&.teardown
+              @gitaly_secondary_node&.teardown
+              @gitaly_tertiary_node&.teardown
+            end
+
+            def run_gitaly_cluster(release, rspec_args) # rubocop:disable Metrics/AbcSize
               # The postgres container starts in seconds so not essential to parallelize it
               # This also ensure that the docker network is created here, avoiding any potential race conditions later
               #  if the gitaly-cluster and GitLab containers attempt to create a network in parallel
@@ -57,21 +83,15 @@ module Gitlab
                   end
                 end
               end
-            ensure
-              @praefect_node&.teardown
-              @sql_node&.teardown
-              @gitaly_primary_node&.teardown
-              @gitaly_secondary_node&.teardown
-              @gitaly_tertiary_node&.teardown
             end
 
             private
 
             def start_gitaly_cluster(release)
               Runtime::Logger.info("Starting Gitaly Cluster")
-              @gitaly_primary_node = gitaly(@primary_node_name, release)
-              @gitaly_secondary_node = gitaly(@secondary_node_name, release)
-              @gitaly_tertiary_node = gitaly(@tertiary_node_name, release)
+              @gitaly_primary_node = gitaly(@primary_node_name, @primary_node_port, release)
+              @gitaly_secondary_node = gitaly(@secondary_node_name, @secondary_node_port, release)
+              @gitaly_tertiary_node = gitaly(@tertiary_node_name, @tertiary_node_port, release)
 
               @praefect_node = praefect(@praefect_node_name, release)
               Runtime::Logger.info("Gitaly Cluster Ready")
@@ -97,31 +117,31 @@ module Gitlab
                 #{disable_other_services}
                 gitaly['enable'] = false;
                 praefect['enable'] = true;
-                praefect['listen_addr'] = '0.0.0.0:2305';
+                praefect['listen_addr'] = '0.0.0.0:#{@praefect_port}';
                 praefect['prometheus_listen_addr'] = '0.0.0.0:9652';
                 praefect['auth_token'] = 'PRAEFECT_EXTERNAL_TOKEN';
                 praefect['reconciliation_scheduling_interval'] = '10s';
-                praefect['database_host'] = '#{@database}.#{@network}';
+                praefect['database_host'] = '#{@database_addr}';
                 praefect['database_user'] = 'postgres';
-                praefect['database_port'] = 5432;
+                praefect['database_port'] = #{@database_port};
                 praefect['database_password'] = 'SQL_PASSWORD';
                 praefect['database_dbname'] = 'praefect_production';
                 praefect['database_sslmode'] = 'disable';
-                praefect['database_direct_host'] = '#{@database}.#{@network}';
-                praefect['database_direct_port'] = 5432;
+                praefect['database_direct_host'] = '#{@database_addr}';
+                praefect['database_direct_port'] = #{@database_port};
                 praefect['virtual_storages'] = {
                   'default' => {
                     'nodes' => {
                       '#{@primary_node_name}' => {
-                        'address' => 'tcp://#{@primary_node_name}.#{@network}:8075',
+                        'address' => 'tcp://#{@primary_node_addr}:#{@primary_node_port}',
                         'token'   => 'PRAEFECT_INTERNAL_TOKEN'
                       },
                       '#{@secondary_node_name}' => {
-                        'address' => 'tcp://#{@secondary_node_name}.#{@network}:8075',
+                        'address' => 'tcp://#{@secondary_node_addr}:#{@secondary_node_port}',
                         'token'   => 'PRAEFECT_INTERNAL_TOKEN'
                       },
                       '#{@tertiary_node_name}' => {
-                        'address' => 'tcp://#{@tertiary_node_name}.#{@network}:8075',
+                        'address' => 'tcp://#{@tertiary_node_addr}:#{@tertiary_node_port}',
                         'token'   => 'PRAEFECT_INTERNAL_TOKEN'
                       }
                     }
@@ -130,12 +150,12 @@ module Gitlab
               OMNIBUS
             end
 
-            def gitaly_omnibus_configuration
+            def gitaly_omnibus_configuration(listen_port)
               <<~OMNIBUS
                 #{disable_other_services.sub(/(prometheus\['enable'\]) = false/, '\1 = true')}
                 prometheus_monitoring['enable'] = false;
                 gitaly['enable'] = true;
-                gitaly['listen_addr'] = '0.0.0.0:8075';
+                gitaly['listen_addr'] = '0.0.0.0:#{listen_port}';
                 gitaly['prometheus_listen_addr'] = '0.0.0.0:9236';
                 gitaly['auth_token'] = 'PRAEFECT_INTERNAL_TOKEN';
                 gitlab_shell['secret_token'] = 'GITLAB_SHELL_SECRET_TOKEN';
@@ -160,7 +180,7 @@ module Gitlab
 
                 git_data_dirs({
                   'default' => {
-                    'gitaly_address' => 'tcp://#{@praefect_node_name}.#{@network}:2305',
+                    'gitaly_address' => 'tcp://#{@praefect_addr}:#{@praefect_port}',
                     'gitaly_token' => 'PRAEFECT_EXTERNAL_TOKEN'
                   }
                 });
@@ -171,7 +191,7 @@ module Gitlab
                     'job_name' => 'praefect',
                     'static_configs' => [
                       'targets' => [
-                        '#{@praefect_node_name}.#{@network}:9652'
+                        '#{@praefect_addr}:#{@praefect_port}'
                       ]
                     ]
                   },
@@ -215,14 +235,14 @@ module Gitlab
               end
             end
 
-            def gitaly(name, release)
+            def gitaly(name, port, release)
               Component::Gitlab.new.tap do |gitaly|
                 gitaly.release = QA::Release.new(release)
                 gitaly.name = name
                 gitaly.network = @network
                 gitaly.skip_availability_check = true
                 gitaly.seed_admin_token = false
-                gitaly.omnibus_configuration << gitaly_omnibus_configuration
+                gitaly.omnibus_configuration << gitaly_omnibus_configuration(port)
                 gitaly.instance(skip_teardown: true)
               end
             end

data/lib/gitlab/qa/scenario/test/integration/praefect.rb

@@ -21,7 +21,7 @@ module Gitlab
 
                 git_data_dirs({
                   'default' => {
-                    'gitaly_address' => 'tcp://#{@praefect_node_name}.#{@network}:2305',
+                    'gitaly_address' => 'tcp://#{@praefect_addr}:#{@praefect_port}',
                     'gitaly_token' => 'PRAEFECT_EXTERNAL_TOKEN'
                   },
                   'gitaly' => {
@@ -45,7 +45,7 @@ module Gitlab
                     'job_name' => 'praefect',
                     'static_configs' => [
                       'targets' => [
-                        '#{@praefect_node_name}.#{@network}:9652'
+                        '#{@praefect_addr}:9652'
                       ]
                     ]
                   },
@@ -53,9 +53,9 @@ module Gitlab
                     'job_name' => 'praefect-gitaly',
                     'static_configs' => [
                       'targets' => [
-                        '#{@primary_node_name}.#{@network}:9236',
-                        '#{@secondary_node_name}.#{@network}:9236',
-                        '#{@tertiary_node_name}.#{@network}:9236'
+                        '#{@primary_node_addr}:9236',
+                        '#{@secondary_node_addr}:9236',
+                        '#{@tertiary_node_addr}:9236'
                       ]
                     ]
                   }

data/lib/gitlab/qa/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   module QA
-    VERSION = '8.7.0'
+    VERSION = '8.8.0'
   end
 end

data/scripts/generate-qa-jobs.rb

@@ -47,6 +47,7 @@ class GenerateQAJobs
     jobs.merge!(load_yml_contents('large_setup', should_automatically_run?('test_instance_all_can_use_large_setup')))
     jobs.merge!(load_yml_contents('cloud_activation', should_automatically_run?('test_instance_all_cloud_activation')))
     jobs.merge!(load_yml_contents('registry_with_cdn', should_automatically_run?('test_integration_registrywithcdn')))
+    jobs.merge!(load_yml_contents('chaos', should_automatically_run?('test_integration_chaos')))
     jobs.merge!(load_yml_contents('staging'))
     jobs.merge!(load_yml_contents('metrics', should_automatically_run?('test_instance_all_metrics')))
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 8.7.0
+  version: 8.8.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-10-12 00:00:00.000000000 Z
+date: 2022-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -182,16 +182,22 @@ dependencies:
   name: rainbow
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3'
+    - - "<"
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.0
+        version: '3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: table_print
   requirement: !ruby/object:Gem::Requirement
@@ -207,19 +213,39 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.5.7
 - !ruby/object:Gem::Dependency
-  name: zeitwerk
+  name: toxiproxy
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.4'
+        version: 2.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.4'
+        version: 2.0.2
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
 description: 
 email:
 - gitlab-qa@gmail.com
@@ -233,6 +259,7 @@ files:
 - ".gitignore"
 - ".gitlab-ci.yml"
 - ".gitlab/ci/jobs/base.gitlab-ci.yml"
+- ".gitlab/ci/jobs/chaos.gitlab-ci.yml"
 - ".gitlab/ci/jobs/cloud_activation.gitlab-ci.yml"
 - ".gitlab/ci/jobs/custom_parallel.gitlab-ci.yml"
 - ".gitlab/ci/jobs/decomposition_multiple_db.gitlab-ci.yml"
@@ -311,6 +338,7 @@ files:
 - gitlab-qa.gemspec
 - lib/gitlab/qa.rb
 - lib/gitlab/qa/component/base.rb
+- lib/gitlab/qa/component/chaos.rb
 - lib/gitlab/qa/component/elasticsearch.rb
 - lib/gitlab/qa/component/gitlab.rb
 - lib/gitlab/qa/component/jira.rb
@@ -383,6 +411,7 @@ files:
 - lib/gitlab/qa/scenario/test/instance/staging_geo.rb
 - lib/gitlab/qa/scenario/test/instance/staging_ref.rb
 - lib/gitlab/qa/scenario/test/instance/staging_ref_geo.rb
+- lib/gitlab/qa/scenario/test/integration/chaos.rb
 - lib/gitlab/qa/scenario/test/integration/client_ssl.rb
 - lib/gitlab/qa/scenario/test/integration/elasticsearch.rb
 - lib/gitlab/qa/scenario/test/integration/geo.rb