gitlab-qa 8.13.1 → 8.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eff4fd2e6e9b9ab05a26c87b04de43cc2724ff6bbfcdfa55309f6ea4dd6ca22f
-  data.tar.gz: f7c23ec439a8d5bbbcffac5942f9af3bd11761df457d6bd5316dcc2accb14d2e
+  metadata.gz: b7fcc8c586127f3c1883786b303610fd2360293a4e2ee2727225a0f50a865947
+  data.tar.gz: 7928dcd357359e99bc3e8c921bf3cda138d1cf5dee9746760a097c34c92be323
 SHA512:
-  metadata.gz: a796e6144671878b4940c6cd529abe4bff1f2b38fe06914c069c4291932d27c2b7527603a625d5a6631f42131e87d48bd63773b7abc62a55628f006863f3acab
-  data.tar.gz: eab90710e729b81e2ba9716462327bf770289c17c197f7743b79fb88023c6711e84751f1bb02fd62414f2ff1ec327b0e318eac060fba28f3ebaade42ee0b5847
+  metadata.gz: 65877f46a7c942ada94490ffcbdcdc74f757cc51000879e4f5030030fcbd0fb59e5be9f8ebd1550d0effe332cad61ff043eb7f8434aaddc72b6094d269048850
+  data.tar.gz: 837d053e553827cd507ba10e12056ae8bb8dbe32b22903f89c5eb67e59f57230a561897e34eb97f6ba67b5b67b8d66562a773c5fc487525ca1a0c83a006656d9

data/.gitlab/ci/jobs/airgapped.gitlab-ci.yml

@@ -0,0 +1,23 @@
+ce:airgapped:
+  extends:
+    - .rules:ce-never-when-triggered-by-feature-flag-definition-change
+    - .test
+    - .high-capacity
+    - .ce-variables
+    - .rspec-report-opts
+  parallel: 10
+  variables:
+    QA_SCENARIO: "Test::Instance::Airgapped"
+    QA_RSPEC_TAGS: "--tag '~github' --tag '~skip_live_env'"
+
+ee:airgapped:
+  extends:
+    - .rules:ee-never-when-triggered-by-feature-flag-definition-change
+    - .test
+    - .high-capacity
+    - .ee-variables
+    - .rspec-report-opts
+  parallel: 10
+  variables:
+    QA_SCENARIO: "Test::Instance::Airgapped"
+    QA_RSPEC_TAGS: "--tag '~github' --tag '~skip_live_env'"

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-qa (8.13.1)
+    gitlab-qa (8.14.1)
       activesupport (~> 6.1)
       gitlab (~> 4.18.0)
       http (~> 5.0)

data/docs/what_tests_can_be_run.md

@@ -95,12 +95,13 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `JIRA_ADMIN_PASSWORD` |-  | Password for authenticating with Jira server as admin. | No|
 | `CACHE_NAMESPACE_NAME` | `true` | Cache namespace name for groups. | No|
 | `DEPLOY_VERSION` |- | The version of GitLab being tested against. | No|
-| `GITLAB_QA_USER_AGENT` |- | The browser user-agent to use instead of the default Chrome user-agent. | No|
+| `GITLAB_QA_USER_AGENT` |- | The browser user-agent to use instead of the default Chrome user-agent. When set to the appropriate value (stored in 1Password), this allows tests to bypass certain login challenges (e.g., reCAPTCHA and ArkoseLabs). | No|
 | `GEO_FAILOVER` | `false` | Set to `true` when a Geo secondary site has been promoted to a Geo primary site. | No|
 | `GITLAB_INITIAL_ROOT_PASSWORD` | `5iveL!fe` | Initial root password for Omnibus installations | No|
 | `COLORIZED_LOGS` | `false` | Colors GitLab QA and test logs to improve readability | No|
 | `QA_DOCKER_ADD_HOSTS` |- | Comma separated list of hosts to add to /etc/hosts in docker container | No|
 | `FIPS` |- | Set to `1` or `true` to indicate that the test is running under FIPS mode | No|
+| `JH_ENV` | `false` | Set to `true` to indicate tests or scenarios are running under JH env | No|
 
 ## [Supported Remote Grid environment variables](./running_against_remote_grid.md)
 

data/lib/gitlab/qa/component/base.rb

@@ -7,7 +7,7 @@ module Gitlab
         include Scenario::Actable
 
         attr_reader :docker
-        attr_accessor :volumes, :ports, :network, :environment, :runner_network
+        attr_accessor :volumes, :ports, :network, :environment, :runner_network, :airgapped_network
         attr_writer :name, :exec_commands
 
         def initialize
@@ -72,15 +72,22 @@ module Gitlab
         end
 
         def prepare_network
-          if runner_network && !docker.network_exists?(runner_network) # rubocop:disable Style/IfUnlessModifier
-            docker.network_create("--driver=bridge --internal #{runner_network}")
-          end
+          prepare_airgapped_network
+          prepare_runner_network
 
           return if docker.network_exists?(network)
 
           docker.network_create(network)
         end
 
+        def prepare_airgapped_network
+          docker.network_create("--driver=bridge --internal #{network}") if airgapped_network && !docker.network_exists?(network)
+        end
+
+        def prepare_runner_network
+          docker.network_create("--driver=bridge --internal #{runner_network}") if runner_network && !docker.network_exists?(runner_network)
+        end
+
         def prepare_docker_container
           return unless docker.container_exists?(name)
 

data/lib/gitlab/qa/component/gitaly.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Component
+      class Gitaly < Base
+        extend Forwardable
+        using Rainbow
+        attr_reader :release
+        attr_accessor :gitaly_port, :gitlab_name, :cluster_config
+        attr_writer :name
+
+        def_delegators :release, :tag, :image, :edition
+
+        def initialize
+          super
+          self.release = 'EE'
+          @cluster_config = Component::GitalyCluster::GitalyClusterConfig.new
+          @gitaly_port = 8075
+          @ports = [gitaly_port]
+        end
+
+        def name
+          @name || "gitaly-#{SecureRandom.hex(4)}"
+        end
+
+        def release=(release)
+          @release = QA::Release.new(release)
+        end
+
+        def reconfigure
+          setup_omnibus
+          @docker.attach(name) do |line, wait|
+            # TODO, workaround which allows to detach from the container
+            break if /gitlab Reconfigured!/.match?(line)
+          end
+        end
+
+        def setup_omnibus
+          @docker.write_files(name) do |f|
+            f.write('/etc/gitlab/gitlab.rb', gitaly_omnibus_configuration)
+          end
+        end
+
+        def process_exec_commands
+          exec_commands << Support::ConfigScripts.add_git_server_hooks(docker, name)
+
+          commands = exec_commands.flatten.uniq
+          return if commands.empty?
+
+          Runtime::Logger.info("Running exec_commands...")
+          commands.each { |command| docker.exec(name, command) }
+        end
+
+        def gitaly_omnibus_configuration
+          <<~OMNIBUS
+            #{GitalyCluster.disable_other_omnibus_services}
+            praefect['enable'] = false;
+            prometheus['enable'] = true;
+            gitaly['enable'] = true;
+            gitaly['listen_addr'] = '0.0.0.0:#{gitaly_port}';
+            gitaly['prometheus_listen_addr'] = '0.0.0.0:9236';
+            gitaly['auth_token'] = 'PRAEFECT_INTERNAL_TOKEN';
+            gitlab_shell['secret_token'] = 'GITLAB_SHELL_SECRET_TOKEN';
+            gitlab_rails['internal_api_url'] = 'http://#{cluster_config.gitlab_name}.#{cluster_config.network}';
+            git_data_dirs({
+              '#{cluster_config.primary_node_name}' => {
+                'path' => '/var/opt/gitlab/git-data'
+              },
+              '#{cluster_config.secondary_node_name}' => {
+                'path' => '/var/opt/gitlab/git-data'
+              },
+              '#{cluster_config.tertiary_node_name}' => {
+                'path' => '/var/opt/gitlab/git-data'
+              }
+            });
+          OMNIBUS
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/component/gitaly_cluster.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Component
+      class GitalyCluster
+        class GitalyClusterConfig
+          attr_accessor :gitlab_name, :network, :airgapped_network,
+                        :praefect_node_name, :praefect_port, :praefect_ip,
+                        :primary_node_name, :primary_node_port,
+                        :secondary_node_name, :secondary_node_port,
+                        :tertiary_node_name, :tertiary_node_port,
+                        :database_node_name, :database_port
+
+          attr_reader :praefect_addr, :primary_node_addr, :secondary_node_addr, :tertiary_node_addr, :database_node_addr
+
+          def initialize(params = {}) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
+            @gitlab_name = params[:gitlab_name] || 'gitlab-gitaly-cluster'
+            @network = params[:network] || 'test'
+            @airgapped_network = params[:airgapped_network] || false
+
+            @praefect_node_name = params[:praefect_node_name] || 'praefect'
+            @praefect_port = params[:praefect_port] || 2305
+
+            @primary_node_name = params[:primary_node_name] || 'gitaly1'
+            @primary_node_port = params[:primary_node_port] || 8075
+
+            @secondary_node_name = params[:secondary_node_name] || 'gitaly2'
+            @secondary_node_port = params[:secondary_node_port] || 8075
+
+            @tertiary_node_name = params[:tertiary_node_name] || 'gitaly3'
+            @tertiary_node_port = params[:tertiary_node_port] || 8075
+
+            @database_node_name = params[:database_node_name] || 'postgres'
+            @database_port = params[:database_port] || 5432
+
+            @praefect_addr = "#{praefect_node_name}.#{network}"
+            @primary_node_addr = "#{primary_node_name}.#{network}"
+            @secondary_node_addr = "#{secondary_node_name}.#{network}"
+            @tertiary_node_addr = "#{tertiary_node_name}.#{network}"
+            @database_node_addr = "#{database_node_name}.#{network}"
+          end
+        end
+
+        include Scenario::Actable
+        using Rainbow
+
+        attr_accessor :release, :exec_commands, :gitlab_name, :config
+        attr_reader :gitaly_primary_node, :gitaly_secondary_node, :gitaly_tertiary_node, :praefect_node, :database_node
+
+        def initialize(config = GitalyClusterConfig.new)
+          @spec_suite = 'Test::Instance::All'
+          @env = {}
+          @tag = 'gitaly_cluster'
+          @release = 'EE'
+          @config = config
+        end
+
+        # @param [Boolean] parallel_gitaly controls whether we start gitaly nodes in parallel to improve startup time
+        def instance(parallel_gitaly = false)
+          run_gitaly_cluster(QA::Release.new(release), parallel_gitaly)
+        end
+
+        # @param [Boolean] parallel_gitaly controls whether we start gitaly nodes in parallel to improve startup time
+        def run_gitaly_cluster(release, parallel_gitaly = false)
+          # This also ensure that the docker network is created here, avoiding any potential race conditions later
+          #  if the gitaly-cluster and GitLab containers attempt to create a network in parallel
+          @database_node = postgres
+
+          Thread.new do
+            Thread.current.abort_on_exception = true
+            start_gitaly_cluster(release, parallel_gitaly)
+          end
+        end
+
+        # @param [Boolean] parallel_gitaly controls whether we start gitaly nodes in parallel to improve startup time
+        def start_gitaly_cluster(release, parallel_gitaly = false) # rubocop:disable Metrics/AbcSize
+          Runtime::Logger.info("Starting Gitaly Cluster")
+
+          if parallel_gitaly
+            threads = []
+            threads << Thread.new { @gitaly_primary_node = gitaly(config.primary_node_name, config.primary_node_port, release) }
+            threads << Thread.new { @gitaly_secondary_node = gitaly(config.secondary_node_name, config.secondary_node_port, release) }
+            threads << Thread.new { @gitaly_tertiary_node = gitaly(config.tertiary_node_name, config.tertiary_node_port, release) }
+            threads.each(&:join)
+          else
+            @gitaly_primary_node = gitaly(config.primary_node_name, config.primary_node_port, release)
+            @gitaly_secondary_node = gitaly(config.secondary_node_name, config.secondary_node_port, release)
+            @gitaly_tertiary_node = gitaly(config.tertiary_node_name, config.tertiary_node_port, release)
+          end
+
+          @praefect_node = praefect(release)
+          config.praefect_ip = praefect_node.ip_address
+          Runtime::Logger.info("Gitaly Cluster Ready")
+        end
+
+        def postgres
+          Component::PostgreSQL.new.tap do |sql|
+            sql.name = config.database_node_name
+            sql.airgapped_network = config.airgapped_network
+            sql.network = config.network
+            sql.instance(skip_teardown: true) do
+              sql.run_psql '-d template1 -c "CREATE DATABASE praefect_production OWNER postgres"'
+            end
+          end
+        end
+
+        def gitaly(gitaly_name, port, release) # rubocop:disable Metrics/AbcSize
+          Component::Gitaly.new.tap do |gitaly|
+            gitaly.cluster_config = config
+            gitaly.release = release
+            gitaly.name = gitaly_name
+            gitaly.gitaly_port = port
+            gitaly.airgapped_network = config.airgapped_network
+            gitaly.network = config.network
+            gitaly.gitlab_name = config.gitlab_name
+            gitaly.instance(skip_teardown: true)
+          end
+        end
+
+        def praefect(release)
+          Component::Praefect.new.tap do |praefect|
+            praefect.cluster_config = config
+            praefect.name = config.praefect_node_name
+            praefect.airgapped_network = config.airgapped_network
+            praefect.network = config.network
+            praefect.release = release
+            praefect.instance(skip_teardown: true)
+          end
+        end
+
+        # Helper configuration for omnibus config to disable all non GitalyCluster related omnibus services
+        def self.disable_other_omnibus_services
+          <<~OMNIBUS
+            postgresql['enable'] = false;
+            redis['enable'] = false;
+            nginx['enable'] = false;
+            grafana['enable'] = false;
+            puma['enable'] = false;
+            sidekiq['enable'] = false;
+            gitlab_workhorse['enable'] = false;
+            gitlab_rails['rake_cache_clear'] = false;
+            gitlab_rails['auto_migrate'] = false;
+            gitlab_exporter['enable'] = false;
+          OMNIBUS
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/component/gitlab.rb

@@ -202,7 +202,7 @@ module Gitlab
           exec_commands << seed_admin_token_command if seed_admin_token
           exec_commands << seed_test_data_command if seed_db
           exec_commands << Runtime::Scenario.omnibus_exec_commands
-          exec_commands << add_git_server_hooks unless skip_server_hooks
+          exec_commands << Support::ConfigScripts.add_git_server_hooks(docker, name) unless skip_server_hooks
 
           commands = exec_commands.flatten.uniq
           return if commands.empty?
@@ -280,28 +280,6 @@ module Gitlab
           ["gitlab-rails runner #{DATA_PATH}/admin_access_token_seed.rb"]
         end
 
-        def add_git_server_hooks
-          global_server_prereceive_hook = <<~SCRIPT
-            #!/usr/bin/env bash
-
-            if [[ \\\$GL_PROJECT_PATH =~ 'reject-prereceive' ]]; then
-              echo 'GL-HOOK-ERR: Custom error message rejecting prereceive hook for projects with GL_PROJECT_PATH matching pattern reject-prereceive'
-              exit 1
-            fi
-          SCRIPT
-
-          [
-            @docker.exec(name, 'mkdir -p /opt/gitlab/embedded/service/gitlab-shell/hooks/pre-receive.d'),
-            @docker.write_files(name) do |f|
-              f.write(
-                '/opt/gitlab/embedded/service/gitlab-shell/hooks/pre-receive.d/pre-receive.d',
-                global_server_prereceive_hook, false
-              )
-            end,
-            @docker.exec(name, 'chmod +x /opt/gitlab/embedded/service/gitlab-shell/hooks/pre-receive.d/*')
-          ]
-        end
-
         class Availability
           def initialize(name, relative_path: '', scheme: 'http', protocol_port: 80)
             @docker = Docker::Engine.new

data/lib/gitlab/qa/component/praefect.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Component
+      class Praefect < Base
+        extend Forwardable
+        using Rainbow
+        attr_reader :release
+        attr_accessor :cluster_config
+        attr_writer :name
+
+        def_delegators :release, :tag, :image, :edition
+
+        def initialize
+          super
+          self.release = 'EE'
+          @cluster_config = Component::GitalyCluster::GitalyClusterConfig.new
+          @ports = [cluster_config.praefect_port]
+        end
+
+        def name
+          @name || "praefect-#{SecureRandom.hex(4)}"
+        end
+
+        def release=(release)
+          @release = QA::Release.new(release)
+        end
+
+        def reconfigure
+          setup_omnibus
+          @docker.attach(name) do |line|
+            # TODO, workaround which allows to detach from the container
+            break if /gitlab Reconfigured!/.match?(line)
+          end
+        end
+
+        def setup_omnibus
+          @docker.write_files(name) do |f|
+            f.write('/etc/gitlab/gitlab.rb', praefect_omnibus_configuration)
+          end
+        end
+
+        def wait_until_ready
+          @docker.exec(name, 'praefect -config /var/opt/gitlab/praefect/cluster_config.toml check || true') do |resp|
+            Runtime::Logger.info(resp)
+            break if /All checks passed/.match?(line)
+          end
+        end
+
+        def praefect_omnibus_configuration # rubocop:disable Metrics/AbcSize
+          <<~OMNIBUS
+              #{GitalyCluster.disable_other_omnibus_services}
+              gitaly['enable'] = false;
+              prometheus['enable'] = true;
+              praefect['enable'] = true;
+              praefect['listen_addr'] = '0.0.0.0:#{cluster_config.praefect_port}';
+              praefect['prometheus_listen_addr'] = '0.0.0.0:9652';
+              praefect['auth_token'] = 'PRAEFECT_EXTERNAL_TOKEN';
+              praefect['reconciliation_scheduling_interval'] = '10s';
+              praefect['database_host'] = '#{cluster_config.database_node_addr}';
+              praefect['database_user'] = 'postgres';
+              praefect['database_port'] = #{cluster_config.database_port};
+              praefect['database_password'] = 'SQL_PASSWORD';
+              praefect['database_dbname'] = 'praefect_production';
+              praefect['database_sslmode'] = 'disable';
+              praefect['database_direct_host'] = '#{cluster_config.database_node_addr}';
+              praefect['database_direct_port'] = #{cluster_config.database_port};
+              praefect['virtual_storages'] = {
+                'default' => {
+                  'nodes' => {
+                    '#{cluster_config.primary_node_name}' => {
+                      'address' => 'tcp://#{cluster_config.primary_node_addr}:#{cluster_config.primary_node_port}',
+                      'token'   => 'PRAEFECT_INTERNAL_TOKEN'
+                    },
+                    '#{cluster_config.secondary_node_name}' => {
+                      'address' => 'tcp://#{cluster_config.secondary_node_addr}:#{cluster_config.secondary_node_port}',
+                      'token'   => 'PRAEFECT_INTERNAL_TOKEN'
+                    },
+                    '#{cluster_config.tertiary_node_name}' => {
+                      'address' => 'tcp://#{cluster_config.tertiary_node_addr}:#{cluster_config.tertiary_node_port}',
+                      'token'   => 'PRAEFECT_INTERNAL_TOKEN'
+                    }
+                  }
+                }
+              };
+          OMNIBUS
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/release.rb

@@ -7,7 +7,7 @@ module Gitlab
     class Release
       CANONICAL_REGEX = /
         \A
-          (?<edition>ce|ee)
+          (?<edition>ce|ee|jh)
           (-qa)?
           (:(?<tag>.+))?
         \z
@@ -18,7 +18,7 @@ module Gitlab
             (?<registry>[^/:]+(:(?<port>\d+))?)
             (?<project>.+)
             gitlab-
-            (?<edition>ce|ee)
+            (?<edition>ce|ee|jh)
           )
           (-qa)?
           (:(?<tag>.+))?
@@ -37,7 +37,7 @@ module Gitlab
       # version
       DEV_OFFICIAL_TAG_REGEX = /
         \A
-          (?<version>\d+\.\d+.\d+(?:-rc\d+)?)-(?<edition>ce|ee)
+          (?<version>\d+\.\d+.\d+(?:-rc\d+)?)-(?<edition>ce|ee|jh)
         \z
       /xi.freeze
 
@@ -57,8 +57,8 @@ module Gitlab
 
       DEFAULT_TAG = 'latest'
       DEFAULT_CANONICAL_TAG = 'nightly'
-      DEV_REGISTRY = 'dev.gitlab.org:5005'
-      COM_REGISTRY = 'registry.gitlab.com'
+      DEV_REGISTRY = Gitlab::QA::Runtime::Env.qa_dev_registry
+      COM_REGISTRY = Gitlab::QA::Runtime::Env.qa_com_registry
 
       InvalidImageNameError = Class.new(RuntimeError)
 

data/lib/gitlab/qa/runtime/env.rb

@@ -138,7 +138,8 @@ module Gitlab
           'NO_ADMIN' => :no_admin,
           'CHROME_DISABLE_DEV_SHM' => :chrome_disable_dev_shm,
           'COLORIZED_LOGS' => :colorized_logs,
-          'FIPS' => :fips
+          'FIPS' => :fips,
+          'JH_ENV' => :jh_env
         }.freeze
 
         ENV_VARIABLES.each do |env_name, method_name|
@@ -357,6 +358,18 @@ module Gitlab
           (env_var_value_if_defined('QA_DOCKER_ADD_HOSTS') || '').split(',')
         end
 
+        def jh_env?
+          enabled?(env_var_value_if_defined('JH_ENV'), default: false)
+        end
+
+        def qa_dev_registry
+          env_var_value_if_defined('QA_DEV_REGISTRY') || 'dev.gitlab.org:5005'
+        end
+
+        def qa_com_registry
+          env_var_value_if_defined('QA_COM_REGISTRY') || 'registry.gitlab.com'
+        end
+
         private
 
         def enabled?(value, default: true)

data/lib/gitlab/qa/scenario/test/instance/airgapped.rb

@@ -7,15 +7,61 @@ module Gitlab
         module Instance
           class Airgapped < Scenario::Template
             require 'resolv'
-            attr_accessor :commands
+            attr_reader :config, :gitlab_air_gap_commands, :iptables_restricted_network, :airgapped_network_name
 
             def initialize
-              gitlab_ip = Resolv.getaddress('registry.gitlab.com')
+              # Uses https://docs.docker.com/engine/reference/commandline/network_create/#network-internal-mode
+              @airgapped_network_name = 'airgapped'
+              # Uses iptables to deny all network traffic, with a number of exceptions for required ports and IPs
+              @iptables_restricted_network = 'test'
+              @config = Component::GitalyCluster::GitalyClusterConfig.new(
+                gitlab_name: "gitlab-airgapped-#{SecureRandom.hex(4)}",
+                airgapped_network: true,
+                network: airgapped_network_name
+              )
+            end
+
+            def perform(release, *rspec_args)
+              Component::Gitlab.perform do |gitlab|
+                Component::GitalyCluster.perform do |cluster|
+                  cluster.config = @config
+                  # we need to get an IP for praefect before proceeding so it cannot be run in parallel with gitlab
+                  cluster.instance(true).join
+                end
+                gitlab.name = config.gitlab_name
+                gitlab.release = release
+                gitlab.network = iptables_restricted_network # we use iptables to restrict access on the gitlab instance
+                gitlab.runner_network = config.network
+                gitlab.exec_commands = airgap_gitlab_commands
+                gitlab.skip_availability_check = true
+                gitlab.omnibus_configuration << gitlab_omnibus_configuration
+                rspec_args << "--" unless rspec_args.include?('--')
+                rspec_args << %w[--tag ~orchestrated]
+                gitlab.instance do
+                  Component::Specs.perform do |specs|
+                    specs.suite = 'Test::Instance::Airgapped'
+                    specs.release = gitlab.release
+                    specs.network = gitlab.network
+                    specs.runner_network = gitlab.runner_network
+                    specs.args = [gitlab.address, *rspec_args]
+                  end
+                end
+              end
+            end
+
+            private
+
+            def airgap_gitlab_commands
+              gitlab_ip = Resolv.getaddress('gitlab.com')
+              gitlab_registry_ip = Resolv.getaddress(QA::Release::COM_REGISTRY)
+              dev_gitlab_registry_ip = Resolv.getaddress(QA::Release::DEV_REGISTRY.split(':')[0])
+              praefect_ip = config.praefect_ip
               @commands = <<~AIRGAP_AND_VERIFY_COMMAND.split(/\n+/)
                 # Should not fail before airgapping due to eg. DNS failure
                 # Ping and wget check
-                apt-get update && apt-get install -y iptables nmap
-                nmap -sT #{gitlab_ip} -p 80 && (echo \"Regular connectivity nmap check passed.\" && exit 0) || (echo \"Regular connectivity nmap check failed.\" && exit 1)
+                apt-get update && apt-get install -y iptables ncat
+                if ncat -zv -w 10 #{gitlab_ip} 80; then echo 'Airgapped connectivity check passed.'; else echo 'Airgapped connectivity check failed - should be able to access gitlab_ip'; exit 1; fi;
+
                 echo "Checking regular connectivity..." \
                   && wget --retry-connrefused --waitretry=1 --read-timeout=15 --timeout=10 -t 2 http://registry.gitlab.com > /dev/null 2>&1 \
                   && (echo "Regular connectivity wget check passed." && exit 0) || (echo "Regular connectivity wget check failed." && exit 1)
@@ -33,34 +79,59 @@ module Gitlab
                 iptables -A OUTPUT -p tcp -m tcp --sport 80 -m state --state NEW,ESTABLISHED -j ACCEPT
                 iptables -A INPUT -p tcp -m tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT
 
-                # Should now fail to ping and wget, port 80 should be open
-                nmap -sT #{gitlab_ip} -p 80 && (echo \"Airgapped network faulty. Connectivity nmap check failed.\" && exit 1) || (echo \"Connectivity nmap check passed.\" && exit 0)
-                nmap -sT  127.0.0.1 -p 22 && (echo "Airgapped connectivity port 22 check passed." && exit 0) || (echo "Airgapped connectivity port 22 check failed." && exit 1)
-                nmap -sT 10 127.0.0.1 -p 80 && (echo "Airgapped connectivity port 80 check passed." && exit 0) || (echo "Airgapped connectivity port 80 check failed." && exit 1)
+                # some exceptions to allow runners access network https://gitlab.com/gitlab-org/gitlab-qa/-/issues/700 
+                iptables -A OUTPUT -p tcp -d  #{gitlab_registry_ip} -j ACCEPT
+                iptables -A OUTPUT -p tcp -d  #{dev_gitlab_registry_ip} -j ACCEPT
+                # allow access to praefect node 
+                iptables -A OUTPUT -p tcp -d #{praefect_ip} -j ACCEPT
+
+                # Should now fail to ping gitlab_ip, port 22/80 should be open
+                if ncat -zv -w 10 #{gitlab_ip} 80; then echo 'Airgapped connectivity check failed - should not be able to access gitlab_ip'; exit 1; else echo 'Airgapped connectivity check passed.'; fi;
+                if ncat -zv -w 10 127.0.0.1 22; then echo 'Airgapped connectivity port 22 check passed.'; else echo 'Airgapped connectivity port 22 check failed.'; exit 1;  fi;
+                if ncat -zv -w 10 127.0.0.1 80; then echo 'Airgapped connectivity port 80 check passed.'; else echo 'Airgapped connectivity port 80 check failed.'; exit 1 ; fi;
+                if ncat -zv -w 10 #{gitlab_registry_ip} 80; then echo 'Airgapped connectivity port gitlab_registry_ip check passed.'; else echo 'Airgapped connectivity port 80 check failed.'; exit 1; fi;
+
                 echo "Checking airgapped connectivity..." \
                   && wget --retry-connrefused --waitretry=1 --read-timeout=15 --timeout=10 -t 2 http://registry.gitlab.com > /dev/null 2>&1 \
                   && (echo "Airgapped network faulty. Connectivity wget check failed." && exit 1) || (echo "Airgapped network confirmed. Connectivity wget check passed." && exit 0)
               AIRGAP_AND_VERIFY_COMMAND
             end
 
-            def perform(release, *rspec_args)
-              Component::Gitlab.perform do |gitlab|
-                gitlab.release = release
-                gitlab.network = 'test'
-                gitlab.runner_network = 'airgapped'
-                gitlab.exec_commands = @commands
-                rspec_args << "--" unless rspec_args.include?('--')
-                rspec_args << %w[--tag ~orchestrated]
-                gitlab.instance do
-                  Component::Specs.perform do |specs|
-                    specs.suite = 'Test::Instance::Airgapped'
-                    specs.release = gitlab.release
-                    specs.network = gitlab.network
-                    specs.runner_network = gitlab.runner_network
-                    specs.args = [gitlab.address, *rspec_args]
-                  end
-                end
-              end
+            def gitlab_omnibus_configuration
+              <<~OMNIBUS
+                external_url 'http://#{config.gitlab_name}.#{iptables_restricted_network}';
+
+                git_data_dirs({
+                  'default' => {
+                    'gitaly_address' => 'tcp://#{config.praefect_addr}:#{config.praefect_port}',
+                    'gitaly_token' => 'PRAEFECT_EXTERNAL_TOKEN'
+                  }
+                });
+                gitaly['enable'] = false;
+                gitlab_shell['secret_token'] = 'GITLAB_SHELL_SECRET_TOKEN';
+                prometheus['scrape_configs'] = [
+                  {
+                    'job_name' => 'praefect',
+                    'static_configs' => [
+                      'targets' => [
+                        '#{config.praefect_addr}:9652'
+                      ]
+                    ]
+                  },
+                  {
+                    'job_name' => 'praefect-gitaly',
+                    'static_configs' => [
+                      'targets' => [
+                        '#{config.primary_node_addr}:9236',
+                        '#{config.secondary_node_addr}:9236',
+                        '#{config.tertiary_node_addr}:9236'
+                      ]
+                    ]
+                  }
+                ];
+                grafana['disable_login_form'] = false;
+                grafana['admin_password'] = 'GRAFANA_ADMIN_PASSWORD';
+              OMNIBUS
             end
           end
         end

data/lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb

@@ -6,67 +6,24 @@ module Gitlab
       module Test
         module Integration
           class GitalyCluster < Scenario::Template
-            attr_reader :gitlab_name, :spec_suite
-
-            def initialize # rubocop:disable Metrics/AbcSize
-              @gitlab_name = 'gitlab-gitaly-cluster'
-
-              @primary_node_name = 'gitaly1'
-              @primary_node_addr = "#{@primary_node_name}.#{@network}"
-              @primary_node_port = 8075
-
-              @secondary_node_name = 'gitaly2'
-              @secondary_node_addr = "#{@secondary_node_name}.#{@network}"
-              @secondary_node_port = 8075
-
-              @tertiary_node_name = 'gitaly3'
-              @tertiary_node_addr = "#{@tertiary_node_name}.#{@network}"
-              @tertiary_node_port = 8075
-
-              @praefect_node_name = 'praefect'
-              @praefect_port = 2305
-              @praefect_addr = "#{@praefect_node_name}.#{@network}"
-
-              @database = 'postgres'
-              @database_port = 5432
-              @database_addr = "#{@database}.#{@network}"
+            attr_reader :gitlab_name, :spec_suite, :name, :config
 
+            def initialize
               @spec_suite = 'Test::Instance::All'
-              @network = 'test'
               @env = {}
               @tag = 'gitaly_cluster'
+              @config = Component::GitalyCluster::GitalyClusterConfig.new
             end
 
             def perform(release, *rspec_args)
-              run_gitaly_cluster(release, rspec_args)
-            ensure
-              @praefect_node&.teardown
-              @sql_node&.teardown
-              @gitaly_primary_node&.teardown
-              @gitaly_secondary_node&.teardown
-              @gitaly_tertiary_node&.teardown
-            end
-
-            def run_gitaly_cluster(release, rspec_args) # rubocop:disable Metrics/AbcSize
-              # The postgres container starts in seconds so not essential to parallelize it
-              # This also ensure that the docker network is created here, avoiding any potential race conditions later
-              #  if the gitaly-cluster and GitLab containers attempt to create a network in parallel
-              @sql_node = postgres
-
-              gitaly_cluster = Thread.new do
-                Thread.current.abort_on_exception = true
-                start_gitaly_cluster(release)
-              end
-
               Component::Gitlab.perform do |gitlab|
                 gitlab.release = QA::Release.new(release)
-                gitlab.name = gitlab_name
-                gitlab.network = @network
-
+                gitlab.name = config.gitlab_name
+                gitlab.network = config.network
                 gitlab.omnibus_configuration << gitlab_omnibus_configuration
+                cluster = Component::GitalyCluster.perform(&:instance)
                 gitlab.instance do
-                  # Wait for gitaly cluster to finish booting, before attempting to run specs
-                  gitaly_cluster.join
+                  cluster.join
                   Runtime::Logger.info('Running Gitaly Cluster specs!')
 
                   if @tag
@@ -87,100 +44,13 @@ module Gitlab
 
             private
 
-            def start_gitaly_cluster(release)
-              Runtime::Logger.info("Starting Gitaly Cluster")
-              @gitaly_primary_node = gitaly(@primary_node_name, @primary_node_port, release)
-              @gitaly_secondary_node = gitaly(@secondary_node_name, @secondary_node_port, release)
-              @gitaly_tertiary_node = gitaly(@tertiary_node_name, @tertiary_node_port, release)
-
-              @praefect_node = praefect(@praefect_node_name, release)
-              Runtime::Logger.info("Gitaly Cluster Ready")
-            end
-
-            def disable_other_services
-              <<~OMNIBUS
-                postgresql['enable'] = false;
-                redis['enable'] = false;
-                nginx['enable'] = false;
-                prometheus['enable'] = false;
-                grafana['enable'] = false;
-                puma['enable'] = false;
-                sidekiq['enable'] = false;
-                gitlab_workhorse['enable'] = false;
-                gitlab_rails['rake_cache_clear'] = false;
-                gitlab_rails['auto_migrate'] = false;
-              OMNIBUS
-            end
-
-            def praefect_omnibus_configuration
-              <<~OMNIBUS
-                #{disable_other_services}
-                gitaly['enable'] = false;
-                praefect['enable'] = true;
-                praefect['listen_addr'] = '0.0.0.0:#{@praefect_port}';
-                praefect['prometheus_listen_addr'] = '0.0.0.0:9652';
-                praefect['auth_token'] = 'PRAEFECT_EXTERNAL_TOKEN';
-                praefect['reconciliation_scheduling_interval'] = '10s';
-                praefect['database_host'] = '#{@database_addr}';
-                praefect['database_user'] = 'postgres';
-                praefect['database_port'] = #{@database_port};
-                praefect['database_password'] = 'SQL_PASSWORD';
-                praefect['database_dbname'] = 'praefect_production';
-                praefect['database_sslmode'] = 'disable';
-                praefect['database_direct_host'] = '#{@database_addr}';
-                praefect['database_direct_port'] = #{@database_port};
-                praefect['virtual_storages'] = {
-                  'default' => {
-                    'nodes' => {
-                      '#{@primary_node_name}' => {
-                        'address' => 'tcp://#{@primary_node_addr}:#{@primary_node_port}',
-                        'token'   => 'PRAEFECT_INTERNAL_TOKEN'
-                      },
-                      '#{@secondary_node_name}' => {
-                        'address' => 'tcp://#{@secondary_node_addr}:#{@secondary_node_port}',
-                        'token'   => 'PRAEFECT_INTERNAL_TOKEN'
-                      },
-                      '#{@tertiary_node_name}' => {
-                        'address' => 'tcp://#{@tertiary_node_addr}:#{@tertiary_node_port}',
-                        'token'   => 'PRAEFECT_INTERNAL_TOKEN'
-                      }
-                    }
-                  }
-                };
-              OMNIBUS
-            end
-
-            def gitaly_omnibus_configuration(listen_port)
-              <<~OMNIBUS
-                #{disable_other_services.sub(/(prometheus\['enable'\]) = false/, '\1 = true')}
-                prometheus_monitoring['enable'] = false;
-                gitaly['enable'] = true;
-                gitaly['listen_addr'] = '0.0.0.0:#{listen_port}';
-                gitaly['prometheus_listen_addr'] = '0.0.0.0:9236';
-                gitaly['auth_token'] = 'PRAEFECT_INTERNAL_TOKEN';
-                gitlab_shell['secret_token'] = 'GITLAB_SHELL_SECRET_TOKEN';
-                gitlab_rails['internal_api_url'] = 'http://#{@gitlab_name}.#{@network}';
-                git_data_dirs({
-                  '#{@primary_node_name}' => {
-                    'path' => '/var/opt/gitlab/git-data'
-                  },
-                  '#{@secondary_node_name}' => {
-                    'path' => '/var/opt/gitlab/git-data'
-                  },
-                  '#{@tertiary_node_name}' => {
-                    'path' => '/var/opt/gitlab/git-data'
-                  }
-                });
-              OMNIBUS
-            end
-
             def gitlab_omnibus_configuration
               <<~OMNIBUS
-                external_url 'http://#{@gitlab_name}.#{@network}';
+                external_url 'http://#{config.gitlab_name}.#{config.network}';
 
                 git_data_dirs({
                   'default' => {
-                    'gitaly_address' => 'tcp://#{@praefect_addr}:#{@praefect_port}',
+                    'gitaly_address' => 'tcp://#{config.praefect_addr}:#{config.praefect_port}',
                     'gitaly_token' => 'PRAEFECT_EXTERNAL_TOKEN'
                   }
                 });
@@ -191,7 +61,7 @@ module Gitlab
                     'job_name' => 'praefect',
                     'static_configs' => [
                       'targets' => [
-                        '#{@praefect_node_name}.#{@network}:9652'
+                        '#{config.praefect_addr}:9652'
                       ]
                     ]
                   },
@@ -199,9 +69,9 @@ module Gitlab
                     'job_name' => 'praefect-gitaly',
                     'static_configs' => [
                       'targets' => [
-                        '#{@primary_node_name}.#{@network}:9236',
-                        '#{@secondary_node_name}.#{@network}:9236',
-                        '#{@tertiary_node_name}.#{@network}:9236'
+                        '#{config.primary_node_addr}:9236',
+                        '#{config.secondary_node_addr}:9236',
+                        '#{config.tertiary_node_addr}:9236'
                       ]
                     ]
                   }
@@ -210,42 +80,6 @@ module Gitlab
                 grafana['admin_password'] = 'GRAFANA_ADMIN_PASSWORD';
               OMNIBUS
             end
-
-            def praefect(name, release)
-              Component::Gitlab.new.tap do |praefect|
-                praefect.release = QA::Release.new(release)
-                praefect.name = name
-                praefect.network = @network
-                praefect.skip_availability_check = true
-                praefect.seed_admin_token = false
-
-                praefect.omnibus_configuration << praefect_omnibus_configuration
-
-                praefect.instance(skip_teardown: true)
-              end
-            end
-
-            def postgres
-              Component::PostgreSQL.new.tap do |sql|
-                sql.name = @database
-                sql.network = @network
-                sql.instance(skip_teardown: true) do
-                  sql.run_psql '-d template1 -c "CREATE DATABASE praefect_production OWNER postgres"'
-                end
-              end
-            end
-
-            def gitaly(name, port, release)
-              Component::Gitlab.new.tap do |gitaly|
-                gitaly.release = QA::Release.new(release)
-                gitaly.name = name
-                gitaly.network = @network
-                gitaly.skip_availability_check = true
-                gitaly.seed_admin_token = false
-                gitaly.omnibus_configuration << gitaly_omnibus_configuration(port)
-                gitaly.instance(skip_teardown: true)
-              end
-            end
           end
         end
       end

data/lib/gitlab/qa/scenario/test/integration/praefect.rb

@@ -15,17 +15,17 @@ module Gitlab
               @env = { QA_PRAEFECT_REPOSITORY_STORAGE: 'default' }
             end
 
-            def gitlab_omnibus_configuration
+            def gitlab_omnibus_configuration # rubocop:disable Metrics/AbcSize
               <<~OMNIBUS
-                external_url 'http://#{@gitlab_name}.#{@network}';
+                external_url 'http://#{config.gitlab_name}.#{config.network}';
 
                 git_data_dirs({
                   'default' => {
-                    'gitaly_address' => 'tcp://#{@praefect_addr}:#{@praefect_port}',
+                    'gitaly_address' => 'tcp://#{config.praefect_addr}:#{config.praefect_port}',
                     'gitaly_token' => 'PRAEFECT_EXTERNAL_TOKEN'
                   },
                   'gitaly' => {
-                    'gitaly_address' => 'tcp://#{@gitlab_name}.#{@network}:8075',
+                    'gitaly_address' => 'tcp://#{config.gitlab_name}.#{config.network}:8075',
                     'path' => '/var/opt/gitlab/git-data'
                   }
                 });
@@ -45,7 +45,7 @@ module Gitlab
                     'job_name' => 'praefect',
                     'static_configs' => [
                       'targets' => [
-                        '#{@praefect_addr}:9652'
+                        '#{config.praefect_addr}:9652'
                       ]
                     ]
                   },
@@ -53,9 +53,9 @@ module Gitlab
                     'job_name' => 'praefect-gitaly',
                     'static_configs' => [
                       'targets' => [
-                        '#{@primary_node_addr}:9236',
-                        '#{@secondary_node_addr}:9236',
-                        '#{@tertiary_node_addr}:9236'
+                        '#{config.primary_node_addr}:9236',
+                        '#{config.secondary_node_addr}:9236',
+                        '#{config.tertiary_node_addr}:9236'
                       ]
                     ]
                   }

data/lib/gitlab/qa/support/config_scripts.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Support
+      module ConfigScripts
+        # Add a git server hooks with a custom error message
+        # See https://docs.gitlab.com/ee/administration/server_hooks.html for details
+        def self.add_git_server_hooks(docker, name)
+          global_server_prereceive_hook = <<~SCRIPT
+            #!/usr/bin/env bash
+
+            if [[ \\\$GL_PROJECT_PATH =~ 'reject-prereceive' ]]; then
+              echo 'GL-HOOK-ERR: Custom error message rejecting prereceive hook for projects with GL_PROJECT_PATH matching pattern reject-prereceive'
+              exit 1
+            fi
+          SCRIPT
+
+          [
+            docker.exec(name, 'mkdir -p /opt/gitlab/embedded/service/gitlab-shell/hooks/pre-receive.d'),
+            docker.write_files(name) do |f|
+              f.write(
+                '/opt/gitlab/embedded/service/gitlab-shell/hooks/pre-receive.d/pre-receive.d',
+                global_server_prereceive_hook, false
+              )
+            end,
+            docker.exec(name, 'chmod +x /opt/gitlab/embedded/service/gitlab-shell/hooks/pre-receive.d/*')
+          ]
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   module QA
-    VERSION = '8.13.1'
+    VERSION = '8.14.1'
   end
 end

data/scripts/generate-qa-jobs.rb

@@ -23,6 +23,7 @@ class GenerateQAJobs
     jobs.merge!(load_yml_contents('update', should_automatically_run?('test_instance_all')))
     jobs.merge!(load_yml_contents('omnibus_upgrade'))
     jobs.merge!(load_yml_contents('ee_previous_to_ce_update'))
+    jobs.merge!(load_yml_contents('airgapped', should_automatically_run?('test_instance_all')))
     jobs.merge!(load_yml_contents('mattermost', should_automatically_run?('test_integration_mattermost')))
     jobs.merge!(load_yml_contents('service_ping_disabled', should_automatically_run?('test_integration_servicepingdisabled')))
     jobs.merge!(load_yml_contents('ldap_no_tls', should_automatically_run?('test_integration_ldapnotls')))

data/support/data/license_usage_seed.rb

@@ -24,7 +24,7 @@ class LicenseUsageSeed
 
     User.create!(
       email: "#{name}@test.com",
-      password: 'password',
+      password: SecureRandom.hex.slice(0, 16),
       username: name,
       name: "User #{name}",
       confirmed_at: Time.current

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 8.13.1
+  version: 8.14.1
 platform: ruby
 authors:
 - GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-11-23 00:00:00.000000000 Z
+date: 2022-12-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -244,6 +244,7 @@ files:
 - ".dockerignore"
 - ".gitignore"
 - ".gitlab-ci.yml"
+- ".gitlab/ci/jobs/airgapped.gitlab-ci.yml"
 - ".gitlab/ci/jobs/base.gitlab-ci.yml"
 - ".gitlab/ci/jobs/chaos.gitlab-ci.yml"
 - ".gitlab/ci/jobs/cloud_activation.gitlab-ci.yml"
@@ -328,6 +329,8 @@ files:
 - lib/gitlab/qa/component/base.rb
 - lib/gitlab/qa/component/chaos.rb
 - lib/gitlab/qa/component/elasticsearch.rb
+- lib/gitlab/qa/component/gitaly.rb
+- lib/gitlab/qa/component/gitaly_cluster.rb
 - lib/gitlab/qa/component/gitlab.rb
 - lib/gitlab/qa/component/jira.rb
 - lib/gitlab/qa/component/ldap.rb
@@ -336,6 +339,7 @@ files:
 - lib/gitlab/qa/component/mock_server.rb
 - lib/gitlab/qa/component/opensearch.rb
 - lib/gitlab/qa/component/postgresql.rb
+- lib/gitlab/qa/component/praefect.rb
 - lib/gitlab/qa/component/preprod.rb
 - lib/gitlab/qa/component/production.rb
 - lib/gitlab/qa/component/release.rb
@@ -436,6 +440,7 @@ files:
 - lib/gitlab/qa/service/cluster_provider/k3d.rb
 - lib/gitlab/qa/service/kubernetes_cluster.rb
 - lib/gitlab/qa/slack/post_to_slack.rb
+- lib/gitlab/qa/support/config_scripts.rb
 - lib/gitlab/qa/support/get_request.rb
 - lib/gitlab/qa/support/gitlab_upgrade_path.rb
 - lib/gitlab/qa/support/gitlab_version_info.rb