gitlab-qa 5.13.7 → 5.17.0

data/lib/gitlab/qa/docker/command.rb

@@ -4,8 +4,9 @@ module Gitlab
       class Command
         attr_reader :args
 
-        def initialize(cmd = nil)
+        def initialize(cmd = nil, mask_secrets: nil)
           @args = Array(cmd)
+          @mask_secrets = Array(mask_secrets)
         end
 
         def <<(*args)
@@ -28,6 +29,17 @@ module Gitlab
           "docker #{@args.join(' ')}"
         end
 
+        # Returns a masked string form of a Command
+        #
+        # @example
+        #   Command.new('a docker command', mask_secrets: 'command').mask_secrets #=> 'a docker *****'
+        #   Command.new('a docker command', mask_secrets: %w[docker command]).mask_secrets #=> 'a ***** *****'
+        #
+        # @return [String] The masked command string
+        def mask_secrets
+          @mask_secrets.each_with_object(to_s) { |secret, s| s.gsub!(secret, '*****') }
+        end
+
         def ==(other)
           to_s == other.to_s
         end
@@ -36,8 +48,8 @@ module Gitlab
           Docker::Shellout.new(self).execute!(&block)
         end
 
-        def self.execute(cmd, &block)
-          new(cmd).execute!(&block)
+        def self.execute(cmd, mask_secrets: nil, &block)
+          new(cmd, mask_secrets: mask_secrets).execute!(&block)
         end
       end
     end

data/lib/gitlab/qa/docker/engine.rb

@@ -3,13 +3,14 @@ module Gitlab
     module Docker
       class Engine
         DOCKER_HOST = ENV['DOCKER_HOST'] || 'http://localhost'
+        PRIVILEGED_COMMANDS = [/^iptables.*/].freeze
 
         def hostname
           URI(DOCKER_HOST).host
         end
 
         def login(username:, password:, registry:)
-          Docker::Command.execute(%(login --username "#{username}" --password "#{password}" #{registry}))
+          Docker::Command.execute(%(login --username "#{username}" --password "#{password}" #{registry}), mask_secrets: password)
         end
 
         def pull(image, tag)
@@ -27,8 +28,18 @@ module Gitlab
           end
         end
 
+        def privileged_command?(command)
+          PRIVILEGED_COMMANDS.each do |privileged_regex|
+            return true if command.match(privileged_regex)
+          end
+
+          false
+        end
+
         def exec(name, command)
-          Docker::Command.execute("exec #{name} bash -c '#{command}'")
+          cmd = ['exec']
+          cmd << '--privileged' if privileged_command?(command)
+          Docker::Command.execute("#{cmd.join(' ')} #{name} bash -c '#{command}'")
         end
 
         def read_file(image, tag, path, &block)
@@ -71,6 +82,10 @@ module Gitlab
         def running?(name)
           Docker::Command.execute("ps -f name=#{name}").include?(name)
         end
+
+        def ps(name = nil)
+          Docker::Command.execute(['ps', name].compact.join(' '))
+        end
       end
     end
   end

data/lib/gitlab/qa/docker/shellout.rb

@@ -10,7 +10,7 @@ module Gitlab
           @command = command
           @output = []
 
-          puts "Docker shell command: `#{@command}`"
+          puts "Docker shell command: `#{@command.mask_secrets}`"
         end
 
         def execute!
@@ -28,7 +28,7 @@ module Gitlab
             end
 
             if wait.value.exited? && wait.value.exitstatus.nonzero?
-              raise StatusError, "Docker command `#{@command}` failed!"
+              raise StatusError, "Docker command `#{@command.mask_secrets}` failed!"
             end
           end
 

data/lib/gitlab/qa/release.rb

@@ -136,6 +136,8 @@ module Gitlab
       end
 
       def login_params
+        return if Runtime::Env.skip_pull?
+
         if dev_gitlab_org?
           Runtime::Env.require_qa_dev_access_token!
 
@@ -145,11 +147,18 @@ module Gitlab
             registry: DEV_REGISTRY
           }
         elsif omnibus_mirror?
-          Runtime::Env.require_gitlab_bot_multi_project_pipeline_polling_token!
-
+          username, password = if Runtime::Env.ci_job_token && Runtime::Env.ci_pipeline_source == 'pipeline'
+                                 ['gitlab-ci-token', Runtime::Env.ci_job_token]
+                               elsif Runtime::Env.qa_container_registry_access_token
+                                 [Runtime::Env.gitlab_username, Runtime::Env.qa_container_registry_access_token]
+                               else
+                                 Runtime::Env.require_qa_access_token!
+
+                                 [Runtime::Env.gitlab_username, Runtime::Env.qa_access_token]
+                               end
           {
-            username: Runtime::Env.gitlab_username,
-            password: Runtime::Env.gitlab_bot_multi_project_pipeline_polling_token,
+            username: username,
+            password: password,
             registry: COM_REGISTRY
           }
         end
@@ -167,6 +176,10 @@ module Gitlab
         canonical? || release.match?(CUSTOM_GITLAB_IMAGE_REGEX)
       end
 
+      def api_project_name
+        project_name.gsub('ce', 'foss').gsub('-ee', '')
+      end
+
       private
 
       def canonical?

data/lib/gitlab/qa/runner.rb

@@ -4,22 +4,20 @@ module Gitlab
   module QA
     # rubocop:disable Metrics/AbcSize
     class Runner
-      # These options are implemented in the QA framework (i.e., in the CE/EE codebase)
-      # They're included here so that gitlab-qa treats them as valid options
-      PASS_THROUGH_OPTS = [
-        ['--address URL', 'Address of the instance to test'],
-        ['--enable-feature FEATURE_FLAG', 'Enable a feature before running tests'],
-        ['--mattermost-address URL', 'Address of the Mattermost server'],
-        ['--parallel', 'Execute tests in parallel'],
-        ['--loop', 'Execute tests in a loop']
-      ].freeze
-
       def self.run(args)
-        options = OptionParser.new do |opts|
+        Runtime::Scenario.define(:teardown, true)
+        Runtime::Scenario.define(:run_tests, true)
+
+        @options = OptionParser.new do |opts|
           opts.banner = 'Usage: gitlab-qa [options] Scenario URL [[--] path] [rspec_options]'
 
-          PASS_THROUGH_OPTS.each do |opt|
-            opts.on(*opt)
+          opts.on('--no-teardown', 'Skip teardown of containers after the scenario completes.') do
+            Runtime::Scenario.define(:teardown, false)
+          end
+
+          opts.on('--no-tests', 'Orchestrates the docker containers but does not run the tests. Implies --no-teardown') do
+            Runtime::Scenario.define(:run_tests, false)
+            Runtime::Scenario.define(:teardown, false)
           end
 
           opts.on_tail('-v', '--version', 'Show the version') do
@@ -33,19 +31,29 @@ module Gitlab
             exit
           end
 
-          opts.parse(args)
+          begin
+            opts.parse(args)
+          rescue OptionParser::InvalidOption
+            # Ignore invalid options and options that are passed through to the tests
+          end
         end
 
+        args.reject! { |arg| gitlab_qa_options.include?(arg) }
+
         if args.size >= 1
           Scenario
             .const_get(args.shift)
             .perform(*args)
         else
-          puts options
+          puts @options
           exit 1
         end
       end
       # rubocop:enable Metrics/AbcSize
+
+      def self.gitlab_qa_options
+        @gitlab_qa_options ||= @options.top.list.map(&:long).flatten
+      end
     end
   end
 end

data/lib/gitlab/qa/runtime/env.rb

@@ -6,6 +6,8 @@ module Gitlab
       module Env
         extend self
 
+        # Variables that are used in tests and are passed through to the docker container that executes the tests.
+        # These variables should be listed in /docs/what_tests_can_be_run.md#supported-gitlab-environment-variables
         ENV_VARIABLES = {
           'QA_REMOTE_GRID' => :remote_grid,
           'QA_REMOTE_GRID_USERNAME' => :remote_grid_username,
@@ -38,6 +40,7 @@ module Gitlab
           'SIGNUP_DISABLED' => :signup_disabled,
           'QA_ADDITIONAL_REPOSITORY_STORAGE' => :qa_additional_repository_storage,
           'QA_PRAEFECT_REPOSITORY_STORAGE' => :qa_praefect_repository_storage,
+          'QA_GITALY_NON_CLUSTER_STORAGE' => :qa_gitaly_non_cluster_storage,
           'QA_COOKIES' => :qa_cookie,
           'QA_DEBUG' => :qa_debug,
           'QA_LOG_PATH' => :qa_log_path,
@@ -104,10 +107,18 @@ module Gitlab
           ENV['CI_JOB_NAME']
         end
 
+        def ci_job_token
+          ENV['CI_JOB_TOKEN']
+        end
+
         def ci_job_url
           ENV['CI_JOB_URL']
         end
 
+        def ci_pipeline_source
+          ENV['CI_PIPELINE_SOURCE']
+        end
+
         def ci_project_name
           ENV['CI_PROJECT_NAME']
         end
@@ -144,6 +155,10 @@ module Gitlab
           ENV['GITLAB_QA_DEV_ACCESS_TOKEN']
         end
 
+        def qa_container_registry_access_token
+          ENV['GITLAB_QA_CONTAINER_REGISTRY_ACCESS_TOKEN']
+        end
+
         def host_artifacts_dir
           @host_artifacts_dir ||= File.join(ENV['QA_ARTIFACTS_DIR'] || '/tmp/gitlab-qa', Runtime::Env.run_id)
         end
@@ -207,12 +222,6 @@ module Gitlab
           end
         end
 
-        def require_gitlab_bot_multi_project_pipeline_polling_token!
-          return unless ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN'].to_s.strip.empty?
-
-          raise ArgumentError, "Please provide GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN"
-        end
-
         def skip_pull?
           enabled?(ENV['QA_SKIP_PULL'], default: false)
         end

data/lib/gitlab/qa/scenario/test/instance/airgapped.rb

@@ -0,0 +1,68 @@
+module Gitlab
+  module QA
+    module Scenario
+      module Test
+        module Instance
+          class Airgapped < Scenario::Template
+            require 'resolv'
+            attr_accessor :commands
+
+            def initialize
+              gitlab_ip = Resolv.getaddress('registry.gitlab.com')
+              @commands = <<~AIRGAP_AND_VERIFY_COMMAND.split(/\n+/)
+                # Should not fail before airgapping due to eg. DNS failure
+                # Ping and wget check
+                apt-get update && apt-get install -y iptables netcat
+                nc -zv -w 10 #{gitlab_ip} 80 && (echo \"Regular connectivity netcat check passed.\" && exit 0) || (echo \"Regular connectivity netcat check failed.\" && exit 1)
+                echo "Checking regular connectivity..." \
+                  && wget --retry-connrefused --waitretry=1 --read-timeout=15 --timeout=10 -t 2 http://registry.gitlab.com > /dev/null 2>&1 \
+                  && (echo "Regular connectivity wget check passed." && exit 0) || (echo "Regular connectivity wget check failed." && exit 1)
+
+                iptables -P INPUT DROP && iptables -P OUTPUT DROP
+                iptables -A INPUT -i lo -j ACCEPT && iptables -A OUTPUT -o lo -j ACCEPT # LOOPBACK
+                iptables -I INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
+                iptables -A FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+
+                # Jenkins on port 8080 and 50000
+                iptables -A OUTPUT -p tcp -m tcp --dport 8080 -m state --state NEW,ESTABLISHED -j ACCEPT \
+                  && iptables -A OUTPUT -p tcp -m tcp --dport 50000 -m state --state NEW,ESTABLISHED -j ACCEPT
+                iptables -A OUTPUT -p tcp -m tcp --sport 22 -m state --state NEW,ESTABLISHED -j ACCEPT
+                iptables -A INPUT -p tcp -m tcp --dport 22 -m state --state NEW,ESTABLISHED -j ACCEPT
+                iptables -A OUTPUT -p tcp -m tcp --sport 80 -m state --state NEW,ESTABLISHED -j ACCEPT
+                iptables -A INPUT -p tcp -m tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT
+
+                # Should now fail to ping and wget, port 80 should be open
+                nc -zv -w 10 #{gitlab_ip} 80 && (echo \"Airgapped network faulty. Connectivity netcat check failed.\" && exit 1) || (echo \"Connectivity netcat check passed.\" && exit 0)
+                nc -zv -w 10 127.0.0.1 22 && (echo "Airgapped connectivity port 22 check passed." && exit 0) || (echo "Airgapped connectivity port 22 check failed." && exit 1)
+                nc -zv -w 10 127.0.0.1 80 && (echo "Airgapped connectivity port 80 check passed." && exit 0) || (echo "Airgapped connectivity port 80 check failed." && exit 1)
+                echo "Checking airgapped connectivity..." \
+                  && wget --retry-connrefused --waitretry=1 --read-timeout=15 --timeout=10 -t 2 http://registry.gitlab.com > /dev/null 2>&1 \
+                  && (echo "Airgapped network faulty. Connectivity wget check failed." && exit 1) || (echo "Airgapped network confirmed. Connectivity wget check passed." && exit 0)
+              AIRGAP_AND_VERIFY_COMMAND
+            end
+
+            def perform(release, *rspec_args)
+              Component::Gitlab.perform do |gitlab|
+                gitlab.release = release
+                gitlab.network = 'test'
+                gitlab.runner_network = 'airgapped'
+                gitlab.exec_commands = @commands
+                rspec_args << "--" unless rspec_args.include?('--')
+                rspec_args << %w[--tag ~orchestrated]
+                gitlab.instance do
+                  Component::Specs.perform do |specs|
+                    specs.suite = 'Test::Instance::Airgapped'
+                    specs.release = gitlab.release
+                    specs.network = gitlab.network
+                    specs.runner_network = gitlab.runner_network
+                    specs.args = [gitlab.address, *rspec_args]
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/scenario/test/instance/geo.rb

@@ -8,9 +8,6 @@ module Gitlab
 
           class Geo < Scenario::Template
             def perform(release, primary_address, secondary_address, *rspec_args)
-              # Geo requires an EE license
-              Runtime::Env.require_license!
-
               Component::Specs.perform do |specs|
                 specs.suite = 'QA::EE::Scenario::Test::Geo'
                 specs.release = QA::Release.new(release)

data/lib/gitlab/qa/scenario/test/integration/geo.rb

@@ -27,13 +27,14 @@ module Gitlab
                   gitlab_rails['db_pool'] = 5;
                   gitlab_rails['geo_node_name'] = '#{primary.name}';
                   gitlab_rails['monitoring_whitelist'] = ['0.0.0.0/0'];
+                  gitlab_rails['packages_enabled'] = true;
                   postgresql['listen_address'] = '0.0.0.0';
                   postgresql['max_replication_slots'] = 1;
                   postgresql['md5_auth_cidr_addresses'] = ['0.0.0.0/0'];
                   postgresql['sql_user_password'] = 'e1d1469ec5f533651918b4567a3ed1ae';
                   postgresql['trust_auth_cidr_addresses'] = ['0.0.0.0/0','0.0.0.0/0'];
                   sidekiq['concurrency'] = 2;
-                  unicorn['worker_processes'] = 2;
+                  puma['worker_processes'] = 2;
                 OMNIBUS
                 primary.exec_commands = fast_ssh_key_lookup_commands + git_lfs_install_commands
 
@@ -50,11 +51,12 @@ module Gitlab
                       gitlab_rails['db_pool'] = 5;
                       gitlab_rails['geo_node_name'] = '#{secondary.name}';
                       gitlab_rails['monitoring_whitelist'] = ['0.0.0.0/0'];
+                      gitlab_rails['packages_enabled'] = true;
                       postgresql['listen_address'] = '0.0.0.0';
                       postgresql['md5_auth_cidr_addresses'] = ['0.0.0.0/0'];
                       postgresql['sql_user_password'] = 'e1d1469ec5f533651918b4567a3ed1ae';
                       sidekiq['concurrency'] = 2;
-                      unicorn['worker_processes'] = 2;
+                      puma['worker_processes'] = 2;
                     OMNIBUS
                     secondary.exec_commands = fast_ssh_key_lookup_commands + git_lfs_install_commands
 

data/lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb

@@ -0,0 +1,201 @@
+module Gitlab
+  module QA
+    module Scenario
+      module Test
+        module Integration
+          class GitalyCluster < Scenario::Template
+            attr_reader :gitlab_name, :spec_suite
+
+            def initialize
+              @gitlab_name = 'gitlab-gitaly-ha'
+              @primary_node_name = 'gitaly1'
+              @secondary_node_name = 'gitaly2'
+              @tertiary_node_name = 'gitaly3'
+              @praefect_node_name = 'praefect'
+              @database = 'postgres'
+              @spec_suite = 'Test::Integration::GitalyHA'
+              @network = 'test'
+            end
+
+            # rubocop:disable Metrics/AbcSize
+            def perform(release, *rspec_args)
+              gitaly_primary_node = gitaly(@primary_node_name, release)
+              gitaly_secondary_node = gitaly(@secondary_node_name, release)
+              gitaly_tertiary_node = gitaly(@tertiary_node_name, release)
+
+              sql_node = Component::PostgreSQL.new.tap do |sql|
+                sql.name = @database
+                sql.network = @network
+                sql.instance_no_teardown do
+                  sql.run_psql '-d template1 -c "CREATE DATABASE praefect_production OWNER postgres"'
+                end
+              end
+
+              praefect_node = Component::Gitlab.new.tap do |praefect|
+                praefect.release = QA::Release.new(release)
+                praefect.name = @praefect_node_name
+                praefect.network = @network
+                praefect.skip_availability_check = true
+
+                praefect.omnibus_config = praefect_omnibus_configuration
+
+                praefect.instance_no_teardown
+              end
+
+              Component::Gitlab.perform do |gitlab|
+                gitlab.release = QA::Release.new(release)
+                gitlab.name = gitlab_name
+                gitlab.network = @network
+
+                gitlab.omnibus_config = gitlab_omnibus_configuration
+                gitlab.instance do
+                  puts "Running Gitaly HA specs!"
+
+                  Component::Specs.perform do |specs|
+                    specs.suite = spec_suite
+                    specs.release = gitlab.release
+                    specs.network = gitlab.network
+                    specs.args = [gitlab.address, *rspec_args]
+                  end
+                end
+              end
+            ensure
+              praefect_node&.teardown
+              sql_node&.teardown
+              gitaly_primary_node&.teardown
+              gitaly_secondary_node&.teardown
+              gitaly_tertiary_node&.teardown
+            end
+            # rubocop:enable Metrics/AbcSize
+
+            private
+
+            def disable_other_services
+              <<~OMNIBUS
+                postgresql['enable'] = false;
+                redis['enable'] = false;
+                nginx['enable'] = false;
+                prometheus['enable'] = false;
+                grafana['enable'] = false;
+                puma['enable'] = false;
+                sidekiq['enable'] = false;
+                gitlab_workhorse['enable'] = false;
+                gitlab_rails['rake_cache_clear'] = false;
+                gitlab_rails['auto_migrate'] = false;
+              OMNIBUS
+            end
+
+            def praefect_omnibus_configuration
+              <<~OMNIBUS
+                #{disable_other_services}
+                gitaly['enable'] = false;
+                praefect['enable'] = true;
+                praefect['listen_addr'] = '0.0.0.0:2305';
+                praefect['prometheus_listen_addr'] = '0.0.0.0:9652';
+                praefect['auth_token'] = 'PRAEFECT_EXTERNAL_TOKEN';
+                praefect['database_host'] = '#{@database}.#{@network}';
+                praefect['database_user'] = 'postgres';
+                praefect['database_port'] = 5432;
+                praefect['database_password'] = 'SQL_PASSWORD';
+                praefect['database_dbname'] = 'praefect_production';
+                praefect['database_sslmode'] = 'disable';
+                praefect['postgres_queue_enabled'] = true;
+                praefect['failover_enabled'] = true;
+                praefect['virtual_storages'] = {
+                  'default' => {
+                    '#{@primary_node_name}' => {
+                      'address' => 'tcp://#{@primary_node_name}.#{@network}:8075',
+                      'token'   => 'PRAEFECT_INTERNAL_TOKEN',
+                      'primary' => true
+                    },
+                    '#{@secondary_node_name}' => {
+                      'address' => 'tcp://#{@secondary_node_name}.#{@network}:8075',
+                      'token'   => 'PRAEFECT_INTERNAL_TOKEN'
+                    },
+                    '#{@tertiary_node_name}' => {
+                      'address' => 'tcp://#{@tertiary_node_name}.#{@network}:8075',
+                      'token'   => 'PRAEFECT_INTERNAL_TOKEN'
+                    }
+                  }
+                };
+              OMNIBUS
+            end
+
+            def gitaly_omnibus_configuration
+              <<~OMNIBUS
+                #{disable_other_services}
+                prometheus['enable'] = true;
+                prometheus_monitoring['enable'] = false;
+                gitaly['enable'] = true;
+                gitaly['listen_addr'] = '0.0.0.0:8075';
+                gitaly['prometheus_listen_addr'] = '0.0.0.0:9236';
+                gitaly['auth_token'] = 'PRAEFECT_INTERNAL_TOKEN';
+                gitlab_shell['secret_token'] = 'GITLAB_SHELL_SECRET_TOKEN';
+                gitlab_rails['internal_api_url'] = 'http://#{@gitlab_name}.#{@network}';
+                git_data_dirs({
+                  '#{@primary_node_name}' => {
+                    'path' => '/var/opt/gitlab/git-data'
+                  },
+                  '#{@secondary_node_name}' => {
+                    'path' => '/var/opt/gitlab/git-data'
+                  },
+                  '#{@tertiary_node_name}' => {
+                    'path' => '/var/opt/gitlab/git-data'
+                  }
+                });
+              OMNIBUS
+            end
+
+            def gitlab_omnibus_configuration
+              <<~OMNIBUS
+                external_url 'http://#{@gitlab_name}.#{@network}';
+
+                git_data_dirs({
+                  'default' => {
+                    'gitaly_address' => 'tcp://#{@praefect_node_name}.#{@network}:2305',
+                    'gitaly_token' => 'PRAEFECT_EXTERNAL_TOKEN'
+                  }
+                });
+                gitaly['listen_addr'] = '0.0.0.0:8075';
+                gitlab_shell['secret_token'] = 'GITLAB_SHELL_SECRET_TOKEN';
+                prometheus['scrape_configs'] = [
+                  {
+                    'job_name' => 'praefect',
+                    'static_configs' => [
+                      'targets' => [
+                        '#{@praefect_node_name}.#{@network}:9652'
+                      ]
+                    ]
+                  },
+                  {
+                    'job_name' => 'praefect-gitaly',
+                    'static_configs' => [
+                      'targets' => [
+                        '#{@primary_node_name}.#{@network}:9236',
+                        '#{@secondary_node_name}.#{@network}:9236',
+                        '#{@tertiary_node_name}.#{@network}:9236'
+                      ]
+                    ]
+                  }
+                ];
+                grafana['disable_login_form'] = false;
+                grafana['admin_password'] = 'GRAFANA_ADMIN_PASSWORD';
+              OMNIBUS
+            end
+
+            def gitaly(name, release)
+              Component::Gitlab.new.tap do |gitaly|
+                gitaly.release = QA::Release.new(release)
+                gitaly.name = name
+                gitaly.network = @network
+                gitaly.skip_availability_check = true
+                gitaly.omnibus_config = gitaly_omnibus_configuration
+                gitaly.instance_no_teardown
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end