gitlab-qa 7.24.6 → 7.25.0

data/docs/what_tests_can_be_run.md

@@ -818,6 +818,21 @@ $ export GITLAB_PASSWORD="$GITLAB_QA_PASSWORD"
 $ gitlab-qa Test::Instance::StagingGeo
 ```
 
+### `Test::Instance::StagingRefGeo`
+
+This scenario tests that the Geo staging deployment (with [`staging-ref.gitlab.com`](https://staging-ref.gitlab.com) as the primary site and [`geo.staging-ref.gitlab.com`](https://geo.staging-ref.gitlab.com) as the secondary site) works as expected by running tests tagged `:geo` against it. This is done by spinning up a GitLab QA (`gitlab/gitlab-qa`) container and running the `QA::EE::Scenario::Test::Geo` scenario. Note that the Geo setup steps in the `QA::EE::Scenario::Test::Geo` scenario are skipped when testing a live Geo deployment.
+
+Scenario requirements are the same as [`Test::Instance::StagingGeo`](#testinstancestaginggeo) described above.
+
+```
+$ export GITLAB_QA_ACCESS_TOKEN=your_api_access_token
+$ export GITLAB_QA_DEV_ACCESS_TOKEN=your_dev_registry_access_token
+$ export GITLAB_USERNAME="gitlab-qa"
+$ export GITLAB_PASSWORD="$GITLAB_QA_PASSWORD"
+
+$ gitlab-qa Test::Instance::StagingRefGeo
+```
+
 ### `Test::Instance::Production`
 
 This scenario functions the same as `Test::Instance::Staging`

data/lib/gitlab/qa/component/gitlab.rb

@@ -5,6 +5,7 @@ require 'net/http'
 require 'uri'
 require 'forwardable'
 require 'openssl'
+require 'tempfile'
 
 module Gitlab
   module QA
@@ -19,9 +20,11 @@ module Gitlab
         def_delegators :release, :tag, :image, :edition
 
         CERTIFICATES_PATH = File.expand_path('../../../../tls_certificates', __dir__)
+        DATA_SEED_PATH = File.expand_path('../../../../support/data', __dir__)
 
         SSL_PATH = '/etc/gitlab/ssl'
         TRUSTED_PATH = '/etc/gitlab/trusted-certs'
+        DATA_PATH = '/tmp/data-seeds'
 
         def initialize
           super
@@ -40,6 +43,7 @@ module Gitlab
           @volumes[@authority_cert_path] = TRUSTED_PATH
 
           self.release = 'CE'
+          self.exec_commands += seed_test_data_command if Runtime::Scenario.seed_db
         end
 
         def set_formless_login_token
@@ -166,6 +170,12 @@ module Gitlab
           end
         end
 
+        def process_exec_commands
+          @docker.copy(name, DATA_SEED_PATH, DATA_PATH) if Runtime::Scenario.seed_db
+
+          exec_commands.each { |command| @docker.exec(name, command) }
+        end
+
         def sha_version
           json = @docker.read_file(
             @release.image, @release.tag,
@@ -176,6 +186,19 @@ module Gitlab
           manifest['software']['gitlab-rails']['locked_version']
         end
 
+        def copy_key_file(env_key)
+          key_dir = ENV['CI_PROJECT_DIR'] || Dir.tmpdir
+          key_file = Tempfile.new(env_key.downcase, key_dir)
+          key_file.write(ENV.fetch(env_key))
+          key_file.close
+
+          File.chmod(0o744, key_file.path)
+
+          @volumes[key_file.path] = key_file.path
+
+          key_file.path
+        end
+
         private
 
         # Copy certs to a temporary directory in current working directory.
@@ -196,6 +219,18 @@ module Gitlab
           end
         end
 
+        def seed_test_data_command
+          cmd = []
+
+          Runtime::Scenario.seed_db.each do |file_patterns|
+            Dir["#{DATA_SEED_PATH}/#{file_patterns}"].map { |f| File.basename f }.each do |file|
+              cmd << "gitlab-rails runner #{DATA_PATH}/#{file}"
+            end
+          end
+
+          cmd.uniq
+        end
+
         class Availability
           def initialize(name, relative_path: '', scheme: 'http', protocol_port: 80)
             @docker = Docker::Engine.new

data/lib/gitlab/qa/component/staging_ref.rb

@@ -3,6 +3,7 @@ module Gitlab
     module Component
       class StagingRef < Staging
         ADDRESS = 'https://staging-ref.gitlab.com'.freeze
+        GEO_SECONDARY_ADDRESS = 'https://geo.staging-ref.gitlab.com'.freeze
       end
     end
   end

data/lib/gitlab/qa/docker/engine.rb

@@ -76,6 +76,10 @@ module Gitlab
           Docker::Command.execute("attach --sig-proxy=false #{name}", &block)
         end
 
+        def copy(name, src_path, dest_path)
+          Docker::Command.execute("cp #{src_path} #{name}:#{dest_path}")
+        end
+
         def restart(name)
           Docker::Command.execute("restart #{name}")
         end

data/lib/gitlab/qa/release.rb

@@ -147,7 +147,7 @@ module Gitlab
             registry: DEV_REGISTRY
           }
         elsif omnibus_mirror?
-          username, password = if Runtime::Env.ci_job_token && Runtime::Env.ci_pipeline_source == 'pipeline'
+          username, password = if Runtime::Env.ci_job_token && Runtime::Env.ci_pipeline_source.include?('pipeline')
                                  ['gitlab-ci-token', Runtime::Env.ci_job_token]
                                elsif Runtime::Env.qa_container_registry_access_token
                                  [Runtime::Env.gitlab_username, Runtime::Env.qa_container_registry_access_token]

data/lib/gitlab/qa/runner.rb

@@ -12,9 +12,11 @@ module Gitlab
         Runtime::Scenario.define(:run_tests, true)
         Runtime::Scenario.define(:qa_image, Runtime::Env.qa_image) if Runtime::Env.qa_image
         Runtime::Scenario.define(:omnibus_configuration, Runtime::OmnibusConfiguration.new)
+        Runtime::Scenario.define(:seed_db, false)
 
         # Omnibus Configurators specified by flags
         @active_configurators = []
+        @seed_scripts = []
         @omnibus_configurations = %w[default] # always load default configuration
 
         @options = OptionParser.new do |opts|
@@ -45,6 +47,14 @@ module Gitlab
             end
           end
 
+          opts.on('--seed-db search_pattern1[,search_pattern2,...]', 'Seed application database with sample test data') do |file_pattern|
+            file_pattern.split(',').each do |pattern|
+              @seed_scripts << pattern
+            end
+
+            Runtime::Scenario.define(:seed_db, @seed_scripts)
+          end
+
           opts.on_tail('-h', '--help', 'Show the usage') do
             puts opts
             exit

data/lib/gitlab/qa/runtime/env.rb

@@ -127,6 +127,11 @@ module Gitlab
           'GOOGLE_PROJECT' => :google_project,
           'GOOGLE_CLIENT_EMAIL' => :google_client_email,
           'GOOGLE_JSON_KEY' => :google_json_key,
+          'GOOGLE_CDN_JSON_KEY' => :google_cdn_json_key,
+          'GOOGLE_CDN_LB' => :google_cdn_load_balancer,
+          'GOOGLE_CDN_SIGNURL_KEY' => :google_cdn_signurl_key,
+          'GOOGLE_CDN_SIGNURL_KEY_NAME' => :google_cdn_signurl_key_name,
+          'GCS_CDN_BUCKET_NAME' => :gcs_cdn_bucket_name,
           'GCS_BUCKET_NAME' => :gcs_bucket_name,
           'SMOKE_ONLY' => :smoke_only,
           'NO_ADMIN' => :no_admin,
@@ -275,6 +280,14 @@ module Gitlab
           end
         end
 
+        def require_gcs_with_cdn_environment!
+          %w[GOOGLE_CDN_JSON_KEY GCS_CDN_BUCKET_NAME GOOGLE_CDN_LB GOOGLE_CDN_SIGNURL_KEY GOOGLE_CDN_SIGNURL_KEY_NAME].each do |env_key|
+            unless ENV.key?(env_key)
+              raise ArgumentError, "Environment variable #{env_key} must be set to run GCS with CDN enabled scenario"
+            end
+          end
+        end
+
         def require_initial_password!
           return unless env_var_value_if_defined('GITLAB_INITIAL_ROOT_PASSWORD').to_s.strip.empty?
 

data/lib/gitlab/qa/scenario/test/instance/staging_ref_geo.rb

@@ -0,0 +1,27 @@
+module Gitlab
+  module QA
+    module Scenario
+      module Test
+        module Instance
+          class StagingRefGeo < DeploymentBase
+            def initialize
+              @suite = 'QA::EE::Scenario::Test::Geo'
+            end
+
+            def deployment_component
+              Component::StagingRef
+            end
+
+            def non_rspec_args
+              [
+                '--primary-address', deployment_component::ADDRESS,
+                '--secondary-address', deployment_component::GEO_SECONDARY_ADDRESS,
+                '--without-setup'
+              ]
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/scenario/test/integration/registry_with_cdn.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Scenario
+      module Test
+        module Integration
+          class RegistryWithCDN < Scenario::Template
+            def perform(release, *rspec_args)
+              Runtime::Env.require_gcs_with_cdn_environment!
+
+              Component::Gitlab.perform do |gitlab|
+                gitlab.release = release
+                gitlab.network = 'test'
+                gitlab.name = 'gitlab'
+                sign_url_key_path = gitlab.copy_key_file('GOOGLE_CDN_SIGNURL_KEY')
+                cdn_gcloud_path = gitlab.copy_key_file('GOOGLE_CDN_JSON_KEY')
+
+                gitlab.omnibus_configuration << <<~OMNIBUS
+                  external_url 'http://#{gitlab.name}.#{gitlab.network}';
+                  registry_external_url 'http://#{gitlab.name}.#{gitlab.network}:5050';
+
+                  registry['middleware'] = { 'storage' => [{ 'name' => 'googlecdn', 'options' => { 'baseurl' => '#{Runtime::Env.google_cdn_load_balancer}', 'privatekey' => '#{sign_url_key_path}', 'keyname' => '#{Runtime::Env.google_cdn_signurl_key_name}' } }] }
+                  registry['storage'] = { 'gcs' => { 'bucket' => '#{Runtime::Env.gcs_cdn_bucket_name}', 'keyfile' => '#{cdn_gcloud_path}' } }
+                OMNIBUS
+
+                gitlab.instance do
+                  Component::Specs.perform do |specs|
+                    specs.suite = 'Test::Integration::RegistryWithCDN'
+                    specs.release = gitlab.release
+                    specs.network = gitlab.network
+                    specs.args = [gitlab.address, *rspec_args]
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   module QA
-    VERSION = '7.24.6'
+    VERSION = '7.25.0'
   end
 end

data/lib/gitlab/qa.rb

@@ -43,6 +43,7 @@ module Gitlab
           autoload :Release, 'gitlab/qa/scenario/test/instance/release'
           autoload :Geo, 'gitlab/qa/scenario/test/instance/geo'
           autoload :StagingGeo, 'gitlab/qa/scenario/test/instance/staging_geo'
+          autoload :StagingRefGeo, 'gitlab/qa/scenario/test/instance/staging_ref_geo'
           autoload :Airgapped, 'gitlab/qa/scenario/test/instance/airgapped'
         end
 
@@ -76,6 +77,7 @@ module Gitlab
           autoload :RegistryTLS, 'gitlab/qa/scenario/test/integration/registry_tls'
           autoload :ServicePingDisabled, 'gitlab/qa/scenario/test/integration/service_ping_disabled'
           autoload :Integrations, 'gitlab/qa/scenario/test/integration/integrations'
+          autoload :RegistryWithCDN, 'gitlab/qa/scenario/test/integration/registry_with_cdn'
         end
 
         module Sanity