gitlab-qa 7.24.6 → 7.26.0

data/lib/gitlab/qa/scenario/test/integration/registry_with_cdn.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Scenario
+      module Test
+        module Integration
+          class RegistryWithCDN < Scenario::Template
+            def perform(release, *rspec_args)
+              Runtime::Env.require_gcs_with_cdn_environment!
+
+              Component::Gitlab.perform do |gitlab|
+                gitlab.release = release
+                gitlab.network = 'test'
+                gitlab.name = 'gitlab'
+                sign_url_key_path = gitlab.copy_key_file('GOOGLE_CDN_SIGNURL_KEY')
+                cdn_gcloud_path = gitlab.copy_key_file('GOOGLE_CDN_JSON_KEY')
+
+                gitlab.omnibus_configuration << <<~OMNIBUS
+                  external_url 'http://#{gitlab.name}.#{gitlab.network}';
+                  registry_external_url 'http://#{gitlab.name}.#{gitlab.network}:5050';
+
+                  registry['middleware'] = { 'storage' => [{ 'name' => 'googlecdn', 'options' => { 'baseurl' => '#{Runtime::Env.google_cdn_load_balancer}', 'privatekey' => '#{sign_url_key_path}', 'keyname' => '#{Runtime::Env.google_cdn_signurl_key_name}' } }] }
+                  registry['storage'] = { 'gcs' => { 'bucket' => '#{Runtime::Env.gcs_cdn_bucket_name}', 'keyfile' => '#{cdn_gcloud_path}' } }
+                OMNIBUS
+
+                gitlab.instance do
+                  Component::Specs.perform do |specs|
+                    specs.suite = 'Test::Integration::RegistryWithCDN'
+                    specs.release = gitlab.release
+                    specs.network = gitlab.network
+                    specs.args = [gitlab.address, *rspec_args]
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/scenario/test/omnibus/update.rb

@@ -16,6 +16,7 @@ module Gitlab
                   gitlab.release = previous_release
                   gitlab.volumes = volumes
                   gitlab.network = 'test'
+                  gitlab.seed_admin_token = false
                   gitlab.launch_and_teardown_instance
                 end
 

data/lib/gitlab/qa/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   module QA
-    VERSION = '7.24.6'
+    VERSION = '7.26.0'
   end
 end

data/lib/gitlab/qa.rb

@@ -20,6 +20,7 @@ module Gitlab
         autoload :ObjectStorageGcs, 'gitlab/qa/runtime/omnibus_configurations/object_storage_gcs'
         autoload :LicenseMode, 'gitlab/qa/runtime/omnibus_configurations/license_mode'
         autoload :RegistryObjectStorage, 'gitlab/qa/runtime/omnibus_configurations/registry_object_storage'
+        autoload :CiDecomposition, 'gitlab/qa/runtime/omnibus_configurations/ci_decomposition'
       end
     end
 
@@ -43,6 +44,7 @@ module Gitlab
           autoload :Release, 'gitlab/qa/scenario/test/instance/release'
           autoload :Geo, 'gitlab/qa/scenario/test/instance/geo'
           autoload :StagingGeo, 'gitlab/qa/scenario/test/instance/staging_geo'
+          autoload :StagingRefGeo, 'gitlab/qa/scenario/test/instance/staging_ref_geo'
           autoload :Airgapped, 'gitlab/qa/scenario/test/instance/airgapped'
         end
 
@@ -76,6 +78,7 @@ module Gitlab
           autoload :RegistryTLS, 'gitlab/qa/scenario/test/integration/registry_tls'
           autoload :ServicePingDisabled, 'gitlab/qa/scenario/test/integration/service_ping_disabled'
           autoload :Integrations, 'gitlab/qa/scenario/test/integration/integrations'
+          autoload :RegistryWithCDN, 'gitlab/qa/scenario/test/integration/registry_with_cdn'
         end
 
         module Sanity

data/scripts/generate-qa-jobs.rb

@@ -0,0 +1,72 @@
+#!/usr/bin/env ruby
+
+class GenerateQAJobs
+  def initialize(no_of_example_files)
+    @no_of_example_files = no_of_example_files
+    puts "no_of_example_files: #{@no_of_example_files}"
+  end
+
+  # rubocop:disable Metrics/AbcSize
+  # rubocop:disable Metrics/PerceivedComplexity
+  # rubocop:disable Metrics/CyclomaticComplexity
+  def execute
+    jobs = load_yml_contents('base')
+    jobs.concat(load_yml_contents('sanity_framework'))
+    jobs.concat(load_yml_contents('custom_parallel'))
+    jobs.concat(load_yml_contents('instance')) if should_run?('test_instance_all')
+    jobs.concat(load_yml_contents('relative_url')) if should_run?('test_instance_all')
+    jobs.concat(load_yml_contents('repository_storage')) if should_run?('test_instance_all_repository_storage')
+    jobs.concat(load_yml_contents('omnibus_image'))
+    jobs.concat(load_yml_contents('update')) if should_run?('test_instance_all')
+    jobs.concat(load_yml_contents('omnibus_upgrade'))
+    jobs.concat(load_yml_contents('ee_previous_to_ce_update'))
+    jobs.concat(load_yml_contents('mattermost')) if should_run?('test_integration_mattermost')
+    jobs.concat(load_yml_contents('service_ping_disabled')) if should_run?('test_integration_servicepingdisabled')
+    jobs.concat(load_yml_contents('ldap_no_tls')) if should_run?('test_integration_ldapnotls')
+    jobs.concat(load_yml_contents('ldap_tls')) if should_run?('test_integration_ldaptls')
+    jobs.concat(load_yml_contents('ldap_no_server')) if should_run?('test_integration_ldapnoserver')
+    jobs.concat(load_yml_contents('instance_saml')) if should_run?('test_integration_instancesaml')
+    jobs.concat(load_yml_contents('group_saml')) if should_run?('test_integration_groupsaml')
+    jobs.concat(load_yml_contents('object_storage')) if should_run?('test_instance_all_object_storage')
+    jobs.concat(load_yml_contents('object_storage_aws')) if should_run?('test_instance_all_object_storage')
+    jobs.concat(load_yml_contents('object_storage_gcs')) if should_run?('test_instance_all_object_storage')
+    jobs.concat(load_yml_contents('object_storage_registry_tls')) if should_run?('test_integration_registrytls')
+    jobs.concat(load_yml_contents('registry')) if should_run?('test_integration_registry')
+    jobs.concat(load_yml_contents('packages')) if should_run?('test_instance_all_packages')
+    jobs.concat(load_yml_contents('actioncable')) if should_run?('test_instance_all_actioncable')
+    jobs.concat(load_yml_contents('elasticsearch')) if should_run?('test_integration_elasticsearch')
+    jobs.concat(load_yml_contents('praefect')) if should_run?('test_instance_all')
+    jobs.concat(load_yml_contents('gitaly_cluster')) if should_run?('test_instance_all')
+    jobs.concat(load_yml_contents('mtls')) if should_run?('test_instance_all_mtls')
+    jobs.concat(load_yml_contents('smtp')) if should_run?('test_integration_smtp')
+    jobs.concat(load_yml_contents('jira')) if should_run?('test_instance_all_jira')
+    jobs.concat(load_yml_contents('integrations')) if should_run?('test_instance_all_integrations')
+    jobs.concat(load_yml_contents('large_setup')) if should_run?('test_instance_all_can_use_large_setup')
+    jobs.concat(load_yml_contents('cloud_activation')) if should_run?('test_instance_all_cloud_activation')
+    jobs.concat(load_yml_contents('registry_with_cdn')) if should_run?('test_integration_registrywithcdn')
+    jobs.concat(load_yml_contents('staging'))
+
+    # Disabling geo jobs temporarily due to https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/774
+    # base.concat(load_yml_contents('geo')) if should_run?('scenario_test_geo')
+
+    jobs
+  end
+  # rubocop:enable Metrics/AbcSize
+  # rubocop:enable Metrics/PerceivedComplexity
+  # rubocop:enable Metrics/CyclomaticComplexity
+
+  private
+
+  def should_run?(example_file_name)
+    @no_of_example_files.include?(example_file_name)
+  end
+
+  def load_yml_contents(file_prefix)
+    jobs_dir_path = File.expand_path('../.gitlab/ci/jobs', __dir__)
+    File.read(File.join(jobs_dir_path, "#{file_prefix}.gitlab-ci.yml"))
+  end
+end
+
+jobs = GenerateQAJobs.new(Dir.glob('no_of_examples/*').map { |s| File.basename(s, '.*') }).execute
+
+File.open('generated-qa-jobs.yml', 'w') { |f| f.write(jobs) }

data/support/data/admin_access_token_seed.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+class AdminAccessTokenSeed
+  TOKEN_VALUE = 'yF4CpxN55A1V0sHMuYQa'
+  SCOPES = Gitlab::Auth.all_available_scopes
+
+  def self.seed!
+    admin_user = User.find_by(username: 'root')
+
+    token_params = {
+      scopes: SCOPES.map(&:to_s),
+      name: 'admin-api-token'
+    }
+
+    admin_user.personal_access_tokens.build(token_params).tap do |pat|
+      pat.set_token(TOKEN_VALUE)
+      pat.save!
+    end
+
+    puts 'Personal access token seeded for root user.'
+  end
+end
+
+AdminAccessTokenSeed.seed!

data/support/data/license_usage_seed.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+# rubocop:disable Metrics/AbcSize
+
+class LicenseUsageSeed
+  def self.seed!
+    admin_user = User.find_by(username: 'root')
+    group_count = seed_groups.count
+    user_count = seed_users.count
+
+    puts 'Start seeding license usage data...'
+
+    rand(5..20).times { create_group(admin_user) }
+    create_users_and_members
+
+    puts 'Creating License usage record...'
+    create_license_usage_record
+
+    puts "Created #{seed_users.count - user_count} users and #{seed_groups.count - group_count} groups."
+    puts 'License usage data seeding completed.'
+  end
+
+  def self.create_user
+    name = "test-user#{SecureRandom.hex(8)}"
+
+    User.create!(
+      email: "#{name}@test.com",
+      password: 'password',
+      username: name,
+      name: "User #{name}",
+      confirmed_at: Time.current
+    )
+  end
+
+  def self.create_group(user)
+    name = "test-group#{SecureRandom.hex(8)}"
+    group_params =
+      {
+        name: name,
+        path: name
+      }
+    ::Groups::CreateService.new(user, group_params).execute
+  end
+
+  def self.create_users_and_members
+    seed_groups.pluck(:id).each do |group_id|
+      3.times { create_member(create_user, group_id) }
+    end
+  end
+
+  def self.create_member(user, group_id)
+    roles = Gitlab::Access.values
+
+    GroupMember.create(user_id: user.id, access_level: roles.sample, source_id: group_id)
+  end
+
+  def self.seed_users
+    User.where('username ~* ?', '^test-user')
+  end
+
+  def self.seed_groups
+    Group.where('name ~* ?', '^test-group')
+  end
+
+  def self.create_license_usage_record
+    # Force update daily billable users and historical license data
+    identifier = Analytics::UsageTrends::Measurement.identifiers[:billable_users]
+    ::Analytics::UsageTrends::CounterJobWorker.new.perform(identifier, User.minimum(:id), User.maximum(:id), Time.zone.now)
+
+    HistoricalData.track!
+  end
+end
+
+LicenseUsageSeed.seed!
+
+# rubocop:enable Metrics/AbcSize

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 7.24.6
+  version: 7.26.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-03-30 00:00:00.000000000 Z
+date: 2022-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -218,6 +218,42 @@ files:
 - ".dockerignore"
 - ".gitignore"
 - ".gitlab-ci.yml"
+- ".gitlab/ci/jobs/actioncable.gitlab-ci.yml"
+- ".gitlab/ci/jobs/base.gitlab-ci.yml"
+- ".gitlab/ci/jobs/cloud_activation.gitlab-ci.yml"
+- ".gitlab/ci/jobs/custom_parallel.gitlab-ci.yml"
+- ".gitlab/ci/jobs/ee_previous_to_ce_update.gitlab-ci.yml"
+- ".gitlab/ci/jobs/elasticsearch.gitlab-ci.yml"
+- ".gitlab/ci/jobs/geo.gitlab-ci.yml"
+- ".gitlab/ci/jobs/gitaly_cluster.gitlab-ci.yml"
+- ".gitlab/ci/jobs/group_saml.gitlab-ci.yml"
+- ".gitlab/ci/jobs/instance.gitlab-ci.yml"
+- ".gitlab/ci/jobs/instance_saml.gitlab-ci.yml"
+- ".gitlab/ci/jobs/integrations.gitlab-ci.yml"
+- ".gitlab/ci/jobs/jira.gitlab-ci.yml"
+- ".gitlab/ci/jobs/large_setup.gitlab-ci.yml"
+- ".gitlab/ci/jobs/ldap_no_server.gitlab-ci.yml"
+- ".gitlab/ci/jobs/ldap_no_tls.gitlab-ci.yml"
+- ".gitlab/ci/jobs/ldap_tls.gitlab-ci.yml"
+- ".gitlab/ci/jobs/mattermost.gitlab-ci.yml"
+- ".gitlab/ci/jobs/mtls.gitlab-ci.yml"
+- ".gitlab/ci/jobs/object_storage.gitlab-ci.yml"
+- ".gitlab/ci/jobs/object_storage_aws.gitlab-ci.yml"
+- ".gitlab/ci/jobs/object_storage_gcs.gitlab-ci.yml"
+- ".gitlab/ci/jobs/object_storage_registry_tls.gitlab-ci.yml"
+- ".gitlab/ci/jobs/omnibus_image.gitlab-ci.yml"
+- ".gitlab/ci/jobs/omnibus_upgrade.gitlab-ci.yml"
+- ".gitlab/ci/jobs/packages.gitlab-ci.yml"
+- ".gitlab/ci/jobs/praefect.gitlab-ci.yml"
+- ".gitlab/ci/jobs/registry.gitlab-ci.yml"
+- ".gitlab/ci/jobs/registry_with_cdn.gitlab-ci.yml"
+- ".gitlab/ci/jobs/relative_url.gitlab-ci.yml"
+- ".gitlab/ci/jobs/repository_storage.gitlab-ci.yml"
+- ".gitlab/ci/jobs/sanity_framework.gitlab-ci.yml"
+- ".gitlab/ci/jobs/service_ping_disabled.gitlab-ci.yml"
+- ".gitlab/ci/jobs/smtp.gitlab-ci.yml"
+- ".gitlab/ci/jobs/staging.gitlab-ci.yml"
+- ".gitlab/ci/jobs/update.gitlab-ci.yml"
 - ".gitlab/ci/rules.gitlab-ci.yml"
 - ".gitlab/issue_templates/Default.md"
 - ".gitlab/merge_request_templates/Default.md"
@@ -235,7 +271,7 @@ files:
 - bin/expect_exit_code_and_text
 - bin/gen-cert.sh
 - bin/merge_html_reports
-- bin/notify_upstream_commit
+- bin/notify_upstream
 - bin/setup
 - bin/slack
 - docs/README.md
@@ -299,6 +335,7 @@ files:
 - lib/gitlab/qa/runtime/env.rb
 - lib/gitlab/qa/runtime/logger.rb
 - lib/gitlab/qa/runtime/omnibus_configuration.rb
+- lib/gitlab/qa/runtime/omnibus_configurations/ci_decomposition.rb
 - lib/gitlab/qa/runtime/omnibus_configurations/default.rb
 - lib/gitlab/qa/runtime/omnibus_configurations/license_mode.rb
 - lib/gitlab/qa/runtime/omnibus_configurations/object_storage.rb
@@ -325,6 +362,7 @@ files:
 - lib/gitlab/qa/scenario/test/instance/staging.rb
 - lib/gitlab/qa/scenario/test/instance/staging_geo.rb
 - lib/gitlab/qa/scenario/test/instance/staging_ref.rb
+- lib/gitlab/qa/scenario/test/instance/staging_ref_geo.rb
 - lib/gitlab/qa/scenario/test/integration/actioncable.rb
 - lib/gitlab/qa/scenario/test/integration/client_ssl.rb
 - lib/gitlab/qa/scenario/test/integration/elasticsearch.rb
@@ -344,6 +382,7 @@ files:
 - lib/gitlab/qa/scenario/test/integration/praefect.rb
 - lib/gitlab/qa/scenario/test/integration/registry.rb
 - lib/gitlab/qa/scenario/test/integration/registry_tls.rb
+- lib/gitlab/qa/scenario/test/integration/registry_with_cdn.rb
 - lib/gitlab/qa/scenario/test/integration/saml.rb
 - lib/gitlab/qa/scenario/test/integration/service_ping_disabled.rb
 - lib/gitlab/qa/scenario/test/integration/smtp.rb
@@ -359,6 +398,9 @@ files:
 - lib/gitlab/qa/support/invalid_response_error.rb
 - lib/gitlab/qa/test_logger.rb
 - lib/gitlab/qa/version.rb
+- scripts/generate-qa-jobs.rb
+- support/data/admin_access_token_seed.rb
+- support/data/license_usage_seed.rb
 - tls_certificates/authority/ca.crt
 - tls_certificates/authority/ca.key
 - tls_certificates/authority/ca.pem

data/bin/notify_upstream_commit

@@ -1,61 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'gitlab'
-
-# Default to "Multi-pipeline (from 'gitlab-org/gitlab-qa' 'notify_upstream_commit:*' jobs)" from current project
-api_token = ENV.fetch('GITLAB_QA_PROJECT_ACCESS_TOKEN') do
-  puts "The $GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN variable is deprecated in favor of a dedicated project access token: $GITLAB_QA_PROJECT_ACCESS_TOKEN."
-  puts "See https://gitlab.com/groups/gitlab-org/quality/-/epics/17 for more details."
-  ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
-end
-
-# Configure credentials to be used with gitlab gem
-Gitlab.configure do |config|
-  config.endpoint = 'https://gitlab.com/api/v4'
-  config.private_token = api_token
-end
-
-class CommitComment
-  def self.post!(status)
-    unless ENV['TOP_UPSTREAM_SOURCE_SHA']
-      puts "The 'TOP_UPSTREAM_SOURCE_SHA' environment variable is missing, cannot post a comment on a missing upstream commit."
-      return
-    end
-
-    top_upstream_source_sha = ENV['TOP_UPSTREAM_SOURCE_SHA']
-
-    unless ENV['TOP_UPSTREAM_SOURCE_PROJECT']
-      puts "The 'TOP_UPSTREAM_SOURCE_PROJECT' environment variable is missing, cannot post a comment on the upstream #{top_upstream_source_sha} commit."
-      return
-    end
-
-    top_upstream_source_project = ENV['TOP_UPSTREAM_SOURCE_PROJECT']
-
-    status_with_icon =
-      case status
-      when :success
-        "passed. :white_check_mark:"
-      when :failure
-        "failed! :boom:"
-      end
-
-    # The HTML report can't be opened as a web page so we're forced to
-    # download it. See https://gitlab.com/gitlab-org/gitlab/issues/25192
-    Gitlab.create_commit_comment(
-      top_upstream_source_project,
-      top_upstream_source_sha,
-      <<~COMMENT
-        The [`gitlab-qa` downstream pipeline](#{ENV['CI_PIPELINE_URL']}) #{status_with_icon}.
-      COMMENT
-    )
-  rescue Gitlab::Error::Error => error
-    puts "Ignoring the following error: #{error}"
-  end
-end
-
-status = ARGV.shift.to_s.strip
-
-abort "Please provide a status!" if status == ''
-
-CommitComment.post!(status.to_sym)