gitlab-qa 8.0.0 → 8.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0982270bb1256eb950e30c84028a70fca0663c744fff1bf00cb1937f89936f43'
-  data.tar.gz: c2051a27b8027df5d47359a6b27a54338e25bfc37471dae2fe600087e118b392
+  metadata.gz: f11ebd2677dc7ba62bd4b55efc03ad8d9560c38a3e601897a9a201d3c09c8eae
+  data.tar.gz: f8db94175a79e62ae625e8b975fc49b03c4077ff608f40a85fa7ea7a5b3f859d
 SHA512:
-  metadata.gz: c11dd80b24b285fcd8b93c94c0fabac93be215576eb307804c485ad9a5dbba4a7b6cd9f6b016f13082ba4c146f8189a2e7f6bf9b41ef85172a735696e2e558af
-  data.tar.gz: 0cf57dcfa1f30e37f2acc66015ed03c0d2e3d2bd65284f7be92d71b1028fadbceb9f72feaa7a4dad29905d7554a9f985885b6d95ca096944c1a955ffa0ac244d
+  metadata.gz: 21199373f3dff6e7e9204062fec4e204309e33d3bd50994285691aa8b833e926262ac9359810ba5f0cc1a624168693a7e4f195e3abc819088b5bc2f6736f4d07
+  data.tar.gz: a7dc574dceebed5e61b2383030cd114df7cabd00c974d5e548bed6364a031d6617f8495716fc8b7eb6043b6ad33eb8db33adc0f7cc7b0125871dca8a704c6f26

data/.gitlab/ci/jobs/cloud_activation.gitlab-ci.yml

@@ -1,7 +1,4 @@
 ee:cloud-activation:
-  before_script:
-    - unset EE_LICENSE
-    - !reference [default, before_script]
   extends:
     - .rules:ee-never-when-triggered-by-feature-flag-definition-change
     - .test
@@ -14,9 +11,6 @@ ee:cloud-activation:
     GITLAB_QA_OPTIONS_COMBINED: "$GITLAB_QA_OPTIONS --seed-db license*.rb"
 
 ee:cloud-activation-quarantine:
-  before_script:
-    - unset EE_LICENSE
-    - !reference [default, before_script]
   extends:
     - .rules:ee-never-when-triggered-by-feature-flag-definition-change
     - .test

data/.gitlab/ci/jobs/metrics.gitlab-ci.yml

@@ -0,0 +1,41 @@
+ce:metrics:
+  extends:
+    - .rules:ce-never-when-triggered-by-feature-flag-definition-change
+    - .test
+    - .high-capacity
+    - .ce-variables
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::Metrics"
+
+ce:metrics-quarantine:
+  extends:
+    - .rules:ce-never-when-triggered-by-feature-flag-definition-change
+    - .test
+    - .high-capacity
+    - .ce-variables
+    - .quarantine
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::Metrics"
+
+ee:metrics:
+  extends:
+    - .rules:ee-never-when-triggered-by-feature-flag-definition-change
+    - .test
+    - .high-capacity
+    - .ee-variables
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::Metrics"
+
+ee:metrics-quarantine:
+  extends:
+    - .rules:ee-never-when-triggered-by-feature-flag-definition-change
+    - .test
+    - .high-capacity
+    - .ee-variables
+    - .quarantine
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::Metrics"

data/.gitlab/ci/jobs/opensearch.gitlab-ci.yml

@@ -0,0 +1,20 @@
+ee:opensearch:
+  extends:
+    - .rules:ee-never-when-triggered-by-feature-flag-definition-change
+    - .test
+    - .high-capacity
+    - .ee-variables
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::Opensearch"
+
+ee:opensearch-quarantine:
+  extends:
+    - .rules:ee-never-when-triggered-by-feature-flag-definition-change
+    - .test
+    - .high-capacity
+    - .ee-variables
+    - .quarantine
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::Opensearch"

data/.gitlab-ci.yml

@@ -105,6 +105,7 @@ generate-jobs:
     - bundle exec bin/qa Test::Instance::All http://dummy.test --count-examples-only -- $QA_TESTS --tag can_use_large_setup
     - bundle exec bin/qa Test::Instance::All http://dummy.test --count-examples-only -- $QA_TESTS --tag cloud_activation
     - bundle exec bin/qa Test::Instance::All http://dummy.test --count-examples-only -- $QA_TESTS --tag object_storage
+    - bundle exec bin/qa Test::Instance::All http://dummy.test --count-examples-only -- $QA_TESTS --tag metrics
     - cp -r no_of_examples $CI_PROJECT_DIR
 
     # TOP_UPSTREAM_DEFAULT_BRANCH is the default branch name of the original project that triggered a pipeline in this project.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-qa (8.0.0)
+    gitlab-qa (8.3.0)
       activesupport (~> 6.1)
       gitlab (~> 4.18.0)
       http (~> 5.0)
@@ -13,7 +13,7 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.6)
+    activesupport (6.1.6.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -72,11 +72,11 @@ GEM
     faraday-em_http (1.0.0)
     faraday-em_synchrony (1.0.0)
     faraday-excon (1.1.0)
-    faraday-http-cache (2.2.0)
+    faraday-http-cache (2.4.0)
       faraday (>= 0.8)
     faraday-httpclient (1.0.1)
-    faraday-multipart (1.0.3)
-      multipart-post (>= 1.2, < 3)
+    faraday-multipart (1.0.4)
+      multipart-post (~> 2)
     faraday-net_http (1.0.1)
     faraday-net_http_persistent (1.2.0)
     faraday-patron (1.0.0)
@@ -101,7 +101,7 @@ GEM
       rubocop-rails (~> 2.9)
       rubocop-rspec (~> 1.44)
     hashdiff (1.0.1)
-    http (5.0.4)
+    http (5.1.0)
       addressable (~> 2.8)
       http-cookie (~> 1.0)
       http-form_data (~> 2.2)
@@ -112,7 +112,7 @@ GEM
     httparty (0.20.0)
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
-    i18n (1.10.0)
+    i18n (1.12.0)
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.4)
     kramdown (2.4.0)
@@ -127,17 +127,17 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2022.0105)
     mini_portile2 (2.8.0)
-    minitest (5.15.0)
+    minitest (5.16.2)
     multi_xml (0.6.0)
-    multipart-post (2.1.1)
+    multipart-post (2.2.3)
     nap (1.1.0)
     no_proxy_fix (0.1.2)
-    nokogiri (1.13.6)
+    nokogiri (1.13.8)
       mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
-    octokit (4.22.0)
-      faraday (>= 0.9)
-      sawyer (~> 0.8.0, >= 0.5.3)
+    octokit (4.25.1)
+      faraday (>= 1, < 3)
+      sawyer (~> 0.9)
     open4 (1.3.4)
     parallel (1.22.1)
     parser (3.1.2.0)
@@ -147,11 +147,11 @@ GEM
       method_source (~> 1.0)
     public_suffix (4.0.7)
     racc (1.6.0)
-    rack (2.2.3)
+    rack (2.2.4)
     rainbow (3.0.0)
     rake (13.0.6)
     rchardet (1.8.0)
-    regexp_parser (2.4.0)
+    regexp_parser (2.5.0)
     reverse_markdown (2.1.1)
       nokogiri
     rexml (3.2.5)
@@ -177,7 +177,7 @@ GEM
       rubocop-ast (>= 0.6.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (1.18.0)
+    rubocop-ast (1.19.1)
       parser (>= 3.1.1.0)
     rubocop-gitlab-security (0.1.1)
       rubocop (>= 0.51)
@@ -193,9 +193,9 @@ GEM
       rubocop-ast (>= 0.7.1)
     ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.5)
-    sawyer (0.8.2)
+    sawyer (0.9.2)
       addressable (>= 2.3.5)
-      faraday (> 0.8, < 2.0)
+      faraday (>= 0.17.3, < 3)
     solargraph (0.45.0)
       backport (~> 1.2)
       benchmark
@@ -215,8 +215,8 @@ GEM
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
     thor (1.2.1)
-    tilt (2.0.10)
-    tzinfo (2.0.4)
+    tilt (2.0.11)
+    tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)
     unf (0.1.4)
       unf_ext
@@ -227,9 +227,9 @@ GEM
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
     webrick (1.7.0)
-    yard (0.9.27)
+    yard (0.9.28)
       webrick (~> 1.7.0)
-    zeitwerk (2.5.4)
+    zeitwerk (2.6.0)
 
 PLATFORMS
   ruby

data/docs/release_process.md

@@ -23,6 +23,7 @@ when we make a change - no matter the size of the change.
   - If there is one, update it if necessary.
   - If not, update [`lib/gitlab/qa/version.rb`] to an appropriate [semantic version](https://semver.org) in a new merge request using the [release template](https://gitlab.com/gitlab-org/gitlab-qa/blob/master/.gitlab/merge_request_templates/Release.md)
     and title the MR like `"Bump version to <version>"`.
+    - Run `bundle update` to update the `Gemfile.lock` with the new version number of `gitlab-qa`.
 - Merge the merge request.
 - The new version should automatically be tagged and pushed to Rubygems by the `gem-release` job in the merge commit pipeline.
 - Update the release notes for the newly created tag (https://gitlab.com/gitlab-org/gitlab-qa/-/tags):

data/docs/trainings.md

@@ -1,5 +1,7 @@
 # GitLab QA trainings
 
+The handbook has [information about the entire test automation framework](https://about.gitlab.com/handbook/engineering/quality/#test-automation-framework), including GitLab QA. There you can find links to additional documentation and videos.
+
 ## Build team training on GitLab QA
 
 A great introduction to the GitLab QA project, its internal structure, how it's

data/docs/what_tests_can_be_run.md

@@ -84,6 +84,7 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `QA_INFLUXDB_TOKEN` |- | Influxdb token for test metrics reporting | No|
 | `QA_RUN_TYPE` |- | QA run type like `staging-full`, `canary`, `production` etc. Used in test metrics reporting | No|
 | `QA_VALIDATE_RESOURCE_REUSE` | `false` | Set to `true` to [validate resource reuse](https://docs.gitlab.com/ee/development/testing_guide/end_to_end/resources.html#reusable-resources) after a test suite | No |
+| `QA_USE_PUBLIC_IP_API` | `false` | When performing Instance tests against a remote/pre-existing instance, use an API to detect the public API for requests coming from gitlab-qa.  Used by tests that exercise IP-address restrictions and similar
 | `GITHUB_USERNAME` |-  | Username for authenticating with GitHub. | No|
 | `GITHUB_PASSWORD` |-  | Password for authenticating with GitHub. | No|
 | `GITLAB_QA_LOOP_RUNNER_MINUTES` | `1` | Minutes to run and repeat a spec while using the '--loop' option; default value is 1 minute. | No|
@@ -98,6 +99,7 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `GEO_FAILOVER` | `false` | Set to `true` when a Geo secondary site has been promoted to a Geo primary site. | No|
 | `GITLAB_INITIAL_ROOT_PASSWORD` | `5iveL!fe` | Initial root password for Omnibus installations | No|
 | `COLORIZED_LOGS` | `false` | Colors GitLab QA and test logs to improve readability | No|
+| `QA_DOCKER_ADD_HOSTS` |- | Comma separated list of hosts to add to /etc/hosts in docker container | No|
 
 ## [Supported Remote Grid environment variables](./running_against_remote_grid.md)
 

data/lib/gitlab/qa/component/base.rb

@@ -7,13 +7,14 @@ module Gitlab
         include Scenario::Actable
 
         attr_reader :docker
-        attr_accessor :volumes, :network, :environment, :runner_network
+        attr_accessor :volumes, :ports, :network, :environment, :runner_network
         attr_writer :name, :exec_commands
 
         def initialize
           @docker = Docker::Engine.new
           @environment = {}
           @volumes = {}
+          @ports = []
           @network_aliases = []
           @exec_commands = []
         end
@@ -85,6 +86,10 @@ module Gitlab
             command << "--net #{network}"
             command << "--hostname #{hostname}"
 
+            @ports.each do |mapping|
+              command.port(mapping)
+            end
+
             @volumes.to_h.each do |to, from|
               command.volume(to, from, 'Z')
             end

data/lib/gitlab/qa/component/gitlab.rb

@@ -7,6 +7,7 @@ require 'uri'
 require 'forwardable'
 require 'openssl'
 require 'tempfile'
+require 'json'
 
 module Gitlab
   module QA
@@ -16,7 +17,7 @@ module Gitlab
         using Rainbow
 
         attr_reader :release, :omnibus_configuration, :omnibus_gitlab_rails_env
-        attr_accessor :tls, :skip_availability_check, :runner_network, :seed_admin_token, :seed_db
+        attr_accessor :tls, :skip_availability_check, :runner_network, :seed_admin_token, :seed_db, :skip_server_hooks
         attr_writer :name, :relative_path
 
         def_delegators :release, :tag, :image, :edition
@@ -46,6 +47,7 @@ module Gitlab
 
           @seed_admin_token = Runtime::Scenario.seed_admin_token
           @seed_db = Runtime::Scenario.seed_db
+          @skip_server_hooks = Runtime::Scenario.skip_server_hooks
 
           self.release = 'CE'
         end
@@ -83,8 +85,8 @@ module Gitlab
           tls ? 'https' : 'http'
         end
 
-        def port
-          tls ? '443:443' : '80'
+        def gitlab_port
+          tls ? ["443:443"] : ["80"]
         end
 
         def gitaly_tls
@@ -119,6 +121,10 @@ module Gitlab
           super
         end
 
+        def exist?(image, tag)
+          docker.manifest_exists?("#{image}:#{tag}")
+        end
+
         def prepare_gitlab_omnibus_config
           set_formless_login_token
           set_license_mode
@@ -131,11 +137,15 @@ module Gitlab
           ensure_configured!
 
           docker.run(image: image, tag: tag) do |command|
-            command << "-d -p #{port}"
+            command << "-d"
             command << "--name #{name}"
             command << "--net #{network}"
             command << "--hostname #{hostname}"
 
+            [*@ports, *gitlab_port].each do |mapping|
+              command.port(mapping)
+            end
+
             @volumes.to_h.each do |to, from|
               command.volume(to, from, 'Z')
             end
@@ -174,7 +184,7 @@ module Gitlab
             name,
             relative_path: relative_path,
             scheme: scheme,
-            protocol_port: port.to_i
+            protocol_port: gitlab_port.first.to_i
           )
 
           Runtime::Logger.info("Waiting for GitLab to become healthy ...")
@@ -192,7 +202,7 @@ module Gitlab
           exec_commands << seed_admin_token_command if seed_admin_token
           exec_commands << seed_test_data_command if seed_db
           exec_commands << Runtime::Scenario.omnibus_exec_commands
-          exec_commands << add_git_server_hooks unless Runtime::Scenario.skip_server_hooks
+          exec_commands << add_git_server_hooks unless skip_server_hooks
 
           commands = exec_commands.flatten.uniq
           return if commands.empty?
@@ -202,18 +212,18 @@ module Gitlab
         end
 
         def rails_version
-          json = @docker.read_file(
-            @release.image, @release.tag,
-            '/opt/gitlab/version-manifest.json'
-          )
-
-          manifest = JSON.parse(json)
+          manifest = JSON.parse(read_package_manifest)
           {
             sha: manifest['software']['gitlab-rails']['locked_version'],
             source: manifest['software']['gitlab-rails']['locked_source']['git']
           }
         end
 
+        def package_version
+          manifest = JSON.parse(read_package_manifest)
+          manifest['software']['package-scripts']['locked_version']
+        end
+
         def copy_key_file(env_key)
           key_dir = ENV['CI_PROJECT_DIR'] || Dir.tmpdir
           key_file = Tempfile.new(env_key.downcase, key_dir)
@@ -229,6 +239,13 @@ module Gitlab
 
         private
 
+        def read_package_manifest
+          @docker.read_file(
+            @release.image, @release.tag,
+            '/opt/gitlab/version-manifest.json'
+          )
+        end
+
         # Copy certs to a temporary directory in current working directory.
         # This is needed for docker-in-docker ci environments where mount points outside of build dir are not accessible
         #

data/lib/gitlab/qa/component/opensearch.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Component
+      class Opensearch < Base
+        DOCKER_IMAGE = 'opensearchproject/opensearch'
+
+        def name
+          @name ||= "elastic68"
+        end
+
+        def tag
+          Runtime::Env.opensearch_version
+        end
+
+        def start
+          @docker.run(image: image, tag: tag) do |command|
+            command << "-d"
+            command << "--name #{name}"
+            command << "--net #{network}"
+            command << "--publish 9200:9200"
+            command << "--publish 9300:9300"
+
+            command.env("discovery.type", "single-node")
+            command.env("ES_JAVA_OPTS", "-Xms512m -Xmx512m")
+            command.env("plugins.security.disabled", "true")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/component/specs.rb

@@ -23,9 +23,7 @@ module Gitlab
 
           raise ArgumentError unless [suite, release].all?
 
-          @docker.login(**release.login_params) if release.login_params
-
-          @docker.pull(image: qa_image) unless Runtime::Env.skip_pull?
+          docker_pull_qa_image_if_needed
 
           Runtime::Logger.info("Running test suite `#{suite}` for #{release.project_name}")
 
@@ -52,6 +50,11 @@ module Gitlab
                 command.env('QA_HOSTNAME', hostname)
               end
 
+              unless Runtime::Env.docker_add_hosts.empty?
+                hosts = Runtime::Env.docker_add_hosts.map { |host| "--add-host=#{host} " }.join
+                command << hosts # override /etc/hosts in docker container when test runs
+              end
+
               env.merge(Runtime::Env.variables).each do |key, value|
                 command.env(key, value)
               end
@@ -70,6 +73,12 @@ module Gitlab
 
         private
 
+        def docker_pull_qa_image_if_needed
+          @docker.login(**release.login_params) if release.login_params
+
+          @docker.pull(image: qa_image) unless Runtime::Env.skip_pull?
+        end
+
         def args_with_flags(args, feature_flag_set)
           return args if feature_flag_set.empty?
 

data/lib/gitlab/qa/docker/command.rb

@@ -33,6 +33,10 @@ module Gitlab
           tap { @args.push(%(--env #{name}="#{value}")) }
         end
 
+        def port(mapping)
+          tap { @args.push("-p #{mapping}") }
+        end
+
         def to_s
           "docker #{@args.join(' ')}"
         end

data/lib/gitlab/qa/docker/engine.rb

@@ -110,6 +110,14 @@ module Gitlab
           Docker::Command.execute("rm -f #{name}")
         end
 
+        def manifest_exists?(name)
+          Docker::Command.execute("manifest inspect #{name}")
+        rescue Docker::Shellout::StatusError
+          false
+        else
+          true
+        end
+
         def container_exists?(name)
           Docker::Command.execute("container inspect #{name}")
         rescue Docker::Shellout::StatusError

data/lib/gitlab/qa/runtime/env.rb

@@ -36,7 +36,6 @@ module Gitlab
           'QA_SLOW_CONNECTION_LATENCY_MS' => :qa_slow_connection_latency_ms,
           'QA_SLOW_CONNECTION_THROUGHPUT_KBPS' => :qa_slow_connection_throughput_kbps,
           'QA_GENERATE_ALLURE_REPORT' => :generate_allure_report,
-          'QA_EE_ACTIVATION_CODE' => :ee_activation_code,
           'QA_EXPORT_TEST_METRICS' => :qa_export_test_metrics,
           'QA_INFLUXDB_URL' => :qa_influxdb_url,
           'QA_INFLUXDB_TOKEN' => :qa_influxdb_token,
@@ -232,6 +231,10 @@ module Gitlab
           env_var_value_if_defined('ELASTIC_VERSION') || '8.2.0'
         end
 
+        def opensearch_version
+          env_var_value_if_defined('OPENSEARCH_VERSION') || '2.0.1'
+        end
+
         def require_license!
           return if ENV.include?('EE_LICENSE')
 
@@ -357,6 +360,10 @@ module Gitlab
           "#{ci_project_name}-#{test_subset}"
         end
 
+        def docker_add_hosts
+          (env_var_value_if_defined('QA_DOCKER_ADD_HOSTS') || '').split(',')
+        end
+
         private
 
         def enabled?(value, default: true)

data/lib/gitlab/qa/scenario/test/integration/metrics.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Scenario
+      module Test
+        module Integration
+          class Metrics < Scenario::Template
+            PUMA_METRICS_SERVER_PORT = 8083
+            SIDEKIQ_METRICS_SERVER_PORT = 8082
+
+            def perform(release, *rspec_args)
+              Component::Gitlab.perform do |gitlab|
+                gitlab.release = release
+                gitlab.network = 'test'
+                gitlab.name = 'gitlab'
+                gitlab.ports = [PUMA_METRICS_SERVER_PORT, SIDEKIQ_METRICS_SERVER_PORT]
+
+                gitlab.omnibus_configuration << <<~RUBY
+                  puma['exporter_enabled'] = true
+                  puma['exporter_address'] = '0.0.0.0'
+                  puma['exporter_port'] = #{PUMA_METRICS_SERVER_PORT}
+                  sidekiq['metrics_enabled'] = true
+                  sidekiq['listen_address'] = '0.0.0.0'
+                  sidekiq['listen_port'] = #{SIDEKIQ_METRICS_SERVER_PORT}
+                RUBY
+
+                gitlab.instance do
+                  Component::Specs.perform do |specs|
+                    specs.suite = 'Test::Integration::Metrics'
+                    specs.release = gitlab.release
+                    specs.network = gitlab.network
+                    specs.args = [gitlab.address, *rspec_args]
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/scenario/test/integration/opensearch.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Scenario
+      module Test
+        module Integration
+          class Opensearch < Scenario::Template
+            attr_reader :gitlab_name, :spec_suite
+
+            def initialize
+              @gitlab_name = 'gitlab-opensearch'
+              # Currently the test suite that tests Advanced Search features is called 'Elasticsearch' which we hope to abstract to 'Advancedsearch' in the future
+              @spec_suite = 'QA::EE::Scenario::Test::Integration::Elasticsearch'
+            end
+
+            def before_perform(release)
+              raise ArgumentError, 'OpenSearch is an EE only feature!' unless release.ee?
+            end
+
+            def perform(release, *rspec_args)
+              release = QA::Release.new(release)
+              before_perform(release)
+
+              Component::Gitlab.perform do |gitlab|
+                gitlab.release = release
+                setup_opensearch_on gitlab
+
+                Component::Opensearch.perform do |opensearch|
+                  opensearch.network = 'test'
+                  opensearch.instance do
+                    gitlab.instance do
+                      Runtime::Logger.info("Running #{spec_suite} specs!")
+
+                      Component::Specs.perform do |specs|
+                        specs.suite = spec_suite
+                        specs.release = gitlab.release
+                        specs.network = gitlab.network
+                        specs.args = [gitlab.address, *rspec_args]
+                      end
+                    end
+                  end
+                end
+              end
+            end
+
+            def empty_index
+              @empty_index ||= ["gitlab-rake gitlab:elastic:create_empty_index"]
+            end
+
+            def setup_opensearch_on(instance)
+              instance.name = gitlab_name
+              instance.network = 'test'
+              instance.elastic_url = "http://elastic68:9200"
+              instance.exec_commands = empty_index
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/scenario/test/omnibus/update.rb

@@ -10,8 +10,44 @@ module Gitlab
         module Omnibus
           class Update < Scenario::Template
             def perform(from_release, to_release = nil, *rspec_args)
-              previous_release = QA::Release.new(from_release).previous_stable
-              current_release = QA::Release.new(to_release || from_release)
+              version = Component::Gitlab.perform do |gitlab|
+                gitlab.release = from_release
+                gitlab.act do
+                  pull
+                  package_version
+                end
+              end
+
+              if version
+                # Sub any plus symbols in version eg. "15.1.2+ce.0"
+                version.tr!('+', '-')
+                Runtime::Logger.info("Found previous version '#{version}'")
+
+                current_release = QA::Release.new(to_release || from_release)
+
+                type = current_release.ee? ? 'ee' : 'ce'
+
+                existing_previous = Component::Gitlab.perform do |gitlab|
+                  gitlab.act do
+                    next "gitlab/gitlab-#{type}:#{version}" if exist?("gitlab/gitlab-#{type}", version)
+
+                    next "gitlab/gitlab-#{type}:#{version}-#{type}.0" if exist?("gitlab/gitlab-#{type}", "#{version}-#{type}.0")
+
+                    nil
+                  end
+                end
+              else
+                Runtime::Logger.info("Could not find previous image version")
+                existing_previous = nil
+              end
+
+              previous_release = if existing_previous
+                                   Runtime::Logger.info("Using previous image '#{existing_previous}'")
+                                   QA::Release.new(existing_previous)
+                                 else
+                                   Runtime::Logger.info("Using stable as previous image")
+                                   QA::Release.new(from_release).previous_stable
+                                 end
 
               Docker::Volumes.new.with_temporary_volumes do |volumes|
                 Component::Gitlab.perform do |gitlab|

data/lib/gitlab/qa/scenario/test/omnibus/update_from_previous.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Scenario
+      module Test
+        module Omnibus
+          class UpdateFromPrevious < Scenario::Template
+            using Rainbow
+
+            # Test update from N - 1 (major|minor|patch) version to current release
+            # Run smoke test suite on previous release to populate some data in database before update
+            #
+            # @example
+            # perform(gitlab-ee:dev-tag, 15.3.0-pre, major)
+            # => will perform upgrades 14.9.5 -> 15.0.5 -> gitlab-ee:dev-tag
+            #
+            # @param [String] release current release docker image
+            # @param [String] current_version current gitlab version associated with docker image
+            # @param [String] semver_component semver component for N - 1 version detection, major|minor|patch
+            # @param [Array] *rspec_args rspec arguments
+            # @return [void]
+            def perform(release, current_version, semver_component, *rspec_args)
+              @current_release = QA::Release.new(release)
+              @upgrade_path = Support::GitlabUpgradePath.new(
+                current_version,
+                semver_component,
+                @current_release.edition
+              ).fetch
+
+              upgrade_info = "#{[*upgrade_path, current_release].join(' => ')} (#{current_version})".bright
+              Runtime::Logger.info("Performing gitlab update: #{upgrade_info}")
+
+              update(rspec_args)
+            end
+
+            private
+
+            attr_reader :current_release, :upgrade_path
+
+            # Perform update
+            #
+            # @param [Array] rspec_args
+            # @return [void]
+            def update(rspec_args)
+              Docker::Volumes.new.with_temporary_volumes do |volumes|
+                # deploy first release in upgrade path and run specs to populate db
+                run_gitlab(upgrade_path.first, volumes, ["--", "--tag", "smoke"])
+
+                # deploy releases in upgrade path
+                upgrade_path[1..].each { |release| run_gitlab(release, volumes, skip_setup: true) }
+
+                # deploy current release and run tests
+                run_gitlab(current_release, volumes, rspec_args, skip_setup: true)
+              end
+            end
+
+            # Deploy gitlab instance and optionally run specs
+            #
+            # @param [Gitlab::QA::Release] release
+            # @param [Hash] volumes
+            # @return [void]
+            def run_gitlab(release, volumes, rspec_args = [], skip_setup: false)
+              Runtime::Logger.info("Deploying release: #{release.to_s.bright}")
+
+              Component::Gitlab.perform do |gitlab|
+                gitlab.release = release
+                gitlab.volumes = volumes
+                gitlab.network = 'test'
+
+                if skip_setup
+                  gitlab.skip_server_hooks = true
+                  gitlab.seed_db = false
+                  gitlab.seed_admin_token = false
+                end
+
+                next gitlab.launch_and_teardown_instance unless run_specs?(release)
+
+                gitlab.instance { run_specs(gitlab, release, rspec_args) }
+              end
+            end
+
+            # Run specs
+            #
+            # @param [Gitlab::QA::Component::Gitlab] gitlab
+            # @param [Gitlab::QA::Release] release
+            # @param [Array] rspec_args
+            # @return [void]
+            def run_specs(gitlab, release, rspec_args)
+              Component::Specs.perform do |specs|
+                specs.release = release
+                specs.suite = 'Test::Instance::All'
+                specs.hostname = "qa-e2e-specs.#{gitlab.network}"
+                specs.network = gitlab.network
+                specs.args = [gitlab.address, *rspec_args]
+              end
+            rescue Docker::Shellout::StatusError => e
+              raise e if release == current_release # only fail on current release
+
+              Runtime::Logger.warn("Test run for release '#{gitlab.release}' finished with errors!")
+            end
+
+            # Run specs on first release to populate database and release being tested
+            #
+            # @param [Gitlab::QA::Release] release
+            # @return [Boolean]
+            def run_specs?(release)
+              [upgrade_path.first, current_release].any? { |rel| rel == release }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/support/gitlab_upgrade_path.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/module/delegation"
+require "yaml"
+
+module Gitlab
+  module QA
+    module Support
+      class GitlabUpgradePath
+        # Get upgrade path between N - 1 and current version not including current release
+        #
+        # @param [String] current_version
+        # @param [String] semver_component version number component for previous version detection - major|minor|patch
+        # @param [String] edition GitLab edition - ee or ce
+        def initialize(current_version, semver_component, edition)
+          @version_info = GitlabVersionInfo.new(current_version, edition)
+          @current_version = Gem::Version.new(current_version)
+          @semver_component = semver_component
+          @edition = edition
+          @logger = Runtime::Logger.logger
+        end
+
+        # Get upgrade path between releases
+        #
+        # Return array with only previous version for updates from previous minor, patch versions
+        #
+        # @return [Array<QA::Release>]
+        def fetch
+          return [release(latest_patch(previous_version))] unless major_upgrade?
+
+          # get versions between previous major and current version in gitlab upgrade path
+          path = full_upgrade_path.each_with_object([]) do |ver, arr|
+            next if ver <= previous_version || ver >= current_version
+
+            arr << ver
+          end
+
+          [previous_version, *path].map do |ver|
+            release(version_info.latest_patch(ver))
+          end
+        end
+
+        private
+
+        delegate :latest_patch, to: :version_info
+
+        attr_reader :version_info, :current_version, :semver_component, :edition, :logger
+
+        # Upgrade from previous major
+        #
+        # @return [Boolean]
+        def major_upgrade?
+          semver_component == "major"
+        end
+
+        # Docker release image
+        #
+        # @param [String] version
+        # @return [QA::Release]
+        def release(version)
+          QA::Release.new("gitlab/gitlab-#{edition}:#{version}-#{edition}.0")
+        end
+
+        # Previous gitlab version
+        #
+        # @return [Gem::Version]
+        def previous_version
+          @previous_version ||= version_info.previous_version(semver_component)
+        end
+
+        # Gitlab upgrade path
+        #
+        # @return [Array<Gem::Version>]
+        def full_upgrade_path
+          @full_upgrade_path ||= ::YAML
+            .safe_load(upgrade_path_yml, symbolize_names: true)
+            .map { |version| Gem::Version.new("#{version[:major]}.#{version[:minor]}") }
+        end
+
+        # Upgrade path yml
+        #
+        # @return [String]
+        def upgrade_path_yml
+          @upgrade_path_yml ||= begin
+            logger.info("Fetching gitlab upgrade path from 'gitlab-com/support/toolbox/upgrade-path' project")
+            HttpRequest.make_http_request(
+              url: "https://gitlab.com/gitlab-com/support/toolbox/upgrade-path/-/raw/main/upgrade-path.yml"
+            ).body
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/support/gitlab_version_info.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Support
+      class GitlabVersionInfo
+        VERSION_PATTERN = /^(?<version>\d+\.\d+\.\d+)/.freeze
+        COMPONENT_PATTERN = /^(?<major>\d+)\.(?<minor>\d+)\.(?<patch>\d+)/.freeze
+
+        # Get previous gitlab version
+        #
+        # @param [String] current_version
+        # @param [String] edition GitLab edition - ee or ce
+        def initialize(current_version, edition)
+          @current_version = current_version
+          @edition = edition
+          @logger = Runtime::Logger.logger
+        end
+
+        # Get N - 1 version number
+        #
+        # @param [String] semver_component version number component for previous version detection - major|minor|patch
+        # @return [Gem::Version]
+        def previous_version(semver_component)
+          case semver_component
+          when "major"
+            previous_major
+          when "minor"
+            previous_minor
+          when "patch"
+            previous_patch
+          else
+            raise("Unsupported semver component, must be major|minor|patch")
+          end
+        end
+
+        # Get latest patch for specific version number
+        #
+        # @example
+        # latest_patch(Gem::Version.new("14.10")) => "14.10.5"
+        #
+        # @param [Gem::Version] version
+        # @return [String]
+        def latest_patch(version)
+          versions.find { |ver| ver.to_s.match?(/^#{version}/) }
+        end
+
+        private
+
+        attr_reader :current_version, :edition, :logger
+
+        # Current versions major version
+        #
+        # @return [Integer]
+        def current_major
+          @current_major ||= current_version.match(COMPONENT_PATTERN)[:major].to_i
+        end
+
+        # Current versions minor version
+        #
+        # @return [Integer]
+        def current_minor
+          @current_minor ||= current_version.match(COMPONENT_PATTERN)[:minor].to_i
+        end
+
+        # Current versions patch version
+        #
+        # @return [Integer]
+        def current_patch
+          @current_patch ||= current_version.match(COMPONENT_PATTERN)[:patch].to_i
+        end
+
+        # Previous major version
+        #
+        # @return [String]
+        def previous_major
+          return fallback_major unless tags
+
+          versions.find { |version| version.to_s.start_with?((current_major - 1).to_s) }
+        end
+
+        # Previous first major version image
+        #
+        # @return [String]
+        def fallback_major
+          previous_fallback_version(current_major - 1)
+        end
+
+        # Previous minor version
+        #
+        # @return [String]
+        def previous_minor
+          return fallback_minor unless tags
+          return previous_major if current_minor.zero?
+
+          versions.find { |version| version.to_s.match?(/^#{current_major}\.#{current_minor - 1}/) }
+        end
+
+        # Previous first minor version
+        #
+        # @return [String]
+        def fallback_minor
+          return previous_fallback_version(current_major, current_minor - 1) unless current_minor.zero?
+
+          previous_major
+        end
+
+        # Previous patch version
+        #
+        # @return [String]
+        def previous_patch
+          return fallback_patch unless tags
+          return previous_minor if current_patch.zero?
+
+          versions.find { |version| version.to_s.match?(/^#{current_major}\.#{current_minor}\.#{current_patch - 1}/) }
+        end
+
+        # Previous first patch version
+        #
+        # @return [String]
+        def fallback_patch
+          return previous_fallback_version(current_major, current_minor, current_patch - 1) unless current_patch.zero?
+
+          previous_minor
+        end
+
+        # Version number from docker tag
+        #
+        # @param [String] tag
+        # @return [String]
+        def version(tag)
+          tag.match(VERSION_PATTERN)[:version]
+        end
+
+        # Fallback version
+        #
+        # @param [Integer] major_component
+        # @param [Integer] minor_component
+        # @param [Integer] patch_component
+        # @return [Gem::Version]
+        def previous_fallback_version(major_component, minor_component = 0, patch_component = 0)
+          Gem::Version.new("#{major_component}.#{minor_component}.#{patch_component}")
+        end
+
+        # All available gitlab versions
+        #
+        # @return [Array<String>]
+        def versions
+          @versions = tags
+            .map { |tag| Gem::Version.new(tag.match(VERSION_PATTERN)[:version]) }
+            .sort
+            .reverse # reverse array so first match by .find always returns latest version
+        end
+
+        # All available docker tags
+        #
+        # @return [Array<String>]
+        def tags
+          return @tags if defined?(@tags)
+
+          logger.info("Fetching docker tags from 'gitlab/gitlab-#{edition}' registry")
+          response = HttpRequest.make_http_request(
+            url: "https://registry.hub.docker.com/v1/repositories/gitlab/gitlab-#{edition}/tags",
+            fail_on_error: false
+          )
+
+          unless response.code == 200
+            logger.error("  failed to fetch docker tags - code: #{response.code}, response: '#{response.body}'")
+            return @tags = nil
+          end
+
+          @tags = JSON
+            .parse(response.body, symbolize_names: true)
+            .map { |tag| tag[:name] }
+            .select { |tag| tag.match?(VERSION_PATTERN) }
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   module QA
-    VERSION = '8.0.0'
+    VERSION = '8.3.0'
   end
 end

data/scripts/generate-qa-jobs.rb

@@ -37,6 +37,7 @@ class GenerateQAJobs
     jobs.merge!(load_yml_contents('registry', should_automatically_run?('test_integration_registry')))
     jobs.merge!(load_yml_contents('packages', should_automatically_run?('test_instance_all_packages')))
     jobs.merge!(load_yml_contents('elasticsearch', should_automatically_run?('test_integration_elasticsearch')))
+    jobs.merge!(load_yml_contents('opensearch', should_automatically_run?('test_integration_opensearch')))
     jobs.merge!(load_yml_contents('praefect', should_automatically_run?('test_instance_all')))
     jobs.merge!(load_yml_contents('gitaly_cluster', should_automatically_run?('test_instance_all')))
     jobs.merge!(load_yml_contents('mtls', should_automatically_run?('test_instance_all_mtls')))
@@ -47,6 +48,7 @@ class GenerateQAJobs
     jobs.merge!(load_yml_contents('cloud_activation', should_automatically_run?('test_instance_all_cloud_activation')))
     jobs.merge!(load_yml_contents('registry_with_cdn', should_automatically_run?('test_integration_registrywithcdn')))
     jobs.merge!(load_yml_contents('staging'))
+    jobs.merge!(load_yml_contents('metrics', should_automatically_run?('test_instance_all_metrics')))
 
     # Disabling geo jobs temporarily due to https://gitlab.com/gitlab-org/quality/team-tasks/-/issues/774
     # jobs.merge!(load_yml_contents('geo', should_automatically_run?('scenario_test_geo')))

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 8.0.0
+  version: 8.3.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-06-27 00:00:00.000000000 Z
+date: 2022-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -251,6 +251,7 @@ files:
 - ".gitlab/ci/jobs/ldap_no_tls.gitlab-ci.yml"
 - ".gitlab/ci/jobs/ldap_tls.gitlab-ci.yml"
 - ".gitlab/ci/jobs/mattermost.gitlab-ci.yml"
+- ".gitlab/ci/jobs/metrics.gitlab-ci.yml"
 - ".gitlab/ci/jobs/mtls.gitlab-ci.yml"
 - ".gitlab/ci/jobs/object_storage.gitlab-ci.yml"
 - ".gitlab/ci/jobs/object_storage_aws.gitlab-ci.yml"
@@ -258,6 +259,7 @@ files:
 - ".gitlab/ci/jobs/object_storage_registry_tls.gitlab-ci.yml"
 - ".gitlab/ci/jobs/omnibus_image.gitlab-ci.yml"
 - ".gitlab/ci/jobs/omnibus_upgrade.gitlab-ci.yml"
+- ".gitlab/ci/jobs/opensearch.gitlab-ci.yml"
 - ".gitlab/ci/jobs/packages.gitlab-ci.yml"
 - ".gitlab/ci/jobs/praefect.gitlab-ci.yml"
 - ".gitlab/ci/jobs/registry.gitlab-ci.yml"
@@ -316,6 +318,7 @@ files:
 - lib/gitlab/qa/component/ldap.rb
 - lib/gitlab/qa/component/mail_hog.rb
 - lib/gitlab/qa/component/minio.rb
+- lib/gitlab/qa/component/opensearch.rb
 - lib/gitlab/qa/component/postgresql.rb
 - lib/gitlab/qa/component/preprod.rb
 - lib/gitlab/qa/component/production.rb
@@ -394,7 +397,9 @@ files:
 - lib/gitlab/qa/scenario/test/integration/ldap_no_tls.rb
 - lib/gitlab/qa/scenario/test/integration/ldap_tls.rb
 - lib/gitlab/qa/scenario/test/integration/mattermost.rb
+- lib/gitlab/qa/scenario/test/integration/metrics.rb
 - lib/gitlab/qa/scenario/test/integration/mtls.rb
+- lib/gitlab/qa/scenario/test/integration/opensearch.rb
 - lib/gitlab/qa/scenario/test/integration/praefect.rb
 - lib/gitlab/qa/scenario/test/integration/registry.rb
 - lib/gitlab/qa/scenario/test/integration/registry_tls.rb
@@ -405,11 +410,14 @@ files:
 - lib/gitlab/qa/scenario/test/integration/ssh_tunnel.rb
 - lib/gitlab/qa/scenario/test/omnibus/image.rb
 - lib/gitlab/qa/scenario/test/omnibus/update.rb
+- lib/gitlab/qa/scenario/test/omnibus/update_from_previous.rb
 - lib/gitlab/qa/scenario/test/omnibus/upgrade.rb
 - lib/gitlab/qa/scenario/test/sanity/version.rb
 - lib/gitlab/qa/slack/post_to_slack.rb
 - lib/gitlab/qa/support/dev_ee_qa_image.rb
 - lib/gitlab/qa/support/get_request.rb
+- lib/gitlab/qa/support/gitlab_upgrade_path.rb
+- lib/gitlab/qa/support/gitlab_version_info.rb
 - lib/gitlab/qa/support/http_request.rb
 - lib/gitlab/qa/support/invalid_response_error.rb
 - lib/gitlab/qa/test_logger.rb