gitlab-qa 14.6.0 → 14.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d16aa0d0e3acf2f78c85344345d2d7a1637fa9c16154446020d0ffd5649687f9
-  data.tar.gz: 68bbb672fecd7bd17113fd7485cbacf547477cd64362e250296316a897868974
+  metadata.gz: 3632783614831ceda0acb1c2fa312dcea081c8e878dfe2ce41bdb6f9ac70253b
+  data.tar.gz: 5af7c287d9820b050feb5944a9bf93561ef33442f0120bb71e6c7d0c33528ed3
 SHA512:
-  metadata.gz: 260445e5a526267392c3d91d81eee2baa37df00c6981acecef40f64d17ebf7a10d608d9eb3eae61a287633ca5e7446fdd4e8da70bc70d1de8a2c915065910248
-  data.tar.gz: fcb5cf3c6a4aa914bb089316cb1f3f1b6d8c666fc7dccf7621c79a90d92dc5dcfb6bb5aa77631059f1c841321baf898471f0e35f09ad7a6d831394badc983a67
+  metadata.gz: 46a5d0fd06137c2f41f8f504f1b3afea1e82ba010f86f704a5991f859c555a45e74e1ad6eaa320e916ed6c8fa9227ef20e47e7da03441f8bc3ec6e6ef2aa2352
+  data.tar.gz: 8ccadd90b943ca72ccd967fabfd70ae26a834ff8b949456eecdda333af3a7e6754b46a956e0dc029b7b70847c9134fb11dcd54a202299d64cbfe2902620c7ca1

data/.gitlab-ci.yml

@@ -10,7 +10,7 @@
 include:
   - component: gitlab.com/gitlab-org/components/danger-review/danger-review@~latest
     inputs:
-      job_image: "${CI_REGISTRY}/gitlab-org/gitlab-build-images/debian-bullseye-ruby-${RUBY_VERSION}:bundler-2.4"
+      job_image: "${CI_REGISTRY}/gitlab-org/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}:bundler-${BUNDLER_VERSION}"
   - component: gitlab.com/gitlab-org/components/gem-release/gem-release@~latest
 
 stages:
@@ -20,7 +20,7 @@ stages:
   - deploy
 
 default:
-  image: ${CI_REGISTRY}/gitlab-org/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}:bundler-2.3
+  image: ${CI_REGISTRY}/gitlab-org/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}:bundler-${BUNDLER_VERSION}
   tags:
     - gitlab-org
   cache:
@@ -41,8 +41,10 @@ workflow:
     - if: '$CI_PIPELINE_SOURCE == "web"'
 
 variables:
-  DEBIAN_VERSION: bullseye
-  RUBY_VERSION: "3.0"
+  DEBIAN_VERSION: bookworm
+  DOCKER_VERSION: "24.0.5"
+  BUNDLER_VERSION: "2.5"
+  RUBY_VERSION: "3.1"
   BUNDLE_PATH: vendor
   BUNDLE_SILENCE_ROOT_WARNING: "true"
   BUNDLE_SUPPRESS_INSTALL_USING_MESSAGES: "true"
@@ -133,13 +135,17 @@ package-and-test:
     RUN_WITH_BUNDLE: "true"
     SKIP_OMNIBUS_TRIGGER: "true"
     SKIP_REPORT_IN_ISSUES: "true"
-    ALLURE_JOB_NAME: gitlab-qa
     UPDATE_QA_CACHE: $UPDATE_QA_CACHE
     GITLAB_QA_CACHE_KEY: $GITLAB_QA_CACHE_KEY
+    QA_RETRY_FAILED_SPECS: "true"
+    QA_RUN_TYPE: gitlab-qa
+    QA_EXPORT_TEST_METRICS: "false" # skip metrics export as this pipeline is only used to validate gitlab-qa changes
   inherit:
     variables:
       - RUBY_VERSION
       - DEBIAN_VERSION
+      - DOCKER_VERSION
+      - BUNDLER_VERSION
   trigger:
     strategy: depend
     forward:
@@ -153,4 +159,4 @@ package-and-test:
     - if: '$CI_MERGE_REQUEST_IID'
       when: manual
       allow_failure: true
-    - when: always
+    - when: always

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-qa (14.6.0)
+    gitlab-qa (14.8.0)
       activesupport (>= 6.1, < 7.2)
       gitlab (~> 4.19)
       http (~> 5.0)
@@ -127,7 +127,7 @@ GEM
       multi_xml (>= 0.5.2)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    jaro_winkler (1.5.4)
+    jaro_winkler (1.5.6)
     json (2.6.3)
     kramdown (2.4.0)
       rexml

data/docs/running_against_remote_grid.md

@@ -18,7 +18,7 @@ I.e, if you have a Selenium server set up at http://localhost:4444 or if you hav
 | QA_SAVE_ALL_VIDEOS        | Used with Selenoid. Saves video for both passed and failed tests | false    | false, true                    |
 | QA_SELENOID_BROWSER_IMAGE | Used with Selenoid. Sets the browser image to use for video recording. | "selenoid/chrome" | "selenoid/chrome", "registry.gitlab.com/gitlab-org/gitlab-qa/selenoid-chrome-gitlab" |
 | QA_SELENOID_BROWSER_VERSION | Used in conjunction with QA_SELENOID_BROWSER_IMAGE. Version of browser to run against.       | "111.0"  | "latest" "111.0" "mobile-111.0"|
-| QA_VIDEO_RECORDER_IMAGE   | Used with Selenoid. Sets the video recorder image to use for video recording. | "presidenten/selenoid-manual-video-recorder" | "presidenten/selenoid-manual-video-recorder" |
+| QA_VIDEO_RECORDER_IMAGE   | Used with Selenoid. Sets the video recorder image to use for video recording. | "registry.gitlab.com/gitlab-org/gitlab-qa/selenoid-manual-video-recorder" | "registry.gitlab.com/gitlab-org/gitlab-qa/selenoid-manual-video-recorder", "presidenten/selenoid-manual-video-recorder" |
 | QA_VIDEO_RECORDER_VERSION | Used with Selenoid. Sets the video recorder image version to use for video recording. | "latest" | "latest"                     |
 | QA_REMOTE_GRID_USERNAME   | Used with Sauce Labs. Username to specify in the remote grid. "USERNAME@provider:80" |          | "gitlab-sl"                    |
 | QA_REMOTE_GRID_ACCESS_KEY | Used with Sauce Labs. Key/Token paired with `QA_REMOTE_GRID_USERNAME` |          |                                |

data/docs/what_tests_can_be_run.md

@@ -349,11 +349,13 @@ $ gitlab-qa Test::Omnibus::UpdateFromPrevious gitlab-ee:dev-tag 15.6.0-pre major
 
 ### `Test::Integration::Geo EE|<full image address>`
 
-This tests that two GitLab Geo instances work as expected.
+This tests that Geo UI proxying is working as expected.
 
-The scenario spins up a primary and secondary GitLab Geo nodes, and verifies
-that the replications (repository, attachments, project rename etc.) work as
-expected.
+The scenario spins up primary and secondary GitLab Geo nodes and
+can be used to verify that web requests to secondary Geo site return
+data that is present on the primary.
+
+See https://docs.gitlab.com/ee/administration/geo/secondary_proxy
 
 To run tests against the GitLab containers, a GitLab QA (`gitlab/gitlab-qa`)
 container is spun up and tests are run from it by running the

data/lib/gitlab/qa/component/ai_gateway.rb

@@ -24,7 +24,7 @@ module Gitlab
             'AIGW_GITLAB_URL' => "http://#{gitlab_hostname}",
             'AIGW_GITLAB_API_URL' => "http://#{gitlab_hostname}/api/v4",
             'AIGW_CUSTOMER_PORTAL_URL' => Runtime::Env.customer_portal_url,
-            'AIGW_USE_FAKE_MODELS' => true,
+            'AIGW_MOCK_MODEL_RESPONSES' => true,
             'AIGW_LOGGING__LEVEL' => 'debug',
             'AIGW_LOGGING__TO_FILE' => "..#{LOG_DIR}/modelgateway_debug.log"
           }

data/lib/gitlab/qa/component/specs.rb

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
-# rubocop:disable Metrics/CyclomaticComplexity
 # rubocop:disable Metrics/AbcSize
 
 require 'securerandom'
+require 'active_support/core_ext/array/grouping'
 
 module Gitlab
   module QA
@@ -13,7 +13,19 @@ module Gitlab
       # the `qa/` directory located in GitLab CE / EE repositories.
       #
       class Specs < Scenario::Template
-        attr_accessor :suite, :release, :network, :args, :volumes, :env, :runner_network, :hostname, :additional_hosts
+        LAST_RUN_FILE = "examples.txt"
+
+        attr_accessor :suite,
+          :release,
+          :network,
+          :args,
+          :volumes,
+          :env,
+          :runner_network,
+          :hostname,
+          :additional_hosts,
+          :retry_failed_specs,
+          :infer_qa_image_from_release
 
         def initialize
           @docker = Docker::Engine.new(stream_output: true) # stream test output directly instead of through logger
@@ -21,6 +33,7 @@ module Gitlab
           @volumes = {}
           @additional_hosts = []
           @volumes = { '/var/run/docker.sock' => '/var/run/docker.sock' }
+          @retry_failed_specs = Runtime::Env.retry_failed_specs?
 
           include_optional_volumes(
             Runtime::Env.qa_rspec_report_path => 'rspec',
@@ -51,9 +64,30 @@ module Gitlab
           Runtime::Logger.info("Running test suite `#{suite}` for #{release.project_name}")
 
           name = "#{release.project_name}-qa-#{SecureRandom.hex(4)}"
+          run_specs(name)
+        rescue Support::ShellCommand::StatusError => e
+          raise e unless retry_failed_specs
 
-          feature_flag_sets = []
+          results_file = File.join(host_artifacts_dir(name), LAST_RUN_FILE)
+          raise "Failed to find initial run results file #{results_file}" unless File.exist?(results_file)
+
+          Runtime::Logger.warn("Initial test run failed, retrying failed specs in new process!")
+          run_specs(name, retry_process: true, initial_run_results_host_path: results_file)
+        end
+
+        private
 
+        # Ful path to tmp dir inside container
+        #
+        # @return [String]
+        def tmp_dir
+          @tmp_dir ||= File.join(Docker::Volumes::QA_CONTAINER_WORKDIR, 'tmp')
+        end
+
+        def feature_flag_sets
+          return @feature_flag_sets if defined?(@feature_flag_sets)
+
+          feature_flag_sets = []
           # When `args` includes:
           #   `[..., "--disable-feature", "a", "--enable-feature", "b", "--set-feature-flags", "c=enable", ...]`
           # `feature_flag_sets` will be set to:
@@ -62,14 +96,36 @@ module Gitlab
           while (index = args&.index { |x| x =~ /--.*-feature/ })
             feature_flag_sets << args.slice!(index, 2)
           end
-
           # When `args` do not have any feature flag options, we add [] so that test is run exactly once.
           feature_flag_sets << [] unless feature_flag_sets.any?
 
+          @feature_flag_sets = feature_flag_sets
+        end
+
+        def run_specs(name, retry_process: false, initial_run_results_host_path: nil)
+          container_name = retry_process ? "#{name}-retry" : name
+
+          env_vars = if retry_process
+                       Runtime::Env.variables.merge({
+                         **env,
+                         "QA_RSPEC_RETRIED" => "true",
+                         "NO_KNAPSACK" => "true"
+                       })
+                     else
+                       Runtime::Env.variables.merge(env)
+                     end
+
+          env_vars["RSPEC_LAST_RUN_RESULTS_FILE"] = last_run_results_file
+          # TODO: remove once rspec-retry gem is removed
+          env_vars["QA_DISABLE_RSPEC_RETRY"] = "true" if retry_failed_specs
+
+          run_volumes = volumes.to_h.merge({ host_artifacts_dir(container_name) => tmp_dir })
+          run_volumes[initial_run_results_host_path] = last_run_results_file if retry_process
+
           feature_flag_sets.each do |feature_flag_set|
             @docker.run(
               image: qa_image,
-              args: [suite, *args_with_flags(args, feature_flag_set)],
+              args: [suite, *args_with_flags(feature_flag_set, retry_process: retry_process)],
               mask_secrets: Runtime::Env.variables_to_mask
             ) do |command|
               command << "-t --rm --net=#{network || 'bridge'}"
@@ -84,37 +140,50 @@ module Gitlab
                 command << hosts # override /etc/hosts in docker container when test runs
               end
 
-              Runtime::Env.variables.merge(env).each do |key, value|
-                command.env(key, value)
-              end
-
-              command.volume(
-                File.join(Runtime::Env.host_artifacts_dir, name),
-                File.join(Docker::Volumes::QA_CONTAINER_WORKDIR, 'tmp')
-              )
+              env_vars.each { |key, value| command.env(key, value) }
+              run_volumes.each { |to, from| command.volume(to, from) }
 
-              volumes.to_h.each { |to, from| command.volume(to, from) }
-
-              command.name(name)
+              command.name(container_name)
             end
           end
         end
 
-        private
-
         def docker_pull_qa_image_if_needed
           @docker.login(**release.login_params) if release.login_params
 
           @docker.pull(image: qa_image) unless Runtime::Env.skip_pull?
         end
 
-        def args_with_flags(args, feature_flag_set)
-          return args if feature_flag_set.empty?
+        def args_with_flags(feature_flag_set, retry_process: false)
+          return args if feature_flag_set.empty? && !retry_process
+
+          run_args = if !retry_process
+                       args.dup
+                     elsif args.include?("--")
+                       qa_args, rspec_args = args.split("--")
+                       [*qa_args, "--"] + args_without_spec_arguments(rspec_args).push("--only-failures")
+                     else
+                       args.dup.push("--", "--only-failures")
+                     end
+
+          return run_args if feature_flag_set.empty?
 
           Runtime::Logger.info("Running with feature flag: #{feature_flag_set.join(' ')}")
+          run_args.insert(1, *feature_flag_set)
+        end
+
+        # Remove particular spec argument like specific spec/folder or specific tags
+        #
+        # @param [Array] rspec_args
+        # @return [Array]
+        def args_without_spec_arguments(rspec_args)
+          arg_pairs = rspec_args.each_with_object([]) do |arg, new_args|
+            next new_args.push([arg]) if new_args.last.nil? || arg.start_with?("--") || new_args.last.size == 2
+
+            new_args.last.push(arg)
+          end
 
-          args_with_f = args.dup
-          args_with_f.insert(1, *feature_flag_set)
+          arg_pairs.reject { |pair| pair.first == "--tag" || !pair.first.start_with?("--") }.flatten
         end
 
         def skip_tests?
@@ -122,11 +191,10 @@ module Gitlab
         end
 
         def qa_image
-          if Runtime::Scenario.attributes.include?(:qa_image)
-            Runtime::Scenario.qa_image
-          else
-            "#{release.qa_image}:#{release.qa_tag}"
-          end
+          infered_qa_image = "#{release.qa_image}:#{release.qa_tag}"
+          return infered_qa_image if infer_qa_image_from_release || !Runtime::Scenario.attributes.include?(:qa_image)
+
+          Runtime::Scenario.qa_image
         end
 
         # Adds volumes to the volumes hash if the relevant host paths exist
@@ -138,9 +206,23 @@ module Gitlab
             host_path.present? && volumes[host_path] = File.join(Docker::Volumes::QA_CONTAINER_WORKDIR, container_path)
           end
         end
+
+        # Full path to host artifacts dir
+        #
+        # @param [String] name
+        # @return [String]
+        def host_artifacts_dir(name)
+          File.join(Runtime::Env.host_artifacts_dir, name)
+        end
+
+        # Path to save or read run results file within container
+        #
+        # @return [String]
+        def last_run_results_file
+          File.join(tmp_dir, LAST_RUN_FILE)
+        end
       end
     end
   end
 end
-# rubocop:enable Metrics/CyclomaticComplexity
 # rubocop:enable Metrics/AbcSize

data/lib/gitlab/qa/runtime/env.rb

@@ -96,6 +96,7 @@ module Gitlab
           'CI' => :ci,
           'CI_JOB_NAME' => :ci_job_name,
           'CI_JOB_NAME_SLUG' => :ci_job_name_slug,
+          'CI_JOB_ID' => :ci_job_id,
           'CI_JOB_URL' => :ci_job_url,
           'CI_JOB_TOKEN' => :ci_job_token,
           'CI_RUNNER_ID' => :ci_runner_id,
@@ -105,7 +106,9 @@ module Gitlab
           'CI_NODE_TOTAL' => :ci_node_total,
           'CI_PROJECT_NAME' => :ci_project_name,
           'CI_PROJECT_PATH' => :ci_project_path,
+          'CI_PROJECT_PATH_SLUG' => :ci_project_path_slug,
           'CI_PIPELINE_SOURCE' => :ci_pipeline_source,
+          'CI_PIPELINE_ID' => :ci_pipeline_id,
           'CI_PIPELINE_URL' => :ci_pipeline_url,
           'CI_PIPELINE_CREATED_AT' => :ci_pipeline_created_at,
           'CI_MERGE_REQUEST_IID' => :ci_merge_request_iid,
@@ -277,20 +280,29 @@ module Gitlab
 
         def require_aws_s3_environment!
           %w[AWS_S3_REGION AWS_S3_KEY_ID AWS_S3_ACCESS_KEY AWS_S3_BUCKET_NAME].each do |env_key|
-            raise ArgumentError, "Environment variable #{env_key} must be set to run AWS S3 object storage specs" unless ENV.key?(env_key)
+            unless ENV.key?(env_key)
+              raise ArgumentError,
+                "Environment variable #{env_key} must be set to run AWS S3 object storage specs"
+            end
           end
         end
 
         def require_gcs_environment!
           %w[GOOGLE_PROJECT GOOGLE_CLIENT_EMAIL GOOGLE_JSON_KEY GCS_BUCKET_NAME].each do |env_key|
-            raise ArgumentError, "Environment variable #{env_key} must be set to run GCS object storage specs" unless ENV.key?(env_key)
+            unless ENV.key?(env_key)
+              raise ArgumentError,
+                "Environment variable #{env_key} must be set to run GCS object storage specs"
+            end
           end
         end
 
         def require_gcs_with_cdn_environment!
           %w[GOOGLE_CDN_JSON_KEY GCS_CDN_BUCKET_NAME GOOGLE_CDN_LB GOOGLE_CDN_SIGNURL_KEY
             GOOGLE_CDN_SIGNURL_KEY_NAME].each do |env_key|
-            raise ArgumentError, "Environment variable #{env_key} must be set to run GCS with CDN enabled scenario" unless ENV.key?(env_key)
+            unless ENV.key?(env_key)
+              raise ArgumentError,
+                "Environment variable #{env_key} must be set to run GCS with CDN enabled scenario"
+            end
           end
         end
 
@@ -359,7 +371,7 @@ module Gitlab
         end
 
         def video_recorder_image
-          env_var_value_if_defined('QA_VIDEO_RECORDER_IMAGE') || 'presidenten/selenoid-manual-video-recorder'
+          env_var_value_if_defined('QA_VIDEO_RECORDER_IMAGE') || 'registry.gitlab.com/gitlab-org/gitlab-qa/selenoid-manual-video-recorder'
         end
 
         def video_recorder_version
@@ -455,6 +467,10 @@ module Gitlab
           enabled?(env_var_value_if_defined('QA_MOCK_GITHUB'), default: true)
         end
 
+        def retry_failed_specs?
+          enabled?(env_var_value_if_defined('QA_RETRY_FAILED_SPECS'), default: false)
+        end
+
         private
 
         def enabled?(value, default: true)

data/lib/gitlab/qa/scenario/test/omnibus/update_from_previous.rb

@@ -111,8 +111,11 @@ module Gitlab
                 specs.args = [gitlab.address, *rspec_args]
                 next if release == current_release
 
-                # do not generate reports and metrics artifacts for non release runs
+                # do not generate reports and metrics artifacts for non release runs or retry failures
                 specs.env = { 'QA_GENERATE_ALLURE_REPORT' => false, 'QA_SAVE_TEST_METRICS' => false }
+                specs.retry_failed_specs = false
+                # if qa-image was set explicitly, make sure it is not used on initial run for release
+                specs.infer_qa_image_from_release = true
               end
             rescue Support::ShellCommand::StatusError => e
               if release == current_release # only fail on current release

data/lib/gitlab/qa/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   module QA
-    VERSION = '14.6.0'
+    VERSION = '14.8.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 14.6.0
+  version: 14.8.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-04-02 00:00:00.000000000 Z
+date: 2024-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control