gitlab-qa 14.6.0 → 14.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d16aa0d0e3acf2f78c85344345d2d7a1637fa9c16154446020d0ffd5649687f9
-  data.tar.gz: 68bbb672fecd7bd17113fd7485cbacf547477cd64362e250296316a897868974
+  metadata.gz: aca8f96e30555fcac3d968c580f1393b87d2114a3616e8ab6dbb98f913f852d4
+  data.tar.gz: 566a628d9587a7eeb6e6bcb530bed7170aa689aaadc50cecb28eaba17096e26e
 SHA512:
-  metadata.gz: 260445e5a526267392c3d91d81eee2baa37df00c6981acecef40f64d17ebf7a10d608d9eb3eae61a287633ca5e7446fdd4e8da70bc70d1de8a2c915065910248
-  data.tar.gz: fcb5cf3c6a4aa914bb089316cb1f3f1b6d8c666fc7dccf7621c79a90d92dc5dcfb6bb5aa77631059f1c841321baf898471f0e35f09ad7a6d831394badc983a67
+  metadata.gz: 4304f800c1430834f7b335ddae6627e51e6126d883359feefb2863662abe274c8b3de2fadca9328c6f6e9815aeb5df2798a1e839a24cf372ee66cdd5db9bea84
+  data.tar.gz: 1a8325bf53f232d5db6292a63107b15f0acf19a04acb7dc0186474656dbd12744ab429030eb53a4e3f83acc5f9655a7159fc14d89cf81b373650b4045904e95c

data/.gitlab-ci.yml

@@ -133,9 +133,11 @@ package-and-test:
     RUN_WITH_BUNDLE: "true"
     SKIP_OMNIBUS_TRIGGER: "true"
     SKIP_REPORT_IN_ISSUES: "true"
-    ALLURE_JOB_NAME: gitlab-qa
     UPDATE_QA_CACHE: $UPDATE_QA_CACHE
     GITLAB_QA_CACHE_KEY: $GITLAB_QA_CACHE_KEY
+    QA_RETRY_FAILED_SPECS: "true"
+    QA_RUN_TYPE: gitlab-qa
+    QA_EXPORT_TEST_METRICS: "false" # skip metrics export as this pipeline is only used to validate gitlab-qa changes
   inherit:
     variables:
       - RUBY_VERSION
@@ -153,4 +155,4 @@ package-and-test:
     - if: '$CI_MERGE_REQUEST_IID'
       when: manual
       allow_failure: true
-    - when: always
+    - when: always

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-qa (14.6.0)
+    gitlab-qa (14.7.0)
       activesupport (>= 6.1, < 7.2)
       gitlab (~> 4.19)
       http (~> 5.0)
@@ -127,7 +127,7 @@ GEM
       multi_xml (>= 0.5.2)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    jaro_winkler (1.5.4)
+    jaro_winkler (1.5.6)
     json (2.6.3)
     kramdown (2.4.0)
       rexml

data/docs/running_against_remote_grid.md

@@ -18,7 +18,7 @@ I.e, if you have a Selenium server set up at http://localhost:4444 or if you hav
 | QA_SAVE_ALL_VIDEOS        | Used with Selenoid. Saves video for both passed and failed tests | false    | false, true                    |
 | QA_SELENOID_BROWSER_IMAGE | Used with Selenoid. Sets the browser image to use for video recording. | "selenoid/chrome" | "selenoid/chrome", "registry.gitlab.com/gitlab-org/gitlab-qa/selenoid-chrome-gitlab" |
 | QA_SELENOID_BROWSER_VERSION | Used in conjunction with QA_SELENOID_BROWSER_IMAGE. Version of browser to run against.       | "111.0"  | "latest" "111.0" "mobile-111.0"|
-| QA_VIDEO_RECORDER_IMAGE   | Used with Selenoid. Sets the video recorder image to use for video recording. | "presidenten/selenoid-manual-video-recorder" | "presidenten/selenoid-manual-video-recorder" |
+| QA_VIDEO_RECORDER_IMAGE   | Used with Selenoid. Sets the video recorder image to use for video recording. | "registry.gitlab.com/gitlab-org/gitlab-qa/selenoid-manual-video-recorder" | "registry.gitlab.com/gitlab-org/gitlab-qa/selenoid-manual-video-recorder", "presidenten/selenoid-manual-video-recorder" |
 | QA_VIDEO_RECORDER_VERSION | Used with Selenoid. Sets the video recorder image version to use for video recording. | "latest" | "latest"                     |
 | QA_REMOTE_GRID_USERNAME   | Used with Sauce Labs. Username to specify in the remote grid. "USERNAME@provider:80" |          | "gitlab-sl"                    |
 | QA_REMOTE_GRID_ACCESS_KEY | Used with Sauce Labs. Key/Token paired with `QA_REMOTE_GRID_USERNAME` |          |                                |

data/docs/what_tests_can_be_run.md

@@ -349,11 +349,13 @@ $ gitlab-qa Test::Omnibus::UpdateFromPrevious gitlab-ee:dev-tag 15.6.0-pre major
 
 ### `Test::Integration::Geo EE|<full image address>`
 
-This tests that two GitLab Geo instances work as expected.
+This tests that Geo UI proxying is working as expected.
 
-The scenario spins up a primary and secondary GitLab Geo nodes, and verifies
-that the replications (repository, attachments, project rename etc.) work as
-expected.
+The scenario spins up primary and secondary GitLab Geo nodes and
+can be used to verify that web requests to secondary Geo site return
+data that is present on the primary.
+
+See https://docs.gitlab.com/ee/administration/geo/secondary_proxy
 
 To run tests against the GitLab containers, a GitLab QA (`gitlab/gitlab-qa`)
 container is spun up and tests are run from it by running the

data/lib/gitlab/qa/component/ai_gateway.rb

@@ -24,7 +24,7 @@ module Gitlab
             'AIGW_GITLAB_URL' => "http://#{gitlab_hostname}",
             'AIGW_GITLAB_API_URL' => "http://#{gitlab_hostname}/api/v4",
             'AIGW_CUSTOMER_PORTAL_URL' => Runtime::Env.customer_portal_url,
-            'AIGW_USE_FAKE_MODELS' => true,
+            'AIGW_MOCK_MODEL_RESPONSES' => true,
             'AIGW_LOGGING__LEVEL' => 'debug',
             'AIGW_LOGGING__TO_FILE' => "..#{LOG_DIR}/modelgateway_debug.log"
           }

data/lib/gitlab/qa/component/specs.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-# rubocop:disable Metrics/CyclomaticComplexity
 # rubocop:disable Metrics/AbcSize
 
 require 'securerandom'
@@ -13,7 +12,18 @@ module Gitlab
       # the `qa/` directory located in GitLab CE / EE repositories.
       #
       class Specs < Scenario::Template
-        attr_accessor :suite, :release, :network, :args, :volumes, :env, :runner_network, :hostname, :additional_hosts
+        LAST_RUN_FILE = "examples.txt"
+
+        attr_accessor :suite,
+          :release,
+          :network,
+          :args,
+          :volumes,
+          :env,
+          :runner_network,
+          :hostname,
+          :additional_hosts,
+          :retry_failed_specs
 
         def initialize
           @docker = Docker::Engine.new(stream_output: true) # stream test output directly instead of through logger
@@ -21,6 +31,7 @@ module Gitlab
           @volumes = {}
           @additional_hosts = []
           @volumes = { '/var/run/docker.sock' => '/var/run/docker.sock' }
+          @retry_failed_specs = Runtime::Env.retry_failed_specs?
 
           include_optional_volumes(
             Runtime::Env.qa_rspec_report_path => 'rspec',
@@ -51,9 +62,30 @@ module Gitlab
           Runtime::Logger.info("Running test suite `#{suite}` for #{release.project_name}")
 
           name = "#{release.project_name}-qa-#{SecureRandom.hex(4)}"
+          run_specs(name)
+        rescue Support::ShellCommand::StatusError => e
+          raise e unless retry_failed_specs
 
-          feature_flag_sets = []
+          results_file = File.join(host_artifacts_dir(name), LAST_RUN_FILE)
+          raise "Failed to find initial run results file #{results_file}" unless File.exist?(results_file)
+
+          Runtime::Logger.warn("Initial test run failed, retrying failed specs in new process!")
+          run_specs(name, retry_process: true, initial_run_results_host_path: results_file)
+        end
+
+        private
+
+        # Ful path to tmp dir inside container
+        #
+        # @return [String]
+        def tmp_dir
+          @tmp_dir ||= File.join(Docker::Volumes::QA_CONTAINER_WORKDIR, 'tmp')
+        end
+
+        def feature_flag_sets
+          return @feature_flag_sets if defined?(@feature_flag_sets)
 
+          feature_flag_sets = []
           # When `args` includes:
           #   `[..., "--disable-feature", "a", "--enable-feature", "b", "--set-feature-flags", "c=enable", ...]`
           # `feature_flag_sets` will be set to:
@@ -62,14 +94,36 @@ module Gitlab
           while (index = args&.index { |x| x =~ /--.*-feature/ })
             feature_flag_sets << args.slice!(index, 2)
           end
-
           # When `args` do not have any feature flag options, we add [] so that test is run exactly once.
           feature_flag_sets << [] unless feature_flag_sets.any?
 
+          @feature_flag_sets = feature_flag_sets
+        end
+
+        def run_specs(name, retry_process: false, initial_run_results_host_path: nil)
+          container_name = retry_process ? "#{name}-retry" : name
+
+          env_vars = if retry_process
+                       Runtime::Env.variables.merge({
+                         **env,
+                         "QA_RSPEC_RETRIED" => "true",
+                         "NO_KNAPSACK" => "true"
+                       })
+                     else
+                       Runtime::Env.variables.merge(env)
+                     end
+
+          env_vars["RSPEC_LAST_RUN_RESULTS_FILE"] = last_run_results_file
+          # TODO: remove once rspec-retry gem is removed
+          env_vars["QA_DISABLE_RSPEC_RETRY"] = "true" if retry_failed_specs
+
+          run_volumes = volumes.to_h.merge({ host_artifacts_dir(container_name) => tmp_dir })
+          run_volumes[initial_run_results_host_path] = last_run_results_file if retry_process
+
           feature_flag_sets.each do |feature_flag_set|
             @docker.run(
               image: qa_image,
-              args: [suite, *args_with_flags(args, feature_flag_set)],
+              args: [suite, *args_with_flags(feature_flag_set, retry_process: retry_process)],
               mask_secrets: Runtime::Env.variables_to_mask
             ) do |command|
               command << "-t --rm --net=#{network || 'bridge'}"
@@ -84,37 +138,40 @@ module Gitlab
                 command << hosts # override /etc/hosts in docker container when test runs
               end
 
-              Runtime::Env.variables.merge(env).each do |key, value|
-                command.env(key, value)
-              end
-
-              command.volume(
-                File.join(Runtime::Env.host_artifacts_dir, name),
-                File.join(Docker::Volumes::QA_CONTAINER_WORKDIR, 'tmp')
-              )
-
-              volumes.to_h.each { |to, from| command.volume(to, from) }
+              env_vars.each { |key, value| command.env(key, value) }
+              run_volumes.each { |to, from| command.volume(to, from) }
 
-              command.name(name)
+              command.name(container_name)
             end
           end
         end
 
-        private
-
         def docker_pull_qa_image_if_needed
           @docker.login(**release.login_params) if release.login_params
 
           @docker.pull(image: qa_image) unless Runtime::Env.skip_pull?
         end
 
-        def args_with_flags(args, feature_flag_set)
-          return args if feature_flag_set.empty?
+        def args_with_flags(feature_flag_set, retry_process: false)
+          return args if feature_flag_set.empty? && !retry_process
 
-          Runtime::Logger.info("Running with feature flag: #{feature_flag_set.join(' ')}")
+          run_args = args.dup
+
+          if retry_process
+            if run_args.include?("--")
+              run_args.push("--only-failures")
+              first_rspec_arg_index = run_args.index("--") + 1
+              # if first arg after `--` is a not a flag, it's a specific spec/folder which needs to be removed
+              run_args.delete_at(first_rspec_arg_index) unless run_args[first_rspec_arg_index].start_with?("--")
+            else
+              run_args.push("--", "--only-failures")
+            end
+          end
 
-          args_with_f = args.dup
-          args_with_f.insert(1, *feature_flag_set)
+          return run_args if feature_flag_set.empty?
+
+          Runtime::Logger.info("Running with feature flag: #{feature_flag_set.join(' ')}")
+          run_args.insert(1, *feature_flag_set)
         end
 
         def skip_tests?
@@ -138,9 +195,23 @@ module Gitlab
             host_path.present? && volumes[host_path] = File.join(Docker::Volumes::QA_CONTAINER_WORKDIR, container_path)
           end
         end
+
+        # Full path to host artifacts dir
+        #
+        # @param [String] name
+        # @return [String]
+        def host_artifacts_dir(name)
+          File.join(Runtime::Env.host_artifacts_dir, name)
+        end
+
+        # Path to save or read run results file within container
+        #
+        # @return [String]
+        def last_run_results_file
+          File.join(tmp_dir, LAST_RUN_FILE)
+        end
       end
     end
   end
 end
-# rubocop:enable Metrics/CyclomaticComplexity
 # rubocop:enable Metrics/AbcSize

data/lib/gitlab/qa/runtime/env.rb

@@ -96,6 +96,7 @@ module Gitlab
           'CI' => :ci,
           'CI_JOB_NAME' => :ci_job_name,
           'CI_JOB_NAME_SLUG' => :ci_job_name_slug,
+          'CI_JOB_ID' => :ci_job_id,
           'CI_JOB_URL' => :ci_job_url,
           'CI_JOB_TOKEN' => :ci_job_token,
           'CI_RUNNER_ID' => :ci_runner_id,
@@ -106,6 +107,7 @@ module Gitlab
           'CI_PROJECT_NAME' => :ci_project_name,
           'CI_PROJECT_PATH' => :ci_project_path,
           'CI_PIPELINE_SOURCE' => :ci_pipeline_source,
+          'CI_PIPELINE_ID' => :ci_pipeline_id,
           'CI_PIPELINE_URL' => :ci_pipeline_url,
           'CI_PIPELINE_CREATED_AT' => :ci_pipeline_created_at,
           'CI_MERGE_REQUEST_IID' => :ci_merge_request_iid,
@@ -277,20 +279,29 @@ module Gitlab
 
         def require_aws_s3_environment!
           %w[AWS_S3_REGION AWS_S3_KEY_ID AWS_S3_ACCESS_KEY AWS_S3_BUCKET_NAME].each do |env_key|
-            raise ArgumentError, "Environment variable #{env_key} must be set to run AWS S3 object storage specs" unless ENV.key?(env_key)
+            unless ENV.key?(env_key)
+              raise ArgumentError,
+                "Environment variable #{env_key} must be set to run AWS S3 object storage specs"
+            end
           end
         end
 
         def require_gcs_environment!
           %w[GOOGLE_PROJECT GOOGLE_CLIENT_EMAIL GOOGLE_JSON_KEY GCS_BUCKET_NAME].each do |env_key|
-            raise ArgumentError, "Environment variable #{env_key} must be set to run GCS object storage specs" unless ENV.key?(env_key)
+            unless ENV.key?(env_key)
+              raise ArgumentError,
+                "Environment variable #{env_key} must be set to run GCS object storage specs"
+            end
           end
         end
 
         def require_gcs_with_cdn_environment!
           %w[GOOGLE_CDN_JSON_KEY GCS_CDN_BUCKET_NAME GOOGLE_CDN_LB GOOGLE_CDN_SIGNURL_KEY
             GOOGLE_CDN_SIGNURL_KEY_NAME].each do |env_key|
-            raise ArgumentError, "Environment variable #{env_key} must be set to run GCS with CDN enabled scenario" unless ENV.key?(env_key)
+            unless ENV.key?(env_key)
+              raise ArgumentError,
+                "Environment variable #{env_key} must be set to run GCS with CDN enabled scenario"
+            end
           end
         end
 
@@ -359,7 +370,7 @@ module Gitlab
         end
 
         def video_recorder_image
-          env_var_value_if_defined('QA_VIDEO_RECORDER_IMAGE') || 'presidenten/selenoid-manual-video-recorder'
+          env_var_value_if_defined('QA_VIDEO_RECORDER_IMAGE') || 'registry.gitlab.com/gitlab-org/gitlab-qa/selenoid-manual-video-recorder'
         end
 
         def video_recorder_version
@@ -455,6 +466,10 @@ module Gitlab
           enabled?(env_var_value_if_defined('QA_MOCK_GITHUB'), default: true)
         end
 
+        def retry_failed_specs?
+          enabled?(env_var_value_if_defined('QA_RETRY_FAILED_SPECS'), default: false)
+        end
+
         private
 
         def enabled?(value, default: true)

data/lib/gitlab/qa/scenario/test/omnibus/update_from_previous.rb

@@ -111,8 +111,9 @@ module Gitlab
                 specs.args = [gitlab.address, *rspec_args]
                 next if release == current_release
 
-                # do not generate reports and metrics artifacts for non release runs
+                # do not generate reports and metrics artifacts for non release runs or retry failures
                 specs.env = { 'QA_GENERATE_ALLURE_REPORT' => false, 'QA_SAVE_TEST_METRICS' => false }
+                specs.retry_failed_specs = false
               end
             rescue Support::ShellCommand::StatusError => e
               if release == current_release # only fail on current release

data/lib/gitlab/qa/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   module QA
-    VERSION = '14.6.0'
+    VERSION = '14.7.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 14.6.0
+  version: 14.7.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-04-02 00:00:00.000000000 Z
+date: 2024-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control