gitlab-qa 14.5.0 → 14.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b273b35fb530099bd344db6ad9f21b56476f4e5b341677e74f22a52a4fcbcad
-  data.tar.gz: f2712bf79d9fbfa72c89c3fb9d63bce0e4e8b82a164d435be79af21f167d81b6
+  metadata.gz: aca8f96e30555fcac3d968c580f1393b87d2114a3616e8ab6dbb98f913f852d4
+  data.tar.gz: 566a628d9587a7eeb6e6bcb530bed7170aa689aaadc50cecb28eaba17096e26e
 SHA512:
-  metadata.gz: ca2c75c3082c7b613e3ca3ef6f439b0873b23842d200643864cbdbb4f4d264002c8630c8f3879c28a47dbe512f7e3560e223218718faad177502104977fdb254
-  data.tar.gz: bb1aa0fd9f7f850d8b441fd56e0c1691d4cac6aaee03793fc6b4b944c945b9cfb6cf8f79b10811da2a2536262d70d9bd954fdbbf01c152f17e9b633d33c03d57
+  metadata.gz: 4304f800c1430834f7b335ddae6627e51e6126d883359feefb2863662abe274c8b3de2fadca9328c6f6e9815aeb5df2798a1e839a24cf372ee66cdd5db9bea84
+  data.tar.gz: 1a8325bf53f232d5db6292a63107b15f0acf19a04acb7dc0186474656dbd12744ab429030eb53a4e3f83acc5f9655a7159fc14d89cf81b373650b4045904e95c

data/.gitlab-ci.yml

@@ -133,9 +133,11 @@ package-and-test:
     RUN_WITH_BUNDLE: "true"
     SKIP_OMNIBUS_TRIGGER: "true"
     SKIP_REPORT_IN_ISSUES: "true"
-    ALLURE_JOB_NAME: gitlab-qa
     UPDATE_QA_CACHE: $UPDATE_QA_CACHE
     GITLAB_QA_CACHE_KEY: $GITLAB_QA_CACHE_KEY
+    QA_RETRY_FAILED_SPECS: "true"
+    QA_RUN_TYPE: gitlab-qa
+    QA_EXPORT_TEST_METRICS: "false" # skip metrics export as this pipeline is only used to validate gitlab-qa changes
   inherit:
     variables:
       - RUBY_VERSION
@@ -153,4 +155,4 @@ package-and-test:
     - if: '$CI_MERGE_REQUEST_IID'
       when: manual
       allow_failure: true
-    - when: always
+    - when: always

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-qa (14.5.0)
+    gitlab-qa (14.7.0)
       activesupport (>= 6.1, < 7.2)
       gitlab (~> 4.19)
       http (~> 5.0)
@@ -127,7 +127,7 @@ GEM
       multi_xml (>= 0.5.2)
     i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    jaro_winkler (1.5.4)
+    jaro_winkler (1.5.6)
     json (2.6.3)
     kramdown (2.4.0)
       rexml

data/README.md

@@ -121,6 +121,7 @@ All the scenarios you can run are described in the
 
 Note: The GitLab QA tool requires that [Docker](https://docs.docker.com/install/) is installed.
 
+
 ### Command-line options
 
 In addition to the [arguments you can use to specify the scenario and
@@ -166,6 +167,25 @@ useful if you want to run and debug a specific test, for example:
 $ bundle exec bin/qa Test::Instance::All http://localhost:32768 -- qa/specs/features/browser_ui/3_create/merge_request/create_merge_request_spec.rb
 ```
 
+### Lefthook
+
+[Lefthook](https://github.com/evilmartians/lefthook) is a Git hooks manager that allows
+custom logic to be executed prior to Git committing or pushing. This project comes with
+Lefthook configuration checked in (`lefthook.yml`), but it must be installed.
+
+#### Install Lefthook
+
+   ```shell
+   # Install the `lefthook` Ruby gem:
+   bundle install
+   # Initialize the lefthook config and adds to .git/hooks dir
+   bundle exec lefthook install
+   # Verify hook execution works as expected
+   bundle exec lefthook run pre-push
+   ```
+
+For a detailed guide on `lefthook` configuration see https://github.com/evilmartians/lefthook/blob/master/docs/configuration.md
+
 ### How to add new tests
 
 Please see the [Beginner's guide to writing end-to-end tests](https://docs.gitlab.com/ee/development/testing_guide/end_to_end/beginners_guide.html).

data/docs/running_against_remote_grid.md

@@ -18,7 +18,7 @@ I.e, if you have a Selenium server set up at http://localhost:4444 or if you hav
 | QA_SAVE_ALL_VIDEOS        | Used with Selenoid. Saves video for both passed and failed tests | false    | false, true                    |
 | QA_SELENOID_BROWSER_IMAGE | Used with Selenoid. Sets the browser image to use for video recording. | "selenoid/chrome" | "selenoid/chrome", "registry.gitlab.com/gitlab-org/gitlab-qa/selenoid-chrome-gitlab" |
 | QA_SELENOID_BROWSER_VERSION | Used in conjunction with QA_SELENOID_BROWSER_IMAGE. Version of browser to run against.       | "111.0"  | "latest" "111.0" "mobile-111.0"|
-| QA_VIDEO_RECORDER_IMAGE   | Used with Selenoid. Sets the video recorder image to use for video recording. | "presidenten/selenoid-manual-video-recorder" | "presidenten/selenoid-manual-video-recorder" |
+| QA_VIDEO_RECORDER_IMAGE   | Used with Selenoid. Sets the video recorder image to use for video recording. | "registry.gitlab.com/gitlab-org/gitlab-qa/selenoid-manual-video-recorder" | "registry.gitlab.com/gitlab-org/gitlab-qa/selenoid-manual-video-recorder", "presidenten/selenoid-manual-video-recorder" |
 | QA_VIDEO_RECORDER_VERSION | Used with Selenoid. Sets the video recorder image version to use for video recording. | "latest" | "latest"                     |
 | QA_REMOTE_GRID_USERNAME   | Used with Sauce Labs. Username to specify in the remote grid. "USERNAME@provider:80" |          | "gitlab-sl"                    |
 | QA_REMOTE_GRID_ACCESS_KEY | Used with Sauce Labs. Key/Token paired with `QA_REMOTE_GRID_USERNAME` |          |                                |

data/docs/what_tests_can_be_run.md

@@ -349,11 +349,13 @@ $ gitlab-qa Test::Omnibus::UpdateFromPrevious gitlab-ee:dev-tag 15.6.0-pre major
 
 ### `Test::Integration::Geo EE|<full image address>`
 
-This tests that two GitLab Geo instances work as expected.
+This tests that Geo UI proxying is working as expected.
 
-The scenario spins up a primary and secondary GitLab Geo nodes, and verifies
-that the replications (repository, attachments, project rename etc.) work as
-expected.
+The scenario spins up primary and secondary GitLab Geo nodes and
+can be used to verify that web requests to secondary Geo site return
+data that is present on the primary.
+
+See https://docs.gitlab.com/ee/administration/geo/secondary_proxy
 
 To run tests against the GitLab containers, a GitLab QA (`gitlab/gitlab-qa`)
 container is spun up and tests are run from it by running the

data/lib/gitlab/qa/component/ai_gateway.rb

@@ -6,17 +6,27 @@ module Gitlab
       class AiGateway < Base
         DOCKER_IMAGE = 'registry.gitlab.com/gitlab-org/modelops/applied-ml/code-suggestions/ai-assist/model-gateway'
         DOCKER_IMAGE_TAG = 'latest'
+        LOG_DIR = '/var/log'
 
         def name
           @name ||= 'ai-gateway'
         end
 
+        def log_volume
+          @log_volume ||= {
+            src: File.join(Runtime::Env.host_artifacts_dir, @name, 'logs'),
+            dest: LOG_DIR
+          }
+        end
+
         def configure_environment(gitlab_hostname:)
           @environment = {
             'AIGW_GITLAB_URL' => "http://#{gitlab_hostname}",
             'AIGW_GITLAB_API_URL' => "http://#{gitlab_hostname}/api/v4",
             'AIGW_CUSTOMER_PORTAL_URL' => Runtime::Env.customer_portal_url,
-            'AIGW_USE_FAKE_MODELS' => true
+            'AIGW_MOCK_MODEL_RESPONSES' => true,
+            'AIGW_LOGGING__LEVEL' => 'debug',
+            'AIGW_LOGGING__TO_FILE' => "..#{LOG_DIR}/modelgateway_debug.log"
           }
         end
       end

data/lib/gitlab/qa/component/specs.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-# rubocop:disable Metrics/CyclomaticComplexity
 # rubocop:disable Metrics/AbcSize
 
 require 'securerandom'
@@ -13,7 +12,18 @@ module Gitlab
       # the `qa/` directory located in GitLab CE / EE repositories.
       #
       class Specs < Scenario::Template
-        attr_accessor :suite, :release, :network, :args, :volumes, :env, :runner_network, :hostname, :additional_hosts
+        LAST_RUN_FILE = "examples.txt"
+
+        attr_accessor :suite,
+          :release,
+          :network,
+          :args,
+          :volumes,
+          :env,
+          :runner_network,
+          :hostname,
+          :additional_hosts,
+          :retry_failed_specs
 
         def initialize
           @docker = Docker::Engine.new(stream_output: true) # stream test output directly instead of through logger
@@ -21,6 +31,7 @@ module Gitlab
           @volumes = {}
           @additional_hosts = []
           @volumes = { '/var/run/docker.sock' => '/var/run/docker.sock' }
+          @retry_failed_specs = Runtime::Env.retry_failed_specs?
 
           include_optional_volumes(
             Runtime::Env.qa_rspec_report_path => 'rspec',
@@ -51,9 +62,30 @@ module Gitlab
           Runtime::Logger.info("Running test suite `#{suite}` for #{release.project_name}")
 
           name = "#{release.project_name}-qa-#{SecureRandom.hex(4)}"
+          run_specs(name)
+        rescue Support::ShellCommand::StatusError => e
+          raise e unless retry_failed_specs
 
-          feature_flag_sets = []
+          results_file = File.join(host_artifacts_dir(name), LAST_RUN_FILE)
+          raise "Failed to find initial run results file #{results_file}" unless File.exist?(results_file)
+
+          Runtime::Logger.warn("Initial test run failed, retrying failed specs in new process!")
+          run_specs(name, retry_process: true, initial_run_results_host_path: results_file)
+        end
+
+        private
+
+        # Ful path to tmp dir inside container
+        #
+        # @return [String]
+        def tmp_dir
+          @tmp_dir ||= File.join(Docker::Volumes::QA_CONTAINER_WORKDIR, 'tmp')
+        end
+
+        def feature_flag_sets
+          return @feature_flag_sets if defined?(@feature_flag_sets)
 
+          feature_flag_sets = []
           # When `args` includes:
           #   `[..., "--disable-feature", "a", "--enable-feature", "b", "--set-feature-flags", "c=enable", ...]`
           # `feature_flag_sets` will be set to:
@@ -62,14 +94,36 @@ module Gitlab
           while (index = args&.index { |x| x =~ /--.*-feature/ })
             feature_flag_sets << args.slice!(index, 2)
           end
-
           # When `args` do not have any feature flag options, we add [] so that test is run exactly once.
           feature_flag_sets << [] unless feature_flag_sets.any?
 
+          @feature_flag_sets = feature_flag_sets
+        end
+
+        def run_specs(name, retry_process: false, initial_run_results_host_path: nil)
+          container_name = retry_process ? "#{name}-retry" : name
+
+          env_vars = if retry_process
+                       Runtime::Env.variables.merge({
+                         **env,
+                         "QA_RSPEC_RETRIED" => "true",
+                         "NO_KNAPSACK" => "true"
+                       })
+                     else
+                       Runtime::Env.variables.merge(env)
+                     end
+
+          env_vars["RSPEC_LAST_RUN_RESULTS_FILE"] = last_run_results_file
+          # TODO: remove once rspec-retry gem is removed
+          env_vars["QA_DISABLE_RSPEC_RETRY"] = "true" if retry_failed_specs
+
+          run_volumes = volumes.to_h.merge({ host_artifacts_dir(container_name) => tmp_dir })
+          run_volumes[initial_run_results_host_path] = last_run_results_file if retry_process
+
           feature_flag_sets.each do |feature_flag_set|
             @docker.run(
               image: qa_image,
-              args: [suite, *args_with_flags(args, feature_flag_set)],
+              args: [suite, *args_with_flags(feature_flag_set, retry_process: retry_process)],
               mask_secrets: Runtime::Env.variables_to_mask
             ) do |command|
               command << "-t --rm --net=#{network || 'bridge'}"
@@ -84,37 +138,40 @@ module Gitlab
                 command << hosts # override /etc/hosts in docker container when test runs
               end
 
-              Runtime::Env.variables.merge(env).each do |key, value|
-                command.env(key, value)
-              end
-
-              command.volume(
-                File.join(Runtime::Env.host_artifacts_dir, name),
-                File.join(Docker::Volumes::QA_CONTAINER_WORKDIR, 'tmp')
-              )
-
-              volumes.to_h.each { |to, from| command.volume(to, from) }
+              env_vars.each { |key, value| command.env(key, value) }
+              run_volumes.each { |to, from| command.volume(to, from) }
 
-              command.name(name)
+              command.name(container_name)
             end
           end
         end
 
-        private
-
         def docker_pull_qa_image_if_needed
           @docker.login(**release.login_params) if release.login_params
 
           @docker.pull(image: qa_image) unless Runtime::Env.skip_pull?
         end
 
-        def args_with_flags(args, feature_flag_set)
-          return args if feature_flag_set.empty?
+        def args_with_flags(feature_flag_set, retry_process: false)
+          return args if feature_flag_set.empty? && !retry_process
 
-          Runtime::Logger.info("Running with feature flag: #{feature_flag_set.join(' ')}")
+          run_args = args.dup
+
+          if retry_process
+            if run_args.include?("--")
+              run_args.push("--only-failures")
+              first_rspec_arg_index = run_args.index("--") + 1
+              # if first arg after `--` is a not a flag, it's a specific spec/folder which needs to be removed
+              run_args.delete_at(first_rspec_arg_index) unless run_args[first_rspec_arg_index].start_with?("--")
+            else
+              run_args.push("--", "--only-failures")
+            end
+          end
 
-          args_with_f = args.dup
-          args_with_f.insert(1, *feature_flag_set)
+          return run_args if feature_flag_set.empty?
+
+          Runtime::Logger.info("Running with feature flag: #{feature_flag_set.join(' ')}")
+          run_args.insert(1, *feature_flag_set)
         end
 
         def skip_tests?
@@ -138,9 +195,23 @@ module Gitlab
             host_path.present? && volumes[host_path] = File.join(Docker::Volumes::QA_CONTAINER_WORKDIR, container_path)
           end
         end
+
+        # Full path to host artifacts dir
+        #
+        # @param [String] name
+        # @return [String]
+        def host_artifacts_dir(name)
+          File.join(Runtime::Env.host_artifacts_dir, name)
+        end
+
+        # Path to save or read run results file within container
+        #
+        # @return [String]
+        def last_run_results_file
+          File.join(tmp_dir, LAST_RUN_FILE)
+        end
       end
     end
   end
 end
-# rubocop:enable Metrics/CyclomaticComplexity
 # rubocop:enable Metrics/AbcSize

data/lib/gitlab/qa/runtime/env.rb

@@ -96,6 +96,7 @@ module Gitlab
           'CI' => :ci,
           'CI_JOB_NAME' => :ci_job_name,
           'CI_JOB_NAME_SLUG' => :ci_job_name_slug,
+          'CI_JOB_ID' => :ci_job_id,
           'CI_JOB_URL' => :ci_job_url,
           'CI_JOB_TOKEN' => :ci_job_token,
           'CI_RUNNER_ID' => :ci_runner_id,
@@ -106,6 +107,7 @@ module Gitlab
           'CI_PROJECT_NAME' => :ci_project_name,
           'CI_PROJECT_PATH' => :ci_project_path,
           'CI_PIPELINE_SOURCE' => :ci_pipeline_source,
+          'CI_PIPELINE_ID' => :ci_pipeline_id,
           'CI_PIPELINE_URL' => :ci_pipeline_url,
           'CI_PIPELINE_CREATED_AT' => :ci_pipeline_created_at,
           'CI_MERGE_REQUEST_IID' => :ci_merge_request_iid,
@@ -277,20 +279,29 @@ module Gitlab
 
         def require_aws_s3_environment!
           %w[AWS_S3_REGION AWS_S3_KEY_ID AWS_S3_ACCESS_KEY AWS_S3_BUCKET_NAME].each do |env_key|
-            raise ArgumentError, "Environment variable #{env_key} must be set to run AWS S3 object storage specs" unless ENV.key?(env_key)
+            unless ENV.key?(env_key)
+              raise ArgumentError,
+                "Environment variable #{env_key} must be set to run AWS S3 object storage specs"
+            end
           end
         end
 
         def require_gcs_environment!
           %w[GOOGLE_PROJECT GOOGLE_CLIENT_EMAIL GOOGLE_JSON_KEY GCS_BUCKET_NAME].each do |env_key|
-            raise ArgumentError, "Environment variable #{env_key} must be set to run GCS object storage specs" unless ENV.key?(env_key)
+            unless ENV.key?(env_key)
+              raise ArgumentError,
+                "Environment variable #{env_key} must be set to run GCS object storage specs"
+            end
           end
         end
 
         def require_gcs_with_cdn_environment!
           %w[GOOGLE_CDN_JSON_KEY GCS_CDN_BUCKET_NAME GOOGLE_CDN_LB GOOGLE_CDN_SIGNURL_KEY
             GOOGLE_CDN_SIGNURL_KEY_NAME].each do |env_key|
-            raise ArgumentError, "Environment variable #{env_key} must be set to run GCS with CDN enabled scenario" unless ENV.key?(env_key)
+            unless ENV.key?(env_key)
+              raise ArgumentError,
+                "Environment variable #{env_key} must be set to run GCS with CDN enabled scenario"
+            end
           end
         end
 
@@ -359,7 +370,7 @@ module Gitlab
         end
 
         def video_recorder_image
-          env_var_value_if_defined('QA_VIDEO_RECORDER_IMAGE') || 'presidenten/selenoid-manual-video-recorder'
+          env_var_value_if_defined('QA_VIDEO_RECORDER_IMAGE') || 'registry.gitlab.com/gitlab-org/gitlab-qa/selenoid-manual-video-recorder'
         end
 
         def video_recorder_version
@@ -455,6 +466,10 @@ module Gitlab
           enabled?(env_var_value_if_defined('QA_MOCK_GITHUB'), default: true)
         end
 
+        def retry_failed_specs?
+          enabled?(env_var_value_if_defined('QA_RETRY_FAILED_SPECS'), default: false)
+        end
+
         private
 
         def enabled?(value, default: true)

data/lib/gitlab/qa/scenario/test/omnibus/update_from_previous.rb

@@ -111,8 +111,9 @@ module Gitlab
                 specs.args = [gitlab.address, *rspec_args]
                 next if release == current_release
 
-                # do not generate reports and metrics artifacts for non release runs
+                # do not generate reports and metrics artifacts for non release runs or retry failures
                 specs.env = { 'QA_GENERATE_ALLURE_REPORT' => false, 'QA_SAVE_TEST_METRICS' => false }
+                specs.retry_failed_specs = false
               end
             rescue Support::ShellCommand::StatusError => e
               if release == current_release # only fail on current release

data/lib/gitlab/qa/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   module QA
-    VERSION = '14.5.0'
+    VERSION = '14.7.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 14.5.0
+  version: 14.7.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-03-22 00:00:00.000000000 Z
+date: 2024-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control