gitlab-qa 12.4.0 → 12.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 06d3630b5975bd6c560b29280de76f5c6fdfeec2acf65775ca48646c2b413c06
-  data.tar.gz: 21d8d5abef70cf7fdcc47af4700d1c3392739efb2e923c0de71fe2614317e7e3
+  metadata.gz: 886c387d7119bbeaff903a9396e315d79563805e6582a9d479cb0d827cc3526d
+  data.tar.gz: e0a58c2a59421fbb55ba7d64613dc8240368af30e544a6d573f0ef64baf81f0b
 SHA512:
-  metadata.gz: fad44c2bdf42d67c7b71686907cf62990ee96948e0b759615cb79745d57c0c8ec91ab75298992972712dab3139fa464b69c0eb078cc3d22ef4ae2db5e2efb807
-  data.tar.gz: 2117d2c4e221dc908ef562800302c1c821f55a2cb9040f3b82529989579613da4cceeddd280497a73de45860e44ce3e611934b2907b2ded4c8a7a65e6eaa9d30
+  metadata.gz: 0b61da0814b8ce32f7fadc0f7dac3aaeb1beb280d64bbd051cae5027d3c94942977b3dae19833f4e516b9aff7e084eb3d2acb239783f33eecb64b2eed11d7ab7
+  data.tar.gz: 0ee1817892437a9157cd90ea1139040b171df703c8d4c4733287eb09c6b0c86aa8fa10cb412bd387f2e34606319399b1e09190259d2eddef0e9078b9fd5bddf3

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab-qa (12.4.0)
+    gitlab-qa (12.5.0)
       activesupport (>= 6.1, < 7.1)
       gitlab (~> 4.19)
       http (~> 5.0)

data/README.md

@@ -92,6 +92,7 @@ of tests.
 - [Trainings](docs/trainings.md)
 - [Waits](docs/waits.md)
 - [What tests can be run?](docs/what_tests_can_be_run.md)
+- [Specifics for Mac OS with M1, M2 processors & Docker Desktop](docs/specifics_for_macos_m1_m2.md)
 
 ## How do we use it
 

data/docs/configuring_omnibus.md

@@ -83,6 +83,36 @@ Notes:
 - Extending the class from `Default` is only a nicety that adds a singleton method called `Registry.configuration` and returns the configuration.
   It is recommended to extend from `Default`, but you may also extend from any other Omnibus Configuration class, including `Runtime::OmnibusConfiguration`.
 
+## Setting variables from components in Omnibus configuration using ERB template
+
+Omnibus configurations can use ERB templates to serve as a placeholder for variables. These variable can be replaced with actual value in a component
+when preparing the omnibus config.
+
+for example:
+
+ ```ruby
+#=> lib/gitlab/qa/runtime/omnibus_configurations/github_oauth.rb
+
+module Gitlab
+  module QA
+    module Runtime
+      module OmnibusConfigurations
+        class GithubOauth < Default
+          def configuration
+            <<~OMNIBUS
+              ...
+              external_url '<%= gitlab.address %>';
+            OMNIBUS
+          end
+        end
+      end
+    end
+  end
+end
+```
+
+Here, `<%= gitlab.address %>` will be replaced by calling the `address` method on an instance of the `gitlab` component.
+
 ## Running tests with Omnibus Configured
 
 All Omnibus Configurators can be called by passing arguments into the `gitlab-qa` executable.

data/docs/specifics_for_macos_m1_m2.md

@@ -0,0 +1,44 @@
+# Specifics for Mac OS with M1, M2 processors & Docker Desktop
+
+You might encounter the following errors when running tests locally, using Gitlab QA, on Mac OS with M1 or M2 processors.  These errors usually stem from using Docker images that are based on Linux/AMD64 platforms.
+
+- `MADV_DONTNEED` does not work (memset will be used instead)
+
+    ```
+    <jemalloc>: MADV_DONTNEED does not work (memset will be used instead)
+    <jemalloc>: (This is the expected behaviour if you are running under QEMU)
+    bundler: failed to load command: bin/qa (bin/qa)
+    #0 0x004000724133 <unknown>: unknown error: Chrome failed to start: crashed. (Selenium::WebDriver::Error::UnknownError)
+        (unknown error: DevToolsActivePort file doesn't exist)
+        (The process started from chrome location /usr/bin/google-chrome is no longer running,
+        so ChromeDriver is assuming that Chrome has crashed.)
+    ```
+
+- `QA::Support::Repeater::WaitExceededError`
+
+    ```
+    QA::Support::Repeater::WaitExceededError:
+        Page did not fully load. This could be due to an unending async request or loading icon.
+    ```
+
+- `Selenium::WebDriver::Error::UnknownError`
+
+    ```
+    Selenium::WebDriver::Error::UnknownError:
+        unknown error: session deleted because of page crash
+        from tab crashed
+        (Session info: headless chrome=113.0.5672.126)
+    ```
+
+To resolve these issues:
+
+1. Do not use `/dev/shm` shared memory. Set `CHROME_DISABLE_DEV_SHM` environment variable to `true`.
+    ```shell
+    $ export CHROME_DISABLE_DEV_SHM=true
+    # Disable Chrome shared memory
+    ```
+2. Enable **Use Rosetta for x86/amd64 emulation on Apple Silicon** option in Docker (Using Docker Desktop ~v4.22.1).
+    1. Open **Settings** in Docker Desktop
+    1. Go to **Features in development**
+    1. Enable **Use Rosetta for x86/amd64 emulation on Apple Silicon** setting
+    1. Select **Apply & restart**

data/docs/what_tests_can_be_run.md

@@ -112,6 +112,14 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `RELEASE_REGISTRY_URL` | - | The registry url to fetch the release image for an orchestrated GitLab. | No |
 | `RELEASE_REGISTRY_USERNAME` | - | The username to log in to the registry for pulling the release image for orchestrated GitLab. | No |
 | `RELEASE_REGISTRY_PASSWORD` | - | The password to log in to the registry for pulling the release image for orchestrated GitLab. | No |
+| `WORKSPACES_OAUTH_APP_ID` | - | Client ID for Gitlab-workspaces-proxy OAuth app used for the authentication and authorization of the workspaces running in the cluster. See https://gitlab.com/gitlab-org/remote-development/gitlab-workspaces-proxy for instructions. | No |
+| `WORKSPACES_OAUTH_APP_SECRET` | - | Client Secret for Gitlab-workspaces-proxy OAuth app used for the authentication and authorization of the workspaces running in the cluster. See https://gitlab.com/gitlab-org/remote-development/gitlab-workspaces-proxy for instructions. | No |
+| `WORKSPACES_OAUTH_SIGNING_KEY` | - | Client Signing key for Gitlab-workspaces-proxy OAuth app used for the authentication and authorization of the workspaces running in the cluster. | No |
+| `WORKSPACES_PROXY_DOMAIN` | - | The domain on which gitlab-workspaces-proxy will listen on and it is used to create workspaces url. | No |
+| `WORKSPACES_DOMAIN_CERT` | - | The fullchain.pem SSL Certificates for the `WORKSPACES_PROXY_DOMAIN`. See https://gitlab.com/gitlab-org/remote-development/gitlab-workspaces-proxy for instructions. | No |
+| `WORKSPACES_DOMAIN_KEY` | - | The privkey.pem SSL Certificates for the `WORKSPACES_PROXY_DOMAIN`. See https://gitlab.com/gitlab-org/remote-development/gitlab-workspaces-proxy for instructions. | No |
+| `WORKSPACES_WILDCARD_CERT` | - | The fullchain.pem SSL Certificates for the Wildcard `WORKSPACES_PROXY_DOMAIN`. See https://gitlab.com/gitlab-org/remote-development/gitlab-workspaces-proxy for instructions.| No |
+| `WORKSPACES_WILDCARD_KEY` | - | The privkey.pem SSL Certificates for the Wildcard `WORKSPACES_PROXY_DOMAIN`. See https://gitlab.com/gitlab-org/remote-development/gitlab-workspaces-proxy for instructions. | No |
 
 ## [Supported Remote Grid environment variables](./running_against_remote_grid.md)
 

data/lib/gitlab/qa/component/gitlab.rb

@@ -129,6 +129,7 @@ module Gitlab
         end
 
         def prepare_gitlab_omnibus_config
+          @omnibus_configuration.expand_config_template(self)
           set_formless_login_token
           set_license_mode
           set_qa_user_agent

data/lib/gitlab/qa/component/specs.rb

@@ -20,15 +20,12 @@ module Gitlab
           @env = {}
           @volumes = {}
           @additional_hosts = []
-          @default_volumes = { '/var/run/docker.sock' => '/var/run/docker.sock' }
+          @volumes = { '/var/run/docker.sock' => '/var/run/docker.sock' }
 
-          if Runtime::Env.qa_rspec_report_path.present? # rubocop:disable Style/IfUnlessModifier
-            @default_volumes[Runtime::Env.qa_rspec_report_path] = File.join(Docker::Volumes::QA_CONTAINER_WORKDIR, 'rspec')
-          end
-
-          return if Runtime::Env.qa_knapsack_report_path.blank?
-
-          @default_volumes[Runtime::Env.qa_knapsack_report_path] = File.join(Docker::Volumes::QA_CONTAINER_WORKDIR, 'knapsack')
+          include_optional_volumes(
+            Runtime::Env.qa_rspec_report_path => 'rspec',
+            Runtime::Env.qa_knapsack_report_path => 'knapsack'
+          )
         end
 
         def perform
@@ -70,7 +67,11 @@ module Gitlab
           feature_flag_sets << [] unless feature_flag_sets.any?
 
           feature_flag_sets.each do |feature_flag_set|
-            @docker.run(image: qa_image, args: [suite, *args_with_flags(args, feature_flag_set)]) do |command|
+            @docker.run(
+              image: qa_image,
+              args: [suite, *args_with_flags(args, feature_flag_set)],
+              mask_secrets: Runtime::Env.variables_to_mask
+            ) do |command|
               command << "-t --rm --net=#{network || 'bridge'}"
 
               unless hostname.nil?
@@ -92,7 +93,7 @@ module Gitlab
                 File.join(Docker::Volumes::QA_CONTAINER_WORKDIR, 'tmp')
               )
 
-              volumes.to_h.merge(default_volumes).each { |to, from| command.volume(to, from) }
+              volumes.to_h.each { |to, from| command.volume(to, from) }
 
               command.name(name)
             end
@@ -101,8 +102,6 @@ module Gitlab
 
         private
 
-        attr_reader :default_volumes
-
         def docker_pull_qa_image_if_needed
           @docker.login(**release.login_params) if release.login_params
 
@@ -129,6 +128,16 @@ module Gitlab
             "#{release.qa_image}:#{release.qa_tag}"
           end
         end
+
+        # Adds volumes to the volumes hash if the relevant host paths exist
+        #
+        # @param [Array] *args volume host_path => container_path pairs
+        # @return [void]
+        def include_optional_volumes(args)
+          args.each do |host_path, container_path|
+            host_path.present? && volumes[host_path] = File.join(Docker::Volumes::QA_CONTAINER_WORKDIR, container_path)
+          end
+        end
       end
     end
   end

data/lib/gitlab/qa/docker/command.rb

@@ -13,7 +13,7 @@ module Gitlab
         # @param [Boolean] stream_output stream command output to stdout directly instead of logger
         def initialize(cmd = nil, mask_secrets: nil, stream_output: false)
           @args = Array(cmd)
-          @mask_secrets = Array(mask_secrets)
+          @mask_secrets = mask_secrets
           @stream_output = stream_output
         end
 

data/lib/gitlab/qa/docker/engine.rb

@@ -31,8 +31,8 @@ module Gitlab
           end
         end
 
-        def run(image:, tag: nil, args: [])
-          Docker::Command.new('run', stream_output: stream_output).tap do |command|
+        def run(image:, tag: nil, args: [], mask_secrets: nil)
+          Docker::Command.new('run', stream_output: stream_output, mask_secrets: mask_secrets).tap do |command|
             yield command if block_given?
 
             command << full_image_name(image, tag)

data/lib/gitlab/qa/runtime/env.rb

@@ -156,23 +156,22 @@ module Gitlab
           'RELEASE_REGISTRY_PASSWORD' => :release_registry_password,
           'SELENOID_DIRECTORY' => :selenoid_directory,
           'USE_SELENOID' => :use_selenoid,
-          'SCHEDULE_TYPE' => :schedule_type
+          'SCHEDULE_TYPE' => :schedule_type,
+          'WORKSPACES_OAUTH_APP_ID' => :workspaces_oauth_app_id,
+          'WORKSPACES_OAUTH_APP_SECRET' => :workspaces_oauth_app_secret,
+          'WORKSPACES_OAUTH_SIGNING_KEY' => :workspaces_oauth_signing_key,
+          'WORKSPACES_PROXY_DOMAIN' => :workspaces_proxy_domain,
+          'WORKSPACES_DOMAIN_CERT' => { name: :workspaces_domain_cert, type: :file },
+          'WORKSPACES_DOMAIN_KEY' => { name: :workspaces_domain_key, type: :file },
+          'WORKSPACES_WILDCARD_CERT' => { name: :workspaces_wildcard_cert, type: :file },
+          'WORKSPACES_WILDCARD_KEY' => { name: :workspaces_wildcard_key, type: :file }
         }.freeze
 
-        ENV_VARIABLES.each do |env_name, method_name|
-          attr_writer(method_name)
-
-          define_method(method_name) do
-            env_var_value_if_defined(env_name) || (instance_variable_get("@#{method_name}") if instance_variable_defined?("@#{method_name}"))
-          end
-        end
-
         def variables
-          defined_variables = ENV_VARIABLES.each_with_object({}) do |(name, attribute), vars|
-            # Variables that are overridden in the environment take precedence
-            # over the defaults specified by the QA runtime.
-            value = env_var_name_if_defined(name) || send(attribute) # rubocop:disable GitlabSecurity/PublicSend
-            vars[name] = value if value
+          defined_variables = ENV_VARIABLES.each_with_object({}) do |(env_var_name, attributes), vars|
+            method_name, value = method_name_and_value(env_var_name, attributes, name_as_value: true)
+            value ||= send(method_name) # rubocop:disable GitlabSecurity/PublicSend
+            vars[env_var_name] = value if value
           end
           qa_variables = ENV.each_with_object({}) do |(name, _value), vars|
             next unless name.start_with?('QA_')
@@ -184,6 +183,14 @@ module Gitlab
           qa_variables.merge(defined_variables)
         end
 
+        # Variables that should be masked
+        #
+        # @return [Array] the values of the variables that should be masked
+        def variables_to_mask
+          # Consider all file variables to need masking by default because they're likely to be secrets
+          variables.select { |k, _| ENV_VARIABLES[k].is_a?(Hash) && ENV_VARIABLES[k][:type] == :file }.values
+        end
+
         def debug?
           enabled?(ENV.fetch('QA_DEBUG', nil), default: true)
         end
@@ -298,6 +305,26 @@ module Gitlab
           end
         end
 
+        def require_facebook_environment!
+          %w[QA_FACEBOOK_USERNAME QA_FACEBOOK_PASSWORD QA_FACEBOOK_OAUTH_APP_SECRET
+            QA_FACEBOOK_OAUTH_APP_ID].each do |env_key|
+            unless ENV.key?(env_key)
+              raise ArgumentError,
+                "Environment variable #{env_key} must be set to run OAuth specs"
+            end
+          end
+        end
+
+        def require_github_oauth_environment!
+          %w[QA_GITHUB_OAUTH_APP_ID QA_GITHUB_OAUTH_APP_SECRET QA_GITHUB_USERNAME
+            QA_GITHUB_PASSWORD QA_1P_EMAIL QA_1P_PASSWORD QA_1P_SECRET QA_1P_GITHUB_UUID].each do |env_key|
+            unless ENV.key?(env_key)
+              raise ArgumentError,
+                "Environment variable #{env_key} must be set to run OAuth specs"
+            end
+          end
+        end
+
         def require_initial_password!
           return unless env_var_value_if_defined('GITLAB_INITIAL_ROOT_PASSWORD').to_s.strip.empty?
 
@@ -451,6 +478,49 @@ module Gitlab
           # Pass through the variables if they are defined and not empty in the environment
           return "$#{variable}" if env_var_value_valid?(variable)
         end
+
+        def method_name_and_value(env_var_name, attributes, name_as_value: false)
+          method_name, type = method_name_and_type(attributes)
+
+          # Variables that are overridden in the environment take precedence
+          # over the defaults specified by the QA runtime.
+          value = if type == :file
+                    # If it's a file variable we pass the content of the file to avoid trying to access an invalid
+                    # path from inside the specs Docker container
+                    path = env_var_value_if_defined(env_var_name)
+                    if path.present?
+                      full_path = File.expand_path(path)
+                      File.read(full_path).strip if full_path && File.exist?(full_path)
+                    end
+                  elsif name_as_value
+                    env_var_name_if_defined(env_var_name)
+                  else
+                    env_var_value_if_defined(env_var_name)
+                  end
+
+          [method_name, value]
+        end
+
+        def method_name_and_type(attributes)
+          if attributes.is_a?(Hash) && attributes[:type] == :file
+            [attributes[:name], :file]
+          else
+            [attributes, :env_var]
+          end
+        end
+
+        # Define methods for each variable last so we can use the methods defined just above
+        ENV_VARIABLES.each do |env_var_name, attributes|
+          method_name, _ = method_name_and_type(attributes)
+          next if method_defined?(method_name) # Don't replace methods that were explicitly defined above
+
+          writer_names = attr_writer(method_name)
+          define_method(method_name) do
+            _, value = method_name_and_value(env_var_name, attributes)
+            value || (instance_variable_get("@#{method_name}") if instance_variable_defined?("@#{method_name}"))
+          end
+          public(*writer_names, method_name)
+        end
       end
     end
   end

data/lib/gitlab/qa/runtime/omnibus_configuration.rb

@@ -2,6 +2,8 @@
 
 require 'active_support'
 require 'active_support/core_ext/object/blank'
+require 'active_support/core_ext/string/inflections'
+require 'erb'
 
 module Gitlab
   module QA
@@ -25,6 +27,21 @@ module Gitlab
           sanitize!.join("\n")
         end
 
+        ERB_PATTERN = /<%=?\s?(\S+)\s?%>/
+
+        # Execute the ERB code to produce completed template
+        # @example
+        #  external_url '<%= gitlab.address %>' #=> external_url 'http://gitlab-ee-09d62235.test'
+        #
+        # @param [GitLab::QA::Component::Gitlab] gitlab
+        #
+        # @return [Array]
+        def expand_config_template(gitlab)
+          @config.map! do |item|
+            item.match(ERB_PATTERN) ? ERB.new(item).result(binding) : item
+          end
+        end
+
         def configuration
           raise NotImplementedError
         end
@@ -45,6 +62,7 @@ module Gitlab
         def sanitize!
           sanitized = @config.map do |config|
             next config if config.start_with?('#') || config.match(/\w+\(/) # allow for comments and method invocations
+            next config if config.match(ERB_PATTERN)
 
             # sometimes "=" is part of a Hash. Only split based on the first "="
             k, v = config.split("=", 2)

data/lib/gitlab/qa/runtime/omnibus_configurations/facebook_oauth.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Runtime
+      module OmnibusConfigurations
+        class FacebookOauth < Default
+          def configuration
+            Runtime::Env.require_github_oauth_environment!
+
+            <<~OMNIBUS
+              gitlab_rails['omniauth_enabled'] = true;
+              gitlab_rails['omniauth_allow_single_sign_on'] = [facebook'];
+              gitlab_rails['omniauth_block_auto_created_users'] = false;
+              gitlab_rails['omniauth_providers'] = [
+                {
+                  name: 'facebook',
+                  app_id: '$QA_FACEBOOK_OAUTH_APP_ID',
+                  app_secret: '$QA_FACEBOOK_OAUTH_APP_SECRET',
+                  verify_ssl: false
+                }
+              ];
+              letsencrypt['enable'] = false;
+              external_url '<%= gitlab.address %>';
+            OMNIBUS
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/runtime/omnibus_configurations/github_oauth.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Runtime
+      module OmnibusConfigurations
+        class GithubOauth < Default
+          def configuration
+            Runtime::Env.require_github_oauth_environment!
+
+            <<~OMNIBUS
+              gitlab_rails['omniauth_enabled'] = true
+              gitlab_rails['omniauth_allow_single_sign_on'] = ['github']
+              gitlab_rails['omniauth_block_auto_created_users'] = false
+              gitlab_rails['omniauth_providers'] = [
+                {
+                  name: 'github',
+                  app_id: '$QA_GITHUB_OAUTH_APP_ID',
+                  app_secret: '$QA_GITHUB_OAUTH_APP_SECRET',
+                  url: 'https://github.com/',
+                  verify_ssl: false,
+                  args: { scope: 'user:email' }
+                }
+              ]
+              letsencrypt['enable'] = false
+              external_url '<%= gitlab.address %>'
+            OMNIBUS
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/version.rb

@@ -2,6 +2,6 @@
 
 module Gitlab
   module QA
-    VERSION = '12.4.0'
+    VERSION = '12.5.0'
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 12.4.0
+  version: 12.5.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-08-29 00:00:00.000000000 Z
+date: 2023-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -370,6 +370,7 @@ files:
 - docs/release_process.md
 - docs/run_qa_against_gdk.md
 - docs/running_against_remote_grid.md
+- docs/specifics_for_macos_m1_m2.md
 - docs/trainings.md
 - docs/waits.md
 - docs/what_tests_can_be_run.md
@@ -418,6 +419,8 @@ files:
 - lib/gitlab/qa/runtime/omnibus_configurations/decomposition_multiple_db.rb
 - lib/gitlab/qa/runtime/omnibus_configurations/decomposition_single_db.rb
 - lib/gitlab/qa/runtime/omnibus_configurations/default.rb
+- lib/gitlab/qa/runtime/omnibus_configurations/facebook_oauth.rb
+- lib/gitlab/qa/runtime/omnibus_configurations/github_oauth.rb
 - lib/gitlab/qa/runtime/omnibus_configurations/object_storage.rb
 - lib/gitlab/qa/runtime/omnibus_configurations/object_storage_aws.rb
 - lib/gitlab/qa/runtime/omnibus_configurations/object_storage_gcs.rb