gitlab-qa 7.2.0 → 7.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8a3728bfd7ced04bccf14b94a1b5a175cb99167a356a3ca8780ffdb11d310a67
-  data.tar.gz: b6f04c635b53ca9879b686358d520bfdd0308c5b2c7c1f67d67b947d55539c13
+  metadata.gz: b8251bd9d33c42893133237ccc67188677306e91a30d48cc50aabe5ddbc19daf
+  data.tar.gz: ee371cbd941804d36c6fa9769241d25a1736829a1edce3970ac72175eafe3437
 SHA512:
-  metadata.gz: 9d25df710f1471b921c5a42f5f5479e5ed32131045e429867df35e1846f75fe91a0647bc580d70657e9ec97b26bb6920fc3a97ab66c7ce16b587ebfd1366b11b
-  data.tar.gz: ab823247a5d1d8313e57f46986b6a236e0fce553dc99974eda863a6f06df4927fd048148865def049cd650cdec5a61e57e5e0d132bfa0e49c3df3d9d79445614
+  metadata.gz: b94c4940266e586d1edc0782a878b950606fb6a851cfd383b2e6167287eedfb682e71758111bafd7b7920b1a0ffd403e53152407c39b1a12a7605da5bc0c6a22
+  data.tar.gz: da50ac47ec455fba82cbf2e037761e8cf32ce274995d2a11c58f3d7199e3f6e3f71d24b92d2c8c8655cd6974eebada76d81c0b7d66be4433c579312ab01282fb

data/.gitlab-ci.yml

@@ -44,6 +44,7 @@ variables:
   QA_ARTIFACTS_DIR: $CI_PROJECT_DIR
   QA_CAN_TEST_GIT_PROTOCOL_V2: "true"
   QA_CAN_TEST_PRAEFECT: "false"
+  QA_GENERATE_ALLURE_REPORT: "true"
   QA_TESTCASES_REPORTING_PROJECT: "gitlab-org/quality/testcases"
   QA_TESTCASE_SESSIONS_PROJECT: "gitlab-org/quality/testcase-sessions"
   # QA_DEFAULT_BRANCH is the default branch name of the instance under test.
@@ -153,7 +154,10 @@ rspec:
   variables:
     QA_RSPEC_TAGS: "--tag quarantine"
 
+# Do not generate allure report since it always adds a failed test to the report. The job runs a test where 1 example would always fail and assert exit code via custom script bin/expect_exit_code_and_text
 ce:sanity-framework:
+  variables:
+    QA_GENERATE_ALLURE_REPORT: "false"
   script:
     - ./bin/expect_exit_code_and_text "bundle exec exe/gitlab-qa Test::Instance::Image ${RELEASE:=CE} -- --tag framework" 1 "2 examples, 1 failure"
   extends:
@@ -162,6 +166,8 @@ ce:sanity-framework:
     - .ce-qa
 
 ee:sanity-framework:
+  variables:
+    QA_GENERATE_ALLURE_REPORT: "false"
   script:
     - ./bin/expect_exit_code_and_text "bundle exec exe/gitlab-qa Test::Instance::Image ${RELEASE:=EE} -- --tag framework" 1 "2 examples, 1 failure"
   extends:
@@ -775,6 +781,46 @@ ee:packages-quarantine:
     GITLAB_QA_OPTS: "--omnibus-config packages"
     QA_RSPEC_TAGS: "--tag quarantine --tag packages"
 
+ce:registry:
+  extends:
+    - .test
+    - .high-capacity
+    - .ce-qa
+    - .rspec-report-opts
+  allow_failure: true
+  variables:
+    QA_SCENARIO: "Test::Integration::Registry"
+
+ce:registry-quarantine:
+  extends:
+    - .test
+    - .high-capacity
+    - .ce-qa
+    - .quarantine
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::Registry"
+
+ee:registry:
+  extends:
+    - .test
+    - .high-capacity
+    - .ee-qa
+    - .rspec-report-opts
+  allow_failure: true
+  variables:
+    QA_SCENARIO: "Test::Integration::Registry"
+
+ee:registry-quarantine:
+  extends:
+    - .test
+    - .high-capacity
+    - .ee-qa
+    - .quarantine
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::Registry"
+
 ce:actioncable:
   extends:
     - .test
@@ -990,6 +1036,51 @@ ee:jira-quarantine:
   variables:
     QA_SCENARIO: "Test::Integration::Jira"
 
+generate-allure-report:
+  image:
+    name: ${GITLAB_DEPENDENCY_PROXY}andrcuns/allure-report-publisher:0.3.3
+    entrypoint: [""]
+  stage: report
+  variables:
+    GIT_STRATEGY: none
+    STORAGE_CREDENTIALS: $QA_ALLURE_REPORT_GCS_CREDENTIALS
+    GITLAB_AUTH_TOKEN: $GITLAB_QA_MR_ALLURE_REPORT_TOKEN
+    # Override allure variables for upstream triggers
+    ALLURE_PROJECT_PATH: $TOP_UPSTREAM_SOURCE_PROJECT
+    ALLURE_MERGE_REQUEST_IID: $TOP_UPSTREAM_MERGE_REQUEST_IID
+    ALLURE_COMMIT_SHA: $TOP_UPSTREAM_SOURCE_SHA
+  # Override default before script and don't push cache
+  cache:
+    policy: pull
+  before_script:
+    - echo "Generating and publishing allure test report"
+  script:
+    - |
+      allure-report-publisher upload gcs \
+        --results-glob="gitlab-qa-run-*/**/allure-results/*" \
+        --bucket="gitlab-qa-allure-reports" \
+        --prefix="${ALLURE_JOB_NAME}/${TOP_UPSTREAM_SOURCE_REF:=$CI_COMMIT_REF_SLUG}" \
+        --update-pr="comment" \
+        --copy-latest \
+        --ignore-missing-results \
+        --color
+  allow_failure: true
+  rules:
+    # Don't run report generation on release pipelines
+    - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab-qa" && $RELEASE == null'
+      changes: ["lib/**/version.rb"]
+      when: never
+    - if: '$CI_COMMIT_TAG'
+      when: never
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || ($CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH)'
+      variables:
+        ALLURE_JOB_NAME: gitlab-qa
+      when: always
+    - if: '$TOP_UPSTREAM_SOURCE_PROJECT && $TOP_UPSTREAM_SOURCE_SHA'
+      variables:
+        ALLURE_JOB_NAME: package-and-qa
+      when: always
+
 # This job requires the `GITLAB_QA_ACCESS_TOKEN` and `GITLAB_QA_DEV_ACCESS_TOKEN`
 # variable to be passed when triggered.
 staging:

data/docs/run_qa_against_gdk.md

@@ -37,18 +37,18 @@ make a few changes to your `gdk/gitlab/config/gitlab.yml` file.
   # build the QA image first
   # In gdk/gitlab:
   $ docker build -t gitlab/gitlab-ce-qa:your-custom-tag --file ./qa/Dockerfile ./
-  
-  # Note: You will also need a corresponding GitLab image. 
+
+  # Note: You will also need a corresponding GitLab image.
   # For example, if you haven't changed any application code (e.g., no new elements), you could tag the current nightly build:
   $ docker tag gitlab/gitlab-ce:nightly gitlab/gitlab-ce:your-custom-tag
 
   # Then in gitlab-qa:
   $ exe/gitlab-qa Test::Instance::Any gitlab/gitlab-ce:your-custom-tag http://192.168.0.12:3000 -- qa/specs/features/browser_ui/1_manage/login/log_in_spec.rb
   ```
-  
+
 **Note:** The hostname of the URL provided to `gitlab-qa` must match the hostname configured for GDK.
 If they do not match, a test will be signed out when it visits a page directly because the hostname of the URL visited will be different from the hostname that was used when signing in.
-  
+
 ### Running EE tests
 
 When running EE tests you'll need to have a license available. GitLab engineers can [request a license](https://about.gitlab.com/handbook/developer-onboarding/#working-on-gitlab-ee).
@@ -68,7 +68,7 @@ Run from the `gdk-ee/gitlab/qa` directory with GDK primary and secondary running
 $ bundle exec bin/qa QA::EE::Scenario::Test::Geo --primary-address http://localhost:3001 --secondary-address http://localhost:3002 --primary-name primary --secondary-name secondary --without-setup
 
 # Run in visible Chrome browser
-$ CHROME_HEADLESS=0 bundle exec bin/qa QA::EE::Scenario::Test::Geo --primary-address http://localhost:3001 --secondary-address http://localhost:3002 --primary-name primary --secondary-name secondary --without-setup
+$ WEBDRIVER_HEADLESS=0 bundle exec bin/qa QA::EE::Scenario::Test::Geo --primary-address http://localhost:3001 --secondary-address http://localhost:3002 --primary-name primary --secondary-name secondary --without-setup
 ```
 
 ### QA Tool support on macOS

data/docs/what_tests_can_be_run.md

@@ -59,7 +59,7 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `EE_LICENSE` |-  | Enterprise Edition license. | No|
 | `QA_ARTIFACTS_DIR` |`/tmp/gitlab-qa`| Path to a directory where artifacts (logs and screenshots) for failing tests will be saved. | No|
 | `DOCKER_HOST` |`http://localhost`| Docker host to run tests against. | No|
-| `CHROME_HEADLESS` |-  | When running locally, set to `false` to allow Chrome tests to be visible - watch your tests being run. | No|
+| `WEBDRIVER_HEADLESS` |-  | When running locally, set to `false` to allow Chrome tests to be visible - watch your tests being run. | No|
 | `QA_ADDITIONAL_REPOSITORY_STORAGE` |-  | The name of additional, non-default storage to be used with tests tagged `repository_storage`, run via the `Test::Instance::RepositoryStorage` scenario.  Note: Admin access is required to change repository storage. | No|
 | `QA_PRAEFECT_REPOSITORY_STORAGE` |-  | The name of repository storage using Praefect. Note: Admin access is required to change repository storage. | No|
 | `QA_COOKIES` |-  | Optionally set to "cookie1=value;cookie2=value" in order to add a cookie to every request. This can be used to set the canary cookie by setting it to "gitlab_canary=true". | No|
@@ -73,6 +73,7 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `QA_SLOW_CONNECTION_LATENCY_MS` | `2000` | The additional latency (in ms) of the simulated slow connection. | No|
 | `QA_SLOW_CONNECTION_THROUGHPUT_KBPS` | `32` | The maximum throughput (in kbps) of the simulated slow connection. | No|
 | `QA_SKIP_PULL` | `false` | Set to `true` to skip pulling docker images (e.g., to use one you built locally). | No|
+| `QA_GENERATE_ALLURE_REPORT` | `false` | When running on CI, set to `true` to generate allure reports | No|
 | `GITHUB_USERNAME` |-  | Username for authenticating with GitHub. | No|
 | `GITHUB_PASSWORD` |-  | Password for authenticating with GitHub. | No|
 | `GITLAB_QA_LOOP_RUNNER_MINUTES` | `1` | Minutes to run and repeat a spec while using the '--loop' option; default value is 1 minute. | No|
@@ -87,7 +88,6 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `GEO_FAILOVER` | `false` | Set to `true` when a Geo secondary site has been promoted to a Geo primary site. | No|
 | `GITLAB_INITIAL_ROOT_PASSWORD` | `5iveL!fe` | Initial root password for Omnibus installations | No|
 
-
 ## [Supported Remote Grid environment variables](./running_against_remote_grid.md)
 
 ## Running tests with a feature flag enabled

data/gitlab-qa.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 3.7'
   spec.add_development_dependency 'rubocop', '~> 0.82.0'
   spec.add_development_dependency 'rubocop-rspec', '~> 1.36'
+  spec.add_development_dependency 'solargraph', '~> 0.41'
   spec.add_development_dependency 'webmock', '3.7.0'
   spec.add_runtime_dependency 'activesupport', '~> 6.0.2'
   spec.add_runtime_dependency 'gitlab', '~> 4.16.1'

data/lib/gitlab/qa.rb

@@ -65,6 +65,7 @@ module Gitlab
           autoload :Jira, 'gitlab/qa/scenario/test/integration/jira'
           autoload :SSHTunnel, 'gitlab/qa/scenario/test/integration/ssh_tunnel'
           autoload :MTLS, 'gitlab/qa/scenario/test/integration/mtls'
+          autoload :ClientSSL, 'gitlab/qa/scenario/test/integration/client_ssl'
         end
 
         module Sanity

data/lib/gitlab/qa/component/gitlab.rb

@@ -65,7 +65,7 @@ module Gitlab
         end
 
         def port
-          tls ? '443' : '80'
+          tls ? '443:443' : '80'
         end
 
         def gitaly_tls

data/lib/gitlab/qa/runtime/env.rb

@@ -54,6 +54,7 @@ module Gitlab
           'QA_SIMULATE_SLOW_CONNECTION' => :qa_simulate_slow_connection,
           'QA_SLOW_CONNECTION_LATENCY_MS' => :qa_slow_connection_latency_ms,
           'QA_SLOW_CONNECTION_THROUGHPUT_KBPS' => :qa_slow_connection_throughput_kbps,
+          'QA_GENERATE_ALLURE_REPORT' => :generate_allure_report,
           'GITLAB_QA_USERNAME_1' => :gitlab_qa_username_1,
           'GITLAB_QA_PASSWORD_1' => :gitlab_qa_password_1,
           'GITLAB_QA_USERNAME_2' => :gitlab_qa_username_2,
@@ -106,7 +107,7 @@ module Gitlab
         end
 
         def default_branch
-          ENV['QA_DEFAULT_BRANCH'] || 'master'
+          ENV['QA_DEFAULT_BRANCH'] || 'main'
         end
 
         def gitlab_availability_timeout

data/lib/gitlab/qa/runtime/omnibus_configurations/license_mode.rb

@@ -7,7 +7,7 @@ module Gitlab
         class LicenseMode < Default
           def configuration
             <<~OMNIBUS
-              gitlab_rails['env'] = { 'GITLAB_LICENSE_MODE' => 'test' }
+              gitlab_rails['env'] = { 'GITLAB_LICENSE_MODE' => 'test', 'CUSTOMER_PORTAL_URL' => 'https://customers.stg.gitlab.com' }
             OMNIBUS
           end
         end

data/lib/gitlab/qa/scenario/test/instance/airgapped.rb

@@ -12,8 +12,8 @@ module Gitlab
               @commands = <<~AIRGAP_AND_VERIFY_COMMAND.split(/\n+/)
                 # Should not fail before airgapping due to eg. DNS failure
                 # Ping and wget check
-                apt-get update && apt-get install -y iptables netcat
-                nc -zv -w 10 #{gitlab_ip} 80 && (echo \"Regular connectivity netcat check passed.\" && exit 0) || (echo \"Regular connectivity netcat check failed.\" && exit 1)
+                apt-get update && apt-get install -y iptables nmap
+                nmap -sT #{gitlab_ip} -p 80 && (echo \"Regular connectivity nmap check passed.\" && exit 0) || (echo \"Regular connectivity nmap check failed.\" && exit 1)
                 echo "Checking regular connectivity..." \
                   && wget --retry-connrefused --waitretry=1 --read-timeout=15 --timeout=10 -t 2 http://registry.gitlab.com > /dev/null 2>&1 \
                   && (echo "Regular connectivity wget check passed." && exit 0) || (echo "Regular connectivity wget check failed." && exit 1)
@@ -32,9 +32,9 @@ module Gitlab
                 iptables -A INPUT -p tcp -m tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT
 
                 # Should now fail to ping and wget, port 80 should be open
-                nc -zv -w 10 #{gitlab_ip} 80 && (echo \"Airgapped network faulty. Connectivity netcat check failed.\" && exit 1) || (echo \"Connectivity netcat check passed.\" && exit 0)
-                nc -zv -w 10 127.0.0.1 22 && (echo "Airgapped connectivity port 22 check passed." && exit 0) || (echo "Airgapped connectivity port 22 check failed." && exit 1)
-                nc -zv -w 10 127.0.0.1 80 && (echo "Airgapped connectivity port 80 check passed." && exit 0) || (echo "Airgapped connectivity port 80 check failed." && exit 1)
+                nmap -sT #{gitlab_ip} -p 80 && (echo \"Airgapped network faulty. Connectivity nmap check failed.\" && exit 1) || (echo \"Connectivity nmap check passed.\" && exit 0)
+                nmap -sT  127.0.0.1 -p 22 && (echo "Airgapped connectivity port 22 check passed." && exit 0) || (echo "Airgapped connectivity port 22 check failed." && exit 1)
+                nmap -sT 10 127.0.0.1 -p 80 && (echo "Airgapped connectivity port 80 check passed." && exit 0) || (echo "Airgapped connectivity port 80 check failed." && exit 1)
                 echo "Checking airgapped connectivity..." \
                   && wget --retry-connrefused --waitretry=1 --read-timeout=15 --timeout=10 -t 2 http://registry.gitlab.com > /dev/null 2>&1 \
                   && (echo "Airgapped network faulty. Connectivity wget check failed." && exit 1) || (echo "Airgapped network confirmed. Connectivity wget check passed." && exit 0)

data/lib/gitlab/qa/scenario/test/integration/client_ssl.rb

@@ -0,0 +1,65 @@
+module Gitlab
+  module QA
+    module Scenario
+      module Test
+        module Integration
+          class ClientSSL < Scenario::Template
+            def initialize
+              @gitlab_name = 'gitlab'
+              @spec_suite = 'Test::Instance::All'
+              @network = 'test'
+              @env = {}
+              @tag = 'client_ssl'
+            end
+
+            # rubocop:disable Metrics/AbcSize
+            def perform(release, *rspec_args)
+              Component::Gitlab.perform do |gitlab|
+                gitlab.release = QA::Release.new(release)
+                gitlab.name = @gitlab_name
+                gitlab.network = @network
+                gitlab.skip_availability_check = true
+
+                gitlab.omnibus_configuration << gitlab_omnibus
+
+                gitlab.tls = true
+
+                gitlab.instance do
+                  puts 'Running Client SSL specs!'
+
+                  if @tag
+                    rspec_args << "--" unless rspec_args.include?('--')
+                    rspec_args << "--tag" << @tag
+                  end
+
+                  Component::Specs.perform do |specs|
+                    specs.suite = @spec_suite
+                    specs.release = gitlab.release
+                    specs.network = gitlab.network
+                    specs.args = [gitlab.address, *rspec_args]
+                    specs.env = @env
+                  end
+                end
+              end
+            end
+            # rubocop:enable Metrics/AbcSize
+
+            def gitlab_omnibus
+              <<~OMNIBUS
+                external_url 'https://#{@gitlab_name}.#{@network}';
+                letsencrypt['enable'] = false;
+
+                nginx['ssl_certificate'] = '/etc/gitlab/ssl/gitlab.test.crt';
+                nginx['ssl_certificate_key'] = '/etc/gitlab/ssl/gitlab.test.key';
+
+                nginx['ssl_verify_client'] = 'on';
+                nginx['ssl_client_certificate'] = '/etc/gitlab/authority/ca.pem';
+                nginx['ssl_verify_depth'] = '2';
+              OMNIBUS
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/scenario/test/integration/mtls.rb

@@ -23,7 +23,6 @@ module Gitlab
 
                 gitaly.omnibus_configuration << gitaly_omnibus
                 gitaly.gitaly_tls
-                gitaly.exec_commands = ['gitlab-ctl status']
 
                 gitaly.instance do
                   Component::Gitlab.perform do |gitlab|
@@ -33,7 +32,6 @@ module Gitlab
 
                     gitlab.omnibus_configuration << gitlab_omnibus
                     gitlab.tls = true
-                    gitlab.exec_commands = ['gitlab-ctl status']
 
                     gitlab.instance do
                       puts "Running mTLS specs!"

data/lib/gitlab/qa/version.rb

@@ -1,5 +1,5 @@
 module Gitlab
   module QA
-    VERSION = '7.2.0'.freeze
+    VERSION = '7.5.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 7.2.0
+  version: 7.5.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-04-27 00:00:00.000000000 Z
+date: 2021-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.36'
+- !ruby/object:Gem::Dependency
+  name: solargraph
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.41'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.41'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -299,6 +313,7 @@ files:
 - lib/gitlab/qa/scenario/test/instance/staging.rb
 - lib/gitlab/qa/scenario/test/instance/staging_geo.rb
 - lib/gitlab/qa/scenario/test/integration/actioncable.rb
+- lib/gitlab/qa/scenario/test/integration/client_ssl.rb
 - lib/gitlab/qa/scenario/test/integration/elasticsearch.rb
 - lib/gitlab/qa/scenario/test/integration/geo.rb
 - lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb
@@ -354,7 +369,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Integration tests for GitLab