gitlab-qa 7.2.1 → 7.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0aa197f5188ff5f7026a40b7ac3e4df43bd75ea763ea6aaa368a8a5bfd01ff3c
-  data.tar.gz: bbd5ced3bf3493954930b5e8f57cc753a34e661166a3557d3ec4f9aeb9a48ad8
+  metadata.gz: 1a3edbc7044be8bd41d66f9a132918f13eafb37c31afce7423ec40f494678624
+  data.tar.gz: 4b4884f9b89b1c2575914d3e6257d7b7754bcf7af566e5f15451e18ebfde024b
 SHA512:
-  metadata.gz: 73dc0ec513ae0f3a0ea2f1f7bf3e4ea47a66caa7404b67559b4c5862dd5baa56e83700a78e31a3e16993bacf07f721619484135ee34b603bc3eca3c28bf7fccc
-  data.tar.gz: 37e979ecfebeaf3006cd622939051f22d651403bc8349d9da8bc90ba89d04cb16967c036182c687af851c1e78b6283526d0b3b79233576903336eed9316db1c6
+  metadata.gz: b42783395647923f61c5581e684e2427866ee873577bc9ce7a3ab88b432e8e9719119042e730889e0ac52828a5029731092f422376f45685d99988e335d79c4b
+  data.tar.gz: b4f70c9b678bc6fc4292951564da6f7bbeced7b611a53b555e538824d11bf9fa5abbf38b5215be753b4b673e6724d3083daed6890b7f861840b91b5246f8e195

data/.gitlab-ci.yml

@@ -44,6 +44,7 @@ variables:
   QA_ARTIFACTS_DIR: $CI_PROJECT_DIR
   QA_CAN_TEST_GIT_PROTOCOL_V2: "true"
   QA_CAN_TEST_PRAEFECT: "false"
+  QA_GENERATE_ALLURE_REPORT: "true"
   QA_TESTCASES_REPORTING_PROJECT: "gitlab-org/quality/testcases"
   QA_TESTCASE_SESSIONS_PROJECT: "gitlab-org/quality/testcase-sessions"
   # QA_DEFAULT_BRANCH is the default branch name of the instance under test.
@@ -153,7 +154,10 @@ rspec:
   variables:
     QA_RSPEC_TAGS: "--tag quarantine"
 
+# Do not generate allure report since it always adds a failed test to the report. The job runs a test where 1 example would always fail and assert exit code via custom script bin/expect_exit_code_and_text
 ce:sanity-framework:
+  variables:
+    QA_GENERATE_ALLURE_REPORT: "false"
   script:
     - ./bin/expect_exit_code_and_text "bundle exec exe/gitlab-qa Test::Instance::Image ${RELEASE:=CE} -- --tag framework" 1 "2 examples, 1 failure"
   extends:
@@ -162,6 +166,8 @@ ce:sanity-framework:
     - .ce-qa
 
 ee:sanity-framework:
+  variables:
+    QA_GENERATE_ALLURE_REPORT: "false"
   script:
     - ./bin/expect_exit_code_and_text "bundle exec exe/gitlab-qa Test::Instance::Image ${RELEASE:=EE} -- --tag framework" 1 "2 examples, 1 failure"
   extends:
@@ -775,6 +781,46 @@ ee:packages-quarantine:
     GITLAB_QA_OPTS: "--omnibus-config packages"
     QA_RSPEC_TAGS: "--tag quarantine --tag packages"
 
+ce:registry:
+  extends:
+    - .test
+    - .high-capacity
+    - .ce-qa
+    - .rspec-report-opts
+  allow_failure: true
+  variables:
+    QA_SCENARIO: "Test::Integration::Registry"
+
+ce:registry-quarantine:
+  extends:
+    - .test
+    - .high-capacity
+    - .ce-qa
+    - .quarantine
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::Registry"
+
+ee:registry:
+  extends:
+    - .test
+    - .high-capacity
+    - .ee-qa
+    - .rspec-report-opts
+  allow_failure: true
+  variables:
+    QA_SCENARIO: "Test::Integration::Registry"
+
+ee:registry-quarantine:
+  extends:
+    - .test
+    - .high-capacity
+    - .ee-qa
+    - .quarantine
+    - .rspec-report-opts
+  variables:
+    QA_SCENARIO: "Test::Integration::Registry"
+
 ce:actioncable:
   extends:
     - .test
@@ -990,6 +1036,51 @@ ee:jira-quarantine:
   variables:
     QA_SCENARIO: "Test::Integration::Jira"
 
+generate-allure-report:
+  image:
+    name: ${GITLAB_DEPENDENCY_PROXY}andrcuns/allure-report-publisher:0.3.3
+    entrypoint: [""]
+  stage: report
+  variables:
+    GIT_STRATEGY: none
+    STORAGE_CREDENTIALS: $QA_ALLURE_REPORT_GCS_CREDENTIALS
+    GITLAB_AUTH_TOKEN: $GITLAB_QA_MR_ALLURE_REPORT_TOKEN
+    # Override allure variables for upstream triggers
+    ALLURE_PROJECT_PATH: $TOP_UPSTREAM_SOURCE_PROJECT
+    ALLURE_MERGE_REQUEST_IID: $TOP_UPSTREAM_MERGE_REQUEST_IID
+    ALLURE_COMMIT_SHA: $TOP_UPSTREAM_SOURCE_SHA
+  # Override default before script and don't push cache
+  cache:
+    policy: pull
+  before_script:
+    - echo "Generating and publishing allure test report"
+  script:
+    - |
+      allure-report-publisher upload gcs \
+        --results-glob="gitlab-qa-run-*/**/allure-results/*" \
+        --bucket="gitlab-qa-allure-reports" \
+        --prefix="${ALLURE_JOB_NAME}/${TOP_UPSTREAM_SOURCE_REF:=$CI_COMMIT_REF_SLUG}" \
+        --update-pr="comment" \
+        --copy-latest \
+        --ignore-missing-results \
+        --color
+  allow_failure: true
+  rules:
+    # Don't run report generation on release pipelines
+    - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab-qa" && $RELEASE == null'
+      changes: ["lib/**/version.rb"]
+      when: never
+    - if: '$CI_COMMIT_TAG'
+      when: never
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" || ($CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH)'
+      variables:
+        ALLURE_JOB_NAME: gitlab-qa
+      when: always
+    - if: '$TOP_UPSTREAM_SOURCE_PROJECT && $TOP_UPSTREAM_SOURCE_SHA'
+      variables:
+        ALLURE_JOB_NAME: package-and-qa
+      when: always
+
 # This job requires the `GITLAB_QA_ACCESS_TOKEN` and `GITLAB_QA_DEV_ACCESS_TOKEN`
 # variable to be passed when triggered.
 staging:

data/docs/run_qa_against_gdk.md

@@ -37,18 +37,18 @@ make a few changes to your `gdk/gitlab/config/gitlab.yml` file.
   # build the QA image first
   # In gdk/gitlab:
   $ docker build -t gitlab/gitlab-ce-qa:your-custom-tag --file ./qa/Dockerfile ./
-  
-  # Note: You will also need a corresponding GitLab image. 
+
+  # Note: You will also need a corresponding GitLab image.
   # For example, if you haven't changed any application code (e.g., no new elements), you could tag the current nightly build:
   $ docker tag gitlab/gitlab-ce:nightly gitlab/gitlab-ce:your-custom-tag
 
   # Then in gitlab-qa:
   $ exe/gitlab-qa Test::Instance::Any gitlab/gitlab-ce:your-custom-tag http://192.168.0.12:3000 -- qa/specs/features/browser_ui/1_manage/login/log_in_spec.rb
   ```
-  
+
 **Note:** The hostname of the URL provided to `gitlab-qa` must match the hostname configured for GDK.
 If they do not match, a test will be signed out when it visits a page directly because the hostname of the URL visited will be different from the hostname that was used when signing in.
-  
+
 ### Running EE tests
 
 When running EE tests you'll need to have a license available. GitLab engineers can [request a license](https://about.gitlab.com/handbook/developer-onboarding/#working-on-gitlab-ee).
@@ -68,7 +68,7 @@ Run from the `gdk-ee/gitlab/qa` directory with GDK primary and secondary running
 $ bundle exec bin/qa QA::EE::Scenario::Test::Geo --primary-address http://localhost:3001 --secondary-address http://localhost:3002 --primary-name primary --secondary-name secondary --without-setup
 
 # Run in visible Chrome browser
-$ CHROME_HEADLESS=0 bundle exec bin/qa QA::EE::Scenario::Test::Geo --primary-address http://localhost:3001 --secondary-address http://localhost:3002 --primary-name primary --secondary-name secondary --without-setup
+$ WEBDRIVER_HEADLESS=0 bundle exec bin/qa QA::EE::Scenario::Test::Geo --primary-address http://localhost:3001 --secondary-address http://localhost:3002 --primary-name primary --secondary-name secondary --without-setup
 ```
 
 ### QA Tool support on macOS

data/docs/what_tests_can_be_run.md

@@ -59,7 +59,7 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `EE_LICENSE` |-  | Enterprise Edition license. | No|
 | `QA_ARTIFACTS_DIR` |`/tmp/gitlab-qa`| Path to a directory where artifacts (logs and screenshots) for failing tests will be saved. | No|
 | `DOCKER_HOST` |`http://localhost`| Docker host to run tests against. | No|
-| `CHROME_HEADLESS` |-  | When running locally, set to `false` to allow Chrome tests to be visible - watch your tests being run. | No|
+| `WEBDRIVER_HEADLESS` |-  | When running locally, set to `false` to allow Chrome tests to be visible - watch your tests being run. | No|
 | `QA_ADDITIONAL_REPOSITORY_STORAGE` |-  | The name of additional, non-default storage to be used with tests tagged `repository_storage`, run via the `Test::Instance::RepositoryStorage` scenario.  Note: Admin access is required to change repository storage. | No|
 | `QA_PRAEFECT_REPOSITORY_STORAGE` |-  | The name of repository storage using Praefect. Note: Admin access is required to change repository storage. | No|
 | `QA_COOKIES` |-  | Optionally set to "cookie1=value;cookie2=value" in order to add a cookie to every request. This can be used to set the canary cookie by setting it to "gitlab_canary=true". | No|
@@ -73,6 +73,7 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `QA_SLOW_CONNECTION_LATENCY_MS` | `2000` | The additional latency (in ms) of the simulated slow connection. | No|
 | `QA_SLOW_CONNECTION_THROUGHPUT_KBPS` | `32` | The maximum throughput (in kbps) of the simulated slow connection. | No|
 | `QA_SKIP_PULL` | `false` | Set to `true` to skip pulling docker images (e.g., to use one you built locally). | No|
+| `QA_GENERATE_ALLURE_REPORT` | `false` | When running on CI, set to `true` to generate allure reports | No|
 | `GITHUB_USERNAME` |-  | Username for authenticating with GitHub. | No|
 | `GITHUB_PASSWORD` |-  | Password for authenticating with GitHub. | No|
 | `GITLAB_QA_LOOP_RUNNER_MINUTES` | `1` | Minutes to run and repeat a spec while using the '--loop' option; default value is 1 minute. | No|
@@ -87,7 +88,6 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `GEO_FAILOVER` | `false` | Set to `true` when a Geo secondary site has been promoted to a Geo primary site. | No|
 | `GITLAB_INITIAL_ROOT_PASSWORD` | `5iveL!fe` | Initial root password for Omnibus installations | No|
 
-
 ## [Supported Remote Grid environment variables](./running_against_remote_grid.md)
 
 ## Running tests with a feature flag enabled

data/gitlab-qa.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 3.7'
   spec.add_development_dependency 'rubocop', '~> 0.82.0'
   spec.add_development_dependency 'rubocop-rspec', '~> 1.36'
+  spec.add_development_dependency 'solargraph', '~> 0.41'
   spec.add_development_dependency 'webmock', '3.7.0'
   spec.add_runtime_dependency 'activesupport', '~> 6.0.2'
   spec.add_runtime_dependency 'gitlab', '~> 4.16.1'

data/lib/gitlab/qa.rb

@@ -65,6 +65,8 @@ module Gitlab
           autoload :Jira, 'gitlab/qa/scenario/test/integration/jira'
           autoload :SSHTunnel, 'gitlab/qa/scenario/test/integration/ssh_tunnel'
           autoload :MTLS, 'gitlab/qa/scenario/test/integration/mtls'
+          autoload :ClientSSL, 'gitlab/qa/scenario/test/integration/client_ssl'
+          autoload :Registry, 'gitlab/qa/scenario/test/integration/registry'
         end
 
         module Sanity

data/lib/gitlab/qa/component/gitlab.rb

@@ -65,7 +65,7 @@ module Gitlab
         end
 
         def port
-          tls ? '443' : '80'
+          tls ? '443:443' : '80'
         end
 
         def gitaly_tls

data/lib/gitlab/qa/runtime/env.rb

@@ -54,6 +54,7 @@ module Gitlab
           'QA_SIMULATE_SLOW_CONNECTION' => :qa_simulate_slow_connection,
           'QA_SLOW_CONNECTION_LATENCY_MS' => :qa_slow_connection_latency_ms,
           'QA_SLOW_CONNECTION_THROUGHPUT_KBPS' => :qa_slow_connection_throughput_kbps,
+          'QA_GENERATE_ALLURE_REPORT' => :generate_allure_report,
           'GITLAB_QA_USERNAME_1' => :gitlab_qa_username_1,
           'GITLAB_QA_PASSWORD_1' => :gitlab_qa_password_1,
           'GITLAB_QA_USERNAME_2' => :gitlab_qa_username_2,
@@ -91,7 +92,8 @@ module Gitlab
           'DEPLOY_VERSION' => :deploy_version,
           'GITLAB_QA_USER_AGENT' => :gitlab_qa_user_agent,
           'GEO_FAILOVER' => :geo_failover,
-          'GITLAB_INITIAL_ROOT_PASSWORD' => :initial_root_password
+          'GITLAB_INITIAL_ROOT_PASSWORD' => :initial_root_password,
+          'GITLAB_TLS_CERTIFICATE' => :gitlab_tls_certificate
         }.freeze
 
         ENV_VARIABLES.each do |env_name, method_name|
@@ -106,7 +108,7 @@ module Gitlab
         end
 
         def default_branch
-          ENV['QA_DEFAULT_BRANCH'] || 'master'
+          ENV['QA_DEFAULT_BRANCH'] || 'main'
         end
 
         def gitlab_availability_timeout

data/lib/gitlab/qa/scenario/test/instance/airgapped.rb

@@ -12,8 +12,8 @@ module Gitlab
               @commands = <<~AIRGAP_AND_VERIFY_COMMAND.split(/\n+/)
                 # Should not fail before airgapping due to eg. DNS failure
                 # Ping and wget check
-                apt-get update && apt-get install -y iptables netcat
-                nc -zv -w 10 #{gitlab_ip} 80 && (echo \"Regular connectivity netcat check passed.\" && exit 0) || (echo \"Regular connectivity netcat check failed.\" && exit 1)
+                apt-get update && apt-get install -y iptables nmap
+                nmap -sT #{gitlab_ip} -p 80 && (echo \"Regular connectivity nmap check passed.\" && exit 0) || (echo \"Regular connectivity nmap check failed.\" && exit 1)
                 echo "Checking regular connectivity..." \
                   && wget --retry-connrefused --waitretry=1 --read-timeout=15 --timeout=10 -t 2 http://registry.gitlab.com > /dev/null 2>&1 \
                   && (echo "Regular connectivity wget check passed." && exit 0) || (echo "Regular connectivity wget check failed." && exit 1)
@@ -32,9 +32,9 @@ module Gitlab
                 iptables -A INPUT -p tcp -m tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT
 
                 # Should now fail to ping and wget, port 80 should be open
-                nc -zv -w 10 #{gitlab_ip} 80 && (echo \"Airgapped network faulty. Connectivity netcat check failed.\" && exit 1) || (echo \"Connectivity netcat check passed.\" && exit 0)
-                nc -zv -w 10 127.0.0.1 22 && (echo "Airgapped connectivity port 22 check passed." && exit 0) || (echo "Airgapped connectivity port 22 check failed." && exit 1)
-                nc -zv -w 10 127.0.0.1 80 && (echo "Airgapped connectivity port 80 check passed." && exit 0) || (echo "Airgapped connectivity port 80 check failed." && exit 1)
+                nmap -sT #{gitlab_ip} -p 80 && (echo \"Airgapped network faulty. Connectivity nmap check failed.\" && exit 1) || (echo \"Connectivity nmap check passed.\" && exit 0)
+                nmap -sT  127.0.0.1 -p 22 && (echo "Airgapped connectivity port 22 check passed." && exit 0) || (echo "Airgapped connectivity port 22 check failed." && exit 1)
+                nmap -sT 10 127.0.0.1 -p 80 && (echo "Airgapped connectivity port 80 check passed." && exit 0) || (echo "Airgapped connectivity port 80 check failed." && exit 1)
                 echo "Checking airgapped connectivity..." \
                   && wget --retry-connrefused --waitretry=1 --read-timeout=15 --timeout=10 -t 2 http://registry.gitlab.com > /dev/null 2>&1 \
                   && (echo "Airgapped network faulty. Connectivity wget check failed." && exit 1) || (echo "Airgapped network confirmed. Connectivity wget check passed." && exit 0)

data/lib/gitlab/qa/scenario/test/integration/client_ssl.rb

@@ -0,0 +1,65 @@
+module Gitlab
+  module QA
+    module Scenario
+      module Test
+        module Integration
+          class ClientSSL < Scenario::Template
+            def initialize
+              @gitlab_name = 'gitlab'
+              @spec_suite = 'Test::Instance::All'
+              @network = 'test'
+              @env = {}
+              @tag = 'client_ssl'
+            end
+
+            # rubocop:disable Metrics/AbcSize
+            def perform(release, *rspec_args)
+              Component::Gitlab.perform do |gitlab|
+                gitlab.release = QA::Release.new(release)
+                gitlab.name = @gitlab_name
+                gitlab.network = @network
+                gitlab.skip_availability_check = true
+
+                gitlab.omnibus_configuration << gitlab_omnibus
+
+                gitlab.tls = true
+
+                gitlab.instance do
+                  puts 'Running Client SSL specs!'
+
+                  if @tag
+                    rspec_args << "--" unless rspec_args.include?('--')
+                    rspec_args << "--tag" << @tag
+                  end
+
+                  Component::Specs.perform do |specs|
+                    specs.suite = @spec_suite
+                    specs.release = gitlab.release
+                    specs.network = gitlab.network
+                    specs.args = [gitlab.address, *rspec_args]
+                    specs.env = @env
+                  end
+                end
+              end
+            end
+            # rubocop:enable Metrics/AbcSize
+
+            def gitlab_omnibus
+              <<~OMNIBUS
+                external_url 'https://#{@gitlab_name}.#{@network}';
+                letsencrypt['enable'] = false;
+
+                nginx['ssl_certificate'] = '/etc/gitlab/ssl/gitlab.test.crt';
+                nginx['ssl_certificate_key'] = '/etc/gitlab/ssl/gitlab.test.key';
+
+                nginx['ssl_verify_client'] = 'on';
+                nginx['ssl_client_certificate'] = '/etc/gitlab/authority/ca.pem';
+                nginx['ssl_verify_depth'] = '2';
+              OMNIBUS
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/scenario/test/integration/registry.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Scenario
+      module Test
+        module Integration
+          class Registry < Scenario::Template
+            def perform(release, *rspec_args)
+              Component::Gitlab.perform do |gitlab|
+                gitlab.release = release
+                gitlab.network = 'test'
+                gitlab.name = 'gitlab'
+                gitlab.tls = true
+
+                gitlab.omnibus_configuration << <<~OMNIBUS
+                  external_url 'https://#{gitlab.name}.#{gitlab.network}';
+                  registry_external_url 'https://#{gitlab.name}.#{gitlab.network}:5050';
+
+                  letsencrypt['enable'] = false;
+
+                  nginx['redirect_http_to_https'] = true;
+                  nginx['ssl_certificate'] = "/etc/gitlab/ssl/gitlab.test.crt";
+                  nginx['ssl_certificate_key'] = "/etc/gitlab/ssl/gitlab.test.key";
+                  registry_nginx['ssl_certificate'] = "/etc/gitlab/ssl/gitlab.test.crt";
+                  registry_nginx['ssl_certificate_key'] = "/etc/gitlab/ssl/gitlab.test.key";
+                OMNIBUS
+
+                gitlab.instance do
+                  Component::Specs.perform do |specs|
+                    specs.suite = 'Test::Integration::Registry'
+                    specs.release = gitlab.release
+                    specs.network = gitlab.network
+                    specs.args = [gitlab.address, *rspec_args]
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/version.rb

@@ -1,5 +1,5 @@
 module Gitlab
   module QA
-    VERSION = '7.2.1'.freeze
+    VERSION = '7.6.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 7.2.1
+  version: 7.6.0
 platform: ruby
 authors:
 - GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-05-11 00:00:00.000000000 Z
+date: 2021-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -108,6 +108,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.36'
+- !ruby/object:Gem::Dependency
+  name: solargraph
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.41'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.41'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -299,6 +313,7 @@ files:
 - lib/gitlab/qa/scenario/test/instance/staging.rb
 - lib/gitlab/qa/scenario/test/instance/staging_geo.rb
 - lib/gitlab/qa/scenario/test/integration/actioncable.rb
+- lib/gitlab/qa/scenario/test/integration/client_ssl.rb
 - lib/gitlab/qa/scenario/test/integration/elasticsearch.rb
 - lib/gitlab/qa/scenario/test/integration/geo.rb
 - lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb
@@ -313,6 +328,7 @@ files:
 - lib/gitlab/qa/scenario/test/integration/mattermost.rb
 - lib/gitlab/qa/scenario/test/integration/mtls.rb
 - lib/gitlab/qa/scenario/test/integration/praefect.rb
+- lib/gitlab/qa/scenario/test/integration/registry.rb
 - lib/gitlab/qa/scenario/test/integration/saml.rb
 - lib/gitlab/qa/scenario/test/integration/smtp.rb
 - lib/gitlab/qa/scenario/test/integration/ssh_tunnel.rb
@@ -354,7 +370,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Integration tests for GitLab