gitlab-qa 7.0.0 → 7.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce83d37d177260e577ff1ffbc8e606eab4bc449953d8fc9de669404371433014
-  data.tar.gz: ef2a8ba114e9799c4d8b4cc71f3c098ac708b79e8573486305b9802d6bdf1826
+  metadata.gz: 748a1ab7cef2f22dba11a3989e6ff8216eab6bc06eb017b1ba52ee9fbfff031c
+  data.tar.gz: 3e95822a025fb825039e3615548eab3f5ba6046b340fdecf7fc5d65c8e16b912
 SHA512:
-  metadata.gz: 22d88835fee7158e9e5829e5d5e415abfa85d157148514c21eb904eecf69a8e8c4a49dc6e3aaae0e35824fc8f3a2be2d0c983f2f8057c915bcd0332a2fbea41f
-  data.tar.gz: 6e8007813c6258741ba516b011b8a5fc7efa1ee1ada5c385580477a18b02c35ac8e232370a80395fd3259f98058abaaffe11941c7ec9f8108a19dd36aafa4df3
+  metadata.gz: b80fc3ecf1eb1934126ed816b73c7f73d55a9364fec8afe035bb7d36c34d99e25003f036dba237dea3e6888102948aa303e3d18d931ae92b72ef306b1538bee6
+  data.tar.gz: 7997a9183233fa52f52e7cf71b6a8a43577ae0631297244a2b9ac20d65a17ad4e7a300ea9af3361a67681be668e7b252e95d27ed1a458b8294ec4af56cea1800

data/.gitlab-ci.yml

@@ -75,9 +75,9 @@ rspec:
     reports:
       junit: gitlab-qa-run-*/**/rspec-*.xml
   script:
-    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
-    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
-    - exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
+    - 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
+    - bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
+    - bundle exec exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
     - exit $test_run_exit_code
@@ -153,7 +153,7 @@ rspec:
 
 ce:sanity-framework:
   script:
-    - ./bin/expect_exit_code_and_text "exe/gitlab-qa Test::Instance::Image ${RELEASE:=CE} -- --tag framework" 1 "2 examples, 1 failure"
+    - ./bin/expect_exit_code_and_text "bundle exec exe/gitlab-qa Test::Instance::Image ${RELEASE:=CE} -- --tag framework" 1 "2 examples, 1 failure"
   extends:
     - .test
     - .high-capacity
@@ -161,7 +161,7 @@ ce:sanity-framework:
 
 ee:sanity-framework:
   script:
-    - ./bin/expect_exit_code_and_text "exe/gitlab-qa Test::Instance::Image ${RELEASE:=EE} -- --tag framework" 1 "2 examples, 1 failure"
+    - ./bin/expect_exit_code_and_text "bundle exec exe/gitlab-qa Test::Instance::Image ${RELEASE:=EE} -- --tag framework" 1 "2 examples, 1 failure"
   extends:
     - .test
     - .high-capacity
@@ -170,8 +170,8 @@ ee:sanity-framework:
 # The custom jobs are for manually running specific/alternative tests in MRs, so we don't report them in issues
 ce:custom-parallel:
   script:
-    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
-    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
+    - 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
+    - bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
   extends:
     - .test
     - .high-capacity
@@ -182,8 +182,8 @@ ce:custom-parallel:
 
 ee:custom-parallel:
   script:
-    - 'echo "Running: exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
-    - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
+    - 'echo "Running: bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"'
+    - bundle exec exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS
   extends:
     - .test
     - .high-capacity
@@ -315,14 +315,14 @@ ee:repository_storage-quarantine:
 # The Test::Omnibus::Image scenarios don't run the E2E tests so they don't need to report test results
 ce:image:
   script:
-    - exe/gitlab-qa Test::Omnibus::Image ${RELEASE:=CE}
+    - bundle exec exe/gitlab-qa Test::Omnibus::Image ${RELEASE:=CE}
   extends:
     - .test
     - .ce-qa
 
 ee:image:
   script:
-    - exe/gitlab-qa Test::Omnibus::Image ${RELEASE:=EE}
+    - bundle exec exe/gitlab-qa Test::Omnibus::Image ${RELEASE:=EE}
   extends:
     - .test
     - .ee-qa
@@ -331,7 +331,7 @@ ee:image:
 # So instead we include the script here again, with two release variables
 ce:update:
   script:
-    - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
+    - bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
     - exit $test_run_exit_code
@@ -345,7 +345,7 @@ ce:update:
 
 ce:update-quarantine:
   script:
-    - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
+    - bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
     - exit $test_run_exit_code
@@ -358,7 +358,7 @@ ce:update-quarantine:
 
 ee:update:
   script:
-    - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
+    - bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
     - exit $test_run_exit_code
@@ -372,7 +372,7 @@ ee:update:
 
 ee:update-quarantine:
   script:
-    - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
+    - bundle exec exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
     - if [ "$TOP_UPSTREAM_SOURCE_REF" == $TOP_UPSTREAM_DEFAULT_BRANCH ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
     - exit $test_run_exit_code
@@ -386,7 +386,7 @@ ee:update-quarantine:
 # The Test::Omnibus::Upgrade scenario isn't run on master (because it always uses the latest CE/EE image) so we don't report the test results in issues
 ce:upgrade:
   script:
-    - exe/gitlab-qa Test::Omnibus::Upgrade CE -- $RSPEC_REPORT_OPTS
+    - bundle exec exe/gitlab-qa Test::Omnibus::Upgrade CE -- $RSPEC_REPORT_OPTS
   extends:
     - .test
     - .high-capacity
@@ -397,7 +397,7 @@ ce:upgrade:
 
 ce:upgrade-quarantine:
   script:
-    - exe/gitlab-qa Test::Omnibus::Upgrade CE -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS
+    - bundle exec exe/gitlab-qa Test::Omnibus::Upgrade CE -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS
   extends:
     - .test
     - .high-capacity
@@ -407,7 +407,7 @@ ce:upgrade-quarantine:
 
 ee-previous-to-ce:update:
   script:
-    - exe/gitlab-qa Test::Omnibus::Update EE CE -- $RSPEC_REPORT_OPTS
+    - bundle exec exe/gitlab-qa Test::Omnibus::Update EE CE -- $RSPEC_REPORT_OPTS
   extends:
     - .test
     - .high-capacity
@@ -416,7 +416,7 @@ ee-previous-to-ce:update:
 
 ee-previous-to-ce:update-quarantine:
   script:
-    - exe/gitlab-qa Test::Omnibus::Update EE CE -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS
+    - bundle exec exe/gitlab-qa Test::Omnibus::Update EE CE -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS
   extends:
     - .test
     - .high-capacity
@@ -719,7 +719,7 @@ ce:object_storage:
     - .ce-qa
     - .rspec-report-opts
   variables:
-    QA_SCENARIO: "Test::Integration::ObjectStorage"
+    GITLAB_QA_OPTS: "--omnibus-config object_storage"
 
 ce:object_storage-quarantine:
   extends:
@@ -729,7 +729,7 @@ ce:object_storage-quarantine:
     - .quarantine
     - .rspec-report-opts
   variables:
-    QA_SCENARIO: "Test::Integration::ObjectStorage"
+    GITLAB_QA_OPTS: "--omnibus-config object_storage"
 
 ee:object_storage:
   extends:
@@ -738,7 +738,7 @@ ee:object_storage:
     - .ee-qa
     - .rspec-report-opts
   variables:
-    QA_SCENARIO: "Test::Integration::ObjectStorage"
+    GITLAB_QA_OPTS: "--omnibus-config object_storage"
 
 ee:object_storage-quarantine:
   extends:
@@ -748,7 +748,7 @@ ee:object_storage-quarantine:
     - .quarantine
     - .rspec-report-opts
   variables:
-    QA_SCENARIO: "Test::Integration::ObjectStorage"
+    GITLAB_QA_OPTS: "--omnibus-config object_storage"
 
 ee:packages:
   extends:
@@ -757,7 +757,7 @@ ee:packages:
     - .ee-qa
     - .rspec-report-opts
   variables:
-    QA_SCENARIO: "Test::Integration::Packages"
+    GITLAB_QA_OPTS: "--omnibus-config packages"
 
 ee:packages-quarantine:
   extends:
@@ -767,7 +767,7 @@ ee:packages-quarantine:
     - .quarantine
     - .rspec-report-opts
   variables:
-    QA_SCENARIO: "Test::Integration::Packages"
+    GITLAB_QA_OPTS: "--omnibus-config packages"
 
 ce:actioncable:
   extends:
@@ -989,8 +989,8 @@ ee:jira-quarantine:
 staging:
   script:
     - unset EE_LICENSE
-    - 'echo "Running: exe/gitlab-qa Test::Instance::Staging ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS"'
-    - exe/gitlab-qa Test::Instance::Staging ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS
+    - 'echo "Running: bundle exec exe/gitlab-qa Test::Instance::Staging ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS"'
+    - bundle exec exe/gitlab-qa Test::Instance::Staging ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS
   extends:
     - .test
     - .high-capacity
@@ -1011,7 +1011,7 @@ generate_test_session:
       - REPORT_ISSUE_URL
   script:
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - exe/gitlab-qa-report --generate-test-session "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASE_SESSIONS_PROJECT"
+    - bundle exec exe/gitlab-qa-report --generate-test-session "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASE_SESSIONS_PROJECT"
 
 relate_test_failures:
   stage: report
@@ -1029,7 +1029,7 @@ relate_test_failures:
     QA_FAILURES_REPORTER_OPTIONS: ""
   script:
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" --max-diff-ratio "$QA_FAILURES_MAX_DIFF_RATIO" $QA_FAILURES_REPORTER_OPTIONS
+    - bundle exec exe/gitlab-qa-report --relate-failure-issue "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_FAILURES_REPORTING_PROJECT" --max-diff-ratio "$QA_FAILURES_MAX_DIFF_RATIO" $QA_FAILURES_REPORTER_OPTIONS
 
 .notify_upstream_commit:
   stage: notify

data/docs/what_tests_can_be_run.md

@@ -534,6 +534,18 @@ $ gitlab-qa Test::Integration::Mattermost EE
 
 ### `Test::Integration::Packages CE|EE|<full image address>`
 
+**Note: This Scenario no longer exists.  See https://gitlab.com/gitlab-org/gitlab-qa/-/merge_requests/662**
+
+To run Packages tests, you may [configure Omnibus](configuring_omnibus.md) to use the [Packages](https://gitlab.com/gitlab-org/gitlab-qa/-/blob/master/lib/gitlab/qa/runtime/omnibus_configurations/packages.rb) configurator.
+
+Example:
+
+```
+$ export EE_LICENSE=$(cat /path/to/Geo.gitlab_license)
+
+$ gitlab-qa Test::Instance::Image EE --omnibus-config packages
+```
+
 This tests the GitLab Package Registry feature by setting
 `gitlab_rails['packages_enabled'] = true` in the Omnibus configuration
 before starting the GitLab container.
@@ -547,14 +559,6 @@ which runs only the tests with `:packages` metadata.
 
 - `EE_LICENSE`: A valid EE license.
 
-Example:
-
-```
-$ export EE_LICENSE=$(cat /path/to/Geo.gitlab_license)
-
-$ gitlab-qa Test::Integration::Packages EE
-```
-
 ### `Test::Integration::Praefect CE|EE|<full image address>`
 
 This tests [Praefect](https://docs.gitlab.com/ee/administration/gitaly/praefect.html),

data/gitlab-qa.gemspec

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'gitlab/qa/version'
@@ -5,11 +7,11 @@ require 'gitlab/qa/version'
 Gem::Specification.new do |spec|
   spec.name          = 'gitlab-qa'
   spec.version       = Gitlab::QA::VERSION
-  spec.authors       = ['Grzegorz Bizon']
-  spec.email         = ['grzesiek.bizon@gmail.com']
+  spec.authors       = ['GitLab Quality']
+  spec.email         = ['gitlab-qa@gmail.com']
 
   spec.summary       = 'Integration tests for GitLab'
-  spec.homepage      = 'http://about.gitlab.com'
+  spec.homepage      = 'http://about.gitlab.com/'
   spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`

data/lib/gitlab/qa/runtime/omnibus_configuration.rb

@@ -33,14 +33,15 @@ module Gitlab
         # rubocop:disable Metrics/AbcSize
         def sanitize!
           sanitized = @config.map do |config|
-            next config if config.start_with?('#') # allow for comment
+            next config if config.start_with?('#') || config.match(/\w+\(/) # allow for comments and method invocations
 
             # sometimes "=" is part of a Hash. Only split based on the first "="
             k, v = config.split("=", 2)
             # make sure each config is well-formed
             # e.g., gitlab_rails['packages_enabled'] = true
             #   NOT gitlab_rails['packages_enabled']=true
-            "#{k.strip} = #{v.strip.tr('"', "'")}".strip
+
+            v.nil? ? k.strip : "#{k.strip} = #{v.strip.tr('"', "'")}".strip
           end.uniq
 
           errors = []

data/lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb

@@ -131,8 +131,7 @@ module Gitlab
 
             def gitaly_omnibus_configuration
               <<~OMNIBUS
-                #{disable_other_services}
-                prometheus['enable'] = true;
+                #{disable_other_services.sub(/(prometheus\['enable'\]) = false/, '\1 = true')}
                 prometheus_monitoring['enable'] = false;
                 gitaly['enable'] = true;
                 gitaly['listen_addr'] = '0.0.0.0:8075';

data/lib/gitlab/qa/scenario/test/integration/ldap.rb

@@ -27,7 +27,7 @@ module Gitlab
             end
 
             def ldap_servers_omnibus_config
-              config = YAML.safe_load <<~CFG
+              YAML.safe_load <<~CFG
                 main:
                   label: LDAP
                   host: #{ldap_hostname}
@@ -44,9 +44,6 @@ module Gitlab
                   external_groups: ''
                   sync_ssh_keys: false
               CFG
-
-              # Quotes get eaten up when the string is set in the environment
-              config.to_s.gsub("\"", "\\\"")
             end
 
             def ldap_hostname

data/lib/gitlab/qa/version.rb

@@ -1,5 +1,5 @@
 module Gitlab
   module QA
-    VERSION = '7.0.0'.freeze
+    VERSION = '7.0.1'.freeze
   end
 end

metadata

@@ -1,10 +1,10 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 7.0.0
+  version: 7.0.1
 platform: ruby
 authors:
-- Grzegorz Bizon
+- GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
@@ -194,7 +194,7 @@ dependencies:
         version: 1.5.7
 description: 
 email:
-- grzesiek.bizon@gmail.com
+- gitlab-qa@gmail.com
 executables:
 - gitlab-qa
 - gitlab-qa-report
@@ -333,7 +333,7 @@ files:
 - tls_certificates/gitlab/gitlab.test.crt
 - tls_certificates/gitlab/gitlab.test.csr
 - tls_certificates/gitlab/gitlab.test.key
-homepage: http://about.gitlab.com
+homepage: http://about.gitlab.com/
 licenses:
 - MIT
 metadata: {}