gitlab-qa 2.13.1 → 3.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b30f0beb5c4bf5dbcafff124f80654b462e5c5d91a9e3bfda44c2fe1b644427a
-  data.tar.gz: f1b1f4776f80efac054598b1c70bf65a3a0d2d26ed8dfce726c860ad2828eea2
+  metadata.gz: 6bf71cde5fe8fd87db4bcd8b13638c710bd77be238a15569c7a0e8794319c361
+  data.tar.gz: 63e82bc93eafed386924bf7c66ab91f67b49b8d26f2dda5ac8a34c47cbc84779
 SHA512:
-  metadata.gz: 70b019a6ffa35174cd17751138b3180a7b155b7ac5246ce3d773be0ee8595d75db4e618422ca2eb47d74ea6a22d279670b1140642aa14eaeaebaea07239a8f4f
-  data.tar.gz: 716111739e1c955d51decc30c078f5d6aeeb15797683069ada7dcbd0acea857470b082dad6c663085d1ee0ee4cc60aa033dd5585ab620ec42e664f1b1641bf92
+  metadata.gz: bf7e5745d3e082ad7a00521e0c7b1cbc68c742064d3e5cb04422b6a7ab56bfa1a08c625019876a4aa6b7142101257d971f7f3a8cb05d7c20655383ae05f0323b
+  data.tar.gz: 073ff4e869d2d86a517e8642e25c446712e0a04e660142ac3f8662d98f61562e5f3b498836098b91eaa19ceeaed73d20ad008297b5826c81a9997d98f2a88530

data/.gitlab-ci.yml

@@ -76,14 +76,14 @@ check:rspec:
 
 ce:sanity-framework:
   script:
-    - ./bin/expect_exit_code_and_text "bin/qa Test::Instance::Image ${RELEASE:=CE} --tag framework" 1 "2 examples, 1 failure"
+    - ./bin/expect_exit_code_and_text "bin/qa Test::Instance::Image ${RELEASE:=CE} -- --tag framework" 1 "2 examples, 1 failure"
   <<: *test
   <<: *high-capacity
   <<: *ce-qa
 
 ee:sanity-framework:
   script:
-    - ./bin/expect_exit_code_and_text "bin/qa Test::Instance::Image ${RELEASE:=EE} --tag framework" 1 "2 examples, 1 failure"
+    - ./bin/expect_exit_code_and_text "bin/qa Test::Instance::Image ${RELEASE:=EE} -- --tag framework" 1 "2 examples, 1 failure"
   <<: *test
   <<: *high-capacity
   <<: *ee-qa
@@ -97,7 +97,7 @@ ce:instance:
 
 ce:instance-quarantine:
   script:
-    - bin/qa Test::Instance::Image ${RELEASE:=CE} --tag quarantine --tag ~orchestrated
+    - bin/qa Test::Instance::Image ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated
   <<: *test
   <<: *high-capacity
   <<: *ce-qa
@@ -112,7 +112,7 @@ ee:instance:
 
 ee:instance-quarantine:
   script:
-    - bin/qa Test::Instance::Image ${RELEASE:=EE} --tag quarantine --tag ~orchestrated
+    - bin/qa Test::Instance::Image ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated
   <<: *test
   <<: *high-capacity
   <<: *ee-qa
@@ -120,14 +120,14 @@ ee:instance-quarantine:
 
 ce:docker:
   script:
-    - bin/qa Test::Instance::Image ${RELEASE:=CE} --tag docker
+    - bin/qa Test::Instance::Image ${RELEASE:=CE} -- --tag docker
   <<: *test
   <<: *high-capacity
   <<: *ce-qa
 
 ee:docker:
   script:
-    - bin/qa Test::Instance::Image ${RELEASE:=EE} --tag docker
+    - bin/qa Test::Instance::Image ${RELEASE:=EE} -- --tag docker
   <<: *test
   <<: *high-capacity
   <<: *ee-qa
@@ -141,7 +141,7 @@ ce:relative_url:
 
 ce:relative_url-quarantine:
   script:
-    - bin/qa Test::Instance::RelativeUrl ${RELEASE:=CE} --tag quarantine --tag ~orchestrated
+    - bin/qa Test::Instance::RelativeUrl ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated
   <<: *test
   <<: *high-capacity
   <<: *ce-qa
@@ -156,7 +156,7 @@ ee:relative_url:
 
 ee:relative_url-quarantine:
   script:
-    - bin/qa Test::Instance::RelativeUrl ${RELEASE:=EE} --tag quarantine --tag ~orchestrated
+    - bin/qa Test::Instance::RelativeUrl ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated
   <<: *test
   <<: *high-capacity
   <<: *ee-qa
@@ -183,7 +183,7 @@ ce:update:
 
 ce:update-quarantine:
   script:
-    - bin/qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} --tag quarantine --tag ~orchestrated
+    - bin/qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated
   <<: *test
   <<: *high-capacity
   <<: *ce-qa
@@ -198,7 +198,7 @@ ee:update:
 
 ee:update-quarantine:
   script:
-    - bin/qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} --tag quarantine --tag ~orchestrated
+    - bin/qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated
   <<: *test
   <<: *high-capacity
   <<: *ee-qa
@@ -213,7 +213,7 @@ ce:upgrade:
 
 ce:upgrade-quarantine:
   script:
-    - bin/qa Test::Omnibus::Upgrade CE --tag quarantine --tag ~orchestrated
+    - bin/qa Test::Omnibus::Upgrade CE -- --tag quarantine --tag ~orchestrated
   <<: *test
   <<: *high-capacity
   <<: *only-qa
@@ -228,7 +228,7 @@ ee-previous-to-ce:update:
 
 ee-previous-to-ce:update-quarantine:
   script:
-     - bin/qa Test::Omnibus::Update EE CE --tag quarantine --tag ~orchestrated
+     - bin/qa Test::Omnibus::Update EE CE -- --tag quarantine --tag ~orchestrated
   <<: *test
   <<: *high-capacity
   <<: *only-qa
@@ -243,7 +243,7 @@ ce:mattermost:
 
 ce:mattermost-quarantine:
   script:
-    - bin/qa Test::Integration::Mattermost ${RELEASE:=CE} --tag quarantine
+    - bin/qa Test::Integration::Mattermost ${RELEASE:=CE} -- --tag quarantine
   <<: *test
   <<: *high-capacity
   <<: *ce-qa
@@ -258,7 +258,7 @@ ee:mattermost:
 
 ee:mattermost-quarantine:
   script:
-    - bin/qa Test::Integration::Mattermost ${RELEASE:=EE} --tag quarantine
+    - bin/qa Test::Integration::Mattermost ${RELEASE:=EE} -- --tag quarantine
   <<: *test
   <<: *high-capacity
   <<: *ee-qa
@@ -288,7 +288,7 @@ ce:ldap_no_tls:
 
 ce:ldap_no_tls-quarantine:
   script:
-    - bin/qa Test::Integration::LDAPNoTLS ${RELEASE:=CE} --tag quarantine
+    - bin/qa Test::Integration::LDAPNoTLS ${RELEASE:=CE} -- --tag quarantine
   <<: *test
   <<: *high-capacity
   <<: *ce-qa
@@ -303,7 +303,7 @@ ee:ldap_no_tls:
 
 ee:ldap_no_tls-quarantine:
   script:
-    - bin/qa Test::Integration::LDAPNoTLS ${RELEASE:=EE} --tag quarantine
+    - bin/qa Test::Integration::LDAPNoTLS ${RELEASE:=EE} -- --tag quarantine
   <<: *test
   <<: *high-capacity
   <<: *ee-qa
@@ -318,7 +318,7 @@ ce:ldap_tls:
 
 ce:ldap_tls-quarantine:
   script:
-    - bin/qa Test::Integration::LDAPTLS ${RELEASE:=CE} --tag quarantine
+    - bin/qa Test::Integration::LDAPTLS ${RELEASE:=CE} -- --tag quarantine
   <<: *test
   <<: *high-capacity
   <<: *ce-qa
@@ -333,7 +333,7 @@ ee:ldap_tls:
 
 ee:ldap_tls-quarantine:
   script:
-    - bin/qa Test::Integration::LDAPTLS ${RELEASE:=EE} --tag quarantine
+    - bin/qa Test::Integration::LDAPTLS ${RELEASE:=EE} -- --tag quarantine
   <<: *test
   <<: *high-capacity
   <<: *ee-qa
@@ -348,7 +348,7 @@ ce:instance_saml:
 
 ce:instance_saml-quarantine:
   script:
-  - bin/qa Test::Integration::InstanceSAML ${RELEASE:=CE} --tag quarantine
+  - bin/qa Test::Integration::InstanceSAML ${RELEASE:=CE} -- --tag quarantine
   <<: *test
   <<: *high-capacity
   <<: *ce-qa
@@ -363,7 +363,7 @@ ee:instance_saml:
 
 ee:instance_saml-quarantine:
   script:
-  - bin/qa Test::Integration::InstanceSAML ${RELEASE:=EE} --tag quarantine
+  - bin/qa Test::Integration::InstanceSAML ${RELEASE:=EE} -- --tag quarantine
   <<: *test
   <<: *high-capacity
   <<: *ee-qa
@@ -378,7 +378,7 @@ ee:group_saml:
 
 ee:group_saml-quarantine:
   script:
-  - bin/qa Test::Integration::GroupSAML ${RELEASE:=EE} --tag quarantine
+  - bin/qa Test::Integration::GroupSAML ${RELEASE:=EE} -- --tag quarantine
   <<: *test
   <<: *high-capacity
   <<: *ee-qa
@@ -393,7 +393,7 @@ ce:kubernetes:
 
 ce:kubernetes-quarantine:
   script:
-    - bin/qa Test::Integration::Kubernetes ${RELEASE:=CE} --tag quarantine
+    - bin/qa Test::Integration::Kubernetes ${RELEASE:=CE} -- --tag quarantine
   <<: *test
   <<: *high-capacity
   <<: *ce-qa
@@ -408,7 +408,7 @@ ee:kubernetes:
 
 ee:kubernetes-quarantine:
   script:
-    - bin/qa Test::Integration::Kubernetes ${RELEASE:=EE} --tag quarantine
+    - bin/qa Test::Integration::Kubernetes ${RELEASE:=EE} -- --tag quarantine
   <<: *test
   <<: *high-capacity
   <<: *ee-qa
@@ -423,7 +423,7 @@ ce:object_storage:
 
 ce:object_storage-quarantine:
   script:
-    - bin/qa Test::Integration::ObjectStorage ${RELEASE:=CE} --tag quarantine
+    - bin/qa Test::Integration::ObjectStorage ${RELEASE:=CE} -- --tag quarantine
   <<: *test
   <<: *high-capacity
   <<: *ce-qa
@@ -438,7 +438,7 @@ ee:object_storage:
 
 ee:object_storage-quarantine:
   script:
-    - bin/qa Test::Integration::ObjectStorage ${RELEASE:=EE} --tag quarantine
+    - bin/qa Test::Integration::ObjectStorage ${RELEASE:=EE} -- --tag quarantine
   <<: *test
   <<: *high-capacity
   <<: *ee-qa
@@ -453,7 +453,7 @@ ce:oauth:
 
 ce:oauth-quarantine:
   script:
-    - bin/qa Test::Integration::OAuth ${RELEASE:=CE} --tag quarantine
+    - bin/qa Test::Integration::OAuth ${RELEASE:=CE} -- --tag quarantine
   <<: *test
   <<: *high-capacity
   <<: *ce-qa
@@ -468,7 +468,7 @@ ee:oauth:
 
 ee:oauth-quarantine:
   script:
-    - bin/qa Test::Integration::OAuth ${RELEASE:=EE} --tag quarantine
+    - bin/qa Test::Integration::OAuth ${RELEASE:=EE} -- --tag quarantine
   <<: *test
   <<: *high-capacity
   <<: *ee-qa
@@ -495,20 +495,3 @@ notify_upstream_commit:failure:
   script:
     - bin/notify_upstream_commit failure
   when: on_failure
-
-.notify_slack: &notify_slack
-  image: alpine
-  stage: notify
-  dependencies: []
-  cache: {}
-  before_script:
-    - apk update && apk add git curl bash
-  only:
-    variables:
-      - $TOP_UPSTREAM_SOURCE_JOB
-
-notify_slack:
-  <<: *notify_slack
-  script:
-    - bin/slack quality-alerts "☠️ Pipeline on \`$CI_BUILD_REF_NAME\` failed! ☠️ Commit \`$(git log -1 --oneline | sed 's|\"|\\\\\"|g')\` See $CI_PIPELINE_URL (triggered from $TOP_UPSTREAM_SOURCE_JOB)" ci_failing
-  when: on_failure

data/docs/what_tests_can_be_run.md

@@ -57,6 +57,11 @@ For more details on the internals, please read the
 
 ## [Supported Remote Grid environment variables](./running_against_remote_grid.md)
 
+## Running tests with a feature flag enabled
+
+It is possible to enable a feature flag before running tests. See the [QA
+framework documentation](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/qa/README.md#running-tests-with-a-feature-flag-enabled) for details.
+
 ## Examples
 
 ### `Test::Instance::Image CE|EE|<full image address>`
@@ -200,7 +205,7 @@ $ gitlab-qa Test::Integration::LDAPNoTLS EE
 
 ### `Test::Integration::LDAPTLS CE|EE|<full image address>`
 
-This tests that a TLS enabled GitLab instance works as expected with an external TLS enabled LDAP server. 
+This tests that a TLS enabled GitLab instance works as expected with an external TLS enabled LDAP server.
 The self signed TLS certificate used for the Gitlab instance and the private key is located at: [`gitlab-org/gitlab-qa@tls_certificates/gitlab`][test-integration-ldap-tls-certs]
 
 The certificate was generated with openssl using this command:

data/exe/gitlab-qa

@@ -3,6 +3,24 @@
 require 'gitlab/qa'
 require 'optparse'
 
+# We use OptionParser here in gitlab-qa and in the QA framework in CE/EE, so we
+# need to include a separator, --, before rspec options so that OptionParser
+# doesn't reject them.
+#
+# Since we invoke OptionParser twice, the original command would need 2
+# separators otherwise the 2nd invocation of OptionParser would reject the rspec
+# tags that follow the separator.
+#
+# Rather than have to include 2 separators when we run gitlab-qa, the following
+# adds a 2nd separator if there is one in the command. It is then removed by
+# OptionParser, but when the args are passed on to the QA framework there is
+# still 1 separator so the rspec options don't cause an error when OptionParser
+# parses the options again.
+
+args = ARGV
+index = args.index('--')
+args.insert(index, '--') if index && args.count('--') == 1
+
 options = OptionParser.new do |opts|
   opts.banner = 'Usage: gitlab-qa [options] Scenario URL [[--] path] [rspec_options]'
 
@@ -17,13 +35,13 @@ options = OptionParser.new do |opts|
     exit
   end
 
-  opts.parse!
+  opts.parse!(args)
 end
 
 if ARGV.size >= 1
   Gitlab::QA::Scenario
-    .const_get(ARGV.shift)
-    .perform(*ARGV)
+    .const_get(args.shift)
+    .perform(*args)
 else
   puts options
   exit 1

data/lib/gitlab/qa/scenario/test/instance/any.rb

@@ -10,7 +10,7 @@ module Gitlab
           class Any < Scenario::Template
             def perform(edition_and_tag, address, *rspec_args)
               Component::Specs.perform do |specs|
-                specs.suite = 'Test::Instance'
+                specs.suite = 'Test::Instance::All'
                 specs.release = Release.new(edition_and_tag)
                 specs.args = [address, *rspec_args]
               end

data/lib/gitlab/qa/scenario/test/instance/image.rb

@@ -18,7 +18,7 @@ module Gitlab
 
                 gitlab.instance do
                   Component::Specs.perform do |specs|
-                    specs.suite = 'Test::Instance'
+                    specs.suite = 'Test::Instance::All'
                     specs.release = gitlab.release
                     specs.network = gitlab.network
                     specs.args = [gitlab.address, *rspec_args]

data/lib/gitlab/qa/scenario/test/instance/production.rb

@@ -24,7 +24,7 @@ module Gitlab
               end
 
               Component::Specs.perform do |specs|
-                specs.suite = 'Test::Instance'
+                specs.suite = 'Test::Instance::All'
                 specs.release = release
                 specs.args = [Component::Production::ADDRESS, *rspec_args]
               end

data/lib/gitlab/qa/scenario/test/instance/relative_url.rb

@@ -16,7 +16,7 @@ module Gitlab
 
                 gitlab.instance do
                   Component::Specs.perform do |specs|
-                    specs.suite = 'Test::Instance'
+                    specs.suite = 'Test::Instance::All'
                     specs.release = gitlab.release
                     specs.network = gitlab.network
                     specs.args = [gitlab.address, *rspec_args]

data/lib/gitlab/qa/scenario/test/instance/staging.rb

@@ -24,7 +24,7 @@ module Gitlab
               end
 
               Component::Specs.perform do |specs|
-                specs.suite = 'Test::Instance'
+                specs.suite = 'Test::Instance::All'
                 specs.release = release
                 specs.args = [Component::Staging::ADDRESS, *rspec_args]
               end

data/lib/gitlab/qa/version.rb

@@ -1,5 +1,5 @@
 module Gitlab
   module QA
-    VERSION = '2.13.1'.freeze
+    VERSION = '3.0.1'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 2.13.1
+  version: 3.0.1
 platform: ruby
 authors:
 - Grzegorz Bizon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-02-28 00:00:00.000000000 Z
+date: 2019-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -208,8 +208,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.2
 signing_key: 
 specification_version: 4
 summary: Integration tests for GitLab