gitlab-qa 6.0.0 → 6.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 87249baf75070cf8c5362c34b9995f6163370e9b6e353a393104beefe62d4bd8
-  data.tar.gz: 6e00eee6993c9f5c6aeb4cb187c708941bfd27cb9b59340165b23b02c30f583a
+  metadata.gz: 20da7f808929fe216ed49cefd35aeeab33c0e0a2c35517aa5ef15569271e2d17
+  data.tar.gz: d87695289a8d8f13f4c9baa5547a46e45f4777e07adcbfc78570f4f35a0db71f
 SHA512:
-  metadata.gz: 6d621d51ca18fa402e397b13c86dda6d52039a11ca259c6a46f3e9caffeab0faa68b6b9ff58ec4f727b2dae82c386442655122571936a47c366647b3ad895a50
-  data.tar.gz: 73bbcc44b560368e64d8bd335b3fac6d52128e8c05f6ddf582ea6139032f570d80563b6fbc9c96f41598fee2c2c345ec9155f5990f3ea4df1bde2167bede9a27
+  metadata.gz: ba356e43063cf62c035364bf6e929d2545d4ad9d6042159cf68b58cce15af7099843ab556622076b705fa5f633d7defc3158cbdaaa948fd2014e79eb5bfbe290
+  data.tar.gz: ea5048047d6c5816e0401d0bf104757be344145740063255581fbbadc27c69334ad737a23b81049ede31f488f3f13858411a1126ca07b3ee14c6a87ad8e9b030

data/.gitlab-ci.yml

@@ -94,7 +94,7 @@ release:
     - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
     - exit $test_run_exit_code
 
 .ce-qa:
@@ -143,7 +143,7 @@ release:
 .rspec-report-opts:
   variables:
     FILE_SAFE_JOB_NAME: $(echo $CI_JOB_NAME | sed 's/[ /]/_/g')
-    RSPEC_REPORT_OPTS: "--format RspecJunitFormatter --out \"tmp/rspec-${CI_JOB_ID}.xml\" --format html --out \"tmp/rspec-${FILE_SAFE_JOB_NAME}.htm\" --color --format documentation"
+    RSPEC_REPORT_OPTS: "--format QA::Support::JsonFormatter --out \"tmp/rspec-${CI_JOB_ID}.json\" --format RspecJunitFormatter --out \"tmp/rspec-${CI_JOB_ID}.xml\" --format html --out \"tmp/rspec-${FILE_SAFE_JOB_NAME}.htm\" --color --format documentation"
 
 .quarantine:
   allow_failure: true
@@ -229,44 +229,6 @@ ee:instance-quarantine:
   variables:
     QA_RSPEC_TAGS: "--tag quarantine --tag ~orchestrated"
 
-ce:docker:
-  extends:
-    - .test
-    - .high-capacity
-    - .ce-qa
-    - .rspec-report-opts
-  variables:
-    QA_RSPEC_TAGS: "--tag docker"
-
-ce:docker-quarantine:
-  extends:
-    - .test
-    - .high-capacity
-    - .ce-qa
-    - .quarantine
-    - .rspec-report-opts
-  variables:
-    QA_RSPEC_TAGS: "--tag docker --tag quarantine"
-
-ee:docker:
-  extends:
-    - .test
-    - .high-capacity
-    - .ee-qa
-    - .rspec-report-opts
-  variables:
-    QA_RSPEC_TAGS: "--tag docker"
-
-ee:docker-quarantine:
-  extends:
-    - .test
-    - .high-capacity
-    - .ee-qa
-    - .quarantine
-    - .rspec-report-opts
-  variables:
-    QA_RSPEC_TAGS: "--tag docker --tag quarantine"
-
 ce:relative_url:
   extends:
     - .test
@@ -370,7 +332,7 @@ ce:update:
   script:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -384,7 +346,7 @@ ce:update-quarantine:
   script:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -397,7 +359,7 @@ ee:update:
   script:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -411,7 +373,7 @@ ee:update-quarantine:
   script:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ] || [[ "$TOP_UPSTREAM_SOURCE_JOB" == https://ops.gitlab.net* ]]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -676,6 +638,7 @@ ce:kubernetes:
     - .high-capacity
     - .ce-qa
     - .rspec-report-opts
+  allow_failure: true
   variables:
     QA_SCENARIO: "Test::Integration::Kubernetes"
 
@@ -695,6 +658,7 @@ ee:kubernetes:
     - .high-capacity
     - .ee-qa
     - .rspec-report-opts
+  allow_failure: true
   variables:
     QA_SCENARIO: "Test::Integration::Kubernetes"
 

data/docs/what_tests_can_be_run.md

@@ -82,6 +82,8 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `GEO_MAX_DB_REPLICATION_TIME` | `120` | Maximum time that a test will wait for database data to appear on a Geo secondary node. | No|
 | `JIRA_ADMIN_USERNAME` |-  | Username for authenticating with Jira server as admin. | No|
 | `JIRA_ADMIN_PASSWORD` |-  | Password for authenticating with Jira server as admin. | No|
+| `CACHE_NAMESPACE_NAME` | `true` | Cache namespace name for groups. | No|
+| `DEPLOY_VERSION` |- | The version of GitLab being tested against. | No|
 
 ## [Supported Remote Grid environment variables](./running_against_remote_grid.md)
 
@@ -135,6 +137,10 @@ For example, the following command would use the image named `gitlab/gitlab-ee:n
 $ gitlab-qa Test::Instance::Image EE
 ```
 
+To run EE tests, the `EE_LICENSE` environment variable needs to be set:
+
+`$ export EE_LICENSE=$(cat /path/to/GitLab.gitlab_license)`
+
 ## Examples
 
 ### `Test::Instance::Image CE|EE|<full image address>`

data/lib/gitlab/qa/report/results_in_issues.rb

@@ -30,12 +30,11 @@ module Gitlab
         RETRY_BACK_OFF_DELAY = 60
         MAX_RETRY_ATTEMPTS = 3
 
-        def initialize(token:, input_files:, project: nil, input_format: :junit)
+        def initialize(token:, input_files:, project: nil)
           @token = token
           @files = Array(input_files)
           @project = project
           @retry_backoff = 0
-          @input_format = input_format.to_sym
         end
 
         def invoke!
@@ -47,11 +46,15 @@ module Gitlab
 
           Dir.glob(files).each do |file|
             puts "Reporting tests in #{file}"
-            case input_format
-            when :json
+            extension = File.extname(file)
+
+            case extension
+            when '.json'
               test_results = Report::JsonTestResults.new(file)
-            when :junit
+            when '.xml'
               test_results = Report::JUnitTestResults.new(file)
+            else
+              raise "Unknown extension #{extension}"
             end
 
             test_results.each do |test|
@@ -62,7 +65,7 @@ module Gitlab
 
         private
 
-        attr_reader :files, :token, :project, :input_format
+        attr_reader :files, :token, :project
 
         def validate_input!
           assert_project!
@@ -137,8 +140,11 @@ module Gitlab
           end
         end
 
+        # rubocop:disable Metrics/AbcSize
         def find_issue(test)
           handle_gitlab_client_exceptions do
+            return Gitlab.issue(project, id_from_testcase_url(test.testcase)) if test.testcase
+
             issues = Gitlab.issues(project, { search: search_term(test) })
               .auto_paginate
               .select { |issue| issue.state == 'opened' && issue.title.strip == title_from_test(test) }
@@ -148,6 +154,11 @@ module Gitlab
             issues.first
           end
         end
+        # rubocop:enable Metrics/AbcSize
+
+        def id_from_testcase_url(url)
+          url.split('/').last.to_i
+        end
 
         def search_term(test)
           %("#{test.file}" "#{search_safe(test.name)}")
@@ -275,7 +286,7 @@ module Gitlab
           # This error could be raised in assert_user_permission!
           # If so, we want it to terminate at that point
           raise
-        rescue SystemCallError, OpenSSL::SSL::SSLError, Net::ReadTimeout, Gitlab::Error::Parsing => e
+        rescue SystemCallError, OpenSSL::SSL::SSLError, Net::OpenTimeout, Net::ReadTimeout, Gitlab::Error::InternalServerError, Gitlab::Error::Parsing => e
           @retry_backoff += RETRY_BACK_OFF_DELAY
 
           raise if @retry_backoff > RETRY_BACK_OFF_DELAY * MAX_RETRY_ATTEMPTS

data/lib/gitlab/qa/report/test_result.rb

@@ -4,7 +4,7 @@ module Gitlab
   module QA
     module Report
       class TestResult
-        attr_accessor :name, :file, :skipped, :failures
+        attr_accessor :name, :file, :skipped, :failures, :testcase
 
         def self.from_json(test)
           new.tap do |test_result|
@@ -12,6 +12,7 @@ module Gitlab
             test_result.file = test['file_path']
             test_result.skipped = test['status'] == 'pending'
             test_result.failures = failures_from_json_exceptions(test)
+            test_result.testcase = test['testcase']
           end
         end
 
@@ -28,9 +29,13 @@ module Gitlab
           return [] unless test.key?('exceptions')
 
           test['exceptions'].map do |exception|
+            spec_file_first_index = exception['backtrace'].rindex do |line|
+              line.include?(File.basename(test['file_path']))
+            end
+
             {
               'message' => "#{exception['class']}: #{exception['message']}",
-              'stacktrace' => exception['backtrace'].join('\n')
+              'stacktrace' => "#{exception['message_lines'].join("\n")}\n#{exception['backtrace'].slice(0..spec_file_first_index).join("\n")}"
             }
           end
         end
@@ -41,9 +46,14 @@ module Gitlab
           return [] if failures.empty?
 
           failures.map do |exception|
+            trace = exception.content.split("\n").map(&:strip)
+            spec_file_first_index = trace.rindex do |line|
+              line.include?(File.basename(test['file']))
+            end
+
             {
               'message' => "#{exception['type']}: #{exception['message']}",
-              'stacktrace' => exception.content
+              'stacktrace' => trace.slice(0..spec_file_first_index).join("\n")
             }
           end
         end

data/lib/gitlab/qa/runtime/env.rb

@@ -80,7 +80,9 @@ module Gitlab
           'GEO_MAX_DB_REPLICATION_TIME' => :geo_max_db_replication_time,
           'JIRA_HOSTNAME' => :jira_hostname,
           'JIRA_ADMIN_USERNAME' => :jira_admin_username,
-          'JIRA_ADMIN_PASSWORD' => :jira_admin_password
+          'JIRA_ADMIN_PASSWORD' => :jira_admin_password,
+          'CACHE_NAMESPACE_NAME' => :cache_namespace_name,
+          'DEPLOY_VERSION' => :deploy_version
         }.freeze
 
         ENV_VARIABLES.each_value do |accessor|

data/lib/gitlab/qa/version.rb

@@ -1,5 +1,5 @@
 module Gitlab
   module QA
-    VERSION = '6.0.0'.freeze
+    VERSION = '6.2.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 6.0.0
+  version: 6.2.0
 platform: ruby
 authors:
 - Grzegorz Bizon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-08-13 00:00:00.000000000 Z
+date: 2020-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control