gitlab-qa 5.17.0 → 6.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a9c4251f7eaa756abc42d5cb76b0b318ec3b14004331d32a92e125135055ecef
-  data.tar.gz: 04bb9549728a2533cfb34b2e5ade00260511fda1d6e4d052769d9682b73b497b
+  metadata.gz: 6f02b51e013ff64f6e9080771f02e439efcf9093cedba40ca625fd10a932436e
+  data.tar.gz: 7089a36078ed197e4abb318f78d67d8f04289803c5bbfb6598fcd0761d071e6b
 SHA512:
-  metadata.gz: dfb8a2afb17ac4d47e0c770c8da39a3290d6ac58d6306f3a930d50b08180660c85995ff248b1f50bdd6f8d9fe26b29f8680016b03ff07f85d5600e1bd5c7be90
-  data.tar.gz: a84cd2cef18c8f229826b47236edd8749345de2bfb51be1705109d623b0fefec7b09088f46244df206f85665420fb22dbdc3d8a250e617a8ab45876e95fc99d0
+  metadata.gz: 4906ac6a4ddacb5061c870a2e7e484741c60b488a8efb5051187d768c7a9a990d73b4bb5ba47ff9cf48980a63197da25890c428b27aaddb5caf69e06931ced0e
+  data.tar.gz: 8dd810d48fcbbd70f858ff4fd511a9c79f29131f13968b2e990483a79e927cd6f7841fcbf47dd6ea63933ea09b7319ff630b1d337085e4a4a731eb606a9c8cc8

data/.gitlab-ci.yml

@@ -94,7 +94,7 @@ release:
     - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
     - exit $test_run_exit_code
 
 .ce-qa:
@@ -143,7 +143,7 @@ release:
 .rspec-report-opts:
   variables:
     FILE_SAFE_JOB_NAME: $(echo $CI_JOB_NAME | sed 's/[ /]/_/g')
-    RSPEC_REPORT_OPTS: "--format RspecJunitFormatter --out \"tmp/rspec-${CI_JOB_ID}.xml\" --format html --out \"tmp/rspec-${FILE_SAFE_JOB_NAME}.htm\" --color --format documentation"
+    RSPEC_REPORT_OPTS: "--format QA::Support::JsonFormatter --out \"tmp/rspec-${CI_JOB_ID}.json\" --format RspecJunitFormatter --out \"tmp/rspec-${CI_JOB_ID}.xml\" --format html --out \"tmp/rspec-${FILE_SAFE_JOB_NAME}.htm\" --color --format documentation"
 
 .quarantine:
   allow_failure: true
@@ -370,7 +370,7 @@ ce:update:
   script:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -384,7 +384,7 @@ ce:update-quarantine:
   script:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=CE} ${RELEASE:=CE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -397,7 +397,7 @@ ee:update:
   script:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi
     - exit $test_run_exit_code
   extends:
     - .test
@@ -411,7 +411,7 @@ ee:update-quarantine:
   script:
     - exe/gitlab-qa Test::Omnibus::Update ${RELEASE:=EE} ${RELEASE:=EE} -- --tag quarantine --tag ~orchestrated $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.json" --project "$QA_TESTCASES_REPORTING_PROJECT"; fi
     - exit $test_run_exit_code
   extends:
     - .test

data/docs/what_tests_can_be_run.md

@@ -82,6 +82,7 @@ All environment variables used by GitLab QA should be defined in [`lib/gitlab/qa
 | `GEO_MAX_DB_REPLICATION_TIME` | `120` | Maximum time that a test will wait for database data to appear on a Geo secondary node. | No|
 | `JIRA_ADMIN_USERNAME` |-  | Username for authenticating with Jira server as admin. | No|
 | `JIRA_ADMIN_PASSWORD` |-  | Password for authenticating with Jira server as admin. | No|
+| `CACHE_NAMESPACE_NAME` | `true` | Cache namespace name for groups. | No|
 
 ## [Supported Remote Grid environment variables](./running_against_remote_grid.md)
 
@@ -135,6 +136,10 @@ For example, the following command would use the image named `gitlab/gitlab-ee:n
 $ gitlab-qa Test::Instance::Image EE
 ```
 
+To run EE tests, the `EE_LICENSE` environment variable needs to be set:
+
+`$ export EE_LICENSE=$(cat /path/to/GitLab.gitlab_license)`
+
 ## Examples
 
 ### `Test::Instance::Image CE|EE|<full image address>`
@@ -240,6 +245,25 @@ $ gitlab-qa Test::Integration::Geo EE
 
 [test-geo]: https://gitlab.com/gitlab-org/gitlab-ee/blob/master/qa/qa/ee/scenario/test/geo.rb
 
+### `Test::Integration::GitalyCluster CE|EE|<full image address>`
+
+This tests [Gitaly Cluster](https://docs.gitlab.com/ee/administration/gitaly/praefect.html),
+a clustered configuration of the Gitaly repository storage service.
+
+The scenario configures and starts several docker containers to
+represent the [recommended minimum configuration](https://docs.gitlab.com/ee/administration/gitaly/praefect.html#requirements-for-configuring-a-gitaly-cluster)
+of a Gitaly Cluster.
+
+To run tests against the GitLab container, a GitLab QA (`gitlab/gitlab-qa`)
+container is spun up and tests are run from it by running the
+`Test::Instance::All` scenario with the `:gitaly_cluster` tag.
+
+Example:
+
+```
+$ gitlab-qa Test::Integration::GitalyCluster EE
+```
+
 ### `Test::Integration::LDAPNoTLS CE|EE|<full image address>`
 
 This tests that a GitLab instance works as expected with an external

data/lib/gitlab/qa.rb

@@ -97,9 +97,12 @@ module Gitlab
     end
 
     module Report
+      autoload :JsonTestResults, 'gitlab/qa/report/json_test_results'
+      autoload :JUnitTestResults, 'gitlab/qa/report/junit_test_results'
       autoload :PrepareStageReports, 'gitlab/qa/report/prepare_stage_reports'
       autoload :ResultsInIssues, 'gitlab/qa/report/results_in_issues'
       autoload :SummaryTable, 'gitlab/qa/report/summary_table'
+      autoload :TestResult, 'gitlab/qa/report/test_result'
       autoload :UpdateScreenshotPath, 'gitlab/qa/report/update_screenshot_path'
     end
 

data/lib/gitlab/qa/component/base.rb

@@ -41,26 +41,16 @@ module Gitlab
           raise NotImplementedError, "#{self.class.name} must specify a docker image tag as DOCKER_IMAGE_TAG"
         end
 
-        def instance
+        def instance(skip_teardown: false)
           instance_no_teardown do
             yield self if block_given?
           end
         ensure
-          teardown
+          teardown unless skip_teardown
         end
 
         alias_method :launch_and_teardown_instance, :instance
 
-        def instance_no_teardown
-          prepare
-          start
-          reconfigure
-          wait_until_ready
-          process_exec_commands
-
-          yield self if block_given?
-        end
-
         def prepare
           prepare_docker_image
           prepare_network
@@ -147,6 +137,16 @@ module Gitlab
           raise 'Invalid instance name!' unless name
         end
 
+        def instance_no_teardown
+          prepare
+          start
+          reconfigure
+          wait_until_ready
+          process_exec_commands
+
+          yield self if block_given?
+        end
+
         def teardown?
           !Runtime::Scenario.attributes.include?(:teardown) || Runtime::Scenario.teardown
         end

data/lib/gitlab/qa/report/json_test_results.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Gitlab
+  module QA
+    module Report
+      class JsonTestResults
+        include Enumerable
+
+        def initialize(file)
+          @testcases = JSON.parse(File.read(file))['examples'].map { |test| TestResult.from_json(test) }
+        end
+
+        def each(&block)
+          @testcases.each(&block)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/report/junit_test_results.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+
+module Gitlab
+  module QA
+    module Report
+      class JUnitTestResults
+        include Enumerable
+
+        def initialize(file)
+          @testcases = Nokogiri::XML(File.read(file)).xpath('//testcase').map { |test| TestResult.from_junit(test) }
+        end
+
+        def each(&block)
+          @testcases.each(&block)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/report/results_in_issues.rb

@@ -46,7 +46,18 @@ module Gitlab
 
           Dir.glob(files).each do |file|
             puts "Reporting tests in #{file}"
-            Nokogiri::XML(File.open(file)).xpath('//testcase').each do |test|
+            extension = File.extname(file)
+
+            case extension
+            when '.json'
+              test_results = Report::JsonTestResults.new(file)
+            when '.xml'
+              test_results = Report::JUnitTestResults.new(file)
+            else
+              raise "Unknown extension #{extension}"
+            end
+
+            test_results.each do |test|
               report_test(test)
             end
           end
@@ -99,9 +110,9 @@ module Gitlab
         end
 
         def report_test(test)
-          return if test.search('skipped').any?
+          return if test.skipped
 
-          puts "Reporting test: #{test['file']} | #{test['name']}"
+          puts "Reporting test: #{test.file} | #{test.name}"
 
           issue = find_issue(test)
           if issue
@@ -124,29 +135,37 @@ module Gitlab
             Gitlab.create_issue(
               project,
               title_from_test(test),
-              { description: "### Full description\n\n#{search_safe(test['name'])}\n\n### File path\n\n#{test['file']}", labels: 'status::automated' }
+              { description: "### Full description\n\n#{search_safe(test.name)}\n\n### File path\n\n#{test.file}", labels: 'status::automated' }
             )
           end
         end
 
+        # rubocop:disable Metrics/AbcSize
         def find_issue(test)
           handle_gitlab_client_exceptions do
+            return Gitlab.issue(project, id_from_testcase_url(test.testcase)) if test.testcase
+
             issues = Gitlab.issues(project, { search: search_term(test) })
               .auto_paginate
               .select { |issue| issue.state == 'opened' && issue.title.strip == title_from_test(test) }
 
-            warn(%(Too many issues found with the file path "#{test['file']}" and name "#{test['name']}")) if issues.many?
+            warn(%(Too many issues found with the file path "#{test.file}" and name "#{test.name}")) if issues.many?
 
             issues.first
           end
         end
+        # rubocop:enable Metrics/AbcSize
+
+        def id_from_testcase_url(url)
+          url.split('/').last.to_i
+        end
 
         def search_term(test)
-          %("#{test['file']}" "#{search_safe(test['name'])}")
+          %("#{test.file}" "#{search_safe(test.name)}")
         end
 
         def title_from_test(test)
-          title = "#{partial_file_path(test['file'])} | #{search_safe(test['name'])}".strip
+          title = "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
 
           return title unless title.length > MAX_TITLE_LENGTH
 
@@ -162,7 +181,7 @@ module Gitlab
         end
 
         def note_status(issue, test)
-          return if failures(test).empty?
+          return if test.failures.empty?
 
           note = note_content(test)
 
@@ -176,7 +195,7 @@ module Gitlab
         end
 
         def note_content(test)
-          errors = failures(test).each_with_object([]) do |failure, text|
+          errors = test.failures.each_with_object([]) do |failure, text|
             text << <<~TEXT
               Error:
               ```
@@ -185,7 +204,7 @@ module Gitlab
 
               Stacktrace:
               ```
-              #{failure.content}
+              #{failure['stacktrace']}
               ```
             TEXT
           end.join("\n\n")
@@ -231,7 +250,7 @@ module Gitlab
         def update_labels(issue, test)
           labels = issue.labels
           labels.delete_if { |label| label.start_with?("#{pipeline}::") }
-          labels << (failures(test).empty? ? "#{pipeline}::passed" : "#{pipeline}::failed")
+          labels << (test.failures.empty? ? "#{pipeline}::passed" : "#{pipeline}::failed")
           labels << "Enterprise Edition" if ee_test?(test)
           quarantine_job? ? labels << "quarantine" : labels.delete("quarantine")
 
@@ -242,11 +261,7 @@ module Gitlab
         # rubocop:enable Metrics/AbcSize
 
         def ee_test?(test)
-          test['file'] =~ %r{features/ee/(api|browser_ui)}
-        end
-
-        def failures(test)
-          test.search('failure')
+          test.file =~ %r{features/ee/(api|browser_ui)}
         end
 
         def pipeline

data/lib/gitlab/qa/report/test_result.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Report
+      class TestResult
+        attr_accessor :name, :file, :skipped, :failures, :testcase
+
+        def self.from_json(test)
+          new.tap do |test_result|
+            test_result.name = test['full_description']
+            test_result.file = test['file_path']
+            test_result.skipped = test['status'] == 'pending'
+            test_result.failures = failures_from_json_exceptions(test)
+            test_result.testcase = test['testcase']
+          end
+        end
+
+        def self.from_junit(test)
+          new.tap do |test_result|
+            test_result.name = test['name']
+            test_result.file = test['file']
+            test_result.skipped = test.search('skipped').any?
+            test_result.failures = failures_from_junit_exceptions(test)
+          end
+        end
+
+        def self.failures_from_json_exceptions(test)
+          return [] unless test.key?('exceptions')
+
+          test['exceptions'].map do |exception|
+            spec_file_first_index = exception['backtrace'].rindex do |line|
+              line.include?(File.basename(test['file_path']))
+            end
+
+            {
+              'message' => "#{exception['class']}: #{exception['message']}",
+              'stacktrace' => exception['backtrace'].slice(0..spec_file_first_index).join("\n")
+            }
+          end
+        end
+        private_class_method :failures_from_json_exceptions
+
+        def self.failures_from_junit_exceptions(test)
+          failures = test.search('failure')
+          return [] if failures.empty?
+
+          failures.map do |exception|
+            trace = exception.content.split("\n").map(&:strip)
+            spec_file_first_index = trace.rindex do |line|
+              line.include?(File.basename(test['file']))
+            end
+
+            {
+              'message' => "#{exception['type']}: #{exception['message']}",
+              'stacktrace' => trace.slice(0..spec_file_first_index).join("\n")
+            }
+          end
+        end
+        private_class_method :failures_from_junit_exceptions
+      end
+    end
+  end
+end

data/lib/gitlab/qa/runtime/env.rb

@@ -80,7 +80,8 @@ module Gitlab
           'GEO_MAX_DB_REPLICATION_TIME' => :geo_max_db_replication_time,
           'JIRA_HOSTNAME' => :jira_hostname,
           'JIRA_ADMIN_USERNAME' => :jira_admin_username,
-          'JIRA_ADMIN_PASSWORD' => :jira_admin_password
+          'JIRA_ADMIN_PASSWORD' => :jira_admin_password,
+          'CACHE_NAMESPACE_NAME' => :cache_namespace_name
         }.freeze
 
         ENV_VARIABLES.each_value do |accessor|

data/lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb

@@ -7,13 +7,13 @@ module Gitlab
             attr_reader :gitlab_name, :spec_suite
 
             def initialize
-              @gitlab_name = 'gitlab-gitaly-ha'
+              @gitlab_name = 'gitlab-gitaly-cluster'
               @primary_node_name = 'gitaly1'
               @secondary_node_name = 'gitaly2'
               @tertiary_node_name = 'gitaly3'
               @praefect_node_name = 'praefect'
               @database = 'postgres'
-              @spec_suite = 'Test::Integration::GitalyHA'
+              @spec_suite = 'Test::Instance::All'
               @network = 'test'
             end
 
@@ -26,7 +26,7 @@ module Gitlab
               sql_node = Component::PostgreSQL.new.tap do |sql|
                 sql.name = @database
                 sql.network = @network
-                sql.instance_no_teardown do
+                sql.instance(skip_teardown: true) do
                   sql.run_psql '-d template1 -c "CREATE DATABASE praefect_production OWNER postgres"'
                 end
               end
@@ -39,7 +39,7 @@ module Gitlab
 
                 praefect.omnibus_config = praefect_omnibus_configuration
 
-                praefect.instance_no_teardown
+                praefect.instance(skip_teardown: true)
               end
 
               Component::Gitlab.perform do |gitlab|
@@ -49,7 +49,10 @@ module Gitlab
 
                 gitlab.omnibus_config = gitlab_omnibus_configuration
                 gitlab.instance do
-                  puts "Running Gitaly HA specs!"
+                  puts "Running Gitaly Cluster specs!"
+
+                  rspec_args << "--" unless rspec_args.include?('--')
+                  rspec_args << %w[--tag gitaly_cluster]
 
                   Component::Specs.perform do |specs|
                     specs.suite = spec_suite
@@ -190,7 +193,7 @@ module Gitlab
                 gitaly.network = @network
                 gitaly.skip_availability_check = true
                 gitaly.omnibus_config = gitaly_omnibus_configuration
-                gitaly.instance_no_teardown
+                gitaly.instance(skip_teardown: true)
               end
             end
           end

data/lib/gitlab/qa/scenario/test/integration/praefect.rb

@@ -65,6 +65,7 @@ module Gitlab
             def omnibus_config_with_praefect
               <<~OMNIBUS
                 gitaly['enable'] = true;
+                gitaly['listen_addr'] = '0.0.0.0:8075';
                 gitaly['auth_token'] = 'secret-token';
                 gitaly['storage'] = [
                   {
@@ -82,7 +83,7 @@ module Gitlab
                 praefect['virtual_storages'] = {
                   'default' => {
                     'praefect-gitaly-0' => {
-                      'address' => 'unix:/var/opt/gitlab/gitaly/gitaly.socket',
+                      'address' => 'tcp://localhost:8075',
                       'token'   => 'secret-token',
                       'primary' => true
                     }

data/lib/gitlab/qa/version.rb

@@ -1,5 +1,5 @@
 module Gitlab
   module QA
-    VERSION = '5.17.0'.freeze
+    VERSION = '6.1.2'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 5.17.0
+  version: 6.1.2
 platform: ruby
 authors:
 - Grzegorz Bizon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-07-28 00:00:00.000000000 Z
+date: 2020-09-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -257,9 +257,12 @@ files:
 - lib/gitlab/qa/docker/shellout.rb
 - lib/gitlab/qa/docker/volumes.rb
 - lib/gitlab/qa/release.rb
+- lib/gitlab/qa/report/json_test_results.rb
+- lib/gitlab/qa/report/junit_test_results.rb
 - lib/gitlab/qa/report/prepare_stage_reports.rb
 - lib/gitlab/qa/report/results_in_issues.rb
 - lib/gitlab/qa/report/summary_table.rb
+- lib/gitlab/qa/report/test_result.rb
 - lib/gitlab/qa/report/update_screenshot_path.rb
 - lib/gitlab/qa/reporter.rb
 - lib/gitlab/qa/runner.rb