gitlab-qa 5.17.1 → 6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d14680b847d514a92a3db02e62d17566d165c3aa2fa7772cbf4cdb5eb3746d6f
-  data.tar.gz: f8669ae124311588e59bd52f743b5aafb41a2f7ca40073f12eae28f860e7f54a
+  metadata.gz: 87249baf75070cf8c5362c34b9995f6163370e9b6e353a393104beefe62d4bd8
+  data.tar.gz: 6e00eee6993c9f5c6aeb4cb187c708941bfd27cb9b59340165b23b02c30f583a
 SHA512:
-  metadata.gz: be1aa1c90f7bbfe3e9fab71640f14517d212d5dbfcd5d412e2576b1c6553ad445a2530088f9999fee9fa02cf3bc813f3ee23ae1f2474b2d48a4e2979710163c6
-  data.tar.gz: 138a8b071f801af2bf9db1bed412976ba29be624352bef902f97d5a041ecb6cb45e1e1e32373af6c17e47847900aef791fe3e7219a53736781035fa143425449
+  metadata.gz: 6d621d51ca18fa402e397b13c86dda6d52039a11ca259c6a46f3e9caffeab0faa68b6b9ff58ec4f727b2dae82c386442655122571936a47c366647b3ad895a50
+  data.tar.gz: 73bbcc44b560368e64d8bd335b3fac6d52128e8c05f6ddf582ea6139032f570d80563b6fbc9c96f41598fee2c2c345ec9155f5990f3ea4df1bde2167bede9a27

data/.gitlab-ci.yml

@@ -94,7 +94,7 @@ release:
     - exe/gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} ${RELEASE:=$DEFAULT_RELEASE} -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS || test_run_exit_code=$?
     - exe/gitlab-qa-report --update-screenshot-path "gitlab-qa-run-*/**/rspec-*.xml"
     - export GITLAB_QA_ACCESS_TOKEN="$GITLAB_QA_PRODUCTION_ACCESS_TOKEN"
-    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then $(exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT") || true; fi
+    - if [ "$TOP_UPSTREAM_SOURCE_REF" == "master" ]; then exe/gitlab-qa-report --report-in-issues "gitlab-qa-run-*/**/rspec-*.xml" --project "$QA_TESTCASES_REPORTING_PROJECT" || true; fi
     - exit $test_run_exit_code
 
 .ce-qa:

data/docs/what_tests_can_be_run.md

@@ -251,8 +251,7 @@ of a Gitaly Cluster.
 
 To run tests against the GitLab container, a GitLab QA (`gitlab/gitlab-qa`)
 container is spun up and tests are run from it by running the
-`Test::Integration::GitalyHA` scenario, which includes tests written
-specifically to test Gitaly Cluster features.
+`Test::Instance::All` scenario with the `:gitaly_cluster` tag.
 
 Example:
 

data/lib/gitlab/qa.rb

@@ -97,9 +97,12 @@ module Gitlab
     end
 
     module Report
+      autoload :JsonTestResults, 'gitlab/qa/report/json_test_results'
+      autoload :JUnitTestResults, 'gitlab/qa/report/junit_test_results'
       autoload :PrepareStageReports, 'gitlab/qa/report/prepare_stage_reports'
       autoload :ResultsInIssues, 'gitlab/qa/report/results_in_issues'
       autoload :SummaryTable, 'gitlab/qa/report/summary_table'
+      autoload :TestResult, 'gitlab/qa/report/test_result'
       autoload :UpdateScreenshotPath, 'gitlab/qa/report/update_screenshot_path'
     end
 

data/lib/gitlab/qa/component/base.rb

@@ -41,26 +41,16 @@ module Gitlab
           raise NotImplementedError, "#{self.class.name} must specify a docker image tag as DOCKER_IMAGE_TAG"
         end
 
-        def instance
+        def instance(skip_teardown: false)
           instance_no_teardown do
             yield self if block_given?
           end
         ensure
-          teardown
+          teardown unless skip_teardown
         end
 
         alias_method :launch_and_teardown_instance, :instance
 
-        def instance_no_teardown
-          prepare
-          start
-          reconfigure
-          wait_until_ready
-          process_exec_commands
-
-          yield self if block_given?
-        end
-
         def prepare
           prepare_docker_image
           prepare_network
@@ -147,6 +137,16 @@ module Gitlab
           raise 'Invalid instance name!' unless name
         end
 
+        def instance_no_teardown
+          prepare
+          start
+          reconfigure
+          wait_until_ready
+          process_exec_commands
+
+          yield self if block_given?
+        end
+
         def teardown?
           !Runtime::Scenario.attributes.include?(:teardown) || Runtime::Scenario.teardown
         end

data/lib/gitlab/qa/report/json_test_results.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Gitlab
+  module QA
+    module Report
+      class JsonTestResults
+        include Enumerable
+
+        def initialize(file)
+          @testcases = JSON.parse(File.read(file))['examples'].map { |test| TestResult.from_json(test) }
+        end
+
+        def each(&block)
+          @testcases.each(&block)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/report/junit_test_results.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+
+module Gitlab
+  module QA
+    module Report
+      class JUnitTestResults
+        include Enumerable
+
+        def initialize(file)
+          @testcases = Nokogiri::XML(File.read(file)).xpath('//testcase').map { |test| TestResult.from_junit(test) }
+        end
+
+        def each(&block)
+          @testcases.each(&block)
+        end
+      end
+    end
+  end
+end

data/lib/gitlab/qa/report/results_in_issues.rb

@@ -30,11 +30,12 @@ module Gitlab
         RETRY_BACK_OFF_DELAY = 60
         MAX_RETRY_ATTEMPTS = 3
 
-        def initialize(token:, input_files:, project: nil)
+        def initialize(token:, input_files:, project: nil, input_format: :junit)
           @token = token
           @files = Array(input_files)
           @project = project
           @retry_backoff = 0
+          @input_format = input_format.to_sym
         end
 
         def invoke!
@@ -46,7 +47,14 @@ module Gitlab
 
           Dir.glob(files).each do |file|
             puts "Reporting tests in #{file}"
-            Nokogiri::XML(File.open(file)).xpath('//testcase').each do |test|
+            case input_format
+            when :json
+              test_results = Report::JsonTestResults.new(file)
+            when :junit
+              test_results = Report::JUnitTestResults.new(file)
+            end
+
+            test_results.each do |test|
               report_test(test)
             end
           end
@@ -54,7 +62,7 @@ module Gitlab
 
         private
 
-        attr_reader :files, :token, :project
+        attr_reader :files, :token, :project, :input_format
 
         def validate_input!
           assert_project!
@@ -99,9 +107,9 @@ module Gitlab
         end
 
         def report_test(test)
-          return if test.search('skipped').any?
+          return if test.skipped
 
-          puts "Reporting test: #{test['file']} | #{test['name']}"
+          puts "Reporting test: #{test.file} | #{test.name}"
 
           issue = find_issue(test)
           if issue
@@ -124,7 +132,7 @@ module Gitlab
             Gitlab.create_issue(
               project,
               title_from_test(test),
-              { description: "### Full description\n\n#{search_safe(test['name'])}\n\n### File path\n\n#{test['file']}", labels: 'status::automated' }
+              { description: "### Full description\n\n#{search_safe(test.name)}\n\n### File path\n\n#{test.file}", labels: 'status::automated' }
             )
           end
         end
@@ -135,18 +143,18 @@ module Gitlab
               .auto_paginate
               .select { |issue| issue.state == 'opened' && issue.title.strip == title_from_test(test) }
 
-            warn(%(Too many issues found with the file path "#{test['file']}" and name "#{test['name']}")) if issues.many?
+            warn(%(Too many issues found with the file path "#{test.file}" and name "#{test.name}")) if issues.many?
 
             issues.first
           end
         end
 
         def search_term(test)
-          %("#{test['file']}" "#{search_safe(test['name'])}")
+          %("#{test.file}" "#{search_safe(test.name)}")
         end
 
         def title_from_test(test)
-          title = "#{partial_file_path(test['file'])} | #{search_safe(test['name'])}".strip
+          title = "#{partial_file_path(test.file)} | #{search_safe(test.name)}".strip
 
           return title unless title.length > MAX_TITLE_LENGTH
 
@@ -162,7 +170,7 @@ module Gitlab
         end
 
         def note_status(issue, test)
-          return if failures(test).empty?
+          return if test.failures.empty?
 
           note = note_content(test)
 
@@ -176,7 +184,7 @@ module Gitlab
         end
 
         def note_content(test)
-          errors = failures(test).each_with_object([]) do |failure, text|
+          errors = test.failures.each_with_object([]) do |failure, text|
             text << <<~TEXT
               Error:
               ```
@@ -185,7 +193,7 @@ module Gitlab
 
               Stacktrace:
               ```
-              #{failure.content}
+              #{failure['stacktrace']}
               ```
             TEXT
           end.join("\n\n")
@@ -231,7 +239,7 @@ module Gitlab
         def update_labels(issue, test)
           labels = issue.labels
           labels.delete_if { |label| label.start_with?("#{pipeline}::") }
-          labels << (failures(test).empty? ? "#{pipeline}::passed" : "#{pipeline}::failed")
+          labels << (test.failures.empty? ? "#{pipeline}::passed" : "#{pipeline}::failed")
           labels << "Enterprise Edition" if ee_test?(test)
           quarantine_job? ? labels << "quarantine" : labels.delete("quarantine")
 
@@ -242,11 +250,7 @@ module Gitlab
         # rubocop:enable Metrics/AbcSize
 
         def ee_test?(test)
-          test['file'] =~ %r{features/ee/(api|browser_ui)}
-        end
-
-        def failures(test)
-          test.search('failure')
+          test.file =~ %r{features/ee/(api|browser_ui)}
         end
 
         def pipeline

data/lib/gitlab/qa/report/test_result.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module QA
+    module Report
+      class TestResult
+        attr_accessor :name, :file, :skipped, :failures
+
+        def self.from_json(test)
+          new.tap do |test_result|
+            test_result.name = test['full_description']
+            test_result.file = test['file_path']
+            test_result.skipped = test['status'] == 'pending'
+            test_result.failures = failures_from_json_exceptions(test)
+          end
+        end
+
+        def self.from_junit(test)
+          new.tap do |test_result|
+            test_result.name = test['name']
+            test_result.file = test['file']
+            test_result.skipped = test.search('skipped').any?
+            test_result.failures = failures_from_junit_exceptions(test)
+          end
+        end
+
+        def self.failures_from_json_exceptions(test)
+          return [] unless test.key?('exceptions')
+
+          test['exceptions'].map do |exception|
+            {
+              'message' => "#{exception['class']}: #{exception['message']}",
+              'stacktrace' => exception['backtrace'].join('\n')
+            }
+          end
+        end
+        private_class_method :failures_from_json_exceptions
+
+        def self.failures_from_junit_exceptions(test)
+          failures = test.search('failure')
+          return [] if failures.empty?
+
+          failures.map do |exception|
+            {
+              'message' => "#{exception['type']}: #{exception['message']}",
+              'stacktrace' => exception.content
+            }
+          end
+        end
+        private_class_method :failures_from_junit_exceptions
+      end
+    end
+  end
+end

data/lib/gitlab/qa/scenario/test/integration/gitaly_cluster.rb

@@ -7,13 +7,13 @@ module Gitlab
             attr_reader :gitlab_name, :spec_suite
 
             def initialize
-              @gitlab_name = 'gitlab-gitaly-ha'
+              @gitlab_name = 'gitlab-gitaly-cluster'
               @primary_node_name = 'gitaly1'
               @secondary_node_name = 'gitaly2'
               @tertiary_node_name = 'gitaly3'
               @praefect_node_name = 'praefect'
               @database = 'postgres'
-              @spec_suite = 'Test::Integration::GitalyHA'
+              @spec_suite = 'Test::Instance::All'
               @network = 'test'
             end
 
@@ -26,7 +26,7 @@ module Gitlab
               sql_node = Component::PostgreSQL.new.tap do |sql|
                 sql.name = @database
                 sql.network = @network
-                sql.instance_no_teardown do
+                sql.instance(skip_teardown: true) do
                   sql.run_psql '-d template1 -c "CREATE DATABASE praefect_production OWNER postgres"'
                 end
               end
@@ -39,7 +39,7 @@ module Gitlab
 
                 praefect.omnibus_config = praefect_omnibus_configuration
 
-                praefect.instance_no_teardown
+                praefect.instance(skip_teardown: true)
               end
 
               Component::Gitlab.perform do |gitlab|
@@ -49,7 +49,10 @@ module Gitlab
 
                 gitlab.omnibus_config = gitlab_omnibus_configuration
                 gitlab.instance do
-                  puts "Running Gitaly HA specs!"
+                  puts "Running Gitaly Cluster specs!"
+
+                  rspec_args << "--" unless rspec_args.include?('--')
+                  rspec_args << %w[--tag gitaly_cluster]
 
                   Component::Specs.perform do |specs|
                     specs.suite = spec_suite
@@ -190,7 +193,7 @@ module Gitlab
                 gitaly.network = @network
                 gitaly.skip_availability_check = true
                 gitaly.omnibus_config = gitaly_omnibus_configuration
-                gitaly.instance_no_teardown
+                gitaly.instance(skip_teardown: true)
               end
             end
           end

data/lib/gitlab/qa/version.rb

@@ -1,5 +1,5 @@
 module Gitlab
   module QA
-    VERSION = '5.17.1'.freeze
+    VERSION = '6.0.0'.freeze
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab-qa
 version: !ruby/object:Gem::Version
-  version: 5.17.1
+  version: 6.0.0
 platform: ruby
 authors:
 - Grzegorz Bizon
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-08-10 00:00:00.000000000 Z
+date: 2020-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -257,9 +257,12 @@ files:
 - lib/gitlab/qa/docker/shellout.rb
 - lib/gitlab/qa/docker/volumes.rb
 - lib/gitlab/qa/release.rb
+- lib/gitlab/qa/report/json_test_results.rb
+- lib/gitlab/qa/report/junit_test_results.rb
 - lib/gitlab/qa/report/prepare_stage_reports.rb
 - lib/gitlab/qa/report/results_in_issues.rb
 - lib/gitlab/qa/report/summary_table.rb
+- lib/gitlab/qa/report/test_result.rb
 - lib/gitlab/qa/report/update_screenshot_path.rb
 - lib/gitlab/qa/reporter.rb
 - lib/gitlab/qa/runner.rb