gitlab_quality-test_tooling 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c393338cd20306bc419cf1bbb0c4dd3e09f51312f0c6bbc2a58e60b062a8170d
-  data.tar.gz: 3e43e01fc39e8c1544f9fdf552f061b98c7056710377ffeaa5a229f2ca08d587
+  metadata.gz: ed3ddf73e69dbf985ff11b7192c25055281b4559b53f3961ff7c4de033872e70
+  data.tar.gz: 30de36dd668dd20ab52a19c12cbf3b68f87041cd12bf495692686f2e813a3332
 SHA512:
-  metadata.gz: b78a87a21945eaacdf4e7ff609b248cb427c7e4c3ccf19c8f4784aa19f0219856144ff2c35079f9556907308f11a72fdf10c60ab3e1f79799f6eede53556bf2b
-  data.tar.gz: 7362e44b09a26d0b4893a05cafc819f9aaff457a7f9dc0c39f645851b2b3362872e2acf14ed3b0e1e35a00f17e02753ff2b42badf4de27672433d6288f39828e
+  metadata.gz: 1dcb1a3f77b47b5ecafd5a425b2de398cedc6caf3b7fa4e4f0a351d467f723329e7382a592936064ff4969b25976e9b3b88553610d33b88db2e22a42c0e2917d
+  data.tar.gz: 0bb812067345d488cfea73ad4d25e396aeea79eaa2a0790de5607e20e3ce08a1e2a96990f8ee1523d52d1c95112e7b99755dfa8857731337a05466d90bbb0925

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (0.7.0)
+    gitlab_quality-test_tooling (0.8.0)
       activesupport (~> 6.1)
       gitlab (~> 4.19)
       http (~> 5.0)

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -18,6 +18,7 @@ module GitlabQuality
         include Concerns::FindSetDri
 
         DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
+        SYSTEMIC_EXCEPTIONS_THRESHOLD = 10
         SPAM_THRESHOLD_FOR_FAILURE_ISSUES = 3
         FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m
         ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)/m
@@ -25,7 +26,10 @@ module GitlabQuality
         FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
         REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
         NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2]).freeze
-        IGNORE_EXCEPTIONS = ['Net::ReadTimeout', '403 Forbidden - Your account has been blocked'].freeze
+        IGNORE_EXCEPTIONS = [
+          'Net::ReadTimeout',
+          '403 Forbidden - Your account has been blocked'
+        ].freeze
         SCREENSHOT_IGNORED_ERRORS = ['500 Internal Server Error', 'fabricate_via_api!', 'Error Code 500'].freeze
 
         MultipleIssuesFound = Class.new(StandardError)
@@ -47,16 +51,27 @@ module GitlabQuality
           puts "Reporting test failures in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
 
           TestResults::Builder.new(files).test_results_per_file do |test_results|
-            puts "=> Reporting tests in #{test_results.path}"
+            puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
+
+            systemic_exceptions = systemic_exceptions_for_test_results(test_results)
 
             test_results.each do |test|
-              relate_failure_to_issue(test) if should_report?(test)
+              relate_failure_to_issue(test) if should_report?(test, systemic_exceptions)
             end
 
             test_results.write
           end
         end
 
+        def systemic_exceptions_for_test_results(test_results)
+          test_results
+            .flat_map { |test| test.report['exceptions']&.map { |exception| exception['message'] } }
+            .compact
+            .tally
+            .select { |_e, count| count >= SYSTEMIC_EXCEPTIONS_THRESHOLD }
+            .keys
+        end
+
         def relate_failure_to_issue(test)
           puts " => Relating issues for test '#{test.name}'..."
 
@@ -377,14 +392,17 @@ module GitlabQuality
         #
         # @return [TrueClass|FalseClass] false if the test was skipped or failed because of a transient error that can be ignored.
         # Otherwise returns true.
-        def should_report?(test)
+        def should_report?(test, systemic_exceptions)
           return false if test.failures.empty?
 
+          puts "  => Systemic exceptions detected: #{systemic_exceptions}" if systemic_exceptions.any?
+          exceptions_to_ignore = IGNORE_EXCEPTIONS + systemic_exceptions
+
           if test.report.key?('exceptions')
-            reason = ignore_failure_reason(test.report['exceptions'])
+            reason = ignore_failure_reason(test.report['exceptions'], exceptions_to_ignore)
 
             if reason
-              puts "Failure reporting skipped because #{reason}"
+              puts "  => Failure reporting skipped because #{reason}"
 
               return false
             end
@@ -397,9 +415,9 @@ module GitlabQuality
         #
         # @param [Array<Hash>] exceptions the exceptions associated with the failure.
         # @return [String] the reason to ignore the exceptions, or `nil` if any exceptions should not be ignored.
-        def ignore_failure_reason(exceptions)
+        def ignore_failure_reason(exceptions, ignored_exceptions)
           exception_messages = exceptions
-            .filter_map { |exception| exception['message'] if IGNORE_EXCEPTIONS.any? { |e| exception['message'].include?(e) } }
+            .filter_map { |exception| exception['message'] if ignored_exceptions.any? { |e| exception['message'].include?(e) } }
             .compact
           return if exception_messages.empty? || exception_messages.size < exceptions.size
 

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "0.7.0"
+    VERSION = "0.8.0"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - GitLab Quality