gitlab_quality-test_tooling 0.7.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c393338cd20306bc419cf1bbb0c4dd3e09f51312f0c6bbc2a58e60b062a8170d
-  data.tar.gz: 3e43e01fc39e8c1544f9fdf552f061b98c7056710377ffeaa5a229f2ca08d587
+  metadata.gz: bc873ad79077521f3fc428ee0c71eceb10b6f03b14b002f4cb2519e974492902
+  data.tar.gz: 4ff2af5013269670822407cc3300bca290920a38902d70410883dcca25d2d462
 SHA512:
-  metadata.gz: b78a87a21945eaacdf4e7ff609b248cb427c7e4c3ccf19c8f4784aa19f0219856144ff2c35079f9556907308f11a72fdf10c60ab3e1f79799f6eede53556bf2b
-  data.tar.gz: 7362e44b09a26d0b4893a05cafc819f9aaff457a7f9dc0c39f645851b2b3362872e2acf14ed3b0e1e35a00f17e02753ff2b42badf4de27672433d6288f39828e
+  metadata.gz: 19c0969b35e7c9d429d7a2bb0a46a514868e4947a29ce05c72492184f97e844afe54a7bcb2a4461b349993c593fffc21c9e6da45877c13498e1b71f3fd47d941
+  data.tar.gz: e653c176e564bc51853f3e58cbe87ef599d14bb3a58bece011e0099201e212e3a55a9313cbb4f682748a6ea84faf4663fa7dee0c20efd510ad3137b8070f129a

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    gitlab_quality-test_tooling (0.7.0)
-      activesupport (~> 6.1)
+    gitlab_quality-test_tooling (0.8.1)
+      activesupport (>= 6.1, < 7.1)
       gitlab (~> 4.19)
       http (~> 5.0)
       nokogiri (~> 1.10)

data/lib/gitlab_quality/test_tooling/report/relate_failure_issue.rb

@@ -18,6 +18,7 @@ module GitlabQuality
         include Concerns::FindSetDri
 
         DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
+        SYSTEMIC_EXCEPTIONS_THRESHOLD = 10
         SPAM_THRESHOLD_FOR_FAILURE_ISSUES = 3
         FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m
         ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)/m
@@ -25,7 +26,10 @@ module GitlabQuality
         FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
         REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
         NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2]).freeze
-        IGNORE_EXCEPTIONS = ['Net::ReadTimeout', '403 Forbidden - Your account has been blocked'].freeze
+        IGNORE_EXCEPTIONS = [
+          'Net::ReadTimeout',
+          '403 Forbidden - Your account has been blocked'
+        ].freeze
         SCREENSHOT_IGNORED_ERRORS = ['500 Internal Server Error', 'fabricate_via_api!', 'Error Code 500'].freeze
 
         MultipleIssuesFound = Class.new(StandardError)
@@ -47,16 +51,27 @@ module GitlabQuality
           puts "Reporting test failures in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
 
           TestResults::Builder.new(files).test_results_per_file do |test_results|
-            puts "=> Reporting tests in #{test_results.path}"
+            puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
+
+            systemic_exceptions = systemic_exceptions_for_test_results(test_results)
 
             test_results.each do |test|
-              relate_failure_to_issue(test) if should_report?(test)
+              relate_failure_to_issue(test) if should_report?(test, systemic_exceptions)
             end
 
             test_results.write
           end
         end
 
+        def systemic_exceptions_for_test_results(test_results)
+          test_results
+            .flat_map { |test| test.report['exceptions']&.map { |exception| exception['message'] } }
+            .compact
+            .tally
+            .select { |_e, count| count >= SYSTEMIC_EXCEPTIONS_THRESHOLD }
+            .keys
+        end
+
         def relate_failure_to_issue(test)
           puts " => Relating issues for test '#{test.name}'..."
 
@@ -377,14 +392,17 @@ module GitlabQuality
         #
         # @return [TrueClass|FalseClass] false if the test was skipped or failed because of a transient error that can be ignored.
         # Otherwise returns true.
-        def should_report?(test)
+        def should_report?(test, systemic_exceptions)
           return false if test.failures.empty?
 
+          puts "  => Systemic exceptions detected: #{systemic_exceptions}" if systemic_exceptions.any?
+          exceptions_to_ignore = IGNORE_EXCEPTIONS + systemic_exceptions
+
           if test.report.key?('exceptions')
-            reason = ignore_failure_reason(test.report['exceptions'])
+            reason = ignore_failure_reason(test.report['exceptions'], exceptions_to_ignore)
 
             if reason
-              puts "Failure reporting skipped because #{reason}"
+              puts "  => Failure reporting skipped because #{reason}"
 
               return false
             end
@@ -397,9 +415,9 @@ module GitlabQuality
         #
         # @param [Array<Hash>] exceptions the exceptions associated with the failure.
         # @return [String] the reason to ignore the exceptions, or `nil` if any exceptions should not be ignored.
-        def ignore_failure_reason(exceptions)
+        def ignore_failure_reason(exceptions, ignored_exceptions)
           exception_messages = exceptions
-            .filter_map { |exception| exception['message'] if IGNORE_EXCEPTIONS.any? { |e| exception['message'].include?(e) } }
+            .filter_map { |exception| exception['message'] if ignored_exceptions.any? { |e| exception['message'].include?(e) } }
             .compact
           return if exception_messages.empty? || exception_messages.size < exceptions.size
 

data/lib/gitlab_quality/test_tooling/version.rb

@@ -2,6 +2,6 @@
 
 module GitlabQuality
   module TestTooling
-    VERSION = "0.7.0"
+    VERSION = "0.8.1"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitlab_quality-test_tooling
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.1
 platform: ruby
 authors:
 - GitLab Quality
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-06-09 00:00:00.000000000 Z
+date: 2023-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: climate_control
@@ -182,16 +182,22 @@ dependencies:
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '6.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '6.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7.1'
 - !ruby/object:Gem::Dependency
   name: gitlab
   requirement: !ruby/object:Gem::Requirement