gitlab_quality-test_tooling 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c393338cd20306bc419cf1bbb0c4dd3e09f51312f0c6bbc2a58e60b062a8170d
4
- data.tar.gz: 3e43e01fc39e8c1544f9fdf552f061b98c7056710377ffeaa5a229f2ca08d587
3
+ metadata.gz: ed3ddf73e69dbf985ff11b7192c25055281b4559b53f3961ff7c4de033872e70
4
+ data.tar.gz: 30de36dd668dd20ab52a19c12cbf3b68f87041cd12bf495692686f2e813a3332
5
5
  SHA512:
6
- metadata.gz: b78a87a21945eaacdf4e7ff609b248cb427c7e4c3ccf19c8f4784aa19f0219856144ff2c35079f9556907308f11a72fdf10c60ab3e1f79799f6eede53556bf2b
7
- data.tar.gz: 7362e44b09a26d0b4893a05cafc819f9aaff457a7f9dc0c39f645851b2b3362872e2acf14ed3b0e1e35a00f17e02753ff2b42badf4de27672433d6288f39828e
6
+ metadata.gz: 1dcb1a3f77b47b5ecafd5a425b2de398cedc6caf3b7fa4e4f0a351d467f723329e7382a592936064ff4969b25976e9b3b88553610d33b88db2e22a42c0e2917d
7
+ data.tar.gz: 0bb812067345d488cfea73ad4d25e396aeea79eaa2a0790de5607e20e3ce08a1e2a96990f8ee1523d52d1c95112e7b99755dfa8857731337a05466d90bbb0925
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- gitlab_quality-test_tooling (0.7.0)
4
+ gitlab_quality-test_tooling (0.8.0)
5
5
  activesupport (~> 6.1)
6
6
  gitlab (~> 4.19)
7
7
  http (~> 5.0)
@@ -18,6 +18,7 @@ module GitlabQuality
18
18
  include Concerns::FindSetDri
19
19
 
20
20
  DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
21
+ SYSTEMIC_EXCEPTIONS_THRESHOLD = 10
21
22
  SPAM_THRESHOLD_FOR_FAILURE_ISSUES = 3
22
23
  FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m
23
24
  ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)/m
@@ -25,7 +26,10 @@ module GitlabQuality
25
26
  FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
26
27
  REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
27
28
  NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2]).freeze
28
- IGNORE_EXCEPTIONS = ['Net::ReadTimeout', '403 Forbidden - Your account has been blocked'].freeze
29
+ IGNORE_EXCEPTIONS = [
30
+ 'Net::ReadTimeout',
31
+ '403 Forbidden - Your account has been blocked'
32
+ ].freeze
29
33
  SCREENSHOT_IGNORED_ERRORS = ['500 Internal Server Error', 'fabricate_via_api!', 'Error Code 500'].freeze
30
34
 
31
35
  MultipleIssuesFound = Class.new(StandardError)
@@ -47,16 +51,27 @@ module GitlabQuality
47
51
  puts "Reporting test failures in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
48
52
 
49
53
  TestResults::Builder.new(files).test_results_per_file do |test_results|
50
- puts "=> Reporting tests in #{test_results.path}"
54
+ puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
55
+
56
+ systemic_exceptions = systemic_exceptions_for_test_results(test_results)
51
57
 
52
58
  test_results.each do |test|
53
- relate_failure_to_issue(test) if should_report?(test)
59
+ relate_failure_to_issue(test) if should_report?(test, systemic_exceptions)
54
60
  end
55
61
 
56
62
  test_results.write
57
63
  end
58
64
  end
59
65
 
66
+ def systemic_exceptions_for_test_results(test_results)
67
+ test_results
68
+ .flat_map { |test| test.report['exceptions']&.map { |exception| exception['message'] } }
69
+ .compact
70
+ .tally
71
+ .select { |_e, count| count >= SYSTEMIC_EXCEPTIONS_THRESHOLD }
72
+ .keys
73
+ end
74
+
60
75
  def relate_failure_to_issue(test)
61
76
  puts " => Relating issues for test '#{test.name}'..."
62
77
 
@@ -377,14 +392,17 @@ module GitlabQuality
377
392
  #
378
393
  # @return [TrueClass|FalseClass] false if the test was skipped or failed because of a transient error that can be ignored.
379
394
  # Otherwise returns true.
380
- def should_report?(test)
395
+ def should_report?(test, systemic_exceptions)
381
396
  return false if test.failures.empty?
382
397
 
398
+ puts " => Systemic exceptions detected: #{systemic_exceptions}" if systemic_exceptions.any?
399
+ exceptions_to_ignore = IGNORE_EXCEPTIONS + systemic_exceptions
400
+
383
401
  if test.report.key?('exceptions')
384
- reason = ignore_failure_reason(test.report['exceptions'])
402
+ reason = ignore_failure_reason(test.report['exceptions'], exceptions_to_ignore)
385
403
 
386
404
  if reason
387
- puts "Failure reporting skipped because #{reason}"
405
+ puts " => Failure reporting skipped because #{reason}"
388
406
 
389
407
  return false
390
408
  end
@@ -397,9 +415,9 @@ module GitlabQuality
397
415
  #
398
416
  # @param [Array<Hash>] exceptions the exceptions associated with the failure.
399
417
  # @return [String] the reason to ignore the exceptions, or `nil` if any exceptions should not be ignored.
400
- def ignore_failure_reason(exceptions)
418
+ def ignore_failure_reason(exceptions, ignored_exceptions)
401
419
  exception_messages = exceptions
402
- .filter_map { |exception| exception['message'] if IGNORE_EXCEPTIONS.any? { |e| exception['message'].include?(e) } }
420
+ .filter_map { |exception| exception['message'] if ignored_exceptions.any? { |e| exception['message'].include?(e) } }
403
421
  .compact
404
422
  return if exception_messages.empty? || exception_messages.size < exceptions.size
405
423
 
@@ -2,6 +2,6 @@
2
2
 
3
3
  module GitlabQuality
4
4
  module TestTooling
5
- VERSION = "0.7.0"
5
+ VERSION = "0.8.0"
6
6
  end
7
7
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: gitlab_quality-test_tooling
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.0
4
+ version: 0.8.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - GitLab Quality