gitlab_quality-test_tooling 0.7.0 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ed3ddf73e69dbf985ff11b7192c25055281b4559b53f3961ff7c4de033872e70
|
4
|
+
data.tar.gz: 30de36dd668dd20ab52a19c12cbf3b68f87041cd12bf495692686f2e813a3332
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1dcb1a3f77b47b5ecafd5a425b2de398cedc6caf3b7fa4e4f0a351d467f723329e7382a592936064ff4969b25976e9b3b88553610d33b88db2e22a42c0e2917d
|
7
|
+
data.tar.gz: 0bb812067345d488cfea73ad4d25e396aeea79eaa2a0790de5607e20e3ce08a1e2a96990f8ee1523d52d1c95112e7b99755dfa8857731337a05466d90bbb0925
|
data/Gemfile.lock
CHANGED
@@ -18,6 +18,7 @@ module GitlabQuality
|
|
18
18
|
include Concerns::FindSetDri
|
19
19
|
|
20
20
|
DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
|
21
|
+
SYSTEMIC_EXCEPTIONS_THRESHOLD = 10
|
21
22
|
SPAM_THRESHOLD_FOR_FAILURE_ISSUES = 3
|
22
23
|
FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m
|
23
24
|
ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)/m
|
@@ -25,7 +26,10 @@ module GitlabQuality
|
|
25
26
|
FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
|
26
27
|
REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
|
27
28
|
NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2]).freeze
|
28
|
-
IGNORE_EXCEPTIONS = [
|
29
|
+
IGNORE_EXCEPTIONS = [
|
30
|
+
'Net::ReadTimeout',
|
31
|
+
'403 Forbidden - Your account has been blocked'
|
32
|
+
].freeze
|
29
33
|
SCREENSHOT_IGNORED_ERRORS = ['500 Internal Server Error', 'fabricate_via_api!', 'Error Code 500'].freeze
|
30
34
|
|
31
35
|
MultipleIssuesFound = Class.new(StandardError)
|
@@ -47,16 +51,27 @@ module GitlabQuality
|
|
47
51
|
puts "Reporting test failures in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
48
52
|
|
49
53
|
TestResults::Builder.new(files).test_results_per_file do |test_results|
|
50
|
-
puts "=> Reporting tests in #{test_results.path}"
|
54
|
+
puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
|
55
|
+
|
56
|
+
systemic_exceptions = systemic_exceptions_for_test_results(test_results)
|
51
57
|
|
52
58
|
test_results.each do |test|
|
53
|
-
relate_failure_to_issue(test) if should_report?(test)
|
59
|
+
relate_failure_to_issue(test) if should_report?(test, systemic_exceptions)
|
54
60
|
end
|
55
61
|
|
56
62
|
test_results.write
|
57
63
|
end
|
58
64
|
end
|
59
65
|
|
66
|
+
def systemic_exceptions_for_test_results(test_results)
|
67
|
+
test_results
|
68
|
+
.flat_map { |test| test.report['exceptions']&.map { |exception| exception['message'] } }
|
69
|
+
.compact
|
70
|
+
.tally
|
71
|
+
.select { |_e, count| count >= SYSTEMIC_EXCEPTIONS_THRESHOLD }
|
72
|
+
.keys
|
73
|
+
end
|
74
|
+
|
60
75
|
def relate_failure_to_issue(test)
|
61
76
|
puts " => Relating issues for test '#{test.name}'..."
|
62
77
|
|
@@ -377,14 +392,17 @@ module GitlabQuality
|
|
377
392
|
#
|
378
393
|
# @return [TrueClass|FalseClass] false if the test was skipped or failed because of a transient error that can be ignored.
|
379
394
|
# Otherwise returns true.
|
380
|
-
def should_report?(test)
|
395
|
+
def should_report?(test, systemic_exceptions)
|
381
396
|
return false if test.failures.empty?
|
382
397
|
|
398
|
+
puts " => Systemic exceptions detected: #{systemic_exceptions}" if systemic_exceptions.any?
|
399
|
+
exceptions_to_ignore = IGNORE_EXCEPTIONS + systemic_exceptions
|
400
|
+
|
383
401
|
if test.report.key?('exceptions')
|
384
|
-
reason = ignore_failure_reason(test.report['exceptions'])
|
402
|
+
reason = ignore_failure_reason(test.report['exceptions'], exceptions_to_ignore)
|
385
403
|
|
386
404
|
if reason
|
387
|
-
puts "Failure reporting skipped because #{reason}"
|
405
|
+
puts " => Failure reporting skipped because #{reason}"
|
388
406
|
|
389
407
|
return false
|
390
408
|
end
|
@@ -397,9 +415,9 @@ module GitlabQuality
|
|
397
415
|
#
|
398
416
|
# @param [Array<Hash>] exceptions the exceptions associated with the failure.
|
399
417
|
# @return [String] the reason to ignore the exceptions, or `nil` if any exceptions should not be ignored.
|
400
|
-
def ignore_failure_reason(exceptions)
|
418
|
+
def ignore_failure_reason(exceptions, ignored_exceptions)
|
401
419
|
exception_messages = exceptions
|
402
|
-
.filter_map { |exception| exception['message'] if
|
420
|
+
.filter_map { |exception| exception['message'] if ignored_exceptions.any? { |e| exception['message'].include?(e) } }
|
403
421
|
.compact
|
404
422
|
return if exception_messages.empty? || exception_messages.size < exceptions.size
|
405
423
|
|