gitlab_quality-test_tooling 0.7.0 → 0.8.1
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bc873ad79077521f3fc428ee0c71eceb10b6f03b14b002f4cb2519e974492902
|
4
|
+
data.tar.gz: 4ff2af5013269670822407cc3300bca290920a38902d70410883dcca25d2d462
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 19c0969b35e7c9d429d7a2bb0a46a514868e4947a29ce05c72492184f97e844afe54a7bcb2a4461b349993c593fffc21c9e6da45877c13498e1b71f3fd47d941
|
7
|
+
data.tar.gz: e653c176e564bc51853f3e58cbe87ef599d14bb3a58bece011e0099201e212e3a55a9313cbb4f682748a6ea84faf4663fa7dee0c20efd510ad3137b8070f129a
|
data/Gemfile.lock
CHANGED
@@ -18,6 +18,7 @@ module GitlabQuality
|
|
18
18
|
include Concerns::FindSetDri
|
19
19
|
|
20
20
|
DEFAULT_MAX_DIFF_RATIO_FOR_DETECTION = 0.15
|
21
|
+
SYSTEMIC_EXCEPTIONS_THRESHOLD = 10
|
21
22
|
SPAM_THRESHOLD_FOR_FAILURE_ISSUES = 3
|
22
23
|
FAILURE_STACKTRACE_REGEX = %r{(?:(?:.*Failure/Error:(?<stacktrace>.+))|(?<stacktrace>.+))}m
|
23
24
|
ISSUE_STACKTRACE_REGEX = /### Stack trace\s*(```)#{FAILURE_STACKTRACE_REGEX}(```)/m
|
@@ -25,7 +26,10 @@ module GitlabQuality
|
|
25
26
|
FAILED_JOB_DESCRIPTION_REGEX = /First happened in #{JOB_URL_REGEX}\./m
|
26
27
|
REPORT_ITEM_REGEX = /^1\. \d{4}-\d{2}-\d{2}: #{JOB_URL_REGEX} \((?<pipeline_url>.+)\)$/
|
27
28
|
NEW_ISSUE_LABELS = Set.new(%w[test failure::new priority::2]).freeze
|
28
|
-
IGNORE_EXCEPTIONS = [
|
29
|
+
IGNORE_EXCEPTIONS = [
|
30
|
+
'Net::ReadTimeout',
|
31
|
+
'403 Forbidden - Your account has been blocked'
|
32
|
+
].freeze
|
29
33
|
SCREENSHOT_IGNORED_ERRORS = ['500 Internal Server Error', 'fabricate_via_api!', 'Error Code 500'].freeze
|
30
34
|
|
31
35
|
MultipleIssuesFound = Class.new(StandardError)
|
@@ -47,16 +51,27 @@ module GitlabQuality
|
|
47
51
|
puts "Reporting test failures in `#{files.join(',')}` as issues in project `#{project}` via the API at `#{Runtime::Env.gitlab_api_base}`."
|
48
52
|
|
49
53
|
TestResults::Builder.new(files).test_results_per_file do |test_results|
|
50
|
-
puts "=> Reporting tests in #{test_results.path}"
|
54
|
+
puts "=> Reporting #{test_results.count} tests in #{test_results.path}"
|
55
|
+
|
56
|
+
systemic_exceptions = systemic_exceptions_for_test_results(test_results)
|
51
57
|
|
52
58
|
test_results.each do |test|
|
53
|
-
relate_failure_to_issue(test) if should_report?(test)
|
59
|
+
relate_failure_to_issue(test) if should_report?(test, systemic_exceptions)
|
54
60
|
end
|
55
61
|
|
56
62
|
test_results.write
|
57
63
|
end
|
58
64
|
end
|
59
65
|
|
66
|
+
def systemic_exceptions_for_test_results(test_results)
|
67
|
+
test_results
|
68
|
+
.flat_map { |test| test.report['exceptions']&.map { |exception| exception['message'] } }
|
69
|
+
.compact
|
70
|
+
.tally
|
71
|
+
.select { |_e, count| count >= SYSTEMIC_EXCEPTIONS_THRESHOLD }
|
72
|
+
.keys
|
73
|
+
end
|
74
|
+
|
60
75
|
def relate_failure_to_issue(test)
|
61
76
|
puts " => Relating issues for test '#{test.name}'..."
|
62
77
|
|
@@ -377,14 +392,17 @@ module GitlabQuality
|
|
377
392
|
#
|
378
393
|
# @return [TrueClass|FalseClass] false if the test was skipped or failed because of a transient error that can be ignored.
|
379
394
|
# Otherwise returns true.
|
380
|
-
def should_report?(test)
|
395
|
+
def should_report?(test, systemic_exceptions)
|
381
396
|
return false if test.failures.empty?
|
382
397
|
|
398
|
+
puts " => Systemic exceptions detected: #{systemic_exceptions}" if systemic_exceptions.any?
|
399
|
+
exceptions_to_ignore = IGNORE_EXCEPTIONS + systemic_exceptions
|
400
|
+
|
383
401
|
if test.report.key?('exceptions')
|
384
|
-
reason = ignore_failure_reason(test.report['exceptions'])
|
402
|
+
reason = ignore_failure_reason(test.report['exceptions'], exceptions_to_ignore)
|
385
403
|
|
386
404
|
if reason
|
387
|
-
puts "Failure reporting skipped because #{reason}"
|
405
|
+
puts " => Failure reporting skipped because #{reason}"
|
388
406
|
|
389
407
|
return false
|
390
408
|
end
|
@@ -397,9 +415,9 @@ module GitlabQuality
|
|
397
415
|
#
|
398
416
|
# @param [Array<Hash>] exceptions the exceptions associated with the failure.
|
399
417
|
# @return [String] the reason to ignore the exceptions, or `nil` if any exceptions should not be ignored.
|
400
|
-
def ignore_failure_reason(exceptions)
|
418
|
+
def ignore_failure_reason(exceptions, ignored_exceptions)
|
401
419
|
exception_messages = exceptions
|
402
|
-
.filter_map { |exception| exception['message'] if
|
420
|
+
.filter_map { |exception| exception['message'] if ignored_exceptions.any? { |e| exception['message'].include?(e) } }
|
403
421
|
.compact
|
404
422
|
return if exception_messages.empty? || exception_messages.size < exceptions.size
|
405
423
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gitlab_quality-test_tooling
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.8.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- GitLab Quality
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-06-
|
11
|
+
date: 2023-06-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: climate_control
|
@@ -182,16 +182,22 @@ dependencies:
|
|
182
182
|
name: activesupport
|
183
183
|
requirement: !ruby/object:Gem::Requirement
|
184
184
|
requirements:
|
185
|
-
- - "
|
185
|
+
- - ">="
|
186
186
|
- !ruby/object:Gem::Version
|
187
187
|
version: '6.1'
|
188
|
+
- - "<"
|
189
|
+
- !ruby/object:Gem::Version
|
190
|
+
version: '7.1'
|
188
191
|
type: :runtime
|
189
192
|
prerelease: false
|
190
193
|
version_requirements: !ruby/object:Gem::Requirement
|
191
194
|
requirements:
|
192
|
-
- - "
|
195
|
+
- - ">="
|
193
196
|
- !ruby/object:Gem::Version
|
194
197
|
version: '6.1'
|
198
|
+
- - "<"
|
199
|
+
- !ruby/object:Gem::Version
|
200
|
+
version: '7.1'
|
195
201
|
- !ruby/object:Gem::Dependency
|
196
202
|
name: gitlab
|
197
203
|
requirement: !ruby/object:Gem::Requirement
|