minitest-distributed 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +4 -0
- data/Gemfile +1 -1
- data/README.md +29 -13
- data/bin/setup +0 -2
- data/lib/minitest/distributed/configuration.rb +49 -4
- data/lib/minitest/distributed/coordinators/coordinator_interface.rb +3 -0
- data/lib/minitest/distributed/coordinators/memory_coordinator.rb +29 -9
- data/lib/minitest/distributed/coordinators/redis_coordinator.rb +258 -156
- data/lib/minitest/distributed/enqueued_runnable.rb +193 -41
- data/lib/minitest/distributed/filters/exclude_filter.rb +4 -4
- data/lib/minitest/distributed/filters/filter_interface.rb +3 -3
- data/lib/minitest/distributed/filters/include_filter.rb +4 -4
- data/lib/minitest/distributed/reporters/distributed_progress_reporter.rb +2 -2
- data/lib/minitest/distributed/reporters/distributed_summary_reporter.rb +49 -10
- data/lib/minitest/distributed/reporters/redis_coordinator_warnings_reporter.rb +11 -16
- data/lib/minitest/distributed/result_aggregate.rb +38 -9
- data/lib/minitest/distributed/result_type.rb +76 -2
- data/lib/minitest/distributed/test_selector.rb +4 -6
- data/lib/minitest/distributed/version.rb +1 -1
- data/lib/minitest/distributed_plugin.rb +1 -25
- data/sorbet/rbi/minitest.rbi +18 -3
- data/sorbet/rbi/redis.rbi +19 -4
- metadata +2 -2
@@ -3,85 +3,237 @@
|
|
3
3
|
|
4
4
|
module Minitest
|
5
5
|
module Distributed
|
6
|
+
class PendingExecution < T::Struct
|
7
|
+
extend T::Sig
|
8
|
+
|
9
|
+
const :worker_id, String
|
10
|
+
const :entry_id, String
|
11
|
+
const :elapsed_time_ms, Integer
|
12
|
+
const :attempt, Integer
|
13
|
+
|
14
|
+
sig { returns(String) }
|
15
|
+
def attempt_id
|
16
|
+
"#{entry_id}/#{attempt}"
|
17
|
+
end
|
18
|
+
|
19
|
+
sig { params(xpending_result: T::Hash[String, T.untyped]).returns(T.attached_class) }
|
20
|
+
def self.from_xpending(xpending_result)
|
21
|
+
new(
|
22
|
+
worker_id: xpending_result.fetch('consumer'),
|
23
|
+
entry_id: xpending_result.fetch('entry_id'),
|
24
|
+
elapsed_time_ms: xpending_result.fetch('elapsed'),
|
25
|
+
attempt: xpending_result.fetch('count'),
|
26
|
+
)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
# This module defines some helper methods to deal with Minitest::Runnable
|
31
|
+
module DefinedRunnable
|
32
|
+
extend T::Sig
|
33
|
+
|
34
|
+
sig { params(name: String).returns(T.class_of(Minitest::Runnable)) }
|
35
|
+
def self.find_class(name)
|
36
|
+
name.split('::')
|
37
|
+
.reduce(Object) { |ns, const| ns.const_get(const) } # rubocop:disable Sorbet/ConstantsFromStrings
|
38
|
+
end
|
39
|
+
|
40
|
+
sig { params(runnable: Minitest::Runnable).returns(String) }
|
41
|
+
def self.identifier(runnable)
|
42
|
+
"#{T.must(runnable.class.name)}##{runnable.name}"
|
43
|
+
end
|
44
|
+
|
45
|
+
sig { params(identifier: String).returns(Minitest::Runnable) }
|
46
|
+
def self.from_identifier(identifier)
|
47
|
+
class_name, method_name = identifier.split('#', 2)
|
48
|
+
find_class(T.must(class_name)).new(T.must(method_name))
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
6
52
|
class EnqueuedRunnable < T::Struct
|
7
|
-
class
|
53
|
+
class Result < T::Struct
|
54
|
+
class Commit
|
55
|
+
extend T::Sig
|
56
|
+
|
57
|
+
sig { params(block: T.proc.returns(T::Boolean)).void }
|
58
|
+
def initialize(&block)
|
59
|
+
@block = block
|
60
|
+
end
|
61
|
+
|
62
|
+
sig { returns(T::Boolean) }
|
63
|
+
def success?
|
64
|
+
@success = T.let(@success, T.nilable(T::Boolean))
|
65
|
+
@success ||= @block.call
|
66
|
+
end
|
67
|
+
|
68
|
+
sig { returns(T::Boolean) }
|
69
|
+
def failure?
|
70
|
+
!success?
|
71
|
+
end
|
72
|
+
|
73
|
+
sig { returns(Commit) }
|
74
|
+
def self.success
|
75
|
+
@success = T.let(@success, T.nilable(Commit))
|
76
|
+
@success ||= new { true }
|
77
|
+
end
|
78
|
+
|
79
|
+
sig { returns(Commit) }
|
80
|
+
def self.failure
|
81
|
+
@failure = T.let(@failure, T.nilable(Commit))
|
82
|
+
@failure ||= new { false }
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
8
86
|
extend T::Sig
|
9
87
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
88
|
+
const :enqueued_runnable, EnqueuedRunnable
|
89
|
+
const :initial_result, Minitest::Result
|
90
|
+
const :commit, Commit
|
91
|
+
|
92
|
+
sig { returns(String) }
|
93
|
+
def entry_id
|
94
|
+
enqueued_runnable.entry_id
|
95
|
+
end
|
96
|
+
|
97
|
+
sig { returns(T::Boolean) }
|
98
|
+
def final?
|
99
|
+
!requeue?
|
17
100
|
end
|
18
101
|
|
19
|
-
sig {
|
20
|
-
def
|
21
|
-
|
22
|
-
class_name: T.must(runnable.class.name),
|
23
|
-
method_name: runnable.name,
|
24
|
-
)
|
102
|
+
sig { returns(T::Boolean) }
|
103
|
+
def requeue?
|
104
|
+
ResultType.of(initial_result) == ResultType::Requeued
|
25
105
|
end
|
26
106
|
|
27
|
-
sig {
|
28
|
-
def
|
29
|
-
|
107
|
+
sig { returns(Minitest::Result) }
|
108
|
+
def committed_result
|
109
|
+
@committed_result = T.let(@committed_result, T.nilable(Minitest::Result))
|
110
|
+
@committed_result ||= if final? && commit.failure?
|
111
|
+
# If a runnable result is final, but the acked failed, we will discard the result.
|
112
|
+
Minitest::Discard.wrap(initial_result, test_timeout_seconds: enqueued_runnable.test_timeout_seconds)
|
113
|
+
else
|
114
|
+
initial_result
|
115
|
+
end
|
30
116
|
end
|
117
|
+
end
|
118
|
+
|
119
|
+
class << self
|
120
|
+
extend T::Sig
|
121
|
+
|
122
|
+
sig do
|
123
|
+
params(
|
124
|
+
claims: T::Array[[String, T::Hash[String, String]]],
|
125
|
+
pending_messages: T::Hash[String, PendingExecution],
|
126
|
+
configuration: Configuration,
|
127
|
+
).returns(T::Array[T.attached_class])
|
128
|
+
end
|
129
|
+
def from_redis_stream_claim(claims, pending_messages = {}, configuration:)
|
130
|
+
claims.map do |entry_id, runnable_method_info|
|
131
|
+
# `attempt` will be set to the current attempt of a different worker that has timed out.
|
132
|
+
# The attempt we are going to try will be the next one, so add one.
|
133
|
+
attempt = pending_messages.key?(entry_id) ? pending_messages.fetch(entry_id).attempt + 1 : 1
|
31
134
|
|
32
|
-
sig { params(claims: T::Array[[String, T::Hash[String, String]]]).returns(T::Array[T.attached_class]) }
|
33
|
-
def from_redis_stream_claim(claims)
|
34
|
-
claims.map do |id, runnable_method_info|
|
35
135
|
new(
|
36
136
|
class_name: runnable_method_info.fetch('class_name'),
|
37
137
|
method_name: runnable_method_info.fetch('method_name'),
|
38
|
-
|
138
|
+
entry_id: entry_id,
|
139
|
+
attempt: attempt,
|
140
|
+
max_attempts: configuration.max_attempts,
|
141
|
+
test_timeout_seconds: configuration.test_timeout_seconds,
|
39
142
|
)
|
40
143
|
end
|
41
144
|
end
|
42
|
-
|
43
|
-
sig { params(name: String).returns(T.class_of(Minitest::Runnable)) }
|
44
|
-
def find_runnable_class(name)
|
45
|
-
name.split('::')
|
46
|
-
.reduce(Object) { |ns, const| ns.const_get(const) } # rubocop:disable Sorbet/ConstantsFromStrings
|
47
|
-
end
|
48
145
|
end
|
49
146
|
|
50
147
|
extend T::Sig
|
51
148
|
|
52
149
|
const :class_name, String
|
53
150
|
const :method_name, String
|
54
|
-
const :
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
prop :canned_failure, T.nilable(Minitest::Assertion), dont_store: true
|
151
|
+
const :entry_id, String, factory: -> { SecureRandom.uuid }, dont_store: true
|
152
|
+
const :attempt, Integer, default: 1, dont_store: true
|
153
|
+
const :max_attempts, Integer, dont_store: true
|
154
|
+
const :test_timeout_seconds, Float, dont_store: true
|
59
155
|
|
60
156
|
sig { returns(String) }
|
61
157
|
def identifier
|
62
158
|
"#{class_name}##{method_name}"
|
63
159
|
end
|
64
160
|
|
161
|
+
sig { returns(String) }
|
162
|
+
def attempt_id
|
163
|
+
"#{entry_id}/#{attempt}"
|
164
|
+
end
|
165
|
+
|
65
166
|
sig { returns(T.class_of(Minitest::Runnable)) }
|
66
167
|
def runnable_class
|
67
|
-
|
168
|
+
DefinedRunnable.find_class(class_name)
|
68
169
|
end
|
69
170
|
|
70
171
|
sig { returns(Minitest::Runnable) }
|
71
|
-
def
|
172
|
+
def instantiate_runnable
|
72
173
|
runnable_class.new(method_name)
|
73
174
|
end
|
74
175
|
|
176
|
+
sig { returns(T::Boolean) }
|
177
|
+
def attempts_exhausted?
|
178
|
+
attempt > max_attempts
|
179
|
+
end
|
180
|
+
|
181
|
+
sig { returns(T::Boolean) }
|
182
|
+
def final_attempt?
|
183
|
+
attempt == max_attempts
|
184
|
+
end
|
185
|
+
|
75
186
|
sig { returns(Minitest::Result) }
|
76
|
-
def
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
187
|
+
def attempts_exhausted_result
|
188
|
+
assertion = Minitest::AttemptsExhausted.new(<<~EOM.chomp)
|
189
|
+
This test takes too long to run (> #{test_timeout_seconds}s).
|
190
|
+
|
191
|
+
We have tried running this test #{max_attempts} on different workers, but every time the worker has not reported back a result within #{test_timeout_seconds}s.
|
192
|
+
Try to make the test faster, or increase the test timeout.
|
193
|
+
EOM
|
194
|
+
assertion.set_backtrace(caller)
|
195
|
+
|
196
|
+
runnable = instantiate_runnable
|
197
|
+
runnable.time = 0.0
|
198
|
+
runnable.failures = [assertion]
|
199
|
+
|
200
|
+
Minitest::Result.from(runnable)
|
201
|
+
end
|
202
|
+
|
203
|
+
sig do
|
204
|
+
params(
|
205
|
+
block: T.proc.params(arg0: Minitest::Result).returns(EnqueuedRunnable::Result::Commit)
|
206
|
+
).returns(EnqueuedRunnable::Result)
|
207
|
+
end
|
208
|
+
def run(&block)
|
209
|
+
initial_result = if attempts_exhausted?
|
210
|
+
attempts_exhausted_result
|
82
211
|
else
|
83
|
-
Minitest.run_one_method(runnable_class, method_name)
|
212
|
+
result = Minitest.run_one_method(runnable_class, method_name)
|
213
|
+
result_type = ResultType.of(result)
|
214
|
+
if (result_type == ResultType::Error || result_type == ResultType::Failed) && !final_attempt?
|
215
|
+
result = Minitest::Requeue.wrap(result, attempt: attempt, max_attempts: max_attempts)
|
216
|
+
end
|
217
|
+
result
|
84
218
|
end
|
219
|
+
|
220
|
+
EnqueuedRunnable::Result.new(
|
221
|
+
enqueued_runnable: self,
|
222
|
+
initial_result: initial_result,
|
223
|
+
commit: block.call(initial_result),
|
224
|
+
)
|
225
|
+
end
|
226
|
+
|
227
|
+
sig { returns(T.self_type) }
|
228
|
+
def next_attempt
|
229
|
+
self.class.new(
|
230
|
+
class_name: class_name,
|
231
|
+
method_name: method_name,
|
232
|
+
entry_id: entry_id,
|
233
|
+
attempt: attempt + 1,
|
234
|
+
max_attempts: max_attempts,
|
235
|
+
test_timeout_seconds: test_timeout_seconds,
|
236
|
+
)
|
85
237
|
end
|
86
238
|
end
|
87
239
|
end
|
@@ -19,13 +19,13 @@ module Minitest
|
|
19
19
|
end
|
20
20
|
end
|
21
21
|
|
22
|
-
sig { override.params(
|
23
|
-
def call(
|
22
|
+
sig { override.params(runnable: Minitest::Runnable).returns(T::Array[Runnable]) }
|
23
|
+
def call(runnable)
|
24
24
|
# rubocop:disable Style/CaseEquality
|
25
|
-
if filter ===
|
25
|
+
if filter === runnable.name || filter === DefinedRunnable.identifier(runnable)
|
26
26
|
[]
|
27
27
|
else
|
28
|
-
[
|
28
|
+
[runnable]
|
29
29
|
end
|
30
30
|
# rubocop:enable Style/CaseEquality
|
31
31
|
end
|
@@ -9,7 +9,7 @@ module Minitest
|
|
9
9
|
# array of runnables.
|
10
10
|
#
|
11
11
|
# - If it returns an empty array, the runnable will not be run.
|
12
|
-
# - If it returns a single
|
12
|
+
# - If it returns a single element array with the passed ion runnable to make no changes.
|
13
13
|
# - It can return an array of enumerables to expand the number of runnables in this test run,
|
14
14
|
# We use this for grinding tests, for instance.
|
15
15
|
module FilterInterface
|
@@ -17,8 +17,8 @@ module Minitest
|
|
17
17
|
extend T::Helpers
|
18
18
|
interface!
|
19
19
|
|
20
|
-
sig { abstract.params(
|
21
|
-
def call(
|
20
|
+
sig { abstract.params(runnable: Minitest::Runnable).returns(T::Array[Minitest::Runnable]) }
|
21
|
+
def call(runnable); end
|
22
22
|
end
|
23
23
|
end
|
24
24
|
end
|
@@ -19,11 +19,11 @@ module Minitest
|
|
19
19
|
end
|
20
20
|
end
|
21
21
|
|
22
|
-
sig { override.params(
|
23
|
-
def call(
|
22
|
+
sig { override.params(runnable: Minitest::Runnable).returns(T::Array[Minitest::Runnable]) }
|
23
|
+
def call(runnable)
|
24
24
|
# rubocop:disable Style/CaseEquality
|
25
|
-
if filter ===
|
26
|
-
[
|
25
|
+
if filter === runnable.name || filter === DefinedRunnable.identifier(runnable)
|
26
|
+
[runnable]
|
27
27
|
else
|
28
28
|
[]
|
29
29
|
end
|
@@ -50,9 +50,9 @@ module Minitest
|
|
50
50
|
case (result_type = ResultType.of(result))
|
51
51
|
when ResultType::Passed
|
52
52
|
# TODO: warn for tests that are slower than the test timeout.
|
53
|
-
when ResultType::Skipped
|
53
|
+
when ResultType::Skipped, ResultType::Discarded
|
54
54
|
io.puts("#{result}\n") if options[:verbose]
|
55
|
-
when ResultType::Error, ResultType::Failed
|
55
|
+
when ResultType::Error, ResultType::Failed, ResultType::Requeued
|
56
56
|
io.puts("#{result}\n")
|
57
57
|
else
|
58
58
|
T.absurd(result_type)
|
@@ -7,14 +7,10 @@ module Minitest
|
|
7
7
|
class DistributedSummaryReporter < Minitest::Reporter
|
8
8
|
extend T::Sig
|
9
9
|
|
10
|
-
sig { returns(Coordinators::CoordinatorInterface) }
|
11
|
-
attr_reader :coordinator
|
12
|
-
|
13
10
|
sig { params(io: IO, options: T::Hash[Symbol, T.untyped]).void }
|
14
11
|
def initialize(io, options)
|
15
12
|
super
|
16
13
|
io.sync = true
|
17
|
-
@coordinator = T.let(options[:distributed].coordinator, Coordinators::CoordinatorInterface)
|
18
14
|
@start_time = T.let(0.0, Float)
|
19
15
|
end
|
20
16
|
|
@@ -26,21 +22,64 @@ module Minitest
|
|
26
22
|
|
27
23
|
sig { override.void }
|
28
24
|
def report
|
29
|
-
|
25
|
+
print_discard_warning if local_results.discards > 0
|
26
|
+
|
27
|
+
if configuration.coordinator.aborted?
|
28
|
+
io.puts("Cannot retry a run that was cut short during the previous attempt.")
|
29
|
+
io.puts
|
30
|
+
elsif combined_results.abort?
|
31
|
+
io.puts("The run was cut short after reaching the limit of #{configuration.max_failures} test failures.")
|
32
|
+
io.puts
|
33
|
+
end
|
30
34
|
|
31
|
-
|
32
|
-
combined_results = coordinator.combined_results
|
35
|
+
formatted_duration = format("(in %0.3fs)", Minitest.clock_time - @start_time)
|
33
36
|
if combined_results == local_results
|
34
|
-
io.puts("Results: #{combined_results} #{
|
37
|
+
io.puts("Results: #{combined_results} #{formatted_duration}")
|
35
38
|
else
|
36
|
-
io.puts("This worker: #{local_results} #{
|
39
|
+
io.puts("This worker: #{local_results} #{formatted_duration}")
|
37
40
|
io.puts("Combined results: #{combined_results}")
|
38
41
|
end
|
39
42
|
end
|
40
43
|
|
41
44
|
sig { override.returns(T::Boolean) }
|
42
45
|
def passed?
|
43
|
-
coordinator.
|
46
|
+
return false if configuration.coordinator.aborted?
|
47
|
+
|
48
|
+
# Generally, we want the workers to fail that had at least one failed or errored
|
49
|
+
# test. We have to trust that another worker will fail (and fail the build) if it
|
50
|
+
# encountered a failed test. We trust that the other worker will do this correctly,
|
51
|
+
# but we do verify that the statistics for the complete run are valid,
|
52
|
+
# to have some protection against unknown edge cases and bugs.
|
53
|
+
local_results.passed? && combined_results.valid?
|
54
|
+
end
|
55
|
+
|
56
|
+
protected
|
57
|
+
|
58
|
+
sig { void }
|
59
|
+
def print_discard_warning
|
60
|
+
io.puts(<<~WARNING)
|
61
|
+
WARNING: This worker was not able to ack all the tests it ran with the coordinator,
|
62
|
+
and had to discard the results of those tests. This means that some of your tests may
|
63
|
+
take too long to run. Make sure that all your tests complete well within #{configuration.test_timeout_seconds}s.
|
64
|
+
|
65
|
+
WARNING
|
66
|
+
end
|
67
|
+
|
68
|
+
sig { returns(ResultAggregate) }
|
69
|
+
def local_results
|
70
|
+
@local_results = T.let(@local_results, T.nilable(ResultAggregate))
|
71
|
+
@local_results ||= configuration.coordinator.local_results
|
72
|
+
end
|
73
|
+
|
74
|
+
sig { returns(ResultAggregate) }
|
75
|
+
def combined_results
|
76
|
+
@combined_results = T.let(@combined_results, T.nilable(ResultAggregate))
|
77
|
+
@combined_results ||= configuration.coordinator.combined_results
|
78
|
+
end
|
79
|
+
|
80
|
+
sig { returns(Configuration) }
|
81
|
+
def configuration
|
82
|
+
T.let(options[:distributed], Configuration)
|
44
83
|
end
|
45
84
|
end
|
46
85
|
end
|
@@ -9,9 +9,10 @@ module Minitest
|
|
9
9
|
|
10
10
|
sig { override.void }
|
11
11
|
def report
|
12
|
-
[
|
13
|
-
|
12
|
+
warnings = [reclaim_timeout_warning, reclaim_failed_warning].compact
|
13
|
+
warnings.each do |warning|
|
14
14
|
io.puts(warning)
|
15
|
+
io.puts
|
15
16
|
end
|
16
17
|
end
|
17
18
|
|
@@ -28,30 +29,24 @@ module Minitest
|
|
28
29
|
end
|
29
30
|
|
30
31
|
sig { returns(T.nilable(String)) }
|
31
|
-
def
|
32
|
-
if redis_coordinator.
|
32
|
+
def reclaim_timeout_warning
|
33
|
+
if redis_coordinator.reclaimed_timeout_tests.any?
|
33
34
|
<<~WARNING
|
34
35
|
WARNING: The following tests were reclaimed from another worker:
|
35
|
-
#{redis_coordinator.
|
36
|
+
#{redis_coordinator.reclaimed_timeout_tests.map { |test| "- #{test.identifier}" }.join("\n")}
|
36
37
|
|
37
|
-
The original worker did not complete running
|
38
|
+
The original worker did not complete running these tests in #{configuration.test_timeout_seconds}s.
|
38
39
|
This either means that the worker unexpectedly went away, or that the test is too slow.
|
39
40
|
WARNING
|
40
41
|
end
|
41
42
|
end
|
42
43
|
|
43
44
|
sig { returns(T.nilable(String)) }
|
44
|
-
def
|
45
|
-
|
46
|
-
if local_results.acks < local_results.size
|
45
|
+
def reclaim_failed_warning
|
46
|
+
if redis_coordinator.reclaimed_failed_tests.any?
|
47
47
|
<<~WARNING
|
48
|
-
WARNING:
|
49
|
-
|
50
|
-
This means that this worker took too long to report the status of one or more tests,
|
51
|
-
and these tests were claimed by other workers. As a result, the total number of
|
52
|
-
reported runs may be larger than the size of the test suite.
|
53
|
-
|
54
|
-
Make sure that all your tests complete within #{configuration.test_timeout}ms.
|
48
|
+
WARNING: The following tests were reclaimed from another worker because they failed:
|
49
|
+
#{redis_coordinator.reclaimed_failed_tests.map { |test| "- #{test.identifier}" }.join("\n")}
|
55
50
|
WARNING
|
56
51
|
end
|
57
52
|
end
|