minitest-distributed 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,85 +3,237 @@
3
3
 
4
4
  module Minitest
5
5
  module Distributed
6
+ class PendingExecution < T::Struct
7
+ extend T::Sig
8
+
9
+ const :worker_id, String
10
+ const :entry_id, String
11
+ const :elapsed_time_ms, Integer
12
+ const :attempt, Integer
13
+
14
+ sig { returns(String) }
15
+ def attempt_id
16
+ "#{entry_id}/#{attempt}"
17
+ end
18
+
19
+ sig { params(xpending_result: T::Hash[String, T.untyped]).returns(T.attached_class) }
20
+ def self.from_xpending(xpending_result)
21
+ new(
22
+ worker_id: xpending_result.fetch('consumer'),
23
+ entry_id: xpending_result.fetch('entry_id'),
24
+ elapsed_time_ms: xpending_result.fetch('elapsed'),
25
+ attempt: xpending_result.fetch('count'),
26
+ )
27
+ end
28
+ end
29
+
30
+ # This module defines some helper methods to deal with Minitest::Runnable
31
+ module DefinedRunnable
32
+ extend T::Sig
33
+
34
+ sig { params(name: String).returns(T.class_of(Minitest::Runnable)) }
35
+ def self.find_class(name)
36
+ name.split('::')
37
+ .reduce(Object) { |ns, const| ns.const_get(const) } # rubocop:disable Sorbet/ConstantsFromStrings
38
+ end
39
+
40
+ sig { params(runnable: Minitest::Runnable).returns(String) }
41
+ def self.identifier(runnable)
42
+ "#{T.must(runnable.class.name)}##{runnable.name}"
43
+ end
44
+
45
+ sig { params(identifier: String).returns(Minitest::Runnable) }
46
+ def self.from_identifier(identifier)
47
+ class_name, method_name = identifier.split('#', 2)
48
+ find_class(T.must(class_name)).new(T.must(method_name))
49
+ end
50
+ end
51
+
6
52
  class EnqueuedRunnable < T::Struct
7
- class << self
53
+ class Result < T::Struct
54
+ class Commit
55
+ extend T::Sig
56
+
57
+ sig { params(block: T.proc.returns(T::Boolean)).void }
58
+ def initialize(&block)
59
+ @block = block
60
+ end
61
+
62
+ sig { returns(T::Boolean) }
63
+ def success?
64
+ @success = T.let(@success, T.nilable(T::Boolean))
65
+ @success ||= @block.call
66
+ end
67
+
68
+ sig { returns(T::Boolean) }
69
+ def failure?
70
+ !success?
71
+ end
72
+
73
+ sig { returns(Commit) }
74
+ def self.success
75
+ @success = T.let(@success, T.nilable(Commit))
76
+ @success ||= new { true }
77
+ end
78
+
79
+ sig { returns(Commit) }
80
+ def self.failure
81
+ @failure = T.let(@failure, T.nilable(Commit))
82
+ @failure ||= new { false }
83
+ end
84
+ end
85
+
8
86
  extend T::Sig
9
87
 
10
- sig { params(identifier: String).returns(T.attached_class) }
11
- def from_identifier(identifier)
12
- class_name, method_name = identifier.split('#', 2)
13
- new(
14
- class_name: T.must(class_name),
15
- method_name: T.must(method_name),
16
- )
88
+ const :enqueued_runnable, EnqueuedRunnable
89
+ const :initial_result, Minitest::Result
90
+ const :commit, Commit
91
+
92
+ sig { returns(String) }
93
+ def entry_id
94
+ enqueued_runnable.entry_id
95
+ end
96
+
97
+ sig { returns(T::Boolean) }
98
+ def final?
99
+ !requeue?
17
100
  end
18
101
 
19
- sig { params(runnable: Minitest::Runnable).returns(T.attached_class) }
20
- def from_runnable(runnable)
21
- new(
22
- class_name: T.must(runnable.class.name),
23
- method_name: runnable.name,
24
- )
102
+ sig { returns(T::Boolean) }
103
+ def requeue?
104
+ ResultType.of(initial_result) == ResultType::Requeued
25
105
  end
26
106
 
27
- sig { params(result: Minitest::Result).returns(T.attached_class) }
28
- def from_result(result)
29
- new(class_name: result.class_name, method_name: result.name)
107
+ sig { returns(Minitest::Result) }
108
+ def committed_result
109
+ @committed_result = T.let(@committed_result, T.nilable(Minitest::Result))
110
+ @committed_result ||= if final? && commit.failure?
111
+ # If a runnable result is final, but the acked failed, we will discard the result.
112
+ Minitest::Discard.wrap(initial_result, test_timeout_seconds: enqueued_runnable.test_timeout_seconds)
113
+ else
114
+ initial_result
115
+ end
30
116
  end
117
+ end
118
+
119
+ class << self
120
+ extend T::Sig
121
+
122
+ sig do
123
+ params(
124
+ claims: T::Array[[String, T::Hash[String, String]]],
125
+ pending_messages: T::Hash[String, PendingExecution],
126
+ configuration: Configuration,
127
+ ).returns(T::Array[T.attached_class])
128
+ end
129
+ def from_redis_stream_claim(claims, pending_messages = {}, configuration:)
130
+ claims.map do |entry_id, runnable_method_info|
131
+ # `attempt` will be set to the current attempt of a different worker that has timed out.
132
+ # The attempt we are going to try will be the next one, so add one.
133
+ attempt = pending_messages.key?(entry_id) ? pending_messages.fetch(entry_id).attempt + 1 : 1
31
134
 
32
- sig { params(claims: T::Array[[String, T::Hash[String, String]]]).returns(T::Array[T.attached_class]) }
33
- def from_redis_stream_claim(claims)
34
- claims.map do |id, runnable_method_info|
35
135
  new(
36
136
  class_name: runnable_method_info.fetch('class_name'),
37
137
  method_name: runnable_method_info.fetch('method_name'),
38
- execution_id: id,
138
+ entry_id: entry_id,
139
+ attempt: attempt,
140
+ max_attempts: configuration.max_attempts,
141
+ test_timeout_seconds: configuration.test_timeout_seconds,
39
142
  )
40
143
  end
41
144
  end
42
-
43
- sig { params(name: String).returns(T.class_of(Minitest::Runnable)) }
44
- def find_runnable_class(name)
45
- name.split('::')
46
- .reduce(Object) { |ns, const| ns.const_get(const) } # rubocop:disable Sorbet/ConstantsFromStrings
47
- end
48
145
  end
49
146
 
50
147
  extend T::Sig
51
148
 
52
149
  const :class_name, String
53
150
  const :method_name, String
54
- const :execution_id, T.nilable(String), dont_store: true
55
-
56
- # By setting canned failure, we will not actually run the runnable,
57
- # but immediately return a result with the canned assertion.x
58
- prop :canned_failure, T.nilable(Minitest::Assertion), dont_store: true
151
+ const :entry_id, String, factory: -> { SecureRandom.uuid }, dont_store: true
152
+ const :attempt, Integer, default: 1, dont_store: true
153
+ const :max_attempts, Integer, dont_store: true
154
+ const :test_timeout_seconds, Float, dont_store: true
59
155
 
60
156
  sig { returns(String) }
61
157
  def identifier
62
158
  "#{class_name}##{method_name}"
63
159
  end
64
160
 
161
+ sig { returns(String) }
162
+ def attempt_id
163
+ "#{entry_id}/#{attempt}"
164
+ end
165
+
65
166
  sig { returns(T.class_of(Minitest::Runnable)) }
66
167
  def runnable_class
67
- self.class.find_runnable_class(class_name)
168
+ DefinedRunnable.find_class(class_name)
68
169
  end
69
170
 
70
171
  sig { returns(Minitest::Runnable) }
71
- def runnable
172
+ def instantiate_runnable
72
173
  runnable_class.new(method_name)
73
174
  end
74
175
 
176
+ sig { returns(T::Boolean) }
177
+ def attempts_exhausted?
178
+ attempt > max_attempts
179
+ end
180
+
181
+ sig { returns(T::Boolean) }
182
+ def final_attempt?
183
+ attempt == max_attempts
184
+ end
185
+
75
186
  sig { returns(Minitest::Result) }
76
- def run
77
- if canned_failure
78
- canned_runnable = runnable
79
- canned_runnable.time = 0.0
80
- canned_runnable.failures << canned_failure
81
- Minitest::Result.from(canned_runnable)
187
+ def attempts_exhausted_result
188
+ assertion = Minitest::AttemptsExhausted.new(<<~EOM.chomp)
189
+ This test takes too long to run (> #{test_timeout_seconds}s).
190
+
191
+ We have tried running this test #{max_attempts} on different workers, but every time the worker has not reported back a result within #{test_timeout_seconds}s.
192
+ Try to make the test faster, or increase the test timeout.
193
+ EOM
194
+ assertion.set_backtrace(caller)
195
+
196
+ runnable = instantiate_runnable
197
+ runnable.time = 0.0
198
+ runnable.failures = [assertion]
199
+
200
+ Minitest::Result.from(runnable)
201
+ end
202
+
203
+ sig do
204
+ params(
205
+ block: T.proc.params(arg0: Minitest::Result).returns(EnqueuedRunnable::Result::Commit)
206
+ ).returns(EnqueuedRunnable::Result)
207
+ end
208
+ def run(&block)
209
+ initial_result = if attempts_exhausted?
210
+ attempts_exhausted_result
82
211
  else
83
- Minitest.run_one_method(runnable_class, method_name)
212
+ result = Minitest.run_one_method(runnable_class, method_name)
213
+ result_type = ResultType.of(result)
214
+ if (result_type == ResultType::Error || result_type == ResultType::Failed) && !final_attempt?
215
+ result = Minitest::Requeue.wrap(result, attempt: attempt, max_attempts: max_attempts)
216
+ end
217
+ result
84
218
  end
219
+
220
+ EnqueuedRunnable::Result.new(
221
+ enqueued_runnable: self,
222
+ initial_result: initial_result,
223
+ commit: block.call(initial_result),
224
+ )
225
+ end
226
+
227
+ sig { returns(T.self_type) }
228
+ def next_attempt
229
+ self.class.new(
230
+ class_name: class_name,
231
+ method_name: method_name,
232
+ entry_id: entry_id,
233
+ attempt: attempt + 1,
234
+ max_attempts: max_attempts,
235
+ test_timeout_seconds: test_timeout_seconds,
236
+ )
85
237
  end
86
238
  end
87
239
  end
@@ -19,13 +19,13 @@ module Minitest
19
19
  end
20
20
  end
21
21
 
22
- sig { override.params(enqueued_runnable: EnqueuedRunnable).returns(T::Array[EnqueuedRunnable]) }
23
- def call(enqueued_runnable)
22
+ sig { override.params(runnable: Minitest::Runnable).returns(T::Array[Runnable]) }
23
+ def call(runnable)
24
24
  # rubocop:disable Style/CaseEquality
25
- if filter === enqueued_runnable.method_name || filter === enqueued_runnable.identifier
25
+ if filter === runnable.name || filter === DefinedRunnable.identifier(runnable)
26
26
  []
27
27
  else
28
- [enqueued_runnable]
28
+ [runnable]
29
29
  end
30
30
  # rubocop:enable Style/CaseEquality
31
31
  end
@@ -9,7 +9,7 @@ module Minitest
9
9
  # array of runnables.
10
10
  #
11
11
  # - If it returns an empty array, the runnable will not be run.
12
- # - If it returns a single elemnt array with the passed ion runnable to make no changes.
12
+ # - If it returns a single element array with the passed ion runnable to make no changes.
13
13
  # - It can return an array of enumerables to expand the number of runnables in this test run,
14
14
  # We use this for grinding tests, for instance.
15
15
  module FilterInterface
@@ -17,8 +17,8 @@ module Minitest
17
17
  extend T::Helpers
18
18
  interface!
19
19
 
20
- sig { abstract.params(runnable_method: EnqueuedRunnable).returns(T::Array[EnqueuedRunnable]) }
21
- def call(runnable_method); end
20
+ sig { abstract.params(runnable: Minitest::Runnable).returns(T::Array[Minitest::Runnable]) }
21
+ def call(runnable); end
22
22
  end
23
23
  end
24
24
  end
@@ -19,11 +19,11 @@ module Minitest
19
19
  end
20
20
  end
21
21
 
22
- sig { override.params(enqueued_runnable: EnqueuedRunnable).returns(T::Array[EnqueuedRunnable]) }
23
- def call(enqueued_runnable)
22
+ sig { override.params(runnable: Minitest::Runnable).returns(T::Array[Minitest::Runnable]) }
23
+ def call(runnable)
24
24
  # rubocop:disable Style/CaseEquality
25
- if filter === enqueued_runnable.method_name || filter === enqueued_runnable.identifier
26
- [enqueued_runnable]
25
+ if filter === runnable.name || filter === DefinedRunnable.identifier(runnable)
26
+ [runnable]
27
27
  else
28
28
  []
29
29
  end
@@ -50,9 +50,9 @@ module Minitest
50
50
  case (result_type = ResultType.of(result))
51
51
  when ResultType::Passed
52
52
  # TODO: warn for tests that are slower than the test timeout.
53
- when ResultType::Skipped
53
+ when ResultType::Skipped, ResultType::Discarded
54
54
  io.puts("#{result}\n") if options[:verbose]
55
- when ResultType::Error, ResultType::Failed
55
+ when ResultType::Error, ResultType::Failed, ResultType::Requeued
56
56
  io.puts("#{result}\n")
57
57
  else
58
58
  T.absurd(result_type)
@@ -7,14 +7,10 @@ module Minitest
7
7
  class DistributedSummaryReporter < Minitest::Reporter
8
8
  extend T::Sig
9
9
 
10
- sig { returns(Coordinators::CoordinatorInterface) }
11
- attr_reader :coordinator
12
-
13
10
  sig { params(io: IO, options: T::Hash[Symbol, T.untyped]).void }
14
11
  def initialize(io, options)
15
12
  super
16
13
  io.sync = true
17
- @coordinator = T.let(options[:distributed].coordinator, Coordinators::CoordinatorInterface)
18
14
  @start_time = T.let(0.0, Float)
19
15
  end
20
16
 
@@ -26,21 +22,64 @@ module Minitest
26
22
 
27
23
  sig { override.void }
28
24
  def report
29
- duration = format("(in %0.3fs)", Minitest.clock_time - @start_time)
25
+ print_discard_warning if local_results.discards > 0
26
+
27
+ if configuration.coordinator.aborted?
28
+ io.puts("Cannot retry a run that was cut short during the previous attempt.")
29
+ io.puts
30
+ elsif combined_results.abort?
31
+ io.puts("The run was cut short after reaching the limit of #{configuration.max_failures} test failures.")
32
+ io.puts
33
+ end
30
34
 
31
- local_results = coordinator.local_results
32
- combined_results = coordinator.combined_results
35
+ formatted_duration = format("(in %0.3fs)", Minitest.clock_time - @start_time)
33
36
  if combined_results == local_results
34
- io.puts("Results: #{combined_results} #{duration}")
37
+ io.puts("Results: #{combined_results} #{formatted_duration}")
35
38
  else
36
- io.puts("This worker: #{local_results} #{duration}")
39
+ io.puts("This worker: #{local_results} #{formatted_duration}")
37
40
  io.puts("Combined results: #{combined_results}")
38
41
  end
39
42
  end
40
43
 
41
44
  sig { override.returns(T::Boolean) }
42
45
  def passed?
43
- coordinator.combined_results.passed?
46
+ return false if configuration.coordinator.aborted?
47
+
48
+ # Generally, we want the workers to fail that had at least one failed or errored
49
+ # test. We have to trust that another worker will fail (and fail the build) if it
50
+ # encountered a failed test. We trust that the other worker will do this correctly,
51
+ # but we do verify that the statistics for the complete run are valid,
52
+ # to have some protection against unknown edge cases and bugs.
53
+ local_results.passed? && combined_results.valid?
54
+ end
55
+
56
+ protected
57
+
58
+ sig { void }
59
+ def print_discard_warning
60
+ io.puts(<<~WARNING)
61
+ WARNING: This worker was not able to ack all the tests it ran with the coordinator,
62
+ and had to discard the results of those tests. This means that some of your tests may
63
+ take too long to run. Make sure that all your tests complete well within #{configuration.test_timeout_seconds}s.
64
+
65
+ WARNING
66
+ end
67
+
68
+ sig { returns(ResultAggregate) }
69
+ def local_results
70
+ @local_results = T.let(@local_results, T.nilable(ResultAggregate))
71
+ @local_results ||= configuration.coordinator.local_results
72
+ end
73
+
74
+ sig { returns(ResultAggregate) }
75
+ def combined_results
76
+ @combined_results = T.let(@combined_results, T.nilable(ResultAggregate))
77
+ @combined_results ||= configuration.coordinator.combined_results
78
+ end
79
+
80
+ sig { returns(Configuration) }
81
+ def configuration
82
+ T.let(options[:distributed], Configuration)
44
83
  end
45
84
  end
46
85
  end
@@ -9,9 +9,10 @@ module Minitest
9
9
 
10
10
  sig { override.void }
11
11
  def report
12
- [reclaim_warning, missing_acks_warning].compact.each do |warning|
13
- io.puts
12
+ warnings = [reclaim_timeout_warning, reclaim_failed_warning].compact
13
+ warnings.each do |warning|
14
14
  io.puts(warning)
15
+ io.puts
15
16
  end
16
17
  end
17
18
 
@@ -28,30 +29,24 @@ module Minitest
28
29
  end
29
30
 
30
31
  sig { returns(T.nilable(String)) }
31
- def reclaim_warning
32
- if redis_coordinator.reclaimed_tests.any?
32
+ def reclaim_timeout_warning
33
+ if redis_coordinator.reclaimed_timeout_tests.any?
33
34
  <<~WARNING
34
35
  WARNING: The following tests were reclaimed from another worker:
35
- #{redis_coordinator.reclaimed_tests.map { |test| "- #{test.identifier}" }.join("\n")}
36
+ #{redis_coordinator.reclaimed_timeout_tests.map { |test| "- #{test.identifier}" }.join("\n")}
36
37
 
37
- The original worker did not complete running this test in #{configuration.test_timeout}ms.
38
+ The original worker did not complete running these tests in #{configuration.test_timeout_seconds}s.
38
39
  This either means that the worker unexpectedly went away, or that the test is too slow.
39
40
  WARNING
40
41
  end
41
42
  end
42
43
 
43
44
  sig { returns(T.nilable(String)) }
44
- def missing_acks_warning
45
- local_results = redis_coordinator.local_results
46
- if local_results.acks < local_results.size
45
+ def reclaim_failed_warning
46
+ if redis_coordinator.reclaimed_failed_tests.any?
47
47
  <<~WARNING
48
- WARNING: This worker was not able to ack all the test it ran with the coordinator (#{local_results.acks}/#{local_results.size}).
49
-
50
- This means that this worker took too long to report the status of one or more tests,
51
- and these tests were claimed by other workers. As a result, the total number of
52
- reported runs may be larger than the size of the test suite.
53
-
54
- Make sure that all your tests complete within #{configuration.test_timeout}ms.
48
+ WARNING: The following tests were reclaimed from another worker because they failed:
49
+ #{redis_coordinator.reclaimed_failed_tests.map { |test| "- #{test.identifier}" }.join("\n")}
55
50
  WARNING
56
51
  end
57
52
  end