padded_cell 0.1.0-aarch64-linux-gnu → 0.3.0-aarch64-linux-gnu
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +38 -0
- data/README.md +170 -8
- data/lib/padded_cell/4.0/padded_cell_native.so +0 -0
- data/lib/padded_cell/asylum.rb +93 -0
- data/lib/padded_cell/cell.rb +23 -14
- data/lib/padded_cell/chart/accumulator.rb +103 -0
- data/lib/padded_cell/chart/config.rb +129 -0
- data/lib/padded_cell/chart/delta.rb +148 -0
- data/lib/padded_cell/chart/document.rb +64 -0
- data/lib/padded_cell/chart/filter.rb +55 -0
- data/lib/padded_cell/chart/merge.rb +63 -0
- data/lib/padded_cell/chart/probe.rb +70 -0
- data/lib/padded_cell/chart/raw.rb +48 -0
- data/lib/padded_cell/chart/report.rb +128 -0
- data/lib/padded_cell/chart/source_files.rb +90 -0
- data/lib/padded_cell/chart.rb +38 -0
- data/lib/padded_cell/cli.rb +70 -4
- data/lib/padded_cell/limits.rb +45 -0
- data/lib/padded_cell/loader.rb +5 -1
- data/lib/padded_cell/pool.rb +82 -58
- data/lib/padded_cell/reporters.rb +20 -3
- data/lib/padded_cell/results.rb +13 -0
- data/lib/padded_cell/runner.rb +150 -22
- data/lib/padded_cell/version.rb +1 -1
- data/lib/padded_cell/worker.rb +63 -0
- data/lib/padded_cell.rb +8 -0
- metadata +15 -1
data/lib/padded_cell/pool.rb
CHANGED
|
@@ -13,18 +13,34 @@ module PaddedCell
|
|
|
13
13
|
#
|
|
14
14
|
# With isolation :test a worker forks a Cell per test; with :worker it runs
|
|
15
15
|
# tests itself, faster but sharing process state. A worker that dies
|
|
16
|
-
# mid-test, or runs past
|
|
17
|
-
# is replaced; its replacement continues the class.
|
|
16
|
+
# mid-test, or runs past its deadline (--timeout, --budget), gets its test
|
|
17
|
+
# reported as an error and is replaced; its replacement continues the class.
|
|
18
|
+
#
|
|
19
|
+
# Replaced, up to a point: workers that keep dying (a boot that segfaults,
|
|
20
|
+
# an after_fork hook that exits) would turn every remaining test into a
|
|
21
|
+
# crash report. Past PaddedCellNative::Restarts' limit the pool gives up.
|
|
22
|
+
# Once the run's budget is spent, it dispatches nothing more.
|
|
18
23
|
class Pool
|
|
19
|
-
Slot = Struct.new(:worker, :pid, :socket, :test, :started, :batch)
|
|
24
|
+
Slot = Struct.new(:worker, :pid, :socket, :test, :started, :batch, :deadline)
|
|
25
|
+
|
|
26
|
+
# Tests left undispatched by stop! (--fail-fast, the budget, giving up).
|
|
27
|
+
attr_reader :undispatched
|
|
28
|
+
|
|
29
|
+
# Why the pool gave up on its workers, or nil.
|
|
30
|
+
attr_reader :gave_up
|
|
20
31
|
|
|
21
32
|
# tests: [[klass, method_name], ...], dispatched by index.
|
|
22
|
-
|
|
33
|
+
# limits: the run's Limits
|
|
34
|
+
# coverage: a Chart::Accumulator when measuring coverage
|
|
35
|
+
def initialize(tests, jobs:, isolation:, seed:, limits: Limits.new, coverage: nil)
|
|
23
36
|
@tests = tests
|
|
24
37
|
@jobs = jobs
|
|
25
38
|
@isolation = isolation
|
|
26
39
|
@seed = seed
|
|
27
|
-
@
|
|
40
|
+
@limits = limits
|
|
41
|
+
@coverage = coverage
|
|
42
|
+
@restarts = PaddedCellNative::Restarts.new(jobs)
|
|
43
|
+
@undispatched = 0
|
|
28
44
|
end
|
|
29
45
|
|
|
30
46
|
# Runs batches of test indices in order (consuming them), yielding
|
|
@@ -34,7 +50,7 @@ module PaddedCell
|
|
|
34
50
|
@on_result = on_result
|
|
35
51
|
@sockets = []
|
|
36
52
|
Hooks.run_before_fork
|
|
37
|
-
@slots = Array.new([@jobs, batches.sum(&:size)].min) { spawn(Slot.new(it, nil, nil, nil, nil, [])) }
|
|
53
|
+
@slots = Array.new([@jobs, batches.sum(&:size)].min) { spawn(Slot.new(it, nil, nil, nil, nil, [], nil)) }
|
|
38
54
|
@slots.each { feed(it) }
|
|
39
55
|
loop do
|
|
40
56
|
busy = @slots.select(&:test)
|
|
@@ -47,8 +63,11 @@ module PaddedCell
|
|
|
47
63
|
shutdown(force: !$!.nil?)
|
|
48
64
|
end
|
|
49
65
|
|
|
66
|
+
def success? = @success
|
|
67
|
+
|
|
50
68
|
# Dispatch nothing more; tests already running still report.
|
|
51
69
|
def stop!
|
|
70
|
+
@undispatched += @queue.sum(&:size) + @slots.sum { it.batch.size }
|
|
52
71
|
@queue.clear
|
|
53
72
|
@slots.each { it.batch.clear }
|
|
54
73
|
end
|
|
@@ -63,7 +82,8 @@ module PaddedCell
|
|
|
63
82
|
# A worker holding another worker's socket would keep it from ever
|
|
64
83
|
# seeing EOF: keep only its own end.
|
|
65
84
|
[parent, *@sockets].each(&:close)
|
|
66
|
-
|
|
85
|
+
Worker.new(@tests, isolation: @isolation, seed: @seed, limits: @limits, probe: @coverage&.probe)
|
|
86
|
+
.run(slot.worker, child)
|
|
67
87
|
end
|
|
68
88
|
child.close
|
|
69
89
|
@sockets << parent
|
|
@@ -71,57 +91,26 @@ module PaddedCell
|
|
|
71
91
|
slot
|
|
72
92
|
end
|
|
73
93
|
|
|
74
|
-
# The worker process.
|
|
75
|
-
def work(worker, socket)
|
|
76
|
-
status = 1
|
|
77
|
-
begin
|
|
78
|
-
srand(@seed)
|
|
79
|
-
setup_error = after_fork(worker)
|
|
80
|
-
while (line = socket.gets)
|
|
81
|
-
payload = Results.dump(perform(Integer(line), socket, setup_error))
|
|
82
|
-
socket.write([payload.bytesize].pack('N'), payload)
|
|
83
|
-
end
|
|
84
|
-
Hooks.run_cleanup(worker) unless setup_error
|
|
85
|
-
status = 0
|
|
86
|
-
rescue Exception => e # anything: report it, then exit!
|
|
87
|
-
warn "padded_cell: worker #{worker} failed: #{e.class}: #{e.message}"
|
|
88
|
-
ensure
|
|
89
|
-
exit!(status)
|
|
90
|
-
end
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
# A failing after_fork hook fails every test of the worker, the way Rails
|
|
94
|
-
# parallel testing reports a broken parallelize_setup.
|
|
95
|
-
def after_fork(worker)
|
|
96
|
-
Hooks.run_after_fork(worker)
|
|
97
|
-
nil
|
|
98
|
-
rescue StandardError => e
|
|
99
|
-
e
|
|
100
|
-
end
|
|
101
|
-
|
|
102
|
-
def perform(index, socket, setup_error)
|
|
103
|
-
klass, name = @tests.fetch(index)
|
|
104
|
-
if setup_error
|
|
105
|
-
Results.crashed(klass, name, "after_fork hook failed: #{setup_error.class}: #{setup_error.message}")
|
|
106
|
-
elsif @isolation == :test
|
|
107
|
-
Cell.run(klass, name, seed: @seed, timeout: @timeout, close: [socket])
|
|
108
|
-
else
|
|
109
|
-
Cell.execute(klass, name)
|
|
110
|
-
end
|
|
111
|
-
end
|
|
112
|
-
|
|
113
94
|
def feed(slot)
|
|
95
|
+
return stop! if @limits.spent?
|
|
96
|
+
|
|
114
97
|
slot.batch = @queue.shift || steal if slot.batch.empty?
|
|
115
98
|
slot.test = slot.batch.shift
|
|
116
99
|
return unless slot.test
|
|
117
100
|
|
|
118
101
|
slot.started = Minitest.clock_time
|
|
102
|
+
slot.deadline = @limits.for_test
|
|
119
103
|
begin
|
|
120
104
|
slot.socket.puts(slot.test)
|
|
121
105
|
rescue Errno::EPIPE, Errno::ECONNRESET
|
|
122
|
-
# The worker died between tests: its replacement gets this one
|
|
106
|
+
# The worker died between tests: its replacement gets this one, unless
|
|
107
|
+
# it dies too. Restarts bounds the retries.
|
|
123
108
|
reap(slot)
|
|
124
|
-
|
|
109
|
+
@coverage&.lost_worker
|
|
110
|
+
return abandon(slot) unless @restarts.died(Limits.now)
|
|
111
|
+
|
|
112
|
+
spawn(slot)
|
|
113
|
+
retry
|
|
125
114
|
end
|
|
126
115
|
end
|
|
127
116
|
|
|
@@ -132,17 +121,24 @@ module PaddedCell
|
|
|
132
121
|
end
|
|
133
122
|
|
|
134
123
|
def receive(slot)
|
|
135
|
-
header = slot
|
|
124
|
+
header = read_frame(slot, 4)
|
|
136
125
|
return died(slot) unless header&.bytesize == 4
|
|
137
126
|
|
|
138
127
|
size = header.unpack1('N')
|
|
139
|
-
payload = slot
|
|
128
|
+
payload = read_frame(slot, size)
|
|
140
129
|
return died(slot) unless payload&.bytesize == size
|
|
141
130
|
|
|
142
131
|
report(slot, Results.load(payload, *@tests.fetch(slot.test)))
|
|
143
132
|
feed(slot)
|
|
144
133
|
end
|
|
145
134
|
|
|
135
|
+
# nil when the worker died: EOF, or a reset when it left data unread.
|
|
136
|
+
def read_frame(slot, size)
|
|
137
|
+
slot.socket.read(size)
|
|
138
|
+
rescue Errno::ECONNRESET
|
|
139
|
+
nil
|
|
140
|
+
end
|
|
141
|
+
|
|
146
142
|
def report(slot, result)
|
|
147
143
|
index = slot.test
|
|
148
144
|
slot.test = nil
|
|
@@ -151,32 +147,51 @@ module PaddedCell
|
|
|
151
147
|
|
|
152
148
|
def died(slot)
|
|
153
149
|
status = reap(slot)
|
|
150
|
+
give_up unless @restarts.died(Limits.now)
|
|
154
151
|
replace(slot, Results.describe(status, 'worker'), Minitest.clock_time - slot.started)
|
|
155
152
|
end
|
|
156
153
|
|
|
157
|
-
#
|
|
158
|
-
|
|
154
|
+
# Stops replacing workers: whatever was not dispatched is not run.
|
|
155
|
+
def give_up
|
|
156
|
+
window = PaddedCellNative::Restarts.window_ns / Limits::NS
|
|
157
|
+
@gave_up = "workers died more than #{@restarts.limit} times in #{window}s"
|
|
158
|
+
stop!
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# A test taken for a worker that died before receiving it, and is not
|
|
162
|
+
# replaced.
|
|
163
|
+
def abandon(slot)
|
|
164
|
+
slot.test = nil
|
|
165
|
+
@undispatched += 1
|
|
166
|
+
give_up
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Cells meet their deadline inside their worker; with :worker isolation
|
|
170
|
+
# the runner kills the worker itself.
|
|
159
171
|
def wait_time(busy)
|
|
160
|
-
return unless @
|
|
172
|
+
return unless @isolation == :worker
|
|
161
173
|
|
|
162
|
-
|
|
163
|
-
|
|
174
|
+
now = Limits.now
|
|
175
|
+
busy.filter_map { Limits.remaining(it.deadline, now) }.min
|
|
164
176
|
end
|
|
165
177
|
|
|
166
178
|
def expire(busy)
|
|
179
|
+
now = Limits.now
|
|
167
180
|
busy.each do |slot|
|
|
168
|
-
next
|
|
181
|
+
next unless slot.deadline.expired?(now)
|
|
169
182
|
|
|
170
183
|
Process.kill(:KILL, slot.pid)
|
|
171
184
|
reap(slot)
|
|
172
|
-
replace(slot,
|
|
185
|
+
replace(slot, @limits.message(slot.deadline), Minitest.clock_time - slot.started)
|
|
173
186
|
end
|
|
174
187
|
end
|
|
175
188
|
|
|
176
189
|
# Reports the running test of a dead worker and starts a new one.
|
|
177
190
|
def replace(slot, message, time)
|
|
191
|
+
@coverage&.lost_worker
|
|
178
192
|
klass, name = @tests.fetch(slot.test)
|
|
179
193
|
report(slot, Results.crashed(klass, name, message, time:))
|
|
194
|
+
stop! if @limits.spent?
|
|
180
195
|
return if @queue.empty? && @slots.all? { it.batch.empty? }
|
|
181
196
|
|
|
182
197
|
spawn(slot)
|
|
@@ -184,8 +199,16 @@ module PaddedCell
|
|
|
184
199
|
end
|
|
185
200
|
|
|
186
201
|
def reap(slot)
|
|
202
|
+
close_socket(slot)
|
|
203
|
+
wait_for(slot)
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def close_socket(slot)
|
|
187
207
|
@sockets.delete(slot.socket)
|
|
188
208
|
slot.socket.close
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def wait_for(slot)
|
|
189
212
|
_, status = Process.wait2(slot.pid)
|
|
190
213
|
slot.pid = nil
|
|
191
214
|
status
|
|
@@ -196,7 +219,8 @@ module PaddedCell
|
|
|
196
219
|
def shutdown(force:)
|
|
197
220
|
live = Array(@slots).select(&:pid)
|
|
198
221
|
live.each { Process.kill(:KILL, it.pid) } if force
|
|
199
|
-
live.each {
|
|
222
|
+
live.each { close_socket(it) }
|
|
223
|
+
@success = live.map { wait_for(it) }.all?(&:success?)
|
|
200
224
|
end
|
|
201
225
|
end
|
|
202
226
|
end
|
|
@@ -36,6 +36,12 @@ module PaddedCell
|
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
def relative(file) = file.delete_prefix("#{Dir.pwd}/")
|
|
39
|
+
|
|
40
|
+
# --asylum's report on a test that passed on retry, if it did.
|
|
41
|
+
def flaky(result) = result.metadata? ? result.metadata[:flaky] : nil
|
|
42
|
+
|
|
43
|
+
# --asylum's attempts at a test that never passed.
|
|
44
|
+
def asylum(result) = result.metadata? ? result.metadata[:asylum] : nil
|
|
39
45
|
end
|
|
40
46
|
|
|
41
47
|
# JUnit XML, as read by GitHub, GitLab, Jenkins and friends.
|
|
@@ -67,10 +73,20 @@ module PaddedCell
|
|
|
67
73
|
file, line = file_and_line(result)
|
|
68
74
|
attributes = %(classname="#{escape(result.klass)}" name="#{escape(result.name)}" file="#{escape(file)}" ) +
|
|
69
75
|
%(line="#{line}" time="#{format('%.6f', result.time)}" assertions="#{result.assertions}")
|
|
70
|
-
body = outcome(result)
|
|
76
|
+
body = outcome(result) + flaky_failure(result)
|
|
71
77
|
body.empty? ? %( <testcase #{attributes}/>\n) : %( <testcase #{attributes}>\n#{body} </testcase>\n)
|
|
72
78
|
end
|
|
73
79
|
|
|
80
|
+
# Surefire's element for a test that failed, then passed on retry.
|
|
81
|
+
def flaky_failure(result)
|
|
82
|
+
info = flaky(result)
|
|
83
|
+
return '' unless info
|
|
84
|
+
|
|
85
|
+
first = info[:first_failure]
|
|
86
|
+
%( <flakyFailure type="#{escape(first[:type])}" message="#{escape(first[:message])}">) +
|
|
87
|
+
%(#{escape(info[:cause])}</flakyFailure>\n)
|
|
88
|
+
end
|
|
89
|
+
|
|
74
90
|
def outcome(result)
|
|
75
91
|
failure = result.failure
|
|
76
92
|
case status(result)
|
|
@@ -98,8 +114,9 @@ module PaddedCell
|
|
|
98
114
|
{
|
|
99
115
|
id: "#{result.klass}##{result.name}", class: result.klass, name: result.name, file:, line:,
|
|
100
116
|
status: status(result), time: result.time, assertions: result.assertions,
|
|
101
|
-
failures: result.failures.map { { type: it.result_label, message: it.message.scrub } }
|
|
102
|
-
|
|
117
|
+
failures: result.failures.map { { type: it.result_label, message: it.message.scrub } },
|
|
118
|
+
flaky: flaky(result), asylum: asylum(result)
|
|
119
|
+
}.compact
|
|
103
120
|
end
|
|
104
121
|
end
|
|
105
122
|
end
|
data/lib/padded_cell/results.rb
CHANGED
|
@@ -74,6 +74,19 @@ module PaddedCell
|
|
|
74
74
|
placeholders << [owner, last]
|
|
75
75
|
end
|
|
76
76
|
|
|
77
|
+
# What a cell sends its worker: its coverage delta (nil without
|
|
78
|
+
# coverage) beside its result. The outer layer holds only core types, so
|
|
79
|
+
# it loads even when the result does not.
|
|
80
|
+
def envelope(delta, result) = Marshal.dump([delta, dump(result)])
|
|
81
|
+
|
|
82
|
+
# [delta, result] from an envelope; never raises.
|
|
83
|
+
def unwrap(data, klass, name)
|
|
84
|
+
delta, payload = Marshal.load(data) # written by our own cell
|
|
85
|
+
[delta, load(payload, klass, name)]
|
|
86
|
+
rescue StandardError => e
|
|
87
|
+
[nil, crashed(klass, name, "its result could not be loaded: #{e.class}: #{e.message}")]
|
|
88
|
+
end
|
|
89
|
+
|
|
77
90
|
# Marshal.dump, after replacing failures that cannot cross a pipe (an
|
|
78
91
|
# exception holding a Proc, an IO...) with plain copies.
|
|
79
92
|
def dump(result)
|
data/lib/padded_cell/runner.rb
CHANGED
|
@@ -4,8 +4,11 @@ require 'etc'
|
|
|
4
4
|
require 'minitest'
|
|
5
5
|
require_relative '../padded_cell'
|
|
6
6
|
require_relative 'results'
|
|
7
|
+
require_relative 'limits'
|
|
7
8
|
require_relative 'cell'
|
|
9
|
+
require_relative 'worker'
|
|
8
10
|
require_relative 'pool'
|
|
11
|
+
require_relative 'asylum'
|
|
9
12
|
require_relative 'loader'
|
|
10
13
|
require_relative 'timings'
|
|
11
14
|
require_relative 'reporters'
|
|
@@ -18,8 +21,9 @@ module PaddedCell
|
|
|
18
21
|
class Runner
|
|
19
22
|
DEFAULTS = {
|
|
20
23
|
paths: [], jobs: Etc.nprocessors, seed: nil, include: nil, exclude: nil, shard: [0, 1],
|
|
21
|
-
isolation: nil, timeout: nil, fail_fast: false, verbose: false, show_skips: false,
|
|
22
|
-
list: false, junit: nil, json: nil, timings: 'tmp/padded_cell/timings.tsv',
|
|
24
|
+
isolation: nil, timeout: nil, budget: nil, asylum: nil, fail_fast: false, verbose: false, show_skips: false,
|
|
25
|
+
list: false, junit: nil, json: nil, timings: 'tmp/padded_cell/timings.tsv', coverage: nil,
|
|
26
|
+
coverage_dir: nil, io: $stdout
|
|
23
27
|
}.freeze
|
|
24
28
|
|
|
25
29
|
attr_reader :options
|
|
@@ -29,33 +33,54 @@ module PaddedCell
|
|
|
29
33
|
@options[:seed] ||= (ENV['SEED'] || Random.new_seed).to_i % 0xFFFF
|
|
30
34
|
end
|
|
31
35
|
|
|
32
|
-
# Exit status: 0 when everything passed, 1 otherwise,
|
|
36
|
+
# Exit status: 0 when everything passed, 1 otherwise, 3 when only
|
|
37
|
+
# coverage is below a minimum, 130 on Ctrl-C.
|
|
33
38
|
def run
|
|
39
|
+
# The budget covers the whole run, loading included.
|
|
40
|
+
@limits = Limits.new(timeout: options[:timeout], budget: options[:budget])
|
|
34
41
|
selection = Selection.parse(options[:paths])
|
|
35
42
|
refs = Discovery.call(selection.files)
|
|
36
|
-
|
|
43
|
+
filter = Filter.new(selection, refs, include: options[:include], exclude: options[:exclude])
|
|
44
|
+
return list(selection, refs, filter) if options[:list]
|
|
37
45
|
raise Unsupported, 'padded_cell needs fork, which this Ruby does not have' unless Process.respond_to?(:fork)
|
|
38
46
|
|
|
47
|
+
@coverage = start_coverage if options[:coverage]
|
|
39
48
|
Loader.call(selection.files, seed: options[:seed])
|
|
49
|
+
refuse_simplecov if @coverage
|
|
40
50
|
if defined?(ActiveSupport::TestCase)
|
|
41
51
|
require_relative 'active_support_hooks'
|
|
42
52
|
ActiveSupportHooks.install
|
|
43
53
|
end
|
|
44
54
|
options[:isolation] ||= default_isolation
|
|
45
|
-
|
|
55
|
+
# Coverage thresholds hold for the whole suite only.
|
|
56
|
+
@whole = !selection.line_filtered? && !options[:include] && !options[:exclude]
|
|
46
57
|
execute(Loader.tests.select { filter.call(*it) })
|
|
47
58
|
end
|
|
48
59
|
|
|
49
60
|
private
|
|
50
61
|
|
|
62
|
+
# Before the suite loads: nothing loaded earlier is measured.
|
|
63
|
+
def start_coverage
|
|
64
|
+
require_relative 'chart'
|
|
65
|
+
Chart::Accumulator.new(options[:coverage]).start
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Two tools cannot share Ruby's Coverage.
|
|
69
|
+
def refuse_simplecov
|
|
70
|
+
return unless defined?(::SimpleCov) && ::SimpleCov.respond_to?(:pid) && ::SimpleCov.pid
|
|
71
|
+
|
|
72
|
+
raise Unsupported, 'SimpleCov is running, and --coverage replaces it: remove SimpleCov.start (and any ' \
|
|
73
|
+
'SimpleCov.result in parallelize_teardown) from the test helper'
|
|
74
|
+
end
|
|
75
|
+
|
|
51
76
|
# A fork per test discards Active Record's connection pool, and with it
|
|
52
77
|
# the fixtures loaded through it: every test would reload them. With
|
|
53
78
|
# fixtures around, tests share their worker instead.
|
|
54
79
|
def default_isolation = defined?(ActiveRecord::FixtureSet) ? :worker : :test
|
|
55
80
|
|
|
56
81
|
# Tests as they would run (static discovery, no loading), in file order.
|
|
57
|
-
def list(selection, refs)
|
|
58
|
-
refs = refs.select { selection.covers?(it.file, it.lines) }
|
|
82
|
+
def list(selection, refs, filter)
|
|
83
|
+
refs = refs.select { selection.covers?(it.file, it.lines) && filter.matches_name?(it.class_name, it.name) }
|
|
59
84
|
kept = Scheduler.batches(refs.map(&:class_name), seed: options[:seed], shard: options[:shard]).flatten.sort
|
|
60
85
|
kept.each do |index|
|
|
61
86
|
ref = refs[index]
|
|
@@ -69,37 +94,134 @@ module PaddedCell
|
|
|
69
94
|
timings = Timings.load(options[:timings])
|
|
70
95
|
batches = Scheduler.batches(tests.map { |klass, _| klass.to_s }, durations: ids.map { timings[it] },
|
|
71
96
|
seed: options[:seed], shard: options[:shard])
|
|
97
|
+
nothing_ran = batches.empty?
|
|
72
98
|
reporter = build_reporter
|
|
73
99
|
reporter.start
|
|
74
|
-
|
|
100
|
+
retried = []
|
|
101
|
+
@coverage&.snapshot
|
|
102
|
+
status = dispatch(tests, batches, reporter, retried) { |index, result| timings[ids[index]] = result.time }
|
|
75
103
|
reporter.report
|
|
104
|
+
report_asylum(retried)
|
|
105
|
+
covered = @coverage ? report_coverage(status, reporter) : true
|
|
76
106
|
Timings.save(options[:timings], timings)
|
|
77
107
|
after_run
|
|
78
|
-
finish(reporter,
|
|
108
|
+
result = finish(reporter, status, nothing_ran)
|
|
109
|
+
result.zero? && !covered ? 3 : result
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Writes the coverage reports, even for a failing run; false when the
|
|
113
|
+
# coverage of a whole, passing, complete run is below a minimum.
|
|
114
|
+
def report_coverage(status, reporter)
|
|
115
|
+
raw = @coverage.finish(interrupted: status == 130)
|
|
116
|
+
return true unless raw
|
|
117
|
+
|
|
118
|
+
dir = options[:coverage_dir] || Chart.config.directory
|
|
119
|
+
document = Chart::Document.build(raw, config: Chart.config, shard: options[:shard], whole: @whole)
|
|
120
|
+
document.write(dir)
|
|
121
|
+
check = status.zero? && reporter.passed? && document.complete?
|
|
122
|
+
Chart::Report.new(document, dir:, io: options[:io]).call(check:)
|
|
79
123
|
end
|
|
80
124
|
|
|
81
|
-
# Yields each result
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
125
|
+
# Yields each result as it arrives. With --asylum a failure is held back:
|
|
126
|
+
# only its final result, after the retries, is reported. Returns 1 when a
|
|
127
|
+
# worker failed (its cleanup hook) or tests were left out (the budget ran
|
|
128
|
+
# out, workers kept dying), 130 when interrupted, 0 otherwise.
|
|
129
|
+
def dispatch(tests, batches, reporter, retried)
|
|
130
|
+
held = {}
|
|
131
|
+
record = lambda do |index, result|
|
|
85
132
|
reporter.prerecord(*tests[index])
|
|
86
133
|
reporter.record(result)
|
|
87
|
-
|
|
88
|
-
pool&.stop! if options[:fail_fast] && !result.passed? && !result.skipped?
|
|
134
|
+
retried << result if result.metadata? && (result.metadata[:flaky] || result.metadata[:asylum])
|
|
89
135
|
end
|
|
90
136
|
pool = Pool.new(tests, jobs: options[:jobs], isolation: options[:isolation], seed: options[:seed],
|
|
91
|
-
|
|
92
|
-
pool.run(batches,
|
|
93
|
-
|
|
137
|
+
limits: @limits, coverage: @coverage)
|
|
138
|
+
pool.run(batches) do |index, result|
|
|
139
|
+
yield index, result
|
|
140
|
+
failed = failed?(result)
|
|
141
|
+
options[:asylum] && failed ? held[index] = result : record.call(index, result)
|
|
142
|
+
pool.stop! if options[:fail_fast] && failed
|
|
143
|
+
end
|
|
144
|
+
complete = report_left_out(pool)
|
|
145
|
+
retries_ok = retry_held(tests, held, record)
|
|
146
|
+
pool.success? && retries_ok && complete ? 0 : 1
|
|
94
147
|
rescue Interrupt
|
|
95
148
|
warn 'Interrupted. Exiting...'
|
|
96
|
-
|
|
149
|
+
# Failures still waiting for a retry are failures.
|
|
150
|
+
held.each { |index, result| record.call(index, result) }
|
|
151
|
+
130
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# False when tests never ran because the budget ran out or workers kept
|
|
155
|
+
# dying. --fail-fast leaving them out is what it asked for.
|
|
156
|
+
def report_left_out(pool)
|
|
157
|
+
left_out = pool.undispatched
|
|
158
|
+
if pool.gave_up
|
|
159
|
+
warn "padded_cell: #{pool.gave_up}; gave up, #{left_out} tests not run"
|
|
160
|
+
elsif @limits.spent? && left_out.positive?
|
|
161
|
+
warn "padded_cell: run budget of #{@limits.budget}s spent, #{left_out} tests not run"
|
|
162
|
+
else
|
|
163
|
+
return true
|
|
164
|
+
end
|
|
165
|
+
false
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# False when a retry's worker failed.
|
|
169
|
+
def retry_held(tests, held, record)
|
|
170
|
+
return true if held.empty?
|
|
171
|
+
|
|
172
|
+
# No budget left to retry in: failures stay failures.
|
|
173
|
+
if @limits.spent?
|
|
174
|
+
held.each { |index, result| record.call(index, result) }
|
|
175
|
+
return true
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
asylum = Asylum.new(tests, attempts: options[:asylum], seed: options[:seed], limits: @limits)
|
|
179
|
+
asylum.call(held) do |index, result|
|
|
180
|
+
held.delete(index)
|
|
181
|
+
record.call(index, result)
|
|
182
|
+
end
|
|
183
|
+
asylum.success?
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def failed?(result) = !result.passed? && !result.skipped?
|
|
187
|
+
|
|
188
|
+
# After Minitest's summary: what --asylum found, attempt by attempt.
|
|
189
|
+
def report_asylum(retried)
|
|
190
|
+
flaky, failing = retried.partition { it.metadata[:flaky] }
|
|
191
|
+
report_flaky(flaky) if flaky.any?
|
|
192
|
+
report_failing(failing) if failing.any?
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def report_flaky(flaky)
|
|
196
|
+
io = options[:io]
|
|
197
|
+
tests = flaky.size == 1 ? '1 test' : "#{flaky.size} tests"
|
|
198
|
+
io.puts "\nYou're not crazy, the world is. #{tests} failed, then passed:"
|
|
199
|
+
width = flaky.flat_map { it.metadata[:flaky][:history] }.map { it[:seed].to_s.size }.max
|
|
200
|
+
flaky.each do |result|
|
|
201
|
+
info = result.metadata[:flaky]
|
|
202
|
+
io.puts "\n #{result.klass}##{result.name}"
|
|
203
|
+
info[:history].each do |attempt|
|
|
204
|
+
io.puts " seed #{attempt[:seed].to_s.ljust(width)} #{attempt_line(attempt, info[:cause])}"
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def attempt_line(attempt, cause)
|
|
210
|
+
return "passed, alone → #{cause}" if attempt[:outcome] == 'passed'
|
|
211
|
+
|
|
212
|
+
attempt[:alone] ? "#{attempt[:outcome]} (alone)" : attempt[:outcome]
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def report_failing(failing)
|
|
216
|
+
io = options[:io]
|
|
217
|
+
io.puts "\nIt's not the world, it's the test:"
|
|
218
|
+
failing.each { io.puts " #{it.klass}##{it.name} failed all #{it.metadata[:asylum][:attempts]} attempts" }
|
|
97
219
|
end
|
|
98
220
|
|
|
99
221
|
def build_reporter
|
|
100
222
|
reporter = Minitest::CompositeReporter.new
|
|
101
223
|
minitest_options = {
|
|
102
|
-
io: options[:io], seed: options[:seed], args:
|
|
224
|
+
io: options[:io], seed: options[:seed], args: run_options,
|
|
103
225
|
verbose: options[:verbose], show_skips: options[:show_skips]
|
|
104
226
|
}
|
|
105
227
|
reporter << Minitest::SummaryReporter.new(options[:io], minitest_options)
|
|
@@ -113,13 +235,19 @@ module PaddedCell
|
|
|
113
235
|
reporter
|
|
114
236
|
end
|
|
115
237
|
|
|
238
|
+
# What a rerun needs to reproduce this run's order and settings.
|
|
239
|
+
def run_options
|
|
240
|
+
args = "--seed #{options[:seed]} --isolation #{options[:isolation]}"
|
|
241
|
+
options[:asylum] ? "#{args} --asylum=#{options[:asylum]}" : args
|
|
242
|
+
end
|
|
243
|
+
|
|
116
244
|
# Minitest.after_run blocks, which autorun would have called.
|
|
117
245
|
def after_run
|
|
118
246
|
Minitest.class_variable_get(:@@after_run).reverse_each(&:call)
|
|
119
247
|
end
|
|
120
248
|
|
|
121
|
-
def finish(reporter,
|
|
122
|
-
return
|
|
249
|
+
def finish(reporter, status, nothing_ran)
|
|
250
|
+
return status unless status.zero?
|
|
123
251
|
if nothing_ran && options[:include]
|
|
124
252
|
warn "Nothing ran for filter: #{options[:include]}"
|
|
125
253
|
return 1
|
data/lib/padded_cell/version.rb
CHANGED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PaddedCell
|
|
4
|
+
# A Pool's worker process. Reads test indices from its socket, runs each
|
|
5
|
+
# (in a Cell, or itself with isolation :worker) and writes back its
|
|
6
|
+
# Minitest::Result as a length-prefixed Marshal frame. Exits once the
|
|
7
|
+
# socket closes.
|
|
8
|
+
class Worker
|
|
9
|
+
# limits: the run's Limits; probe: a Chart::Probe when measuring coverage
|
|
10
|
+
def initialize(tests, isolation:, seed:, limits:, probe:)
|
|
11
|
+
@tests = tests
|
|
12
|
+
@isolation = isolation
|
|
13
|
+
@seed = seed
|
|
14
|
+
@limits = limits
|
|
15
|
+
@probe = probe
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Never returns.
|
|
19
|
+
def run(worker, socket)
|
|
20
|
+
status = 1
|
|
21
|
+
begin
|
|
22
|
+
srand(@seed)
|
|
23
|
+
@probe&.start
|
|
24
|
+
setup_error = after_fork(worker)
|
|
25
|
+
while (line = socket.gets)
|
|
26
|
+
payload = Results.dump(perform(Integer(line), socket, setup_error))
|
|
27
|
+
socket.write([payload.bytesize].pack('N'), payload)
|
|
28
|
+
@probe&.tick
|
|
29
|
+
end
|
|
30
|
+
# Before cleanup, which may stop Coverage.
|
|
31
|
+
@probe&.flush
|
|
32
|
+
Hooks.run_cleanup(worker) unless setup_error
|
|
33
|
+
status = 0
|
|
34
|
+
rescue Exception => e # anything: report it, then exit!
|
|
35
|
+
warn "padded_cell: worker #{worker} failed: #{e.class}: #{e.message}"
|
|
36
|
+
ensure
|
|
37
|
+
exit!(status)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
# A failing after_fork hook fails every test of the worker, the way Rails
|
|
44
|
+
# parallel testing reports a broken parallelize_setup.
|
|
45
|
+
def after_fork(worker)
|
|
46
|
+
Hooks.run_after_fork(worker)
|
|
47
|
+
nil
|
|
48
|
+
rescue StandardError => e
|
|
49
|
+
e
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def perform(index, socket, setup_error)
|
|
53
|
+
klass, name = @tests.fetch(index)
|
|
54
|
+
if setup_error
|
|
55
|
+
Results.crashed(klass, name, "after_fork hook failed: #{setup_error.class}: #{setup_error.message}")
|
|
56
|
+
elsif @isolation == :test
|
|
57
|
+
Cell.run(klass, name, seed: @seed, limits: @limits, close: [socket], probe: @probe)
|
|
58
|
+
else
|
|
59
|
+
Cell.execute(klass, name)
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
data/lib/padded_cell.rb
CHANGED
|
@@ -23,6 +23,14 @@ module PaddedCell
|
|
|
23
23
|
def after_fork(&) = Hooks.after_fork(&)
|
|
24
24
|
|
|
25
25
|
def cleanup(&) = Hooks.cleanup(&)
|
|
26
|
+
|
|
27
|
+
# Configures the coverage report (see Chart::Config). Inert unless
|
|
28
|
+
# padded_cell runs with --coverage. A helper also run without
|
|
29
|
+
# padded_cell guards it: `if defined?(PaddedCell.coverage)`.
|
|
30
|
+
def coverage(profile = nil, &)
|
|
31
|
+
require_relative 'padded_cell/chart'
|
|
32
|
+
Chart.configure(profile, &)
|
|
33
|
+
end
|
|
26
34
|
end
|
|
27
35
|
end
|
|
28
36
|
|