padded_cell 0.1.0-aarch64-linux-gnu → 0.3.0-aarch64-linux-gnu
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +38 -0
- data/README.md +170 -8
- data/lib/padded_cell/4.0/padded_cell_native.so +0 -0
- data/lib/padded_cell/asylum.rb +93 -0
- data/lib/padded_cell/cell.rb +23 -14
- data/lib/padded_cell/chart/accumulator.rb +103 -0
- data/lib/padded_cell/chart/config.rb +129 -0
- data/lib/padded_cell/chart/delta.rb +148 -0
- data/lib/padded_cell/chart/document.rb +64 -0
- data/lib/padded_cell/chart/filter.rb +55 -0
- data/lib/padded_cell/chart/merge.rb +63 -0
- data/lib/padded_cell/chart/probe.rb +70 -0
- data/lib/padded_cell/chart/raw.rb +48 -0
- data/lib/padded_cell/chart/report.rb +128 -0
- data/lib/padded_cell/chart/source_files.rb +90 -0
- data/lib/padded_cell/chart.rb +38 -0
- data/lib/padded_cell/cli.rb +70 -4
- data/lib/padded_cell/limits.rb +45 -0
- data/lib/padded_cell/loader.rb +5 -1
- data/lib/padded_cell/pool.rb +82 -58
- data/lib/padded_cell/reporters.rb +20 -3
- data/lib/padded_cell/results.rb +13 -0
- data/lib/padded_cell/runner.rb +150 -22
- data/lib/padded_cell/version.rb +1 -1
- data/lib/padded_cell/worker.rb +63 -0
- data/lib/padded_cell.rb +8 -0
- metadata +15 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8989d567fd19a8fd3d7713dc0c008b0f7c7c58567b837d69b39f2d1ac7f8db69
|
|
4
|
+
data.tar.gz: 98254fdb4722ef9a0fcc50c302c3f2496e8ffcc75eb92330844bc3ec6aabfba2
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1031e64b53b8d5c156057513bea76d2585a4d05fcb6f73edc672f598f32331ff630189bf93dd8683cf83b66594b91c4346a1915539fe1565d843a505287e3f72
|
|
7
|
+
data.tar.gz: 44194137c0688850c83d563b153becb8ac3aef01d1d2b38648e5a54064ae203e3cac5bf159ad8a46e9e0a7a44989d70b8accf67a0ec0d590143ff6b1398b8861
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,43 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.3.0 (2026-11-08)
|
|
4
|
+
|
|
5
|
+
- `--budget SECONDS`: a deadline for the whole run, loading included. Once
|
|
6
|
+
it's spent, nothing more is dispatched and running tests are killed. Tests
|
|
7
|
+
it left out fail the run (exit 1). `--timeout` never outlives it.
|
|
8
|
+
- Workers that keep dying (more than 4 per worker, at least 16, in 10s) are
|
|
9
|
+
no longer replaced forever. padded_cell gives up, reports how many tests
|
|
10
|
+
never ran, and exits 1. Uses `supervisor-machines`.
|
|
11
|
+
- A replacement worker that died before its first test crashed the run with
|
|
12
|
+
`Errno::EPIPE` (exit 2). A worker reset mid-frame is now reported as dead.
|
|
13
|
+
- `--asylum` retry seeds keep all 64 bits: they were folded modulo 0xffff,
|
|
14
|
+
so large `--seed`s shared the same few retry seeds.
|
|
15
|
+
|
|
16
|
+
## 0.2.0 (2026-10-11)
|
|
17
|
+
|
|
18
|
+
- Built-in coverage: `--coverage[=lines,branches,methods,eval]` (or
|
|
19
|
+
`PADDED_CELL_COVERAGE`) measures the runner, every worker and every cell,
|
|
20
|
+
and sums them exactly. Configured with `PaddedCell.coverage` in the test
|
|
21
|
+
helper: tracked files, filters, groups, the `:rails` profile. Honors
|
|
22
|
+
`# :nocov:` blocks and `# simplecov:disable` comments. Writes
|
|
23
|
+
`coverage.md` (the code that never ran, for agents, reviews and CI
|
|
24
|
+
summaries), `lcov.info`, and `coverage.json` / `.resultset.json`.
|
|
25
|
+
`minimum`, `minimum_by_file`, `minimum_by_group` and `maximum_drop` fail a
|
|
26
|
+
whole passing run with exit 3. `--coverage-merge` merges shards' reports.
|
|
27
|
+
- `--seed` must be 0 to 2**64 - 1; others crashed the planner.
|
|
28
|
+
- `--asylum[=N]`: retry failed tests (N times, default 2) after the run,
|
|
29
|
+
alone and in fresh cells. Tests that pass on retry count as passed and are
|
|
30
|
+
reported as flaky, with the likely cause (timing and shared state, or
|
|
31
|
+
randomness and order) and their first failure; JUnit gets
|
|
32
|
+
`<flakyFailure>`, JSON a `flaky` field.
|
|
33
|
+
- A `cleanup` hook that raises fails the run (exit 1), `--asylum` retries
|
|
34
|
+
included. It exited 0 when the tests passed.
|
|
35
|
+
- A passing run filtered with `--name` exits 0: it exited 1, reporting that
|
|
36
|
+
nothing ran.
|
|
37
|
+
- `--list` applies `--name` and `--exclude`, before `--shard`, as a run does.
|
|
38
|
+
- An invalid `--name` or `--exclude` regexp is a usage error (exit 2), not a
|
|
39
|
+
crash.
|
|
40
|
+
|
|
3
41
|
## 0.1.0 (2026-10-11)
|
|
4
42
|
|
|
5
43
|
First release, for CRuby 4.0+ and Minitest 6 on Linux, macOS and FreeBSD: a
|
data/README.md
CHANGED
|
@@ -69,14 +69,18 @@ $ padded_cell user_test.rb:12-30:41 # tests overlapping 12..30, or spanni
|
|
|
69
69
|
$ padded_cell -n /refund/ -e test_slow # Minitest's --include / --exclude
|
|
70
70
|
$ padded_cell -j 8 --seed 1234 # 8 workers, reproducible order
|
|
71
71
|
$ padded_cell --timeout 30 # kill and fail anything running 30s
|
|
72
|
+
$ padded_cell --budget 600 # the whole run gets 10 minutes; what's left out fails it
|
|
72
73
|
$ padded_cell --fail-fast
|
|
74
|
+
$ padded_cell --asylum # retry failures; report the flaky ones (see below)
|
|
73
75
|
$ padded_cell --list # what would run, without loading anything
|
|
74
76
|
$ padded_cell --shard 2/4 # CI: the second quarter of the classes
|
|
75
77
|
$ padded_cell --junit tmp/junit.xml --json tmp/results.jsonl
|
|
78
|
+
$ PADDED_CELL_JSON=tmp/results.jsonl padded_cell # same, from a CI runner's env
|
|
79
|
+
$ padded_cell --coverage # line and branch coverage (see Coverage)
|
|
76
80
|
```
|
|
77
81
|
|
|
78
82
|
Exit status: 0 when everything passed, 1 otherwise, 2 for usage or load
|
|
79
|
-
errors, 130 when interrupted.
|
|
83
|
+
errors, 3 when only coverage is below a threshold, 130 when interrupted.
|
|
80
84
|
|
|
81
85
|
### How a run works
|
|
82
86
|
|
|
@@ -108,11 +112,60 @@ cheap, and order-dependent tests stop depending on order.
|
|
|
108
112
|
Rails' `parallelize`. A worker that crashes or times out is still reported
|
|
109
113
|
and replaced.
|
|
110
114
|
|
|
115
|
+
Replaced up to a point. If workers keep dying, say a boot that segfaults or an
|
|
116
|
+
`after_fork` hook that calls `exit`, every remaining test would come back as a
|
|
117
|
+
crash report, one fork at a time. So past 4 deaths per worker in 10 seconds
|
|
118
|
+
(16 at least), padded_cell gives up: it lists how many tests never ran and
|
|
119
|
+
exits 1. The decision comes from
|
|
120
|
+
[supervisor_machines](https://github.com/seuros/supervisor_machines), the same
|
|
121
|
+
one-for-one restart intensity OTP uses.
|
|
122
|
+
|
|
123
|
+
### Budget
|
|
124
|
+
|
|
125
|
+
`--budget SECONDS` caps the whole run, loading included. Once it's spent,
|
|
126
|
+
nothing more is dispatched, and a test still running is killed and fails
|
|
127
|
+
("run budget of 600.0s spent"). A test's `--timeout` never outlives the
|
|
128
|
+
budget. If the budget left tests out, the run fails and says how many. The
|
|
129
|
+
deadline math comes from
|
|
130
|
+
[deadline_machines](https://github.com/seuros/deadline_machines).
|
|
131
|
+
|
|
111
132
|
The default is `test`, except when Active Record fixtures are loaded: a fork
|
|
112
133
|
discards Active Record's connection pool, and every test would reload its
|
|
113
134
|
fixtures, so those suites default to `worker`. The `Run options:` line shows
|
|
114
135
|
which one a run uses; `--isolation` overrides it.
|
|
115
136
|
|
|
137
|
+
### Flaky tests
|
|
138
|
+
|
|
139
|
+
`--asylum[=N]` gives failed tests up to N more attempts (default 2) once the
|
|
140
|
+
run is over: one test at a time, each in a fresh cell. A test that passes on
|
|
141
|
+
retry counts as passed, but padded_cell shows every attempt, its seed, and
|
|
142
|
+
what made the difference:
|
|
143
|
+
|
|
144
|
+
```console
|
|
145
|
+
4 runs, 4 assertions, 1 failures, 0 errors, 0 skips
|
|
146
|
+
|
|
147
|
+
You're not crazy, the world is. 2 tests failed, then passed:
|
|
148
|
+
|
|
149
|
+
FlakyTest#test_passes_when_retried
|
|
150
|
+
seed 1 lost the race
|
|
151
|
+
seed 1 passed, alone → timing, contention or state left by other tests
|
|
152
|
+
|
|
153
|
+
FlakyTest#test_depends_on_the_seed
|
|
154
|
+
seed 1 unlucky seed.
|
|
155
|
+
seed 1 unlucky seed. (alone)
|
|
156
|
+
seed 15472 passed, alone → randomness or test order
|
|
157
|
+
|
|
158
|
+
It's not the world, it's the test:
|
|
159
|
+
FlakyTest#test_always_fails failed all 3 attempts
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
The first retry keeps the run's seed and runs alone, so passing there points
|
|
163
|
+
at timing, contention or leftover state; later retries change the seed, so
|
|
164
|
+
passing only there points at randomness or order. A test that fails every
|
|
165
|
+
attempt keeps its first failure, and the run fails. JUnit reports carry a
|
|
166
|
+
`<flakyFailure>` per flaky test; JSON lines carry the attempts in a `flaky`
|
|
167
|
+
field, or an `asylum` field for a test that never passed.
|
|
168
|
+
|
|
116
169
|
### Hooks
|
|
117
170
|
|
|
118
171
|
For anything that must not be shared across forks:
|
|
@@ -125,7 +178,115 @@ PaddedCell.cleanup { |worker| Database.drop("app_test_#{worker}") }
|
|
|
125
178
|
```
|
|
126
179
|
|
|
127
180
|
`before_fork` runs in the runner, `after_fork` and `cleanup` once per worker
|
|
128
|
-
(`0...jobs`). Cells run no hooks: they inherit their worker's state.
|
|
181
|
+
(`0...jobs`). Cells run no hooks: they inherit their worker's state. A
|
|
182
|
+
failing `after_fork` fails every test of its worker; a failing `cleanup`
|
|
183
|
+
fails the run, even when every test passed.
|
|
184
|
+
|
|
185
|
+
## Coverage
|
|
186
|
+
|
|
187
|
+
Ruby's `Coverage`, measured in the runner, in every worker and in every
|
|
188
|
+
cell, and summed exactly: a parallel run reports the counts of a serial one.
|
|
189
|
+
|
|
190
|
+
```console
|
|
191
|
+
$ padded_cell --coverage test/ # lines and branches
|
|
192
|
+
$ padded_cell --coverage=lines,branches,methods test/
|
|
193
|
+
$ PADDED_CELL_COVERAGE=1 bin/rails padded_cell
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
Measuring starts before the suite loads, so what is measured (`lines`,
|
|
197
|
+
`branches`, `methods`, and `eval` for templates, experimental) comes from the
|
|
198
|
+
command line. The report is configured in the test helper:
|
|
199
|
+
|
|
200
|
+
```ruby
|
|
201
|
+
# test/test_helper.rb
|
|
202
|
+
PaddedCell.coverage(:rails) do
|
|
203
|
+
track "{app,lib}/**/*.rb" # report files never loaded, at 0%
|
|
204
|
+
skip "app/legacy/" # also `filter`
|
|
205
|
+
group "Operations", "app/concepts" # string, regexp, array or block
|
|
206
|
+
group("Contracts") { |file| file.filename.include?("contract") }
|
|
207
|
+
minimum line: 90, branch: 80
|
|
208
|
+
minimum_by_file line: 50
|
|
209
|
+
minimum_by_group "Operations", line: 95
|
|
210
|
+
maximum_drop line: 1 # since the previous whole run
|
|
211
|
+
formats :markdown, :lcov, :json
|
|
212
|
+
dir "coverage" # --coverage-dir wins
|
|
213
|
+
end
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
- Paths are matched relative to the project root. A string matches whole
|
|
217
|
+
path segments (`"app/models"` takes `app/models/user.rb`, not
|
|
218
|
+
`app/models_old/`), `"lib/"` a directory anywhere, `"contract.rb"` a file
|
|
219
|
+
name ending; a regexp matches the path; a block gets the file
|
|
220
|
+
(`filename`, `project_filename`, `src`).
|
|
221
|
+
- Left out by default: `vendor/bundle`, hidden files, `test/`, `spec/`,
|
|
222
|
+
`features/`. `clear_filters` drops these.
|
|
223
|
+
- `:rails` skips `config/` and `db/`, groups `app/channels`,
|
|
224
|
+
`app/controllers`, `app/helpers`, `app/jobs` and `app/workers`, `lib/`,
|
|
225
|
+
`app/mailers`, `app/models` and `app/views`, and tracks
|
|
226
|
+
`{app,lib}/**/*.rb`. A file in no group is Ungrouped.
|
|
227
|
+
- Thresholds compare percents floored to 2 decimals.
|
|
228
|
+
|
|
229
|
+
`PaddedCell.coverage` configures only; without `--coverage` nothing is
|
|
230
|
+
measured. Outside padded_cell, guard it with `if defined?(PaddedCell.coverage)`.
|
|
231
|
+
|
|
232
|
+
Into `coverage/` go, by format:
|
|
233
|
+
|
|
234
|
+
- `markdown`: `coverage.md`, for agents, reviews and CI summaries
|
|
235
|
+
(`cat coverage/coverage.md >> $GITHUB_STEP_SUMMARY`): the totals and
|
|
236
|
+
groups, then every file with code that never ran, the most missed first,
|
|
237
|
+
with its missed lines, branches and methods, and that code. It stops
|
|
238
|
+
listing files at 1 MB, the size of a GitHub job summary.
|
|
239
|
+
- `lcov`: `lcov.info` (`SF:./path`), for CI and editors.
|
|
240
|
+
- `json`: `coverage.json` and `.resultset.json`, in SimpleCov's formats, for
|
|
241
|
+
the tools reading them.
|
|
242
|
+
|
|
243
|
+
Also, every run: `.padded_cell.json` (what the report is made of, and what
|
|
244
|
+
`--coverage-merge` reads), and after a whole passing run `.last_run.json`.
|
|
245
|
+
|
|
246
|
+
Lines and branches inside `# :nocov:` blocks, and under
|
|
247
|
+
`# simplecov:disable [line|branch|method]` / `# simplecov:enable` comments,
|
|
248
|
+
are skipped. A directive after code skips its own line.
|
|
249
|
+
|
|
250
|
+
A run below a threshold exits 3. Thresholds hold for a whole, passing,
|
|
251
|
+
complete run only: they are not checked when tests failed, when tests were
|
|
252
|
+
picked by name or line, under `--shard`, or when a worker or cell died
|
|
253
|
+
before handing over what it measured (the report then says it is partial).
|
|
254
|
+
Reports are written either way, failing runs included.
|
|
255
|
+
|
|
256
|
+
A suite that starts SimpleCov as well is refused (exit 2): two tools cannot
|
|
257
|
+
share Ruby's `Coverage`.
|
|
258
|
+
|
|
259
|
+
### Shards
|
|
260
|
+
|
|
261
|
+
Each shard reports on its own; merge them, at the same commit, without
|
|
262
|
+
running anything:
|
|
263
|
+
|
|
264
|
+
```console
|
|
265
|
+
$ padded_cell --coverage --coverage-dir coverage/1 --shard 1/2 test/ # machine 1
|
|
266
|
+
$ padded_cell --coverage --coverage-dir coverage/2 --shard 2/2 test/ # machine 2
|
|
267
|
+
$ padded_cell --coverage-merge coverage/1,coverage/2 --coverage-dir coverage
|
|
268
|
+
```
|
|
269
|
+
|
|
270
|
+
The merge refuses files that differ between shards or from the checkout,
|
|
271
|
+
and checks thresholds when every shard of the run is there. Lines that run
|
|
272
|
+
while the suite loads count once per shard.
|
|
273
|
+
|
|
274
|
+
### Cost
|
|
275
|
+
|
|
276
|
+
Measuring is opt-in: Ruby's instrumentation slows tests down. On a Rails
|
|
277
|
+
suite of 6,300 tests, lines and branches took the run from 465 s to 638 s of
|
|
278
|
+
CPU.
|
|
279
|
+
|
|
280
|
+
A worker hands over what it measured every 10 seconds or more: each time,
|
|
281
|
+
Ruby copies the coverage of every file it loaded, gems included (a few
|
|
282
|
+
milliseconds in a small app, a few hundred in a big Rails one), so the
|
|
283
|
+
interval stretches to keep this under 1% of the worker's time. A worker that
|
|
284
|
+
dies loses what it measured since. With `--isolation test` every cell pays
|
|
285
|
+
one such copy, at its end.
|
|
286
|
+
|
|
287
|
+
Files never loaded report their branches in the order Ruby compiles them.
|
|
288
|
+
With `eval`, code that `class_eval` runs from a heredoc counts as lines of
|
|
289
|
+
its file; directives in templates are not read.
|
|
129
290
|
|
|
130
291
|
## Rails
|
|
131
292
|
|
|
@@ -144,8 +305,7 @@ $ bin/rails padded_cell ARGS="-j 8 test/models"
|
|
|
144
305
|
Rails suites with fixtures run with `--isolation worker` unless told
|
|
145
306
|
otherwise (see [Isolation](#isolation)).
|
|
146
307
|
|
|
147
|
-
|
|
148
|
-
see only the runner, as with Rails' process parallelization.
|
|
308
|
+
For coverage, see [Coverage](#coverage).
|
|
149
309
|
|
|
150
310
|
## Library
|
|
151
311
|
|
|
@@ -164,7 +324,9 @@ PaddedCell::Scheduler.batches(class_names, durations: seconds, seed: 42, shard:
|
|
|
164
324
|
## Rust
|
|
165
325
|
|
|
166
326
|
The extension lives in `ext/padded_cell_native`: `core` does discovery and
|
|
167
|
-
scheduling in plain Rust (no Ruby types), `ffi` binds it to Ruby.
|
|
327
|
+
scheduling in plain Rust (no Ruby types), `ffi` binds it to Ruby. Worker
|
|
328
|
+
supervision and deadlines come from the published `supervisor-machines` and
|
|
329
|
+
`deadline-machines` crates. Neither is
|
|
168
330
|
published as a crate.
|
|
169
331
|
|
|
170
332
|
## Development
|
|
@@ -173,9 +335,9 @@ Needs Rust 1.99+ (`mise install` reads `mise.toml`).
|
|
|
173
335
|
|
|
174
336
|
```console
|
|
175
337
|
bundle install
|
|
176
|
-
rake test:all # cargo test, then
|
|
177
|
-
rake
|
|
178
|
-
rake
|
|
338
|
+
rake test:all # cargo test, then this suite run by padded_cell itself
|
|
339
|
+
rake dogfood # this suite, run by padded_cell (compiles the extension first)
|
|
340
|
+
rake test # this suite on plain Minitest, for when padded_cell itself is broken
|
|
179
341
|
rake lint # dictator, cargo fmt --check, clippy
|
|
180
342
|
```
|
|
181
343
|
|
|
Binary file
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module PaddedCell
|
|
4
|
+
# --asylum: tests that failed get more attempts once the run is over, one
|
|
5
|
+
# at a time, each in a fresh cell. A test that passes on retry counts as
|
|
6
|
+
# passed but is reported as flaky, with every attempt and what changed:
|
|
7
|
+
#
|
|
8
|
+
# attempt 1, same seed, run alone: timing, contention, or state left by
|
|
9
|
+
# the tests that ran around it
|
|
10
|
+
# attempt 2+, another seed: randomness or test order
|
|
11
|
+
#
|
|
12
|
+
# A test that fails every attempt keeps its first failure.
|
|
13
|
+
class Asylum
|
|
14
|
+
ALONE = 'timing, contention or state left by other tests'
|
|
15
|
+
RESEEDED = 'randomness or test order'
|
|
16
|
+
SEEDS = 2**64
|
|
17
|
+
|
|
18
|
+
# The seed of retry attempt, derived from the run's seed so a report can
|
|
19
|
+
# be replayed. Attempt 1 keeps the run's seed. Wraps within the planner's
|
|
20
|
+
# 64-bit seeds, as --seed accepts them.
|
|
21
|
+
def self.retry_seed(seed, attempt)
|
|
22
|
+
attempt == 1 ? seed : (seed + (attempt * 0x9e37)) % SEEDS
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def initialize(tests, attempts:, seed:, limits: Limits.new)
|
|
26
|
+
@tests = tests
|
|
27
|
+
@attempts = attempts
|
|
28
|
+
@seed = seed
|
|
29
|
+
@limits = limits
|
|
30
|
+
@success = true
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# False when a retry's worker failed (its cleanup hook).
|
|
34
|
+
def success? = @success
|
|
35
|
+
|
|
36
|
+
# failures: {test index => first failing result}. Yields (index, result)
|
|
37
|
+
# once per test, with its final result. Its metadata holds the attempts:
|
|
38
|
+
# :flaky when it passed on retry, :asylum when it never did. Both are
|
|
39
|
+
# plain data, for Marshal and the JUnit and JSON reports.
|
|
40
|
+
def call(failures)
|
|
41
|
+
history = failures.to_h { |index, result| [index, [attempt_entry(@seed, false, result)]] }
|
|
42
|
+
pending = failures.keys
|
|
43
|
+
1.upto(@attempts) do |attempt|
|
|
44
|
+
break if pending.empty?
|
|
45
|
+
|
|
46
|
+
pending -= retry_round(pending, history, attempt) do |index, result|
|
|
47
|
+
result.metadata[:flaky] = flaky(failures[index], history[index], attempt)
|
|
48
|
+
yield index, result
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
pending.each do |index|
|
|
52
|
+
failures[index].metadata[:asylum] = { attempts: history[index].size, history: history[index] }
|
|
53
|
+
yield index, failures[index]
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
# Runs the pending tests once more, alone; returns the ones that passed.
|
|
60
|
+
def retry_round(pending, history, attempt)
|
|
61
|
+
seed = self.class.retry_seed(@seed, attempt)
|
|
62
|
+
passed = []
|
|
63
|
+
pool = Pool.new(@tests, jobs: 1, isolation: :test, seed:, limits: @limits)
|
|
64
|
+
pool.run([pending.dup]) do |index, result|
|
|
65
|
+
recovered = result.passed? && !result.skipped?
|
|
66
|
+
history[index] << attempt_entry(seed, true, recovered ? nil : result)
|
|
67
|
+
next unless recovered
|
|
68
|
+
|
|
69
|
+
passed << index
|
|
70
|
+
yield index, result
|
|
71
|
+
end
|
|
72
|
+
@success &&= pool.success?
|
|
73
|
+
passed
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# One attempt: its seed, whether it ran alone, and how it ended.
|
|
77
|
+
def attempt_entry(seed, alone, failed)
|
|
78
|
+
{ seed:, alone:, outcome: failed ? summary(failed.failure)[:message] : 'passed' }
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def flaky(first, history, attempt)
|
|
82
|
+
{
|
|
83
|
+
attempts: history.size, seed: history.last[:seed], cause: attempt == 1 ? ALONE : RESEEDED,
|
|
84
|
+
first_failure: summary(first.failure), history:
|
|
85
|
+
}
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def summary(failure)
|
|
89
|
+
type = failure.is_a?(Minitest::UnexpectedError) ? failure.error.class.name : failure.class.name
|
|
90
|
+
{ type:, message: failure.message.lines.first.to_s.strip }
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
data/lib/padded_cell/cell.rb
CHANGED
|
@@ -9,18 +9,23 @@ module PaddedCell
|
|
|
9
9
|
module_function
|
|
10
10
|
|
|
11
11
|
# Runs klass#name in a fork and returns its Minitest::Result, or an error
|
|
12
|
-
# result when the cell died or ran past
|
|
12
|
+
# result when the cell died or ran past its deadline.
|
|
13
13
|
#
|
|
14
|
-
#
|
|
15
|
-
|
|
14
|
+
# limits: the run's Limits (--timeout, --budget); nil: no deadline
|
|
15
|
+
# close: descriptors the cell must not keep open (the worker's socket)
|
|
16
|
+
# probe: the worker's Chart::Probe, which gets the cell's coverage
|
|
17
|
+
def run(klass, name, seed:, limits: nil, close: [], probe: nil)
|
|
16
18
|
# Binary: the result is Marshal data, and with Encoding.default_internal
|
|
17
19
|
# set (Rails sets it) a text-mode pipe transcodes what is written to it.
|
|
18
20
|
reader, writer = IO.pipe
|
|
19
21
|
[reader, writer].each(&:binmode)
|
|
20
22
|
started = Minitest.clock_time
|
|
21
|
-
|
|
23
|
+
deadline = limits&.for_test
|
|
24
|
+
pid = fork_cell(klass, name, seed, reader, writer, close, probe)
|
|
22
25
|
writer.close
|
|
23
|
-
collect(pid, reader, klass, name, started,
|
|
26
|
+
delta, result = collect(pid, reader, klass, name, started, deadline) { limits.message(deadline) }
|
|
27
|
+
delta ? probe.absorb(delta) : probe&.lost
|
|
28
|
+
result
|
|
24
29
|
ensure
|
|
25
30
|
reader&.close
|
|
26
31
|
writer&.close unless writer&.closed?
|
|
@@ -34,7 +39,7 @@ module PaddedCell
|
|
|
34
39
|
Results.crashed(klass, name, "exit(#{e.status}) called during the test")
|
|
35
40
|
end
|
|
36
41
|
|
|
37
|
-
def fork_cell(klass, name, seed, reader, writer, close)
|
|
42
|
+
def fork_cell(klass, name, seed, reader, writer, close, probe)
|
|
38
43
|
$stdout.flush
|
|
39
44
|
$stderr.flush
|
|
40
45
|
Process.fork do
|
|
@@ -43,7 +48,9 @@ module PaddedCell
|
|
|
43
48
|
reader.close
|
|
44
49
|
close.each(&:close)
|
|
45
50
|
srand(seed)
|
|
46
|
-
|
|
51
|
+
probe&.start
|
|
52
|
+
result = execute(klass, name)
|
|
53
|
+
writer.write(Results.envelope(probe&.capture, result))
|
|
47
54
|
status = 0
|
|
48
55
|
rescue Exception => e # anything: report it, then exit!
|
|
49
56
|
warn "padded_cell: cell #{klass}##{name} failed to report: #{e.class}: #{e.message}"
|
|
@@ -54,26 +61,28 @@ module PaddedCell
|
|
|
54
61
|
end
|
|
55
62
|
end
|
|
56
63
|
|
|
57
|
-
|
|
58
|
-
|
|
64
|
+
# [coverage delta or nil, result]. Yields for the message of a cell
|
|
65
|
+
# killed at its deadline.
|
|
66
|
+
def collect(pid, reader, klass, name, started, deadline)
|
|
67
|
+
data = read(reader, deadline)
|
|
59
68
|
elapsed = Minitest.clock_time - started
|
|
60
69
|
if data.nil?
|
|
61
70
|
Process.kill(:KILL, pid)
|
|
62
71
|
Process.wait(pid)
|
|
63
|
-
return Results.crashed(klass, name,
|
|
72
|
+
return [nil, Results.crashed(klass, name, yield, time: elapsed)]
|
|
64
73
|
end
|
|
65
74
|
|
|
66
75
|
_, status = Process.wait2(pid)
|
|
67
|
-
return Results.
|
|
76
|
+
return Results.unwrap(data, klass, name) unless data.empty?
|
|
68
77
|
|
|
69
|
-
Results.crashed(klass, name, Results.describe(status, 'cell'), time: elapsed)
|
|
78
|
+
[nil, Results.crashed(klass, name, Results.describe(status, 'cell'), time: elapsed)]
|
|
70
79
|
end
|
|
71
80
|
|
|
72
|
-
# Everything until EOF, or nil when the deadline passes first.
|
|
81
|
+
# Everything until EOF, or nil when the deadline (if any) passes first.
|
|
73
82
|
def read(io, deadline)
|
|
74
83
|
data = String.new(encoding: Encoding::BINARY)
|
|
75
84
|
loop do
|
|
76
|
-
remaining = deadline && (deadline
|
|
85
|
+
remaining = deadline && Limits.remaining(deadline)
|
|
77
86
|
return if remaining&.<= 0
|
|
78
87
|
next unless io.wait_readable(remaining)
|
|
79
88
|
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'coverage'
|
|
4
|
+
require 'fileutils'
|
|
5
|
+
require 'tmpdir'
|
|
6
|
+
|
|
7
|
+
module PaddedCell
|
|
8
|
+
module Chart
|
|
9
|
+
# The runner's side. Starts Ruby's Coverage before the suite loads, hands
|
|
10
|
+
# each worker a Probe, then sums its own measure (loading the suite) with
|
|
11
|
+
# every frame the workers wrote.
|
|
12
|
+
class Accumulator
|
|
13
|
+
attr_reader :criteria
|
|
14
|
+
|
|
15
|
+
# criteria: what Ruby's Coverage measures, :lines always among them
|
|
16
|
+
def initialize(criteria)
|
|
17
|
+
@criteria = criteria
|
|
18
|
+
@root = File.realpath(Dir.pwd)
|
|
19
|
+
@roots = [Dir.pwd, @root].uniq.map { "#{it}/" }
|
|
20
|
+
@kept = {}
|
|
21
|
+
@lost = 0
|
|
22
|
+
@known = Set.new
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Nothing loaded before is measured.
|
|
26
|
+
def start
|
|
27
|
+
::Coverage.setup(**@criteria.to_h { [it, true] })
|
|
28
|
+
::Coverage.resume
|
|
29
|
+
@frames = Dir.mktmpdir('padded_cell-coverage')
|
|
30
|
+
self
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# After the suite loaded, before the first fork: the files whose
|
|
34
|
+
# structure the runner will report itself. Forks inherit the set.
|
|
35
|
+
def snapshot
|
|
36
|
+
::Coverage.peek_result.each_key { @known << it if keep?(it) }
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# For a worker, right after its fork.
|
|
40
|
+
def probe
|
|
41
|
+
Probe.new(file: File.join(@frames, "#{Process.pid}.cov"), known: @known, keep: method(:keep?),
|
|
42
|
+
methods: @criteria.include?(:methods), interval: Chart.config.interval)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# A worker died: what it measured since its last flush is gone.
|
|
46
|
+
def lost_worker = @lost += 1
|
|
47
|
+
|
|
48
|
+
# Sums everything (see Raw); nil for an interrupted run.
|
|
49
|
+
def finish(interrupted: false)
|
|
50
|
+
return if interrupted
|
|
51
|
+
|
|
52
|
+
total = Delta.capture(::Coverage.peek_result, Set.new, method(:keep?))
|
|
53
|
+
cells = 0
|
|
54
|
+
frames.each do |frame|
|
|
55
|
+
Delta.merge!(total, frame[:delta])
|
|
56
|
+
cells += frame[:lost]
|
|
57
|
+
end
|
|
58
|
+
Raw.build(normalize(total), root: @root, criteria: @criteria, lost: { workers: @lost, cells: })
|
|
59
|
+
ensure
|
|
60
|
+
FileUtils.rm_rf(@frames) if @frames
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
private
|
|
64
|
+
|
|
65
|
+
# Files under the project root, which exist (not eval'd code).
|
|
66
|
+
def keep?(path)
|
|
67
|
+
@kept.fetch(path) { @kept[path] = @roots.any? { path.start_with?(it) } && File.file?(path) }
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Every complete frame of every worker. A worker killed while writing
|
|
71
|
+
# leaves a truncated last frame, skipped.
|
|
72
|
+
def frames
|
|
73
|
+
Dir[File.join(@frames, '*.cov')].flat_map do |file|
|
|
74
|
+
data = File.binread(file)
|
|
75
|
+
offset = 0
|
|
76
|
+
frames = []
|
|
77
|
+
while offset + 4 <= data.bytesize
|
|
78
|
+
size = data.unpack1('N', offset:)
|
|
79
|
+
break if offset + 4 + size > data.bytesize
|
|
80
|
+
|
|
81
|
+
frames << Marshal.load(data.byteslice(offset + 4, size)) # written by our own workers
|
|
82
|
+
offset += 4 + size
|
|
83
|
+
end
|
|
84
|
+
frames
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Keys by real path relative to the root: a file required through a
|
|
89
|
+
# symlink and through its real path is one file. A file deleted during
|
|
90
|
+
# the run is not reported.
|
|
91
|
+
def normalize(total)
|
|
92
|
+
total.each_with_object({}) do |(path, entry), files|
|
|
93
|
+
relative = File.realpath(path).delete_prefix("#{@root}/")
|
|
94
|
+
next if relative.start_with?('/')
|
|
95
|
+
|
|
96
|
+
files[relative] ? Delta.merge!(files, { relative => entry }) : files[relative] = entry
|
|
97
|
+
rescue SystemCallError
|
|
98
|
+
next
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|