saboteur 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +399 -0
- data/README.md +202 -42
- data/exe/saboteur +34 -7
- data/lib/saboteur/baseline/comparison.rb +1 -0
- data/lib/saboteur/baseline/evidence.rb +11 -1
- data/lib/saboteur/baseline/run_evidence.rb +1 -1
- data/lib/saboteur/baseline/workflow.rb +2 -1
- data/lib/saboteur/baseline_health.rb +40 -6
- data/lib/saboteur/cli/exit_status.rb +49 -4
- data/lib/saboteur/cli/help_options.rb +16 -0
- data/lib/saboteur/cli/presets.rb +36 -0
- data/lib/saboteur/cli/run_budget_options.rb +37 -0
- data/lib/saboteur/cli/run_reporting.rb +75 -6
- data/lib/saboteur/cli.rb +80 -48
- data/lib/saboteur/config.rb +88 -13
- data/lib/saboteur/crash_report.rb +154 -0
- data/lib/saboteur/custom_map.rb +3 -1
- data/lib/saboteur/doctor.rb +64 -2
- data/lib/saboteur/engine/baseline_guard.rb +79 -5
- data/lib/saboteur/engine/boot.rb +12 -1
- data/lib/saboteur/engine/canary.rb +41 -10
- data/lib/saboteur/engine/candidate_selection.rb +9 -1
- data/lib/saboteur/engine/candidates.rb +4 -1
- data/lib/saboteur/engine/coverage_guard.rb +29 -0
- data/lib/saboteur/engine/coverage_store.rb +48 -6
- data/lib/saboteur/engine/diagnostics.rb +23 -2
- data/lib/saboteur/engine/diff_scope.rb +24 -5
- data/lib/saboteur/engine/distribution.rb +124 -8
- data/lib/saboteur/engine/early_stop_recording.rb +46 -0
- data/lib/saboteur/engine/result_cache.rb +2 -1
- data/lib/saboteur/engine/run_budget.rb +63 -0
- data/lib/saboteur/engine/run_fingerprint.rb +5 -1
- data/lib/saboteur/engine/sandbox_policy.rb +18 -11
- data/lib/saboteur/engine/scheduler.rb +114 -18
- data/lib/saboteur/engine/subject_selection.rb +84 -5
- data/lib/saboteur/engine/suite_files.rb +3 -1
- data/lib/saboteur/engine/test_guess.rb +183 -23
- data/lib/saboteur/engine/thread_warnings.rb +53 -0
- data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
- data/lib/saboteur/engine/verdict_check.rb +54 -18
- data/lib/saboteur/engine.rb +175 -96
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/runner.rb +179 -17
- data/lib/saboteur/execution/sandbox.rb +6 -1
- data/lib/saboteur/execution/supervisor.rb +18 -3
- data/lib/saboteur/execution/worker_pool.rb +87 -7
- data/lib/saboteur/frameworks/database_hosts.rb +30 -0
- data/lib/saboteur/frameworks/rails.rb +8 -10
- data/lib/saboteur/help.rb +55 -10
- data/lib/saboteur/ignore_list.rb +11 -1
- data/lib/saboteur/install.rb +4 -3
- data/lib/saboteur/integration/base.rb +7 -2
- data/lib/saboteur/integration/rspec.rb +40 -4
- data/lib/saboteur/mutation_source/builtin.rb +22 -13
- data/lib/saboteur/mutation_source/hazard_guard.rb +21 -3
- data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
- data/lib/saboteur/mutation_source/method_swap.rb +45 -0
- data/lib/saboteur/mutation_source/mutation.rb +5 -3
- data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
- data/lib/saboteur/mutation_source/operator_examples.rb +38 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +36 -8
- data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
- data/lib/saboteur/preview.rb +12 -2
- data/lib/saboteur/reporter/agent_commands.rb +49 -16
- data/lib/saboteur/reporter/agent_incomplete.rb +27 -2
- data/lib/saboteur/reporter/agent_summary.rb +1 -1
- data/lib/saboteur/reporter/history_policy.rb +35 -0
- data/lib/saboteur/reporter/history_record.rb +4 -5
- data/lib/saboteur/reporter/next_steps.rb +14 -4
- data/lib/saboteur/reporter/score_history.rb +19 -3
- data/lib/saboteur/reporter/stryker.rb +19 -8
- data/lib/saboteur/reporter/survivor_hints.rb +3 -0
- data/lib/saboteur/reporter/terminal.rb +43 -10
- data/lib/saboteur/result.rb +25 -7
- data/lib/saboteur/selection/coverage_map.rb +2 -1
- data/lib/saboteur/selection/git_diff.rb +15 -5
- data/lib/saboteur/selection/untracked_changes.rb +126 -0
- data/lib/saboteur/subject.rb +11 -0
- data/lib/saboteur/text.rb +42 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur.rb +16 -1
- metadata +20 -2
data/README.md
CHANGED
|
@@ -47,11 +47,17 @@ accepted mutants or change their scores. See [Opt-in survivor baseline](#opt-in-
|
|
|
47
47
|
|
|
48
48
|
### The unchanged suite is checked first
|
|
49
49
|
|
|
50
|
-
Every run with selected subjects runs the loaded unchanged suite twice (the specs chosen with `--test
|
|
50
|
+
Every run with selected subjects runs the loaded unchanged suite twice (the specs chosen with `--test` if given,
|
|
51
|
+
otherwise the inferred specs) in fresh
|
|
51
52
|
supervised children before coverage lookup or incremental verdict reuse. This
|
|
52
53
|
costs two suite runs even on a cache hit, plus a coverage-building suite run on
|
|
53
|
-
a cache miss.
|
|
54
|
-
|
|
54
|
+
a cache miss. Before the first pass, a run over the whole suite announces the example
|
|
55
|
+
count and the real number of full passes (a cached map only counts as
|
|
56
|
+
reused when it loads clean); once the first baseline pass is timed, the
|
|
57
|
+
run estimates the time left for the remaining baseline and coverage
|
|
58
|
+
passes from it - mutation time is not part of that estimate. Past 1,000
|
|
59
|
+
examples the announcement also names `--test` as the way to scope the run. Both baseline passes must be green, complete and in the same
|
|
60
|
+
RSpec order at the run seed. Set `c.baseline_passes = 1` (or `--baseline-passes 1`) for a faster PR loop: the run says so up front and gives up the order-stability comparison between passes (#55). A failure exits 3, stops mutation, prints the failing example ids and the first ten lines of the
|
|
55
61
|
failure inline, and names the log and fix. Long phases
|
|
56
62
|
print their elapsed time about every 30 seconds, with the examples finished so far, the percentage and a rough estimate of the time left (it assumes equally long examples) once the suite child has logged its first count, and each phase's log holds
|
|
57
63
|
the suite's progress and full failure detail. Each suite pass
|
|
@@ -82,21 +88,24 @@ hazard guard is a syntactic filter. It recognizes destructive calls written
|
|
|
82
88
|
as direct calls: `FileUtils.rm*`/`remove_entry*` (also as bare names under
|
|
83
89
|
`include FileUtils`), `File.delete`/`unlink`, `Dir.rmdir`/`delete`/`unlink`,
|
|
84
90
|
`rmtree`, `system`/`exec`/`spawn` (bare or on `Kernel`/`Process`), backticks
|
|
85
|
-
and `%x`, `IO.popen`, any `Open3` method,
|
|
91
|
+
and `%x`, `IO.popen`, any `Open3` method, `open` with a leading `|`, and
|
|
92
|
+
`send`/`public_send`/`__send__` with a literal symbol or string selector
|
|
93
|
+
naming one of these (dynamic selectors stay out of scope). It
|
|
86
94
|
suppresses any mutant that would leave such a call in place while altering
|
|
87
95
|
code inside it or anywhere else in the same method. A mutant that removes
|
|
88
96
|
every hazardous call in the method still runs, provided its replacement
|
|
89
97
|
introduces no new hazard. The match is by name on the written call: calls
|
|
90
|
-
reached through aliases, variables
|
|
98
|
+
reached through aliases, variables or other indirection are not
|
|
91
99
|
recognized, so the guard cannot promise to find every destructive call. The
|
|
92
100
|
summary reports how many mutants were suppressed. Extend or replace the list
|
|
93
101
|
with `c.hazard` (same tag shapes: `"FileUtils.rm_rf"`, or a bare name like
|
|
94
102
|
`"system"`); `--allow-hazardous` runs the suppressed mutants anyway.
|
|
95
103
|
|
|
96
104
|
Behind the guard, a canary snapshots the tracked files (or every file,
|
|
97
|
-
outside a repository) before mutating and
|
|
98
|
-
|
|
99
|
-
|
|
105
|
+
outside a repository) before mutating and probes the snapshot after every
|
|
106
|
+
mutant - a cheap stat pass, no tree walk. If anything vanished, the run
|
|
107
|
+
aborts with exit 3 and names the digest of the mutant that just ran, not
|
|
108
|
+
the method's last mutant. A fuller check still runs after each method. That check happens after the damage and restores
|
|
100
109
|
nothing; it limits how far a bad mutant gets. Recover from your own backup,
|
|
101
110
|
a fresh checkout or a reclone - a deleted `.git` or uncommitted work is gone
|
|
102
111
|
unless you have another copy. The canary is the backstop for the destructive
|
|
@@ -104,8 +113,18 @@ forms the syntactic guard cannot see.
|
|
|
104
113
|
|
|
105
114
|
On Linux 6.2 and later (Landlock ABI 3) each mutant child also runs under
|
|
106
115
|
a Landlock sandbox: the child cannot create, write, rename, unlink or
|
|
107
|
-
truncate files in the project tree, except `tmp
|
|
108
|
-
system temp dir, plus anything in `c.sandbox_writable`.
|
|
116
|
+
truncate files in the project tree, except `tmp/` and the
|
|
117
|
+
system temp dir, plus anything in `c.sandbox_writable`. The project
|
|
118
|
+
`tmp/` grant applies only when `tmp/` already exists - the run never
|
|
119
|
+
creates it for the grant - so create it before the run if your suite
|
|
120
|
+
writes there, or grant the directory with `c.sandbox_writable`. The cache dir
|
|
121
|
+
is denied too - a mutant cannot delete the baseline, the reports or
|
|
122
|
+
other children's logs; its own log is opened before the fork and
|
|
123
|
+
arrives as an inherited descriptor. Grants are additive, though: a
|
|
124
|
+
cache dir placed inside `tmp/`, the system temp dir or a
|
|
125
|
+
`c.sandbox_writable` entry stays writable, and the tool cannot deny
|
|
126
|
+
what another grant allows - keep the cache outside every grant for
|
|
127
|
+
the isolation to bite. Every other path
|
|
109
128
|
is denied for writing, `/dev/shm` and `/dev/tty` included; `/dev/null`
|
|
110
129
|
stays writable. Inherited
|
|
111
130
|
descriptors already open for writing outside the granted dirs are closed
|
|
@@ -123,9 +142,9 @@ grant that directory with `c.sandbox_writable` or opt out with
|
|
|
123
142
|
sockets and IPC are never restricted - and a checkout inside the system
|
|
124
143
|
temp dir cannot be protected, because everything under it stays writable
|
|
125
144
|
so that real test suites keep working - the run warns when the effective
|
|
126
|
-
write set (every grant realpath'd, the
|
|
145
|
+
write set (every grant realpath'd, the system temp dir
|
|
127
146
|
included) contains the project root, however it happened: a checkout
|
|
128
|
-
inside the system temp dir,
|
|
147
|
+
inside the system temp dir, a symlinked `tmp`, or an
|
|
129
148
|
overlapping `c.sandbox_writable` grant. `c.sandbox = "strict"` refuses to
|
|
130
149
|
run at all in that situation, since the sandbox would protect nothing.
|
|
131
150
|
The sandbox applies to every mutant
|
|
@@ -255,10 +274,12 @@ above the limit, with `exitCause` `maxTimeouts` or `maxErrors` in the JSON repor
|
|
|
255
274
|
their own causes when they fail too. The gates cannot be combined with `--agent`.
|
|
256
275
|
The GitHub Action has no input for them yet.
|
|
257
276
|
|
|
258
|
-
Other flags: `--jobs N` (concurrent workers), `--timeout-factor N`,
|
|
277
|
+
Other flags: `--jobs N` (concurrent workers), `--timeout-factor N`, `--baseline-passes N`, `--max-duration DUR`,
|
|
259
278
|
`--config PATH`, `--integration NAME`, `--reeval class_eval|nesting`,
|
|
260
279
|
`--json PATH`, `--no-cache` (rebuild the coverage map), `--allow-focus`.
|
|
261
280
|
|
|
281
|
+
Two presets bundle the common loops (#244): `saboteur pr [REF]` is the pull-request loop - `--since REF` (default `main`), one baseline check and a ten-minute `--max-duration` that opts out of the budget-specific exit 4 (normal and incomplete verdicts still fail the run); `saboteur audit` is the whole project: `--all`, all operators, both baseline checks and both reports (an explicit `--test` replaces the injected `--all`; configured selection limits such as `since`, subjects, exclusions and a configured budget still apply). The run prints the expansion up front, and an explicit flag after the preset wins.
|
|
282
|
+
|
|
262
283
|
A stray `fit`, `fdescribe` or `:focus` example, with `filter_run_when_matching :focus`, makes RSpec skip
|
|
263
284
|
every other example. The baseline stays green and the score would cover only the focused ones, so
|
|
264
285
|
saboteur exits 2 and names the focused examples. Remove the focus, or pass `--allow-focus` (config
|
|
@@ -269,24 +290,58 @@ check compares what RSpec runs with what it would run without the stray focus, s
|
|
|
269
290
|
|
|
270
291
|
For pull requests, start with `--since` against your target branch. This is a
|
|
271
292
|
workflow recommendation, not a CLI default: plain `run` still checks the whole
|
|
272
|
-
configured project. To scope the specs
|
|
293
|
+
configured project. To scope the specs yourself, pass them with `--test` and no
|
|
273
294
|
source operands: `bundle exec saboteur run --since origin/main --test spec/billing_spec.rb`.
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
295
|
+
A changed Ruby file whose changed lines cover no method mutates nothing; the run
|
|
296
|
+
warns and names the file instead of silently selecting zero subjects. An untracked
|
|
297
|
+
file inside the configured `include` set counts as fully changed: a brand-new,
|
|
298
|
+
never-committed source file is selected in full rather than silently skipped,
|
|
299
|
+
because a diff cannot see it (#295). The union reads race-free
|
|
300
|
+
(O_NOFOLLOW/O_NONBLOCK, validated on the opened descriptor) and streams its
|
|
301
|
+
line count, so a swapped-in symlink or FIFO is skipped quietly and a huge
|
|
302
|
+
untracked fixture outside `include` is never even opened. Comment-only
|
|
303
|
+
or formatting-only changes within discovered methods are skipped without this
|
|
304
|
+
warning; the run reports those skipped methods. A changed Ruby file that an
|
|
305
|
+
include pattern covers but `c.exclude` removes mutates nothing either, and the
|
|
306
|
+
run warns and names up to five such files - narrow or remove the `c.exclude`
|
|
307
|
+
entry, or pass the file as a positional source operand (configured excludes are
|
|
308
|
+
then ignored). `--subject` cannot help here: it filters subjects inside the
|
|
309
|
+
files left after excludes.
|
|
310
|
+
|
|
311
|
+
Without `--test`, the run infers the specs: the spec named after each mutated
|
|
312
|
+
file (`lib/a/b.rb` runs `spec/a/b_spec.rb`; `app/` files and `spec/lib/` work
|
|
313
|
+
too) runs for the baseline passes, the coverage pass and every mutant, as
|
|
314
|
+
`--test` would. With `--since` only the changed files count; without it every
|
|
315
|
+
included file does. It is a naming guess, not coverage: a spec that reaches the
|
|
316
|
+
mutated file only through another file does not run, so a mutant only that
|
|
317
|
+
spec would kill can survive, and a line none of the inferred specs executes is
|
|
318
|
+
reported as NoCoverage. A file needs no inferred spec when it has no mutation
|
|
319
|
+
subjects (with `--since`: none on its changed lines), such as a comment-only
|
|
320
|
+
change within a method or a deletion leaving only blank/comment context; the
|
|
321
|
+
run says so and skips those files. When no file has a mutation subject at all,
|
|
322
|
+
the run reports that there is nothing to mutate. A `--subject` run infers only
|
|
323
|
+
from the named subjects' files: an unrelated spec-less file never stops a
|
|
324
|
+
one-method run, and a subject file with no conventional spec stops the run
|
|
325
|
+
like any other unmatched file. With `--since` only named subjects on changed
|
|
326
|
+
lines count: an unchanged subject file, one whose changes sit in other
|
|
327
|
+
methods, or one with a comment-only change is skipped, not stopped.
|
|
328
|
+
|
|
329
|
+
A file that needs a spec and has none stops the run before it boots (exit 2)
|
|
330
|
+
and names both ways out: `--test` with the specs to run, or `--all` for the
|
|
331
|
+
whole suite. An unmatched file no longer silently narrows the selected source
|
|
332
|
+
scope or widens the test run to the whole suite. Whole-project audits should
|
|
333
|
+
use `--all` explicitly. The agent JSON records the inferred specs as its
|
|
334
|
+
`tests`, so its follow-up commands keep the same scope.
|
|
335
|
+
|
|
336
|
+
`--all` (config `c.test_selection = :all`) runs the whole configured suite.
|
|
337
|
+
With `--since` or `--all` and no inferred or named specs, the run warns before
|
|
338
|
+
it starts that the two baseline passes run the loaded suite (a cold coverage
|
|
339
|
+
cache adds one more pass). `--guess-tests` is a deprecated alias for the
|
|
340
|
+
default inference and `--no-guess-tests` warns and is ignored; use `--all` for
|
|
341
|
+
the whole suite. `--all` cannot be combined with `--test` or `--guess-tests`,
|
|
342
|
+
and `--guess-tests` cannot be combined with `--test`. Keep whole-project runs
|
|
343
|
+
on main: diff mode does not check every method affected by a dependency change.
|
|
344
|
+
[The pull-request guide](docs/pull-requests.md) covers
|
|
290
345
|
checkout history, a GitHub Actions example, configuration and limits.
|
|
291
346
|
|
|
292
347
|
Doctor warns when an ignore subject matches no method discovered in the included
|
|
@@ -326,8 +381,21 @@ printed once as `path:line in Subject`, with the source line and the tests that
|
|
|
326
381
|
covered it, then one compact line per mutation: `operator original -> replacement digest`,
|
|
327
382
|
and a hint on what kind of test would kill it. The summary has the counts per
|
|
328
383
|
status, the score, a table of scores per file (worst first) and a one-line
|
|
329
|
-
legend.
|
|
330
|
-
|
|
384
|
+
legend. When survivors remain, the Next steps footer prints the complete
|
|
385
|
+
replay command - ref, subjects, specs, sources, seed and the run settings
|
|
386
|
+
the agent protocol records, including the CLI-only switches that change
|
|
387
|
+
candidate availability or boot (`--allow-hazardous`, `--allow-focus`) -
|
|
388
|
+
shell-escaped and printed in full however long it gets, so running it
|
|
389
|
+
as-is replays the survivor with the same selection. Framework and
|
|
390
|
+
generation settings from the config file (arid/hazard patterns,
|
|
391
|
+
whole-file, require, rspec_args, rails) ride along through the command's
|
|
392
|
+
`--config` or the default config path. Only knobs that cannot change a
|
|
393
|
+
by-digest replay (`--max-mutants-per-method`, `--fail-fast`,
|
|
394
|
+
`--threshold`) are left out. The agent protocol instead omits a command
|
|
395
|
+
longer than 1024 bytes, and the agent rebuilds it from the run's
|
|
396
|
+
provenance. `--verbose` adds the score split, mean time, whole-body score and
|
|
397
|
+
kill causes, and prints one line per finished subject with its result
|
|
398
|
+
marks, so a survivor is attributable to a method before the run ends. Colour is on when stdout is a terminal and `NO_COLOR` is not set;
|
|
331
399
|
`--color` and `--no-color` override that. While mutants run, stderr shows
|
|
332
400
|
`done/total (percent) eta`: a live counter on a terminal, and dots with a
|
|
333
401
|
counter every 50 mutants otherwise. Skipped and reused mutants are not
|
|
@@ -353,7 +421,13 @@ survivor's notes. The JSON report goes to `.saboteur/report.json` and a self-con
|
|
|
353
421
|
report (one file, no scripts, no network) to `.saboteur/report.html`. Change
|
|
354
422
|
the paths with `--json` / `--html` or `c.json_report` / `c.html_report` in
|
|
355
423
|
`config/saboteur.rb`; set either to `nil` to turn that report off. The two paths
|
|
356
|
-
must differ.
|
|
424
|
+
must differ. When an artifact cannot be written - a read-only tree inside a
|
|
425
|
+
sandboxed run - the HTML report and the score history warn and the run still
|
|
426
|
+
exits by its results. An explicitly requested JSON report (`--json` or
|
|
427
|
+
`c.json_report`) is different: it is the completion evidence an outer harness
|
|
428
|
+
trusts, so a failed write fails the run with exit 3 rather than leaving a
|
|
429
|
+
stale or partial file presented as this run's; the default
|
|
430
|
+
`.saboteur/report.json` stays best-effort. The HTML page shows the covered mutation score
|
|
357
431
|
`(killed + timeout) / (killed + survived + timeout)`, which leaves NoCoverage,
|
|
358
432
|
RanAtLoad and Unreachable out and keeps survivors accepted on the ignore list
|
|
359
433
|
in the denominator. A tool
|
|
@@ -415,10 +489,33 @@ changed line and prints none.
|
|
|
415
489
|
|
|
416
490
|
New to survivors? The [reading survivors guide](docs/guides/reading-survivors.md) walks one from the diff to the test that kills it.
|
|
417
491
|
|
|
418
|
-
Exit codes: 0 no survivors (or all on the ignore list), or the score
|
|
419
|
-
`--threshold`; 1 survivors, or the score is below `--threshold` or
|
|
420
|
-
2 bad configuration or a missing source or test; 3 the tool
|
|
421
|
-
`--threshold`, also any errored or resource-limited
|
|
492
|
+
Exit codes: 0 no survivors (or all on the ignore list), or the score
|
|
493
|
+
meets `--threshold`; 1 survivors, or the score is below `--threshold` or
|
|
494
|
+
missing; 2 bad configuration or a missing source or test; 3 the tool
|
|
495
|
+
failed (with `--threshold`, also any errored or resource-limited
|
|
496
|
+
mutant). Unexpected `StandardError` or `LoadError` failures print
|
|
497
|
+
`saboteur: internal error (...) - please report it` and exit 3, rather
|
|
498
|
+
than dying with a bare stack trace and an accidental exit code. The full
|
|
499
|
+
backtrace lands in `<cache_dir>/internal-error.log` (default
|
|
500
|
+
`.saboteur/`), and `--verbose` also prints it to stderr. When the cache
|
|
501
|
+
dir cannot hold the log, a private pid-keyed fallback in the system temp
|
|
502
|
+
dir takes it (created with `O_EXCL | O_NOFOLLOW`, mode 0600, so it never
|
|
503
|
+
follows or overwrites a pre-existing path). Logs are always written
|
|
504
|
+
private, and a nested run reads them back only through a no-follow,
|
|
505
|
+
non-blocking descriptor - a regular file, same owner, no group/other
|
|
506
|
+
bits (or, where the platform lacks those flags, only the `O_EXCL` temp
|
|
507
|
+
fallback is written and no excerpt is read). The message names the log only
|
|
508
|
+
after a successful write: an unset `cache_dir`, or a log writable nowhere,
|
|
509
|
+
leaves the one-liner standing alone. The executable
|
|
510
|
+
loads the crash reporter before the CLI, so a failure while it is
|
|
511
|
+
loading, being built, or parsing arguments gets the same report;
|
|
512
|
+
failures earlier than that - coverage setup or the reporter's own load -
|
|
513
|
+
get only the bare one-liner and still exit 3. Once the crash
|
|
514
|
+
reporter has loaded, agent mode reports the same failure as agent
|
|
515
|
+
JSON, using a standalone payload if the agent reporter has not
|
|
516
|
+
loaded. Before the crash reporter loads, only the one-line
|
|
517
|
+
diagnostic and exit 3 are available.
|
|
518
|
+
`--max-duration 10m` budgets the whole run, baselines included: the budget is checked before every subject and between mutants inside one, so past the budget the run finishes only the mutant in flight (bounded by its timeout), then stops before the next subject. Admitted phases (baseline checks, the coverage map build) are capped at the time left, as are verdict checks and timeout calibrations - admission is per check and per sequence, so nothing forks with no time left. A phase that hits only its budget-shortened cap stops the run as budget-exhausted rather than failing as a suite error. A kill or timeout the budget leaves uncheckable is reported as `unverified` - outside the score, never cached, listed under the early stop - instead of becoming an error, reports which selected subjects never ran and which mutants were omitted, marks the JSON and agent output partial (`earlyStop.reason: "max_duration"`), and exits 4; `--max-duration-ok` (or `c.max_duration_ok = true`) leaves the exit to the plain verdict instead, which can still fail the run - 1 on survivors, 3 when the budget stopped the run before any mutant verdict was verified. In agent mode the exit code stays verdict-driven and the budget shows up in `incomplete[]` and `earlyStop` (#245).
|
|
422
519
|
|
|
423
520
|
### Rejected kills
|
|
424
521
|
|
|
@@ -452,6 +549,9 @@ Saboteur.configure do |c|
|
|
|
452
549
|
c.sandbox_writable = [] # extra dirs mutant children may write, e.g. ["db", "log"]
|
|
453
550
|
c.ignore_file = "config/saboteur_ignore.rb"
|
|
454
551
|
c.seed = 1 # Ruby's random seed in every test run
|
|
552
|
+
c.baseline_passes = 2 # unchanged-suite checks before mutating; 1 skips the order-stability check (#55)
|
|
553
|
+
c.max_duration = nil # wall-clock budget in seconds, e.g. 600; an exhausted budget exits 4 (#245)
|
|
554
|
+
c.max_duration_ok = false # true: an exhausted budget does not fail the run
|
|
455
555
|
c.allow_focus = false # true runs a suite narrowed by fit/fdescribe/:focus instead of exit 2
|
|
456
556
|
c.fingerprint_extra = [] # extra globs whose contents invalidate caches, e.g. ["data/*.csv"]
|
|
457
557
|
end
|
|
@@ -500,6 +600,13 @@ Doctor boots your app, checks the config and the test database host, and runs
|
|
|
500
600
|
your suite twice before any mutation. Fix whatever it reports first. Turn
|
|
501
601
|
SimpleCov off for the run if you use it.
|
|
502
602
|
|
|
603
|
+
The database check refuses any test database whose host is not local
|
|
604
|
+
(`localhost`, `127.0.0.1`, `::1`, `0.0.0.0` or a socket), so a remote
|
|
605
|
+
database is never mutated by accident. `0.0.0.0` counts as local: compose
|
|
606
|
+
setups write it into database URLs, and connecting to it reaches this
|
|
607
|
+
machine. For another disposable test database host, assert it yourself:
|
|
608
|
+
`c.allow_db_hosts = ["db.internal"]` in `config/saboteur.rb`.
|
|
609
|
+
|
|
503
610
|
**4. Test what your branch changed**
|
|
504
611
|
|
|
505
612
|
```sh
|
|
@@ -663,6 +770,27 @@ An entry without a reason is rejected.
|
|
|
663
770
|
saboteur needs Ruby's `Coverage` for itself. Disable SimpleCov for the run
|
|
664
771
|
(most projects already guard it with an environment variable).
|
|
665
772
|
|
|
773
|
+
### Threads alive at fork time
|
|
774
|
+
|
|
775
|
+
Threads do not survive `fork`, so a run names any that are alive when the
|
|
776
|
+
workers fork. Two are well known:
|
|
777
|
+
|
|
778
|
+
- `AR Pool Reaper` only maintains the parent's connection pool. It is
|
|
779
|
+
expected harmless when children reconnect the database after fork -
|
|
780
|
+
saboteur's rails mode does exactly that. Without verified
|
|
781
|
+
reconnection, do not assume the inherited connection state is safe.
|
|
782
|
+
- `DEBUGGER__::SESSION@server` is the debug gem's remote console. What
|
|
783
|
+
it does in children depends on `RUBY_DEBUG_FORK_MODE`: with `parent`
|
|
784
|
+
no debugger runs in children; with `child` or `both` (the gem's
|
|
785
|
+
default) the debugger activates in children and can pause or
|
|
786
|
+
interfere with supervised mutation runs. For unattended runs, do not
|
|
787
|
+
start the remote debugger at all: drop `RUBY_DEBUG_OPEN` or require
|
|
788
|
+
`debug` without `:open`.
|
|
789
|
+
|
|
790
|
+
Any other thread gets a generic caveat: a thread that does not run in a
|
|
791
|
+
child cannot do background work a mutant's test may rely on, so results
|
|
792
|
+
can differ when one is left behind.
|
|
793
|
+
|
|
666
794
|
## Filter mutations
|
|
667
795
|
|
|
668
796
|
The built-in source also swaps `select` / `reject` and `select!` / `reject!`.
|
|
@@ -725,6 +853,10 @@ and `tap` links are never removed. Tests can observe frozenness, copies and a
|
|
|
725
853
|
query chain, and dropping such a link mostly yields noise, so this operator
|
|
726
854
|
leaves them out. Arid statements and nested definitions are skipped.
|
|
727
855
|
|
|
856
|
+
If you confirm the shorter chain is equivalent across the intended
|
|
857
|
+
inputs (for example a duplicated column), simplify the code or ignore
|
|
858
|
+
the mutant.
|
|
859
|
+
|
|
728
860
|
Chain removal is on by default, like `argument_removal` and `pair_removal`.
|
|
729
861
|
Its proposals follow the other fine proposals, with the same validation,
|
|
730
862
|
deduplication and `max_mutants_per_method` cap.
|
|
@@ -779,11 +911,12 @@ Public API, stable across minor versions:
|
|
|
779
911
|
- `Saboteur::Integration::Base` for other test frameworks (RSpec is the only one
|
|
780
912
|
that ships).
|
|
781
913
|
- `Saboteur.hook(:before_boot | :after_boot | :before_fork | :after_fork |
|
|
782
|
-
:around_mutant)` in files listed under `hooks:`.
|
|
914
|
+
:around_mutant)` in files listed under `hooks:`. An extensionless hooks
|
|
915
|
+
path resolves its `.rb`, the way `require` does.
|
|
783
916
|
|
|
784
917
|
## 🌍 Where it has run
|
|
785
918
|
|
|
786
|
-
|
|
919
|
+
Tested on the RSpec suites of plain gems
|
|
787
920
|
(dry-inflector, tty-color, hashie, addressable) and of two Rails 8.1 apps:
|
|
788
921
|
human-essentials on Postgres (4,080 mutants, 6 workers; the 2026-10-10 run, before
|
|
789
922
|
the baseline check and the coarse-gate removal) and lobsters on SQLite,
|
|
@@ -885,6 +1018,14 @@ details are in [docs/design.md](docs/design.md), prior art in
|
|
|
885
1018
|
|
|
886
1019
|
See [CONTRIBUTING.md](CONTRIBUTING.md). Commits need a DCO `Signed-off-by` line.
|
|
887
1020
|
|
|
1021
|
+
Before you ask for review, run the local preflight check on your exact head:
|
|
1022
|
+
|
|
1023
|
+
```sh
|
|
1024
|
+
ruby script/preflight --base <full main sha> --head <full head sha> --base-ref main --mode full
|
|
1025
|
+
```
|
|
1026
|
+
|
|
1027
|
+
It runs scope, sign-off, SPDX, full RuboCop and the specs, and writes a report under `tmp/preflight/`. A pass is not an approval; fast mode is partial. See [docs/preflight.md](docs/preflight.md).
|
|
1028
|
+
|
|
888
1029
|
## Acknowledgements
|
|
889
1030
|
|
|
890
1031
|
Thanks to [Glomo](https://glomo.one) for providing the AI tokens used to build saboteur.
|
|
@@ -932,13 +1073,30 @@ The builtin `endpoint_replacement` operator swaps explicit, no-argument `first`/
|
|
|
932
1073
|
|
|
933
1074
|
JSON findings are ordered by a deterministic [review ranking](docs/survivor-ranking.md): coarse first, then covering-test count, edited-line impact and changed-line proximity, with file/line/digest ties. The terminal groups survivors by file, method and line first; the ranking only orders the survivors on one line, and every survivor is listed, in review mode too. Only consecutive survivors with matching evidence share a test/evidence block; repeated evidence after another group is printed again to keep that order. Ranking does not change scores or execution.
|
|
934
1075
|
|
|
1076
|
+
Cache and baseline identities include the policy. Re-propose and review old survivor baselines with `baseline propose`; never auto-accept new survivors. Mutation digests for unchanged edits stay stable.
|
|
1077
|
+
|
|
935
1078
|
### Preview and choose operators
|
|
936
1079
|
|
|
937
1080
|
`bundle exec saboteur run --dry-run` discovers source methods and generates candidates without booting the app, running tests, forking, or writing reports/caches. It honors `--since`, `--subject` and source arguments. It prints per-method counts and cap omissions. After a matching normal run, a cached estimate shows baseline covering-test work (cached original-test durations times planned mutants; it is not a bound on mutant runtime and excludes boot and fork costs). If any file cannot be parsed, the total is marked INCOMPLETE and no estimate is given.
|
|
938
1081
|
|
|
939
1082
|
Trust boundary: `--dry-run` never boots the app or runs its tests, but custom `sources` entries (`path.rb:ClassName`) are still `require`d to generate candidates, so their code runs. Only preview with sources you trust.
|
|
940
1083
|
|
|
941
|
-
`bundle exec saboteur run --list-operators` lists every builtin name and description.
|
|
1084
|
+
`bundle exec saboteur run --list-operators` lists every builtin name and description. All builtin operators run by default, including [`range`](docs/operators/range.md), [`operator_assignment`](docs/operators/operator_assignment.md), [`condition_negation`](docs/operators/condition_negation.md), [`method_swap`](docs/operators/method_swap.md), [`operand_replacement`](docs/operators/operand_replacement.md), and [`interpolation_removal`](docs/operators/interpolation_removal.md). `--operators arithmetic,comparison` or `c.operators = ["arithmetic", "comparison"]` still selects a subset; `all` selects every name. There is no opt-in tier. `--exclude-operators range,condition_negation` or `c.exclude_operators = ["range", "condition_negation"]` removes named operators from that selection; exclusion wins if a name is in both lists. An empty selection means all, while an empty exclusion means none. CLI lists replace their corresponding config lists. These controls apply to builtin operators, not custom source plugins. Selection and exclusions invalidate cached verdict identity and are carried into agent rerun/replay commands.
|
|
1085
|
+
|
|
1086
|
+
|
|
1087
|
+
All operators run unless you select a smaller set; exclusions always remove operators from that set.
|
|
1088
|
+
Repeated `--operators` or `--exclude-operators` flags use the last list, not a union.
|
|
1089
|
+
To re-enable an operator excluded in config: `bundle exec saboteur run --operators range --exclude-operators ''`.
|
|
1090
|
+
To keep the 0.5.0 default inventory: `bundle exec saboteur run --exclude-operators range,operator_assignment,condition_negation`.
|
|
1091
|
+
The old `core` and `extended` names now give a migration error, not a changed selection in disguise.
|
|
1092
|
+
An empty effective selection is a usage error before boot. `--list-operators` loads current config and shows on/off plus examples; preview and the run header show the effective set.
|
|
1093
|
+
|
|
1094
|
+
```ruby
|
|
1095
|
+
Saboteur.configure do |c|
|
|
1096
|
+
c.operators = [] # all by default; or ["arithmetic", "range"] for a subset
|
|
1097
|
+
c.exclude_operators = ["condition_negation"]
|
|
1098
|
+
end
|
|
1099
|
+
```
|
|
942
1100
|
|
|
943
1101
|
Inspired by mutineer's --dry-run, --list-operators and --operators tiers (github.com/davidteren/mutineer, MIT, by David Teren).
|
|
944
1102
|
|
|
@@ -957,9 +1115,11 @@ step; the bundled Action's upload behavior is unchanged.
|
|
|
957
1115
|
|
|
958
1116
|
Score history now refuses non-regular files, symlink paths and report/history
|
|
959
1117
|
inode aliases before running the engine or writing reports. Imported records
|
|
960
|
-
have a 64 KiB per-line bound and strict version
|
|
961
|
-
counts, scores and duplicate keys. History is an output, excluded from baseline
|
|
1118
|
+
have a 64 KiB per-line bound and strict validation of supported version 1
|
|
1119
|
+
and version 2 records, including counts, scores and duplicate keys. History is an output, excluded from baseline
|
|
962
1120
|
fingerprints. See [history safety and comparison limits](docs/scores.md).
|
|
1121
|
+
|
|
1122
|
+
Operator policy version 2, selected names, excluded names and effective names are in JSON/agent/baseline evidence and fresh score-history v2 rows. Old history v1 rows remain readable but are not compared across policy versions.
|
|
963
1123
|
### Inline suppression comments
|
|
964
1124
|
|
|
965
1125
|
`total = a + b # saboteur:disable-line arithmetic reason="rounding checked elsewhere"` suppresses edits starting on that line, restricted to the named operators. Omit names to suppress all operators. Multiple names are space-separated. Only actual Ruby comments count; directive text in strings does not. A missing or blank reason or unknown operator is a usage error naming file:line. Terminal, JSON and HTML list every suppressed candidate with its reason and digest, outside the score. Moving the commented code keeps the suppression. Multi-line edits use their starting line.
|
data/exe/saboteur
CHANGED
|
@@ -2,15 +2,42 @@
|
|
|
2
2
|
# SPDX-License-Identifier: MIT
|
|
3
3
|
# frozen_string_literal: true
|
|
4
4
|
|
|
5
|
-
#
|
|
6
|
-
#
|
|
5
|
+
# Coverage comes first because it only measures files loaded after it -
|
|
6
|
+
# the crash reporter must stay measurable - and it loads inside the same
|
|
7
|
+
# guarded begin as the reporter: every later failure, even while the CLI
|
|
8
|
+
# is being required, gets the full report (the bounded one-liner, the log
|
|
9
|
+
# file, the --verbose backtrace and agent JSON when the CLI loaded far
|
|
10
|
+
# enough). If even the reporter cannot load, a bare normalized one-liner
|
|
11
|
+
# (one line, invalid bytes scrubbed, bounded length) still exits 3.
|
|
12
|
+
begin
|
|
13
|
+
require "coverage"
|
|
14
|
+
Coverage.setup(lines: true, methods: true, branches: true) if Coverage.state == :idle
|
|
15
|
+
|
|
16
|
+
require "saboteur/crash_report"
|
|
17
|
+
rescue LoadError, StandardError => e
|
|
18
|
+
message = e.message.to_s.dup.force_encoding(Encoding::UTF_8).scrub("?").gsub(/\s+/, " ").strip
|
|
19
|
+
message = "#{message[0...160]}..." if message.length > 160
|
|
20
|
+
warn "saboteur: internal error (#{e.class}: #{message}) - please report it: " \
|
|
21
|
+
"https://github.com/vamsipavanmahesh/saboteur/issues"
|
|
22
|
+
exit 3
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# For `run` coverage counts from here: what executes while the app and
|
|
7
26
|
# spec files load is how ran-at-load mutants are told apart from uncovered
|
|
8
27
|
# ones. The engine suspends counting right after boot and reads the
|
|
9
28
|
# counters as the map's load lines (a forked child would not inherit them).
|
|
10
|
-
|
|
11
|
-
Coverage.
|
|
12
|
-
Coverage.resume if ARGV.first == "run" && Coverage.state == :suspended
|
|
29
|
+
begin
|
|
30
|
+
Coverage.resume if ARGV.first == "run" && Coverage.state == :suspended
|
|
13
31
|
|
|
14
|
-
require "saboteur/cli"
|
|
32
|
+
require "saboteur/cli"
|
|
15
33
|
|
|
16
|
-
exit Saboteur::CLI.new(ARGV).call
|
|
34
|
+
exit Saboteur::CLI.new(ARGV).call
|
|
35
|
+
rescue LoadError, StandardError => e
|
|
36
|
+
# Agent mode is visible in the raw arguments; the CLI may never have
|
|
37
|
+
# loaded, so this scan must not depend on it.
|
|
38
|
+
agent = ARGV.include?("--agent") || ARGV.include?("--format=agent") ||
|
|
39
|
+
ARGV.each_cons(2).any? { |flag, value| flag == "--format" && value == "agent" }
|
|
40
|
+
exit Saboteur::CrashReport.call(e, verbose: ARGV.include?("--verbose"), agent: agent,
|
|
41
|
+
err: $stderr, out: $stdout,
|
|
42
|
+
log_path: File.join(Dir.pwd, ".saboteur", "internal-error.log"))
|
|
43
|
+
end
|
|
@@ -19,6 +19,7 @@ module Saboteur
|
|
|
19
19
|
@issues = current.fetch("issues").dup
|
|
20
20
|
@categories = {}
|
|
21
21
|
unless baseline.fetch("compatibility") == current.fetch("compatibility")
|
|
22
|
+
@issues << "baseline inputs or operator policy changed; re-propose with baseline propose"
|
|
22
23
|
@state = "incompatible"
|
|
23
24
|
return
|
|
24
25
|
end
|
|
@@ -56,7 +56,7 @@ module Saboteur
|
|
|
56
56
|
compatibility = document.fetch("compatibility")
|
|
57
57
|
config = compatibility.fetch("config")
|
|
58
58
|
runtime = compatibility.fetch("runtime")
|
|
59
|
-
|
|
59
|
+
validate_operator_policy(config)
|
|
60
60
|
fields!(runtime, %w[engine version platform patchlevel saboteur])
|
|
61
61
|
raise ConfigError, "baseline nonfresh config" if config["since"] || config["incremental"]
|
|
62
62
|
|
|
@@ -69,6 +69,16 @@ module Saboteur
|
|
|
69
69
|
validate_generator(compatibility.fetch("generator"), config, runtime)
|
|
70
70
|
end
|
|
71
71
|
|
|
72
|
+
def validate_operator_policy(config)
|
|
73
|
+
unless config["operator_policy"] && config["operator_policy"]["version"] == MutationSource::OperatorRegistry::POLICY_VERSION
|
|
74
|
+
raise ConfigError, "baseline operator policy is incompatible; re-propose with baseline propose"
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
fields!(config, Config::DEFAULTS.keys - RunEvidence::OUTPUT_SETTINGS + ["operator_policy"])
|
|
78
|
+
expected = Config.new(config.except("operator_policy")).operator_policy
|
|
79
|
+
raise ConfigError, "baseline operator policy mismatch" unless expected == config["operator_policy"]
|
|
80
|
+
end
|
|
81
|
+
|
|
72
82
|
def validate_generator(generator, config, runtime)
|
|
73
83
|
fields!(generator, %w[saboteur sources plugins])
|
|
74
84
|
plugins = document.fetch("snapshot").fetch("inventory").select { |entry| entry["kind"] == "source_plugin" }
|
|
@@ -95,7 +95,7 @@ module Saboteur
|
|
|
95
95
|
entry.kind == "source_plugin"
|
|
96
96
|
end.map(&:to_h) },
|
|
97
97
|
"runtime" => Engine::RunFingerprint.runtime_identity,
|
|
98
|
-
"config" => @config.
|
|
98
|
+
"config" => @config.selection_settings.except(*OUTPUT_SETTINGS),
|
|
99
99
|
"scope" => { "include" => @config["include"], "exclude" => @config["exclude"],
|
|
100
100
|
"subjects" => @config["subjects"] } }
|
|
101
101
|
end
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
require_relative "acceptance"
|
|
5
5
|
require_relative "comparison"
|
|
6
6
|
require_relative "../engine"
|
|
7
|
+
require_relative "../text"
|
|
7
8
|
|
|
8
9
|
module Saboteur
|
|
9
10
|
module Baseline
|
|
@@ -70,7 +71,7 @@ module Saboteur
|
|
|
70
71
|
|
|
71
72
|
bytes = File.read(path)
|
|
72
73
|
@baseline_digest = Digest::SHA256.hexdigest(bytes)
|
|
73
|
-
manifest = Manifest.new(JSON.parse(bytes))
|
|
74
|
+
manifest = Manifest.new(JSON.parse(Text.utf8_strict(bytes)))
|
|
74
75
|
metadata = manifest.to_h.fetch("metadata")
|
|
75
76
|
candidates = manifest.candidates.values
|
|
76
77
|
restored = metadata.slice("snapshot", "subjects", "complete", "issues").merge(
|
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
require "json"
|
|
5
5
|
require "tempfile"
|
|
6
6
|
|
|
7
|
+
require_relative "text"
|
|
8
|
+
|
|
7
9
|
module Saboteur
|
|
8
10
|
# Two fresh supervised children check the full suite without changing methods
|
|
9
11
|
# or writing caches. This samples stability; it cannot prove absence of flakes.
|
|
@@ -49,8 +51,9 @@ module Saboteur
|
|
|
49
51
|
|
|
50
52
|
def check_run(number, expected)
|
|
51
53
|
Tempfile.create("saboteur-baseline") do |log|
|
|
52
|
-
result = @supervisor.run(timeout: @config["baseline_timeout"]
|
|
53
|
-
|
|
54
|
+
result = @supervisor.run(timeout: @config["baseline_timeout"],
|
|
55
|
+
label: "baseline child died during setup/run") { |writer| record(writer, log.path) }
|
|
56
|
+
data = read_result(result, number, log)
|
|
54
57
|
verify_execution(data, number, expected)
|
|
55
58
|
raise Error, "baseline run #{number} left processes running; fix suite cleanup" if result.leftover_pids.any?
|
|
56
59
|
|
|
@@ -64,9 +67,12 @@ module Saboteur
|
|
|
64
67
|
raise Error, "baseline check failed: #{e.class}: #{e.message}; check suite hooks and setup"
|
|
65
68
|
end
|
|
66
69
|
|
|
67
|
-
def read_result(result, number)
|
|
70
|
+
def read_result(result, number, log = nil)
|
|
68
71
|
raise Error, "baseline run #{number} stopped: #{result.detail}" if result.stopped_by
|
|
69
|
-
|
|
72
|
+
if result.payload.empty?
|
|
73
|
+
raise Error, "baseline child died during setup/run: baseline run #{number} exited without a " \
|
|
74
|
+
"result#{exit_detail(result.exit_status)}; check suite boot and exits#{log_tail(log)}"
|
|
75
|
+
end
|
|
70
76
|
if result.child_failed?
|
|
71
77
|
raise Error, "baseline run #{number} exited unsuccessfully; fix suite cleanup and exit handlers"
|
|
72
78
|
end
|
|
@@ -77,6 +83,30 @@ module Saboteur
|
|
|
77
83
|
data
|
|
78
84
|
end
|
|
79
85
|
|
|
86
|
+
# The child's own output when it died without answering: the redirect
|
|
87
|
+
# puts its last words in this log, not on the terminal (#289). Bounded
|
|
88
|
+
# (about 2 KB, 20 lines) and scrubbed before it joins the error. Read
|
|
89
|
+
# through the parent's open handle, and strictly best-effort: a dead
|
|
90
|
+
# handle must never mask the real exit or signal error.
|
|
91
|
+
def log_tail(log)
|
|
92
|
+
return "" unless log
|
|
93
|
+
|
|
94
|
+
log.seek([log.size - 2048, 0].max)
|
|
95
|
+
tail = Text.utf8(log.read(2048).to_s).lines.last(20).map(&:chomp).join(" / ")
|
|
96
|
+
tail.empty? ? "" : "; child log tail: #{tail}"
|
|
97
|
+
rescue SystemCallError, IOError
|
|
98
|
+
""
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# The child's ending when the supervisor reaped one ("exit 1",
|
|
102
|
+
# "signal SIGKILL", ...): an empty result's cause is usually there (#255).
|
|
103
|
+
def exit_detail(status)
|
|
104
|
+
return "" unless status.is_a?(Process::Status)
|
|
105
|
+
return " (signal #{Signal.signame(status.termsig)})" if status.signaled?
|
|
106
|
+
|
|
107
|
+
" (exit #{status.exitstatus})"
|
|
108
|
+
end
|
|
109
|
+
|
|
80
110
|
def verify_execution(data, number, expected)
|
|
81
111
|
unless data["passed"] == true
|
|
82
112
|
raise Error, "baseline run #{number} failed: #{data['error'] || 'fix the unchanged suite'}"
|
|
@@ -93,8 +123,12 @@ module Saboteur
|
|
|
93
123
|
exception = example.execution_result.exception
|
|
94
124
|
next unless exception
|
|
95
125
|
|
|
96
|
-
|
|
97
|
-
|
|
126
|
+
# Exception text and example names can arrive locale-tagged or as
|
|
127
|
+
# raw path bytes (e.g. Errno::EILSEQ): scrub each field before any
|
|
128
|
+
# strip, truncation or interpolation - all three raise on bad tags.
|
|
129
|
+
# This detail ships in a JSON payload, so it must be valid UTF-8.
|
|
130
|
+
message = Text.utf8(exception.message.to_s).lines.map(&:strip).find { |line| !line.empty? }.to_s[0, 200]
|
|
131
|
+
"#{Text.utf8(example.id.to_s)}: #{Text.utf8(example.full_description.to_s)[0, 200]}: #{message}"
|
|
98
132
|
end.first(5).join("; ")
|
|
99
133
|
end
|
|
100
134
|
|