saboteur 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +367 -0
  3. data/README.md +166 -40
  4. data/exe/saboteur +34 -7
  5. data/lib/saboteur/baseline/workflow.rb +2 -1
  6. data/lib/saboteur/baseline_health.rb +40 -6
  7. data/lib/saboteur/cli/exit_status.rb +49 -4
  8. data/lib/saboteur/cli/help_options.rb +16 -0
  9. data/lib/saboteur/cli/presets.rb +36 -0
  10. data/lib/saboteur/cli/run_budget_options.rb +37 -0
  11. data/lib/saboteur/cli/run_reporting.rb +74 -5
  12. data/lib/saboteur/cli.rb +67 -46
  13. data/lib/saboteur/config.rb +66 -11
  14. data/lib/saboteur/crash_report.rb +154 -0
  15. data/lib/saboteur/custom_map.rb +3 -1
  16. data/lib/saboteur/doctor.rb +40 -1
  17. data/lib/saboteur/engine/baseline_guard.rb +79 -5
  18. data/lib/saboteur/engine/boot.rb +12 -1
  19. data/lib/saboteur/engine/canary.rb +41 -10
  20. data/lib/saboteur/engine/candidate_selection.rb +9 -1
  21. data/lib/saboteur/engine/coverage_guard.rb +29 -0
  22. data/lib/saboteur/engine/coverage_store.rb +48 -6
  23. data/lib/saboteur/engine/diagnostics.rb +23 -2
  24. data/lib/saboteur/engine/diff_scope.rb +24 -5
  25. data/lib/saboteur/engine/distribution.rb +124 -8
  26. data/lib/saboteur/engine/early_stop_recording.rb +46 -0
  27. data/lib/saboteur/engine/result_cache.rb +2 -1
  28. data/lib/saboteur/engine/run_budget.rb +63 -0
  29. data/lib/saboteur/engine/run_fingerprint.rb +4 -0
  30. data/lib/saboteur/engine/sandbox_policy.rb +18 -11
  31. data/lib/saboteur/engine/scheduler.rb +114 -18
  32. data/lib/saboteur/engine/subject_selection.rb +84 -5
  33. data/lib/saboteur/engine/suite_files.rb +3 -1
  34. data/lib/saboteur/engine/test_guess.rb +145 -22
  35. data/lib/saboteur/engine/thread_warnings.rb +53 -0
  36. data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
  37. data/lib/saboteur/engine/verdict_check.rb +54 -18
  38. data/lib/saboteur/engine.rb +166 -95
  39. data/lib/saboteur/execution/protocol.rb +2 -1
  40. data/lib/saboteur/execution/runner.rb +179 -17
  41. data/lib/saboteur/execution/sandbox.rb +6 -1
  42. data/lib/saboteur/execution/supervisor.rb +18 -3
  43. data/lib/saboteur/execution/worker_pool.rb +87 -7
  44. data/lib/saboteur/help.rb +55 -11
  45. data/lib/saboteur/ignore_list.rb +11 -1
  46. data/lib/saboteur/install.rb +4 -3
  47. data/lib/saboteur/integration/base.rb +7 -2
  48. data/lib/saboteur/integration/rspec.rb +40 -4
  49. data/lib/saboteur/mutation_source/builtin.rb +20 -11
  50. data/lib/saboteur/mutation_source/hazard_guard.rb +21 -3
  51. data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
  52. data/lib/saboteur/mutation_source/method_swap.rb +45 -0
  53. data/lib/saboteur/mutation_source/mutation.rb +5 -3
  54. data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
  55. data/lib/saboteur/mutation_source/operator_examples.rb +4 -1
  56. data/lib/saboteur/mutation_source/operator_registry.rb +4 -1
  57. data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
  58. data/lib/saboteur/preview.rb +2 -1
  59. data/lib/saboteur/reporter/agent_commands.rb +38 -13
  60. data/lib/saboteur/reporter/agent_incomplete.rb +27 -2
  61. data/lib/saboteur/reporter/next_steps.rb +14 -4
  62. data/lib/saboteur/reporter/score_history.rb +5 -1
  63. data/lib/saboteur/reporter/stryker.rb +13 -4
  64. data/lib/saboteur/reporter/survivor_hints.rb +3 -0
  65. data/lib/saboteur/reporter/terminal.rb +34 -10
  66. data/lib/saboteur/result.rb +22 -7
  67. data/lib/saboteur/selection/coverage_map.rb +2 -1
  68. data/lib/saboteur/selection/git_diff.rb +15 -5
  69. data/lib/saboteur/selection/untracked_changes.rb +126 -0
  70. data/lib/saboteur/subject.rb +11 -0
  71. data/lib/saboteur/text.rb +42 -0
  72. data/lib/saboteur/version.rb +1 -1
  73. data/lib/saboteur.rb +16 -1
  74. metadata +17 -2
data/README.md CHANGED
@@ -47,11 +47,17 @@ accepted mutants or change their scores. See [Opt-in survivor baseline](#opt-in-
47
47
 
48
48
  ### The unchanged suite is checked first
49
49
 
50
- Every run with selected subjects runs the loaded unchanged suite twice (the specs chosen with `--test`, if given) in fresh
50
+ Every run with selected subjects runs the loaded unchanged suite twice (the specs chosen with `--test` if given,
51
+ otherwise the inferred specs) in fresh
51
52
  supervised children before coverage lookup or incremental verdict reuse. This
52
53
  costs two suite runs even on a cache hit, plus a coverage-building suite run on
53
- a cache miss. Both baseline passes must be green, complete and in the same
54
- RSpec order at the run seed. A failure exits 3, stops mutation, prints the failing example ids and the first ten lines of the
54
+ a cache miss. Before the first pass, a run over the whole suite announces the example
55
+ count and the real number of full passes (a cached map only counts as
56
+ reused when it loads clean); once the first baseline pass is timed, the
57
+ run estimates the time left for the remaining baseline and coverage
58
+ passes from it - mutation time is not part of that estimate. Past 1,000
59
+ examples the announcement also names `--test` as the way to scope the run. Both baseline passes must be green, complete and in the same
60
+ RSpec order at the run seed. Set `c.baseline_passes = 1` (or `--baseline-passes 1`) for a faster PR loop: the run says so up front and gives up the order-stability comparison between passes (#55). A failure exits 3, stops mutation, prints the failing example ids and the first ten lines of the
55
61
  failure inline, and names the log and fix. Long phases
56
62
  print their elapsed time about every 30 seconds, with the examples finished so far, the percentage and a rough estimate of the time left (it assumes equally long examples) once the suite child has logged its first count, and each phase's log holds
57
63
  the suite's progress and full failure detail. Each suite pass
@@ -82,21 +88,24 @@ hazard guard is a syntactic filter. It recognizes destructive calls written
82
88
  as direct calls: `FileUtils.rm*`/`remove_entry*` (also as bare names under
83
89
  `include FileUtils`), `File.delete`/`unlink`, `Dir.rmdir`/`delete`/`unlink`,
84
90
  `rmtree`, `system`/`exec`/`spawn` (bare or on `Kernel`/`Process`), backticks
85
- and `%x`, `IO.popen`, any `Open3` method, and `open` with a leading `|`. It
91
+ and `%x`, `IO.popen`, any `Open3` method, `open` with a leading `|`, and
92
+ `send`/`public_send`/`__send__` with a literal symbol or string selector
93
+ naming one of these (dynamic selectors stay out of scope). It
86
94
  suppresses any mutant that would leave such a call in place while altering
87
95
  code inside it or anywhere else in the same method. A mutant that removes
88
96
  every hazardous call in the method still runs, provided its replacement
89
97
  introduces no new hazard. The match is by name on the written call: calls
90
- reached through aliases, variables, `send` or other indirection are not
98
+ reached through aliases, variables or other indirection are not
91
99
  recognized, so the guard cannot promise to find every destructive call. The
92
100
  summary reports how many mutants were suppressed. Extend or replace the list
93
101
  with `c.hazard` (same tag shapes: `"FileUtils.rm_rf"`, or a bare name like
94
102
  `"system"`); `--allow-hazardous` runs the suppressed mutants anyway.
95
103
 
96
104
  Behind the guard, a canary snapshots the tracked files (or every file,
97
- outside a repository) before mutating and re-checks after each method's
98
- mutants. If anything vanished, the run aborts with exit 3 and names the most
99
- recent mutant's digest. That check happens after the damage and restores
105
+ outside a repository) before mutating and probes the snapshot after every
106
+ mutant - a cheap stat pass, no tree walk. If anything vanished, the run
107
+ aborts with exit 3 and names the digest of the mutant that just ran, not
108
+ the method's last mutant. A fuller check still runs after each method. That check happens after the damage and restores
100
109
  nothing; it limits how far a bad mutant gets. Recover from your own backup,
101
110
  a fresh checkout or a reclone - a deleted `.git` or uncommitted work is gone
102
111
  unless you have another copy. The canary is the backstop for the destructive
@@ -104,8 +113,18 @@ forms the syntactic guard cannot see.
104
113
 
105
114
  On Linux 6.2 and later (Landlock ABI 3) each mutant child also runs under
106
115
  a Landlock sandbox: the child cannot create, write, rename, unlink or
107
- truncate files in the project tree, except `tmp/`, the cache dir and the
108
- system temp dir, plus anything in `c.sandbox_writable`. Every other path
116
+ truncate files in the project tree, except `tmp/` and the
117
+ system temp dir, plus anything in `c.sandbox_writable`. The project
118
+ `tmp/` grant applies only when `tmp/` already exists - the run never
119
+ creates it for the grant - so create it before the run if your suite
120
+ writes there, or grant the directory with `c.sandbox_writable`. The cache dir
121
+ is denied too - a mutant cannot delete the baseline, the reports or
122
+ other children's logs; its own log is opened before the fork and
123
+ arrives as an inherited descriptor. Grants are additive, though: a
124
+ cache dir placed inside `tmp/`, the system temp dir or a
125
+ `c.sandbox_writable` entry stays writable, and the tool cannot deny
126
+ what another grant allows - keep the cache outside every grant for
127
+ the isolation to bite. Every other path
109
128
  is denied for writing, `/dev/shm` and `/dev/tty` included; `/dev/null`
110
129
  stays writable. Inherited
111
130
  descriptors already open for writing outside the granted dirs are closed
@@ -123,9 +142,9 @@ grant that directory with `c.sandbox_writable` or opt out with
123
142
  sockets and IPC are never restricted - and a checkout inside the system
124
143
  temp dir cannot be protected, because everything under it stays writable
125
144
  so that real test suites keep working - the run warns when the effective
126
- write set (every grant realpath'd, the cache dir and the system temp dir
145
+ write set (every grant realpath'd, the system temp dir
127
146
  included) contains the project root, however it happened: a checkout
128
- inside the system temp dir, `cache_dir = "."`, a symlinked `tmp`, or an
147
+ inside the system temp dir, a symlinked `tmp`, or an
129
148
  overlapping `c.sandbox_writable` grant. `c.sandbox = "strict"` refuses to
130
149
  run at all in that situation, since the sandbox would protect nothing.
131
150
  The sandbox applies to every mutant
@@ -255,10 +274,12 @@ above the limit, with `exitCause` `maxTimeouts` or `maxErrors` in the JSON repor
255
274
  their own causes when they fail too. The gates cannot be combined with `--agent`.
256
275
  The GitHub Action has no input for them yet.
257
276
 
258
- Other flags: `--jobs N` (concurrent workers), `--timeout-factor N`,
277
+ Other flags: `--jobs N` (concurrent workers), `--timeout-factor N`, `--baseline-passes N`, `--max-duration DUR`,
259
278
  `--config PATH`, `--integration NAME`, `--reeval class_eval|nesting`,
260
279
  `--json PATH`, `--no-cache` (rebuild the coverage map), `--allow-focus`.
261
280
 
281
+ Two presets bundle the common loops (#244): `saboteur pr [REF]` is the pull-request loop - `--since REF` (default `main`), one baseline check and a ten-minute `--max-duration` that opts out of the budget-specific exit 4 (normal and incomplete verdicts still fail the run); `saboteur audit` is the whole project: `--all`, all operators, both baseline checks and both reports (an explicit `--test` replaces the injected `--all`; configured selection limits such as `since`, subjects, exclusions and a configured budget still apply). The run prints the expansion up front, and an explicit flag after the preset wins.
282
+
262
283
  A stray `fit`, `fdescribe` or `:focus` example, with `filter_run_when_matching :focus`, makes RSpec skip
263
284
  every other example. The baseline stays green and the score would cover only the focused ones, so
264
285
  saboteur exits 2 and names the focused examples. Remove the focus, or pass `--allow-focus` (config
@@ -269,24 +290,58 @@ check compares what RSpec runs with what it would run without the stray focus, s
269
290
 
270
291
  For pull requests, start with `--since` against your target branch. This is a
271
292
  workflow recommendation, not a CLI default: plain `run` still checks the whole
272
- configured project. To scope the specs as well, pass them with `--test` and no
293
+ configured project. To scope the specs yourself, pass them with `--test` and no
273
294
  source operands: `bundle exec saboteur run --since origin/main --test spec/billing_spec.rb`.
274
- With `--since` and no `--test`, the run warns before it starts that the two baseline passes
275
- run the loaded suite (a cold coverage cache adds one more pass), and how to
276
- scope them.
277
-
278
- `--since origin/main --guess-tests` does that scoping for you: it picks the spec
279
- named after each changed source file (`lib/a/b.rb` runs `spec/a/b_spec.rb`; `app/`
280
- files and `spec/lib/` work too) and runs only those for the baseline passes, the
281
- coverage pass and every mutant, as `--test` would. It is a naming guess, not
282
- coverage: a spec that reaches the changed file only through another file does not
283
- run: a mutant only that spec would kill can survive, and a line that none of the
284
- guessed specs executes is reported as NoCoverage. The agent JSON records the guessed specs as its
285
- `tests`, so its follow-up commands keep the same `--test` scope (with `--no-guess-tests`, so a `guess_tests`
286
- in the config file cannot conflict) instead of widening to the whole suite. If any changed file has no such
287
- spec, or nothing changed, the whole suite runs as before and the run says so.
288
- `--guess-tests` needs `--since` and cannot be combined with `--test`. Keep whole-project runs on main: diff mode does not check every method affected by a
289
- dependency change. [The pull-request guide](docs/pull-requests.md) covers
295
+ A changed Ruby file whose changed lines cover no method mutates nothing; the run
296
+ warns and names the file instead of silently selecting zero subjects. An untracked
297
+ file inside the configured `include` set counts as fully changed: a brand-new,
298
+ never-committed source file is selected in full rather than silently skipped,
299
+ because a diff cannot see it (#295). The union reads race-free
300
+ (O_NOFOLLOW/O_NONBLOCK, validated on the opened descriptor) and streams its
301
+ line count, so a swapped-in symlink or FIFO is skipped quietly and a huge
302
+ untracked fixture outside `include` is never even opened. Comment-only
303
+ or formatting-only changes within discovered methods are skipped without this
304
+ warning; the run reports those skipped methods. A changed Ruby file that an
305
+ include pattern covers but `c.exclude` removes mutates nothing either, and the
306
+ run warns and names up to five such files - narrow or remove the `c.exclude`
307
+ entry, or pass the file as a positional source operand (configured excludes are
308
+ then ignored). `--subject` cannot help here: it filters subjects inside the
309
+ files left after excludes.
310
+
311
+ Without `--test`, the run infers the specs: the spec named after each mutated
312
+ file (`lib/a/b.rb` runs `spec/a/b_spec.rb`; `app/` files and `spec/lib/` work
313
+ too) runs for the baseline passes, the coverage pass and every mutant, as
314
+ `--test` would. With `--since` only the changed files count; without it every
315
+ included file does. It is a naming guess, not coverage: a spec that reaches the
316
+ mutated file only through another file does not run, so a mutant only that
317
+ spec would kill can survive, and a line none of the inferred specs executes is
318
+ reported as NoCoverage. A file needs no inferred spec when it has no mutation
319
+ subjects (with `--since`: none on its changed lines), such as a comment-only
320
+ change within a method or a deletion leaving only blank/comment context; the
321
+ run says so and skips those files. When no file has a mutation subject at all,
322
+ the run reports that there is nothing to mutate. A `--subject` run infers only
323
+ from the named subjects' files: an unrelated spec-less file never stops a
324
+ one-method run, and a subject file with no conventional spec stops the run
325
+ like any other unmatched file. With `--since` only named subjects on changed
326
+ lines count: an unchanged subject file, one whose changes sit in other
327
+ methods, or one with a comment-only change is skipped, not stopped.
328
+
329
+ A file that needs a spec and has none stops the run before it boots (exit 2)
330
+ and names both ways out: `--test` with the specs to run, or `--all` for the
331
+ whole suite. An unmatched file no longer silently narrows the selected source
332
+ scope or widens the test run to the whole suite. Whole-project audits should
333
+ use `--all` explicitly. The agent JSON records the inferred specs as its
334
+ `tests`, so its follow-up commands keep the same scope.
335
+
336
+ `--all` (config `c.test_selection = :all`) runs the whole configured suite.
337
+ With `--since` or `--all` and no inferred or named specs, the run warns before
338
+ it starts that the two baseline passes run the loaded suite (a cold coverage
339
+ cache adds one more pass). `--guess-tests` is a deprecated alias for the
340
+ default inference and `--no-guess-tests` warns and is ignored; use `--all` for
341
+ the whole suite. `--all` cannot be combined with `--test` or `--guess-tests`,
342
+ and `--guess-tests` cannot be combined with `--test`. Keep whole-project runs
343
+ on main: diff mode does not check every method affected by a dependency change.
344
+ [The pull-request guide](docs/pull-requests.md) covers
290
345
  checkout history, a GitHub Actions example, configuration and limits.
291
346
 
292
347
  Doctor warns when an ignore subject matches no method discovered in the included
@@ -326,8 +381,21 @@ printed once as `path:line in Subject`, with the source line and the tests that
326
381
  covered it, then one compact line per mutation: `operator original -> replacement digest`,
327
382
  and a hint on what kind of test would kill it. The summary has the counts per
328
383
  status, the score, a table of scores per file (worst first) and a one-line
329
- legend. `--verbose` adds the score split, mean time, whole-body score and
330
- kill causes. Colour is on when stdout is a terminal and `NO_COLOR` is not set;
384
+ legend. When survivors remain, the Next steps footer prints the complete
385
+ replay command - ref, subjects, specs, sources, seed and the run settings
386
+ the agent protocol records, including the CLI-only switches that change
387
+ candidate availability or boot (`--allow-hazardous`, `--allow-focus`) -
388
+ shell-escaped and printed in full however long it gets, so running it
389
+ as-is replays the survivor with the same selection. Framework and
390
+ generation settings from the config file (arid/hazard patterns,
391
+ whole-file, require, rspec_args, rails) ride along through the command's
392
+ `--config` or the default config path. Only knobs that cannot change a
393
+ by-digest replay (`--max-mutants-per-method`, `--fail-fast`,
394
+ `--threshold`) are left out. The agent protocol instead omits a command
395
+ longer than 1024 bytes, and the agent rebuilds it from the run's
396
+ provenance. `--verbose` adds the score split, mean time, whole-body score and
397
+ kill causes, and prints one line per finished subject with its result
398
+ marks, so a survivor is attributable to a method before the run ends. Colour is on when stdout is a terminal and `NO_COLOR` is not set;
331
399
  `--color` and `--no-color` override that. While mutants run, stderr shows
332
400
  `done/total (percent) eta`: a live counter on a terminal, and dots with a
333
401
  counter every 50 mutants otherwise. Skipped and reused mutants are not
@@ -353,7 +421,13 @@ survivor's notes. The JSON report goes to `.saboteur/report.json` and a self-con
353
421
  report (one file, no scripts, no network) to `.saboteur/report.html`. Change
354
422
  the paths with `--json` / `--html` or `c.json_report` / `c.html_report` in
355
423
  `config/saboteur.rb`; set either to `nil` to turn that report off. The two paths
356
- must differ. The HTML page shows the covered mutation score
424
+ must differ. When an artifact cannot be written - a read-only tree inside a
425
+ sandboxed run - the HTML report and the score history warn and the run still
426
+ exits by its results. An explicitly requested JSON report (`--json` or
427
+ `c.json_report`) is different: it is the completion evidence an outer harness
428
+ trusts, so a failed write fails the run with exit 3 rather than leaving a
429
+ stale or partial file presented as this run's; the default
430
+ `.saboteur/report.json` stays best-effort. The HTML page shows the covered mutation score
357
431
  `(killed + timeout) / (killed + survived + timeout)`, which leaves NoCoverage,
358
432
  RanAtLoad and Unreachable out and keeps survivors accepted on the ignore list
359
433
  in the denominator. A tool
@@ -415,10 +489,33 @@ changed line and prints none.
415
489
 
416
490
  New to survivors? The [reading survivors guide](docs/guides/reading-survivors.md) walks one from the diff to the test that kills it.
417
491
 
418
- Exit codes: 0 no survivors (or all on the ignore list), or the score meets
419
- `--threshold`; 1 survivors, or the score is below `--threshold` or missing;
420
- 2 bad configuration or a missing source or test; 3 the tool failed (with
421
- `--threshold`, also any errored or resource-limited mutant).
492
+ Exit codes: 0 no survivors (or all on the ignore list), or the score
493
+ meets `--threshold`; 1 survivors, or the score is below `--threshold` or
494
+ missing; 2 bad configuration or a missing source or test; 3 the tool
495
+ failed (with `--threshold`, also any errored or resource-limited
496
+ mutant). Unexpected `StandardError` or `LoadError` failures print
497
+ `saboteur: internal error (...) - please report it` and exit 3, rather
498
+ than dying with a bare stack trace and an accidental exit code. The full
499
+ backtrace lands in `<cache_dir>/internal-error.log` (default
500
+ `.saboteur/`), and `--verbose` also prints it to stderr. When the cache
501
+ dir cannot hold the log, a private pid-keyed fallback in the system temp
502
+ dir takes it (created with `O_EXCL | O_NOFOLLOW`, mode 0600, so it never
503
+ follows or overwrites a pre-existing path). Logs are always written
504
+ private, and a nested run reads them back only through a no-follow,
505
+ non-blocking descriptor - a regular file, same owner, no group/other
506
+ bits (or, where the platform lacks those flags, only the `O_EXCL` temp
507
+ fallback is written and no excerpt is read). The message names the log only
508
+ after a successful write: an unset `cache_dir`, or a log writable nowhere,
509
+ leaves the one-liner standing alone. The executable
510
+ loads the crash reporter before the CLI, so a failure while it is
511
+ loading, being built, or parsing arguments gets the same report;
512
+ failures earlier than that - coverage setup or the reporter's own load -
513
+ get only the bare one-liner and still exit 3. Once the crash
514
+ reporter has loaded, agent mode reports the same failure as agent
515
+ JSON, using a standalone payload if the agent reporter has not
516
+ loaded. Before the crash reporter loads, only the one-line
517
+ diagnostic and exit 3 are available.
518
+ `--max-duration 10m` budgets the whole run, baselines included: the budget is checked before every subject and between mutants inside one, so past the budget the run finishes only the mutant in flight (bounded by its timeout), then stops before the next subject. Admitted phases (baseline checks, the coverage map build) are capped at the time left, as are verdict checks and timeout calibrations - admission is per check and per sequence, so nothing forks with no time left. A phase that hits only its budget-shortened cap stops the run as budget-exhausted rather than failing as a suite error. A kill or timeout the budget leaves uncheckable is reported as `unverified` - outside the score, never cached, listed under the early stop - instead of becoming an error, reports which selected subjects never ran and which mutants were omitted, marks the JSON and agent output partial (`earlyStop.reason: "max_duration"`), and exits 4; `--max-duration-ok` (or `c.max_duration_ok = true`) leaves the exit to the plain verdict instead, which can still fail the run - 1 on survivors, 3 when the budget stopped the run before any mutant verdict was verified. In agent mode the exit code stays verdict-driven and the budget shows up in `incomplete[]` and `earlyStop` (#245).
422
519
 
423
520
  ### Rejected kills
424
521
 
@@ -452,6 +549,9 @@ Saboteur.configure do |c|
452
549
  c.sandbox_writable = [] # extra dirs mutant children may write, e.g. ["db", "log"]
453
550
  c.ignore_file = "config/saboteur_ignore.rb"
454
551
  c.seed = 1 # Ruby's random seed in every test run
552
+ c.baseline_passes = 2 # unchanged-suite checks before mutating; 1 skips the order-stability check (#55)
553
+ c.max_duration = nil # wall-clock budget in seconds, e.g. 600; an exhausted budget exits 4 (#245)
554
+ c.max_duration_ok = false # true: an exhausted budget does not fail the run
455
555
  c.allow_focus = false # true runs a suite narrowed by fit/fdescribe/:focus instead of exit 2
456
556
  c.fingerprint_extra = [] # extra globs whose contents invalidate caches, e.g. ["data/*.csv"]
457
557
  end
@@ -670,6 +770,27 @@ An entry without a reason is rejected.
670
770
  saboteur needs Ruby's `Coverage` for itself. Disable SimpleCov for the run
671
771
  (most projects already guard it with an environment variable).
672
772
 
773
+ ### Threads alive at fork time
774
+
775
+ Threads do not survive `fork`, so a run names any that are alive when the
776
+ workers fork. Two are well known:
777
+
778
+ - `AR Pool Reaper` only maintains the parent's connection pool. It is
779
+ expected harmless when children reconnect the database after fork -
780
+ saboteur's rails mode does exactly that. Without verified
781
+ reconnection, do not assume the inherited connection state is safe.
782
+ - `DEBUGGER__::SESSION@server` is the debug gem's remote console. What
783
+ it does in children depends on `RUBY_DEBUG_FORK_MODE`: with `parent`
784
+ no debugger runs in children; with `child` or `both` (the gem's
785
+ default) the debugger activates in children and can pause or
786
+ interfere with supervised mutation runs. For unattended runs, do not
787
+ start the remote debugger at all: drop `RUBY_DEBUG_OPEN` or require
788
+ `debug` without `:open`.
789
+
790
+ Any other thread gets a generic caveat: a thread that does not run in a
791
+ child cannot do background work a mutant's test may rely on, so results
792
+ can differ when one is left behind.
793
+
673
794
  ## Filter mutations
674
795
 
675
796
  The built-in source also swaps `select` / `reject` and `select!` / `reject!`.
@@ -732,6 +853,10 @@ and `tap` links are never removed. Tests can observe frozenness, copies and a
732
853
  query chain, and dropping such a link mostly yields noise, so this operator
733
854
  leaves them out. Arid statements and nested definitions are skipped.
734
855
 
856
+ If you confirm the shorter chain is equivalent across the intended
857
+ inputs (for example a duplicated column), simplify the code or ignore
858
+ the mutant.
859
+
735
860
  Chain removal is on by default, like `argument_removal` and `pair_removal`.
736
861
  Its proposals follow the other fine proposals, with the same validation,
737
862
  deduplication and `max_mutants_per_method` cap.
@@ -786,11 +911,12 @@ Public API, stable across minor versions:
786
911
  - `Saboteur::Integration::Base` for other test frameworks (RSpec is the only one
787
912
  that ships).
788
913
  - `Saboteur.hook(:before_boot | :after_boot | :before_fork | :after_fork |
789
- :around_mutant)` in files listed under `hooks:`.
914
+ :around_mutant)` in files listed under `hooks:`. An extensionless hooks
915
+ path resolves its `.rb`, the way `require` does.
790
916
 
791
917
  ## 🌍 Where it has run
792
918
 
793
- An early release (0.4.0). Tested on the RSpec suites of plain gems
919
+ Tested on the RSpec suites of plain gems
794
920
  (dry-inflector, tty-color, hashie, addressable) and of two Rails 8.1 apps:
795
921
  human-essentials on Postgres (4,080 mutants, 6 workers; the 2026-10-10 run, before
796
922
  the baseline check and the coarse-gate removal) and lobsters on SQLite,
@@ -955,7 +1081,7 @@ Cache and baseline identities include the policy. Re-propose and review old surv
955
1081
 
956
1082
  Trust boundary: `--dry-run` never boots the app or runs its tests, but custom `sources` entries (`path.rb:ClassName`) are still `require`d to generate candidates, so their code runs. Only preview with sources you trust.
957
1083
 
958
- `bundle exec saboteur run --list-operators` lists every builtin name and description. All builtin operators run by default, including [`range`](docs/operators/range.md), [`operator_assignment`](docs/operators/operator_assignment.md), and [`condition_negation`](docs/operators/condition_negation.md). `--operators arithmetic,comparison` or `c.operators = ["arithmetic", "comparison"]` still selects a subset; `all` selects every name. There is no opt-in tier. `--exclude-operators range,condition_negation` or `c.exclude_operators = ["range", "condition_negation"]` removes named operators from that selection; exclusion wins if a name is in both lists. An empty selection means all, while an empty exclusion means none. CLI lists replace their corresponding config lists. These controls apply to builtin operators, not custom source plugins. Selection and exclusions invalidate cached verdict identity and are carried into agent rerun/replay commands.
1084
+ `bundle exec saboteur run --list-operators` lists every builtin name and description. All builtin operators run by default, including [`range`](docs/operators/range.md), [`operator_assignment`](docs/operators/operator_assignment.md), [`condition_negation`](docs/operators/condition_negation.md), [`method_swap`](docs/operators/method_swap.md), [`operand_replacement`](docs/operators/operand_replacement.md), and [`interpolation_removal`](docs/operators/interpolation_removal.md). `--operators arithmetic,comparison` or `c.operators = ["arithmetic", "comparison"]` still selects a subset; `all` selects every name. There is no opt-in tier. `--exclude-operators range,condition_negation` or `c.exclude_operators = ["range", "condition_negation"]` removes named operators from that selection; exclusion wins if a name is in both lists. An empty selection means all, while an empty exclusion means none. CLI lists replace their corresponding config lists. These controls apply to builtin operators, not custom source plugins. Selection and exclusions invalidate cached verdict identity and are carried into agent rerun/replay commands.
959
1085
 
960
1086
 
961
1087
  All operators run unless you select a smaller set; exclusions always remove operators from that set.
data/exe/saboteur CHANGED
@@ -2,15 +2,42 @@
2
2
  # SPDX-License-Identifier: MIT
3
3
  # frozen_string_literal: true
4
4
 
5
- # Line coverage must be set up before any target code loads, so it happens
6
- # first. For `run` it also counts from here: what executes while the app and
5
+ # Coverage comes first because it only measures files loaded after it -
6
+ # the crash reporter must stay measurable - and it loads inside the same
7
+ # guarded begin as the reporter: every later failure, even while the CLI
8
+ # is being required, gets the full report (the bounded one-liner, the log
9
+ # file, the --verbose backtrace and agent JSON when the CLI loaded far
10
+ # enough). If even the reporter cannot load, a bare normalized one-liner
11
+ # (one line, invalid bytes scrubbed, bounded length) still exits 3.
12
+ begin
13
+ require "coverage"
14
+ Coverage.setup(lines: true, methods: true, branches: true) if Coverage.state == :idle
15
+
16
+ require "saboteur/crash_report"
17
+ rescue LoadError, StandardError => e
18
+ message = e.message.to_s.dup.force_encoding(Encoding::UTF_8).scrub("?").gsub(/\s+/, " ").strip
19
+ message = "#{message[0...160]}..." if message.length > 160
20
+ warn "saboteur: internal error (#{e.class}: #{message}) - please report it: " \
21
+ "https://github.com/vamsipavanmahesh/saboteur/issues"
22
+ exit 3
23
+ end
24
+
25
+ # For `run` coverage counts from here: what executes while the app and
7
26
  # spec files load is how ran-at-load mutants are told apart from uncovered
8
27
  # ones. The engine suspends counting right after boot and reads the
9
28
  # counters as the map's load lines (a forked child would not inherit them).
10
- require "coverage"
11
- Coverage.setup(lines: true, methods: true, branches: true) if Coverage.state == :idle
12
- Coverage.resume if ARGV.first == "run" && Coverage.state == :suspended
29
+ begin
30
+ Coverage.resume if ARGV.first == "run" && Coverage.state == :suspended
13
31
 
14
- require "saboteur/cli"
32
+ require "saboteur/cli"
15
33
 
16
- exit Saboteur::CLI.new(ARGV).call
34
+ exit Saboteur::CLI.new(ARGV).call
35
+ rescue LoadError, StandardError => e
36
+ # Agent mode is visible in the raw arguments; the CLI may never have
37
+ # loaded, so this scan must not depend on it.
38
+ agent = ARGV.include?("--agent") || ARGV.include?("--format=agent") ||
39
+ ARGV.each_cons(2).any? { |flag, value| flag == "--format" && value == "agent" }
40
+ exit Saboteur::CrashReport.call(e, verbose: ARGV.include?("--verbose"), agent: agent,
41
+ err: $stderr, out: $stdout,
42
+ log_path: File.join(Dir.pwd, ".saboteur", "internal-error.log"))
43
+ end
@@ -4,6 +4,7 @@
4
4
  require_relative "acceptance"
5
5
  require_relative "comparison"
6
6
  require_relative "../engine"
7
+ require_relative "../text"
7
8
 
8
9
  module Saboteur
9
10
  module Baseline
@@ -70,7 +71,7 @@ module Saboteur
70
71
 
71
72
  bytes = File.read(path)
72
73
  @baseline_digest = Digest::SHA256.hexdigest(bytes)
73
- manifest = Manifest.new(JSON.parse(bytes))
74
+ manifest = Manifest.new(JSON.parse(Text.utf8_strict(bytes)))
74
75
  metadata = manifest.to_h.fetch("metadata")
75
76
  candidates = manifest.candidates.values
76
77
  restored = metadata.slice("snapshot", "subjects", "complete", "issues").merge(
@@ -4,6 +4,8 @@
4
4
  require "json"
5
5
  require "tempfile"
6
6
 
7
+ require_relative "text"
8
+
7
9
  module Saboteur
8
10
  # Two fresh supervised children check the full suite without changing methods
9
11
  # or writing caches. This samples stability; it cannot prove absence of flakes.
@@ -49,8 +51,9 @@ module Saboteur
49
51
 
50
52
  def check_run(number, expected)
51
53
  Tempfile.create("saboteur-baseline") do |log|
52
- result = @supervisor.run(timeout: @config["baseline_timeout"]) { |writer| record(writer, log.path) }
53
- data = read_result(result, number)
54
+ result = @supervisor.run(timeout: @config["baseline_timeout"],
55
+ label: "baseline child died during setup/run") { |writer| record(writer, log.path) }
56
+ data = read_result(result, number, log)
54
57
  verify_execution(data, number, expected)
55
58
  raise Error, "baseline run #{number} left processes running; fix suite cleanup" if result.leftover_pids.any?
56
59
 
@@ -64,9 +67,12 @@ module Saboteur
64
67
  raise Error, "baseline check failed: #{e.class}: #{e.message}; check suite hooks and setup"
65
68
  end
66
69
 
67
- def read_result(result, number)
70
+ def read_result(result, number, log = nil)
68
71
  raise Error, "baseline run #{number} stopped: #{result.detail}" if result.stopped_by
69
- raise Error, "baseline run #{number} exited without a result; check suite boot and exits" if result.payload.empty?
72
+ if result.payload.empty?
73
+ raise Error, "baseline child died during setup/run: baseline run #{number} exited without a " \
74
+ "result#{exit_detail(result.exit_status)}; check suite boot and exits#{log_tail(log)}"
75
+ end
70
76
  if result.child_failed?
71
77
  raise Error, "baseline run #{number} exited unsuccessfully; fix suite cleanup and exit handlers"
72
78
  end
@@ -77,6 +83,30 @@ module Saboteur
77
83
  data
78
84
  end
79
85
 
86
+ # The child's own output when it died without answering: the redirect
87
+ # puts its last words in this log, not on the terminal (#289). Bounded
88
+ # (about 2 KB, 20 lines) and scrubbed before it joins the error. Read
89
+ # through the parent's open handle, and strictly best-effort: a dead
90
+ # handle must never mask the real exit or signal error.
91
+ def log_tail(log)
92
+ return "" unless log
93
+
94
+ log.seek([log.size - 2048, 0].max)
95
+ tail = Text.utf8(log.read(2048).to_s).lines.last(20).map(&:chomp).join(" / ")
96
+ tail.empty? ? "" : "; child log tail: #{tail}"
97
+ rescue SystemCallError, IOError
98
+ ""
99
+ end
100
+
101
+ # The child's ending when the supervisor reaped one ("exit 1",
102
+ # "signal SIGKILL", ...): an empty result's cause is usually there (#255).
103
+ def exit_detail(status)
104
+ return "" unless status.is_a?(Process::Status)
105
+ return " (signal #{Signal.signame(status.termsig)})" if status.signaled?
106
+
107
+ " (exit #{status.exitstatus})"
108
+ end
109
+
80
110
  def verify_execution(data, number, expected)
81
111
  unless data["passed"] == true
82
112
  raise Error, "baseline run #{number} failed: #{data['error'] || 'fix the unchanged suite'}"
@@ -93,8 +123,12 @@ module Saboteur
93
123
  exception = example.execution_result.exception
94
124
  next unless exception
95
125
 
96
- message = exception.message.to_s.lines.map(&:strip).find { |line| !line.empty? }.to_s[0, 200]
97
- "#{example.id}: #{example.full_description.to_s[0, 200]}: #{message}"
126
+ # Exception text and example names can arrive locale-tagged or as
127
+ # raw path bytes (e.g. Errno::EILSEQ): scrub each field before any
128
+ # strip, truncation or interpolation - all three raise on bad tags.
129
+ # This detail ships in a JSON payload, so it must be valid UTF-8.
130
+ message = Text.utf8(exception.message.to_s).lines.map(&:strip).find { |line| !line.empty? }.to_s[0, 200]
131
+ "#{Text.utf8(example.id.to_s)}: #{Text.utf8(example.full_description.to_s)[0, 200]}: #{message}"
98
132
  end.first(5).join("; ")
99
133
  end
100
134
 
@@ -8,7 +8,26 @@ module Saboteur
8
8
  module ExitStatus
9
9
  private
10
10
 
11
+ # Last resort: a bug must not escape as a bare stack trace and an accidental exit 1.
12
+ def internal_error(error)
13
+ CrashReport.call(error, verbose: @verbose, agent: agent?, err: @err, out: @out, log_path: crash_log_path)
14
+ end
15
+
16
+ # The crash log lives with the other run state. A broken config (or a
17
+ # nil cache_dir) must not swallow the crash report itself, so fall
18
+ # back to no log.
19
+ def crash_log_path
20
+ dir = config.path("cache_dir")
21
+ dir && File.join(dir, CrashReport::LOG_NAME)
22
+ rescue StandardError
23
+ nil
24
+ end
25
+
11
26
  def exit_code(report)
27
+ return EXIT_BUDGET if budget_exhausted?(report)
28
+
29
+ return EXIT_TOOL if untested_partial?(report)
30
+
12
31
  return EXIT_TOOL if mostly_errored?(report)
13
32
 
14
33
  return EXIT_TOOL if incomplete_under_threshold?(report)
@@ -16,6 +35,24 @@ module Saboteur
16
35
  failed_quality_gate?(report) || gates.failed?(report, @err) ? EXIT_SURVIVORS : EXIT_OK
17
36
  end
18
37
 
38
+ # A spent --max-duration budget exits 4 on its own, before the gates:
39
+ # the run is partial by design, and --max-duration-ok opts back into
40
+ # the plain verdict.
41
+ def budget_exhausted?(report)
42
+ report.early_stop&.reason == "max_duration" && !config["max_duration_ok"]
43
+ end
44
+
45
+ # A run stopped before any mutant was tested has no verdict to fall
46
+ # back on: --max-duration-ok must not turn it green. Unverified
47
+ # results are not tested verdicts, so an unverified-only stop has no
48
+ # score either (#296). Exit 3, like the other incomplete shapes.
49
+ def untested_partial?(report)
50
+ return false unless report.early_stop && report.score_breakdown.tested.zero?
51
+
52
+ @err.puts("saboteur: the run stopped before any mutant was tested, so there is no score to judge")
53
+ true
54
+ end
55
+
19
56
  def gates = (@gates ||= Gates.new(config))
20
57
 
21
58
  def mostly_errored?(report)
@@ -33,9 +70,11 @@ module Saboteur
33
70
  # incomplete run that mostly errored keeps its distinct cause, so the
34
71
  # harness rule means the same thing in both modes.
35
72
  def agent_exit_cause(report, payload)
73
+ return "budgetExhausted" if budget_exhausted?(report)
74
+
36
75
  case payload[:verdict]
37
76
  when "clean" then "ok"
38
- when "survivors" then report.early_stop ? "failFast" : "survivors"
77
+ when "survivors" then report.early_stop&.reason == "fail_fast" ? "failFast" : "survivors"
39
78
  else report.majority_errored? ? "errorMajority" : "incomplete"
40
79
  end
41
80
  end
@@ -45,22 +84,28 @@ module Saboteur
45
84
  def exit_cause(report, code)
46
85
  case code
47
86
  when EXIT_OK then "ok"
87
+ when EXIT_BUDGET then "budgetExhausted"
48
88
  when EXIT_SURVIVORS then survivors_cause(report)
49
- else report.majority_errored? ? "errorMajority" : "incompleteUnderThreshold"
89
+ else
90
+ if report.majority_errored?
91
+ "errorMajority"
92
+ else
93
+ report.early_stop && report.score_breakdown.tested.zero? ? "incomplete" : "incompleteUnderThreshold"
94
+ end
50
95
  end
51
96
  end
52
97
 
53
98
  # A run stopped by --fail-fast always exits 1: it stopped on a survivor, and
54
99
  # its score covers only the subjects that ran, so no threshold can pass it.
55
100
  def survivors_cause(report)
56
- return "failFast" if report.early_stop
101
+ return "failFast" if report.early_stop&.reason == "fail_fast"
57
102
 
58
103
  gates.cause || (config["threshold"].nil? ? "survivors" : "belowThreshold")
59
104
  end
60
105
 
61
106
  # With --threshold the score decides; without it any unignored survivor does.
62
107
  def failed_quality_gate?(report)
63
- return true if report.early_stop
108
+ return true if report.early_stop&.reason == "fail_fast"
64
109
 
65
110
  threshold = config["threshold"]
66
111
  return report.unignored_survivors.any? if threshold.nil?
@@ -0,0 +1,16 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # The option map `saboteur help` prints per command. run, pr and audit
7
+ # share the run parser: the presets take every run option.
8
+ module HelpOptions
9
+ def option_lines
10
+ run = parser.summarize
11
+ { "run" => run, "pr" => run, "audit" => run, "doctor" => doctor_parser.summarize,
12
+ "baseline" => baseline_parser.summarize }
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,36 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # Opinionated bundles (#244): pr is the fast changed-lines loop (one
7
+ # baseline check, a 10-minute budget that does not fail the build),
8
+ # audit is the whole project with both reports. Preset flags go first,
9
+ # so an explicit flag after them wins on the usual last-flag rule.
10
+ module Presets
11
+ PRESETS = {
12
+ "pr" => %w[--baseline-passes 1 --max-duration 10m --max-duration-ok],
13
+ "audit" => %w[--all --operators all --baseline-passes 2 --json .saboteur/audit-report.json
14
+ --html .saboteur/audit-report.html]
15
+ }.freeze
16
+
17
+ private
18
+
19
+ def preset(name)
20
+ flags = PRESETS.fetch(name).dup
21
+ if name == "pr"
22
+ ref = @argv.first.nil? || @argv.first.start_with?("-") ? "main" : @argv.shift
23
+ flags += ["--since", ref]
24
+ end
25
+ # An explicit scope beats the injected whole-suite --all, which
26
+ # would otherwise reject the combination: named tests, and the
27
+ # deprecated inference alias (#245).
28
+ explicit_scope = @argv.any? { |arg| arg == "--test" || arg.start_with?("--test=") || arg == "--guess-tests" }
29
+ flags.delete("--all") if explicit_scope
30
+ @err.puts("saboteur: preset '#{name}': #{flags.join(' ')} (explicit flags override)")
31
+ @argv = flags + @argv
32
+ run_command
33
+ end
34
+ end
35
+ end
36
+ end