evilution 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +41 -0
- data/CHANGELOG.md +65 -0
- data/README.md +149 -13
- data/docs/architecture.md +20 -5
- data/docs/isolation.md +3 -4
- data/exe/evil +8 -1
- data/exe/evilution +8 -1
- data/lib/evilution/ast/local_reads.rb +43 -0
- data/lib/evilution/cli/exit_guard.rb +59 -0
- data/lib/evilution/cli/parser/options_builder.rb +1 -0
- data/lib/evilution/cli.rb +1 -0
- data/lib/evilution/config.rb +3 -2
- data/lib/evilution/diagnostic.rb +22 -0
- data/lib/evilution/integration/loading/concern_state_cleaner.rb +20 -4
- data/lib/evilution/integration/loading/redefinition_recovery.rb +1 -1
- data/lib/evilution/integration/loading/reeval_warning_filter.rb +72 -0
- data/lib/evilution/integration/loading/source_evaluator.rb +4 -1
- data/lib/evilution/integration/minitest.rb +2 -1
- data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +9 -1
- data/lib/evilution/integration/rspec/state_guard/configuration_streams.rb +4 -1
- data/lib/evilution/integration/rspec/unresolved_spec_warner.rb +2 -1
- data/lib/evilution/integration/rspec.rb +48 -2
- data/lib/evilution/integration/test_unit/test_file_resolver.rb +2 -1
- data/lib/evilution/isolation/fork.rb +29 -8
- data/lib/evilution/mcp/complete_result_server.rb +41 -0
- data/lib/evilution/mcp/server.rb +2 -1
- data/lib/evilution/mutator/base.rb +14 -2
- data/lib/evilution/mutator/operator/block_destructuring_expansion.rb +85 -0
- data/lib/evilution/mutator/operator/block_parameter_drop.rb +96 -0
- data/lib/evilution/mutator/operator/boolean_expression_to_nil.rb +22 -0
- data/lib/evilution/mutator/operator/boolean_operand_promotion.rb +31 -0
- data/lib/evilution/mutator/operator/case_in.rb +64 -0
- data/lib/evilution/mutator/operator/case_when.rb +72 -1
- data/lib/evilution/mutator/operator/conditional_branch.rb +20 -7
- data/lib/evilution/mutator/operator/forwarding_super_to_explicit.rb +71 -0
- data/lib/evilution/mutator/operator/if_branch_swap.rb +48 -0
- data/lib/evilution/mutator/operator/loop_body_to_raise.rb +68 -0
- data/lib/evilution/mutator/operator/method_body_replacement.rb +10 -1
- data/lib/evilution/mutator/operator/method_body_to_raise.rb +59 -0
- data/lib/evilution/mutator/operator/method_body_to_super.rb +150 -0
- data/lib/evilution/mutator/operator/optional_default_injection.rb +71 -0
- data/lib/evilution/mutator/operator/optional_parameter_to_required.rb +41 -0
- data/lib/evilution/mutator/operator/pattern_predicate.rb +28 -0
- data/lib/evilution/mutator/operator/typed_default_return.rb +84 -0
- data/lib/evilution/mutator/primitives.rb +52 -0
- data/lib/evilution/mutator/registry.rb +14 -0
- data/lib/evilution/process_supervisor.rb +20 -8
- data/lib/evilution/reporter/cli/item_formatters/neutral_group.rb +23 -0
- data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +42 -0
- data/lib/evilution/reporter/cli/item_formatters/subject_score_group.rb +16 -0
- data/lib/evilution/reporter/cli/line_formatters/infra_retry_notice.rb +19 -0
- data/lib/evilution/reporter/cli/line_formatters/result_line.rb +27 -3
- data/lib/evilution/reporter/cli/line_formatters/score.rb +19 -1
- data/lib/evilution/reporter/cli/line_formatters/unresolved_targets.rb +35 -0
- data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
- data/lib/evilution/reporter/cli/trailer.rb +11 -7
- data/lib/evilution/reporter/cli.rb +20 -4
- data/lib/evilution/reporter/json/subjects.rb +29 -0
- data/lib/evilution/reporter/json.rb +23 -1
- data/lib/evilution/result/mutation_result.rb +3 -2
- data/lib/evilution/result/neutral_reason.rb +35 -0
- data/lib/evilution/result/subject_score.rb +28 -0
- data/lib/evilution/result/subject_scorer.rb +37 -0
- data/lib/evilution/result/summary.rb +44 -2
- data/lib/evilution/runner/canary.rb +52 -5
- data/lib/evilution/runner/mutation_executor/infra_retry.rb +54 -0
- data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +9 -3
- data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +18 -1
- data/lib/evilution/runner/mutation_executor/result_cache.rb +11 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +19 -1
- data/lib/evilution/runner/mutation_executor.rb +32 -4
- data/lib/evilution/runner/report_publisher.rb +31 -9
- data/lib/evilution/runner/target_spec_audit.rb +43 -0
- data/lib/evilution/runner.rb +10 -1
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution.rb +18 -0
- data/scripts/compare_targeting +4 -2
- metadata +33 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 4fe2a5694f081965211f6471734e4f1b11d4e02d2a33617b19a0e6fb3b3c685f
|
|
4
|
+
data.tar.gz: 3e2637aa9ebb40904cb8def655dc2b5df7e9e0727f22add31267e8183553fe79
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 530e85eaa20e6d6fa18a6b633d4fdca79c5b55a6ca5515f6a0e79951ece9582b077a14921fc9217df3d96567974a256df9b27deb46d014da263fc0cee253da1a
|
|
7
|
+
data.tar.gz: 0a4ba7c748b0fa7f7c664cd0a25e7ed68f3d3c59e51536185fbc869c81b20b2d77f6391fe6b6aabe3ae3a848d6ffa3a70ef5d07760d28e520426b871daa97100
|
data/.beads/interactions.jsonl
CHANGED
|
@@ -448,3 +448,44 @@
|
|
|
448
448
|
{"id":"int-616f39e1","kind":"field_change","created_at":"2026-07-15T13:33:53.68007233Z","actor":"Denis Kiselev","issue_id":"EV-y194","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
449
449
|
{"id":"int-abcaeb3a","kind":"field_change","created_at":"2026-07-15T13:33:54.363899883Z","actor":"Denis Kiselev","issue_id":"EV-j2kz","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
450
450
|
{"id":"int-b8027311","kind":"field_change","created_at":"2026-07-15T13:34:47.737536249Z","actor":"Denis Kiselev","issue_id":"EV-w07i","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}
|
|
451
|
+
{"id":"int-e7b1f137","kind":"field_change","created_at":"2026-08-14T05:36:05.821384373Z","actor":"Denis Kiselev","issue_id":"EV-4ild","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
452
|
+
{"id":"int-c003e7fa","kind":"field_change","created_at":"2026-08-22T04:23:28.006710286Z","actor":"Denis Kiselev","issue_id":"EV-170m.1","extra":{"field":"status","new_value":"in_progress","old_value":"open"}}
|
|
453
|
+
{"id":"int-f912a48c","kind":"field_change","created_at":"2026-08-22T15:26:32.589708305Z","actor":"Denis Kiselev","issue_id":"EV-170m.1","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
454
|
+
{"id":"int-d74d2ac8","kind":"field_change","created_at":"2026-08-22T17:21:54.118527965Z","actor":"Denis Kiselev","issue_id":"EV-170m.2","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
455
|
+
{"id":"int-d694451a","kind":"field_change","created_at":"2026-08-22T17:21:59.468788556Z","actor":"Denis Kiselev","issue_id":"EV-170m.3","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
456
|
+
{"id":"int-9cd4dc49","kind":"field_change","created_at":"2026-08-22T17:22:05.341826256Z","actor":"Denis Kiselev","issue_id":"EV-170m.5","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
457
|
+
{"id":"int-9a64fd8d","kind":"field_change","created_at":"2026-08-23T05:54:32.093591437Z","actor":"Denis Kiselev","issue_id":"EV-170m.14","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
458
|
+
{"id":"int-990ddf2a","kind":"field_change","created_at":"2026-08-23T05:54:35.35056636Z","actor":"Denis Kiselev","issue_id":"EV-170m.15","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
459
|
+
{"id":"int-aa07e9f8","kind":"field_change","created_at":"2026-08-23T05:54:38.655416932Z","actor":"Denis Kiselev","issue_id":"EV-170m.16","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
460
|
+
{"id":"int-e7ba9d2f","kind":"field_change","created_at":"2026-08-23T05:54:41.957571538Z","actor":"Denis Kiselev","issue_id":"EV-170m.10","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
461
|
+
{"id":"int-c4cf46e8","kind":"field_change","created_at":"2026-08-23T05:54:43.220109392Z","actor":"Denis Kiselev","issue_id":"EV-170m.11","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
462
|
+
{"id":"int-46ac995b","kind":"field_change","created_at":"2026-08-23T05:54:46.601240327Z","actor":"Denis Kiselev","issue_id":"EV-170m.12","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
463
|
+
{"id":"int-ee1da7ed","kind":"field_change","created_at":"2026-08-23T05:54:58.551793942Z","actor":"Denis Kiselev","issue_id":"EV-170m.8","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
464
|
+
{"id":"int-a2659f70","kind":"field_change","created_at":"2026-08-23T06:17:48.940199465Z","actor":"Denis Kiselev","issue_id":"EV-zfh6","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
465
|
+
{"id":"int-3ef2119d","kind":"field_change","created_at":"2026-08-23T06:22:32.612716343Z","actor":"Denis Kiselev","issue_id":"EV-9mrs","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
466
|
+
{"id":"int-4f24cfc1","kind":"field_change","created_at":"2026-08-23T06:22:36.754337802Z","actor":"Denis Kiselev","issue_id":"EV-170m.4","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}
|
|
467
|
+
{"id":"int-db1a15c4","kind":"field_change","created_at":"2026-08-23T06:22:40.226188047Z","actor":"Denis Kiselev","issue_id":"EV-170m.6","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}
|
|
468
|
+
{"id":"int-ee6c5593","kind":"field_change","created_at":"2026-08-23T06:22:43.665223358Z","actor":"Denis Kiselev","issue_id":"EV-170m.7","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}
|
|
469
|
+
{"id":"int-a9687ce8","kind":"field_change","created_at":"2026-08-23T06:22:47.016572566Z","actor":"Denis Kiselev","issue_id":"EV-170m.9","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}
|
|
470
|
+
{"id":"int-6979f453","kind":"field_change","created_at":"2026-08-23T06:22:50.346314707Z","actor":"Denis Kiselev","issue_id":"EV-170m.13","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}
|
|
471
|
+
{"id":"int-c0c4ce61","kind":"field_change","created_at":"2026-08-23T06:22:53.658409885Z","actor":"Denis Kiselev","issue_id":"EV-170m.17","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}
|
|
472
|
+
{"id":"int-9fe2b310","kind":"field_change","created_at":"2026-08-23T15:26:14.173757324Z","actor":"Denis Kiselev","issue_id":"EV-secd","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}
|
|
473
|
+
{"id":"int-e3df2816","kind":"field_change","created_at":"2026-08-23T15:26:14.758723969Z","actor":"Denis Kiselev","issue_id":"EV-v2rc","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}
|
|
474
|
+
{"id":"int-4964c024","kind":"field_change","created_at":"2026-08-23T16:53:26.159738103Z","actor":"Denis Kiselev","issue_id":"EV-df7u","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
475
|
+
{"id":"int-09aacdba","kind":"field_change","created_at":"2026-08-23T17:31:02.425168724Z","actor":"Denis Kiselev","issue_id":"EV-65nf","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
476
|
+
{"id":"int-f248abda","kind":"field_change","created_at":"2026-09-19T02:33:35.442608624Z","actor":"Denis Kiselev","issue_id":"EV-0y0p.1","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
477
|
+
{"id":"int-2295820f","kind":"field_change","created_at":"2026-09-19T05:36:52.469484387Z","actor":"Denis Kiselev","issue_id":"EV-0y0p.2","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
478
|
+
{"id":"int-c34dface","kind":"field_change","created_at":"2026-09-19T06:40:49.582010237Z","actor":"Denis Kiselev","issue_id":"EV-0y0p.3","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
479
|
+
{"id":"int-b54a1b1b","kind":"field_change","created_at":"2026-09-19T09:16:07.408034725Z","actor":"Denis Kiselev","issue_id":"EV-0y0p.7","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}
|
|
480
|
+
{"id":"int-d07f50fb","kind":"field_change","created_at":"2026-09-20T05:54:55.207588665Z","actor":"Denis Kiselev","issue_id":"EV-0y0p.4","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
481
|
+
{"id":"int-8c521e96","kind":"field_change","created_at":"2026-09-20T07:27:33.837283926Z","actor":"Denis Kiselev","issue_id":"EV-0y0p.5","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
482
|
+
{"id":"int-0082d40c","kind":"field_change","created_at":"2026-09-20T11:10:19.819992902Z","actor":"Denis Kiselev","issue_id":"EV-0y0p.6","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}
|
|
483
|
+
{"id":"int-b1fbb9d9","kind":"field_change","created_at":"2026-09-20T12:09:22.863657064Z","actor":"Denis Kiselev","issue_id":"EV-0y0p.8","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}
|
|
484
|
+
{"id":"int-0341459f","kind":"field_change","created_at":"2026-09-20T15:42:35.81801432Z","actor":"Denis Kiselev","issue_id":"EV-p4sm","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
485
|
+
{"id":"int-37d6d94c","kind":"field_change","created_at":"2026-09-20T16:28:47.475124108Z","actor":"Denis Kiselev","issue_id":"EV-39t1","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
486
|
+
{"id":"int-3c77e0d2","kind":"field_change","created_at":"2026-09-20T17:08:50.170870939Z","actor":"Denis Kiselev","issue_id":"EV-j0bv","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
487
|
+
{"id":"int-9a5aa97e","kind":"field_change","created_at":"2026-09-21T05:03:01.265493211Z","actor":"Denis Kiselev","issue_id":"EV-g8ya","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
488
|
+
{"id":"int-2b83bb6b","kind":"field_change","created_at":"2026-09-21T06:59:17.164808817Z","actor":"Denis Kiselev","issue_id":"EV-nlx1","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
489
|
+
{"id":"int-9a7df3ab","kind":"field_change","created_at":"2026-09-21T07:43:13.101312655Z","actor":"Denis Kiselev","issue_id":"EV-5pob","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
490
|
+
{"id":"int-6c08ef8f","kind":"field_change","created_at":"2026-09-21T08:47:57.19015326Z","actor":"Denis Kiselev","issue_id":"EV-f8h3","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
491
|
+
{"id":"int-41e28161","kind":"field_change","created_at":"2026-09-21T09:13:10.822154641Z","actor":"Denis Kiselev","issue_id":"EV-vk1f","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
data/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,71 @@
|
|
|
2
2
|
|
|
3
3
|
Versioning policy: see [docs/versioning.md](docs/versioning.md).
|
|
4
4
|
|
|
5
|
+
## [1.2.0] - 2026-09-21
|
|
6
|
+
|
|
7
|
+
Reporting honesty release. A run now says what it did **not** measure instead of letting the headline score speak for the whole target set: files that resolved to no spec fail the run, subjects nothing reached are named, neutral results record why, and the printed verdict and the exit code finally share one threshold. The `default` profile also grows from 80 to 88 operators, so mutation scores will move — every new operator produces mutants your suite has never been measured against. Pin the gem version and the operator profile if you need a stable score across runs.
|
|
8
|
+
|
|
9
|
+
Most of this release comes from a field report on 1.1.0 ([discussion #1584](https://github.com/marinazzio/evilution/discussions/1584)): eight months of side-by-side runs against `mutant` on a private Rails application, filed as GH #1603–#1608.
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- **Eight new mutation operators for method definitions (`default` profile: 80 -> 88)** — the operator set mutated method bodies only to `nil`, `self` or `super`, and never touched parameter lists (GH #1419):
|
|
14
|
+
- **`method_body_to_raise`** — whole body to a bare `raise`. Where body-to-nil asks whether the return value matters, this asks the prior question of whether the method is called at all on a path the suite asserts; a method whose result is discarded survives the nil mutation but not this one (PR #1609, GH #1447)
|
|
15
|
+
- **`method_body_to_super`** — whole body to a bare `super`, for plain overrides. `method_body_replacement` emits the same replacement only when the body *already* calls super, so the case worth probing was never reached. Emitted only where a super target exists: an explicit superclass, or `include`/`prepend` for instance methods and `extend` for singleton ones (PR #1610, GH #1448)
|
|
16
|
+
- **`typed_default_return`** — single-expression body to the empty value of the type its trailing call returns (`users.map(&:name)` -> `[]`, `users.count` -> `0`). A survivor means the suite asserts the shape of what comes back but never its content (PR #1611, GH #1449)
|
|
17
|
+
- **`block_parameter_drop`** — drops a block's single parameter: `users.each { |u| touch(u) }` -> `users.each { touch(u) }`. A survivor means the block never ran on an asserted path (PR #1612, GH #1453)
|
|
18
|
+
- **`optional_parameter_to_required`** — `def f(a = 1)` -> `def f(a)`. A survivor means no example calls the method without that argument, so the default is never exercised (PR #1613, GH #1450)
|
|
19
|
+
- **`optional_default_injection`** — overwrites an optional parameter with its own default at the top of the body. The mirror of the above: it asks whether any value *other* than the default is ever asserted (PR #1614, GH #1451)
|
|
20
|
+
- **`block_destructuring_expansion`** — `|(k, v), i|` -> `|k, v, i|`, emitted only where the group has a sibling parameter, since a lone group and a flat list bind identically for a single yielded array (PR #1615, GH #1452)
|
|
21
|
+
- **`forwarding_super_to_explicit`** — bare `super` to `super()`. Bare super forwards the current arguments, `super()` forwards none, so a survivor means the forwarded arguments never reach an assertion (PR #1616, GH #1454)
|
|
22
|
+
- **Per-subject score breakdown** — the run's score is computed per file, which says nothing about a method inside it that no example reaches; a well-tested file that gains untested methods still reported 100%. Every report now scores each subject separately. The text report names the subjects the file-level score does not speak for, and JSON carries every subject under `subjects` with `killed`, `verified`, `score` and `reached` (PR #1621, GH #1605)
|
|
23
|
+
- **Neutral results record why they are neutral** — `neutral` covered both "the spec was already red" and "the test process died on infrastructure", which want opposite responses. Each neutral now carries a reason; the text report groups by it and names the failing spec or the crash class, and JSON entries carry `neutral_reason` as `{ kind, detail }` (PR #1622, GH #1606)
|
|
24
|
+
- **`--output FILE`** — writes the report to a file instead of stdout, for projects whose preloaded spec helper writes to stdout on exit (PR #1620, GH #1608)
|
|
25
|
+
|
|
26
|
+
### Changed
|
|
27
|
+
|
|
28
|
+
- **A target file that resolves to no spec now fails the run** — previously it contributed a handful of `unresolved` mutations and disappeared behind the other files' score: adding one well-tested file to the same command turned a 0% file into a `PASS`. Such files are now named, and the run exits non-zero whatever the score. `--fallback-full-suite` suppresses this, since nothing goes untested there. JSON carries `summary.unresolved_target_files` (PR #1617, GH #1603)
|
|
29
|
+
- **The printed verdict and the exit code share one threshold** — `Result:` was rendered against a hard-coded 80% while the exit code used `min_score`, whose default is `0.0`, so a run could print `FAIL (score 0.00% < 80.00%)` and exit 0. The reporter now uses the run's own `min_score`, and with no gate configured it prints the score instead of a verdict: `Result: 66.67% (no minimum score set)`. Exit-code semantics are unchanged (PR #1618, GH #1604)
|
|
30
|
+
- **The score line says how much of the run it covers** — `Score: 100.00% (10/10 verified of 17 mutations, 7 neutral)` whenever mutations were left out of the denominator. A run that measured everything still prints the plain pair (PR #1622, GH #1606)
|
|
31
|
+
- **Evilution owns its exit status and, in JSON mode, stdout** — `--preload` loads the project's spec helper into the parent process, and an at-exit hook it installs (SimpleCov calls `exit` with its own status, and prints its coverage report) replaced evilution's exit code and appended text after the JSON document. The executable now installs a guard that has the final word on the status, and JSON mode points stdout at stderr once the document is written (PR #1620, GH #1608)
|
|
32
|
+
|
|
33
|
+
### Fixed
|
|
34
|
+
|
|
35
|
+
- **Survivors are confirmed against the whole spec file** — per-mutation targeting selects examples by matching the enclosing method's name against example bodies, which keys on incidental identifier text: on evilution's own code a local variable named `row` selected two examples and hid six kills, reporting survivors the suite actually covered. A survivor is now re-run against the whole resolved file before it is reported. Only survivors pay for it, and only where the subset was narrower than the file (PR #1629, GH #1624)
|
|
36
|
+
- **Neutral counts no longer move with `--jobs`** — parallel workers contending on a shared database crash the test process, and those crashes are demoted to `:neutral` so they cannot inflate the kill count (GH #814). Since the contention exists only while the pool runs, the same workload scored 24 killed / 0 neutral at `-j 1` and 6 killed / 18 neutral at `-j 4`. Mutations neutralised that way are now re-run one at a time once the pool is done, and the run reports how many it had to redo. Such crashes are also never written to the `--incremental` cache, where a cached `:killed` would have hidden a real survivor on the next run (PR #1619, GH #1607)
|
|
37
|
+
- **`--format json` is parseable under `in_process` isolation** — RSpec redirects its own output only when the stream its configuration holds is the current `$stdout`; in-process isolation swaps `$stdout` for a null IO, and `--preload` builds the configuration before that swap, so RSpec kept writing to the real stdout ahead of the document. The run now claims RSpec's streams outright (PR #1631, GH #1627)
|
|
38
|
+
- **`method_body_replacement` no longer emits a bare `super` for a method that has no parent to call** — a `super` inside a *nested* def made the enclosing method eligible, producing a mutant that raises `NoMethodError` on contact: a kill that proves nothing (PR #1630, GH #1625)
|
|
39
|
+
- **Canary failures name the underlying error** — the proof-of-life check reported that it failed without saying why (PR #1592)
|
|
40
|
+
- **Advisory warnings bypass `Kernel#warn`** — a project whose `Warning` handler raises could turn an evilution advisory into a fatal error mid-run (PR #1590)
|
|
41
|
+
- **`ConcernStateCleaner` handles ActiveSupport deprecation proxies** — sending messages to a module that intercepts them left concern state uncleared (PR #1587)
|
|
42
|
+
- **Empty results from a child report exit status and signal** — a worker that died without producing a result said only "empty result from child" (PR #1585, GH #1580)
|
|
43
|
+
|
|
44
|
+
## [1.1.0] - 2026-08-23
|
|
45
|
+
|
|
46
|
+
Control-flow and pattern-matching operator expansion: the `default` profile grows from 74 to 80 operators, and three existing operators gain mutations they were silently missing. Adding operators to `default` is a MINOR change under [docs/versioning.md](docs/versioning.md), and mutation scores will move — every new operator produces mutants your suite has never been measured against. Pin the gem version and the operator profile if you need a stable score across runs.
|
|
47
|
+
|
|
48
|
+
### Added
|
|
49
|
+
|
|
50
|
+
- **Six new mutation operators for control flow and pattern matching (`default` profile: 74 -> 80)** — closing the value-level and branch-level gaps in an operator set that previously mutated control-flow nodes only at the keyword level (epic EV-170m, GH #1418):
|
|
51
|
+
- **`boolean_operand_promotion`** — drops one side of a compound boolean: `a && b` becomes `a` and `b`, and the same for `||` and the `and` / `or` keyword forms. Kills tests that only ever exercise one side of a condition, which `boolean_operator_replacement` cannot: it swaps the operator but always keeps both operands (EV-170m.2, PR #1565, GH #1431)
|
|
52
|
+
- **`boolean_expression_to_nil`** — replaces a whole compound boolean with `nil`, targeting conditions that are run for their side effects rather than their value (EV-170m.3, PR #1566, GH #1432)
|
|
53
|
+
- **`if_branch_swap`** — replaces the if-branch with the else body and drops the else: `if c; x; else; y; end` becomes `if c; y; end`. Both outcomes of the condition change, which is out of reach of `conditional_negation` (pins the predicate to one branch) and `conditional_branch` (blanks one body to `nil`) (EV-170m.5, PR #1567, GH #1434)
|
|
54
|
+
- **`loop_body_to_raise`** — replaces a `while` / `until` body with a bare `raise`, so a survivor means no test ever enters the loop. The raise ends the loop on its first iteration, so the mutant cannot spin (EV-170m.8, PR #1569, GH #1437)
|
|
55
|
+
- **`case_in`** — the first operator to visit Ruby's pattern-matching grammar (`Prism::CaseMatchNode`): drops one `in` clause from a `case/in`, or drops its `else`. Input that used to match then falls through to a later arm, to the `else`, or — with neither — raises `NoMatchingPatternError` (EV-170m.14 / EV-170m.15, PRs #1573 / #1574, GH #1443 / #1444)
|
|
56
|
+
- **`pattern_predicate`** — one-line pattern match to `false`: `x in Integer` becomes `false`. The mutant differs from the original only on inputs the pattern actually matches, so a survivor means no test ever feeds it a matching value (EV-170m.16, PR #1575, GH #1445)
|
|
57
|
+
- **`case_when` covers two more shapes** — an empty `when` arm now gets a `raise` inserted, and a multi-condition arm is shortened one value at a time (`when a, b` becomes `when a` and `when b`). Neither was previously reachable: dropping an empty arm is indistinguishable from falling through to a missing `else`, and whole-arm removal cannot tell which of several listed values a test exercises (EV-170m.11 / EV-170m.12, PRs #1571 / #1572, GH #1440 / #1441)
|
|
58
|
+
|
|
59
|
+
### Fixed
|
|
60
|
+
|
|
61
|
+
- **`conditional_branch` ignored `unless` entirely** — the operator defined only `visit_if_node`, so neither `unless c; x; end` nor `unless c; x; else; y; end` received any branch-body mutation, while the equivalent `if` forms received one per branch. Prism gives `unless` its own node type and names the else slot `else_clause` rather than `IfNode`'s `subsequent`; both forms now share the branch handling, so `unless` bodies — including the modifier form — get the same branch-to-`nil` mutants (EV-9mrs, PR #1578, GH #1568)
|
|
62
|
+
- **MCP list responses omitted the mandatory `resultType` field** — protocol revision 2026-07-28 (SEP-2322) makes `resultType` mandatory on list results, and the `mcp` gem echoes back that revision while still omitting the field. Strict clients therefore rejected the whole list and saw a server exposing no tools at all. `Evilution::MCP::CompleteResultServer` now marks every list result `"complete"` (PR #1417)
|
|
63
|
+
|
|
64
|
+
### Changed
|
|
65
|
+
|
|
66
|
+
- **Post-form loops (`begin ... end while c`) verified across the loop operators** — `loop_body_to_raise` reaches through the `BeginNode` that Prism reports as the loop's statements, so the wrapper, and with it the guaranteed first iteration, survives the mutation; editing the outer span would have rewritten the loop as `raise while c`, which tests the condition first. `loop_flip` and `begin_unwrap` gained the post-form coverage they never had, including the fact that unwrapping such a loop legitimately drops its run-once guarantee. A sweep of every operator over post-form loops found no mutation that fails to parse (EV-170m.10, PR #1570, GH #1439)
|
|
67
|
+
- **Shared operator primitives (`Mutator::Primitives`)** — `mutate_to_nil` and `promote_child`, built on `Base#add_mutation(skip_unparseable: true)`. Operators built on them skip rather than emit when the result would not parse in its surrounding context, so a promotion that cannot parse no longer reaches the `unparseable` bucket that the point operators still populate (EV-170m.1, PR #1564, GH #1430)
|
|
68
|
+
- **Dependency bumps** — `mcp` 0.24.0 -> 1.2.0 (PRs #1411, #1413, #1414, #1563), CI Ruby versions and gem dependencies (PR #1415), `ruby/setup-ruby` 1.318.0 -> 1.321.0 (PRs #1409, #1412), `actions/checkout` 7.0.0 -> 7.0.1 (PR #1410), `rubygems/release-gem` 1.4.0 -> 1.4.1 (PR #1562)
|
|
69
|
+
|
|
5
70
|
## [1.0.0] - 2026-07-15
|
|
6
71
|
|
|
7
72
|
First stable release. From `1.0.0` onward evilution follows [Semantic Versioning](https://semver.org): the public contract — CLI commands and flags, `.evilution.yml` configuration keys, session JSON files, the MCP tool schemas, and process exit codes — is frozen and covered by the SemVer guarantees and deprecation cycle in [docs/versioning.md](docs/versioning.md). The `1.0.0` milestone is the culmination of the readiness work that shipped across the `0.31`–`0.35` line (config and session-JSON schema versioning, MCP tool-contract stabilization, the CLI flag deprecation sweep, real-world Rails validation, the parallel/isolation stress suite, and running evilution against its own suite to a mutation-score target).
|
data/README.md
CHANGED
|
@@ -104,6 +104,7 @@ Every command, subcommand, and flag listed in this section is part of evilution'
|
|
|
104
104
|
| `-t`, `--timeout N` | Integer | 30 | Per-mutation timeout in seconds. |
|
|
105
105
|
| `-f`, `--format FORMAT` | String | `text` | Output format: `text`, `json`, or `html`. |
|
|
106
106
|
| `--target EXPR` | String | _(none)_ | Only mutate matching methods. Supports method name (`Foo::Bar#calculate`), class (`Foo`), namespace wildcards (`Foo::Bar*`), method-type selectors (`Foo#`, `Foo.`), descendants (`descendants:Foo`), and source globs (`source:lib/**/*.rb`). |
|
|
107
|
+
| `--output FILE` | String | _(stdout)_ | Write the report to FILE instead of stdout. Useful when a preloaded spec helper writes to stdout on exit. |
|
|
107
108
|
| `--min-score FLOAT` | Float | 0.0 | Minimum mutation score (0.0–1.0) to pass. |
|
|
108
109
|
| `--spec FILES` | Array | _(none)_ | Spec files to run (comma-separated). Defaults to auto-detection via `SpecResolver`, which also resolves non-mirrored (`spec/unit`, `test/unit`), dir-grouped (`test/unit/<class>/*_test.rb`), and flat `test_`-prefixed (`test/test_connection_pool_timed_stack.rb`) layouts. |
|
|
109
110
|
| `--spec-dir DIR` | String | _(none)_ | Include all `*_spec.rb` files in DIR recursively. Composable with `--spec`. |
|
|
@@ -157,7 +158,7 @@ Every command, subcommand, and flag listed in this section is part of evilution'
|
|
|
157
158
|
|
|
158
159
|
Two profiles ship out of the box:
|
|
159
160
|
|
|
160
|
-
- **`default`** — the
|
|
161
|
+
- **`default`** — the 88 stable operators registered in `Mutator::Registry.default`. Suitable for everyday CI runs; balances coverage signal against survivor noise.
|
|
161
162
|
- **`strict`** — adds extra truthiness mutators on top of `default`. Currently `PredicateToNil` (replaces every `x.predicate?` call with `nil` to surface tests that only assert truthiness rather than exact return values). Use for pre-merge audits where you want maximum sensitivity at the cost of more survivors.
|
|
162
163
|
|
|
163
164
|
Set via `--profile=strict`, the `--strict` shortcut, or `profile: strict` in `.evilution.yml`.
|
|
@@ -167,9 +168,23 @@ Set via `--profile=strict`, the `--strict` shortcut, or `profile: strict` in `.e
|
|
|
167
168
|
| Code | Meaning | Agent action |
|
|
168
169
|
|------|-----------------------------------------------|---------------------------------------|
|
|
169
170
|
| 0 | Mutation score meets or exceeds `--min-score` | Success. No action needed. |
|
|
170
|
-
| 1 | Mutation score below `--min-score
|
|
171
|
+
| 1 | Mutation score below `--min-score`, or a target file resolved to no spec | Parse output, fix surviving mutants. |
|
|
171
172
|
| 2 | Tool error (bad config, parse failure, etc.) | Check stderr, fix invocation. |
|
|
172
173
|
|
|
174
|
+
`min_score` defaults to `0.0`, so **no score gate is armed unless you set one**. The `Result:` line says so rather than implying a threshold nobody configured:
|
|
175
|
+
|
|
176
|
+
```
|
|
177
|
+
$ evilution run lib/half_tested.rb # no gate
|
|
178
|
+
Result: 66.67% (no minimum score set) # exit 0
|
|
179
|
+
|
|
180
|
+
$ evilution run lib/half_tested.rb --min-score 0.8
|
|
181
|
+
Result: FAIL (score 66.67% < 80.00%) # exit 1
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
Evilution owns the exit status: `--preload` loads the project's own spec helper into the parent process, and an at-exit hook it installs (SimpleCov calls `exit` with its own status when coverage is below the minimum) would otherwise replace the status evilution computed (GH #1608).
|
|
185
|
+
|
|
186
|
+
The printed verdict and the exit code always use the same threshold. Previously the line was printed against a hard-coded 80% that the exit code did not share, so a failing-looking run still exited 0 (GH #1604).
|
|
187
|
+
|
|
173
188
|
## Configuration
|
|
174
189
|
|
|
175
190
|
Generate default config: `bundle exec evilution init`
|
|
@@ -224,6 +239,7 @@ All keys recognised under `schema_version: 1`:
|
|
|
224
239
|
| `timeout` | Integer | `30` | Per-mutation timeout in seconds. |
|
|
225
240
|
| `format` | String | `text` | Output format: `text`, `json`, `html`. |
|
|
226
241
|
| `target` | String / null | `null` | Filter expression: method (`Foo#bar`), class (`Foo`), namespace (`Foo*`), descendants (`descendants:Foo`), source glob (`source:**/*.rb`). |
|
|
242
|
+
| `output` | String | _(stdout)_ | Write the report to this file instead of stdout. |
|
|
227
243
|
| `min_score` | Float | `0.0` | Minimum mutation score (0.0–1.0) for exit code 0. |
|
|
228
244
|
| `integration` | String | `rspec` | Test framework: `rspec`, `minitest`, or `test_unit`. |
|
|
229
245
|
| `verbose` | Boolean | `false` | Verbose output (RSS/GC stats per phase, error details for errored mutations). |
|
|
@@ -299,6 +315,8 @@ Schema:
|
|
|
299
315
|
"neutral": "integer — mutations whose tests already failed before mutation (baseline failure)",
|
|
300
316
|
"equivalent": "integer — mutations proven to have identical behavior to the original",
|
|
301
317
|
"unresolved": "integer — mutations where no spec file resolved (coverage gap, not a failure)",
|
|
318
|
+
"unresolved_target_files": "array of strings (optional) — target files that resolved to no spec at all; present only when non-empty, and the run fails when it is",
|
|
319
|
+
"infra_retried": "integer (optional) — mutations a parallel pass could not judge because the test process crashed on infrastructure, re-run serially afterwards; present only when non-zero",
|
|
302
320
|
"unparseable": "integer — mutations whose mutated source did not parse (short-circuited, never executed)",
|
|
303
321
|
"score": "float — killed / (total - errors - neutral - equivalent - unresolved - unparseable), range 0.0-1.0, rounded to 4 decimals",
|
|
304
322
|
"duration": "float — total wall-clock seconds, rounded to 4 decimals",
|
|
@@ -313,7 +331,20 @@ Schema:
|
|
|
313
331
|
"duration": "float — seconds this mutation took, rounded to 4 decimals",
|
|
314
332
|
"diff": "string — legacy +/- diff snippet",
|
|
315
333
|
"unified_diff": "string (optional, survived only) — git-style unified diff with `--- a/file`, `+++ b/file`, `@@` hunk header and sdiff body; omitted when source slices are unavailable",
|
|
316
|
-
"suggestion": "string — actionable hint for surviving mutants (survived only)"
|
|
334
|
+
"suggestion": "string — actionable hint for surviving mutants (survived only)",
|
|
335
|
+
"neutral_reason": "object (optional, neutral only) — { kind: 'baseline_failure' | 'infra_error', detail: string|null — the failing spec file or the crash class; null when the run was given explicit --spec files and no single spec can be named }"
|
|
336
|
+
}
|
|
337
|
+
],
|
|
338
|
+
"subjects": [
|
|
339
|
+
{
|
|
340
|
+
"name": "string — subject name (e.g. 'Foo#bar')",
|
|
341
|
+
"file": "string — relative path to source file",
|
|
342
|
+
"total": "integer — mutations generated for this subject",
|
|
343
|
+
"killed": "integer — mutations detected",
|
|
344
|
+
"verified": "integer — mutations that got a verdict (killed + survived + timed out)",
|
|
345
|
+
"survived": "integer — mutations that went undetected",
|
|
346
|
+
"score": "float — killed / verified, 0.0 when nothing was verified, rounded to 4 decimals",
|
|
347
|
+
"reached": "boolean — false when no mutation of this subject got a verdict at all"
|
|
317
348
|
}
|
|
318
349
|
],
|
|
319
350
|
"coverage_gaps": [
|
|
@@ -361,6 +392,10 @@ Sessions saved by `--save-session` (under `.evilution/results/*.json`) and consu
|
|
|
361
392
|
|
|
362
393
|
Saved sessions also omit the per-status arrays (`killed`, `neutral`, `equivalent`, `unresolved`, `unparseable`, `timed_out`, `errors`) — only `survived` and `coverage_gaps` are persisted. The score, totals, and timestamps are stable for diff/compare consumers.
|
|
363
394
|
|
|
395
|
+
#### stdout in JSON mode
|
|
396
|
+
|
|
397
|
+
With `--format json`, stdout carries the JSON document and nothing else. Each mutation's test run writes to buffers evilution owns, whatever the isolation mode: under `in_process` the framework's configuration can outlive a single run — `--preload` builds it before isolation swaps `$stdout` — so the run claims RSpec's output and error streams outright rather than relying on RSpec to redirect them (GH #1627). Once the document is written, stdout is pointed at stderr, so anything a preloaded spec helper prints on the way out — SimpleCov's coverage report, for example — lands on stderr instead of after the document where it would leave `JSON.parse` with nothing to work with. `--output FILE` writes the document to a file and leaves stdout alone entirely.
|
|
398
|
+
|
|
364
399
|
#### Schema versioning
|
|
365
400
|
|
|
366
401
|
Every session and stdout JSON document carries a top-level `schema_version` integer (currently `1`). On read:
|
|
@@ -383,14 +418,70 @@ Compatibility policy for the `1.x` gem line:
|
|
|
383
418
|
| `survived` | No test failed — gap in coverage | denominator only |
|
|
384
419
|
| `timeout` | Test run exceeded `--timeout` — treated like survived for scoring | denominator only |
|
|
385
420
|
| `error` | Mutation caused an unexpected error (syntax error, boot failure, etc.) | excluded from denominator |
|
|
386
|
-
| `neutral` | Baseline tests already failed before mutation
|
|
421
|
+
| `neutral` | Baseline tests already failed before mutation, or the test process crashed on infrastructure (DB lock, statement timeout) rather than on the mutation. Every neutral records which of the two, and the report groups them by it | excluded |
|
|
387
422
|
| `equivalent` | Mutation is provably identical to the original (e.g. no-op replacement) | excluded |
|
|
388
423
|
| `unresolved` | No spec file resolved for the mutated source — **coverage gap, not a failure**. Use `--fallback-full-suite` to run the full suite instead. | excluded |
|
|
389
424
|
| `unparseable` | Mutated source failed to parse (e.g. dangling heredoc opener after `method_body_replacement`). Short-circuited — never executed. | excluded |
|
|
390
425
|
|
|
391
426
|
Unresolved mutations indicate a missing test mapping — the file has no corresponding test file that the resolver could find (for example, an RSpec `_spec.rb` file or a Minitest `_test.rb` file, depending on configuration). The resolver searches the `lib/`-mirrored path, common non-mirrored buckets (`spec/unit`, `spec/lib`, `test/unit`, `test/lib`), and the flat `test_`-prefixed Minitest/Test::Unit convention (`test/test_connection_pool_timed_stack.rb`), so a high unresolved rate usually means a genuinely missing or unconventionally-placed test; a run that leaves many mutations unresolved prints an unresolved-rate warning with a best-guess spec path per source file. They are reported separately so you can act on them (add a test, adjust test naming, pass `--spec`, or opt in to the full-suite fallback) without inflating the error count.
|
|
392
427
|
|
|
393
|
-
|
|
428
|
+
A *target file* that resolves to no test at all is a stronger condition than an individual unresolved mutation, and is reported on its own terms: evilution names the file and **fails the run**, whatever the mutations it did measure scored. Without this, adding one well-tested file to the command dilutes the unresolved rate and the untested file disappears behind a `PASS` (GH #1603):
|
|
429
|
+
|
|
430
|
+
```
|
|
431
|
+
$ evilution run app/services/untested.rb app/services/well_tested.rb
|
|
432
|
+
Mutations: 98 total, 82 killed, 0 survived, 0 timed out, 2 neutral, 14 unresolved
|
|
433
|
+
Score: 100.00% (82/82)
|
|
434
|
+
! 1 of 2 target files has no resolvable spec — it was never tested:
|
|
435
|
+
app/services/untested.rb
|
|
436
|
+
Result: FAIL (1 target file has no resolvable spec)
|
|
437
|
+
$ echo $?
|
|
438
|
+
1
|
|
439
|
+
```
|
|
440
|
+
|
|
441
|
+
The check covers files evilution found something to mutate in; a file it produced no mutations for (a constants-only file, say) is not reported. `--fallback-full-suite` runs such a file against the whole suite instead, so nothing goes untested and nothing is reported. The file list is also in JSON output under `summary.unresolved_target_files`.
|
|
442
|
+
|
|
443
|
+
### Survivor Confirmation
|
|
444
|
+
|
|
445
|
+
Per-mutation targeting runs a subset of a spec file's examples, chosen by matching the enclosing method's name against example bodies. That match keys on identifier text, so it can miss the example that would have caught a mutation — and the mutation is then reported as a survivor the suite actually covers. A survivor nobody can reproduce is worse than a missed kill: it sends the reader to write a test that is already there (GH #1624).
|
|
446
|
+
|
|
447
|
+
Before any survivor is reported, it is therefore re-run against the whole resolved spec file, and that run is the one reported. Only survivors pay for the extra run, and only where the targeted subset was narrower than the file; a mutation the targeted examples already killed is never re-run.
|
|
448
|
+
|
|
449
|
+
On evilution's own `lib/evilution/reporter/json/subjects.rb` this moved the reported score from 74.19% with 8 survivors to 93.55% with 2 — the six that disappeared were killed by an example in the same file all along, and the run now agrees with `--no-example-targeting` instead of contradicting it.
|
|
450
|
+
|
|
451
|
+
### Neutral Mutations
|
|
452
|
+
|
|
453
|
+
Neutral covers two unrelated situations that want opposite responses: a spec file that was already red before any mutation ran, and a test process that died on infrastructure rather than on the mutation. Each neutral records which, and the report groups by it, naming the spec or the error class (GH #1606):
|
|
454
|
+
|
|
455
|
+
```
|
|
456
|
+
Score: 100.00% (10/10 verified of 17 mutations, 7 neutral)
|
|
457
|
+
|
|
458
|
+
Neutral mutations (7, not verified):
|
|
459
|
+
baseline already failing (spec/tally_spec.rb):
|
|
460
|
+
arithmetic_replacement: lib/tally.rb:9
|
|
461
|
+
integer_literal: lib/tally.rb:9
|
|
462
|
+
```
|
|
463
|
+
|
|
464
|
+
The score line names the remainder whenever the run left mutations out of the denominator, because full marks over a fraction of a run otherwise reads as a verdict on all of it. A clean run still prints the plain `Score: 100.00% (17/17)`.
|
|
465
|
+
|
|
466
|
+
Those seven mutations were survivors until the spec file went red — a neutral of this kind is a hidden coverage gap, not a clean bill of health. JSON output carries `neutral_reason` as `{ kind, detail }` on neutral entries that have one; `detail` is null where no single spec can be named (an explicit `--spec` run), and the field is absent on a result recorded without a reason, which the text report shows as `reason not recorded`.
|
|
467
|
+
|
|
468
|
+
### Per-Subject Scores
|
|
469
|
+
|
|
470
|
+
The run's score is computed per file, and test selection resolves per file too, so a well-tested file that gains new untested methods still reports 100% — the number only ever describes what the resolved spec reaches (GH #1605). Every report therefore breaks the run down per subject, and the text report names the subjects the file-level score does not speak for:
|
|
471
|
+
|
|
472
|
+
```
|
|
473
|
+
Mutations: 33 total, 7 killed, 0 survived, 0 timed out, 26 unresolved
|
|
474
|
+
Score: 100.00% (7/7)
|
|
475
|
+
|
|
476
|
+
Subjects needing attention (2 subjects in 1 file):
|
|
477
|
+
lib/helper.rb
|
|
478
|
+
#summary_for 0.00% (0/17) nothing reached this subject
|
|
479
|
+
#total_for 0.00% (0/9) nothing reached this subject
|
|
480
|
+
```
|
|
481
|
+
|
|
482
|
+
A subject is listed when something survived, or when nothing reached it at all — zero verdicts, every mutation unresolved or neutral. Fully-killed subjects are not listed, so the section stays actionable. JSON output carries every subject under `subjects`, whether or not it needs attention, so a CI step can assert on `reached` or on a per-subject `score`.
|
|
483
|
+
|
|
484
|
+
## Mutation Operators (88 total)
|
|
394
485
|
|
|
395
486
|
Each operator name is stable and appears in JSON output under `survived[].operator`.
|
|
396
487
|
|
|
@@ -399,6 +490,8 @@ Each operator name is stable and appears in JSON output under `survived[].operat
|
|
|
399
490
|
| `arithmetic_replacement` | Swap arithmetic operators | `a + b` -> `a - b` |
|
|
400
491
|
| `comparison_replacement` | Swap comparison operators | `a >= b` -> `a > b` |
|
|
401
492
|
| `boolean_operator_replacement` | Swap `&&` / `\|\|` | `a && b` -> `a \|\| b` |
|
|
493
|
+
| `boolean_operand_promotion` | Drop one side of `&&` / `\|\|` | `a && b` -> `a`, `b` |
|
|
494
|
+
| `boolean_expression_to_nil` | Replace a whole compound boolean with `nil` | `a && b` -> `nil` |
|
|
402
495
|
| `boolean_literal_replacement` | Flip boolean literals | `true` -> `false` |
|
|
403
496
|
| `nil_replacement` | Replace `nil` with `true`, `false`, `0`, `""` | `nil` -> `true` |
|
|
404
497
|
| `integer_literal` | Boundary-value integer mutations | `n` -> `0`, `1`, `n+1`, `n-1` |
|
|
@@ -408,7 +501,8 @@ Each operator name is stable and appears in JSON output under `survived[].operat
|
|
|
408
501
|
| `hash_literal` | Empty the hash | `{k: v}` -> `{}` |
|
|
409
502
|
| `symbol_literal` | Replace with sentinel symbol | `:foo` -> `:__evilution_mutated__` |
|
|
410
503
|
| `conditional_negation` | Replace condition with `true`/`false` | `if cond` -> `if true` |
|
|
411
|
-
| `conditional_branch` | Remove if/else branch | Deletes branch body |
|
|
504
|
+
| `conditional_branch` | Remove if/unless/else branch | Deletes branch body |
|
|
505
|
+
| `if_branch_swap` | Replace the if-branch with the else body, drop the else | `if c; x; else; y; end` -> `if c; y; end` |
|
|
412
506
|
| `conditional_flip` | Flip `if` to `unless` and vice versa | `if cond` -> `unless cond` |
|
|
413
507
|
| `statement_deletion` | Remove statements from method bodies | Deletes a statement |
|
|
414
508
|
| `method_body_replacement` | Replace entire method body | Method body -> `nil`, `self`, `super` |
|
|
@@ -460,9 +554,20 @@ Each operator name is stable and appears in JSON output under `survived[].operat
|
|
|
460
554
|
| `defined_check` | Replace `defined?` with `true` | `defined?(x)` -> `true` |
|
|
461
555
|
| `regex_capture` | Swap or nil-ify capture refs | `$1` -> `$2`, `$1` -> `nil` |
|
|
462
556
|
| `loop_flip` | Swap while/until loops | `while cond` -> `until cond` |
|
|
557
|
+
| `loop_body_to_raise` | Replace a loop body with `raise` | `while c; body; end` -> `while c; raise; end` |
|
|
558
|
+
| `method_body_to_raise` | Replace a whole method body with `raise` | `def foo; body; end` -> `def foo; raise; end` |
|
|
559
|
+
| `method_body_to_super` | Replace a method body with bare `super` where a super target exists | `def foo; body; end` -> `def foo; super; end` |
|
|
560
|
+
| `typed_default_return` | Replace a single-expression body with the empty value of its inferred type | `def names(u); u.map(&:name); end` -> `def names(u); []; end` |
|
|
561
|
+
| `block_parameter_drop` | Drop a block's single parameter | `users.each { |u| touch(u) }` -> `users.each { touch(u) }` |
|
|
562
|
+
| `optional_parameter_to_required` | Drop an optional positional parameter's default | `def f(a = 1)` -> `def f(a)` |
|
|
563
|
+
| `optional_default_injection` | Overwrite an optional parameter with its own default at the top of the body | `def f(a = 1); body; end` -> `def f(a = 1); a = 1; body; end` |
|
|
564
|
+
| `block_destructuring_expansion` | Flatten a destructuring group in a block's parameters | `pairs.each_with_index { |(k, v), i| use(k, v, i) }` -> `pairs.each_with_index { |k, v, i| use(k, v, i) }` |
|
|
565
|
+
| `forwarding_super_to_explicit` | Give a forwarding `super` an empty argument list | `def f(a); super; end` -> `def f(a); super(); end` |
|
|
463
566
|
| `string_interpolation` | Replace interpolation content with nil | `"hello #{name}"` -> `"hello #{nil}"` |
|
|
464
567
|
| `retry_removal` | Remove retry statements | `retry` -> `nil` |
|
|
465
|
-
| `case_when` | Remove/replace case/when branches | Remove `when` branch, body -> `nil`, remove `else` |
|
|
568
|
+
| `case_when` | Remove/replace case/when branches | Remove `when` branch, drop one condition from `when a, b`, body -> `nil`, empty body -> `raise`, remove `else` |
|
|
569
|
+
| `case_in` | Drop one `in` clause from a `case/in`, or its `else` | `case x; in Integer; …; in String; …; end` -> drops an arm; `else` -> `NoMatchingPatternError` |
|
|
570
|
+
| `pattern_predicate` | One-line pattern match -> `false` | `x in Integer` -> `false` |
|
|
466
571
|
| `predicate_replacement` | Replace predicate calls with booleans | `x.empty?` -> `true`, `x.empty?` -> `false` |
|
|
467
572
|
| `equality_to_identity` | Replace equality with identity check | `a == b` -> `a.equal?(b)` |
|
|
468
573
|
| `lambda_body` | Replace lambda body with nil | `-> { expr }` -> `-> { nil }` |
|
|
@@ -514,6 +619,12 @@ The `evilution-mutate` tool accepts a `verbosity` parameter to control response
|
|
|
514
619
|
|
|
515
620
|
Use `minimal` when context window budget is tight and you only need to see what survived. The trimmed `errors` sample (each entry: `error_message`, `error_class`, location, plus the first 5 backtrace lines) is added so a partly-broken run is still self-diagnosable without escalating verbosity. Use `full` when you need to inspect killed/neutral/equivalent entries for debugging.
|
|
516
621
|
|
|
622
|
+
What survives trimming matters when you are deciding whether to trust a score:
|
|
623
|
+
|
|
624
|
+
- `neutral` entries — and with them each `neutral_reason` — are dropped at `summary` and `minimal`. Use `full` to see why mutations were neutralised.
|
|
625
|
+
- `subjects` is kept at `full` and `summary`, and dropped at `minimal`, which keeps only `summary` and `survived`.
|
|
626
|
+
- Everything inside `summary` survives at every level, including `unresolved_target_files`, `infra_retried` and the `neutral` count — so even a `minimal` response still says whether a target file went untested and how much of the run the score covers.
|
|
627
|
+
|
|
517
628
|
### Enriched Survived Entries
|
|
518
629
|
|
|
519
630
|
Unlike `evilution --format json`, every survived entry returned by `evilution-mutate` carries extra fields so the agent can act without a second round-trip:
|
|
@@ -599,7 +710,7 @@ Per-tool placement:
|
|
|
599
710
|
- **`evilution-session` `list`** — `{ "schema_version": Integer, "sessions": Array<{ file, timestamp, total, killed, survived, score, duration }> }`. Sessions are reverse-chronological; the array is filtered by `limit` when provided.
|
|
600
711
|
- **`evilution-session` `show`** — the parsed session JSON document, exactly as written under `.evilution/results/*.json`. Field reference: see [Session JSON files](#session-json-files).
|
|
601
712
|
- **`evilution-session` `diff`** — `{ "schema_version": Integer, "summary": { base_score, head_score, score_delta, base_survived, head_survived, base_total, head_total, base_killed, head_killed }, "fixed": Array, "new_survivors": Array, "persistent": Array }`. The mutation arrays carry the same per-mutation fields the session `survived` list uses (`operator`, `file`, `line`, `subject`, `diff`).
|
|
602
|
-
- **`evilution-info` `subjects`** — `{ "schema_version": Integer, "subjects": Array<{ name, file, line, mutations }>, "total_subjects": Integer, "total_mutations": Integer }`.
|
|
713
|
+
- **`evilution-info` `subjects`** — `{ "schema_version": Integer, "subjects": Array<{ name, file, line, mutations }>, "total_subjects": Integer, "total_mutations": Integer }`. Discovery only: this lists what *can* be mutated. The `subjects` array in a run's report is a different shape, carrying what each subject scored.
|
|
603
714
|
- **`evilution-info` `tests`** — `{ "schema_version": Integer, "specs": Array<{ source, spec }>, "unresolved": Array<String>, "total_sources": Integer, "total_specs": Integer }`.
|
|
604
715
|
- **`evilution-info` `environment`** — `{ "schema_version": Integer, "version": String, "ruby": String, "config_file": String|null, ... }` mirroring the effective `Evilution::Config`.
|
|
605
716
|
- **`evilution-info` `statuses`** — `{ "schema_version": Integer, "statuses": Array<{ name, meaning, in_score }> }`.
|
|
@@ -630,7 +741,7 @@ When a parameter, action, or output field on the public MCP contract is deprecat
|
|
|
630
741
|
bundle exec evilution run lib/ --format json --min-score 0.8
|
|
631
742
|
```
|
|
632
743
|
|
|
633
|
-
Parse JSON output. Exit code 0 = pass, 1 =
|
|
744
|
+
Parse JSON output. Exit code 0 = pass, 1 = fail — either the score is below `--min-score`, or a target file resolved to no spec and was never tested (`summary.unresolved_target_files` names them). Without `--min-score` no score gate is armed; the run still fails on an untested target file.
|
|
634
745
|
|
|
635
746
|
### 2. PR / changed-lines scan (fast feedback)
|
|
636
747
|
|
|
@@ -685,8 +796,23 @@ bundle exec evilution run lib/models/user.rb lib/models/account.rb lib/models/or
|
|
|
685
796
|
|
|
686
797
|
Pass multiple file paths on a single invocation to amortise startup cost. The framework (Rails, Sorbet, etc.) and the `preload` chain (`spec/rails_helper.rb` → `spec/spec_helper.rb` → `test/test_helper.rb`) load **once** in the parent process. When `--isolation=fork` is selected (the default `--isolation=auto` resolves to `fork` on Rails projects and packaged gems), every subsequent mutation across all files forks from that warmed parent — materially faster than scripting a `for f in ...; do bundle exec evilution run "$f"; done` loop, which pays the bootstrap per file. With `--isolation=in_process` (default for non-Rails, non-gem projects under `auto`), there is no per-mutation fork, but the parent-process boot still runs once instead of N times. Per-file paths and line numbers are preserved in the report (`survived[].file`, HTML grouping by source file).
|
|
687
798
|
|
|
799
|
+
### What to read before acting on a score
|
|
800
|
+
|
|
801
|
+
A score describes only the mutations that got a verdict. Four fields say what it leaves out, and each points at a different action:
|
|
802
|
+
|
|
803
|
+
| Field | Meaning | What to do |
|
|
804
|
+
|---|---|---|
|
|
805
|
+
| `summary.unresolved_target_files` | A file you named resolved to no spec and was never tested; the run fails on this alone | Write a spec, pass `--spec`, or map it in `spec_mappings` — do not trust the score until this is empty |
|
|
806
|
+
| `subjects[].reached == false` | Mutations were generated for that method but none got a verdict | The method is untested even where its file scores well; start here rather than with `survived[]` |
|
|
807
|
+
| `neutral[].neutral_reason.kind == "baseline_failure"` | The spec file was already red before any mutation ran; `detail` names it | Fix that spec first — nothing about these mutations is measurable until it is green |
|
|
808
|
+
| `neutral[].neutral_reason.kind == "infra_error"` | The test process crashed on infrastructure (DB lock, timeout); `detail` names the class | Not a coverage gap. Give parallel workers their own database, or run `-j 1` |
|
|
809
|
+
|
|
810
|
+
`summary.infra_retried` reports how many mutations had to be re-run serially because of the last case; a large number means the parallel run was fighting shared infrastructure rather than measuring your suite.
|
|
811
|
+
|
|
688
812
|
### 6. Fixing surviving mutants
|
|
689
813
|
|
|
814
|
+
Every entry in `survived[]` has already been re-run against the whole resolved spec file, so it is a gap the suite genuinely does not cover rather than an artefact of per-mutation example targeting.
|
|
815
|
+
|
|
690
816
|
For each entry in `survived[]`:
|
|
691
817
|
1. Read `file` at `line` to understand the code context
|
|
692
818
|
2. Read `operator` to understand what was changed
|
|
@@ -727,7 +853,7 @@ RUBYOPT="-Itest" bundle exec evilution mutate lib/<file>.rb \
|
|
|
727
853
|
--spec test/<dir>/<file>_test.rb
|
|
728
854
|
```
|
|
729
855
|
|
|
730
|
-
`-j 4` parallelises across workers, `-t 10` caps any mutation that pathologically loops at 10 s. Expect the run to print progress only when stderr is a TTY (use `bundle exec evilution mutate ... 2>&1 | tee log` to get progress while still saving output). The historical "Minitest fork hangs on liquid" report (
|
|
856
|
+
`-j 4` parallelises across workers, `-t 10` caps any mutation that pathologically loops at 10 s. Expect the run to print progress only when stderr is a TTY (use `bundle exec evilution mutate ... 2>&1 | tee log` to get progress while still saving output). The historical "Minitest fork hangs on liquid" report (GH #1211) turned out to be a slow run + silent UX, not an actual deadlock — the worker logs show steady forward progress when captured via `--quiet-children --quiet-children-dir DIR`.
|
|
731
857
|
|
|
732
858
|
### 8. CI gate
|
|
733
859
|
|
|
@@ -736,7 +862,9 @@ bundle exec evilution run lib/ --format json --min-score 0.8 --quiet
|
|
|
736
862
|
# Exit code 0 = pass, 1 = fail, 2 = error
|
|
737
863
|
```
|
|
738
864
|
|
|
739
|
-
|
|
865
|
+
Exit 1 covers two conditions: the score missed `--min-score`, and a target file that resolved to no spec. The second fails the run whether or not a score gate is set, so a CI step that names files explicitly cannot silently stop testing one of them.
|
|
866
|
+
|
|
867
|
+
Note: `--quiet` suppresses all stdout output (including JSON). Use it in CI only when you care about the exit code and do not need JSON output. `--output FILE` is the alternative when a preloaded spec helper (SimpleCov, for example) writes to stdout on exit.
|
|
740
868
|
|
|
741
869
|
### 9. Regression tracking across runs (`compare`)
|
|
742
870
|
|
|
@@ -765,7 +893,15 @@ Use in CI to gate merges on `reintroduced` being empty, or to surface `new` surv
|
|
|
765
893
|
|
|
766
894
|
## Parallel Runs with SQLite
|
|
767
895
|
|
|
768
|
-
Running with `-j N` forks worker processes. If your Rails app uses SQLite, every worker opens the same `db/test.sqlite3` file, and concurrent writers collide on the database-level lock. Symptoms: `ActiveRecord::StatementTimeout`, `SQLite3::BusyException`, and slow runs. Evilution classifies these crashes as `:neutral` (see [
|
|
896
|
+
Running with `-j N` forks worker processes. If your Rails app uses SQLite, every worker opens the same `db/test.sqlite3` file, and concurrent writers collide on the database-level lock. Symptoms: `ActiveRecord::StatementTimeout`, `SQLite3::BusyException`, and slow runs. Evilution classifies these crashes as `:neutral` (see [GH #814](https://github.com/marinazzio/evilution/issues/814)) so the mutation score is not polluted, but the wall-clock penalty remains.
|
|
897
|
+
|
|
898
|
+
Because that contention only exists while several workers are running, those mutations are re-run one at a time once the pool is done, and the verdict from the quiet re-run is the one reported. Without it the neutral bucket moved with `-j` on identical input — the same files scoring 24 killed / 0 neutral at `-j 1` and 6 killed / 18 neutral at `-j 4` (GH #1607). The run says how much it had to redo:
|
|
899
|
+
|
|
900
|
+
```
|
|
901
|
+
! 18 mutations hit infrastructure errors under parallel workers; re-ran them serially.
|
|
902
|
+
```
|
|
903
|
+
|
|
904
|
+
The count is in JSON output as `summary.infra_retried`. Such a crash is also never written to the `--incremental` cache: the cache keeps no error class, so a cached `:killed` would be indistinguishable from a real one on the next run and would short-circuit both the demotion and the retry. A neutral from a failing baseline is never re-run — that is a real statement about the spec, not a missed verdict. The retry costs wall-clock time in proportion to the contention, which is another reason to give each worker its own database file:
|
|
769
905
|
|
|
770
906
|
Evilution follows the [`parallel_tests`](https://github.com/grosser/parallel_tests) convention: each worker receives a `TEST_ENV_NUMBER` environment variable (`""` for worker 1, `"2"` for worker 2, `"3"` for worker 3, …). Interpolate it into `config/database.yml` so each worker gets its own SQLite file:
|
|
771
907
|
|
|
@@ -810,7 +946,7 @@ points — see [docs/architecture.md](docs/architecture.md).
|
|
|
810
946
|
1. **Parse** — Prism parses Ruby files into ASTs with exact byte offsets
|
|
811
947
|
2. **Extract** — Methods are identified as mutation subjects
|
|
812
948
|
3. **Filter** — Disable comments, Sorbet `sig` blocks, and AST ignore patterns exclude mutations before execution
|
|
813
|
-
4. **Mutate** —
|
|
949
|
+
4. **Mutate** — 88 operators produce text replacements at precise byte offsets (source-level surgery, no AST unparsing); heredoc literal text is skipped by default. Identical byte-mutations from different operators are deduplicated by `(file_path, mutated_source)` so the count is not inflated by overlap
|
|
814
950
|
5. **Isolate** — Mutations are applied to temporary file copies (never modifying originals); load-path redirection ensures `require` resolves the mutated copy. Default isolation is in-process for plain Ruby projects (no gemspec) and fork for Rails projects and packaged gems (auto-detected); `--isolation fork` forces forked child processes. Both sequential and parallel (`--jobs N`) modes respect the configured isolation strategy
|
|
815
951
|
6. **Test** — The configured test framework (RSpec, Minitest, or Test::Unit) executes against the mutated source
|
|
816
952
|
7. **Collect** — Source strings and AST nodes are released after use to minimize memory retention
|
data/docs/architecture.md
CHANGED
|
@@ -48,13 +48,14 @@ Everything lives under `lib/evilution/`.
|
|
|
48
48
|
| `Config`, `Config::*` | Merge `.evilution.yml` + CLI flags + env, validate, freeze. | `config.rb`, `config/sources.rb`, `config/validators/*` |
|
|
49
49
|
| `Runner`, `Runner::*` | Orchestrate the whole run. Each stage is its own collaborator. | `runner.rb`, `runner/*` |
|
|
50
50
|
| `AST`, `Subject` | Prism parse, find method subjects, source surgery, pattern matching, heredoc spans. | `ast/parser.rb`, `ast/source_surgeon.rb`, `subject.rb` |
|
|
51
|
-
| `Mutator`, `Mutator::Operator::*` |
|
|
51
|
+
| `Mutator`, `Mutator::Operator::*` | 88 operators (default profile) that emit byte-edits; registry + profiles. | `mutator/base.rb`, `mutator/registry.rb`, `mutator/operator/*` |
|
|
52
52
|
| `Mutation` | An immutable mutation record (original/mutated sources, slice, location, parse status). | `mutation.rb` |
|
|
53
53
|
| `SpecResolver`, `SpecSelector` | Map a source file to its covering spec files (layout heuristics + explicit mappings). | `spec_resolver.rb`, `spec_selector.rb` |
|
|
54
54
|
| `Isolation::{Fork,InProcess}`, `ProcessSupervisor` | Run one mutation's tests in isolation; process-group lifecycle, sandboxing, TERM/KILL ladder. | `isolation/fork.rb`, `process_supervisor.rb` |
|
|
55
55
|
| `Integration::{RSpec,Minitest,TestUnit}` | Apply a mutation and run the configured test framework; report the raw outcome. | `integration/base.rb`, `integration/rspec.rb` |
|
|
56
56
|
| `Parallel::{Pool,WorkQueue}` | Fan mutations across worker processes for `jobs > 1`. | `parallel/pool.rb`, `parallel/work_queue.rb` |
|
|
57
57
|
| `Result::{MutationResult,Summary}` | Per-mutation result + aggregated, scored summary. | `result/mutation_result.rb`, `result/summary.rb` |
|
|
58
|
+
| `Result::{SubjectScore,SubjectScorer,NeutralReason}` | Score each subject on its own, and record why a neutral is neutral — what the file-level score does not speak for. | `result/subject_scorer.rb`, `result/neutral_reason.rb` |
|
|
58
59
|
| `Reporter::{CLI,JSON,HTML,Suggestion}` | Render a `Summary` to text / JSON / HTML. | `reporter/*` |
|
|
59
60
|
| `Session`, `Compare` | Persist runs to `.evilution/results/*.json`; diff two sessions. | `session/store.rb`, `compare.rb` |
|
|
60
61
|
| `Coverage`, `Equivalent`, `Baseline`, `Cache`, `Hooks`, `MCP` | Coverage-based example targeting, equivalent-mutation detection, baseline capture, incremental cache, lifecycle hooks, MCP server. | respective dirs |
|
|
@@ -87,7 +88,10 @@ class that owns it.
|
|
|
87
88
|
5. **Spec resolution** — per subject, `SpecSelector#call(source_path)` picks specs
|
|
88
89
|
(explicit `spec_files` → `spec_mappings` → `SpecResolver#resolve_specs` layout
|
|
89
90
|
heuristics). Example-level targeting narrows to examples that reference the
|
|
90
|
-
mutated token (`ExampleFilter` / `CoverageExampleFilter`).
|
|
91
|
+
mutated token (`ExampleFilter` / `CoverageExampleFilter`). Separately,
|
|
92
|
+
`Runner::TargetSpecAudit` asks the same selector once per target file in the
|
|
93
|
+
parent, so a file that resolves to no spec at all is a fact the summary
|
|
94
|
+
carries rather than a handful of `unresolved` mutations.
|
|
91
95
|
6. **Execute** — `Runner::MutationExecutor#call` picks a strategy by `config.jobs`:
|
|
92
96
|
`Strategy::Sequential` for `jobs == 1`, `Strategy::Parallel` (via
|
|
93
97
|
`Parallel::Pool` / `WorkQueue`) for `jobs > 1`. Either way each mutation reaches
|
|
@@ -103,13 +107,24 @@ class that owns it.
|
|
|
103
107
|
guard / unresolved); the final symbol is chosen by
|
|
104
108
|
`Isolation::Fork#classify_status`: `:timeout` → `:killed` (crash) →
|
|
105
109
|
`:unresolved` → `:error` → `:survived` (tests passed) → default `:killed`.
|
|
106
|
-
A `NeutralizationPipeline` can reclassify results
|
|
107
|
-
failed at baseline
|
|
110
|
+
A `NeutralizationPipeline` can reclassify results into `:neutral` — either
|
|
111
|
+
because the covering spec already failed at baseline (`Neutralizer::BaselineFailed`)
|
|
112
|
+
or because the test process crashed on infrastructure rather than on the
|
|
113
|
+
mutation (`Neutralizer::InfraError`). Each records a `Result::NeutralReason`,
|
|
114
|
+
since the two want opposite responses. A survivor is re-run against its whole
|
|
115
|
+
spec file before it is reported (`Integration::RSpec#confirm_survivor?`), so a
|
|
116
|
+
narrowed example set cannot invent one; after a parallel pass,
|
|
117
|
+
`MutationExecutor::InfraRetry` re-runs the infrastructure-neutralised
|
|
118
|
+
mutations serially, once the contention that caused them is gone.
|
|
108
119
|
9. **Aggregate + report** — `Result::Summary` counts each status and computes
|
|
109
120
|
`score = killed / score_denominator` (total minus error/neutral/equivalent/
|
|
110
121
|
unresolved/unparseable). `Runner::ReportPublisher#publish` selects a reporter by
|
|
111
122
|
`config.format` and writes it; `Session::Store` optionally persists the run.
|
|
112
|
-
`
|
|
123
|
+
`Summary#success?` is also false when a target file resolved to no spec, whatever
|
|
124
|
+
the score. `Commands::Run` maps `summary.success?(min_score:)` to exit code
|
|
125
|
+
`0`/`1` (`2` on error), and `CLI::ExitGuard` — installed by the executable before
|
|
126
|
+
anything can preload the project — has the final word on the process status, so a
|
|
127
|
+
preloaded spec helper's at-exit hook cannot replace it.
|
|
113
128
|
|
|
114
129
|
## How to add a new mutator
|
|
115
130
|
|
data/docs/isolation.md
CHANGED
|
@@ -48,7 +48,7 @@ framework must be loaded in the parent before forking — see [Automatic
|
|
|
48
48
|
preload](#automatic-preload) — and `in_process` cannot preload them without
|
|
49
49
|
polluting the host process. A non-Rails gem run under the old `in_process`
|
|
50
50
|
default therefore produced 0 examples / 100% errors out of the box; defaulting
|
|
51
|
-
gems to `fork` lets auto-preload fire (
|
|
51
|
+
gems to `fork` lets auto-preload fire (PR #1375). A plain non-Rails,
|
|
52
52
|
non-gem project (no gemspec) still defaults to `in_process`.
|
|
53
53
|
|
|
54
54
|
The same hazard applies to any Ruby code that uses
|
|
@@ -92,12 +92,11 @@ order, falling back to the gem's library entry point (`lib/<gem>.rb`):
|
|
|
92
92
|
When a gem is detected but none of those helpers exist, evilution prints a
|
|
93
93
|
warning naming the locations it looked in and pointing at `--preload`, so a
|
|
94
94
|
non-standard test layout reads as a fixable configuration issue rather than a
|
|
95
|
-
silent 0% (
|
|
95
|
+
silent 0% (PR #1375).
|
|
96
96
|
|
|
97
97
|
Minitest/Test::Unit helpers that `require "test_helper"` (or any non-relative
|
|
98
98
|
`require "support/..."`) work without `-Itest`: evilution puts the test root
|
|
99
|
-
on `$LOAD_PATH` for the preload just as the test runner would (
|
|
100
|
-
#1373).
|
|
99
|
+
on `$LOAD_PATH` for the preload just as the test runner would (PR #1373).
|
|
101
100
|
|
|
102
101
|
No configuration needed.
|
|
103
102
|
|
data/exe/evil
CHANGED
|
@@ -3,4 +3,11 @@
|
|
|
3
3
|
|
|
4
4
|
require "evilution"
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
# Installed before anything loads the project's spec helper, so this hook is the
|
|
7
|
+
# last one standing and decides the process status — a helper's own at_exit hook
|
|
8
|
+
# (SimpleCov calls exit with its own status) cannot replace it.
|
|
9
|
+
# EV-g8ya / GH #1608.
|
|
10
|
+
guard = Evilution::CLI::ExitGuard.new.install
|
|
11
|
+
guard.status = Evilution::CLI.new(ARGV).call
|
|
12
|
+
|
|
13
|
+
exit guard.status
|
data/exe/evilution
CHANGED
|
@@ -3,4 +3,11 @@
|
|
|
3
3
|
|
|
4
4
|
require "evilution"
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
# Installed before anything loads the project's spec helper, so this hook is the
|
|
7
|
+
# last one standing and decides the process status — a helper's own at_exit hook
|
|
8
|
+
# (SimpleCov calls exit with its own status) cannot replace it.
|
|
9
|
+
# EV-g8ya / GH #1608.
|
|
10
|
+
guard = Evilution::CLI::ExitGuard.new.install
|
|
11
|
+
guard.status = Evilution::CLI.new(ARGV).call
|
|
12
|
+
|
|
13
|
+
exit guard.status
|