evilution 1.1.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +58 -0
- data/.rubocop_todo.yml +9 -0
- data/CHANGELOG.md +86 -0
- data/README.md +202 -14
- data/docs/architecture.md +54 -5
- data/docs/isolation.md +3 -4
- data/exe/evil +8 -1
- data/exe/evilution +8 -1
- data/lib/evilution/ast/local_reads.rb +43 -0
- data/lib/evilution/ast/parser.rb +1 -0
- data/lib/evilution/ast/pattern/filter.rb +1 -0
- data/lib/evilution/ast/pattern/parser.rb +1 -0
- data/lib/evilution/ast/pattern.rb +2 -0
- data/lib/evilution/ast/sorbet_sig_detector.rb +1 -0
- data/lib/evilution/baseline.rb +37 -15
- data/lib/evilution/child_output.rb +1 -1
- data/lib/evilution/cli/command.rb +1 -0
- data/lib/evilution/cli/commands/tests_list.rb +4 -3
- data/lib/evilution/cli/commands.rb +2 -0
- data/lib/evilution/cli/dispatcher.rb +2 -0
- data/lib/evilution/cli/exit_guard.rb +61 -0
- data/lib/evilution/cli/parsed_args.rb +2 -0
- data/lib/evilution/cli/parser/command_extractor.rb +2 -0
- data/lib/evilution/cli/parser/file_args.rb +2 -0
- data/lib/evilution/cli/parser/options_builder.rb +4 -0
- data/lib/evilution/cli/parser/stdin_reader.rb +1 -0
- data/lib/evilution/cli/parser.rb +1 -0
- data/lib/evilution/cli/printers/tests_list.rb +5 -4
- data/lib/evilution/cli/printers.rb +2 -0
- data/lib/evilution/cli/result.rb +2 -0
- data/lib/evilution/cli.rb +6 -0
- data/lib/evilution/config/builders.rb +2 -0
- data/lib/evilution/config/env_loader.rb +2 -0
- data/lib/evilution/config/file_loader.rb +1 -0
- data/lib/evilution/config/sources.rb +1 -0
- data/lib/evilution/config/validators/example_targeting_cache.rb +1 -0
- data/lib/evilution/config/validators/example_targeting_fallback.rb +1 -0
- data/lib/evilution/config/validators/example_targeting_strategy.rb +1 -0
- data/lib/evilution/config/validators/fail_fast.rb +1 -0
- data/lib/evilution/config/validators/hooks.rb +1 -0
- data/lib/evilution/config/validators/ignore_patterns.rb +1 -0
- data/lib/evilution/config/validators/integration.rb +1 -0
- data/lib/evilution/config/validators/isolation.rb +1 -0
- data/lib/evilution/config/validators/jobs.rb +1 -0
- data/lib/evilution/config/validators/preload.rb +1 -0
- data/lib/evilution/config/validators/profile.rb +1 -0
- data/lib/evilution/config/validators/spec_mappings.rb +1 -0
- data/lib/evilution/config/validators/spec_pattern.rb +1 -0
- data/lib/evilution/config/validators/warmup.rb +17 -0
- data/lib/evilution/config/validators.rb +2 -0
- data/lib/evilution/config.rb +7 -5
- data/lib/evilution/coverage.rb +1 -1
- data/lib/evilution/coverage_example_filter.rb +1 -1
- data/lib/evilution/diagnostic.rb +22 -0
- data/lib/evilution/example_filter.rb +1 -1
- data/lib/evilution/integration/loading/body_call_neutralizer.rb +10 -2
- data/lib/evilution/integration/loading/concern_state_cleaner.rb +20 -4
- data/lib/evilution/integration/loading/reeval_warning_filter.rb +72 -0
- data/lib/evilution/integration/loading/source_evaluator.rb +4 -1
- data/lib/evilution/integration/loading/test_load_path.rb +21 -13
- data/lib/evilution/integration/minitest.rb +2 -1
- data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +9 -1
- data/lib/evilution/integration/rspec/state_guard/configuration_state.rb +1 -0
- data/lib/evilution/integration/rspec/state_guard/configuration_streams.rb +5 -1
- data/lib/evilution/integration/rspec/state_guard/example_groups_constants.rb +1 -2
- data/lib/evilution/integration/rspec/state_guard/internals.rb +1 -2
- data/lib/evilution/integration/rspec/state_guard/object_space_example_groups.rb +1 -2
- data/lib/evilution/integration/rspec/state_guard/reporter_arrays.rb +1 -0
- data/lib/evilution/integration/rspec/state_guard/world_example_groups.rb +1 -0
- data/lib/evilution/integration/rspec/state_guard/world_filtered_examples.rb +1 -0
- data/lib/evilution/integration/rspec/state_guard/world_sources_by_path.rb +1 -0
- data/lib/evilution/integration/rspec/state_guard.rb +6 -0
- data/lib/evilution/integration/rspec/unresolved_spec_warner.rb +2 -1
- data/lib/evilution/integration/rspec.rb +48 -2
- data/lib/evilution/integration/test_unit/test_file_resolver.rb +2 -1
- data/lib/evilution/isolation/fork.rb +29 -8
- data/lib/evilution/mcp/info_tool/actions/environment.rb +1 -0
- data/lib/evilution/mcp/info_tool/actions/feedback.rb +1 -0
- data/lib/evilution/mcp/info_tool/actions/statuses.rb +1 -0
- data/lib/evilution/mcp/info_tool/actions/subjects.rb +1 -0
- data/lib/evilution/mcp/info_tool/actions/tests.rb +1 -0
- data/lib/evilution/mutation.rb +1 -1
- data/lib/evilution/mutator/operator/argument_list_removal.rb +52 -0
- data/lib/evilution/mutator/operator/argument_propagation.rb +54 -0
- data/lib/evilution/mutator/operator/array_coercion_to_literal.rb +34 -0
- data/lib/evilution/mutator/operator/attribute_write_to_read.rb +34 -0
- data/lib/evilution/mutator/operator/bang_method.rb +11 -3
- data/lib/evilution/mutator/operator/binary_operand_promotion.rb +63 -0
- data/lib/evilution/mutator/operator/block_body_promotion.rb +45 -0
- data/lib/evilution/mutator/operator/block_body_to_nil.rb +52 -0
- data/lib/evilution/mutator/operator/block_body_to_raise.rb +34 -0
- data/lib/evilution/mutator/operator/block_destructuring_expansion.rb +85 -0
- data/lib/evilution/mutator/operator/block_parameter_drop.rb +96 -0
- data/lib/evilution/mutator/operator/call_to_nil.rb +34 -0
- data/lib/evilution/mutator/operator/coercion_emptying.rb +60 -0
- data/lib/evilution/mutator/operator/collection_replacement.rb +19 -9
- data/lib/evilution/mutator/operator/comparison_replacement.rb +37 -12
- data/lib/evilution/mutator/operator/const_get_to_constant_path.rb +50 -0
- data/lib/evilution/mutator/operator/dig_to_fetch_chain.rb +45 -0
- data/lib/evilution/mutator/operator/double_negation_removal.rb +22 -0
- data/lib/evilution/mutator/operator/dynamic_dispatch_resolution.rb +79 -0
- data/lib/evilution/mutator/operator/forwarding_super_to_explicit.rb +71 -0
- data/lib/evilution/mutator/operator/inequality_to_negated_identity.rb +43 -0
- data/lib/evilution/mutator/operator/keyword_argument_removal.rb +47 -0
- data/lib/evilution/mutator/operator/method_body_replacement.rb +10 -1
- data/lib/evilution/mutator/operator/method_body_to_raise.rb +59 -0
- data/lib/evilution/mutator/operator/method_body_to_super.rb +150 -0
- data/lib/evilution/mutator/operator/optional_default_injection.rb +71 -0
- data/lib/evilution/mutator/operator/optional_parameter_to_required.rb +41 -0
- data/lib/evilution/mutator/operator/proc_to_lambda.rb +43 -0
- data/lib/evilution/mutator/operator/receiver_constructor_swap.rb +51 -0
- data/lib/evilution/mutator/operator/reduce_to_sum.rb +58 -0
- data/lib/evilution/mutator/operator/regexp_anchor_to_predicate.rb +136 -0
- data/lib/evilution/mutator/operator/safe_navigation_removal.rb +67 -0
- data/lib/evilution/mutator/operator/send_mutation.rb +42 -6
- data/lib/evilution/mutator/operator/symbol_to_proc_replacement.rb +59 -0
- data/lib/evilution/mutator/operator/to_i_to_integer.rb +38 -0
- data/lib/evilution/mutator/operator/typed_default_return.rb +84 -0
- data/lib/evilution/mutator/primitives.rb +25 -0
- data/lib/evilution/mutator/registry.rb +32 -1
- data/lib/evilution/parallel/pool.rb +1 -0
- data/lib/evilution/parallel_db_warning.rb +1 -1
- data/lib/evilution/process_supervisor.rb +20 -8
- data/lib/evilution/rails_warmup.rb +61 -0
- data/lib/evilution/reporter/cli/item_formatters/neutral_group.rb +23 -0
- data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +42 -0
- data/lib/evilution/reporter/cli/item_formatters/subject_score_group.rb +16 -0
- data/lib/evilution/reporter/cli/line_formatters/infra_retry_notice.rb +19 -0
- data/lib/evilution/reporter/cli/line_formatters/result_line.rb +27 -3
- data/lib/evilution/reporter/cli/line_formatters/score.rb +19 -1
- data/lib/evilution/reporter/cli/line_formatters/unresolved_targets.rb +35 -0
- data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
- data/lib/evilution/reporter/cli/trailer.rb +11 -7
- data/lib/evilution/reporter/cli.rb +20 -4
- data/lib/evilution/reporter/html/report.rb +1 -0
- data/lib/evilution/reporter/json/subjects.rb +29 -0
- data/lib/evilution/reporter/json.rb +23 -1
- data/lib/evilution/result/coverage_gap_grouper.rb +1 -0
- data/lib/evilution/result/mutation_result.rb +3 -2
- data/lib/evilution/result/neutral_reason.rb +35 -0
- data/lib/evilution/result/subject_score.rb +28 -0
- data/lib/evilution/result/subject_scorer.rb +38 -0
- data/lib/evilution/result/summary.rb +44 -2
- data/lib/evilution/runner/baseline_runner.rb +17 -7
- data/lib/evilution/runner/canary.rb +52 -5
- data/lib/evilution/runner/isolation_resolver.rb +23 -2
- data/lib/evilution/runner/mutation_executor/infra_retry.rb +54 -0
- data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +23 -10
- data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +18 -1
- data/lib/evilution/runner/mutation_executor/result_cache.rb +11 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +19 -1
- data/lib/evilution/runner/mutation_executor.rb +32 -4
- data/lib/evilution/runner/report_publisher.rb +31 -9
- data/lib/evilution/runner/target_spec_audit.rb +43 -0
- data/lib/evilution/runner.rb +10 -1
- data/lib/evilution/source_ast_cache.rb +1 -1
- data/lib/evilution/spec_ast_cache.rb +1 -1
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution.rb +35 -0
- metadata +50 -2
data/README.md
CHANGED
|
@@ -21,9 +21,15 @@ Add to `Gemfile`:
|
|
|
21
21
|
gem "evilution", group: :test
|
|
22
22
|
```
|
|
23
23
|
|
|
24
|
-
Then:
|
|
24
|
+
Then:
|
|
25
|
+
```shell
|
|
26
|
+
bundle install
|
|
27
|
+
```
|
|
25
28
|
|
|
26
|
-
Or standalone:
|
|
29
|
+
Or standalone:
|
|
30
|
+
```shell
|
|
31
|
+
gem install evilution
|
|
32
|
+
```
|
|
27
33
|
|
|
28
34
|
Requires `prism >= 1.5, < 2`. Older Rails apps (e.g. Rails 7.1 pins `prism 0.19`) must upgrade prism — the gemspec constraint forces bundler to resolve a compatible 1.x version. If your app pins `prism 2.x`, bundler will reject the install until evilution widens its upper bound.
|
|
29
35
|
|
|
@@ -88,7 +94,7 @@ Every command, subcommand, and flag listed in this section is part of evilution'
|
|
|
88
94
|
| `init` | Generate `.evilution.yml` config file | |
|
|
89
95
|
| `version` | Print version string | |
|
|
90
96
|
| `subjects [files]` | List mutation subjects with locations and counts | |
|
|
91
|
-
| `tests list [files]` | List spec files
|
|
97
|
+
| `tests list [files]` | List the spec files `run` would use for each source (layout, `spec_mappings`, `spec_pattern`) | |
|
|
92
98
|
| `session list` | List saved session results | |
|
|
93
99
|
| `session show FILE` | Display detailed session results | |
|
|
94
100
|
| `session diff A B` | Compare two sessions (fixed/new/persistent) | |
|
|
@@ -104,6 +110,7 @@ Every command, subcommand, and flag listed in this section is part of evilution'
|
|
|
104
110
|
| `-t`, `--timeout N` | Integer | 30 | Per-mutation timeout in seconds. |
|
|
105
111
|
| `-f`, `--format FORMAT` | String | `text` | Output format: `text`, `json`, or `html`. |
|
|
106
112
|
| `--target EXPR` | String | _(none)_ | Only mutate matching methods. Supports method name (`Foo::Bar#calculate`), class (`Foo`), namespace wildcards (`Foo::Bar*`), method-type selectors (`Foo#`, `Foo.`), descendants (`descendants:Foo`), and source globs (`source:lib/**/*.rb`). |
|
|
113
|
+
| `--output FILE` | String | _(stdout)_ | Write the report to FILE instead of stdout. Useful when a preloaded spec helper writes to stdout on exit. |
|
|
107
114
|
| `--min-score FLOAT` | Float | 0.0 | Minimum mutation score (0.0–1.0) to pass. |
|
|
108
115
|
| `--spec FILES` | Array | _(none)_ | Spec files to run (comma-separated). Defaults to auto-detection via `SpecResolver`, which also resolves non-mirrored (`spec/unit`, `test/unit`), dir-grouped (`test/unit/<class>/*_test.rb`), and flat `test_`-prefixed (`test/test_connection_pool_timed_stack.rb`) layouts. |
|
|
109
116
|
| `--spec-dir DIR` | String | _(none)_ | Include all `*_spec.rb` files in DIR recursively. Composable with `--spec`. |
|
|
@@ -128,6 +135,7 @@ Every command, subcommand, and flag listed in this section is part of evilution'
|
|
|
128
135
|
| `--isolation MODE` | String | `auto` | Isolation strategy: `auto`, `fork`, or `in_process`. `auto` selects `fork` for Rails projects and packaged gems (`*.gemspec`), `in_process` otherwise. See [docs/isolation.md](docs/isolation.md). |
|
|
129
136
|
| `--preload FILE` | String | _(auto)_ | File to require in parent before forking workers. Auto-detect chain for Rails projects: `spec/rails_helper.rb` → `spec/spec_helper.rb` → `test/test_helper.rb`. For non-Rails gems: `spec/spec_helper.rb` → `test/test_helper.rb` → `test/helper.rb`, falling back to the gem entry `lib/<gem>.rb` (with a warning naming the searched paths). Pass `--no-preload` to opt out. |
|
|
130
137
|
| `--no-preload` | Boolean | _(enabled)_ | Disable parent-process preload. |
|
|
138
|
+
| `--warmup NAME` | String | `none` | After preload, warm lazily-initialised framework state once in the parent: `none` or `rails`. See [Warming up Rails before forking](#warming-up-rails-before-forking). |
|
|
131
139
|
| `--skip-heredoc-literals` | Boolean | false | Skip all string literal mutations inside heredocs. |
|
|
132
140
|
| `--show-disabled` | Boolean | false | Report mutations skipped by `# evilution:disable` comments. |
|
|
133
141
|
| `--fallback-full-suite` | Boolean | false | When no matching spec/test resolves for a mutation, run the whole test suite instead of marking it `:unresolved` and skipping. |
|
|
@@ -157,7 +165,7 @@ Every command, subcommand, and flag listed in this section is part of evilution'
|
|
|
157
165
|
|
|
158
166
|
Two profiles ship out of the box:
|
|
159
167
|
|
|
160
|
-
- **`default`** — the
|
|
168
|
+
- **`default`** — the 111 stable operators registered in `Mutator::Registry.default`. Suitable for everyday CI runs; balances coverage signal against survivor noise.
|
|
161
169
|
- **`strict`** — adds extra truthiness mutators on top of `default`. Currently `PredicateToNil` (replaces every `x.predicate?` call with `nil` to surface tests that only assert truthiness rather than exact return values). Use for pre-merge audits where you want maximum sensitivity at the cost of more survivors.
|
|
162
170
|
|
|
163
171
|
Set via `--profile=strict`, the `--strict` shortcut, or `profile: strict` in `.evilution.yml`.
|
|
@@ -167,9 +175,23 @@ Set via `--profile=strict`, the `--strict` shortcut, or `profile: strict` in `.e
|
|
|
167
175
|
| Code | Meaning | Agent action |
|
|
168
176
|
|------|-----------------------------------------------|---------------------------------------|
|
|
169
177
|
| 0 | Mutation score meets or exceeds `--min-score` | Success. No action needed. |
|
|
170
|
-
| 1 | Mutation score below `--min-score
|
|
178
|
+
| 1 | Mutation score below `--min-score`, or a target file resolved to no spec | Parse output, fix surviving mutants. |
|
|
171
179
|
| 2 | Tool error (bad config, parse failure, etc.) | Check stderr, fix invocation. |
|
|
172
180
|
|
|
181
|
+
`min_score` defaults to `0.0`, so **no score gate is armed unless you set one**. The `Result:` line says so rather than implying a threshold nobody configured:
|
|
182
|
+
|
|
183
|
+
```
|
|
184
|
+
$ evilution run lib/half_tested.rb # no gate
|
|
185
|
+
Result: 66.67% (no minimum score set) # exit 0
|
|
186
|
+
|
|
187
|
+
$ evilution run lib/half_tested.rb --min-score 0.8
|
|
188
|
+
Result: FAIL (score 66.67% < 80.00%) # exit 1
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
Evilution owns the exit status: `--preload` loads the project's own spec helper into the parent process, and an at-exit hook it installs (SimpleCov calls `exit` with its own status when coverage is below the minimum) would otherwise replace the status evilution computed (GH #1608).
|
|
192
|
+
|
|
193
|
+
The printed verdict and the exit code always use the same threshold. Previously the line was printed against a hard-coded 80% that the exit code did not share, so a failing-looking run still exited 0 (GH #1604).
|
|
194
|
+
|
|
173
195
|
## Configuration
|
|
174
196
|
|
|
175
197
|
Generate default config: `bundle exec evilution init`
|
|
@@ -187,6 +209,7 @@ schema_version: 1 # opts into strict validation (rejects unknown keys
|
|
|
187
209
|
# isolation: auto # auto | fork | in_process (auto selects fork for Rails + gems)
|
|
188
210
|
# canary: true # proof-of-life synthetic mutation at session start (false to skip)
|
|
189
211
|
# preload: null # path to preload before forking; false to disable; auto-detects for Rails + gems
|
|
212
|
+
# warmup: none # rails: warm I18n, routes, assets and templates once after preload (fork isolation)
|
|
190
213
|
# skip_heredoc_literals: false # skip string literal mutations inside heredocs (recommended for Rails: heredoc SQL/templates rarely have test coverage)
|
|
191
214
|
# show_disabled: false # report mutations skipped by disable comments
|
|
192
215
|
# baseline_session: null # path to session file for HTML comparison
|
|
@@ -224,6 +247,7 @@ All keys recognised under `schema_version: 1`:
|
|
|
224
247
|
| `timeout` | Integer | `30` | Per-mutation timeout in seconds. |
|
|
225
248
|
| `format` | String | `text` | Output format: `text`, `json`, `html`. |
|
|
226
249
|
| `target` | String / null | `null` | Filter expression: method (`Foo#bar`), class (`Foo`), namespace (`Foo*`), descendants (`descendants:Foo`), source glob (`source:**/*.rb`). |
|
|
250
|
+
| `output` | String | _(stdout)_ | Write the report to this file instead of stdout. |
|
|
227
251
|
| `min_score` | Float | `0.0` | Minimum mutation score (0.0–1.0) for exit code 0. |
|
|
228
252
|
| `integration` | String | `rspec` | Test framework: `rspec`, `minitest`, or `test_unit`. |
|
|
229
253
|
| `verbose` | Boolean | `false` | Verbose output (RSS/GC stats per phase, error details for errored mutations). |
|
|
@@ -245,6 +269,7 @@ All keys recognised under `schema_version: 1`:
|
|
|
245
269
|
| `related_specs_heuristic` | Boolean | `false` | Append related request/integration/feature/system specs for `includes(...)` mutations. |
|
|
246
270
|
| `fallback_to_full_suite` | Boolean | `false` | When no matching spec resolves, run the entire suite instead of marking the mutation `:unresolved`. |
|
|
247
271
|
| `preload` | String / Boolean / null | `null` | File to preload in parent before forking. `false` to disable. `null` to auto-detect for Rails projects and packaged gems. |
|
|
272
|
+
| `warmup` | String | `none` | `rails` warms lazily-initialised Rails state once after preload so forks do not each pay it. `none` / `false` to skip. |
|
|
248
273
|
| `spec_mappings` | Hash<String, String/Array> | `{}` | Custom mapping from source path to spec path(s). |
|
|
249
274
|
| `spec_pattern` | String / null | `null` | Glob restricting resolved spec candidates. |
|
|
250
275
|
| `example_targeting` | Boolean | `true` | Per-mutation example-level targeting. |
|
|
@@ -299,6 +324,8 @@ Schema:
|
|
|
299
324
|
"neutral": "integer — mutations whose tests already failed before mutation (baseline failure)",
|
|
300
325
|
"equivalent": "integer — mutations proven to have identical behavior to the original",
|
|
301
326
|
"unresolved": "integer — mutations where no spec file resolved (coverage gap, not a failure)",
|
|
327
|
+
"unresolved_target_files": "array of strings (optional) — target files that resolved to no spec at all; present only when non-empty, and the run fails when it is",
|
|
328
|
+
"infra_retried": "integer (optional) — mutations a parallel pass could not judge because the test process crashed on infrastructure, re-run serially afterwards; present only when non-zero",
|
|
302
329
|
"unparseable": "integer — mutations whose mutated source did not parse (short-circuited, never executed)",
|
|
303
330
|
"score": "float — killed / (total - errors - neutral - equivalent - unresolved - unparseable), range 0.0-1.0, rounded to 4 decimals",
|
|
304
331
|
"duration": "float — total wall-clock seconds, rounded to 4 decimals",
|
|
@@ -313,7 +340,20 @@ Schema:
|
|
|
313
340
|
"duration": "float — seconds this mutation took, rounded to 4 decimals",
|
|
314
341
|
"diff": "string — legacy +/- diff snippet",
|
|
315
342
|
"unified_diff": "string (optional, survived only) — git-style unified diff with `--- a/file`, `+++ b/file`, `@@` hunk header and sdiff body; omitted when source slices are unavailable",
|
|
316
|
-
"suggestion": "string — actionable hint for surviving mutants (survived only)"
|
|
343
|
+
"suggestion": "string — actionable hint for surviving mutants (survived only)",
|
|
344
|
+
"neutral_reason": "object (optional, neutral only) — { kind: 'baseline_failure' | 'infra_error', detail: string|null — the failing spec file or the crash class; null when the run was given explicit --spec files and no single spec can be named }"
|
|
345
|
+
}
|
|
346
|
+
],
|
|
347
|
+
"subjects": [
|
|
348
|
+
{
|
|
349
|
+
"name": "string — subject name (e.g. 'Foo#bar')",
|
|
350
|
+
"file": "string — relative path to source file",
|
|
351
|
+
"total": "integer — mutations generated for this subject",
|
|
352
|
+
"killed": "integer — mutations detected",
|
|
353
|
+
"verified": "integer — mutations that got a verdict (killed + survived + timed out)",
|
|
354
|
+
"survived": "integer — mutations that went undetected",
|
|
355
|
+
"score": "float — killed / verified, 0.0 when nothing was verified, rounded to 4 decimals",
|
|
356
|
+
"reached": "boolean — false when no mutation of this subject got a verdict at all"
|
|
317
357
|
}
|
|
318
358
|
],
|
|
319
359
|
"coverage_gaps": [
|
|
@@ -361,6 +401,10 @@ Sessions saved by `--save-session` (under `.evilution/results/*.json`) and consu
|
|
|
361
401
|
|
|
362
402
|
Saved sessions also omit the per-status arrays (`killed`, `neutral`, `equivalent`, `unresolved`, `unparseable`, `timed_out`, `errors`) — only `survived` and `coverage_gaps` are persisted. The score, totals, and timestamps are stable for diff/compare consumers.
|
|
363
403
|
|
|
404
|
+
#### stdout in JSON mode
|
|
405
|
+
|
|
406
|
+
With `--format json`, stdout carries the JSON document and nothing else. Each mutation's test run writes to buffers evilution owns, whatever the isolation mode: under `in_process` the framework's configuration can outlive a single run — `--preload` builds it before isolation swaps `$stdout` — so the run claims RSpec's output and error streams outright rather than relying on RSpec to redirect them (GH #1627). Once the document is written, stdout is pointed at stderr, so anything a preloaded spec helper prints on the way out — SimpleCov's coverage report, for example — lands on stderr instead of after the document where it would leave `JSON.parse` with nothing to work with. `--output FILE` writes the document to a file and leaves stdout alone entirely.
|
|
407
|
+
|
|
364
408
|
#### Schema versioning
|
|
365
409
|
|
|
366
410
|
Every session and stdout JSON document carries a top-level `schema_version` integer (currently `1`). On read:
|
|
@@ -383,14 +427,70 @@ Compatibility policy for the `1.x` gem line:
|
|
|
383
427
|
| `survived` | No test failed — gap in coverage | denominator only |
|
|
384
428
|
| `timeout` | Test run exceeded `--timeout` — treated like survived for scoring | denominator only |
|
|
385
429
|
| `error` | Mutation caused an unexpected error (syntax error, boot failure, etc.) | excluded from denominator |
|
|
386
|
-
| `neutral` | Baseline tests already failed before mutation
|
|
430
|
+
| `neutral` | Baseline tests already failed before mutation, or the test process crashed on infrastructure (DB lock, statement timeout) rather than on the mutation. Every neutral records which of the two, and the report groups them by it | excluded |
|
|
387
431
|
| `equivalent` | Mutation is provably identical to the original (e.g. no-op replacement) | excluded |
|
|
388
432
|
| `unresolved` | No spec file resolved for the mutated source — **coverage gap, not a failure**. Use `--fallback-full-suite` to run the full suite instead. | excluded |
|
|
389
433
|
| `unparseable` | Mutated source failed to parse (e.g. dangling heredoc opener after `method_body_replacement`). Short-circuited — never executed. | excluded |
|
|
390
434
|
|
|
391
435
|
Unresolved mutations indicate a missing test mapping — the file has no corresponding test file that the resolver could find (for example, an RSpec `_spec.rb` file or a Minitest `_test.rb` file, depending on configuration). The resolver searches the `lib/`-mirrored path, common non-mirrored buckets (`spec/unit`, `spec/lib`, `test/unit`, `test/lib`), and the flat `test_`-prefixed Minitest/Test::Unit convention (`test/test_connection_pool_timed_stack.rb`), so a high unresolved rate usually means a genuinely missing or unconventionally-placed test; a run that leaves many mutations unresolved prints an unresolved-rate warning with a best-guess spec path per source file. They are reported separately so you can act on them (add a test, adjust test naming, pass `--spec`, or opt in to the full-suite fallback) without inflating the error count.
|
|
392
436
|
|
|
393
|
-
|
|
437
|
+
A *target file* that resolves to no test at all is a stronger condition than an individual unresolved mutation, and is reported on its own terms: evilution names the file and **fails the run**, whatever the mutations it did measure scored. Without this, adding one well-tested file to the command dilutes the unresolved rate and the untested file disappears behind a `PASS` (GH #1603):
|
|
438
|
+
|
|
439
|
+
```
|
|
440
|
+
$ evilution run app/services/untested.rb app/services/well_tested.rb
|
|
441
|
+
Mutations: 98 total, 82 killed, 0 survived, 0 timed out, 2 neutral, 14 unresolved
|
|
442
|
+
Score: 100.00% (82/82)
|
|
443
|
+
! 1 of 2 target files has no resolvable spec — it was never tested:
|
|
444
|
+
app/services/untested.rb
|
|
445
|
+
Result: FAIL (1 target file has no resolvable spec)
|
|
446
|
+
$ echo $?
|
|
447
|
+
1
|
|
448
|
+
```
|
|
449
|
+
|
|
450
|
+
The check covers files evilution found something to mutate in; a file it produced no mutations for (a constants-only file, say) is not reported. `--fallback-full-suite` runs such a file against the whole suite instead, so nothing goes untested and nothing is reported. The file list is also in JSON output under `summary.unresolved_target_files`.
|
|
451
|
+
|
|
452
|
+
### Survivor Confirmation
|
|
453
|
+
|
|
454
|
+
Per-mutation targeting runs a subset of a spec file's examples, chosen by matching the enclosing method's name against example bodies. That match keys on identifier text, so it can miss the example that would have caught a mutation — and the mutation is then reported as a survivor the suite actually covers. A survivor nobody can reproduce is worse than a missed kill: it sends the reader to write a test that is already there (GH #1624).
|
|
455
|
+
|
|
456
|
+
Before any survivor is reported, it is therefore re-run against the whole resolved spec file, and that run is the one reported. Only survivors pay for the extra run, and only where the targeted subset was narrower than the file; a mutation the targeted examples already killed is never re-run.
|
|
457
|
+
|
|
458
|
+
On evilution's own `lib/evilution/reporter/json/subjects.rb` this moved the reported score from 74.19% with 8 survivors to 93.55% with 2 — the six that disappeared were killed by an example in the same file all along, and the run now agrees with `--no-example-targeting` instead of contradicting it.
|
|
459
|
+
|
|
460
|
+
### Neutral Mutations
|
|
461
|
+
|
|
462
|
+
Neutral covers two unrelated situations that want opposite responses: a spec file that was already red before any mutation ran, and a test process that died on infrastructure rather than on the mutation. Each neutral records which, and the report groups by it, naming the spec or the error class (GH #1606):
|
|
463
|
+
|
|
464
|
+
```
|
|
465
|
+
Score: 100.00% (10/10 verified of 17 mutations, 7 neutral)
|
|
466
|
+
|
|
467
|
+
Neutral mutations (7, not verified):
|
|
468
|
+
baseline already failing (spec/tally_spec.rb):
|
|
469
|
+
arithmetic_replacement: lib/tally.rb:9
|
|
470
|
+
integer_literal: lib/tally.rb:9
|
|
471
|
+
```
|
|
472
|
+
|
|
473
|
+
The score line names the remainder whenever the run left mutations out of the denominator, because full marks over a fraction of a run otherwise reads as a verdict on all of it. A clean run still prints the plain `Score: 100.00% (17/17)`.
|
|
474
|
+
|
|
475
|
+
Those seven mutations were survivors until the spec file went red — a neutral of this kind is a hidden coverage gap, not a clean bill of health. JSON output carries `neutral_reason` as `{ kind, detail }` on neutral entries that have one; `detail` is null where no single spec can be named (an explicit `--spec` run), and the field is absent on a result recorded without a reason, which the text report shows as `reason not recorded`.
|
|
476
|
+
|
|
477
|
+
### Per-Subject Scores
|
|
478
|
+
|
|
479
|
+
The run's score is computed per file, and test selection resolves per file too, so a well-tested file that gains new untested methods still reports 100% — the number only ever describes what the resolved spec reaches (GH #1605). Every report therefore breaks the run down per subject, and the text report names the subjects the file-level score does not speak for:
|
|
480
|
+
|
|
481
|
+
```
|
|
482
|
+
Mutations: 33 total, 7 killed, 0 survived, 0 timed out, 26 unresolved
|
|
483
|
+
Score: 100.00% (7/7)
|
|
484
|
+
|
|
485
|
+
Subjects needing attention (2 subjects in 1 file):
|
|
486
|
+
lib/helper.rb
|
|
487
|
+
#summary_for 0.00% (0/17) nothing reached this subject
|
|
488
|
+
#total_for 0.00% (0/9) nothing reached this subject
|
|
489
|
+
```
|
|
490
|
+
|
|
491
|
+
A subject is listed when something survived, or when nothing reached it at all — zero verdicts, every mutation unresolved or neutral. Fully-killed subjects are not listed, so the section stays actionable. JSON output carries every subject under `subjects`, whether or not it needs attention, so a CI step can assert on `reached` or on a per-subject `score`.
|
|
492
|
+
|
|
493
|
+
## Mutation Operators (111 total)
|
|
394
494
|
|
|
395
495
|
Each operator name is stable and appears in JSON output under `survived[].operator`.
|
|
396
496
|
|
|
@@ -423,6 +523,29 @@ Each operator name is stable and appears in JSON output under `survived[].operat
|
|
|
423
523
|
| `method_call_removal` | Remove method calls, keep receiver | `obj.foo(x)` -> `obj` |
|
|
424
524
|
| `argument_removal` | Remove individual arguments | `foo(a, b)` -> `foo(b)` |
|
|
425
525
|
| `argument_nil_substitution` | Replace arguments with `nil` | `foo(a, b)` -> `foo(nil, b)` |
|
|
526
|
+
| `call_to_nil` | Replace a method call with `nil` (skips void statements, receivers of another call, and attribute or index writes) | `user.name` -> `nil` |
|
|
527
|
+
| `safe_navigation_removal` | Replace `&.` with a plain call (skips `self` and literal receivers) | `user&.name` -> `user.name` |
|
|
528
|
+
| `attribute_write_to_read` | Replace an attribute or index write with the matching read (skips writes `statement_deletion` already removes) | `a.foo = b` -> `a.foo`, `a[i] = b` -> `a[i]` |
|
|
529
|
+
| `argument_propagation` | Replace a call with its only positional argument (skips operator methods, attribute writes and void statements) | `normalize(value)` -> `value` |
|
|
530
|
+
| `argument_list_removal` | Drop a call's whole argument list, keeping any block (skips operator methods and attribute writes) | `foo(a, b)` -> `foo`, `foo(a, &b)` -> `foo(&b)` |
|
|
531
|
+
| `symbol_to_proc_replacement` | Apply the `send_mutation` and `collection_replacement` selector tables to a symbol block-pass (skips alias swaps) | `map(&:to_s)` -> `map(&:to_i)` |
|
|
532
|
+
| `dynamic_dispatch_resolution` | Resolve `send` / `__send__` / `public_send` with a literal symbol into a direct call, only where visibility checks can differ | `obj.send(:m, a)` -> `obj.m(a)` |
|
|
533
|
+
| `keyword_argument_removal` | Drop one `key: value` argument at a call site (keeps `**opts`; skips a lone keyword) | `f(a: 1, b: 2)` -> `f(b: 2)` |
|
|
534
|
+
| `double_negation_removal` | Replace a double negation with its operand | `!!x` -> `x` |
|
|
535
|
+
| `regexp_anchor_to_predicate` | Rewrite a match against an anchored literal regexp as a prefix / suffix predicate (skips `match?` with `\A` / `\z`, an exact equivalent) | `x =~ /^foo/` -> `x.start_with?("foo")` |
|
|
536
|
+
| `reduce_to_sum` | Rewrite a `+` reduction (`reduce` / `inject` with `:+` or `&:+`) into `sum` | `xs.reduce(:+)` -> `xs.sum`, `xs.inject(0, :+)` -> `xs.sum(0)` |
|
|
537
|
+
| `array_coercion_to_literal` | Replace an `Array()` coercion with an array literal | `Array(x)` -> `[x]` |
|
|
538
|
+
| `to_i_to_integer` | Replace lenient `to_i` with strict `Integer()` (skips numeric literal receivers) | `x.to_i` -> `Integer(x)`, `x.to_i(16)` -> `Integer(x, 16)` |
|
|
539
|
+
| `coercion_emptying` | Replace a conversion with an empty value of its type (`to_a` / `to_ary`, `to_h` / `to_hash`, `to_s` / `to_str`) | `x.to_s` -> `""`, `x.to_h` -> `{}` |
|
|
540
|
+
| `const_get_to_constant_path` | Resolve `const_get` with a literal symbol into a constant path | `X.const_get(:Y)` -> `X::Y` |
|
|
541
|
+
| `proc_to_lambda` | Turn `proc { }` / `Proc.new { }` into `lambda { }` (arity checks, local `return`) | `proc { \|x\| x }` -> `lambda { \|x\| x }` |
|
|
542
|
+
| `dig_to_fetch_chain` | Make the first step of a multi-key `dig` strict | `x.dig(a, b)` -> `x.fetch(a).dig(b)` |
|
|
543
|
+
| `inequality_to_negated_identity` | Rewrite `!=` with the stricter `eql?` / `equal?` (skips nil, boolean and symbol literals) | `a != b` -> `!a.eql?(b)`, `!a.equal?(b)` |
|
|
544
|
+
| `binary_operand_promotion` | Replace an arithmetic or bitwise expression with one of its operands (skips integer identities and void statements) | `a + b` -> `a`, `b` |
|
|
545
|
+
| `receiver_constructor_swap` | Swap `Date` / `DateTime` / `Time.parse` for a stricter sibling constructor on the same receiver | `Date.parse(s)` -> `Date.iso8601(s)`, `Date.strptime(s)`, ... |
|
|
546
|
+
| `block_body_to_nil` | Replace a block body with `nil`, keeping the iteration (skips `loop` and endless `cycle` / `cycle(nil)`, which would hang) | `xs.each { \|x\| log(x) }` -> `xs.each { \|x\| nil }` |
|
|
547
|
+
| `block_body_to_raise` | Replace a block body with `raise` to prove the block is invoked (skips bodies with a `rescue` clause, which would swallow it) | `xs.each { \|x\| log(x) }` -> `xs.each { \|x\| raise }` |
|
|
548
|
+
| `block_body_promotion` | Replace a call with its parameter-less block body, run once (skips blocks with parameters or a `rescue` / `ensure` clause) | `Base.transaction { save! }` -> `save!`, `3.times { poll }` -> `poll` |
|
|
426
549
|
| `keyword_argument` | Remove keyword defaults/params | `def foo(bar: 42)` -> `def foo(bar:)` |
|
|
427
550
|
| `multiple_assignment` | Remove targets or swap order | `a, b = 1, 2` -> `b, a = 1, 2` |
|
|
428
551
|
| `block_removal` | Remove blocks from method calls | `items.map { \|x\| x * 2 }` -> `items.map` |
|
|
@@ -464,6 +587,14 @@ Each operator name is stable and appears in JSON output under `survived[].operat
|
|
|
464
587
|
| `regex_capture` | Swap or nil-ify capture refs | `$1` -> `$2`, `$1` -> `nil` |
|
|
465
588
|
| `loop_flip` | Swap while/until loops | `while cond` -> `until cond` |
|
|
466
589
|
| `loop_body_to_raise` | Replace a loop body with `raise` | `while c; body; end` -> `while c; raise; end` |
|
|
590
|
+
| `method_body_to_raise` | Replace a whole method body with `raise` | `def foo; body; end` -> `def foo; raise; end` |
|
|
591
|
+
| `method_body_to_super` | Replace a method body with bare `super` where a super target exists | `def foo; body; end` -> `def foo; super; end` |
|
|
592
|
+
| `typed_default_return` | Replace a single-expression body with the empty value of its inferred type | `def names(u); u.map(&:name); end` -> `def names(u); []; end` |
|
|
593
|
+
| `block_parameter_drop` | Drop a block's single parameter | `users.each { |u| touch(u) }` -> `users.each { touch(u) }` |
|
|
594
|
+
| `optional_parameter_to_required` | Drop an optional positional parameter's default | `def f(a = 1)` -> `def f(a)` |
|
|
595
|
+
| `optional_default_injection` | Overwrite an optional parameter with its own default at the top of the body | `def f(a = 1); body; end` -> `def f(a = 1); a = 1; body; end` |
|
|
596
|
+
| `block_destructuring_expansion` | Flatten a destructuring group in a block's parameters | `pairs.each_with_index { |(k, v), i| use(k, v, i) }` -> `pairs.each_with_index { |k, v, i| use(k, v, i) }` |
|
|
597
|
+
| `forwarding_super_to_explicit` | Give a forwarding `super` an empty argument list | `def f(a); super; end` -> `def f(a); super(); end` |
|
|
467
598
|
| `string_interpolation` | Replace interpolation content with nil | `"hello #{name}"` -> `"hello #{nil}"` |
|
|
468
599
|
| `retry_removal` | Remove retry statements | `retry` -> `nil` |
|
|
469
600
|
| `case_when` | Remove/replace case/when branches | Remove `when` branch, drop one condition from `when a, b`, body -> `nil`, empty body -> `raise`, remove `else` |
|
|
@@ -520,6 +651,12 @@ The `evilution-mutate` tool accepts a `verbosity` parameter to control response
|
|
|
520
651
|
|
|
521
652
|
Use `minimal` when context window budget is tight and you only need to see what survived. The trimmed `errors` sample (each entry: `error_message`, `error_class`, location, plus the first 5 backtrace lines) is added so a partly-broken run is still self-diagnosable without escalating verbosity. Use `full` when you need to inspect killed/neutral/equivalent entries for debugging.
|
|
522
653
|
|
|
654
|
+
What survives trimming matters when you are deciding whether to trust a score:
|
|
655
|
+
|
|
656
|
+
- `neutral` entries — and with them each `neutral_reason` — are dropped at `summary` and `minimal`. Use `full` to see why mutations were neutralised.
|
|
657
|
+
- `subjects` is kept at `full` and `summary`, and dropped at `minimal`, which keeps only `summary` and `survived`.
|
|
658
|
+
- Everything inside `summary` survives at every level, including `unresolved_target_files`, `infra_retried` and the `neutral` count — so even a `minimal` response still says whether a target file went untested and how much of the run the score covers.
|
|
659
|
+
|
|
523
660
|
### Enriched Survived Entries
|
|
524
661
|
|
|
525
662
|
Unlike `evilution --format json`, every survived entry returned by `evilution-mutate` carries extra fields so the agent can act without a second round-trip:
|
|
@@ -605,7 +742,7 @@ Per-tool placement:
|
|
|
605
742
|
- **`evilution-session` `list`** — `{ "schema_version": Integer, "sessions": Array<{ file, timestamp, total, killed, survived, score, duration }> }`. Sessions are reverse-chronological; the array is filtered by `limit` when provided.
|
|
606
743
|
- **`evilution-session` `show`** — the parsed session JSON document, exactly as written under `.evilution/results/*.json`. Field reference: see [Session JSON files](#session-json-files).
|
|
607
744
|
- **`evilution-session` `diff`** — `{ "schema_version": Integer, "summary": { base_score, head_score, score_delta, base_survived, head_survived, base_total, head_total, base_killed, head_killed }, "fixed": Array, "new_survivors": Array, "persistent": Array }`. The mutation arrays carry the same per-mutation fields the session `survived` list uses (`operator`, `file`, `line`, `subject`, `diff`).
|
|
608
|
-
- **`evilution-info` `subjects`** — `{ "schema_version": Integer, "subjects": Array<{ name, file, line, mutations }>, "total_subjects": Integer, "total_mutations": Integer }`.
|
|
745
|
+
- **`evilution-info` `subjects`** — `{ "schema_version": Integer, "subjects": Array<{ name, file, line, mutations }>, "total_subjects": Integer, "total_mutations": Integer }`. Discovery only: this lists what *can* be mutated. The `subjects` array in a run's report is a different shape, carrying what each subject scored.
|
|
609
746
|
- **`evilution-info` `tests`** — `{ "schema_version": Integer, "specs": Array<{ source, spec }>, "unresolved": Array<String>, "total_sources": Integer, "total_specs": Integer }`.
|
|
610
747
|
- **`evilution-info` `environment`** — `{ "schema_version": Integer, "version": String, "ruby": String, "config_file": String|null, ... }` mirroring the effective `Evilution::Config`.
|
|
611
748
|
- **`evilution-info` `statuses`** — `{ "schema_version": Integer, "statuses": Array<{ name, meaning, in_score }> }`.
|
|
@@ -636,7 +773,7 @@ When a parameter, action, or output field on the public MCP contract is deprecat
|
|
|
636
773
|
bundle exec evilution run lib/ --format json --min-score 0.8
|
|
637
774
|
```
|
|
638
775
|
|
|
639
|
-
Parse JSON output. Exit code 0 = pass, 1 =
|
|
776
|
+
Parse JSON output. Exit code 0 = pass, 1 = fail — either the score is below `--min-score`, or a target file resolved to no spec and was never tested (`summary.unresolved_target_files` names them). Without `--min-score` no score gate is armed; the run still fails on an untested target file.
|
|
640
777
|
|
|
641
778
|
### 2. PR / changed-lines scan (fast feedback)
|
|
642
779
|
|
|
@@ -691,8 +828,23 @@ bundle exec evilution run lib/models/user.rb lib/models/account.rb lib/models/or
|
|
|
691
828
|
|
|
692
829
|
Pass multiple file paths on a single invocation to amortise startup cost. The framework (Rails, Sorbet, etc.) and the `preload` chain (`spec/rails_helper.rb` → `spec/spec_helper.rb` → `test/test_helper.rb`) load **once** in the parent process. When `--isolation=fork` is selected (the default `--isolation=auto` resolves to `fork` on Rails projects and packaged gems), every subsequent mutation across all files forks from that warmed parent — materially faster than scripting a `for f in ...; do bundle exec evilution run "$f"; done` loop, which pays the bootstrap per file. With `--isolation=in_process` (default for non-Rails, non-gem projects under `auto`), there is no per-mutation fork, but the parent-process boot still runs once instead of N times. Per-file paths and line numbers are preserved in the report (`survived[].file`, HTML grouping by source file).
|
|
693
830
|
|
|
831
|
+
### What to read before acting on a score
|
|
832
|
+
|
|
833
|
+
A score describes only the mutations that got a verdict. Four fields say what it leaves out, and each points at a different action:
|
|
834
|
+
|
|
835
|
+
| Field | Meaning | What to do |
|
|
836
|
+
|---|---|---|
|
|
837
|
+
| `summary.unresolved_target_files` | A file you named resolved to no spec and was never tested; the run fails on this alone | Write a spec, pass `--spec`, or map it in `spec_mappings` — do not trust the score until this is empty |
|
|
838
|
+
| `subjects[].reached == false` | Mutations were generated for that method but none got a verdict | The method is untested even where its file scores well; start here rather than with `survived[]` |
|
|
839
|
+
| `neutral[].neutral_reason.kind == "baseline_failure"` | The spec file was already red before any mutation ran; `detail` names it | Fix that spec first — nothing about these mutations is measurable until it is green |
|
|
840
|
+
| `neutral[].neutral_reason.kind == "infra_error"` | The test process crashed on infrastructure (DB lock, timeout); `detail` names the class | Not a coverage gap. Give parallel workers their own database, or run `-j 1` |
|
|
841
|
+
|
|
842
|
+
`summary.infra_retried` reports how many mutations had to be re-run serially because of the last case; a large number means the parallel run was fighting shared infrastructure rather than measuring your suite.
|
|
843
|
+
|
|
694
844
|
### 6. Fixing surviving mutants
|
|
695
845
|
|
|
846
|
+
Every entry in `survived[]` has already been re-run against the whole resolved spec file, so it is a gap the suite genuinely does not cover rather than an artefact of per-mutation example targeting.
|
|
847
|
+
|
|
696
848
|
For each entry in `survived[]`:
|
|
697
849
|
1. Read `file` at `line` to understand the code context
|
|
698
850
|
2. Read `operator` to understand what was changed
|
|
@@ -720,6 +872,17 @@ bundle exec evilution mutate lib/aasm/base.rb --spec spec/unit/event_spec.rb,spe
|
|
|
720
872
|
bundle exec evilution mutate lib/aasm/base.rb --fallback-full-suite
|
|
721
873
|
```
|
|
722
874
|
|
|
875
|
+
To make the mapping permanent, declare it once in `.evilution.yml` under `spec_mappings`; the run, its baseline and `tests list` all honour it. `evilution tests list <file>` prints exactly the spec files a run would use, so it is the quick way to check a mapping before spending a run on it:
|
|
876
|
+
|
|
877
|
+
```yaml
|
|
878
|
+
spec_mappings:
|
|
879
|
+
lib/aasm/base.rb:
|
|
880
|
+
- spec/unit/event_spec.rb
|
|
881
|
+
- spec/unit/callbacks_spec.rb
|
|
882
|
+
```
|
|
883
|
+
|
|
884
|
+
Without `--fallback-full-suite` an unresolved source is also left out of the baseline, which does not run the whole test directory for it.
|
|
885
|
+
|
|
723
886
|
### Long Minitest fork runs — not a hang
|
|
724
887
|
|
|
725
888
|
Minitest projects under `--isolation=fork` re-bootstrap the test environment (`test_helper.rb`, plugins, runnable state) once per mutation. On constant-heavy files (e.g. Shopify/liquid's `lib/liquid/lexer.rb`, ~270 mutations) the wall-clock cost is dominated by that per-fork bootstrap and any mutations that hit a `--timeout` rather than killing the test fast. A single-worker run (`-j 1`) on a few hundred mutations can take 4+ minutes; combined with `--no-progress` and a non-TTY stderr (CI, redirected logs) the run looks silent the entire time.
|
|
@@ -733,7 +896,7 @@ RUBYOPT="-Itest" bundle exec evilution mutate lib/<file>.rb \
|
|
|
733
896
|
--spec test/<dir>/<file>_test.rb
|
|
734
897
|
```
|
|
735
898
|
|
|
736
|
-
`-j 4` parallelises across workers, `-t 10` caps any mutation that pathologically loops at 10 s. Expect the run to print progress only when stderr is a TTY (use `bundle exec evilution mutate ... 2>&1 | tee log` to get progress while still saving output). The historical "Minitest fork hangs on liquid" report (
|
|
899
|
+
`-j 4` parallelises across workers, `-t 10` caps any mutation that pathologically loops at 10 s. On a Rails app, add `--warmup rails` so each fork does not re-initialise I18n, routes and templates on its first request (see [Warming up Rails before forking](#warming-up-rails-before-forking)). Expect the run to print progress only when stderr is a TTY (use `bundle exec evilution mutate ... 2>&1 | tee log` to get progress while still saving output). The historical "Minitest fork hangs on liquid" report (GH #1211) turned out to be a slow run + silent UX, not an actual deadlock — the worker logs show steady forward progress when captured via `--quiet-children --quiet-children-dir DIR`.
|
|
737
900
|
|
|
738
901
|
### 8. CI gate
|
|
739
902
|
|
|
@@ -742,7 +905,9 @@ bundle exec evilution run lib/ --format json --min-score 0.8 --quiet
|
|
|
742
905
|
# Exit code 0 = pass, 1 = fail, 2 = error
|
|
743
906
|
```
|
|
744
907
|
|
|
745
|
-
|
|
908
|
+
Exit 1 covers two conditions: the score missed `--min-score`, and a target file that resolved to no spec. The second fails the run whether or not a score gate is set, so a CI step that names files explicitly cannot silently stop testing one of them.
|
|
909
|
+
|
|
910
|
+
Note: `--quiet` suppresses all stdout output (including JSON). Use it in CI only when you care about the exit code and do not need JSON output. `--output FILE` is the alternative when a preloaded spec helper (SimpleCov, for example) writes to stdout on exit.
|
|
746
911
|
|
|
747
912
|
### 9. Regression tracking across runs (`compare`)
|
|
748
913
|
|
|
@@ -769,9 +934,32 @@ Output buckets:
|
|
|
769
934
|
|
|
770
935
|
Use in CI to gate merges on `reintroduced` being empty, or to surface `new` survivors for reviewer attention without failing the build on `persistent` debt.
|
|
771
936
|
|
|
937
|
+
## Warming up Rails before forking
|
|
938
|
+
|
|
939
|
+
Under `isolation: fork` the parent preloads once and every mutation runs in a fresh fork, which inherits only what the parent had already initialised. Rails initialises a lot lazily on a process's first request: loading locale files, building route helpers, resolving asset paths, compiling templates. Without a warm-up, every forked mutation whose tests make a request pays that cold start again, often several hundred milliseconds each.
|
|
940
|
+
|
|
941
|
+
`warmup: rails` (or `--warmup rails`) does that initialisation once in the parent, right after the preload:
|
|
942
|
+
|
|
943
|
+
- `I18n.eager_load!`
|
|
944
|
+
- the route set's `url_helpers` (and `eager_load!` for lazily drawn routes on Rails 8)
|
|
945
|
+
- an asset path lookup through the controller helpers
|
|
946
|
+
- template precompilation, only if the [`actionview_precompiler`](https://github.com/jhawthorn/actionview_precompiler) gem is installed
|
|
947
|
+
|
|
948
|
+
Each step is skipped if its library is not loaded, and skipped with a one-line note if it raises, so an app that does not fit a step still runs normally.
|
|
949
|
+
|
|
950
|
+
Do not warm up by sending a real request from your preload (`Rails.application.call(...)`). Controller callbacks change process-global state that every fork then inherits: `I18n.locale`, gem request stores such as PaperTrail's, a session row in the database. Tests that depend on that state then fail in every fork, and evilution reports their mutations as killed. The steps above make no request and leave that state untouched.
|
|
951
|
+
|
|
772
952
|
## Parallel Runs with SQLite
|
|
773
953
|
|
|
774
|
-
Running with `-j N` forks worker processes. If your Rails app uses SQLite, every worker opens the same `db/test.sqlite3` file, and concurrent writers collide on the database-level lock. Symptoms: `ActiveRecord::StatementTimeout`, `SQLite3::BusyException`, and slow runs. Evilution classifies these crashes as `:neutral` (see [
|
|
954
|
+
Running with `-j N` forks worker processes. If your Rails app uses SQLite, every worker opens the same `db/test.sqlite3` file, and concurrent writers collide on the database-level lock. Symptoms: `ActiveRecord::StatementTimeout`, `SQLite3::BusyException`, and slow runs. Evilution classifies these crashes as `:neutral` (see [GH #814](https://github.com/marinazzio/evilution/issues/814)) so the mutation score is not polluted, but the wall-clock penalty remains.
|
|
955
|
+
|
|
956
|
+
Because that contention only exists while several workers are running, those mutations are re-run one at a time once the pool is done, and the verdict from the quiet re-run is the one reported. Without it the neutral bucket moved with `-j` on identical input — the same files scoring 24 killed / 0 neutral at `-j 1` and 6 killed / 18 neutral at `-j 4` (GH #1607). The run says how much it had to redo:
|
|
957
|
+
|
|
958
|
+
```
|
|
959
|
+
! 18 mutations hit infrastructure errors under parallel workers; re-ran them serially.
|
|
960
|
+
```
|
|
961
|
+
|
|
962
|
+
The count is in JSON output as `summary.infra_retried`. Such a crash is also never written to the `--incremental` cache: the cache keeps no error class, so a cached `:killed` would be indistinguishable from a real one on the next run and would short-circuit both the demotion and the retry. A neutral from a failing baseline is never re-run — that is a real statement about the spec, not a missed verdict. The retry costs wall-clock time in proportion to the contention, which is another reason to give each worker its own database file:
|
|
775
963
|
|
|
776
964
|
Evilution follows the [`parallel_tests`](https://github.com/grosser/parallel_tests) convention: each worker receives a `TEST_ENV_NUMBER` environment variable (`""` for worker 1, `"2"` for worker 2, `"3"` for worker 3, …). Interpolate it into `config/database.yml` so each worker gets its own SQLite file:
|
|
777
965
|
|
|
@@ -816,7 +1004,7 @@ points — see [docs/architecture.md](docs/architecture.md).
|
|
|
816
1004
|
1. **Parse** — Prism parses Ruby files into ASTs with exact byte offsets
|
|
817
1005
|
2. **Extract** — Methods are identified as mutation subjects
|
|
818
1006
|
3. **Filter** — Disable comments, Sorbet `sig` blocks, and AST ignore patterns exclude mutations before execution
|
|
819
|
-
4. **Mutate** —
|
|
1007
|
+
4. **Mutate** — 111 operators produce text replacements at precise byte offsets (source-level surgery, no AST unparsing); heredoc literal text is skipped by default. Identical byte-mutations from different operators are deduplicated by `(file_path, mutated_source)` so the count is not inflated by overlap
|
|
820
1008
|
5. **Isolate** — Mutations are applied to temporary file copies (never modifying originals); load-path redirection ensures `require` resolves the mutated copy. Default isolation is in-process for plain Ruby projects (no gemspec) and fork for Rails projects and packaged gems (auto-detected); `--isolation fork` forces forked child processes. Both sequential and parallel (`--jobs N`) modes respect the configured isolation strategy
|
|
821
1009
|
6. **Test** — The configured test framework (RSpec, Minitest, or Test::Unit) executes against the mutated source
|
|
822
1010
|
7. **Collect** — Source strings and AST nodes are released after use to minimize memory retention
|
data/docs/architecture.md
CHANGED
|
@@ -48,17 +48,52 @@ Everything lives under `lib/evilution/`.
|
|
|
48
48
|
| `Config`, `Config::*` | Merge `.evilution.yml` + CLI flags + env, validate, freeze. | `config.rb`, `config/sources.rb`, `config/validators/*` |
|
|
49
49
|
| `Runner`, `Runner::*` | Orchestrate the whole run. Each stage is its own collaborator. | `runner.rb`, `runner/*` |
|
|
50
50
|
| `AST`, `Subject` | Prism parse, find method subjects, source surgery, pattern matching, heredoc spans. | `ast/parser.rb`, `ast/source_surgeon.rb`, `subject.rb` |
|
|
51
|
-
| `Mutator`, `Mutator::Operator::*` |
|
|
51
|
+
| `Mutator`, `Mutator::Operator::*` | 111 operators (default profile) that emit byte-edits; registry + profiles. | `mutator/base.rb`, `mutator/registry.rb`, `mutator/operator/*` |
|
|
52
52
|
| `Mutation` | An immutable mutation record (original/mutated sources, slice, location, parse status). | `mutation.rb` |
|
|
53
53
|
| `SpecResolver`, `SpecSelector` | Map a source file to its covering spec files (layout heuristics + explicit mappings). | `spec_resolver.rb`, `spec_selector.rb` |
|
|
54
54
|
| `Isolation::{Fork,InProcess}`, `ProcessSupervisor` | Run one mutation's tests in isolation; process-group lifecycle, sandboxing, TERM/KILL ladder. | `isolation/fork.rb`, `process_supervisor.rb` |
|
|
55
55
|
| `Integration::{RSpec,Minitest,TestUnit}` | Apply a mutation and run the configured test framework; report the raw outcome. | `integration/base.rb`, `integration/rspec.rb` |
|
|
56
56
|
| `Parallel::{Pool,WorkQueue}` | Fan mutations across worker processes for `jobs > 1`. | `parallel/pool.rb`, `parallel/work_queue.rb` |
|
|
57
57
|
| `Result::{MutationResult,Summary}` | Per-mutation result + aggregated, scored summary. | `result/mutation_result.rb`, `result/summary.rb` |
|
|
58
|
+
| `Result::{SubjectScore,SubjectScorer,NeutralReason}` | Score each subject on its own, and record why a neutral is neutral — what the file-level score does not speak for. | `result/subject_scorer.rb`, `result/neutral_reason.rb` |
|
|
58
59
|
| `Reporter::{CLI,JSON,HTML,Suggestion}` | Render a `Summary` to text / JSON / HTML. | `reporter/*` |
|
|
59
60
|
| `Session`, `Compare` | Persist runs to `.evilution/results/*.json`; diff two sessions. | `session/store.rb`, `compare.rb` |
|
|
60
61
|
| `Coverage`, `Equivalent`, `Baseline`, `Cache`, `Hooks`, `MCP` | Coverage-based example targeting, equivalent-mutation detection, baseline capture, incremental cache, lifecycle hooks, MCP server. | respective dirs |
|
|
61
62
|
|
|
63
|
+
## Require conventions
|
|
64
|
+
|
|
65
|
+
Most directories under `lib/evilution/` have a parent file of the same name
|
|
66
|
+
(`runner.rb` next to `runner/`) that declares the namespace. The rules:
|
|
67
|
+
|
|
68
|
+
- **Every child requires its parent.** A file in `runner/` starts with
|
|
69
|
+
`require_relative "../runner"`, a file in `cli/commands/` with
|
|
70
|
+
`require_relative "../commands"`, and so on down the tree.
|
|
71
|
+
- **A parent declares its namespace before requiring its children.** Usually
|
|
72
|
+
the class or module is defined first and children are required at the
|
|
73
|
+
bottom. Where the parent's own body needs its children loaded first (for
|
|
74
|
+
example a constant built from them), it opens with an empty declaration —
|
|
75
|
+
`class Evilution::CLI; end` — then requires the children, then defines the
|
|
76
|
+
rest. `cli.rb` and `integration/rspec/state_guard.rb` do this.
|
|
77
|
+
- **Top-level files (`lib/evilution/*.rb`) are the exception.** Their parent is
|
|
78
|
+
`lib/evilution.rb`, the gem's entry point, which loads everything. A
|
|
79
|
+
top-level file that only needs the root module requires `version.rb`, which
|
|
80
|
+
defines `module Evilution` and nothing else. Only a file that genuinely needs
|
|
81
|
+
the whole gem requires `../evilution` — `runner.rb`, whose collaborators
|
|
82
|
+
reach every operator through `Mutator::Registry`.
|
|
83
|
+
|
|
84
|
+
The parent require and the parent's own `require_relative` of its children
|
|
85
|
+
form a cycle (`runner.rb` → `runner/canary.rb` → `runner.rb`). It is inert:
|
|
86
|
+
Ruby does not load a file twice, so a re-entrant require of a file already
|
|
87
|
+
being loaded returns immediately, and the namespace already exists because the
|
|
88
|
+
parent declared it first. Reviews flagging it as a circular require can be
|
|
89
|
+
answered with this section.
|
|
90
|
+
|
|
91
|
+
Loading an arbitrary file on its own (`require "evilution/cli/command"` in a
|
|
92
|
+
fresh process) is not a goal. The parent requires make most files work that
|
|
93
|
+
way, but not all: a base class whose subclasses are required by the parent
|
|
94
|
+
(`cli/command.rb`) cannot finish defining itself before those subclasses load.
|
|
95
|
+
Load the gem through `require "evilution"`.
|
|
96
|
+
|
|
62
97
|
## Data flow, source → result
|
|
63
98
|
|
|
64
99
|
Driven entirely by `Evilution::Runner#call` (`runner.rb:24`). Each step names the
|
|
@@ -87,7 +122,10 @@ class that owns it.
|
|
|
87
122
|
5. **Spec resolution** — per subject, `SpecSelector#call(source_path)` picks specs
|
|
88
123
|
(explicit `spec_files` → `spec_mappings` → `SpecResolver#resolve_specs` layout
|
|
89
124
|
heuristics). Example-level targeting narrows to examples that reference the
|
|
90
|
-
mutated token (`ExampleFilter` / `CoverageExampleFilter`).
|
|
125
|
+
mutated token (`ExampleFilter` / `CoverageExampleFilter`). Separately,
|
|
126
|
+
`Runner::TargetSpecAudit` asks the same selector once per target file in the
|
|
127
|
+
parent, so a file that resolves to no spec at all is a fact the summary
|
|
128
|
+
carries rather than a handful of `unresolved` mutations.
|
|
91
129
|
6. **Execute** — `Runner::MutationExecutor#call` picks a strategy by `config.jobs`:
|
|
92
130
|
`Strategy::Sequential` for `jobs == 1`, `Strategy::Parallel` (via
|
|
93
131
|
`Parallel::Pool` / `WorkQueue`) for `jobs > 1`. Either way each mutation reaches
|
|
@@ -103,13 +141,24 @@ class that owns it.
|
|
|
103
141
|
guard / unresolved); the final symbol is chosen by
|
|
104
142
|
`Isolation::Fork#classify_status`: `:timeout` → `:killed` (crash) →
|
|
105
143
|
`:unresolved` → `:error` → `:survived` (tests passed) → default `:killed`.
|
|
106
|
-
A `NeutralizationPipeline` can reclassify results
|
|
107
|
-
failed at baseline
|
|
144
|
+
A `NeutralizationPipeline` can reclassify results into `:neutral` — either
|
|
145
|
+
because the covering spec already failed at baseline (`Neutralizer::BaselineFailed`)
|
|
146
|
+
or because the test process crashed on infrastructure rather than on the
|
|
147
|
+
mutation (`Neutralizer::InfraError`). Each records a `Result::NeutralReason`,
|
|
148
|
+
since the two want opposite responses. A survivor is re-run against its whole
|
|
149
|
+
spec file before it is reported (`Integration::RSpec#confirm_survivor?`), so a
|
|
150
|
+
narrowed example set cannot invent one; after a parallel pass,
|
|
151
|
+
`MutationExecutor::InfraRetry` re-runs the infrastructure-neutralised
|
|
152
|
+
mutations serially, once the contention that caused them is gone.
|
|
108
153
|
9. **Aggregate + report** — `Result::Summary` counts each status and computes
|
|
109
154
|
`score = killed / score_denominator` (total minus error/neutral/equivalent/
|
|
110
155
|
unresolved/unparseable). `Runner::ReportPublisher#publish` selects a reporter by
|
|
111
156
|
`config.format` and writes it; `Session::Store` optionally persists the run.
|
|
112
|
-
`
|
|
157
|
+
`Summary#success?` is also false when a target file resolved to no spec, whatever
|
|
158
|
+
the score. `Commands::Run` maps `summary.success?(min_score:)` to exit code
|
|
159
|
+
`0`/`1` (`2` on error), and `CLI::ExitGuard` — installed by the executable before
|
|
160
|
+
anything can preload the project — has the final word on the process status, so a
|
|
161
|
+
preloaded spec helper's at-exit hook cannot replace it.
|
|
113
162
|
|
|
114
163
|
## How to add a new mutator
|
|
115
164
|
|
data/docs/isolation.md
CHANGED
|
@@ -48,7 +48,7 @@ framework must be loaded in the parent before forking — see [Automatic
|
|
|
48
48
|
preload](#automatic-preload) — and `in_process` cannot preload them without
|
|
49
49
|
polluting the host process. A non-Rails gem run under the old `in_process`
|
|
50
50
|
default therefore produced 0 examples / 100% errors out of the box; defaulting
|
|
51
|
-
gems to `fork` lets auto-preload fire (
|
|
51
|
+
gems to `fork` lets auto-preload fire (PR #1375). A plain non-Rails,
|
|
52
52
|
non-gem project (no gemspec) still defaults to `in_process`.
|
|
53
53
|
|
|
54
54
|
The same hazard applies to any Ruby code that uses
|
|
@@ -92,12 +92,11 @@ order, falling back to the gem's library entry point (`lib/<gem>.rb`):
|
|
|
92
92
|
When a gem is detected but none of those helpers exist, evilution prints a
|
|
93
93
|
warning naming the locations it looked in and pointing at `--preload`, so a
|
|
94
94
|
non-standard test layout reads as a fixable configuration issue rather than a
|
|
95
|
-
silent 0% (
|
|
95
|
+
silent 0% (PR #1375).
|
|
96
96
|
|
|
97
97
|
Minitest/Test::Unit helpers that `require "test_helper"` (or any non-relative
|
|
98
98
|
`require "support/..."`) work without `-Itest`: evilution puts the test root
|
|
99
|
-
on `$LOAD_PATH` for the preload just as the test runner would (
|
|
100
|
-
#1373).
|
|
99
|
+
on `$LOAD_PATH` for the preload just as the test runner would (PR #1373).
|
|
101
100
|
|
|
102
101
|
No configuration needed.
|
|
103
102
|
|
data/exe/evil
CHANGED
|
@@ -3,4 +3,11 @@
|
|
|
3
3
|
|
|
4
4
|
require "evilution"
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
# Installed before anything loads the project's spec helper, so this hook is the
|
|
7
|
+
# last one standing and decides the process status — a helper's own at_exit hook
|
|
8
|
+
# (SimpleCov calls exit with its own status) cannot replace it.
|
|
9
|
+
# EV-g8ya / GH #1608.
|
|
10
|
+
guard = Evilution::CLI::ExitGuard.new.install
|
|
11
|
+
guard.status = Evilution::CLI.new(ARGV).call
|
|
12
|
+
|
|
13
|
+
exit guard.status
|
data/exe/evilution
CHANGED
|
@@ -3,4 +3,11 @@
|
|
|
3
3
|
|
|
4
4
|
require "evilution"
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
# Installed before anything loads the project's spec helper, so this hook is the
|
|
7
|
+
# last one standing and decides the process status — a helper's own at_exit hook
|
|
8
|
+
# (SimpleCov calls exit with its own status) cannot replace it.
|
|
9
|
+
# EV-g8ya / GH #1608.
|
|
10
|
+
guard = Evilution::CLI::ExitGuard.new.install
|
|
11
|
+
guard.status = Evilution::CLI.new(ARGV).call
|
|
12
|
+
|
|
13
|
+
exit guard.status
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "prism"
|
|
4
|
+
|
|
5
|
+
require_relative "../ast"
|
|
6
|
+
|
|
7
|
+
# Answers whether a scope body ever reads a given local name — the question an
|
|
8
|
+
# operator asks before changing a parameter, since a parameter nothing reads
|
|
9
|
+
# makes most such mutations behaviour-preserving.
|
|
10
|
+
#
|
|
11
|
+
# Scope-aware in the ways that matter here. A block shares the scope it is
|
|
12
|
+
# written in, so a read inside one counts — unless the block binds the same name
|
|
13
|
+
# itself, as a parameter or a block-local, in which case the read is of the
|
|
14
|
+
# block's own variable. A nested def opens a scope of its own, where a local of
|
|
15
|
+
# the same name is unrelated. A write is not a read.
|
|
16
|
+
#
|
|
17
|
+
# Shadowing is settled by Prism's own answer rather than by inspecting each
|
|
18
|
+
# block's parameter list: a LocalVariableReadNode carries the number of scopes
|
|
19
|
+
# between the read and the variable's declaration, so a read refers to the local
|
|
20
|
+
# in question exactly when that depth matches the number of block scopes
|
|
21
|
+
# descended to reach it.
|
|
22
|
+
class Evilution::AST::LocalReads
|
|
23
|
+
def call(node, name, depth = 0)
|
|
24
|
+
return false if node.nil?
|
|
25
|
+
return true if reads?(node, name, depth)
|
|
26
|
+
|
|
27
|
+
node.compact_child_nodes.any? do |child|
|
|
28
|
+
next false if child.is_a?(Prism::DefNode)
|
|
29
|
+
|
|
30
|
+
call(child, name, depth + (scope?(child) ? 1 : 0))
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
private
|
|
35
|
+
|
|
36
|
+
def reads?(node, name, depth)
|
|
37
|
+
node.is_a?(Prism::LocalVariableReadNode) && node.name.to_s == name && node.depth == depth
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def scope?(node)
|
|
41
|
+
node.is_a?(Prism::BlockNode) || node.is_a?(Prism::LambdaNode)
|
|
42
|
+
end
|
|
43
|
+
end
|
data/lib/evilution/ast/parser.rb
CHANGED