evilution 0.31.0 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +16 -0
- data/CHANGELOG.md +11 -0
- data/README.md +7 -7
- data/docs/integrations.md +126 -0
- data/lib/evilution/cli/parser/options_builder.rb +1 -1
- data/lib/evilution/config/validators/integration.rb +5 -1
- data/lib/evilution/config.rb +1 -1
- data/lib/evilution/integration/loading/mutation_applier.rb +1 -2
- data/lib/evilution/integration/loading/source_evaluator.rb +1 -2
- data/lib/evilution/integration/minitest.rb +1 -2
- data/lib/evilution/integration/rspec/baseline_runner.rb +3 -1
- data/lib/evilution/integration/rspec/framework_loader.rb +5 -1
- data/lib/evilution/integration/test_unit/dispatcher.rb +26 -0
- data/lib/evilution/integration/test_unit/framework_loader.rb +33 -0
- data/lib/evilution/integration/test_unit/result_builder.rb +53 -0
- data/lib/evilution/integration/test_unit/subject_class_registry.rb +26 -0
- data/lib/evilution/integration/test_unit/test_file_resolver.rb +48 -0
- data/lib/evilution/integration/test_unit.rb +124 -0
- data/lib/evilution/integration/test_unit_crash_detector.rb +61 -0
- data/lib/evilution/isolation/fork.rb +17 -1
- data/lib/evilution/mcp/info_tool.rb +2 -2
- data/lib/evilution/mcp/mutate_tool.rb +3 -2
- data/lib/evilution/runner/baseline_runner.rb +3 -1
- data/lib/evilution/runner.rb +6 -1
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution.rb +11 -0
- data/script/run_self_baseline +2 -2
- metadata +10 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: dc948250a218198fef7420e04ab49e5a2f8e01ff50ae279f5299bb85db07ab77
|
|
4
|
+
data.tar.gz: df6e3a2ea4ee5512f9c946f4df514f8a1a2df8308ac77236ceafa3de81b435d8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 43c08e6baa4904c1122d639c957cc1139c9e4760e3e38aed8664a70ff46388aab91b64924816739d4c7a9f00abb049f2da579c50f3b5a585c49b2175f3461dff
|
|
7
|
+
data.tar.gz: e30f5845c332781de9431c9a4238df1ac9048960d3ef36323dac0a8beaf2f5c5d56eb1d3132d618a7e076bb7547520286fc127347499b0b95640331e16912fb8
|
data/.beads/interactions.jsonl
CHANGED
|
@@ -393,3 +393,19 @@
|
|
|
393
393
|
{"id":"int-4ea24fb5","kind":"field_change","created_at":"2026-05-16T16:04:54.395246914Z","actor":"Denis Kiselev","issue_id":"EV-s24s","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Already fixed — both named specs (unparseable_short_circuit_spec.rb, neutralization_integration_spec.rb) pass and destructure via execution.results. Stale in_progress; syncing."}}
|
|
394
394
|
{"id":"int-8b699158","kind":"field_change","created_at":"2026-05-16T17:22:07.345408768Z","actor":"Denis Kiselev","issue_id":"EV-kcuf","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Proof-of-life canary implemented + merged. Runner::Canary runs a synthetic unobservable mutation at session start; aborts if not scored :survived. --[no-]canary flag, config.canary default true."}}
|
|
395
395
|
{"id":"int-e4dcc2b3","kind":"field_change","created_at":"2026-05-27T04:19:00.738936734Z","actor":"Denis Kiselev","issue_id":"EV-qbd6","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Fix landed on master in commit 0f1781f (require 'bundler/setup' at bin/evilution-self:16). Verified 2026-05-27: crash dir lib/evilution/result/*.rb scores 91.11% (744 muts, 656 killed, 64 survived, 24 equivalent, 0 errors); canary passes; dual-runtime harness measures crash dirs again."}}
|
|
396
|
+
{"id":"int-4cb27d5a","kind":"field_change","created_at":"2026-05-28T06:42:48.564599548Z","actor":"Denis Kiselev","issue_id":"EV-pyx6","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Fix merged on master in commit 72b43a3 (PR #1292). Added Evilution.project_base_dir helper + anchored spec/ via Evilution.project_base_dir in framework_loader.rb and baseline_runner.rb. Verified 2026-05-28: bin/evilution-self run lib/evilution/config/env_loader.rb scores 94.59% (was 0 before fix). 4705 examples pass, rubocop clean."}}
|
|
397
|
+
{"id":"int-4cd16cba","kind":"field_change","created_at":"2026-05-29T04:54:00.815774086Z","actor":"Denis Kiselev","issue_id":"EV-8tmc","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Partial fix landed in commit 525787b: 18 of 20 lib/ subdirs now measurable (was 11/20). Two follow-up causes diagnosed and addressed: (1) auto-isolation picking in_process for non-Rails projects + RSpec swallowing Timeout::Error per-example, fixed by script using --isolation=fork explicitly; (2) ChildOutput log_dir relative-path bug under EV-wqxu sandbox CWD, fixed by absolutizing in runner.rb#configure_child_output. Remaining 2 dirs (parallel + toplevel) hang via different root cause — grandchild fork leak — tracked separately as EV-dgjv (GH #1295)."}}
|
|
398
|
+
{"id":"int-991e5acf","kind":"field_change","created_at":"2026-05-30T15:56:24.341278813Z","actor":"Denis Kiselev","issue_id":"EV-auk5","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
399
|
+
{"id":"int-df2bd661","kind":"field_change","created_at":"2026-05-30T15:56:24.686504141Z","actor":"Denis Kiselev","issue_id":"EV-9d62","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
400
|
+
{"id":"int-6a1dc0ce","kind":"field_change","created_at":"2026-05-30T15:56:25.04815512Z","actor":"Denis Kiselev","issue_id":"EV-unar","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
401
|
+
{"id":"int-defdc095","kind":"field_change","created_at":"2026-05-30T15:56:25.406639346Z","actor":"Denis Kiselev","issue_id":"EV-04sc","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
402
|
+
{"id":"int-bc5a7747","kind":"field_change","created_at":"2026-05-30T17:33:33.191722982Z","actor":"Denis Kiselev","issue_id":"EV-8qiy","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
403
|
+
{"id":"int-796e3efc","kind":"field_change","created_at":"2026-05-30T17:33:34.539604057Z","actor":"Denis Kiselev","issue_id":"EV-uv11","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
404
|
+
{"id":"int-5fb67d2b","kind":"field_change","created_at":"2026-05-30T17:46:39.956899342Z","actor":"Denis Kiselev","issue_id":"EV-bcjp","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
405
|
+
{"id":"int-6ba88696","kind":"field_change","created_at":"2026-05-30T17:59:44.432841837Z","actor":"Denis Kiselev","issue_id":"EV-aqxd","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
406
|
+
{"id":"int-34eb133e","kind":"field_change","created_at":"2026-05-31T02:20:26.61836178Z","actor":"Denis Kiselev","issue_id":"EV-vumz","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
407
|
+
{"id":"int-9527af14","kind":"field_change","created_at":"2026-05-31T02:47:48.910182696Z","actor":"Denis Kiselev","issue_id":"EV-zhqc","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
408
|
+
{"id":"int-dfe331ef","kind":"field_change","created_at":"2026-05-31T02:59:12.044281579Z","actor":"Denis Kiselev","issue_id":"EV-akt2","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
409
|
+
{"id":"int-c7bcc360","kind":"field_change","created_at":"2026-05-31T03:12:47.029582311Z","actor":"Denis Kiselev","issue_id":"EV-6az9","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
410
|
+
{"id":"int-fa3ade4f","kind":"field_change","created_at":"2026-05-31T08:47:32.000198989Z","actor":"Denis Kiselev","issue_id":"EV-d3av","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
411
|
+
{"id":"int-b1831323","kind":"field_change","created_at":"2026-05-31T08:47:50.244781995Z","actor":"Denis Kiselev","issue_id":"EV-d7re","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}
|
data/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
Versioning policy: see [docs/versioning.md](docs/versioning.md).
|
|
4
4
|
|
|
5
|
+
## [0.32.0] - 2026-05-31
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- **`--integration test-unit` runs mutations against projects whose suites use the `test-unit` gem** — Rails projects that `require "test/unit/rails/test_help"` make `ActiveSupport::TestCase` inherit from `Test::Unit::TestCase`, and the existing Minitest integration could not dispatch them (`Minitest::Runnable.runnables` is empty because the test classes are not Minitest runnables at all). A dedicated `Evilution::Integration::TestUnit` orchestrator now wires a framework loader (disables the `Test::Unit::AutoRunner` at_exit handler so it does not fire on evilution exit), a subject-class registry (diffs `Test::Unit::TestCase` descendants across `load` to scope each mutation to freshly-loaded suites without a public registry-clear API), a dispatcher (`Test::Unit::UI::Console::TestRunner` with output captured to a `StringIO`), a spec resolver (`test/foo_test.rb` convention, matching Minitest), and a result builder shaping the `passed`/`test_crashed`/`no_tests_ran`/`unresolved` hashes that `classify_status` consumes. `Evilution::Integration::TestUnitCrashDetector` mirrors the Minitest analogue, distinguishing `Test::Unit::Failure` (assertion) from `Test::Unit::Error` (exception/crash) via `TestResult.add_listener(FAULT, ...)`. CLI accepts both `--integration test-unit` and `--integration test_unit`; the MCP `evilution-mutate` tool advertises `test-unit` in its JSON schema enum. Kaminari canary (`kaminari-core/lib/kaminari/models/page_scope_methods.rb`) now scores 63.60% (272 mutations dispatched) where 0.30.x scored 0.00% with the Minitest integration. See [docs/integrations.md](docs/integrations.md). Suggest-tests templates for test-unit are not yet implemented; runs with `--suggest-tests --integration test-unit` fall back to the generic operator-level phrasing (EV-d7re epic + sub-beads EV-8qiy/EV-uv11/EV-bcjp/EV-aqxd/EV-vumz/EV-zhqc/EV-akt2/EV-6az9/EV-d3av, PRs #1314/#1315/#1316/#1318/#1320/#1321, GH #1267)
|
|
10
|
+
|
|
11
|
+
### Fixed
|
|
12
|
+
|
|
13
|
+
- **Pool workers no longer hang indefinitely in `Process.wait` when a per-mutation child writes a payload but does not exit promptly** — `Evilution::Isolation::Fork#reap_and_decode` called unbounded `Process.wait(pid)` after reading a length-prefixed payload from the marshal pipe. If the per-mutation child had written the payload but was still stuck in `execute_in_child` waiting on a subject grandchild the mutation broke (or a grandchild had written garbage bytes that looked like a valid header+body to the parent's inherited write fd), the pool worker blocked forever; the per-mutation timeout had no effect because `wait_for_result` had already returned. `Evilution::Parallel::WorkQueue::Dispatcher`'s `item_timeout` then fired `"worker timed out after 30s"` and SIGKILLed the pool worker, leaving the per-mutation child orphaned to init. `reap_and_decode` now bounds the wait by `REAP_DEADLINE = 1.0`s and falls through to the TERM → GRACE_PERIOD → KILL → safe_wait ladder when the child has not exited, so a stuck child takes ~3s worst-case instead of indefinite. Previously-unmeasurable parallel/ + toplevel/ self-baseline directories (parallel/work_queue.rb, parallel/work_queue/dispatcher.rb, parallel/work_queue/worker.rb, isolation/, toplevel) now all produce real summaries (EV-dgjv, PR #1296, GH #1295)
|
|
14
|
+
- **`script/run_self_baseline` toplevel batch invocation now passes `--isolation=fork`** — every subdir invocation explicitly passed `--isolation=fork`, but the trailing toplevel batch (`lib/evilution/*.rb` files) omitted it. Default isolation auto-picked in_process for the non-Rails self-mutation, which (combined with RSpec's per-example `Timeout::Error` swallow) caused the toplevel batch to hit dispatcher item_timeout and report `NO_SUMMARY`. The toplevel cmd now matches the per-subdir cmd, so all 20 self-baseline dirs are measurable (EV-dgjv, PR #1296, GH #1295)
|
|
15
|
+
|
|
5
16
|
## [0.31.0] - 2026-05-27
|
|
6
17
|
|
|
7
18
|
### Added
|
data/README.md
CHANGED
|
@@ -104,10 +104,10 @@ Every command, subcommand, and flag listed in this section is part of evilution'
|
|
|
104
104
|
| `--[no-]canary` | Boolean | _(enabled)_ | Run a proof-of-life synthetic mutation at session start; abort the run if the pipeline misreports it. Catches misconfigured isolation, broken autoload, and reporter-plugin eviction before any real score is produced. Pass `--no-canary` to skip (e.g. CI speed, or when the canary itself is the thing under test). |
|
|
105
105
|
| `--fail-fast [N]` | Integer | _(none)_ | Stop after N surviving mutants (default 1 if no value given). |
|
|
106
106
|
| `-v`, `--verbose` | Boolean | false | Verbose output with RSS memory and GC stats per phase and per mutation; also prints error class, message, and first 5 backtrace lines for errored mutations. |
|
|
107
|
-
| `--suggest-tests` | Boolean | false | Generate concrete test code in suggestions (RSpec or Minitest, based on `--integration`). |
|
|
107
|
+
| `--suggest-tests` | Boolean | false | Generate concrete test code in suggestions (RSpec or Minitest, based on `--integration`; falls back to generic phrasing for `test-unit`). |
|
|
108
108
|
| `-q`, `--quiet` | Boolean | false | Suppress output. |
|
|
109
109
|
| `--stdin` | Boolean | false | Read target file paths from stdin (one per line). |
|
|
110
|
-
| `--integration NAME` | String | `rspec` | Test framework integration: `rspec` or `
|
|
110
|
+
| `--integration NAME` | String | `rspec` | Test framework integration: `rspec`, `minitest`, or `test-unit`. See [docs/integrations.md](docs/integrations.md). |
|
|
111
111
|
| `--[no-]incremental` | Boolean | false | Cache killed/timeout results; skip unchanged mutations on re-runs. Pass `--no-incremental` to override `incremental: true` from the config file for one invocation (e.g. cold-cache debugging). Last flag wins when both are given. |
|
|
112
112
|
| `--save-session` | Boolean | false | Persist results as timestamped JSON under `.evilution/results/`. |
|
|
113
113
|
| `--no-progress` | Boolean | _(enabled)_ | Disable the TTY progress bar. |
|
|
@@ -169,7 +169,7 @@ schema_version: 1 # opts into strict validation (rejects unknown keys
|
|
|
169
169
|
# timeout: 30 # seconds per mutation
|
|
170
170
|
# format: text # text | json | html
|
|
171
171
|
# min_score: 0.0 # 0.0–1.0
|
|
172
|
-
# integration: rspec # test framework: rspec, minitest
|
|
172
|
+
# integration: rspec # test framework: rspec, minitest, test_unit
|
|
173
173
|
# suggest_tests: false # concrete test code in suggestions (matches integration)
|
|
174
174
|
# save_session: false # persist results under .evilution/results/
|
|
175
175
|
# isolation: auto # auto | fork | in_process (auto selects fork for Rails)
|
|
@@ -213,7 +213,7 @@ All keys recognised under `schema_version: 1`:
|
|
|
213
213
|
| `format` | String | `text` | Output format: `text`, `json`, `html`. |
|
|
214
214
|
| `target` | String / null | `null` | Filter expression: method (`Foo#bar`), class (`Foo`), namespace (`Foo*`), descendants (`descendants:Foo`), source glob (`source:**/*.rb`). |
|
|
215
215
|
| `min_score` | Float | `0.0` | Minimum mutation score (0.0–1.0) for exit code 0. |
|
|
216
|
-
| `integration` | String | `rspec` | Test framework: `rspec` or `
|
|
216
|
+
| `integration` | String | `rspec` | Test framework: `rspec`, `minitest`, or `test_unit`. |
|
|
217
217
|
| `verbose` | Boolean | `false` | Verbose output (RSS/GC stats per phase, error details for errored mutations). |
|
|
218
218
|
| `quiet` | Boolean | `false` | Suppress output. |
|
|
219
219
|
| `jobs` | Integer | `1` | Number of parallel workers. |
|
|
@@ -517,7 +517,7 @@ These fields are added in addition to the existing `operator`, `file`, `line`, `
|
|
|
517
517
|
|
|
518
518
|
The `evilution-mutate` tool accepts a `suggest_tests` boolean parameter (default: `false`). When enabled, survived mutation suggestions contain concrete test code that an agent can drop into a test file, instead of static description text. It currently generates RSpec-style suggestions (`it`/`expect` blocks).
|
|
519
519
|
|
|
520
|
-
Pass `suggest_tests: true` in the `evilution-mutate` call to activate this mode. The CLI also supports `--suggest-tests`; when using the CLI, generated suggestions match the `--integration` setting (RSpec `it`/`expect` blocks or Minitest `def test_`/`assert_equal` methods).
|
|
520
|
+
Pass `suggest_tests: true` in the `evilution-mutate` call to activate this mode. The CLI also supports `--suggest-tests`; when using the CLI, generated suggestions match the `--integration` setting (RSpec `it`/`expect` blocks or Minitest `def test_`/`assert_equal` methods). Concrete templates for `test-unit` are not yet implemented — `test-unit` runs fall back to generic suggestion text.
|
|
521
521
|
|
|
522
522
|
### Project Config File
|
|
523
523
|
|
|
@@ -532,7 +532,7 @@ Pass `skip_config: true` to ignore the project config file. This skips loading `
|
|
|
532
532
|
| Parameter | Purpose |
|
|
533
533
|
|---|---|
|
|
534
534
|
| `incremental` | Cache killed/timeout results across runs — set `true` when iterating on the same files |
|
|
535
|
-
| `integration` | `rspec` or `
|
|
535
|
+
| `integration` | `rspec`, `minitest`, or `test_unit` |
|
|
536
536
|
| `isolation` | `auto`, `fork`, or `in_process` |
|
|
537
537
|
| `baseline` | `false` to skip the baseline suite check when you already know it's green |
|
|
538
538
|
| `save_session` | Persist results to `.evilution/results/` for inspection via `evilution-session` |
|
|
@@ -774,7 +774,7 @@ Tests 4 paths (InProcess isolation, Fork isolation, mutation generation + stripp
|
|
|
774
774
|
3. **Filter** — Disable comments, Sorbet `sig` blocks, and AST ignore patterns exclude mutations before execution
|
|
775
775
|
4. **Mutate** — 74 operators produce text replacements at precise byte offsets (source-level surgery, no AST unparsing); heredoc literal text is skipped by default. Identical byte-mutations from different operators are deduplicated by `(file_path, mutated_source)` so the count is not inflated by overlap
|
|
776
776
|
5. **Isolate** — Mutations are applied to temporary file copies (never modifying originals); load-path redirection ensures `require` resolves the mutated copy. Default isolation is in-process for plain Ruby projects and fork for Rails projects (auto-detected); `--isolation fork` forces forked child processes. Both sequential and parallel (`--jobs N`) modes respect the configured isolation strategy
|
|
777
|
-
6. **Test** — The configured test framework (RSpec or
|
|
777
|
+
6. **Test** — The configured test framework (RSpec, Minitest, or Test::Unit) executes against the mutated source
|
|
778
778
|
7. **Collect** — Source strings and AST nodes are released after use to minimize memory retention
|
|
779
779
|
8. **Report** — Results aggregated into text, JSON, or HTML, including efficiency metrics and peak memory usage
|
|
780
780
|
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# Test Framework Integrations
|
|
2
|
+
|
|
3
|
+
Evilution supports three test framework integrations, selected via the
|
|
4
|
+
`--integration NAME` CLI flag (or the `integration:` key in `.evilution.yml`).
|
|
5
|
+
Each integration owns its own framework loader, dispatcher, spec resolver,
|
|
6
|
+
and result builder so they can evolve independently.
|
|
7
|
+
|
|
8
|
+
| Integration | Flag value | Default test dir | Default test suffix | Gem dep |
|
|
9
|
+
|---------------|---------------|------------------|---------------------|------------------------|
|
|
10
|
+
| RSpec | `rspec` | `spec/` | `_spec.rb` | `rspec-core` |
|
|
11
|
+
| Minitest | `minitest` | `test/` | `_test.rb` | `minitest` |
|
|
12
|
+
| Test::Unit | `test-unit` | `test/` | `_test.rb` | `test-unit` |
|
|
13
|
+
|
|
14
|
+
## When to pick which
|
|
15
|
+
|
|
16
|
+
- **`rspec`** (default) — Suites built on `RSpec.describe` / `it` / `expect`.
|
|
17
|
+
- **`minitest`** — Suites that subclass `Minitest::Test` or use
|
|
18
|
+
`Minitest::Spec`'s `describe` / `it`. Includes Rails apps where
|
|
19
|
+
`ActiveSupport::TestCase` inherits from `Minitest::Test`.
|
|
20
|
+
- **`test-unit`** — Suites that subclass `Test::Unit::TestCase`, including
|
|
21
|
+
Rails projects that `require "test/unit/rails/test_help"` (which makes
|
|
22
|
+
`ActiveSupport::TestCase < Test::Unit::TestCase`). The `test-unit` gem's
|
|
23
|
+
`TestCase` classes are *not* `Minitest::Runnable` subclasses, so the
|
|
24
|
+
`minitest` integration cannot dispatch them — pick this integration whenever
|
|
25
|
+
your test helper pulls in `test/unit/rails/test_help`, `test-unit-activerecord`,
|
|
26
|
+
or similar test-unit-specific glue.
|
|
27
|
+
|
|
28
|
+
## CLI examples
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
# RSpec — default; specs in spec/, named *_spec.rb
|
|
32
|
+
bundle exec evilution run lib/foo.rb
|
|
33
|
+
|
|
34
|
+
# Minitest
|
|
35
|
+
bundle exec evilution run lib/foo.rb \
|
|
36
|
+
--integration minitest --spec test/foo_test.rb
|
|
37
|
+
|
|
38
|
+
# Test::Unit (gem name uses hyphen; symbol value uses underscore)
|
|
39
|
+
bundle exec evilution run lib/foo.rb \
|
|
40
|
+
--integration test-unit --spec test/foo_test.rb
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
The CLI accepts both `test-unit` and `test_unit` strings; the internal config
|
|
44
|
+
symbol is always `:test_unit`.
|
|
45
|
+
|
|
46
|
+
## MCP examples
|
|
47
|
+
|
|
48
|
+
The `evilution-mutate` MCP tool exposes `integration` as a JSON schema enum
|
|
49
|
+
of `["rspec", "minitest", "test-unit"]`:
|
|
50
|
+
|
|
51
|
+
```json
|
|
52
|
+
{
|
|
53
|
+
"tool": "evilution-mutate",
|
|
54
|
+
"args": {
|
|
55
|
+
"files": ["lib/foo.rb"],
|
|
56
|
+
"integration": "test-unit",
|
|
57
|
+
"spec": ["test/foo_test.rb"]
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Spec resolution conventions
|
|
63
|
+
|
|
64
|
+
When `--spec` is not supplied, evilution resolves a test file from the source
|
|
65
|
+
path using the integration's spec resolver. Strips `lib/` or `app/` prefix
|
|
66
|
+
and rewrites the suffix:
|
|
67
|
+
|
|
68
|
+
| Source | rspec | minitest / test-unit |
|
|
69
|
+
|-------------------------------------|-------------------------------------|------------------------------------|
|
|
70
|
+
| `lib/foo.rb` | `spec/foo_spec.rb` | `test/foo_test.rb` |
|
|
71
|
+
| `lib/foo/bar.rb` | `spec/foo/bar_spec.rb` | `test/foo/bar_test.rb` |
|
|
72
|
+
| `app/models/user.rb` | `spec/models/user_spec.rb` | `test/models/user_test.rb` |
|
|
73
|
+
| `app/controllers/users_controller.rb` | `spec/requests/users_spec.rb` *or* `spec/controllers/users_controller_spec.rb` | `test/integration/users_test.rb` *or* `test/controllers/users_controller_test.rb` |
|
|
74
|
+
|
|
75
|
+
For controllers, the resolver tries the request-spec / integration-test
|
|
76
|
+
location first, then falls back to the controller-spec / controller-test
|
|
77
|
+
location.
|
|
78
|
+
|
|
79
|
+
## Suggest-tests caveat
|
|
80
|
+
|
|
81
|
+
The `--suggest-tests` mode emits ready-to-paste test snippets for survived
|
|
82
|
+
mutations. Concrete templates currently exist for **rspec** and **minitest**
|
|
83
|
+
only; the **test-unit** integration falls back to the generic operator-level
|
|
84
|
+
suggestion text. Adding test-unit templates is a small follow-up.
|
|
85
|
+
|
|
86
|
+
## Other Ruby test frameworks
|
|
87
|
+
|
|
88
|
+
No additional integrations are planned at the moment. The three integrations
|
|
89
|
+
above cover the overwhelming majority of Ruby projects with unit / integration
|
|
90
|
+
test suites. Frameworks we considered and deferred:
|
|
91
|
+
|
|
92
|
+
- **Cucumber / Spinach** — BDD scenarios over Gherkin step definitions, a
|
|
93
|
+
fundamentally coarser granularity than mutation testing rewards. Not a
|
|
94
|
+
natural fit; no plans to add.
|
|
95
|
+
- **Sus** — Socketry's async-focused framework, small but actively maintained
|
|
96
|
+
user base. Could be added under the same orchestrator/collaborator pattern
|
|
97
|
+
if a real project surfaces needing it.
|
|
98
|
+
- **Bacon, Test::Spec** — early RSpec-style clones, effectively dormant.
|
|
99
|
+
- **Minitest::Spec** — already covered by the `minitest` integration.
|
|
100
|
+
|
|
101
|
+
If you maintain a project on another framework and would benefit from
|
|
102
|
+
mutation-testing it through evilution, open an issue describing the project
|
|
103
|
+
and the framework's dispatch entry-point — the existing integration layout
|
|
104
|
+
makes new entries cheap to add.
|
|
105
|
+
|
|
106
|
+
## Behind the scenes
|
|
107
|
+
|
|
108
|
+
Each integration has an orchestrator class at
|
|
109
|
+
`lib/evilution/integration/<name>.rb`. RSpec and Test::Unit additionally
|
|
110
|
+
decompose their collaborators into a sibling directory of the same name
|
|
111
|
+
(`lib/evilution/integration/rspec/...`,
|
|
112
|
+
`lib/evilution/integration/test_unit/...`); Minitest remains a single file.
|
|
113
|
+
The test-unit collaborators are:
|
|
114
|
+
|
|
115
|
+
- `framework_loader.rb` — `require "test-unit"` + disables the at_exit
|
|
116
|
+
auto-run handler so it doesn't fire on evilution exit.
|
|
117
|
+
- `subject_class_registry.rb` — ObjectSpace tracking of newly-loaded
|
|
118
|
+
`Test::Unit::TestCase` subclasses (test-unit has no public
|
|
119
|
+
registry-clear analog to `Minitest::Runnable.runnables`).
|
|
120
|
+
- `dispatcher.rb` — assembles a `Test::Unit::TestSuite` from the new
|
|
121
|
+
classes and runs it via `Test::Unit::UI::Console::TestRunner` with
|
|
122
|
+
output captured to a `StringIO`.
|
|
123
|
+
- `test_file_resolver.rb` — explicit-override + spec-selector +
|
|
124
|
+
fallback glob + warn-once for unresolved sources.
|
|
125
|
+
- `result_builder.rb` — shapes the `passed`/`test_crashed`/
|
|
126
|
+
`no_tests_ran`/`unresolved` Hash that flows into `classify_status`.
|
|
@@ -95,7 +95,7 @@ class Evilution::CLI::Parser::OptionsBuilder
|
|
|
95
95
|
end
|
|
96
96
|
|
|
97
97
|
def add_runner_mode_options(opts)
|
|
98
|
-
opts.on("--integration NAME", "Test integration: rspec, minitest (default: rspec)") { |i| @options[:integration] = i }
|
|
98
|
+
opts.on("--integration NAME", "Test integration: rspec, minitest, test-unit (default: rspec)") { |i| @options[:integration] = i }
|
|
99
99
|
opts.on("--isolation STRATEGY", "Isolation: auto, fork, in_process (default: auto)") { |s| @options[:isolation] = s }
|
|
100
100
|
opts.on("--preload FILE", "Preload FILE in the parent process before forking " \
|
|
101
101
|
"(default: auto-detect spec/rails_helper.rb -> spec/spec_helper.rb -> " \
|
|
@@ -3,9 +3,13 @@
|
|
|
3
3
|
require_relative "base"
|
|
4
4
|
|
|
5
5
|
class Evilution::Config::Validators::Integration < Evilution::Config::Validators::Base
|
|
6
|
-
ALLOWED = %i[rspec minitest].freeze
|
|
6
|
+
ALLOWED = %i[rspec minitest test_unit].freeze
|
|
7
7
|
|
|
8
|
+
# CLI users naturally write the gem name `test-unit`; the internal symbol
|
|
9
|
+
# uses underscore form to match the file path and registry key. Normalize
|
|
10
|
+
# hyphenated string input before coercion.
|
|
8
11
|
def self.call(value)
|
|
12
|
+
value = value.tr("-", "_") if value.is_a?(String)
|
|
9
13
|
coerce_symbol!(value, allowed: ALLOWED, name: "integration")
|
|
10
14
|
end
|
|
11
15
|
end
|
data/lib/evilution/config.rb
CHANGED
|
@@ -139,7 +139,7 @@ class Evilution::Config
|
|
|
139
139
|
# (default: true). Set false to skip (e.g. for CI speed).
|
|
140
140
|
# canary: true
|
|
141
141
|
|
|
142
|
-
# Test integration: rspec, minitest (default: rspec)
|
|
142
|
+
# Test integration: rspec, minitest, test_unit (default: rspec)
|
|
143
143
|
# integration: rspec
|
|
144
144
|
|
|
145
145
|
# Number of parallel workers (default: 1)
|
|
@@ -78,8 +78,7 @@ class Evilution::Integration::Loading::MutationApplier
|
|
|
78
78
|
# When the isolator has chdir'd into a per-mutation sandbox (EV-wqxu /
|
|
79
79
|
# GH #1278), anchor against PROJECT_ROOT so File.realpath does not chase
|
|
80
80
|
# file_path into a non-existent /tmp path.
|
|
81
|
-
|
|
82
|
-
absolute = File.realpath(File.expand_path(file_path, base))
|
|
81
|
+
absolute = File.realpath(File.expand_path(file_path, Evilution.project_base_dir))
|
|
83
82
|
$LOADED_FEATURES << absolute unless $LOADED_FEATURES.include?(absolute)
|
|
84
83
|
rescue Errno::ENOENT
|
|
85
84
|
nil
|
|
@@ -16,8 +16,7 @@ class Evilution::Integration::Loading::SourceEvaluator
|
|
|
16
16
|
# When the isolator has chdir'd into a per-mutation sandbox (EV-wqxu /
|
|
17
17
|
# GH #1278), anchor the eval __FILE__ against PROJECT_ROOT so siblings
|
|
18
18
|
# `require_relative` can find each other from the real source tree.
|
|
19
|
-
|
|
20
|
-
absolute = File.expand_path(file_path, base)
|
|
19
|
+
absolute = File.expand_path(file_path, Evilution.project_base_dir)
|
|
21
20
|
eval(source, TOPLEVEL_BINDING, absolute, 1)
|
|
22
21
|
end
|
|
23
22
|
end
|
|
@@ -128,8 +128,7 @@ class Evilution::Integration::Minitest < Evilution::Integration::Base
|
|
|
128
128
|
end
|
|
129
129
|
|
|
130
130
|
def execute_minitest(mutation, files, command)
|
|
131
|
-
|
|
132
|
-
files.each { |f| load(File.expand_path(f, base)) }
|
|
131
|
+
files.each { |f| load(File.expand_path(f, Evilution.project_base_dir)) }
|
|
133
132
|
|
|
134
133
|
detector = reset_crash_detector
|
|
135
134
|
run = run_minitest(build_args(mutation), detector)
|
|
@@ -5,7 +5,9 @@ require_relative "../rspec"
|
|
|
5
5
|
class Evilution::Integration::RSpec::BaselineRunner
|
|
6
6
|
def call(spec_file)
|
|
7
7
|
require "rspec/core"
|
|
8
|
-
|
|
8
|
+
# Anchor against PROJECT_ROOT under EV-wqxu sandbox CWD; see
|
|
9
|
+
# FrameworkLoader#add_spec_load_path for rationale.
|
|
10
|
+
spec_dir = File.expand_path("spec", Evilution.project_base_dir)
|
|
9
11
|
$LOAD_PATH.unshift(spec_dir) unless $LOAD_PATH.include?(spec_dir)
|
|
10
12
|
::RSpec.reset
|
|
11
13
|
status = ::RSpec::Core::Runner.run(
|
|
@@ -22,7 +22,11 @@ class Evilution::Integration::RSpec::FrameworkLoader
|
|
|
22
22
|
private
|
|
23
23
|
|
|
24
24
|
def add_spec_load_path
|
|
25
|
-
|
|
25
|
+
# Anchor against PROJECT_ROOT inside an isolated worker (EV-wqxu /
|
|
26
|
+
# GH #1278) so the project's spec/ dir lands on $LOAD_PATH — otherwise
|
|
27
|
+
# `require "spec_helper"` resolves to a non-existent sandbox/spec and
|
|
28
|
+
# every mutation errors as "loaded 0 examples".
|
|
29
|
+
spec_dir = File.expand_path("spec", Evilution.project_base_dir)
|
|
26
30
|
$LOAD_PATH.unshift(spec_dir) unless $LOAD_PATH.include?(spec_dir)
|
|
27
31
|
end
|
|
28
32
|
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "stringio"
|
|
4
|
+
require_relative "../test_unit"
|
|
5
|
+
|
|
6
|
+
# Builds a Test::Unit::TestSuite from a list of TestCase subclasses and runs
|
|
7
|
+
# it via the console runner with output captured to a StringIO. Owning this
|
|
8
|
+
# responsibility separately keeps the runner library require + suite assembly
|
|
9
|
+
# in one place — used by both the baseline path (Evilution::Integration::TestUnit
|
|
10
|
+
# .run_baseline_test_file) and the per-mutation path (#run_tests).
|
|
11
|
+
module Evilution::Integration::TestUnit::Dispatcher
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
def call(test_case_classes, name: "evilution")
|
|
15
|
+
require "test/unit/ui/console/testrunner"
|
|
16
|
+
suite = ::Test::Unit::TestSuite.new(name)
|
|
17
|
+
test_case_classes.each { |klass| suite << klass.suite }
|
|
18
|
+
out = StringIO.new
|
|
19
|
+
runner = ::Test::Unit::UI::Console::TestRunner.new(suite, output: out)
|
|
20
|
+
runner.start
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def test_method_count(test_case_classes)
|
|
24
|
+
test_case_classes.sum { |klass| klass.suite.tests.length }
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../test_unit"
|
|
4
|
+
|
|
5
|
+
# Loads the test-unit gem and disables its at_exit auto-run handler.
|
|
6
|
+
# Mirrors Evilution::Integration::RSpec::FrameworkLoader's role: framework
|
|
7
|
+
# setup is one responsibility separated from dispatch + result building so
|
|
8
|
+
# integrations can compose it independently in tests.
|
|
9
|
+
class Evilution::Integration::TestUnit::FrameworkLoader
|
|
10
|
+
def loaded?
|
|
11
|
+
@loaded == true
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def call
|
|
15
|
+
return if @loaded
|
|
16
|
+
|
|
17
|
+
require "test-unit"
|
|
18
|
+
self.class.stub_autorun!
|
|
19
|
+
@loaded = true
|
|
20
|
+
rescue LoadError => e
|
|
21
|
+
raise Evilution::Error, "test-unit is required but not available: #{e.message}"
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# User code that `require "test-unit"` installs an at_exit hook that calls
|
|
25
|
+
# Test::Unit::AutoRunner.run when need_auto_run? is true. At evilution exit
|
|
26
|
+
# ARGV still holds evilution flags and the runner prints a misleading banner.
|
|
27
|
+
# Flipping need_auto_run = false prevents the handler from firing.
|
|
28
|
+
def self.stub_autorun!
|
|
29
|
+
return unless defined?(::Test::Unit::AutoRunner)
|
|
30
|
+
|
|
31
|
+
::Test::Unit::AutoRunner.need_auto_run = false
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../test_unit"
|
|
4
|
+
|
|
5
|
+
# Shapes the result Hash that flows back to Evilution::Result::MutationResult
|
|
6
|
+
# / classify_status. Three orthogonal flavours — pass/fail/crash, no tests
|
|
7
|
+
# executed, and unresolved spec — each have their own change axis (e.g. the
|
|
8
|
+
# no-tests-ran error string evolved separately as the test-unit framework
|
|
9
|
+
# diagnostic improved). Putting them behind a single object documents the
|
|
10
|
+
# contract and lets the integration class drop them.
|
|
11
|
+
module Evilution::Integration::TestUnit::ResultBuilder
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
def call(passed:, command:, detector:)
|
|
15
|
+
if passed
|
|
16
|
+
{ passed: true, test_command: command }
|
|
17
|
+
elsif detector.only_crashes?
|
|
18
|
+
crash(command, detector)
|
|
19
|
+
else
|
|
20
|
+
{ passed: false, test_command: command }
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def no_tests_ran(command)
|
|
25
|
+
{
|
|
26
|
+
passed: false,
|
|
27
|
+
error: "no Test::Unit tests executed (0 test methods ran) — the resolved " \
|
|
28
|
+
"spec registered no Test::Unit suite. Check --integration/--spec.",
|
|
29
|
+
error_class: "Evilution::Error",
|
|
30
|
+
test_command: command
|
|
31
|
+
}
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def unresolved(mutation_file_path)
|
|
35
|
+
{
|
|
36
|
+
passed: false,
|
|
37
|
+
unresolved: true,
|
|
38
|
+
error: "no matching test resolved for #{mutation_file_path}",
|
|
39
|
+
test_command: "ruby -Itest (skipped: no test resolved for #{mutation_file_path})"
|
|
40
|
+
}
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def crash(command, detector)
|
|
44
|
+
classes = detector.unique_crash_classes
|
|
45
|
+
{
|
|
46
|
+
passed: false,
|
|
47
|
+
test_crashed: true,
|
|
48
|
+
error: "test crashes: #{detector.crash_summary}",
|
|
49
|
+
error_class: (classes.first if classes.length == 1),
|
|
50
|
+
test_command: command
|
|
51
|
+
}
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../test_unit"
|
|
4
|
+
|
|
5
|
+
# Tracks Test::Unit::TestCase descendants in the host process. Test::Unit has
|
|
6
|
+
# no public registry-clear method analogous to Minitest::Runnable.runnables;
|
|
7
|
+
# the integration scopes each dispatch to classes that appeared during *this*
|
|
8
|
+
# round by diffing the descendant set before and after #load. Keeping that
|
|
9
|
+
# responsibility in its own object makes it cheap to stub in tests and lets
|
|
10
|
+
# the integration's main class read as orchestration.
|
|
11
|
+
module Evilution::Integration::TestUnit::SubjectClassRegistry
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
def descendants
|
|
15
|
+
return [] unless defined?(::Test::Unit::TestCase)
|
|
16
|
+
|
|
17
|
+
ObjectSpace.each_object(Class).select { |c| c < ::Test::Unit::TestCase }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Yields, captures the descendant set before/after, and returns the diff.
|
|
21
|
+
def newly_loaded
|
|
22
|
+
before = descendants
|
|
23
|
+
yield
|
|
24
|
+
descendants - before
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../test_unit"
|
|
4
|
+
|
|
5
|
+
# Resolves the list of test files to load for a given mutation. Encapsulates
|
|
6
|
+
# the explicit-override path, spec-selector lookup, fallback glob, and the
|
|
7
|
+
# warn-once behaviour for unresolved sources. The integration class would
|
|
8
|
+
# otherwise carry both per-instance test resolution state (@test_files,
|
|
9
|
+
# @spec_selector, @warned_files, @fallback_to_full_suite) and dispatch
|
|
10
|
+
# orchestration in the same object; splitting them gives the resolver its
|
|
11
|
+
# own change axis (e.g. adding new resolution heuristics) independent of
|
|
12
|
+
# the runner.
|
|
13
|
+
class Evilution::Integration::TestUnit::TestFileResolver
|
|
14
|
+
def initialize(test_files:, spec_selector:, fallback_to_full_suite:)
|
|
15
|
+
@test_files = test_files
|
|
16
|
+
@spec_selector = spec_selector
|
|
17
|
+
@fallback_to_full_suite = fallback_to_full_suite
|
|
18
|
+
@warned_files = Set.new
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Returns the resolved file list, or nil if the source could not be
|
|
22
|
+
# resolved and fallback is disabled.
|
|
23
|
+
def call(mutation_file_path)
|
|
24
|
+
return @test_files if @test_files
|
|
25
|
+
|
|
26
|
+
resolved = Array(@spec_selector.call(mutation_file_path))
|
|
27
|
+
return resolved unless resolved.empty?
|
|
28
|
+
|
|
29
|
+
warn_unresolved(mutation_file_path)
|
|
30
|
+
@fallback_to_full_suite ? glob_test_files : nil
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def glob_test_files
|
|
36
|
+
files = Dir.glob("test/**/*_test.rb")
|
|
37
|
+
files.empty? ? ["test"] : files
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def warn_unresolved(file_path)
|
|
41
|
+
return if @warned_files.include?(file_path)
|
|
42
|
+
|
|
43
|
+
@warned_files << file_path
|
|
44
|
+
action = @fallback_to_full_suite ? "running full suite" : "marking mutation unresolved"
|
|
45
|
+
warn "[evilution] No matching test found for #{file_path}, #{action}. " \
|
|
46
|
+
"Use --spec to specify the test file."
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "base"
|
|
4
|
+
require_relative "test_unit_crash_detector"
|
|
5
|
+
require_relative "../spec_resolver"
|
|
6
|
+
require_relative "../spec_selector"
|
|
7
|
+
|
|
8
|
+
require_relative "../integration"
|
|
9
|
+
|
|
10
|
+
# Test::Unit integration. Decomposed under lib/evilution/integration/test_unit/
|
|
11
|
+
# mirroring the RSpec integration's layout. This class is the orchestrator:
|
|
12
|
+
# it wires the framework loader, dispatcher, subject-class registry,
|
|
13
|
+
# test-file resolver, and result builder. The class is registered under
|
|
14
|
+
# Evilution::Runner::INTEGRATIONS[:test_unit] and reachable via the
|
|
15
|
+
# `--integration test-unit` CLI flag.
|
|
16
|
+
class Evilution::Integration::TestUnit < Evilution::Integration::Base
|
|
17
|
+
def self.baseline_runner
|
|
18
|
+
->(test_file) { run_baseline_test_file(test_file) }
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# SpecResolver tuned for the dominant Test::Unit layout: tests live under
|
|
22
|
+
# test/, named with the _test.rb suffix (the same convention Minitest uses).
|
|
23
|
+
# Rails plugins on the test-unit gem (e.g. kaminari-core) follow this layout.
|
|
24
|
+
# The test/test_<name>.rb prefix-style convention is rare enough in practice
|
|
25
|
+
# that we defer support to a follow-up if a project surfaces needing it.
|
|
26
|
+
def self.spec_resolver
|
|
27
|
+
Evilution::SpecResolver.new(test_dir: "test", test_suffix: "_test.rb", request_dir: "integration")
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def self.baseline_options
|
|
31
|
+
{
|
|
32
|
+
runner: baseline_runner,
|
|
33
|
+
spec_resolver: spec_resolver,
|
|
34
|
+
fallback_dir: "test"
|
|
35
|
+
}
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def self.run_baseline_test_file(test_file)
|
|
39
|
+
require_relative "test_unit/framework_loader"
|
|
40
|
+
require_relative "test_unit/subject_class_registry"
|
|
41
|
+
require_relative "test_unit/dispatcher"
|
|
42
|
+
FrameworkLoader.new.call
|
|
43
|
+
new_classes = SubjectClassRegistry.newly_loaded do
|
|
44
|
+
baseline_test_files(test_file).each { |f| load(File.expand_path(f)) }
|
|
45
|
+
end
|
|
46
|
+
Dispatcher.call(new_classes, name: "evilution baseline").passed?
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def self.baseline_test_files(test_file)
|
|
50
|
+
File.directory?(test_file) ? Dir.glob(File.join(test_file, "**/*_test.rb")) : [test_file]
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def initialize(test_files: nil, hooks: nil, fallback_to_full_suite: false, spec_selector: nil)
|
|
54
|
+
require_relative "test_unit/framework_loader"
|
|
55
|
+
require_relative "test_unit/subject_class_registry"
|
|
56
|
+
require_relative "test_unit/dispatcher"
|
|
57
|
+
require_relative "test_unit/test_file_resolver"
|
|
58
|
+
require_relative "test_unit/result_builder"
|
|
59
|
+
@framework_loader = FrameworkLoader.new
|
|
60
|
+
@file_resolver = TestFileResolver.new(
|
|
61
|
+
test_files: test_files,
|
|
62
|
+
spec_selector: spec_selector || Evilution::SpecSelector.new(spec_resolver: self.class.spec_resolver),
|
|
63
|
+
fallback_to_full_suite: fallback_to_full_suite
|
|
64
|
+
)
|
|
65
|
+
@crash_detector = nil
|
|
66
|
+
super(hooks: hooks)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
private
|
|
70
|
+
|
|
71
|
+
def ensure_framework_loaded
|
|
72
|
+
return if @framework_loader.loaded?
|
|
73
|
+
|
|
74
|
+
fire_hook(:setup_integration_pre, integration: :test_unit)
|
|
75
|
+
@framework_loader.call
|
|
76
|
+
fire_hook(:setup_integration_post, integration: :test_unit)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def run_tests(mutation)
|
|
80
|
+
ensure_framework_loaded
|
|
81
|
+
reset_state
|
|
82
|
+
files = @file_resolver.call(mutation.file_path)
|
|
83
|
+
return ResultBuilder.unresolved(mutation.file_path) if files.nil?
|
|
84
|
+
|
|
85
|
+
command = "ruby -Itest #{files.join(" ")}"
|
|
86
|
+
execute_test_unit(files, command)
|
|
87
|
+
rescue StandardError => e
|
|
88
|
+
{ passed: false, error: e.message, test_command: command }
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def execute_test_unit(files, command)
|
|
92
|
+
new_classes = SubjectClassRegistry.newly_loaded do
|
|
93
|
+
files.each { |f| load(File.expand_path(f, Evilution.project_base_dir)) }
|
|
94
|
+
end
|
|
95
|
+
return ResultBuilder.no_tests_ran(command) if Dispatcher.test_method_count(new_classes).zero?
|
|
96
|
+
|
|
97
|
+
detector = reset_crash_detector
|
|
98
|
+
result = Dispatcher.call(new_classes, name: "evilution-mutation")
|
|
99
|
+
result.faults.each { |fault| detector.record(fault) }
|
|
100
|
+
ResultBuilder.call(passed: result.passed?, command: command, detector: detector)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Test::Unit has no public registry-clear analogous to
|
|
104
|
+
# Minitest::Runnable.runnables.clear. SubjectClassRegistry's newly_loaded
|
|
105
|
+
# block scopes each dispatch to classes loaded in *this* round, so stale
|
|
106
|
+
# classes from prior mutations sit dormant on ObjectSpace without polluting
|
|
107
|
+
# the run. #reset_state stays as a contract no-op for parity with Minitest.
|
|
108
|
+
def reset_state
|
|
109
|
+
# no-op — see comment above
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def build_args(_mutation)
|
|
113
|
+
[]
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def reset_crash_detector
|
|
117
|
+
if @crash_detector
|
|
118
|
+
@crash_detector.reset
|
|
119
|
+
else
|
|
120
|
+
@crash_detector = Evilution::Integration::TestUnitCrashDetector.new
|
|
121
|
+
end
|
|
122
|
+
@crash_detector
|
|
123
|
+
end
|
|
124
|
+
end
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../integration"
|
|
4
|
+
|
|
5
|
+
# Test::Unit analog of Evilution::Integration::MinitestCrashDetector. Tracks
|
|
6
|
+
# whether a Test::Unit test run produced only crashes (exceptions captured as
|
|
7
|
+
# Test::Unit::Error) vs assertion failures (Test::Unit::Failure). When only
|
|
8
|
+
# crashes occur, the mutation result can carry crash details while remaining
|
|
9
|
+
# classified as :killed — see classify_status / Result::MutationResult.
|
|
10
|
+
#
|
|
11
|
+
# Hook the detector into a Test::Unit::TestResult via .attach(result), or
|
|
12
|
+
# call #record(fault) directly when iterating a finished result's #faults.
|
|
13
|
+
class Evilution::Integration::TestUnitCrashDetector
|
|
14
|
+
def initialize
|
|
15
|
+
reset
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def reset
|
|
19
|
+
@assertion_failures = 0
|
|
20
|
+
@crashes = []
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def attach(test_result)
|
|
24
|
+
require "test/unit/testresult"
|
|
25
|
+
test_result.add_listener(Test::Unit::TestResult::FAULT) { |fault| record(fault) }
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def record(fault)
|
|
29
|
+
if fault.is_a?(Test::Unit::Error)
|
|
30
|
+
@crashes << fault.exception
|
|
31
|
+
elsif fault.is_a?(Test::Unit::Failure)
|
|
32
|
+
@assertion_failures += 1
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def passed?
|
|
37
|
+
@assertion_failures.zero? && @crashes.empty?
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def assertion_failure?
|
|
41
|
+
@assertion_failures.positive?
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def crashed?
|
|
45
|
+
@crashes.any?
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def only_crashes?
|
|
49
|
+
@crashes.any? && @assertion_failures.zero?
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def unique_crash_classes
|
|
53
|
+
@crashes.map { |e| e.class.name }.uniq
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def crash_summary
|
|
57
|
+
return nil if @crashes.empty?
|
|
58
|
+
|
|
59
|
+
"#{unique_crash_classes.join(", ")} (#{@crashes.length} crash#{"es" unless @crashes.length == 1})"
|
|
60
|
+
end
|
|
61
|
+
end
|
|
@@ -11,6 +11,7 @@ require_relative "../isolation"
|
|
|
11
11
|
|
|
12
12
|
class Evilution::Isolation::Fork
|
|
13
13
|
GRACE_PERIOD = 2
|
|
14
|
+
REAP_DEADLINE = 1.0
|
|
14
15
|
|
|
15
16
|
def initialize(hooks: nil)
|
|
16
17
|
@hooks = hooks
|
|
@@ -130,8 +131,23 @@ class Evilution::Isolation::Fork
|
|
|
130
131
|
end
|
|
131
132
|
end
|
|
132
133
|
|
|
134
|
+
# Process.wait without a deadline can hang the parent indefinitely. The
|
|
135
|
+
# per-mutation child may already have written a payload (or a grandchild may
|
|
136
|
+
# have written garbage that looks like one) while the child itself is stuck
|
|
137
|
+
# in execute_in_child waiting on a subject grandchild the mutation broke.
|
|
138
|
+
# wait_for_result has already returned by this point, so the per-mutation
|
|
139
|
+
# timeout cannot fire. Bound the wait and fall back to the TERM/KILL ladder.
|
|
133
140
|
def reap_and_decode(pid, payload)
|
|
134
|
-
|
|
141
|
+
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + REAP_DEADLINE
|
|
142
|
+
loop do
|
|
143
|
+
break if ::Process.waitpid(pid, ::Process::WNOHANG)
|
|
144
|
+
|
|
145
|
+
if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
|
|
146
|
+
terminate_child(pid)
|
|
147
|
+
break
|
|
148
|
+
end
|
|
149
|
+
sleep 0.05
|
|
150
|
+
end
|
|
135
151
|
decode_payload(payload)
|
|
136
152
|
end
|
|
137
153
|
|
|
@@ -58,8 +58,8 @@ class Evilution::MCP::InfoTool < MCP::Tool
|
|
|
58
58
|
},
|
|
59
59
|
integration: {
|
|
60
60
|
type: "string",
|
|
61
|
-
description: "[subjects, tests] Test integration (rspec, minitest) — 'tests' selects " \
|
|
62
|
-
"the matching spec resolver (spec/*_spec.rb for rspec, test/*_test.rb for minitest)"
|
|
61
|
+
description: "[subjects, tests] Test integration (rspec, minitest, test-unit) — 'tests' selects " \
|
|
62
|
+
"the matching spec resolver (spec/*_spec.rb for rspec, test/*_test.rb for minitest/test-unit)"
|
|
63
63
|
},
|
|
64
64
|
skip_config: {
|
|
65
65
|
type: "boolean",
|
|
@@ -72,8 +72,9 @@ class Evilution::MCP::MutateTool < MCP::Tool
|
|
|
72
72
|
},
|
|
73
73
|
integration: {
|
|
74
74
|
type: "string",
|
|
75
|
-
enum: %w[rspec minitest],
|
|
76
|
-
description: "Test integration to use (default: rspec)"
|
|
75
|
+
enum: %w[rspec minitest test-unit],
|
|
76
|
+
description: "Test integration to use (default: rspec). " \
|
|
77
|
+
"Use test-unit for projects whose suites run on the test-unit gem."
|
|
77
78
|
},
|
|
78
79
|
isolation: {
|
|
79
80
|
type: "string",
|
|
@@ -5,6 +5,7 @@ require_relative "../baseline"
|
|
|
5
5
|
require_relative "../spec_resolver"
|
|
6
6
|
require_relative "../integration/rspec"
|
|
7
7
|
require_relative "../integration/minitest"
|
|
8
|
+
require_relative "../integration/test_unit"
|
|
8
9
|
require_relative "../example_filter"
|
|
9
10
|
require_relative "../spec_ast_cache"
|
|
10
11
|
require_relative "../source_ast_cache"
|
|
@@ -12,7 +13,8 @@ require_relative "../source_ast_cache"
|
|
|
12
13
|
unless defined?(Evilution::Runner::INTEGRATIONS)
|
|
13
14
|
Evilution::Runner::INTEGRATIONS = {
|
|
14
15
|
rspec: Evilution::Integration::RSpec,
|
|
15
|
-
minitest: Evilution::Integration::Minitest
|
|
16
|
+
minitest: Evilution::Integration::Minitest,
|
|
17
|
+
test_unit: Evilution::Integration::TestUnit
|
|
16
18
|
}.freeze
|
|
17
19
|
end
|
|
18
20
|
|
data/lib/evilution/runner.rb
CHANGED
|
@@ -154,7 +154,12 @@ class Evilution::Runner
|
|
|
154
154
|
return
|
|
155
155
|
end
|
|
156
156
|
|
|
157
|
-
|
|
157
|
+
# Resolve to absolute now — fork children chdir into a per-mutation
|
|
158
|
+
# sandbox (EV-wqxu / GH #1278) before suppress_child_output runs, so a
|
|
159
|
+
# relative log_dir would reopen $stdout/$stderr at <sandbox>/<log_dir>
|
|
160
|
+
# which does not exist and the child dies with Errno::ENOENT before
|
|
161
|
+
# marshaling any result back.
|
|
162
|
+
dir = File.expand_path(config.quiet_children_dir)
|
|
158
163
|
begin
|
|
159
164
|
FileUtils.rm_rf(dir)
|
|
160
165
|
FileUtils.mkdir_p(dir)
|
data/lib/evilution/version.rb
CHANGED
data/lib/evilution.rb
CHANGED
|
@@ -157,6 +157,17 @@ module Evilution
|
|
|
157
157
|
@in_isolated_worker = previous
|
|
158
158
|
end
|
|
159
159
|
|
|
160
|
+
# Base directory for resolving project-relative paths. An isolated worker
|
|
161
|
+
# has chdir'd into a per-mutation sandbox (EV-wqxu / GH #1278), so callers
|
|
162
|
+
# in that context must anchor against PROJECT_ROOT rather than Dir.pwd —
|
|
163
|
+
# otherwise spec files, source eval __FILE__, and $LOAD_PATH entries
|
|
164
|
+
# resolve into the sandbox and break the run. In any other context (normal
|
|
165
|
+
# use, tests that intentionally chdir into a fixture project layout, etc.)
|
|
166
|
+
# the caller's Dir.pwd remains the truth.
|
|
167
|
+
def self.project_base_dir
|
|
168
|
+
in_isolated_worker? ? PROJECT_ROOT : Dir.pwd
|
|
169
|
+
end
|
|
170
|
+
|
|
160
171
|
class Error < StandardError
|
|
161
172
|
attr_reader :file
|
|
162
173
|
|
data/script/run_self_baseline
CHANGED
|
@@ -38,7 +38,7 @@ dirs.each do |dir|
|
|
|
38
38
|
end
|
|
39
39
|
|
|
40
40
|
log = File.join(LOG_DIR, "#{dir}.self.log")
|
|
41
|
-
cmd = [WRAPPER, "--strict", "--jobs=4", "--timeout=15", "--quiet-children", *files]
|
|
41
|
+
cmd = [WRAPPER, "--strict", "--jobs=4", "--timeout=15", "--quiet-children", "--isolation=fork", *files]
|
|
42
42
|
puts "==> #{dir} (#{files.length} files)"
|
|
43
43
|
pid = spawn(*cmd, out: log, err: %i[child out])
|
|
44
44
|
Process.wait(pid)
|
|
@@ -53,7 +53,7 @@ toplevel_files = Dir.glob(File.join(ROOT, "lib", "evilution", "*.rb"))
|
|
|
53
53
|
toplevel_files.reject! { |f| SKIP_FILES.include?(f) }
|
|
54
54
|
unless toplevel_files.empty?
|
|
55
55
|
log = File.join(LOG_DIR, "toplevel.self.log")
|
|
56
|
-
cmd = [WRAPPER, "--strict", "--jobs=4", "--timeout=15", "--quiet-children", *toplevel_files]
|
|
56
|
+
cmd = [WRAPPER, "--strict", "--jobs=4", "--timeout=15", "--quiet-children", "--isolation=fork", *toplevel_files]
|
|
57
57
|
puts "==> toplevel (#{toplevel_files.length} files)"
|
|
58
58
|
pid = spawn(*cmd, out: log, err: %i[child out])
|
|
59
59
|
Process.wait(pid)
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: evilution
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.32.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Denis Kiselev
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-05-
|
|
11
|
+
date: 2026-05-31 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: diff-lcs
|
|
@@ -104,6 +104,7 @@ files:
|
|
|
104
104
|
- comparison_results/operator_classification.md
|
|
105
105
|
- comparison_results/operator_prioritization.md
|
|
106
106
|
- docs/ast_pattern_syntax.md
|
|
107
|
+
- docs/integrations.md
|
|
107
108
|
- docs/isolation.md
|
|
108
109
|
- docs/mutation_density_benchmark.md
|
|
109
110
|
- docs/versioning.md
|
|
@@ -240,6 +241,13 @@ files:
|
|
|
240
241
|
- lib/evilution/integration/rspec/state_guard/world_sources_by_path.rb
|
|
241
242
|
- lib/evilution/integration/rspec/test_file_resolver.rb
|
|
242
243
|
- lib/evilution/integration/rspec/unresolved_spec_warner.rb
|
|
244
|
+
- lib/evilution/integration/test_unit.rb
|
|
245
|
+
- lib/evilution/integration/test_unit/dispatcher.rb
|
|
246
|
+
- lib/evilution/integration/test_unit/framework_loader.rb
|
|
247
|
+
- lib/evilution/integration/test_unit/result_builder.rb
|
|
248
|
+
- lib/evilution/integration/test_unit/subject_class_registry.rb
|
|
249
|
+
- lib/evilution/integration/test_unit/test_file_resolver.rb
|
|
250
|
+
- lib/evilution/integration/test_unit_crash_detector.rb
|
|
243
251
|
- lib/evilution/isolation.rb
|
|
244
252
|
- lib/evilution/isolation/fork.rb
|
|
245
253
|
- lib/evilution/isolation/in_process.rb
|