evilution 0.31.0 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +28 -0
- data/.rubocop_todo.yml +1 -0
- data/CHANGELOG.md +28 -0
- data/README.md +8 -8
- data/docs/integrations.md +141 -0
- data/docs/isolation.md +15 -0
- data/lib/evilution/baseline.rb +11 -4
- data/lib/evilution/cli/parser/options_builder.rb +1 -1
- data/lib/evilution/config/validators/integration.rb +5 -1
- data/lib/evilution/config.rb +1 -1
- data/lib/evilution/integration/loading/mutation_applier.rb +1 -2
- data/lib/evilution/integration/loading/source_evaluator.rb +1 -2
- data/lib/evilution/integration/loading/test_load_path.rb +76 -0
- data/lib/evilution/integration/minitest.rb +6 -3
- data/lib/evilution/integration/rspec/baseline_runner.rb +3 -1
- data/lib/evilution/integration/rspec/framework_loader.rb +5 -1
- data/lib/evilution/integration/rspec/state_guard/configuration_state.rb +72 -0
- data/lib/evilution/integration/rspec/state_guard/configuration_streams.rb +45 -0
- data/lib/evilution/integration/rspec/state_guard.rb +3 -1
- data/lib/evilution/integration/test_unit/dispatcher.rb +26 -0
- data/lib/evilution/integration/test_unit/framework_loader.rb +33 -0
- data/lib/evilution/integration/test_unit/result_builder.rb +53 -0
- data/lib/evilution/integration/test_unit/subject_class_registry.rb +26 -0
- data/lib/evilution/integration/test_unit/test_file_resolver.rb +48 -0
- data/lib/evilution/integration/test_unit.rb +132 -0
- data/lib/evilution/integration/test_unit_crash_detector.rb +61 -0
- data/lib/evilution/isolation/fork.rb +45 -3
- data/lib/evilution/mcp/info_tool.rb +2 -2
- data/lib/evilution/mcp/mutate_tool.rb +3 -2
- data/lib/evilution/parallel/work_queue/dispatcher.rb +94 -22
- data/lib/evilution/parallel/work_queue/worker.rb +49 -3
- data/lib/evilution/parallel/work_queue/worker_registry.rb +47 -0
- data/lib/evilution/parallel/work_queue.rb +8 -0
- data/lib/evilution/reporter/cli/line_formatters/unresolved_rate_warning.rb +50 -0
- data/lib/evilution/reporter/cli/metrics_block.rb +2 -0
- data/lib/evilution/runner/baseline_runner.rb +3 -1
- data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +28 -1
- data/lib/evilution/runner.rb +12 -1
- data/lib/evilution/spec_resolver.rb +81 -9
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution.rb +11 -0
- data/lib/tasks/stress.rake +15 -0
- data/script/run_self_baseline +2 -2
- metadata +16 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 07051270c176d0d26e28c2978c6bfa2eb0d76dd2e6ba453387f9731f04a0d79b
|
|
4
|
+
data.tar.gz: 1562d91a064d05af5b6264c89eacf93077630bc09b4a149010d6e4e5514409ed
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9957755214557a006e461cadd076d57b5c1d55536e9cec5105df1b1b37580150a6ff5fc8a33c816c0356a8e890eab1ac8e5fcfccac1acb6a46e18c76ba8ec909
|
|
7
|
+
data.tar.gz: 8a2a4000adbaf10f178cbed4cb117644d25bd27ec2730fb02024fa68b3cf8424181b0d5e45bbfbe53b3c9a9a37266e306de7592f16443d0d8545ad46f878f4af
|
data/.beads/interactions.jsonl
CHANGED
|
@@ -393,3 +393,31 @@
|
|
|
393
393
|
{"id":"int-4ea24fb5","kind":"field_change","created_at":"2026-05-16T16:04:54.395246914Z","actor":"Denis Kiselev","issue_id":"EV-s24s","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Already fixed — both named specs (unparseable_short_circuit_spec.rb, neutralization_integration_spec.rb) pass and destructure via execution.results. Stale in_progress; syncing."}}
|
|
394
394
|
{"id":"int-8b699158","kind":"field_change","created_at":"2026-05-16T17:22:07.345408768Z","actor":"Denis Kiselev","issue_id":"EV-kcuf","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Proof-of-life canary implemented + merged. Runner::Canary runs a synthetic unobservable mutation at session start; aborts if not scored :survived. --[no-]canary flag, config.canary default true."}}
|
|
395
395
|
{"id":"int-e4dcc2b3","kind":"field_change","created_at":"2026-05-27T04:19:00.738936734Z","actor":"Denis Kiselev","issue_id":"EV-qbd6","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Fix landed on master in commit 0f1781f (require 'bundler/setup' at bin/evilution-self:16). Verified 2026-05-27: crash dir lib/evilution/result/*.rb scores 91.11% (744 muts, 656 killed, 64 survived, 24 equivalent, 0 errors); canary passes; dual-runtime harness measures crash dirs again."}}
|
|
396
|
+
{"id":"int-4cb27d5a","kind":"field_change","created_at":"2026-05-28T06:42:48.564599548Z","actor":"Denis Kiselev","issue_id":"EV-pyx6","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Fix merged on master in commit 72b43a3 (PR #1292). Added Evilution.project_base_dir helper + anchored spec/ via Evilution.project_base_dir in framework_loader.rb and baseline_runner.rb. Verified 2026-05-28: bin/evilution-self run lib/evilution/config/env_loader.rb scores 94.59% (was 0 before fix). 4705 examples pass, rubocop clean."}}
|
|
397
|
+
{"id":"int-4cd16cba","kind":"field_change","created_at":"2026-05-29T04:54:00.815774086Z","actor":"Denis Kiselev","issue_id":"EV-8tmc","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Partial fix landed in commit 525787b: 18 of 20 lib/ subdirs now measurable (was 11/20). Two follow-up causes diagnosed and addressed: (1) auto-isolation picking in_process for non-Rails projects + RSpec swallowing Timeout::Error per-example, fixed by script using --isolation=fork explicitly; (2) ChildOutput log_dir relative-path bug under EV-wqxu sandbox CWD, fixed by absolutizing in runner.rb#configure_child_output. Remaining 2 dirs (parallel + toplevel) hang via different root cause — grandchild fork leak — tracked separately as EV-dgjv (GH #1295)."}}
|
|
398
|
+
{"id":"int-991e5acf","kind":"field_change","created_at":"2026-05-30T15:56:24.341278813Z","actor":"Denis Kiselev","issue_id":"EV-auk5","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
399
|
+
{"id":"int-df2bd661","kind":"field_change","created_at":"2026-05-30T15:56:24.686504141Z","actor":"Denis Kiselev","issue_id":"EV-9d62","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
400
|
+
{"id":"int-6a1dc0ce","kind":"field_change","created_at":"2026-05-30T15:56:25.04815512Z","actor":"Denis Kiselev","issue_id":"EV-unar","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
401
|
+
{"id":"int-defdc095","kind":"field_change","created_at":"2026-05-30T15:56:25.406639346Z","actor":"Denis Kiselev","issue_id":"EV-04sc","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
402
|
+
{"id":"int-bc5a7747","kind":"field_change","created_at":"2026-05-30T17:33:33.191722982Z","actor":"Denis Kiselev","issue_id":"EV-8qiy","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
403
|
+
{"id":"int-796e3efc","kind":"field_change","created_at":"2026-05-30T17:33:34.539604057Z","actor":"Denis Kiselev","issue_id":"EV-uv11","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
404
|
+
{"id":"int-5fb67d2b","kind":"field_change","created_at":"2026-05-30T17:46:39.956899342Z","actor":"Denis Kiselev","issue_id":"EV-bcjp","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
405
|
+
{"id":"int-6ba88696","kind":"field_change","created_at":"2026-05-30T17:59:44.432841837Z","actor":"Denis Kiselev","issue_id":"EV-aqxd","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
406
|
+
{"id":"int-34eb133e","kind":"field_change","created_at":"2026-05-31T02:20:26.61836178Z","actor":"Denis Kiselev","issue_id":"EV-vumz","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
407
|
+
{"id":"int-9527af14","kind":"field_change","created_at":"2026-05-31T02:47:48.910182696Z","actor":"Denis Kiselev","issue_id":"EV-zhqc","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
408
|
+
{"id":"int-dfe331ef","kind":"field_change","created_at":"2026-05-31T02:59:12.044281579Z","actor":"Denis Kiselev","issue_id":"EV-akt2","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
409
|
+
{"id":"int-c7bcc360","kind":"field_change","created_at":"2026-05-31T03:12:47.029582311Z","actor":"Denis Kiselev","issue_id":"EV-6az9","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
410
|
+
{"id":"int-fa3ade4f","kind":"field_change","created_at":"2026-05-31T08:47:32.000198989Z","actor":"Denis Kiselev","issue_id":"EV-d3av","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
411
|
+
{"id":"int-b1831323","kind":"field_change","created_at":"2026-05-31T08:47:50.244781995Z","actor":"Denis Kiselev","issue_id":"EV-d7re","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}
|
|
412
|
+
{"id":"int-6325348d","kind":"field_change","created_at":"2026-06-05T17:09:56.549588702Z","actor":"Denis Kiselev","issue_id":"EV-rxob","extra":{"field":"status","new_value":"in_progress","old_value":"closed"}}
|
|
413
|
+
{"id":"int-2cd0e23a","kind":"field_change","created_at":"2026-06-06T03:49:11.649710835Z","actor":"Denis Kiselev","issue_id":"EV-axze","extra":{"field":"priority","new_value":"1","old_value":"3"}}
|
|
414
|
+
{"id":"int-586865ca","kind":"field_change","created_at":"2026-06-06T03:49:12.267581064Z","actor":"Denis Kiselev","issue_id":"EV-axze","extra":{"field":"priority","new_value":"2","old_value":"1"}}
|
|
415
|
+
{"id":"int-c295efed","kind":"field_change","created_at":"2026-06-06T03:56:26.595461244Z","actor":"Denis Kiselev","issue_id":"EV-gl1e","extra":{"field":"status","new_value":"in_progress","old_value":"open"}}
|
|
416
|
+
{"id":"int-552b93f8","kind":"field_change","created_at":"2026-06-06T06:34:06.544071647Z","actor":"Denis Kiselev","issue_id":"EV-cnx8","extra":{"field":"status","new_value":"in_progress","old_value":"open"}}
|
|
417
|
+
{"id":"int-38e50f55","kind":"field_change","created_at":"2026-06-06T06:34:58.433491242Z","actor":"Denis Kiselev","issue_id":"EV-gl1e","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Merged via PR #1329"}}
|
|
418
|
+
{"id":"int-121d6c46","kind":"field_change","created_at":"2026-06-06T07:31:53.274033616Z","actor":"Denis Kiselev","issue_id":"EV-cnx8","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Merged via PR #1331"}}
|
|
419
|
+
{"id":"int-17dbc619","kind":"field_change","created_at":"2026-06-06T08:17:59.133451819Z","actor":"Denis Kiselev","issue_id":"EV-jwao","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Merged via PR #1333. Forward INT/TERM to worker process groups before parent dies; WorkerRegistry (lock-free COW pgid registry, signal-safe), register-before-isolate (race fix from review), unregister on reap. Specs + e2e green."}}
|
|
420
|
+
{"id":"int-9eb0b37f","kind":"field_change","created_at":"2026-06-06T10:31:43.322237231Z","actor":"Denis Kiselev","issue_id":"EV-2sh8","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Merged via PR #1335. Mutation child setpgid(0,0) as own group leader; terminate_child group-kills via signal_tree(-pid + pid) on TERM/KILL ladder; sweeps blocking grandchildren on inner timeout. Specs green, memory:check PASS. Flaky-test review hardened (timeout 3s, atomic pid write)."}}
|
|
421
|
+
{"id":"int-cdc02b29","kind":"field_change","created_at":"2026-06-06T15:19:26.153490037Z","actor":"Denis Kiselev","issue_id":"EV-dlnn","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Merged via PR #1344. Flaky /tmp glob replaced with parent-side sandbox capture (spy Dir.mktmpdir, assert the evilution-run dir gone after timeout). Deterministic, fork.rb untouched."}}
|
|
422
|
+
{"id":"int-af4ca762","kind":"field_change","created_at":"2026-06-06T17:23:17.168065946Z","actor":"Denis Kiselev","issue_id":"EV-dwqw","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Merged via PR #1345. Real carrier was RSpec.configuration @preferred_options (color_mode getter reads it first); inner Runner.run --no-color force-merges color_mode=:off in place. StateGuard::ConfigurationState snapshots @preferred_options by dup + stream ivars and restores them; in-process example re-enabled. Visual dots green, suite 4881 green."}}
|
|
423
|
+
{"id":"int-f7253919","kind":"field_change","created_at":"2026-06-07T04:14:56.864785049Z","actor":"Denis Kiselev","issue_id":"EV-z7f5","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Merged PR #1347 (master). opt1+2+3 + Copilot review fixes. Empirical 4-repro re-run remains under EV-rxob canary harness (the dep)."}}
|
data/.rubocop_todo.yml
CHANGED
|
@@ -12,6 +12,7 @@ Metrics/ClassLength:
|
|
|
12
12
|
- "lib/evilution/isolation/fork.rb"
|
|
13
13
|
- "lib/evilution/integration/minitest.rb"
|
|
14
14
|
- "lib/evilution/mcp/mutate_tool.rb"
|
|
15
|
+
- "lib/evilution/parallel/work_queue/dispatcher.rb"
|
|
15
16
|
- "lib/evilution/runner.rb"
|
|
16
17
|
- "lib/evilution/runner/isolation_resolver.rb"
|
|
17
18
|
|
data/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,34 @@
|
|
|
2
2
|
|
|
3
3
|
Versioning policy: see [docs/versioning.md](docs/versioning.md).
|
|
4
4
|
|
|
5
|
+
## [0.33.0] - 2026-06-07
|
|
6
|
+
|
|
7
|
+
### Added
|
|
8
|
+
|
|
9
|
+
- **Auto spec-resolution now finds specs in non-mirrored test layouts (`spec/unit`, `spec/lib`, `test/unit`, `test/lib`) instead of scoring 0% out of the box** — when `--spec` is omitted, `Evilution::SpecResolver` previously only resolved tests that mirrored the `lib/` tree 1:1 (`lib/foo/bar.rb` → `spec/foo/bar_spec.rb`). Projects that park their suites under a `unit`/`lib` bucket (`spec/unit/foo/bar_spec.rb`, `test/unit/foo/bar_test.rb` — a very common convention) resolved nothing, so every mutation reported `:unresolved` and the score collapsed to 0.0 with no actionable signal. The resolver now layers conventional-subdir candidates (`CONVENTIONAL_SUBDIRS = %w[unit lib]`) and parent-directory fallbacks on top of the deterministic mirror, ranking full mirrors above dropped-namespace and bare-basename guesses. When mutations still cannot be paired with a spec, a new **unresolved-rate warning** surfaces in both CLI (`reporter/cli/line_formatters/unresolved_rate_warning.rb`) and HTML (`reporter/html/sections/unresolved_details.rb`) output with a best-guess suggestion per source file, so a high unresolved rate reads as a resolution problem to fix (pass `--spec`, adjust layout) rather than a silently meaningless 0%. Repros that went 0.0 → real scores out of the box: webmock, doorkeeper, concurrent-ruby (EV-z7f5, PR #1347, GH #1325)
|
|
10
|
+
- **Opt-in parallel/isolation stress + load suite (`rake stress`) plus a weekly CI guardrail** — a new `:stress`-tagged spec (`spec/evilution/parallel/stress_spec.rb`) drives `Evilution::Parallel::WorkQueue` and `Evilution::Isolation::Fork` far past the per-class fixtures: 10k items at `-j8` with worker recycling, simultaneous worker-timeout and worker-death cascades (folding the EV-gl1e per-item-timeout recovery regression), deadline precision under scheduling churn, sustained-load RSS bounds, and hundreds of real forked mutation runs mixing fast and blocking children. It asserts the invariants that matter under load — correct ordered results, no deadlocks (every run is `Timeout`-wrapped), no zombie/unreaped workers (non-blocking `waitpid(WNOHANG)`), no FD leaks, bounded memory growth. Excluded from the default run; lifted by `RUN_STRESS=1` (set automatically by `rake stress`). A scheduled `.github/workflows/stress.yml` runs it weekly (Mondays 06:00 UTC) and on demand with CI-tuned scale knobs (`STRESS_JOBS`/`STRESS_ITEMS`/`STRESS_FORK_MUTS`/`STRESS_RUN_TIMEOUT`). The full-scale run surfaced no new defects — the EV-gl1e/EV-cnx8/EV-jwao/EV-2sh8 isolation hardening below holds under load (EV-axze, PR #1348, GH #861)
|
|
11
|
+
|
|
12
|
+
### Fixed
|
|
13
|
+
|
|
14
|
+
- **A single stuck or dead worker no longer aborts the entire parallel run** — `Evilution::Parallel::WorkQueue::Dispatcher` enforced timeouts with a coarse pool-wide watchdog: one mutation that blocked past `item_timeout` (e.g. a checkout-breaking mutation that stalls a child in `ConditionVariable#wait`) raised `"worker timed out after Ns"` as the run's `first_error` and tore down every worker, and a worker that exited unexpectedly (`handle_dead`) did the same. The dispatcher now tracks a **per-worker deadline**: it kills and recycles only the stuck worker, marks just that worker's in-flight item with a `WorkQueue::TIMED_OUT` / `DIED` sentinel, and continues the remaining work. `Strategy::Parallel` translates the sentinels into `:timeout` / `:error` `MutationResult`s. connection_pool jobs=4 (which previously whole-run-aborted) now completes with the stuck mutation scored `:timed_out` and a real non-zero score (EV-gl1e, PR #1329, GH #1324)
|
|
15
|
+
- **Worker SIGKILL now reaps grandchild processes instead of orphaning them** — when the dispatcher SIGKILLed a stuck pool worker, any grandchildren the worker had forked (the per-mutation test subprocess and anything *it* spawned) were left orphaned to init, leaking processes across a long run. Each worker now makes itself a process-group leader (`Process.setpgid(pid, pid)` in `Worker.spawn`) and `Worker#kill` signals the whole group (`-pid`) with a single-pid fallback, so killing a worker takes its entire subtree down (EV-cnx8, PR #1331, GH #1327)
|
|
16
|
+
- **Terminal interrupts (Ctrl-C) now forward to worker process groups instead of leaking them** — once workers ran in their own process groups (above), a SIGINT/SIGTERM to the evilution parent no longer reached the workers' groups, so an interrupted parallel run could leave workers and their mutation children alive. A new `WorkerRegistry` tracks live worker pgids; the `Runner` signal handler forwards the terminating signal to every registered group. The worker registers its pgid **before** `setpgid` to close a trap race where a signal could arrive after the child had changed groups but before the registry knew about it (EV-jwao, PR #1333, GH #1332)
|
|
17
|
+
- **A mutation that blocks a grandchild no longer hangs the per-mutation timeout** — `Evilution::Isolation::Fork` makes the per-mutation child its own process-group leader (`Process.setpgid(0, 0)`) before running the test command, so when the mutation spawns a grandchild that blocks (or traps interrupts), the timeout's TERM → KILL ladder reaches the whole subtree and the child is bounded rather than wedged behind a blocking grandchild (EV-2sh8, PR #1335, GH #1330)
|
|
18
|
+
- **Minitest / test-unit projects whose test helper is loaded via `require "test_helper"` no longer error every mutation with `LoadError`** — the mutation child did not place the project's test root on `$LOAD_PATH`, so a suite that does a bare `require "test_helper"` (the standard Minitest convention) raised `LoadError` and every mutation reported `:error`, collapsing the run. A new `Evilution::Integration::Loading::TestLoadPath` prepends the relevant test directories to `$LOAD_PATH` for the load, and restores it afterward; the Minitest and Test::Unit integrations both route their test-file loads through it. Repro: state_machines (EV-52hf, PR #1341, GH #1326)
|
|
19
|
+
- **In-process isolation no longer leaks RSpec global configuration into the host process** — an `--isolation=in_process` run mutates `RSpec.configuration` while dispatching each mutation, and two of those fields (`color_mode` and `output_stream`) were left mutated after the run, bleeding RSpec's color setting and redirected output stream into the host process that invoked evilution. A new `ConfigurationState` strategy (wired into `StateGuard`) snapshots and restores the affected configuration fields around each in-process run, so the host's RSpec state is unchanged afterward (EV-dwqw, PR #1345/#1346, GH #1343)
|
|
20
|
+
- **Flaky `fork_spec` sandbox-cleanup example decoupled from global `/tmp`** — the spec asserting that a child timeout cleans up its sandbox temp directory keyed off the shared `/tmp`, so concurrent activity made it intermittently fail. It now scopes to its own directory, removing the false negative (test-only) (EV-dlnn, PR #1344, GH #1342)
|
|
21
|
+
|
|
22
|
+
## [0.32.0] - 2026-05-31
|
|
23
|
+
|
|
24
|
+
### Added
|
|
25
|
+
|
|
26
|
+
- **`--integration test-unit` runs mutations against projects whose suites use the `test-unit` gem** — Rails projects that `require "test/unit/rails/test_help"` make `ActiveSupport::TestCase` inherit from `Test::Unit::TestCase`, and the existing Minitest integration could not dispatch them (`Minitest::Runnable.runnables` is empty because the test classes are not Minitest runnables at all). A dedicated `Evilution::Integration::TestUnit` orchestrator now wires a framework loader (disables the `Test::Unit::AutoRunner` at_exit handler so it does not fire on evilution exit), a subject-class registry (diffs `Test::Unit::TestCase` descendants across `load` to scope each mutation to freshly-loaded suites without a public registry-clear API), a dispatcher (`Test::Unit::UI::Console::TestRunner` with output captured to a `StringIO`), a spec resolver (`test/foo_test.rb` convention, matching Minitest), and a result builder shaping the `passed`/`test_crashed`/`no_tests_ran`/`unresolved` hashes that `classify_status` consumes. `Evilution::Integration::TestUnitCrashDetector` mirrors the Minitest analogue, distinguishing `Test::Unit::Failure` (assertion) from `Test::Unit::Error` (exception/crash) via `TestResult.add_listener(FAULT, ...)`. CLI accepts both `--integration test-unit` and `--integration test_unit`; the MCP `evilution-mutate` tool advertises `test-unit` in its JSON schema enum. Kaminari canary (`kaminari-core/lib/kaminari/models/page_scope_methods.rb`) now scores 63.60% (272 mutations dispatched) where 0.30.x scored 0.00% with the Minitest integration. See [docs/integrations.md](docs/integrations.md). Suggest-tests templates for test-unit are not yet implemented; runs with `--suggest-tests --integration test-unit` fall back to the generic operator-level phrasing (EV-d7re epic + sub-beads EV-8qiy/EV-uv11/EV-bcjp/EV-aqxd/EV-vumz/EV-zhqc/EV-akt2/EV-6az9/EV-d3av, PRs #1314/#1315/#1316/#1318/#1320/#1321, GH #1267)
|
|
27
|
+
|
|
28
|
+
### Fixed
|
|
29
|
+
|
|
30
|
+
- **Pool workers no longer hang indefinitely in `Process.wait` when a per-mutation child writes a payload but does not exit promptly** — `Evilution::Isolation::Fork#reap_and_decode` called unbounded `Process.wait(pid)` after reading a length-prefixed payload from the marshal pipe. If the per-mutation child had written the payload but was still stuck in `execute_in_child` waiting on a subject grandchild the mutation broke (or a grandchild had written garbage bytes that looked like a valid header+body to the parent's inherited write fd), the pool worker blocked forever; the per-mutation timeout had no effect because `wait_for_result` had already returned. `Evilution::Parallel::WorkQueue::Dispatcher`'s `item_timeout` then fired `"worker timed out after 30s"` and SIGKILLed the pool worker, leaving the per-mutation child orphaned to init. `reap_and_decode` now bounds the wait by `REAP_DEADLINE = 1.0`s and falls through to the TERM → GRACE_PERIOD → KILL → safe_wait ladder when the child has not exited, so a stuck child takes ~3s worst-case instead of indefinite. Previously-unmeasurable parallel/ + toplevel/ self-baseline directories (parallel/work_queue.rb, parallel/work_queue/dispatcher.rb, parallel/work_queue/worker.rb, isolation/, toplevel) now all produce real summaries (EV-dgjv, PR #1296, GH #1295)
|
|
31
|
+
- **`script/run_self_baseline` toplevel batch invocation now passes `--isolation=fork`** — every subdir invocation explicitly passed `--isolation=fork`, but the trailing toplevel batch (`lib/evilution/*.rb` files) omitted it. Default isolation auto-picked in_process for the non-Rails self-mutation, which (combined with RSpec's per-example `Timeout::Error` swallow) caused the toplevel batch to hit dispatcher item_timeout and report `NO_SUMMARY`. The toplevel cmd now matches the per-subdir cmd, so all 20 self-baseline dirs are measurable (EV-dgjv, PR #1296, GH #1295)
|
|
32
|
+
|
|
5
33
|
## [0.31.0] - 2026-05-27
|
|
6
34
|
|
|
7
35
|
### Added
|
data/README.md
CHANGED
|
@@ -104,10 +104,10 @@ Every command, subcommand, and flag listed in this section is part of evilution'
|
|
|
104
104
|
| `--[no-]canary` | Boolean | _(enabled)_ | Run a proof-of-life synthetic mutation at session start; abort the run if the pipeline misreports it. Catches misconfigured isolation, broken autoload, and reporter-plugin eviction before any real score is produced. Pass `--no-canary` to skip (e.g. CI speed, or when the canary itself is the thing under test). |
|
|
105
105
|
| `--fail-fast [N]` | Integer | _(none)_ | Stop after N surviving mutants (default 1 if no value given). |
|
|
106
106
|
| `-v`, `--verbose` | Boolean | false | Verbose output with RSS memory and GC stats per phase and per mutation; also prints error class, message, and first 5 backtrace lines for errored mutations. |
|
|
107
|
-
| `--suggest-tests` | Boolean | false | Generate concrete test code in suggestions (RSpec or Minitest, based on `--integration`). |
|
|
107
|
+
| `--suggest-tests` | Boolean | false | Generate concrete test code in suggestions (RSpec or Minitest, based on `--integration`; falls back to generic phrasing for `test-unit`). |
|
|
108
108
|
| `-q`, `--quiet` | Boolean | false | Suppress output. |
|
|
109
109
|
| `--stdin` | Boolean | false | Read target file paths from stdin (one per line). |
|
|
110
|
-
| `--integration NAME` | String | `rspec` | Test framework integration: `rspec` or `
|
|
110
|
+
| `--integration NAME` | String | `rspec` | Test framework integration: `rspec`, `minitest`, or `test-unit`. See [docs/integrations.md](docs/integrations.md). |
|
|
111
111
|
| `--[no-]incremental` | Boolean | false | Cache killed/timeout results; skip unchanged mutations on re-runs. Pass `--no-incremental` to override `incremental: true` from the config file for one invocation (e.g. cold-cache debugging). Last flag wins when both are given. |
|
|
112
112
|
| `--save-session` | Boolean | false | Persist results as timestamped JSON under `.evilution/results/`. |
|
|
113
113
|
| `--no-progress` | Boolean | _(enabled)_ | Disable the TTY progress bar. |
|
|
@@ -169,7 +169,7 @@ schema_version: 1 # opts into strict validation (rejects unknown keys
|
|
|
169
169
|
# timeout: 30 # seconds per mutation
|
|
170
170
|
# format: text # text | json | html
|
|
171
171
|
# min_score: 0.0 # 0.0–1.0
|
|
172
|
-
# integration: rspec # test framework: rspec, minitest
|
|
172
|
+
# integration: rspec # test framework: rspec, minitest, test_unit
|
|
173
173
|
# suggest_tests: false # concrete test code in suggestions (matches integration)
|
|
174
174
|
# save_session: false # persist results under .evilution/results/
|
|
175
175
|
# isolation: auto # auto | fork | in_process (auto selects fork for Rails)
|
|
@@ -213,7 +213,7 @@ All keys recognised under `schema_version: 1`:
|
|
|
213
213
|
| `format` | String | `text` | Output format: `text`, `json`, `html`. |
|
|
214
214
|
| `target` | String / null | `null` | Filter expression: method (`Foo#bar`), class (`Foo`), namespace (`Foo*`), descendants (`descendants:Foo`), source glob (`source:**/*.rb`). |
|
|
215
215
|
| `min_score` | Float | `0.0` | Minimum mutation score (0.0–1.0) for exit code 0. |
|
|
216
|
-
| `integration` | String | `rspec` | Test framework: `rspec` or `
|
|
216
|
+
| `integration` | String | `rspec` | Test framework: `rspec`, `minitest`, or `test_unit`. |
|
|
217
217
|
| `verbose` | Boolean | `false` | Verbose output (RSS/GC stats per phase, error details for errored mutations). |
|
|
218
218
|
| `quiet` | Boolean | `false` | Suppress output. |
|
|
219
219
|
| `jobs` | Integer | `1` | Number of parallel workers. |
|
|
@@ -375,7 +375,7 @@ Compatibility policy for the `1.x` gem line:
|
|
|
375
375
|
| `unresolved` | No spec file resolved for the mutated source — **coverage gap, not a failure**. Use `--fallback-full-suite` to run the full suite instead. | excluded |
|
|
376
376
|
| `unparseable` | Mutated source failed to parse (e.g. dangling heredoc opener after `method_body_replacement`). Short-circuited — never executed. | excluded |
|
|
377
377
|
|
|
378
|
-
Unresolved mutations indicate a missing test mapping — the file has no corresponding test file that the resolver could find (for example, an RSpec `_spec.rb` file or a Minitest `_test.rb` file, depending on configuration). They are reported separately so you can act on them (add a test, adjust test naming, or opt in to the full-suite fallback) without inflating the error count.
|
|
378
|
+
Unresolved mutations indicate a missing test mapping — the file has no corresponding test file that the resolver could find (for example, an RSpec `_spec.rb` file or a Minitest `_test.rb` file, depending on configuration). The resolver searches both the `lib/`-mirrored path and common non-mirrored buckets (`spec/unit`, `spec/lib`, `test/unit`, `test/lib`), so a high unresolved rate usually means a genuinely missing or unconventionally-placed test; a run that leaves many mutations unresolved prints an unresolved-rate warning with a best-guess spec path per source file. They are reported separately so you can act on them (add a test, adjust test naming, pass `--spec`, or opt in to the full-suite fallback) without inflating the error count.
|
|
379
379
|
|
|
380
380
|
## Mutation Operators (74 total)
|
|
381
381
|
|
|
@@ -517,7 +517,7 @@ These fields are added in addition to the existing `operator`, `file`, `line`, `
|
|
|
517
517
|
|
|
518
518
|
The `evilution-mutate` tool accepts a `suggest_tests` boolean parameter (default: `false`). When enabled, survived mutation suggestions contain concrete test code that an agent can drop into a test file, instead of static description text. It currently generates RSpec-style suggestions (`it`/`expect` blocks).
|
|
519
519
|
|
|
520
|
-
Pass `suggest_tests: true` in the `evilution-mutate` call to activate this mode. The CLI also supports `--suggest-tests`; when using the CLI, generated suggestions match the `--integration` setting (RSpec `it`/`expect` blocks or Minitest `def test_`/`assert_equal` methods).
|
|
520
|
+
Pass `suggest_tests: true` in the `evilution-mutate` call to activate this mode. The CLI also supports `--suggest-tests`; when using the CLI, generated suggestions match the `--integration` setting (RSpec `it`/`expect` blocks or Minitest `def test_`/`assert_equal` methods). Concrete templates for `test-unit` are not yet implemented — `test-unit` runs fall back to generic suggestion text.
|
|
521
521
|
|
|
522
522
|
### Project Config File
|
|
523
523
|
|
|
@@ -532,7 +532,7 @@ Pass `skip_config: true` to ignore the project config file. This skips loading `
|
|
|
532
532
|
| Parameter | Purpose |
|
|
533
533
|
|---|---|
|
|
534
534
|
| `incremental` | Cache killed/timeout results across runs — set `true` when iterating on the same files |
|
|
535
|
-
| `integration` | `rspec` or `
|
|
535
|
+
| `integration` | `rspec`, `minitest`, or `test_unit` |
|
|
536
536
|
| `isolation` | `auto`, `fork`, or `in_process` |
|
|
537
537
|
| `baseline` | `false` to skip the baseline suite check when you already know it's green |
|
|
538
538
|
| `save_session` | Persist results to `.evilution/results/` for inspection via `evilution-session` |
|
|
@@ -774,7 +774,7 @@ Tests 4 paths (InProcess isolation, Fork isolation, mutation generation + stripp
|
|
|
774
774
|
3. **Filter** — Disable comments, Sorbet `sig` blocks, and AST ignore patterns exclude mutations before execution
|
|
775
775
|
4. **Mutate** — 74 operators produce text replacements at precise byte offsets (source-level surgery, no AST unparsing); heredoc literal text is skipped by default. Identical byte-mutations from different operators are deduplicated by `(file_path, mutated_source)` so the count is not inflated by overlap
|
|
776
776
|
5. **Isolate** — Mutations are applied to temporary file copies (never modifying originals); load-path redirection ensures `require` resolves the mutated copy. Default isolation is in-process for plain Ruby projects and fork for Rails projects (auto-detected); `--isolation fork` forces forked child processes. Both sequential and parallel (`--jobs N`) modes respect the configured isolation strategy
|
|
777
|
-
6. **Test** — The configured test framework (RSpec or
|
|
777
|
+
6. **Test** — The configured test framework (RSpec, Minitest, or Test::Unit) executes against the mutated source
|
|
778
778
|
7. **Collect** — Source strings and AST nodes are released after use to minimize memory retention
|
|
779
779
|
8. **Report** — Results aggregated into text, JSON, or HTML, including efficiency metrics and peak memory usage
|
|
780
780
|
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
# Test Framework Integrations
|
|
2
|
+
|
|
3
|
+
Evilution supports three test framework integrations, selected via the
|
|
4
|
+
`--integration NAME` CLI flag (or the `integration:` key in `.evilution.yml`).
|
|
5
|
+
Each integration owns its own framework loader, dispatcher, spec resolver,
|
|
6
|
+
and result builder so they can evolve independently.
|
|
7
|
+
|
|
8
|
+
| Integration | Flag value | Default test dir | Default test suffix | Gem dep |
|
|
9
|
+
|---------------|---------------|------------------|---------------------|------------------------|
|
|
10
|
+
| RSpec | `rspec` | `spec/` | `_spec.rb` | `rspec-core` |
|
|
11
|
+
| Minitest | `minitest` | `test/` | `_test.rb` | `minitest` |
|
|
12
|
+
| Test::Unit | `test-unit` | `test/` | `_test.rb` | `test-unit` |
|
|
13
|
+
|
|
14
|
+
## When to pick which
|
|
15
|
+
|
|
16
|
+
- **`rspec`** (default) — Suites built on `RSpec.describe` / `it` / `expect`.
|
|
17
|
+
- **`minitest`** — Suites that subclass `Minitest::Test` or use
|
|
18
|
+
`Minitest::Spec`'s `describe` / `it`. Includes Rails apps where
|
|
19
|
+
`ActiveSupport::TestCase` inherits from `Minitest::Test`.
|
|
20
|
+
- **`test-unit`** — Suites that subclass `Test::Unit::TestCase`, including
|
|
21
|
+
Rails projects that `require "test/unit/rails/test_help"` (which makes
|
|
22
|
+
`ActiveSupport::TestCase < Test::Unit::TestCase`). The `test-unit` gem's
|
|
23
|
+
`TestCase` classes are *not* `Minitest::Runnable` subclasses, so the
|
|
24
|
+
`minitest` integration cannot dispatch them — pick this integration whenever
|
|
25
|
+
your test helper pulls in `test/unit/rails/test_help`, `test-unit-activerecord`,
|
|
26
|
+
or similar test-unit-specific glue.
|
|
27
|
+
|
|
28
|
+
## CLI examples
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
# RSpec — default; specs in spec/, named *_spec.rb
|
|
32
|
+
bundle exec evilution run lib/foo.rb
|
|
33
|
+
|
|
34
|
+
# Minitest
|
|
35
|
+
bundle exec evilution run lib/foo.rb \
|
|
36
|
+
--integration minitest --spec test/foo_test.rb
|
|
37
|
+
|
|
38
|
+
# Test::Unit (gem name uses hyphen; symbol value uses underscore)
|
|
39
|
+
bundle exec evilution run lib/foo.rb \
|
|
40
|
+
--integration test-unit --spec test/foo_test.rb
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
The CLI accepts both `test-unit` and `test_unit` strings; the internal config
|
|
44
|
+
symbol is always `:test_unit`.
|
|
45
|
+
|
|
46
|
+
## MCP examples
|
|
47
|
+
|
|
48
|
+
The `evilution-mutate` MCP tool exposes `integration` as a JSON schema enum
|
|
49
|
+
of `["rspec", "minitest", "test-unit"]`:
|
|
50
|
+
|
|
51
|
+
```json
|
|
52
|
+
{
|
|
53
|
+
"tool": "evilution-mutate",
|
|
54
|
+
"args": {
|
|
55
|
+
"files": ["lib/foo.rb"],
|
|
56
|
+
"integration": "test-unit",
|
|
57
|
+
"spec": ["test/foo_test.rb"]
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
## Spec resolution conventions
|
|
63
|
+
|
|
64
|
+
When `--spec` is not supplied, evilution resolves a test file from the source
|
|
65
|
+
path using the integration's spec resolver. Strips `lib/` or `app/` prefix
|
|
66
|
+
and rewrites the suffix:
|
|
67
|
+
|
|
68
|
+
| Source | rspec | minitest / test-unit |
|
|
69
|
+
|-------------------------------------|-------------------------------------|------------------------------------|
|
|
70
|
+
| `lib/foo.rb` | `spec/foo_spec.rb` | `test/foo_test.rb` |
|
|
71
|
+
| `lib/foo/bar.rb` | `spec/foo/bar_spec.rb` | `test/foo/bar_test.rb` |
|
|
72
|
+
| `app/models/user.rb` | `spec/models/user_spec.rb` | `test/models/user_test.rb` |
|
|
73
|
+
| `app/controllers/users_controller.rb` | `spec/requests/users_spec.rb` *or* `spec/controllers/users_controller_spec.rb` | `test/integration/users_test.rb` *or* `test/controllers/users_controller_test.rb` |
|
|
74
|
+
|
|
75
|
+
For controllers, the resolver tries the request-spec / integration-test
|
|
76
|
+
location first, then falls back to the controller-spec / controller-test
|
|
77
|
+
location.
|
|
78
|
+
|
|
79
|
+
### Non-mirrored layouts
|
|
80
|
+
|
|
81
|
+
Many projects do not mirror the `lib/` tree 1:1 — they park suites under a
|
|
82
|
+
`unit` or `lib` bucket (`spec/unit/foo/bar_spec.rb`, `test/unit/foo/bar_test.rb`,
|
|
83
|
+
`spec/lib/...`, `test/lib/...`). On top of the deterministic mirror above, the
|
|
84
|
+
resolver also tries these conventional sub-directories plus parent-directory
|
|
85
|
+
fallbacks, ranking the full mirror highest and bare-basename guesses lowest. So
|
|
86
|
+
a `spec/unit`-style layout resolves out of the box without `--spec`.
|
|
87
|
+
|
|
88
|
+
If mutations still cannot be paired with a spec, the run prints an
|
|
89
|
+
**unresolved-rate warning** (and an "unresolved" section in HTML output) with a
|
|
90
|
+
best-guess candidate per source file — so a high unresolved count reads as a
|
|
91
|
+
resolution problem to fix (pass `--spec`, or adjust the layout) rather than a
|
|
92
|
+
silently meaningless 0% score.
|
|
93
|
+
|
|
94
|
+
## Suggest-tests caveat
|
|
95
|
+
|
|
96
|
+
The `--suggest-tests` mode emits ready-to-paste test snippets for survived
|
|
97
|
+
mutations. Concrete templates currently exist for **rspec** and **minitest**
|
|
98
|
+
only; the **test-unit** integration falls back to the generic operator-level
|
|
99
|
+
suggestion text. Adding test-unit templates is a small follow-up.
|
|
100
|
+
|
|
101
|
+
## Other Ruby test frameworks
|
|
102
|
+
|
|
103
|
+
No additional integrations are planned at the moment. The three integrations
|
|
104
|
+
above cover the overwhelming majority of Ruby projects with unit / integration
|
|
105
|
+
test suites. Frameworks we considered and deferred:
|
|
106
|
+
|
|
107
|
+
- **Cucumber / Spinach** — BDD scenarios over Gherkin step definitions, a
|
|
108
|
+
fundamentally coarser granularity than mutation testing rewards. Not a
|
|
109
|
+
natural fit; no plans to add.
|
|
110
|
+
- **Sus** — Socketry's async-focused framework, small but actively maintained
|
|
111
|
+
user base. Could be added under the same orchestrator/collaborator pattern
|
|
112
|
+
if a real project surfaces needing it.
|
|
113
|
+
- **Bacon, Test::Spec** — early RSpec-style clones, effectively dormant.
|
|
114
|
+
- **Minitest::Spec** — already covered by the `minitest` integration.
|
|
115
|
+
|
|
116
|
+
If you maintain a project on another framework and would benefit from
|
|
117
|
+
mutation-testing it through evilution, open an issue describing the project
|
|
118
|
+
and the framework's dispatch entry-point — the existing integration layout
|
|
119
|
+
makes new entries cheap to add.
|
|
120
|
+
|
|
121
|
+
## Behind the scenes
|
|
122
|
+
|
|
123
|
+
Each integration has an orchestrator class at
|
|
124
|
+
`lib/evilution/integration/<name>.rb`. RSpec and Test::Unit additionally
|
|
125
|
+
decompose their collaborators into a sibling directory of the same name
|
|
126
|
+
(`lib/evilution/integration/rspec/...`,
|
|
127
|
+
`lib/evilution/integration/test_unit/...`); Minitest remains a single file.
|
|
128
|
+
The test-unit collaborators are:
|
|
129
|
+
|
|
130
|
+
- `framework_loader.rb` — `require "test-unit"` + disables the at_exit
|
|
131
|
+
auto-run handler so it doesn't fire on evilution exit.
|
|
132
|
+
- `subject_class_registry.rb` — ObjectSpace tracking of newly-loaded
|
|
133
|
+
`Test::Unit::TestCase` subclasses (test-unit has no public
|
|
134
|
+
registry-clear analog to `Minitest::Runnable.runnables`).
|
|
135
|
+
- `dispatcher.rb` — assembles a `Test::Unit::TestSuite` from the new
|
|
136
|
+
classes and runs it via `Test::Unit::UI::Console::TestRunner` with
|
|
137
|
+
output captured to a `StringIO`.
|
|
138
|
+
- `test_file_resolver.rb` — explicit-override + spec-selector +
|
|
139
|
+
fallback glob + warn-once for unresolved sources.
|
|
140
|
+
- `result_builder.rb` — shapes the `passed`/`test_crashed`/
|
|
141
|
+
`no_tests_ran`/`unresolved` Hash that flows into `classify_status`.
|
data/docs/isolation.md
CHANGED
|
@@ -130,6 +130,21 @@ On by default; toggle with `--[no-]canary` or `canary: true|false` in
|
|
|
130
130
|
`.evilution.yml`. The canary mirrors the configured `--isolation` so
|
|
131
131
|
isolation-specific defects are caught too.
|
|
132
132
|
|
|
133
|
+
## Parallel run resilience (`--jobs N`)
|
|
134
|
+
|
|
135
|
+
Under `--jobs N` a stuck or crashed worker no longer takes the whole run down.
|
|
136
|
+
The work-queue dispatcher tracks a per-worker deadline: if one mutation blocks
|
|
137
|
+
past its timeout (e.g. a mutation that wedges a child in `ConditionVariable#wait`)
|
|
138
|
+
or a worker exits unexpectedly, only that worker is killed and recycled, its
|
|
139
|
+
single in-flight mutation is recorded as `:timeout` / `:error`, and the run
|
|
140
|
+
continues with the remaining work. A single pathological mutation costs you one
|
|
141
|
+
result, not the entire run.
|
|
142
|
+
|
|
143
|
+
Each worker is also its own process-group leader, so killing a worker reaps the
|
|
144
|
+
mutation subprocess and any grandchildren it spawned rather than orphaning them,
|
|
145
|
+
and a terminal interrupt (Ctrl-C) is forwarded to every worker group — an
|
|
146
|
+
aborted parallel run leaves no stray worker or mutation processes behind.
|
|
147
|
+
|
|
133
148
|
## Related flags
|
|
134
149
|
|
|
135
150
|
- `--timeout N` sets the per-mutation time limit. Under `fork`, this drives
|
data/lib/evilution/baseline.rb
CHANGED
|
@@ -113,11 +113,18 @@ class Evilution::Baseline
|
|
|
113
113
|
warned = Set.new
|
|
114
114
|
subjects.map do |s|
|
|
115
115
|
resolved = @spec_resolver.call(s.file_path)
|
|
116
|
-
if resolved.nil? && warned.add?(s.file_path)
|
|
117
|
-
warn "[evilution] No matching test found for #{s.file_path}, running full suite. " \
|
|
118
|
-
"Use --spec to specify the test file."
|
|
119
|
-
end
|
|
116
|
+
warn_no_matching_test(s.file_path) if resolved.nil? && warned.add?(s.file_path)
|
|
120
117
|
resolved || @fallback_dir
|
|
121
118
|
end.uniq
|
|
122
119
|
end
|
|
120
|
+
|
|
121
|
+
def warn_no_matching_test(file_path)
|
|
122
|
+
suggestion = @spec_resolver.suggest(file_path)
|
|
123
|
+
hint = if suggestion
|
|
124
|
+
"Pass --spec #{suggestion} (best guess) or the correct test file."
|
|
125
|
+
else
|
|
126
|
+
"Use --spec to specify the test file."
|
|
127
|
+
end
|
|
128
|
+
warn "[evilution] No matching test found for #{file_path}, running full suite. #{hint}"
|
|
129
|
+
end
|
|
123
130
|
end
|
|
@@ -95,7 +95,7 @@ class Evilution::CLI::Parser::OptionsBuilder
|
|
|
95
95
|
end
|
|
96
96
|
|
|
97
97
|
def add_runner_mode_options(opts)
|
|
98
|
-
opts.on("--integration NAME", "Test integration: rspec, minitest (default: rspec)") { |i| @options[:integration] = i }
|
|
98
|
+
opts.on("--integration NAME", "Test integration: rspec, minitest, test-unit (default: rspec)") { |i| @options[:integration] = i }
|
|
99
99
|
opts.on("--isolation STRATEGY", "Isolation: auto, fork, in_process (default: auto)") { |s| @options[:isolation] = s }
|
|
100
100
|
opts.on("--preload FILE", "Preload FILE in the parent process before forking " \
|
|
101
101
|
"(default: auto-detect spec/rails_helper.rb -> spec/spec_helper.rb -> " \
|
|
@@ -3,9 +3,13 @@
|
|
|
3
3
|
require_relative "base"
|
|
4
4
|
|
|
5
5
|
class Evilution::Config::Validators::Integration < Evilution::Config::Validators::Base
|
|
6
|
-
ALLOWED = %i[rspec minitest].freeze
|
|
6
|
+
ALLOWED = %i[rspec minitest test_unit].freeze
|
|
7
7
|
|
|
8
|
+
# CLI users naturally write the gem name `test-unit`; the internal symbol
|
|
9
|
+
# uses underscore form to match the file path and registry key. Normalize
|
|
10
|
+
# hyphenated string input before coercion.
|
|
8
11
|
def self.call(value)
|
|
12
|
+
value = value.tr("-", "_") if value.is_a?(String)
|
|
9
13
|
coerce_symbol!(value, allowed: ALLOWED, name: "integration")
|
|
10
14
|
end
|
|
11
15
|
end
|
data/lib/evilution/config.rb
CHANGED
|
@@ -139,7 +139,7 @@ class Evilution::Config
|
|
|
139
139
|
# (default: true). Set false to skip (e.g. for CI speed).
|
|
140
140
|
# canary: true
|
|
141
141
|
|
|
142
|
-
# Test integration: rspec, minitest (default: rspec)
|
|
142
|
+
# Test integration: rspec, minitest, test_unit (default: rspec)
|
|
143
143
|
# integration: rspec
|
|
144
144
|
|
|
145
145
|
# Number of parallel workers (default: 1)
|
|
@@ -78,8 +78,7 @@ class Evilution::Integration::Loading::MutationApplier
|
|
|
78
78
|
# When the isolator has chdir'd into a per-mutation sandbox (EV-wqxu /
|
|
79
79
|
# GH #1278), anchor against PROJECT_ROOT so File.realpath does not chase
|
|
80
80
|
# file_path into a non-existent /tmp path.
|
|
81
|
-
|
|
82
|
-
absolute = File.realpath(File.expand_path(file_path, base))
|
|
81
|
+
absolute = File.realpath(File.expand_path(file_path, Evilution.project_base_dir))
|
|
83
82
|
$LOADED_FEATURES << absolute unless $LOADED_FEATURES.include?(absolute)
|
|
84
83
|
rescue Errno::ENOENT
|
|
85
84
|
nil
|
|
@@ -16,8 +16,7 @@ class Evilution::Integration::Loading::SourceEvaluator
|
|
|
16
16
|
# When the isolator has chdir'd into a per-mutation sandbox (EV-wqxu /
|
|
17
17
|
# GH #1278), anchor the eval __FILE__ against PROJECT_ROOT so siblings
|
|
18
18
|
# `require_relative` can find each other from the real source tree.
|
|
19
|
-
|
|
20
|
-
absolute = File.expand_path(file_path, base)
|
|
19
|
+
absolute = File.expand_path(file_path, Evilution.project_base_dir)
|
|
21
20
|
eval(source, TOPLEVEL_BINDING, absolute, 1)
|
|
22
21
|
end
|
|
23
22
|
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../loading"
|
|
4
|
+
|
|
5
|
+
# Mirrors `ruby -Itest` / `-Ispec` for in-process test loading.
|
|
6
|
+
#
|
|
7
|
+
# evilution loads resolved test files with Kernel#load instead of shelling out,
|
|
8
|
+
# so the `-Itest` shown in the displayed command string is never actually
|
|
9
|
+
# applied to $LOAD_PATH. Minitest and Test::Unit suites near-universally
|
|
10
|
+
# `require "test_helper"` (which the suite's own runner satisfies via -Itest);
|
|
11
|
+
# without the test root on $LOAD_PATH that bare require raises LoadError and
|
|
12
|
+
# every mutation errors with score 0.0 (EV-52hf / GH #1326).
|
|
13
|
+
#
|
|
14
|
+
# Anchors against Evilution.project_base_dir, which resolves to PROJECT_ROOT
|
|
15
|
+
# inside an isolated worker (EV-wqxu / GH #1278) and Dir.pwd otherwise, so the
|
|
16
|
+
# same call works on both the baseline (parent) and mutation (child) paths.
|
|
17
|
+
module Evilution::Integration::Loading::TestLoadPath
|
|
18
|
+
ROOT_NAMES = %w[test spec].freeze
|
|
19
|
+
|
|
20
|
+
module_function
|
|
21
|
+
|
|
22
|
+
# Prepend every relevant test directory to $LOAD_PATH (idempotently).
|
|
23
|
+
# Iterate in reverse so the first entry from #dirs_for ends up frontmost,
|
|
24
|
+
# preserving its order (mirrors how `ruby -Ia -Ib` lands a before b).
|
|
25
|
+
def add!(files, base: Evilution.project_base_dir)
|
|
26
|
+
dirs_for(files, base).reverse_each do |dir|
|
|
27
|
+
$LOAD_PATH.unshift(dir) unless $LOAD_PATH.include?(dir)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# The directories to put on $LOAD_PATH for the given resolved test files:
|
|
32
|
+
# the conventional test/ and spec/ roots under base, each file's own
|
|
33
|
+
# directory, and the topmost test/spec ancestor of each file (covers nested
|
|
34
|
+
# layouts like test/unit, spec/lib, spec/unit). Existing directories only,
|
|
35
|
+
# and only those inside the project base -- never a broad outside-project dir
|
|
36
|
+
# (e.g. a /tmp test file), which would over-widen $LOAD_PATH for the whole
|
|
37
|
+
# process (the baseline runs in the long-lived parent).
|
|
38
|
+
def dirs_for(files, base)
|
|
39
|
+
base = File.expand_path(base)
|
|
40
|
+
dirs = conventional_roots(base)
|
|
41
|
+
Array(files).each do |file|
|
|
42
|
+
file_dir = File.dirname(File.expand_path(file, base))
|
|
43
|
+
dirs << file_dir
|
|
44
|
+
root = root_ancestor(file_dir, base)
|
|
45
|
+
dirs << root if root
|
|
46
|
+
end
|
|
47
|
+
dirs.uniq.select { |dir| File.directory?(dir) && within?(dir, base) }
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def within?(dir, base)
|
|
51
|
+
dir == base || dir.start_with?("#{base}/")
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def conventional_roots(base)
|
|
55
|
+
ROOT_NAMES.map { |name| File.join(base, name) }
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Walk from `dir` up to `base`, returning the highest ancestor whose basename
|
|
59
|
+
# is a conventional test root (test/spec). Highest, so test/unit/foo_test.rb
|
|
60
|
+
# yields `test` (matching -Itest), not the intermediate test/unit.
|
|
61
|
+
def root_ancestor(dir, base)
|
|
62
|
+
base = File.expand_path(base)
|
|
63
|
+
found = nil
|
|
64
|
+
current = File.expand_path(dir)
|
|
65
|
+
loop do
|
|
66
|
+
found = current if ROOT_NAMES.include?(File.basename(current))
|
|
67
|
+
break if current == base
|
|
68
|
+
|
|
69
|
+
parent = File.dirname(current)
|
|
70
|
+
break if parent == current
|
|
71
|
+
|
|
72
|
+
current = parent
|
|
73
|
+
end
|
|
74
|
+
found
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require "stringio"
|
|
4
4
|
require_relative "base"
|
|
5
5
|
require_relative "minitest_crash_detector"
|
|
6
|
+
require_relative "loading/test_load_path"
|
|
6
7
|
require_relative "../spec_resolver"
|
|
7
8
|
require_relative "../spec_selector"
|
|
8
9
|
|
|
@@ -18,7 +19,9 @@ class Evilution::Integration::Minitest < Evilution::Integration::Base
|
|
|
18
19
|
require "stringio"
|
|
19
20
|
stub_autorun!
|
|
20
21
|
::Minitest::Runnable.runnables.clear
|
|
21
|
-
baseline_test_files(test_file)
|
|
22
|
+
files = baseline_test_files(test_file)
|
|
23
|
+
Evilution::Integration::Loading::TestLoadPath.add!(files)
|
|
24
|
+
files.each { |f| load(File.expand_path(f)) }
|
|
22
25
|
run_baseline_minitest
|
|
23
26
|
end
|
|
24
27
|
|
|
@@ -128,8 +131,8 @@ class Evilution::Integration::Minitest < Evilution::Integration::Base
|
|
|
128
131
|
end
|
|
129
132
|
|
|
130
133
|
def execute_minitest(mutation, files, command)
|
|
131
|
-
|
|
132
|
-
files.each { |f| load(File.expand_path(f,
|
|
134
|
+
Evilution::Integration::Loading::TestLoadPath.add!(files)
|
|
135
|
+
files.each { |f| load(File.expand_path(f, Evilution.project_base_dir)) }
|
|
133
136
|
|
|
134
137
|
detector = reset_crash_detector
|
|
135
138
|
run = run_minitest(build_args(mutation), detector)
|
|
@@ -5,7 +5,9 @@ require_relative "../rspec"
|
|
|
5
5
|
class Evilution::Integration::RSpec::BaselineRunner
|
|
6
6
|
def call(spec_file)
|
|
7
7
|
require "rspec/core"
|
|
8
|
-
|
|
8
|
+
# Anchor against PROJECT_ROOT under EV-wqxu sandbox CWD; see
|
|
9
|
+
# FrameworkLoader#add_spec_load_path for rationale.
|
|
10
|
+
spec_dir = File.expand_path("spec", Evilution.project_base_dir)
|
|
9
11
|
$LOAD_PATH.unshift(spec_dir) unless $LOAD_PATH.include?(spec_dir)
|
|
10
12
|
::RSpec.reset
|
|
11
13
|
status = ::RSpec::Core::Runner.run(
|
|
@@ -22,7 +22,11 @@ class Evilution::Integration::RSpec::FrameworkLoader
|
|
|
22
22
|
private
|
|
23
23
|
|
|
24
24
|
def add_spec_load_path
|
|
25
|
-
|
|
25
|
+
# Anchor against PROJECT_ROOT inside an isolated worker (EV-wqxu /
|
|
26
|
+
# GH #1278) so the project's spec/ dir lands on $LOAD_PATH — otherwise
|
|
27
|
+
# `require "spec_helper"` resolves to a non-existent sandbox/spec and
|
|
28
|
+
# every mutation errors as "loaded 0 examples".
|
|
29
|
+
spec_dir = File.expand_path("spec", Evilution.project_base_dir)
|
|
26
30
|
$LOAD_PATH.unshift(spec_dir) unless $LOAD_PATH.include?(spec_dir)
|
|
27
31
|
end
|
|
28
32
|
end
|