evilution 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/.migration-hint-ts +1 -1
- data/.beads/issues.jsonl +13 -13
- data/CHANGELOG.md +18 -0
- data/README.md +4 -2
- data/lib/evilution/cli.rb +11 -2
- data/lib/evilution/config.rb +12 -2
- data/lib/evilution/integration/rspec.rb +32 -24
- data/lib/evilution/isolation/fork.rb +3 -6
- data/lib/evilution/mutator/base.rb +1 -1
- data/lib/evilution/mutator/operator/index_to_dig.rb +1 -1
- data/lib/evilution/mutator/operator/string_literal.rb +18 -0
- data/lib/evilution/mutator/registry.rb +9 -2
- data/lib/evilution/runner.rb +28 -1
- data/lib/evilution/temp_dir_tracker.rb +39 -0
- data/lib/evilution/version.rb +1 -1
- metadata +2 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c8f4aa7633e70e4a54aded76fcdfeb152cb4e4ad76d587b5aa0c93bda96246e3
|
|
4
|
+
data.tar.gz: b8e65e5d0837b6873c31e6cae9621160a2a6fe75b3949d08a39f91b3df7db60b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 923d8fa302a830d1b070e27b2494c5ec3227f6c6188b26f1701a0344e54ac230626a75bd4f0ec70d6e4f10af04e5e62d04b05d93c067ee160ce3c653e86faaf6
|
|
7
|
+
data.tar.gz: f63389c729c4d121cb24a38a2bd9bd4d386707f81c16dbfd106868e952d72200278cd84fc0936ad60e666e53b5c51efce7d40a615c093d5fc327e1e910a39fd4
|
data/.beads/.migration-hint-ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
1775646585
|
data/.beads/issues.jsonl
CHANGED
|
@@ -177,7 +177,7 @@
|
|
|
177
177
|
{"id":"EV-234","title":"Conceptual deduplication of survived mutations","description":"Multiple operators can flag the same underlying coverage gap (e.g., .includes removal via method_call_removal and symbol_literal both reveal missing eager-load test). Add a post-processing pass that groups conceptually similar survivors and presents them as a single coverage gap with multiple mutation evidence. Reduces noise in reports.","notes":"GitHub: #510","status":"closed","priority":3,"issue_type":"feature","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:21:58.378005036+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-08T09:21:20.475702136+07:00","closed_at":"2026-04-08T09:21:20.475702136+07:00","close_reason":"Closed"}
|
|
178
178
|
{"id":"EV-235","title":"Bug: Non-deterministic mutation count on same file","description":"Running evilution twice on the same file (HelpController, 9 LOC) produced different mutation counts (18 vs 15). Unclear cause — possibly non-deterministic operator selection or file state difference. Low priority but should be investigated. Reported once in v0.12.0.","notes":"GitHub: #512","status":"closed","priority":4,"issue_type":"bug","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:22:01.930671333+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-04T13:20:37.135440498+07:00","closed_at":"2026-04-04T13:20:37.135440498+07:00","close_reason":"Not a bug. The 18 vs 15 difference was exactly the 3 timed-out mutations — counted in run 1 total but excluded in run 2 summary. Mutation generation is fully deterministic (no rand/shuffle/sample in codepath). Reported once in v0.12.0, never reproduced. Reporting has been significantly improved since then."}
|
|
179
179
|
{"id":"EV-236","title":"--spec-dir flag for directory-level spec inclusion","description":"Add a --spec-dir flag that auto-includes all specs in a directory, reducing the chance of missing coverage from adjacent spec files. Useful when a controller has tests split across spec/requests/, spec/controllers/, and spec/features/. Reported once.","notes":"GitHub: #513","status":"closed","priority":4,"issue_type":"feature","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:22:04.618160285+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-07T09:55:50.99913701+07:00","closed_at":"2026-04-07T09:55:50.99913701+07:00","close_reason":"--spec-dir CLI flag implemented, composes with --spec, validates directory existence. 3 unit tests passing. Merged via GH #513.","dependencies":[{"issue_id":"EV-236","depends_on_id":"EV-227","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
180
|
-
{"id":"EV-237","title":"Temp-file based mutation (don't modify original source)","description":"Evilution mutates source files in-place on the filesystem, which triggers file watchers, linters, and IDE notifications during runs. Even with the ensure-based restore (fixed earlier), race conditions exist if the process is killed. Write mutated source to a tempfile and point the test runner at it via load path manipulation. Never modify the original source file. Reported in 2 sessions (v0.16.1).","notes":"GH issue: #520 (https://github.com/marinazzio/evilution/issues/520)","status":"
|
|
180
|
+
{"id":"EV-237","title":"Temp-file based mutation (don't modify original source)","description":"Evilution mutates source files in-place on the filesystem, which triggers file watchers, linters, and IDE notifications during runs. Even with the ensure-based restore (fixed earlier), race conditions exist if the process is killed. Write mutated source to a tempfile and point the test runner at it via load path manipulation. Never modify the original source file. Reported in 2 sessions (v0.16.1).","notes":"GH issue: #520 (https://github.com/marinazzio/evilution/issues/520)","status":"closed","priority":2,"issue_type":"feature","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:22:06.770551806+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-08T15:52:59.896531797+07:00","closed_at":"2026-04-08T15:52:59.896531797+07:00","close_reason":"Closed","external_ref":"gh-520","labels":["reliability"]}
|
|
181
181
|
{"id":"EV-238","title":"Epic: Research mutation density gap with mutant","description":"Evilution consistently generates 1.8-2.6x fewer mutations than mutant across 25 feedback sessions. While some of mutant's extras are equivalent ([]→fetch, to_i→Integer()), many catch real edge cases. This epic covers: (1) Audit mutant's operator list systematically against evilution's 54 operators, (2) Identify which missing operators catch real bugs vs produce noise, (3) Prioritize operator additions by signal-to-noise ratio, (4) Target closing the gap to <1.5x. Related: existing gap analysis created 15 new operator issues (EV-214 through EV-224, #491-#505).","notes":"GitHub: #515","status":"open","priority":2,"issue_type":"feature","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:22:11.095318733+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-04T11:22:55.790159971+07:00"}
|
|
182
182
|
{"id":"EV-239","title":"Epic: Research and fix high memory baseline","description":"Evilution's memory baseline is 718+ MB even for tiny files in v0.18.0 sessions, and grows across consecutive runs in the same session (718→763→795→800 MB). Previous fixes addressed AST node retention and StringIO leaks but baseline remains high. Mutant peaks at ~200 MB for comparable workloads. This epic covers: (1) Profile memory allocation during boot/setup phase, (2) Identify what's consuming the 718 MB baseline, (3) Investigate cross-run memory growth (session-level leak?), (4) Target bringing baseline under 300 MB. Related: existing rake memory:check infrastructure exists.","notes":"GitHub: #517","status":"closed","priority":2,"issue_type":"feature","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:22:16.602817355+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-06T00:16:38.665740136+07:00","closed_at":"2026-04-06T00:16:38.665740136+07:00","close_reason":"Premise invalid: 718 MB baseline was the MCP host process, not evilution. Standalone evilution baseline is ~30 MB (confirmed via EV-242/245/246 profiling). No memory fix needed. Sub-issues resolved: EV-242 (30 MB boot baseline), EV-243 (single mutation profiled), EV-245 (no cross-run growth), EV-246 (fork has zero parent-side cost), EV-247 (RSS tracking added)."}
|
|
183
183
|
{"id":"EV-24","title":"Epic: JSON Output Improvements","description":"Make JSON output fully machine-parseable in all scenarios, including errors. Add diagnostic fields that help agents debug failures.","status":"closed","priority":1,"issue_type":"feature","owner":"denis.kiselyov@gmail.com","created_at":"2026-03-10T06:17:37.450686472+07:00","created_by":"Denis Kiselev","updated_at":"2026-03-16T11:15:24.900944562+07:00","closed_at":"2026-03-16T11:15:24.900944562+07:00","close_reason":"All children complete: structured errors, test_command in JSON, noise suppression","dependencies":[{"issue_id":"EV-24","depends_on_id":"EV-25","type":"blocks","created_at":"0001-01-01T00:00:00Z"},{"issue_id":"EV-24","depends_on_id":"EV-26","type":"blocks","created_at":"0001-01-01T00:00:00Z"},{"issue_id":"EV-24","depends_on_id":"EV-40","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
@@ -190,28 +190,28 @@
|
|
|
190
190
|
{"id":"EV-246","title":"Investigate parent vs child process memory split","description":"Measure RSS of the parent (coordinator) process separately from forked child (worker) processes. Determine where the 718 MB baseline lives — is it the parent before forking, or do children inherit and grow independently?","notes":"GitHub: #531","status":"in_progress","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:28:59.300347033+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-05T23:45:11.030185257+07:00","dependencies":[{"issue_id":"EV-246","depends_on_id":"EV-239","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
191
191
|
{"id":"EV-247","title":"Add RSS tracking per mutation to JSON output","description":"Include parent_rss_kb and child_rss_kb fields in each mutation result. child_rss_kb partially exists (seen in feedback log) — verify it is accurate and add parent_rss_kb tracking. This provides ongoing observability for memory usage.","notes":"GitHub: #532","status":"in_progress","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:29:08.569266807+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-05T23:49:30.25391422+07:00","dependencies":[{"issue_id":"EV-247","depends_on_id":"EV-239","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
192
192
|
{"id":"EV-248","title":"Implement memory budget CI gate","description":"Add a CI check that runs rake memory:check and fails if peak RSS exceeds a threshold (e.g., 400 MB for a reference fixture). This prevents memory regressions from being merged.","notes":"GitHub: #533","status":"open","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:29:16.177682161+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-06T11:45:18.765568988+07:00","close_reason":"Growth-based leak detection in CI (EV-274/PR #571) is sufficient. Absolute peak RSS budget not needed — standalone baseline is ~30 MB (EV-239 premise was invalid). Per-mutation growth check catches regressions effectively.","dependencies":[{"issue_id":"EV-248","depends_on_id":"EV-239","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
193
|
-
{"id":"EV-249","title":"Audit current SourceSurgeon mutation-and-restore flow","description":"Document the current code path: where the file is read, mutated, written, and restored. Identify all callers and the ensure-based restore mechanism. Map the failure modes (SIGKILL, OOM, etc.).","notes":"
|
|
193
|
+
{"id":"EV-249","title":"Audit current SourceSurgeon mutation-and-restore flow","description":"Document the current code path: where the file is read, mutated, written, and restored. Identify all callers and the ensure-based restore mechanism. Map the failure modes (SIGKILL, OOM, etc.).","notes":"## Audit Findings\n\n### Read Points\n1. **AST::Parser#call** (parser.rb:11) — File.read to parse with Prism\n2. **Mutator::Base#call** (base.rb:18) — File.read to store as original_source in Mutation\n3. **Integration::RSpec#apply_mutation** (rspec.rb:68) — reads before overwrite (direct-overwrite fallback only)\n4. **Isolation::Fork#restore_original_source** (fork.rb:48) — reads to verify if restore needed (defense-in-depth)\n\n### Mutation (In-Memory)\nSourceSurgeon.apply (source_surgeon.rb:6-10) is pure in-memory byte surgery. Never touches filesystem. Called from Mutator::Base#add_mutation.\n\n### Two Write Paths in Integration::RSpec#apply_mutation\n- **Path A (LOAD_PATH shadow, preferred):** Target under $LOAD_PATH → Dir.mktmpdir, write mutated source to mirrored subpath, prepend to $LOAD_PATH. Original file never touched.\n- **Path B (Direct overwrite, fallback):** Not under $LOAD_PATH → acquires exclusive flock, overwrites original file.\n\n### Restore — Two Layers\n- **Layer 1:** Integration::RSpec#restore_original via ensure in #call (rspec.rb:33-35). Path A: removes temp dir from $LOAD_PATH, purges $LOADED_FEATURES, deletes temp dir. Path B: writes @original_content back, releases flock.\n- **Layer 2:** Isolation::Fork#restore_original_source (fork.rb:47-53) — parent-process defense-in-depth. Only in sequential (Fork isolation) path. NOT in parallel path.\n\n### Execution Paths\n- **Sequential (jobs=1):** Runner → Isolation::Fork → fork child → Integration::RSpec#call → ensure restore (child) + ensure restore (parent)\n- **Parallel (jobs>1):** Runner → Parallel::Pool → WorkQueue forks workers → Isolation::InProcess → Integration::RSpec#call → ensure restore only. No parent-side defense-in-depth.\n\n### Failure Modes\n- Normal flow / exception: Safe on both paths\n- SIGKILL child (sequential): Safe (parent restores on direct-overwrite; file untouched on temp-dir)\n- **SIGKILL worker (parallel) + direct-overwrite: FILE CORRUPTED — no recovery**\n- **OOM parallel worker + direct-overwrite: FILE CORRUPTED**\n- **SIGINT/SIGTERM to parent + direct-overwrite: File may be corrupted (zero signal handlers in lib/)**\n- Disk full during restore: File stays corrupted\n\n### Key Findings\n- Zero trap/at_exit/Signal.trap calls in entire lib/ directory\n- Biggest risk: direct-overwrite fallback in parallel mode (no parent-side restore)\n- Epic EV-237 should eliminate direct-overwrite path entirely, making LOAD_PATH shadow the only path","status":"closed","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:29:17.689070042+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-08T15:53:00.183398494+07:00","closed_at":"2026-04-08T15:53:00.183398494+07:00","close_reason":"Closed","dependencies":[{"issue_id":"EV-249","depends_on_id":"EV-237","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
194
194
|
{"id":"EV-25","title":"Structured error responses in JSON mode","description":"When --format json is used and exit code is 2 (error), output a JSON object with error details instead of unstructured stderr text. Schema: { \"error\": { \"type\": \"config_error|parse_error|runtime_error\", \"message\": \"...\", \"file\": \"...\" } }. Agents currently have to regex-parse stderr which is fragile.","status":"closed","priority":1,"issue_type":"feature","owner":"denis.kiselyov@gmail.com","created_at":"2026-03-10T06:17:38.283715502+07:00","created_by":"Denis Kiselev","updated_at":"2026-03-15T22:41:54.370789377+07:00","closed_at":"2026-03-15T22:41:54.370789377+07:00","close_reason":"Merged PR #74 — structured JSON error output in CLI"}
|
|
195
195
|
{"id":"EV-250","title":"Classify mutant's extra mutations by operator category","description":"From the head-to-head data (EV-244), group mutant's extra mutations by category (e.g., receiver mutations, argument permutations, method name substitutions, literal boundary values). Count how many are signal vs noise per category. Produce a table of categories with signal/noise breakdown.","notes":"GitHub: #526","status":"open","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:29:21.320269648+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-04T11:29:32.613448913+07:00","dependencies":[{"issue_id":"EV-250","depends_on_id":"EV-238","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
196
196
|
{"id":"EV-251","title":"Prioritize operator additions by signal-to-noise ratio","description":"Rank the missing operator categories by: (a) frequency of real signal catches, (b) implementation complexity, (c) expected equivalent mutant rate. Produce a prioritized implementation order for new operators.","notes":"GitHub: #534","status":"open","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:29:38.370181376+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-04T11:29:53.283203728+07:00","dependencies":[{"issue_id":"EV-251","depends_on_id":"EV-238","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
197
197
|
{"id":"EV-252","title":"Reproduce and measure per-mutation RSS growth on reference fixture","description":"Create a reproducible benchmark: run evilution on a known fixture file, record RSS after each mutation. Confirm the ~3-8 MB/mutation growth rate. Baseline for measuring fix effectiveness.","notes":"GitHub: #539","status":"in_progress","priority":0,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:29:39.421709567+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-04T15:08:27.260195719+07:00","dependencies":[{"issue_id":"EV-252","depends_on_id":"EV-226","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
198
198
|
{"id":"EV-253","title":"Profile object allocation delta per mutation cycle","description":"Use ObjectSpace.count_objects or memory_profiler to capture what new objects are allocated during one mutation cycle and not released. Identify the top retained object types.","notes":"GitHub: #540 — Profiling complete. Root cause: RSpec ExampleGroup subclass ivars create reference cycles preventing GC (+3380 slots/mutation). Secondary: World#@sources_by_path cache. Fix proven: clearing EG ivars + sources cache after Runner.run = 0 growth.","status":"closed","priority":0,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:29:41.259785614+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-05T15:50:12.517441649+07:00","closed_at":"2026-04-05T15:50:12.517441649+07:00","close_reason":"Profiling complete. Root cause identified: RSpec ExampleGroup reference cycles + World source cache. Findings documented on GH #540.","dependencies":[{"issue_id":"EV-253","depends_on_id":"EV-226","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
199
|
-
{"id":"EV-254","title":"Design temp-file mutation architecture","description":"Design how the temp file is created, where it lives (tmpdir vs .evilution/tmp), how the test process is redirected to load it (Ruby $LOAD_PATH manipulation vs file-level bootsnap override vs ENV-based), and how Rails autoloader interacts with it.","notes":"
|
|
199
|
+
{"id":"EV-254","title":"Design temp-file mutation architecture","description":"Design how the temp file is created, where it lives (tmpdir vs .evilution/tmp), how the test process is redirected to load it (Ruby $LOAD_PATH manipulation vs file-level bootsnap override vs ENV-based), and how Rails autoloader interacts with it.","notes":"## Architecture Design: Temp-File Mutation\n\n### Problem Statement\nTwo write paths exist in Integration::RSpec#apply_mutation:\n- **Path A (LOAD_PATH shadow):** Works when file is under $LOAD_PATH — writes to temp dir, prepends to $LOAD_PATH. Original file never touched. Already safe.\n- **Path B (Direct overwrite):** Fallback when file is NOT under $LOAD_PATH — overwrites original, restores via ensure. Vulnerable to SIGKILL/OOM (especially in parallel mode where no parent-side defense-in-depth exists).\n\n**Goal:** Eliminate Path B entirely. Never modify the original source file.\n\n---\n\n### Design Decisions\n\n#### 1. Temp file location: Dir.mktmpdir (system tmpdir)\n- Use `Dir.mktmpdir('evilution')` (same as current Path A), NOT .evilution/tmp\n- Rationale: system tmpdir is auto-cleaned on reboot; no risk of polluting the project directory; avoids .gitignore concerns; already proven in current Path A\n\n#### 2. Redirection mechanism: LOAD_PATH shadow + explicit load\n**For files under $LOAD_PATH** (most lib/ files): Keep current approach — mirror subpath in temp dir, prepend to $LOAD_PATH. This handles any `require` calls during the test run.\n\n**For files NOT under $LOAD_PATH** (the current fallback case): \n- Write mutated source to temp dir mirroring the relative path from project root\n- Explicitly `load` the temp file in the forked child to redefine the class/module\n- This replaces the direct-overwrite approach entirely\n- The `load` approach works because it always executes the file (unlike `require` which checks $LOADED_FEATURES)\n\n#### 3. SourceSurgeon: No changes needed\nSourceSurgeon.apply is already pure in-memory byte surgery. It returns a mutated string without touching the filesystem. No changes required for this epic.\n\n#### 4. Where file I/O moves\n**Integration::RSpec#apply_mutation** remains the owner of temp-file writes, but the two-path logic changes:\n- Path A (under $LOAD_PATH): unchanged — temp dir + $LOAD_PATH prepend\n- Path B (not under $LOAD_PATH): temp dir + explicit `load` (replaces direct overwrite)\n- Both paths use temp files. Original is never touched.\n\n#### 5. Restore/cleanup strategy (three layers)\n1. **ensure in Integration::RSpec#call** (existing): Remove temp dir from $LOAD_PATH, purge $LOADED_FEATURES entries, FileUtils.rm_rf temp dir. Works for both paths now.\n2. **ensure in Isolation::Fork#call** (existing): Simplify — no longer needs to check/restore original file content. Instead, just verify temp dir cleanup. Keep as defense-in-depth for temp dir leaks.\n3. **at_exit hook** (new): Register a cleanup for the temp base dir pattern (evilution*) in case of unhandled exit. Safety net for leaked temp dirs.\n4. **Signal traps** (new): Trap SIGTERM/SIGINT in the parent process to ensure temp dir cleanup before exit.\n\n#### 6. Isolation::Fork#restore_original_source\n- Remove the file-content comparison and rewrite logic\n- Replace with temp-dir cleanup verification (check if any evilution temp dirs remain, clean them)\n- This is now truly defense-in-depth rather than a critical restore path\n\n#### 7. Parallel mode (InProcess isolation)\n- No special handling needed — each worker is a forked process with its own $LOAD_PATH\n- Temp dirs are per-mutation, isolated across workers\n- The biggest current risk (corrupted original file on worker SIGKILL) is eliminated because the original file is never modified\n\n#### 8. Zeitwerk (Rails autoloader) compatibility\n- Zeitwerk maps file paths to constant names using autoload_paths (which are $LOAD_PATH entries in Rails)\n- For files under Zeitwerk-managed paths: LOAD_PATH shadow works — Zeitwerk will find the temp version first\n- For files NOT under Zeitwerk paths: the explicit `load` approach bypasses Zeitwerk entirely, which is correct since Zeitwerk wouldn't manage those files anyway\n- Edge case: Zeitwerk caches file-to-constant mappings. In a forked child, the cache is inherited. Since we `load` after fork, the class is redefined in-place — Zeitwerk's cache remains valid (same constant, new definition)\n- Need integration test to verify (EV-268)\n\n---\n\n### Implementation Order\n1. **EV-263** (SourceSurgeon temp-file write): Modify apply_mutation to always use temp files. Add explicit `load` for non-LOAD_PATH files. Remove direct-overwrite fallback.\n2. **EV-265** (Load-path redirection): Refine the LOAD_PATH prepend logic. Handle edge cases (multiple LOAD_PATH matches, nested paths).\n3. **EV-267** (Cleanup): Add at_exit hook and signal traps. Simplify Isolation::Fork defense-in-depth.\n4. **EV-266** (Zeitwerk): Test and handle Zeitwerk edge cases.\n5. **EV-268** (Integration tests): Verify original file never modified, cleanup on normal/exceptional/signal exit, Zeitwerk compat.\n\n### Files to modify\n- `lib/evilution/integration/rspec.rb` — primary changes (apply_mutation, restore_original)\n- `lib/evilution/isolation/fork.rb` — simplify restore_original_source\n- `lib/evilution/isolation/in_process.rb` — no changes expected\n- `lib/evilution/ast/source_surgeon.rb` — no changes\n- `lib/evilution/runner.rb` — possibly add at_exit/signal trap registration","status":"closed","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:29:42.91131604+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-08T15:53:00.18355494+07:00","closed_at":"2026-04-08T15:53:00.18355494+07:00","close_reason":"Closed","dependencies":[{"issue_id":"EV-254","depends_on_id":"EV-237","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
200
200
|
{"id":"EV-255","title":"Investigate worker process reuse vs fresh fork","description":"Determine if workers are reused across mutations (accumulating state) or fresh-forked each time. If reused, the parent process is accumulating objects passed to/from workers. If fresh-forked, the parent's memory shouldn't grow — investigate what the parent retains.","notes":"## Investigation Results\n\n**Workers are NOT reused across batches** (fresh fork each batch), but ARE reused within a batch (InProcess isolation, batch_size = jobs count).\n\n### Sequential (jobs=1)\n- Fresh fork per mutation via Isolation::Fork\n- Child exits after 1 mutation — no accumulation\n\n### Parallel (jobs>N) \n- N workers forked per batch (each_slice(config.jobs))\n- Workers loop processing multiple mutations via InProcess isolation\n- Workers killed and re-forked between batches\n- RSpec state accumulates within a worker for batch duration\n- EV-256 fix (clear_examples + release_rspec_state) mitigates in-batch accumulation\n\n### Parent-side accumulation\n- Expected O(N) growth: MutationResult + stripped Mutation per mutation for final report\n- Not a leak — required for report generation\n\n**Conclusion:** Architecture is sound. No unexpected reuse or leak path identified.","status":"closed","priority":0,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:29:44.352572561+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-05T22:33:41.490184078+07:00","closed_at":"2026-04-05T22:33:41.490184078+07:00","close_reason":"Investigation complete. Worker lifecycle is sound: fresh fork per batch, InProcess reuse within batch mitigated by EV-256 cleanup. No action needed.","dependencies":[{"issue_id":"EV-255","depends_on_id":"EV-226","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
201
201
|
{"id":"EV-256","title":"Fix identified leak source(s)","description":"Based on profiling results, fix the leak. This could be: mutation results accumulating without GC, Prism parse trees retained, test output buffers, or reporter state growing.","status":"in_progress","priority":0,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:29:46.524854416+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-05T16:21:16.522830162+07:00","dependencies":[{"issue_id":"EV-256","depends_on_id":"EV-226","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
202
202
|
{"id":"EV-257","title":"Add per-mutation RSS regression test","description":"Add a spec that runs N mutations and asserts RSS growth stays below a threshold (e.g., <1 MB/mutation average). Integrate with rake memory:check.","notes":"GitHub: #544","status":"closed","priority":0,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:29:48.696476684+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-05T23:22:49.927334624+07:00","closed_at":"2026-04-05T23:22:49.927334624+07:00","close_reason":"Merged PR #568. Added RSpec integration per-mutation check to rake memory:check.","dependencies":[{"issue_id":"EV-257","depends_on_id":"EV-226","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
203
|
-
{"id":"EV-258","title":"Identify heredoc nodes in Prism AST","description":"Determine how Prism represents heredocs (interpolated_string_node with heredoc opening?). Document the AST structure for plain heredocs, interpolated heredocs, and squiggly heredocs.","notes":"
|
|
204
|
-
{"id":"EV-259","title":"Add heredoc detection to StringLiteral operator","description":"Modify the StringLiteral operator to detect when the target node is inside a heredoc. Add a heredoc? helper method.","notes":"GitHub: #546","status":"
|
|
203
|
+
{"id":"EV-258","title":"Identify heredoc nodes in Prism AST","description":"Determine how Prism represents heredocs (interpolated_string_node with heredoc opening?). Document the AST structure for plain heredocs, interpolated heredocs, and squiggly heredocs.","notes":"## Findings\n\nPrism represents heredocs using the same node types as regular strings, with a built-in `heredoc?` method for detection:\n\n### Node types\n- **Plain heredoc** (no interpolation): `Prism::StringNode` with `heredoc? = true`\n- **Interpolated heredoc**: `Prism::InterpolatedStringNode` with `heredoc? = true`\n- **Single-quoted heredoc** (`<<~'HEREDOC'`): `Prism::StringNode` with `heredoc? = true` (no interpolation possible)\n\n### Detection\n- `node.heredoc?` — built-in Prism method, works on both `StringNode` and `InterpolatedStringNode`\n- `node.opening` returns the heredoc sigil (e.g. `<<~HEREDOC`, `<<-HEREDOC`, `<<~'HEREDOC'`)\n- Can also check `node.opening&.start_with?(\"<<\")` but `heredoc?` is preferred\n\n### Heredoc variants (all return `heredoc? = true`)\n- `<<HEREDOC` — standard\n- `<<-HEREDOC` — allows indented closing delimiter\n- `<<~HEREDOC` — squiggly (strips leading whitespace)\n- `<<~'HEREDOC'` — single-quoted (no interpolation)\n\n### InterpolatedStringNode parts\nFor interpolated heredocs, `node.parts` contains:\n- `Prism::StringNode` — literal text segments (between interpolations)\n- `Prism::EmbeddedStatementsNode` — interpolated expressions (`#{...}`)\n\n### Key implication for EV-259\nNo custom AST traversal needed — just call `node.heredoc?` in the StringLiteral operator.","status":"closed","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:29:50.91761442+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-08T16:02:56.854568029+07:00","closed_at":"2026-04-08T16:02:56.854568029+07:00","close_reason":"Closed","dependencies":[{"issue_id":"EV-258","depends_on_id":"EV-241","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
204
|
+
{"id":"EV-259","title":"Add heredoc detection to StringLiteral operator","description":"Modify the StringLiteral operator to detect when the target node is inside a heredoc. Add a heredoc? helper method.","notes":"GitHub: #546","status":"closed","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:29:52.767879347+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-08T18:09:45.856242189+07:00","closed_at":"2026-04-08T18:09:45.856242189+07:00","close_reason":"Closed","dependencies":[{"issue_id":"EV-259","depends_on_id":"EV-241","type":"blocks","created_at":"0001-01-01T00:00:00Z"},{"issue_id":"EV-259","depends_on_id":"EV-258","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
205
205
|
{"id":"EV-26","title":"Include test command in mutation result JSON","description":"Add a 'test_command' field to each mutation result in JSON output showing the exact command that was run to test that mutation. Helps agents debug when a mutation errors out or times out.","status":"closed","priority":3,"issue_type":"feature","owner":"denis.kiselyov@gmail.com","created_at":"2026-03-10T06:17:39.227881462+07:00","created_by":"Denis Kiselev","updated_at":"2026-03-16T10:11:01.311631114+07:00","closed_at":"2026-03-16T10:11:01.311631114+07:00","close_reason":"Already merged and released in v0.4.0"}
|
|
206
|
-
{"id":"EV-260","title":"Implement interpolation-only mutation mode for heredocs","description":"When mutating a heredoc, only mutate the interpolated expressions (embedded_statements_node), not the literal text surrounding them. This preserves template structure while still testing dynamic content.","notes":"GitHub: #547","status":"
|
|
207
|
-
{"id":"EV-261","title":"Add --skip-heredoc-literals CLI flag","description":"Add a flag to completely skip string literal mutations inside heredocs. For users who prefer zero heredoc mutations.","notes":"GitHub: #548","status":"
|
|
208
|
-
{"id":"EV-262","title":"Add tests for heredoc mutation behavior","description":"Test: heredoc with no interpolation (skipped or mutated to empty), heredoc with interpolation (only expressions mutated), squiggly heredoc, nested heredoc.","notes":"GitHub: #549","status":"
|
|
209
|
-
{"id":"EV-263","title":"Implement temp-file write in SourceSurgeon","description":"Modify SourceSurgeon.apply to write mutated source to a temp file instead of overwriting the original. Return the temp file path. Original file is never touched.","notes":"GitHub: #537","status":"
|
|
206
|
+
{"id":"EV-260","title":"Implement interpolation-only mutation mode for heredocs","description":"When mutating a heredoc, only mutate the interpolated expressions (embedded_statements_node), not the literal text surrounding them. This preserves template structure while still testing dynamic content.","notes":"GitHub: #547","status":"closed","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:29:54.680419198+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-08T18:25:33.319924888+07:00","closed_at":"2026-04-08T18:25:33.319924888+07:00","close_reason":"Closed","dependencies":[{"issue_id":"EV-260","depends_on_id":"EV-241","type":"blocks","created_at":"0001-01-01T00:00:00Z"},{"issue_id":"EV-260","depends_on_id":"EV-259","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
207
|
+
{"id":"EV-261","title":"Add --skip-heredoc-literals CLI flag","description":"Add a flag to completely skip string literal mutations inside heredocs. For users who prefer zero heredoc mutations.","notes":"GitHub: #548","status":"closed","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:29:56.515382643+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-08T19:20:01.937747302+07:00","closed_at":"2026-04-08T19:20:01.937747302+07:00","close_reason":"Closed","dependencies":[{"issue_id":"EV-261","depends_on_id":"EV-241","type":"blocks","created_at":"0001-01-01T00:00:00Z"},{"issue_id":"EV-261","depends_on_id":"EV-260","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
208
|
+
{"id":"EV-262","title":"Add tests for heredoc mutation behavior","description":"Test: heredoc with no interpolation (skipped or mutated to empty), heredoc with interpolation (only expressions mutated), squiggly heredoc, nested heredoc.","notes":"GitHub: #549","status":"in_progress","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:29:56.333773283+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-08T19:20:02.042056553+07:00","dependencies":[{"issue_id":"EV-262","depends_on_id":"EV-241","type":"blocks","created_at":"0001-01-01T00:00:00Z"},{"issue_id":"EV-262","depends_on_id":"EV-261","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
209
|
+
{"id":"EV-263","title":"Implement temp-file write in SourceSurgeon","description":"Modify SourceSurgeon.apply to write mutated source to a temp file instead of overwriting the original. Return the temp file path. Original file is never touched.","notes":"GitHub: #537","status":"closed","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:29:59.265360981+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-08T15:53:00.183566714+07:00","closed_at":"2026-04-08T15:53:00.183566714+07:00","close_reason":"Closed","dependencies":[{"issue_id":"EV-263","depends_on_id":"EV-237","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
210
210
|
{"id":"EV-264","title":"Define target metric and measurement methodology for mutation density gap","description":"Define what 'closing the gap' means: target ratio (e.g., <1.5x), measurement protocol (which files, which mutant config), and a benchmark script that can be re-run to track progress over time.","notes":"GitHub: #541","status":"open","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:30:08.545241632+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-04T11:30:26.634822515+07:00","dependencies":[{"issue_id":"EV-264","depends_on_id":"EV-238","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
211
|
-
{"id":"EV-265","title":"Implement load-path redirection for forked test process","description":"In the fork isolation, prepend the temp directory to $LOAD_PATH (or use a more targeted mechanism) so that require and load pick up the mutated file instead of the original.","notes":"GitHub: #550","status":"
|
|
212
|
-
{"id":"EV-266","title":"Handle Rails autoloader (Zeitwerk) compatibility","description":"Zeitwerk uses absolute paths. Test that the temp-file approach works with Zeitwerk's file-to-constant mapping. May need to use Zeitwerk's on_load callbacks or file override mechanism.","notes":"GitHub: #551","status":"
|
|
213
|
-
{"id":"EV-267","title":"Add cleanup of temp files after mutation run","description":"Ensure temp files are cleaned up on normal exit, exception, and signal (SIGTERM/SIGINT). Use at_exit hooks and signal traps.","notes":"GitHub: #552","status":"
|
|
214
|
-
{"id":"EV-268","title":"Add integration tests for temp-file mutation","description":"Test: original file never modified during run, mutated code is what the test process sees, cleanup on normal/exceptional exit, Rails autoloader compatibility.","notes":"GitHub: #554","status":"
|
|
211
|
+
{"id":"EV-265","title":"Implement load-path redirection for forked test process","description":"In the fork isolation, prepend the temp directory to $LOAD_PATH (or use a more targeted mechanism) so that require and load pick up the mutated file instead of the original.","notes":"GitHub: #550","status":"closed","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:30:17.522950262+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-08T15:53:00.18357647+07:00","closed_at":"2026-04-08T15:53:00.18357647+07:00","close_reason":"Closed","dependencies":[{"issue_id":"EV-265","depends_on_id":"EV-237","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
212
|
+
{"id":"EV-266","title":"Handle Rails autoloader (Zeitwerk) compatibility","description":"Zeitwerk uses absolute paths. Test that the temp-file approach works with Zeitwerk's file-to-constant mapping. May need to use Zeitwerk's on_load callbacks or file override mechanism.","notes":"GitHub: #551","status":"closed","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:30:40.575881302+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-08T15:53:00.183584148+07:00","closed_at":"2026-04-08T15:53:00.183584148+07:00","close_reason":"Closed","dependencies":[{"issue_id":"EV-266","depends_on_id":"EV-237","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
213
|
+
{"id":"EV-267","title":"Add cleanup of temp files after mutation run","description":"Ensure temp files are cleaned up on normal exit, exception, and signal (SIGTERM/SIGINT). Use at_exit hooks and signal traps.","notes":"GitHub: #552","status":"closed","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:30:57.733824316+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-08T15:53:00.18359655+07:00","closed_at":"2026-04-08T15:53:00.18359655+07:00","close_reason":"Closed","dependencies":[{"issue_id":"EV-267","depends_on_id":"EV-237","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
214
|
+
{"id":"EV-268","title":"Add integration tests for temp-file mutation","description":"Test: original file never modified during run, mutated code is what the test process sees, cleanup on normal/exceptional exit, Rails autoloader compatibility.","notes":"GitHub: #554","status":"closed","priority":2,"issue_type":"task","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T11:31:13.886751721+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-08T15:53:00.183603939+07:00","closed_at":"2026-04-08T15:53:00.183603939+07:00","close_reason":"Closed","dependencies":[{"issue_id":"EV-268","depends_on_id":"EV-237","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
215
215
|
{"id":"EV-269","title":"Add 'evil' as alternative executable name","description":"Add exe/evil as an alternative entry point so users with alias be='bundle exec' can run 'be evil'. Need to consider: (1) thin wrapper vs symlink, (2) whether to document the alias suggestion, (3) Windows compatibility of symlinks. Fun quality-of-life feature, not urgent.","status":"open","priority":4,"issue_type":"feature","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T12:52:10.764169051+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-04T12:52:10.764169051+07:00"}
|
|
216
216
|
{"id":"EV-27","title":"Epic: Workflow Integration","description":"Make evilution easier to invoke from AI agent toolchains — MCP server for direct tool calls, stdin mode for piping.","status":"closed","priority":2,"issue_type":"feature","owner":"denis.kiselyov@gmail.com","created_at":"2026-03-10T06:17:43.883767452+07:00","created_by":"Denis Kiselev","updated_at":"2026-03-16T22:59:12.241444308+07:00","closed_at":"2026-03-16T22:59:12.241444308+07:00","close_reason":"Both child issues done: MCP server (EV-28) and --stdin (EV-29)","dependencies":[{"issue_id":"EV-27","depends_on_id":"EV-28","type":"blocks","created_at":"0001-01-01T00:00:00Z"},{"issue_id":"EV-27","depends_on_id":"EV-29","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
|
217
217
|
{"id":"EV-270","title":"Bug: Pipe buffer deadlock in Fork isolation when Marshal payload exceeds 64KB","description":"In Isolation::Fork#wait_for_result, the parent calls read_io.read (blocking until EOF) while the child calls Marshal.dump(result, write_io). If the serialized result exceeds the ~64KB Linux pipe buffer, the child blocks on write waiting for the parent to drain, while the parent blocks on read waiting for the child to close the pipe. Classic pipe deadlock. This is the most likely cause of occasional evilution hangs. Fix: read in a loop with IO.select, or use a tempfile for large payloads, or use non-blocking IO.","notes":"GitHub: #557","status":"closed","priority":1,"issue_type":"bug","owner":"denis.kiselyov@gmail.com","created_at":"2026-04-04T23:39:33.862423078+07:00","created_by":"Denis Kiselev","updated_at":"2026-04-05T11:36:22.036913254+07:00","closed_at":"2026-04-05T11:36:22.036913254+07:00","close_reason":"False positive: Ruby IO#read drains pipe incrementally, no deadlock occurs. Real bug (double Process.wait) filed as EV-273 / GH #561.","dependencies":[{"issue_id":"EV-270","depends_on_id":"EV-226","type":"blocks","created_at":"0001-01-01T00:00:00Z"}]}
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,23 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.21.0] - 2026-04-08
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
|
|
7
|
+
- **Heredoc-aware string mutations** — `string_literal` operator now skips literal text in heredocs (all variants: `<<HEREDOC`, `<<-HEREDOC`, `<<~HEREDOC`, `<<~'HEREDOC'`); still mutates string literals inside interpolated expressions (`#{"literal"}`); uses Prism's built-in `heredoc?` detection (#522, #545, #546, #547)
|
|
8
|
+
- **`--skip-heredoc-literals` CLI flag** — completely suppresses all string literal mutations inside heredocs, including strings within interpolated expressions; configurable via CLI flag and `.evilution.yml` (#548)
|
|
9
|
+
- **Temp-file mutation approach** — mutations are applied to temporary file copies instead of overwriting original source files; uses load-path redirection (`$LOAD_PATH.unshift`) so `require` resolves the mutated copy; original files are never modified during mutation runs (#537)
|
|
10
|
+
- **Zeitwerk-compatible load-path redirection** — forked test processes redirect the load path to pick up mutated temp files, compatible with Zeitwerk-like autoloaders (#550, #551)
|
|
11
|
+
- **Temp directory cleanup and tracking** — `TempDirTracker` ensures mutation temp directories are cleaned up after each run, preventing temp file accumulation (#552)
|
|
12
|
+
- **Integration tests for temp-file mutation** — end-to-end tests verifying original file protection, temp file cleanup, and sandbox isolation during forked mutation runs (#554)
|
|
13
|
+
- **Integration tests for heredoc mutation behavior** — full-pipeline tests covering plain, squiggly, non-squiggly, dash, single-quote, interpolated, nested, and mixed heredocs (#549)
|
|
14
|
+
|
|
15
|
+
### Changed
|
|
16
|
+
|
|
17
|
+
- **Mutation isolation** — mutation runs no longer modify original source files on disk; all mutations are applied to temporary copies, improving safety for concurrent usage and editor integration
|
|
18
|
+
- **Operator options threading** — `Registry#mutations_for` accepts `operator_options` hash, passed through to operator constructors; enables per-operator configuration like `skip_heredoc_literals`
|
|
19
|
+
- **Dependencies** — bumped `mcp` gem to 0.11.0; bumped `ruby/setup-ruby` CI action to 1.300.0
|
|
20
|
+
|
|
3
21
|
## [0.20.0] - 2026-04-08
|
|
4
22
|
|
|
5
23
|
### Added
|
data/README.md
CHANGED
|
@@ -63,6 +63,7 @@ evilution [command] [options] [files...]
|
|
|
63
63
|
| `--incremental` | Boolean | false | Cache killed/timeout results; skip unchanged mutations on re-runs. |
|
|
64
64
|
| `--save-session` | Boolean | false | Persist results as timestamped JSON under `.evilution/results/`. |
|
|
65
65
|
| `--no-progress` | Boolean | _(enabled)_ | Disable the TTY progress bar. |
|
|
66
|
+
| `--skip-heredoc-literals` | Boolean | false | Skip all string literal mutations inside heredocs. |
|
|
66
67
|
| `--show-disabled` | Boolean | false | Report mutations skipped by `# evilution:disable` comments. |
|
|
67
68
|
| `--baseline-session PATH` | String | _(none)_ | Saved session file for HTML report comparison. |
|
|
68
69
|
| `-e CODE`, `--eval CODE` | String | _(none)_ | Inline Ruby code for `util mutation` command. |
|
|
@@ -88,6 +89,7 @@ Creates `.evilution.yml`:
|
|
|
88
89
|
# integration: rspec # test framework
|
|
89
90
|
# suggest_tests: false # concrete RSpec test code in suggestions
|
|
90
91
|
# save_session: false # persist results under .evilution/results/
|
|
92
|
+
# skip_heredoc_literals: false # skip all string literal mutations inside heredocs
|
|
91
93
|
# show_disabled: false # report mutations skipped by disable comments
|
|
92
94
|
# baseline_session: null # path to session file for HTML comparison
|
|
93
95
|
# ignore_patterns: [] # AST patterns to exclude (see docs/ast_pattern_syntax.md)
|
|
@@ -387,8 +389,8 @@ Tests 4 paths (InProcess isolation, Fork isolation, mutation generation + stripp
|
|
|
387
389
|
1. **Parse** — Prism parses Ruby files into ASTs with exact byte offsets
|
|
388
390
|
2. **Extract** — Methods are identified as mutation subjects
|
|
389
391
|
3. **Filter** — Disable comments, Sorbet `sig` blocks, and AST ignore patterns exclude mutations before execution
|
|
390
|
-
4. **Mutate** — 69 operators produce text replacements at precise byte offsets (source-level surgery, no AST unparsing)
|
|
391
|
-
5. **Isolate** — Default isolation is in-process; `--isolation fork` uses forked child processes. Parallel mode (`--jobs N`) always uses in-process isolation inside pool workers to avoid double forking
|
|
392
|
+
4. **Mutate** — 69 operators produce text replacements at precise byte offsets (source-level surgery, no AST unparsing); heredoc literal text is skipped by default
|
|
393
|
+
5. **Isolate** — Mutations are applied to temporary file copies (never modifying originals); load-path redirection ensures `require` resolves the mutated copy. Default isolation is in-process; `--isolation fork` uses forked child processes. Parallel mode (`--jobs N`) always uses in-process isolation inside pool workers to avoid double forking
|
|
392
394
|
6. **Test** — RSpec executes against the mutated source
|
|
393
395
|
7. **Collect** — Source strings and AST nodes are released after use to minimize memory retention
|
|
394
396
|
8. **Report** — Results aggregated into text, JSON, or HTML, including efficiency metrics and peak memory usage
|
data/lib/evilution/cli.rb
CHANGED
|
@@ -255,6 +255,7 @@ class Evilution::CLI
|
|
|
255
255
|
end
|
|
256
256
|
|
|
257
257
|
def add_extra_flag_options(opts)
|
|
258
|
+
opts.on("--skip-heredoc-literals", "Skip all string literal mutations inside heredocs") { @options[:skip_heredoc_literals] = true }
|
|
258
259
|
opts.on("--show-disabled", "Report mutations skipped by # evilution:disable") { @options[:show_disabled] = true }
|
|
259
260
|
opts.on("--baseline-session PATH", "Compare against a baseline session in HTML report") { |p| @options[:baseline_session] = p }
|
|
260
261
|
opts.on("--save-session", "Save session results to .evilution/results/") { @options[:save_session] = true }
|
|
@@ -298,10 +299,11 @@ class Evilution::CLI
|
|
|
298
299
|
|
|
299
300
|
registry = Evilution::Mutator::Registry.default
|
|
300
301
|
filter = build_subject_filter(config)
|
|
302
|
+
operator_options = build_operator_options(config)
|
|
301
303
|
total_mutations = 0
|
|
302
304
|
|
|
303
305
|
subjects.each do |subj|
|
|
304
|
-
count = registry.mutations_for(subj, filter: filter).length
|
|
306
|
+
count = registry.mutations_for(subj, filter: filter, operator_options: operator_options).length
|
|
305
307
|
total_mutations += count
|
|
306
308
|
label = count == 1 ? "1 mutation" : "#{count} mutations"
|
|
307
309
|
$stdout.puts(" #{subj.name} #{subj.file_path}:#{subj.line_number} (#{label})")
|
|
@@ -317,6 +319,10 @@ class Evilution::CLI
|
|
|
317
319
|
2
|
|
318
320
|
end
|
|
319
321
|
|
|
322
|
+
def build_operator_options(config)
|
|
323
|
+
{ skip_heredoc_literals: config.skip_heredoc_literals? }
|
|
324
|
+
end
|
|
325
|
+
|
|
320
326
|
def build_subject_filter(config)
|
|
321
327
|
return nil if config.ignore_patterns.empty?
|
|
322
328
|
|
|
@@ -425,6 +431,7 @@ class Evilution::CLI
|
|
|
425
431
|
" suggest_tests: #{config.suggest_tests}",
|
|
426
432
|
" save_session: #{config.save_session}",
|
|
427
433
|
" target: #{config.target || "(all files)"}",
|
|
434
|
+
" skip_heredoc_literals: #{config.skip_heredoc_literals}",
|
|
428
435
|
" ignore_patterns: #{config.ignore_patterns.empty? ? "(none)" : config.ignore_patterns.inspect}"
|
|
429
436
|
]
|
|
430
437
|
end
|
|
@@ -481,8 +488,10 @@ class Evilution::CLI
|
|
|
481
488
|
def run_util_mutation
|
|
482
489
|
source, file_path = resolve_util_mutation_source
|
|
483
490
|
subjects = parse_source_to_subjects(source, file_path)
|
|
491
|
+
config = Evilution::Config.new(**@options)
|
|
484
492
|
registry = Evilution::Mutator::Registry.default
|
|
485
|
-
|
|
493
|
+
operator_options = build_operator_options(config)
|
|
494
|
+
mutations = subjects.flat_map { |s| registry.mutations_for(s, operator_options: operator_options) }
|
|
486
495
|
|
|
487
496
|
if mutations.empty?
|
|
488
497
|
$stdout.puts("No mutations generated")
|
data/lib/evilution/config.rb
CHANGED
|
@@ -25,14 +25,16 @@ class Evilution::Config
|
|
|
25
25
|
spec_files: [],
|
|
26
26
|
ignore_patterns: [],
|
|
27
27
|
show_disabled: false,
|
|
28
|
-
baseline_session: nil
|
|
28
|
+
baseline_session: nil,
|
|
29
|
+
skip_heredoc_literals: false
|
|
29
30
|
}.freeze
|
|
30
31
|
|
|
31
32
|
attr_reader :target_files, :timeout, :format,
|
|
32
33
|
:target, :min_score, :integration, :verbose, :quiet,
|
|
33
34
|
:jobs, :fail_fast, :baseline, :isolation, :incremental, :suggest_tests,
|
|
34
35
|
:progress, :save_session, :line_ranges, :spec_files, :hooks,
|
|
35
|
-
:ignore_patterns, :show_disabled, :baseline_session
|
|
36
|
+
:ignore_patterns, :show_disabled, :baseline_session,
|
|
37
|
+
:skip_heredoc_literals
|
|
36
38
|
|
|
37
39
|
def initialize(**options)
|
|
38
40
|
file_options = options.delete(:skip_config_file) ? {} : load_config_file
|
|
@@ -89,6 +91,10 @@ class Evilution::Config
|
|
|
89
91
|
show_disabled
|
|
90
92
|
end
|
|
91
93
|
|
|
94
|
+
def skip_heredoc_literals?
|
|
95
|
+
skip_heredoc_literals
|
|
96
|
+
end
|
|
97
|
+
|
|
92
98
|
def self.file_options
|
|
93
99
|
CONFIG_FILES.each do |path|
|
|
94
100
|
next unless File.exist?(path)
|
|
@@ -131,6 +137,9 @@ class Evilution::Config
|
|
|
131
137
|
# Generate concrete RSpec test code in suggestions (default: false)
|
|
132
138
|
# suggest_tests: false
|
|
133
139
|
|
|
140
|
+
# Skip all string literal mutations inside heredocs (default: false)
|
|
141
|
+
# skip_heredoc_literals: false
|
|
142
|
+
|
|
134
143
|
# Hooks: Ruby files returning a Proc, keyed by lifecycle event
|
|
135
144
|
# hooks:
|
|
136
145
|
# worker_process_start: config/evilution_hooks/worker_start.rb
|
|
@@ -179,6 +188,7 @@ class Evilution::Config
|
|
|
179
188
|
@ignore_patterns = validate_ignore_patterns(merged[:ignore_patterns])
|
|
180
189
|
@show_disabled = merged[:show_disabled]
|
|
181
190
|
@baseline_session = merged[:baseline_session]
|
|
191
|
+
@skip_heredoc_literals = merged[:skip_heredoc_literals]
|
|
182
192
|
@hooks = validate_hooks(merged[:hooks])
|
|
183
193
|
end
|
|
184
194
|
|
|
@@ -7,6 +7,7 @@ require_relative "base"
|
|
|
7
7
|
require_relative "crash_detector"
|
|
8
8
|
require_relative "../spec_resolver"
|
|
9
9
|
require_relative "../related_spec_heuristic"
|
|
10
|
+
require_relative "../temp_dir_tracker"
|
|
10
11
|
|
|
11
12
|
require_relative "../integration"
|
|
12
13
|
|
|
@@ -22,9 +23,7 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
|
22
23
|
end
|
|
23
24
|
|
|
24
25
|
def call(mutation)
|
|
25
|
-
@original_content = nil
|
|
26
26
|
@temp_dir = nil
|
|
27
|
-
@lock_file = nil
|
|
28
27
|
ensure_rspec_loaded
|
|
29
28
|
@hooks.fire(:mutation_insert_pre, mutation: mutation, file_path: mutation.file_path) if @hooks
|
|
30
29
|
apply_mutation(mutation)
|
|
@@ -51,51 +50,60 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
|
51
50
|
end
|
|
52
51
|
|
|
53
52
|
def apply_mutation(mutation)
|
|
53
|
+
@temp_dir = Dir.mktmpdir("evilution")
|
|
54
|
+
Evilution::TempDirTracker.register(@temp_dir)
|
|
55
|
+
@displaced_feature = nil
|
|
54
56
|
subpath = resolve_require_subpath(mutation.file_path)
|
|
55
57
|
|
|
56
58
|
if subpath
|
|
57
|
-
@temp_dir = Dir.mktmpdir("evilution")
|
|
58
59
|
dest = File.join(@temp_dir, subpath)
|
|
59
60
|
FileUtils.mkdir_p(File.dirname(dest))
|
|
60
61
|
File.write(dest, mutation.mutated_source)
|
|
61
62
|
$LOAD_PATH.unshift(@temp_dir)
|
|
63
|
+
displace_loaded_feature(mutation.file_path)
|
|
62
64
|
else
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
@original_content = File.read(mutation.file_path)
|
|
69
|
-
File.write(mutation.file_path, mutation.mutated_source)
|
|
65
|
+
absolute = File.expand_path(mutation.file_path)
|
|
66
|
+
dest = File.join(@temp_dir, absolute)
|
|
67
|
+
FileUtils.mkdir_p(File.dirname(dest))
|
|
68
|
+
File.write(dest, mutation.mutated_source)
|
|
69
|
+
load(dest)
|
|
70
70
|
end
|
|
71
71
|
end
|
|
72
72
|
|
|
73
|
-
def restore_original(mutation)
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
@lock_file = nil
|
|
84
|
-
end
|
|
73
|
+
def restore_original(mutation) # rubocop:disable Lint/UnusedMethodArgument
|
|
74
|
+
return unless @temp_dir
|
|
75
|
+
|
|
76
|
+
$LOAD_PATH.delete(@temp_dir)
|
|
77
|
+
$LOADED_FEATURES.reject! { |f| f.start_with?(@temp_dir) }
|
|
78
|
+
$LOADED_FEATURES << @displaced_feature if @displaced_feature && !$LOADED_FEATURES.include?(@displaced_feature)
|
|
79
|
+
@displaced_feature = nil
|
|
80
|
+
FileUtils.rm_rf(@temp_dir)
|
|
81
|
+
Evilution::TempDirTracker.unregister(@temp_dir)
|
|
82
|
+
@temp_dir = nil
|
|
85
83
|
end
|
|
86
84
|
|
|
87
85
|
def resolve_require_subpath(file_path)
|
|
88
86
|
absolute = File.expand_path(file_path)
|
|
87
|
+
best_subpath = nil
|
|
89
88
|
|
|
90
89
|
$LOAD_PATH.each do |entry|
|
|
91
90
|
dir = File.expand_path(entry)
|
|
92
91
|
prefix = dir.end_with?("/") ? dir : "#{dir}/"
|
|
93
92
|
next unless absolute.start_with?(prefix)
|
|
94
93
|
|
|
95
|
-
|
|
94
|
+
candidate = absolute.delete_prefix(prefix)
|
|
95
|
+
best_subpath = candidate if best_subpath.nil? || candidate.length < best_subpath.length
|
|
96
96
|
end
|
|
97
97
|
|
|
98
|
-
|
|
98
|
+
best_subpath
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def displace_loaded_feature(file_path)
|
|
102
|
+
absolute = File.expand_path(file_path)
|
|
103
|
+
return unless $LOADED_FEATURES.include?(absolute)
|
|
104
|
+
|
|
105
|
+
@displaced_feature = absolute
|
|
106
|
+
$LOADED_FEATURES.delete(absolute)
|
|
99
107
|
end
|
|
100
108
|
|
|
101
109
|
def run_rspec(mutation)
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require "fileutils"
|
|
4
4
|
require "tmpdir"
|
|
5
5
|
require_relative "../memory"
|
|
6
|
+
require_relative "../temp_dir_tracker"
|
|
6
7
|
|
|
7
8
|
require_relative "../isolation"
|
|
8
9
|
|
|
@@ -44,12 +45,8 @@ class Evilution::Isolation::Fork
|
|
|
44
45
|
|
|
45
46
|
private
|
|
46
47
|
|
|
47
|
-
def restore_original_source(mutation)
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
File.write(mutation.file_path, mutation.original_source)
|
|
51
|
-
rescue StandardError => e
|
|
52
|
-
warn("Warning: failed to restore #{mutation.file_path}: #{e.message}")
|
|
48
|
+
def restore_original_source(mutation) # rubocop:disable Lint/UnusedMethodArgument
|
|
49
|
+
Evilution::TempDirTracker.cleanup_all
|
|
53
50
|
end
|
|
54
51
|
|
|
55
52
|
def suppress_child_output
|
|
@@ -3,7 +3,25 @@
|
|
|
3
3
|
require_relative "../operator"
|
|
4
4
|
|
|
5
5
|
class Evilution::Mutator::Operator::StringLiteral < Evilution::Mutator::Base
|
|
6
|
+
def initialize(skip_heredoc_literals: false, **rest)
|
|
7
|
+
super(**rest)
|
|
8
|
+
@skip_heredoc_literals = skip_heredoc_literals
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def visit_interpolated_string_node(node)
|
|
12
|
+
return super unless node.heredoc?
|
|
13
|
+
return if @skip_heredoc_literals
|
|
14
|
+
|
|
15
|
+
node.parts.each do |part|
|
|
16
|
+
next if part.is_a?(Prism::StringNode)
|
|
17
|
+
|
|
18
|
+
visit(part)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
6
22
|
def visit_string_node(node)
|
|
23
|
+
return super if node.heredoc?
|
|
24
|
+
|
|
7
25
|
replacement = node.content.empty? ? '"mutation"' : '""'
|
|
8
26
|
|
|
9
27
|
add_mutation(
|
|
@@ -88,9 +88,10 @@ class Evilution::Mutator::Registry
|
|
|
88
88
|
self
|
|
89
89
|
end
|
|
90
90
|
|
|
91
|
-
def mutations_for(subject, filter: nil)
|
|
91
|
+
def mutations_for(subject, filter: nil, operator_options: {})
|
|
92
92
|
@operators.flat_map do |operator_class|
|
|
93
|
-
operator_class
|
|
93
|
+
operator = build_operator(operator_class, operator_options)
|
|
94
|
+
operator.call(subject, filter: filter)
|
|
94
95
|
end
|
|
95
96
|
end
|
|
96
97
|
|
|
@@ -101,4 +102,10 @@ class Evilution::Mutator::Registry
|
|
|
101
102
|
def operators
|
|
102
103
|
@operators.dup
|
|
103
104
|
end
|
|
105
|
+
|
|
106
|
+
private
|
|
107
|
+
|
|
108
|
+
def build_operator(operator_class, options)
|
|
109
|
+
operator_class.new(**options)
|
|
110
|
+
end
|
|
104
111
|
end
|
data/lib/evilution/runner.rb
CHANGED
|
@@ -21,6 +21,7 @@ require_relative "cache"
|
|
|
21
21
|
require_relative "parallel/pool"
|
|
22
22
|
require_relative "session/store"
|
|
23
23
|
require_relative "ast/pattern/filter"
|
|
24
|
+
require_relative "temp_dir_tracker"
|
|
24
25
|
require_relative "disable_comment"
|
|
25
26
|
require_relative "ast/sorbet_sig_detector"
|
|
26
27
|
|
|
@@ -42,6 +43,7 @@ class Evilution::Runner
|
|
|
42
43
|
end
|
|
43
44
|
|
|
44
45
|
def call
|
|
46
|
+
install_signal_handlers
|
|
45
47
|
start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
46
48
|
|
|
47
49
|
subjects = parse_and_filter_subjects
|
|
@@ -180,8 +182,9 @@ class Evilution::Runner
|
|
|
180
182
|
|
|
181
183
|
def generate_mutations(subjects)
|
|
182
184
|
filter = build_ignore_filter
|
|
185
|
+
operator_options = build_operator_options
|
|
183
186
|
mutations = subjects.flat_map do |subject|
|
|
184
|
-
registry.mutations_for(subject, filter: filter)
|
|
187
|
+
registry.mutations_for(subject, filter: filter, operator_options: operator_options)
|
|
185
188
|
end
|
|
186
189
|
skipped_count = filter ? filter.skipped_count : 0
|
|
187
190
|
|
|
@@ -252,6 +255,10 @@ class Evilution::Runner
|
|
|
252
255
|
end
|
|
253
256
|
end
|
|
254
257
|
|
|
258
|
+
def build_operator_options
|
|
259
|
+
{ skip_heredoc_literals: config.skip_heredoc_literals? }
|
|
260
|
+
end
|
|
261
|
+
|
|
255
262
|
def build_ignore_filter
|
|
256
263
|
patterns = config.ignore_patterns
|
|
257
264
|
return nil if patterns.nil? || patterns.empty?
|
|
@@ -432,6 +439,26 @@ class Evilution::Runner
|
|
|
432
439
|
config.fail_fast? && survived_count >= config.fail_fast
|
|
433
440
|
end
|
|
434
441
|
|
|
442
|
+
def install_signal_handlers
|
|
443
|
+
%w[INT TERM].each { |sig| install_signal_handler(sig) }
|
|
444
|
+
end
|
|
445
|
+
|
|
446
|
+
def install_signal_handler(sig)
|
|
447
|
+
prev_handler = Signal.trap(sig) do
|
|
448
|
+
Evilution::TempDirTracker.cleanup_all
|
|
449
|
+
|
|
450
|
+
case prev_handler
|
|
451
|
+
when Proc, Method
|
|
452
|
+
prev_handler.call
|
|
453
|
+
when "IGNORE"
|
|
454
|
+
# Do nothing — signal is ignored
|
|
455
|
+
else
|
|
456
|
+
Signal.trap(sig, "DEFAULT")
|
|
457
|
+
Process.kill(sig, Process.pid)
|
|
458
|
+
end
|
|
459
|
+
end
|
|
460
|
+
end
|
|
461
|
+
|
|
435
462
|
def build_isolator
|
|
436
463
|
case resolve_isolation
|
|
437
464
|
when :fork then Evilution::Isolation::Fork.new(hooks: @hooks)
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "monitor"
|
|
5
|
+
require_relative "version"
|
|
6
|
+
|
|
7
|
+
module Evilution::TempDirTracker
|
|
8
|
+
@dirs = Set.new
|
|
9
|
+
@monitor = Monitor.new
|
|
10
|
+
@at_exit_registered = false
|
|
11
|
+
|
|
12
|
+
def self.register(dir)
|
|
13
|
+
@monitor.synchronize do
|
|
14
|
+
@dirs << dir
|
|
15
|
+
register_at_exit unless @at_exit_registered
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def self.unregister(dir)
|
|
20
|
+
@monitor.synchronize { @dirs.delete(dir) }
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def self.cleanup_all
|
|
24
|
+
@monitor.synchronize do
|
|
25
|
+
@dirs.each { |d| FileUtils.rm_rf(d) }
|
|
26
|
+
@dirs.clear
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def self.tracked_dirs
|
|
31
|
+
@monitor.synchronize { @dirs.dup }
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def self.register_at_exit
|
|
35
|
+
at_exit { cleanup_all }
|
|
36
|
+
@at_exit_registered = true
|
|
37
|
+
end
|
|
38
|
+
private_class_method :register_at_exit
|
|
39
|
+
end
|
data/lib/evilution/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: evilution
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.21.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Denis Kiselev
|
|
@@ -218,6 +218,7 @@ files:
|
|
|
218
218
|
- lib/evilution/session/store.rb
|
|
219
219
|
- lib/evilution/spec_resolver.rb
|
|
220
220
|
- lib/evilution/subject.rb
|
|
221
|
+
- lib/evilution/temp_dir_tracker.rb
|
|
221
222
|
- lib/evilution/version.rb
|
|
222
223
|
- lib/tasks/memory_check.rake
|
|
223
224
|
- script/memory_check
|