evilution 0.34.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +11 -0
- data/.rubocop_todo.yml +1 -0
- data/CHANGELOG.md +25 -0
- data/README.md +41 -3
- data/assets/logo/evilution-mark-1024.png +0 -0
- data/assets/logo/evilution-mark-64.png +0 -0
- data/assets/logo/evilution-mark-transparent-1024.png +0 -0
- data/assets/logo/evilution-mark-transparent.svg +53 -0
- data/assets/logo/evilution-mark.svg +70 -0
- data/docs/architecture.md +220 -0
- data/docs/isolation.md +1 -0
- data/docs/migration-from-mutant.md +226 -0
- data/docs/public_api.md +50 -0
- data/docs/versioning.md +3 -3
- data/lib/evilution/cli.rb +1 -0
- data/lib/evilution/config/builders/spec_resolver.rb +3 -1
- data/lib/evilution/config.rb +1 -1
- data/lib/evilution/gem_detector.rb +7 -0
- data/lib/evilution/integration/loading/redefinition_recovery.rb +3 -1
- data/lib/evilution/integration/minitest.rb +14 -3
- data/lib/evilution/integration/rspec/unresolved_spec_warner.rb +17 -3
- data/lib/evilution/integration/test_unit/test_file_resolver.rb +14 -3
- data/lib/evilution/related_spec_heuristic.rb +0 -2
- data/lib/evilution/runner/canary.rb +31 -4
- data/lib/evilution/runner/isolation_resolver.rb +15 -1
- data/lib/evilution/runner.rb +1 -0
- data/lib/evilution/spec_resolver.rb +27 -1
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution.rb +6 -0
- metadata +10 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 26a7a721749a19cb1f10d77b46a9d78d394ce5c112ba5d5fab8963de4a1a5f14
|
|
4
|
+
data.tar.gz: 65ee839279c74e07dd84d153015bfff6e29daf56d87929da75991fdef67e49c5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 587598737cb636180b40cfb2466ecd0aac34c827eddb5432b543fc33b2b7e94b2e5b2a03915598fb9c41e9174bab8baad06ffab620f4174d8f686f9173f8fd1b
|
|
7
|
+
data.tar.gz: 85be8b922c45fab5e0e50fd8f17a6d5a572a9b9a2139181f0ca0283dc52c241bcd54b512e0cb3a5be9588edc31cd8b46fedd10437e8caf0c3e3104229c1f7127
|
data/.beads/interactions.jsonl
CHANGED
|
@@ -437,3 +437,14 @@
|
|
|
437
437
|
{"id":"int-555de29c","kind":"field_change","created_at":"2026-06-16T07:54:17.438977358Z","actor":"Denis Kiselev","issue_id":"EV-z03y","extra":{"field":"status","new_value":"in_progress","old_value":"open"}}
|
|
438
438
|
{"id":"int-616ccd27","kind":"field_change","created_at":"2026-06-16T09:31:30.217250542Z","actor":"Denis Kiselev","issue_id":"EV-z03y","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Merged via PR #1375 (master). Dep on EV-rxob is provenance only; code fix shipped."}}
|
|
439
439
|
{"id":"int-6d2f9b35","kind":"field_change","created_at":"2026-06-16T10:00:18.493013642Z","actor":"Denis Kiselev","issue_id":"EV-rxob","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"1.0 real-world validation complete. Rounds 2-4 ran 30+ gems across rspec/minitest/test-unit; no crash/memory blowup. All findings filed+fixed+merged: EV-z7f5/#1347, EV-gl1e/#1329, EV-52hf/#1341, EV-5hk5/#1373, EV-bi41/#1374, EV-z03y/#1375. Round-4 OOB re-verify confirmed fixes compound (state_machines 0->0.922, factory_bot/webmock/http real OOB scores, errors=0). Residual resolution-coverage gaps (behavior-named/flat-prefixed/non-std helper) tracked low-pri: EV-ajby/#1376, EV-6rbd/#1377, EV-no5u/#1378."}}
|
|
440
|
+
{"id":"int-46080989","kind":"field_change","created_at":"2026-07-15T12:59:28.484284407Z","actor":"Denis Kiselev","issue_id":"EV-jj18","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
441
|
+
{"id":"int-f2064cbf","kind":"field_change","created_at":"2026-07-15T13:25:07.606150972Z","actor":"Denis Kiselev","issue_id":"EV-fkw0","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}
|
|
442
|
+
{"id":"int-eb2c5880","kind":"field_change","created_at":"2026-07-15T13:33:14.202699471Z","actor":"Denis Kiselev","issue_id":"EV-axze","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
443
|
+
{"id":"int-fece2dce","kind":"field_change","created_at":"2026-07-15T13:33:14.634226625Z","actor":"Denis Kiselev","issue_id":"EV-7ab0","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
444
|
+
{"id":"int-46a5d7e3","kind":"field_change","created_at":"2026-07-15T13:33:52.254966218Z","actor":"Denis Kiselev","issue_id":"EV-2bx6","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
445
|
+
{"id":"int-2b8ea434","kind":"field_change","created_at":"2026-07-15T13:33:52.673568305Z","actor":"Denis Kiselev","issue_id":"EV-dgjv","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
446
|
+
{"id":"int-9f4fbaea","kind":"field_change","created_at":"2026-07-15T13:33:53.013705815Z","actor":"Denis Kiselev","issue_id":"EV-t1qg","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
447
|
+
{"id":"int-608d35ec","kind":"field_change","created_at":"2026-07-15T13:33:53.352696649Z","actor":"Denis Kiselev","issue_id":"EV-vlbh","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
448
|
+
{"id":"int-616f39e1","kind":"field_change","created_at":"2026-07-15T13:33:53.68007233Z","actor":"Denis Kiselev","issue_id":"EV-y194","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
449
|
+
{"id":"int-abcaeb3a","kind":"field_change","created_at":"2026-07-15T13:33:54.363899883Z","actor":"Denis Kiselev","issue_id":"EV-j2kz","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Closed"}}
|
|
450
|
+
{"id":"int-b8027311","kind":"field_change","created_at":"2026-07-15T13:34:47.737536249Z","actor":"Denis Kiselev","issue_id":"EV-w07i","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Closed"}}
|
data/.rubocop_todo.yml
CHANGED
data/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,31 @@
|
|
|
2
2
|
|
|
3
3
|
Versioning policy: see [docs/versioning.md](docs/versioning.md).
|
|
4
4
|
|
|
5
|
+
## [1.0.0] - 2026-07-15
|
|
6
|
+
|
|
7
|
+
First stable release. From `1.0.0` onward evilution follows [Semantic Versioning](https://semver.org): the public contract — CLI commands and flags, `.evilution.yml` configuration keys, session JSON files, the MCP tool schemas, and process exit codes — is frozen and covered by the SemVer guarantees and deprecation cycle in [docs/versioning.md](docs/versioning.md). The `1.0.0` milestone is the culmination of the readiness work that shipped across the `0.31`–`0.35` line (config and session-JSON schema versioning, MCP tool-contract stabilization, the CLI flag deprecation sweep, real-world Rails validation, the parallel/isolation stress suite, and running evilution against its own suite to a mutation-score target).
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
|
|
11
|
+
- **Contributor architecture guide (`docs/architecture.md`)** — a single internals map for contributors: a high-level flow diagram, a module map of every `Evilution::` namespace, the end-to-end data flow from source file to scored result, and step-by-step recipes for adding a new mutator, reporter, or test integration. Linked from the README "Development" and "Internals" sections (EV-fkw0, PR #1407, GH #863)
|
|
12
|
+
|
|
13
|
+
### Changed
|
|
14
|
+
|
|
15
|
+
- **Public Ruby API frozen as internal (`docs/public_api.md`)** — evilution has no public Ruby API. The entire `Evilution::` namespace is internal and may change in any release; the supported, SemVer-governed surface is the CLI, config file, session JSON, MCP tools, and exit codes. The top-level `Evilution` module and the primary entry points (`Runner`, `CLI`, `Config`) carry an `@api private` YARD marker, the new [docs/public_api.md](docs/public_api.md) documents the contract surfaces, and [docs/versioning.md](docs/versioning.md) is updated to match (the MINOR trigger now covers "introducing a public Ruby facade where none exists" and the deprecation cycle no longer references a Ruby `@deprecated` path) (EV-jj18, PR #1406, GH #854)
|
|
16
|
+
- **Dependency bumps** — `mcp` 0.21.0 → 0.24.0 (PRs #1402/#1403), plus CI/tooling updates (`rubocop` 1.88.2, `ruby/setup-ruby` 1.318.0, `actions/checkout` 7)
|
|
17
|
+
|
|
18
|
+
## [0.35.0] - 2026-06-24
|
|
19
|
+
|
|
20
|
+
### Added
|
|
21
|
+
|
|
22
|
+
- **Test::Unit now scores out-of-the-box: a dedicated canary spec plus a Test::Unit spec resolver** — `--integration test-unit` shared minitest's runtime but not its file conventions, so two things silently broke. (1) The proof-of-life canary always wrote either an RSpec `_spec.rb` or a minitest `_test.rb` and only ever generated RSpec/minitest source; under `test-unit` the synthetic mutation loaded nothing, scored `:error`, and aborted the run before any real mutation ran. The canary now emits a `Test::Unit::TestCase` subclass into a `_test.rb` file for `test-unit` (and keeps the `_test.rb`/`_spec.rb` split keyed on the integration). (2) The config spec-resolver builder fell through to the default `spec/`/`_spec.rb` resolver for `test-unit`, which resolved nothing for a `test/`-rooted suite; it now reuses minitest's `test/` + `_test.rb` resolver since test-unit gems mirror that layout (PR #1386)
|
|
23
|
+
- **Auto-preload finds namespaced gem test helpers (`test/<gem>/helper.rb`)** — gems that namespace their suite keep the bootstrap at `test/<gem>/helper.rb` (e.g. ruby/csv: `test/csv/helper.rb`) rather than a flat `test/test_helper.rb`. The conventional preload chain missed it, so autodetect fell back to the bare gem entry point (`lib/<gem>.rb`) — which never sets up the test framework — and the canary aborted. The gem-helper search now also looks for the namespaced form, deriving the directory from the gem name (dotted form for dash-named gems plus the flat form): `test/<gem>/helper.rb`. `Evilution::GemDetector.gem_name` is exposed publicly so callers can build these conventional paths (PR #1386)
|
|
24
|
+
|
|
25
|
+
### Fixed
|
|
26
|
+
|
|
27
|
+
- **`SpecResolver` resolves flat `test_`-prefixed Test::Unit/minitest files (`test/test_connection_pool_timed_stack.rb`)** — some minitest/Test::Unit gems flatten a nested source's path into a single `test_`-prefixed file at the test root (`lib/connection_pool/timed_stack.rb` → `test/test_connection_pool_timed_stack.rb`, with no `test/connection_pool/` subdir), so the resolver found nothing and scored `:unresolved`. The resolver now derives a flat candidate from the **full** lib-relative path (every segment joined with `_`) — deliberately not from namespace-dropped variants, which would yield bare-basename forms (`test_timed_stack.rb`) that collide across namespaces and are already covered by the existing `test_<name>.rb` convention. Top-level sources produce nothing here, and the flat candidate ranks below the mirrored layouts so a 1:1 file always wins. Minitest suffix only (PR #1385)
|
|
28
|
+
- **Unresolved-mutation warnings now name concrete recovery paths for behaviour-named spec layouts** — `SpecResolver` intentionally does not guess for **behaviour-named** layouts (specs named by behaviour, not by lib path, so nothing ties a source file to a spec — e.g. aasm's `lib/aasm/base.rb`, exercised through the public API by `spec/unit/{event,callbacks,guard,api}_spec.rb`). A fuzzy match there would report a misleading score, so every mutation stays `:unresolved` (a coverage gap, `errors=0`). The minitest, RSpec, and Test::Unit unresolved warnings now spell out the explicit opt-ins — name the covering specs via `--spec`/`--spec-dir`, or run the whole suite per mutation with `--fallback-full-suite` — so a behaviour-named layout reads as a fixable resolution choice rather than a silent 0%. Documented in the README "Unresolved mutations — behaviour-named spec layouts" section (PR #1384)
|
|
29
|
+
|
|
5
30
|
## [0.34.0] - 2026-06-16
|
|
6
31
|
|
|
7
32
|
### Added
|
data/README.md
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/marinazzio/evilution/refs/heads/master/assets/logo/evilution-mark-transparent.svg" width="160" height="160" alt="Evilution mascot">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
1
5
|
[](https://badge.fury.io/rb/evilution)
|
|
2
6
|
|
|
3
7
|
# Evilution — Mutation Testing for Ruby
|
|
@@ -7,7 +11,7 @@
|
|
|
7
11
|
* **License**: MIT (free, no commercial restrictions)
|
|
8
12
|
* **Language**: Ruby >= 3.3
|
|
9
13
|
* **Parser**: Prism (Ruby's official AST parser, ships with Ruby 3.3+)
|
|
10
|
-
* **Test frameworks**: RSpec and
|
|
14
|
+
* **Test frameworks**: RSpec, Minitest, and Test::Unit
|
|
11
15
|
|
|
12
16
|
## Installation
|
|
13
17
|
|
|
@@ -59,6 +63,13 @@ The first command writes a sibling `Gemfile.local.lock`. Decide whether to commi
|
|
|
59
63
|
|
|
60
64
|
The evilution gemspec already declares `prism >= 1.5, < 2`, so adding the `gem "prism"` line above is only necessary on stacks that also pin prism in `Gemfile.lock`.
|
|
61
65
|
|
|
66
|
+
## Migrating from `mutant`
|
|
67
|
+
|
|
68
|
+
Coming from the commercially-licensed [`mutant`](https://github.com/mbj/mutant) gem? See
|
|
69
|
+
[docs/migration-from-mutant.md](docs/migration-from-mutant.md) for a command/config
|
|
70
|
+
mapping, output-terminology differences, and how to diff the two tools' results with
|
|
71
|
+
`evilution compare` (it ingests mutant session JSON directly).
|
|
72
|
+
|
|
62
73
|
## Command Reference
|
|
63
74
|
|
|
64
75
|
```
|
|
@@ -94,7 +105,7 @@ Every command, subcommand, and flag listed in this section is part of evilution'
|
|
|
94
105
|
| `-f`, `--format FORMAT` | String | `text` | Output format: `text`, `json`, or `html`. |
|
|
95
106
|
| `--target EXPR` | String | _(none)_ | Only mutate matching methods. Supports method name (`Foo::Bar#calculate`), class (`Foo`), namespace wildcards (`Foo::Bar*`), method-type selectors (`Foo#`, `Foo.`), descendants (`descendants:Foo`), and source globs (`source:lib/**/*.rb`). |
|
|
96
107
|
| `--min-score FLOAT` | Float | 0.0 | Minimum mutation score (0.0–1.0) to pass. |
|
|
97
|
-
| `--spec FILES` | Array | _(none)_ | Spec files to run (comma-separated). Defaults to auto-detection via `SpecResolver`, which also resolves non-mirrored (`spec/unit`, `test/unit`)
|
|
108
|
+
| `--spec FILES` | Array | _(none)_ | Spec files to run (comma-separated). Defaults to auto-detection via `SpecResolver`, which also resolves non-mirrored (`spec/unit`, `test/unit`), dir-grouped (`test/unit/<class>/*_test.rb`), and flat `test_`-prefixed (`test/test_connection_pool_timed_stack.rb`) layouts. |
|
|
98
109
|
| `--spec-dir DIR` | String | _(none)_ | Include all `*_spec.rb` files in DIR recursively. Composable with `--spec`. |
|
|
99
110
|
| `--spec-pattern GLOB` | String | _(none)_ | Restrict resolved spec candidates to files matching GLOB (e.g. `spec/models/**/*_spec.rb`). |
|
|
100
111
|
| `--no-example-targeting` | Boolean | _(enabled)_ | Disable per-mutation example targeting (always run every example in the resolved spec file). Example targeting scans each example body for symbols from the mutated method and runs only the matching subset. |
|
|
@@ -377,7 +388,7 @@ Compatibility policy for the `1.x` gem line:
|
|
|
377
388
|
| `unresolved` | No spec file resolved for the mutated source — **coverage gap, not a failure**. Use `--fallback-full-suite` to run the full suite instead. | excluded |
|
|
378
389
|
| `unparseable` | Mutated source failed to parse (e.g. dangling heredoc opener after `method_body_replacement`). Short-circuited — never executed. | excluded |
|
|
379
390
|
|
|
380
|
-
Unresolved mutations indicate a missing test mapping — the file has no corresponding test file that the resolver could find (for example, an RSpec `_spec.rb` file or a Minitest `_test.rb` file, depending on configuration). The resolver searches
|
|
391
|
+
Unresolved mutations indicate a missing test mapping — the file has no corresponding test file that the resolver could find (for example, an RSpec `_spec.rb` file or a Minitest `_test.rb` file, depending on configuration). The resolver searches the `lib/`-mirrored path, common non-mirrored buckets (`spec/unit`, `spec/lib`, `test/unit`, `test/lib`), and the flat `test_`-prefixed Minitest/Test::Unit convention (`test/test_connection_pool_timed_stack.rb`), so a high unresolved rate usually means a genuinely missing or unconventionally-placed test; a run that leaves many mutations unresolved prints an unresolved-rate warning with a best-guess spec path per source file. They are reported separately so you can act on them (add a test, adjust test naming, pass `--spec`, or opt in to the full-suite fallback) without inflating the error count.
|
|
381
392
|
|
|
382
393
|
## Mutation Operators (74 total)
|
|
383
394
|
|
|
@@ -687,6 +698,22 @@ For each entry in `survived[]`:
|
|
|
687
698
|
|
|
688
699
|
Entries in the JSON `errors[]` array represent mutations that raised an exception (syntax error, load failure, or runtime crash) rather than producing a test outcome. Each entry includes `error_class`, `error_message`, and the first 5 `error_backtrace` lines. Use these fields to decide whether the error is a bug in the mutation operator (file an issue), a load-time problem in the mutated source (often `NoMethodError: super called outside of method` or constant-redefinition issues), or a genuine crash that the original tests should have caught. Run with `--verbose` to stream the same error details to stderr during the run.
|
|
689
700
|
|
|
701
|
+
### Unresolved mutations — behaviour-named spec layouts
|
|
702
|
+
|
|
703
|
+
`SpecResolver` auto-detects mirrored (`spec/foo_spec.rb`), non-mirrored (`spec/unit`, `spec/lib`, `test/unit`), and dir-grouped (`test/unit/<class>/*_test.rb`) layouts. It deliberately does **not** guess for **behaviour-named** layouts — where specs are named by behaviour rather than by lib path, so no path or class name ties a source file to a spec. The canonical example is [aasm](https://github.com/aasm/aasm): `lib/aasm/base.rb` is the central DSL class, but it has no `base_spec.rb`; its behaviour is exercised through the public API by `spec/unit/{event,callbacks,guard,api}_spec.rb`, none of which name `AASM::Base`. Every mutation in such a file resolves to no spec and is reported `:unresolved` (a coverage gap, not a failure — `errors=0`).
|
|
704
|
+
|
|
705
|
+
This is intentional: a fuzzy content/grep match would pick a wrong-but-plausible spec subset and report a misleading score. `:unresolved` plus the warning is the honest signal. To get a real score, opt into one of the two explicit recovery paths the warning names:
|
|
706
|
+
|
|
707
|
+
```bash
|
|
708
|
+
# run a directory of specs you know covers the file (--spec-dir globs it)
|
|
709
|
+
bundle exec evilution mutate lib/aasm/base.rb --spec-dir spec/unit
|
|
710
|
+
# or name explicit files as a single comma-separated --spec value (no shell glob)
|
|
711
|
+
bundle exec evilution mutate lib/aasm/base.rb --spec spec/unit/event_spec.rb,spec/unit/callbacks_spec.rb
|
|
712
|
+
|
|
713
|
+
# or run the whole suite per mutation (slower, but correct-by-superset)
|
|
714
|
+
bundle exec evilution mutate lib/aasm/base.rb --fallback-full-suite
|
|
715
|
+
```
|
|
716
|
+
|
|
690
717
|
### Long Minitest fork runs — not a hang
|
|
691
718
|
|
|
692
719
|
Minitest projects under `--isolation=fork` re-bootstrap the test environment (`test_helper.rb`, plugins, runnable state) once per mutation. On constant-heavy files (e.g. Shopify/liquid's `lib/liquid/lexer.rb`, ~270 mutations) the wall-clock cost is dominated by that per-fork bootstrap and any mutations that hit a `--timeout` rather than killing the test fast. A single-worker run (`-j 1`) on a few hundred mutations can take 4+ minutes; combined with `--no-progress` and a non-TTY stderr (CI, redirected logs) the run looks silent the entire time.
|
|
@@ -756,6 +783,12 @@ When Evilution detects a parallel run against a SQLite-backed `config/database.y
|
|
|
756
783
|
|
|
757
784
|
## Development
|
|
758
785
|
|
|
786
|
+
### Architecture
|
|
787
|
+
|
|
788
|
+
New to the codebase? [docs/architecture.md](docs/architecture.md) is the
|
|
789
|
+
contributor guide: the end-to-end mutation flow (source → result), a module map,
|
|
790
|
+
and step-by-step recipes for adding a mutator, a reporter, or a test integration.
|
|
791
|
+
|
|
759
792
|
### Memory leak check
|
|
760
793
|
|
|
761
794
|
Run before releasing to verify no memory regressions:
|
|
@@ -771,6 +804,9 @@ Tests 4 paths (InProcess isolation, Fork isolation, mutation generation + stripp
|
|
|
771
804
|
|
|
772
805
|
## Internals (for context, not for direct use)
|
|
773
806
|
|
|
807
|
+
For the full contributor architecture — module map, data flow, and extension
|
|
808
|
+
points — see [docs/architecture.md](docs/architecture.md).
|
|
809
|
+
|
|
774
810
|
1. **Parse** — Prism parses Ruby files into ASTs with exact byte offsets
|
|
775
811
|
2. **Extract** — Methods are identified as mutation subjects
|
|
776
812
|
3. **Filter** — Disable comments, Sorbet `sig` blocks, and AST ignore patterns exclude mutations before execution
|
|
@@ -784,6 +820,8 @@ Tests 4 paths (InProcess isolation, Fork isolation, mutation generation + stripp
|
|
|
784
820
|
|
|
785
821
|
`evilution` follows [Semantic Versioning](https://semver.org). The full policy — what counts as the public contract, what triggers a major bump, how deprecations work — is documented in [docs/versioning.md](docs/versioning.md).
|
|
786
822
|
|
|
823
|
+
There is **no public Ruby API**: the entire `Evilution::` namespace is internal and may change in any release. Drive evilution through the CLI, `.evilution.yml`, the MCP tools, or hooks. See [docs/public_api.md](docs/public_api.md).
|
|
824
|
+
|
|
787
825
|
## Repository
|
|
788
826
|
|
|
789
827
|
https://github.com/marinazzio/evilution
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 256 256" width="256" height="256" role="img" aria-label="Evilution — evil ruby mutant mascot">
|
|
2
|
+
<title>Evilution</title>
|
|
3
|
+
<defs>
|
|
4
|
+
<filter id="glow" x="-60%" y="-60%" width="220%" height="220%">
|
|
5
|
+
<feGaussianBlur stdDeviation="2.4" result="b"/>
|
|
6
|
+
<feMerge><feMergeNode in="b"/><feMergeNode in="SourceGraphic"/></feMerge>
|
|
7
|
+
</filter>
|
|
8
|
+
</defs>
|
|
9
|
+
|
|
10
|
+
<!-- clawed arms (behind gem) -->
|
|
11
|
+
<g stroke="#470B13" stroke-width="2.5" stroke-linejoin="round">
|
|
12
|
+
<path d="M60 124 L40 132 L46 138 L36 142 L48 147 L40 153 L56 150 L64 136 Z" fill="#9A1826"/>
|
|
13
|
+
<path d="M196 124 L216 132 L210 138 L220 142 L208 147 L216 153 L200 150 L192 136 Z" fill="#7F1420"/>
|
|
14
|
+
</g>
|
|
15
|
+
|
|
16
|
+
<!-- faceted ruby body -->
|
|
17
|
+
<g stroke="#470B13" stroke-width="2.5" stroke-linejoin="round">
|
|
18
|
+
<polygon points="98,82 128,82 128,120 88,120" fill="#D5384A"/>
|
|
19
|
+
<polygon points="128,82 158,82 168,120 128,120" fill="#B91C2C"/>
|
|
20
|
+
<polygon points="54,120 88,120 98,82" fill="#E2566B"/>
|
|
21
|
+
<polygon points="168,120 202,120 158,82" fill="#911A28"/>
|
|
22
|
+
<polygon points="54,120 88,120 128,214" fill="#A81B2A"/>
|
|
23
|
+
<polygon points="88,120 128,120 128,214" fill="#911A28"/>
|
|
24
|
+
<polygon points="128,120 168,120 128,214" fill="#7C1320"/>
|
|
25
|
+
<polygon points="168,120 202,120 128,214" fill="#5E0F18"/>
|
|
26
|
+
</g>
|
|
27
|
+
<polygon points="98,82 158,82 202,120 128,214 54,120" fill="none" stroke="#3A0910" stroke-width="3" stroke-linejoin="round"/>
|
|
28
|
+
|
|
29
|
+
<!-- glowing mutant eyes -->
|
|
30
|
+
<g filter="url(#glow)">
|
|
31
|
+
<polygon points="100,99 123,106 118,117 96,110" fill="#22D3EE"/>
|
|
32
|
+
<polygon points="156,99 133,106 138,117 160,110" fill="#22D3EE"/>
|
|
33
|
+
<polygon points="110,105 118,107 116,113 108,111" fill="#0E1A2B"/>
|
|
34
|
+
<polygon points="146,105 138,107 140,113 148,111" fill="#0E1A2B"/>
|
|
35
|
+
</g>
|
|
36
|
+
|
|
37
|
+
<!-- fanged grin -->
|
|
38
|
+
<path d="M106 138 L150 138 L142 149 L134 144 L128 150 L122 144 L114 149 Z" fill="#1B060C"/>
|
|
39
|
+
<polygon points="118,138 125,138 122,148" fill="#EAF2F8"/>
|
|
40
|
+
<polygon points="131,138 138,138 134,148" fill="#EAF2F8"/>
|
|
41
|
+
|
|
42
|
+
<!-- mutation cracks leaking binary -->
|
|
43
|
+
<g filter="url(#glow)" fill="none" stroke="#67E8F9" stroke-width="1.7" stroke-linecap="round" opacity="0.92">
|
|
44
|
+
<polyline points="166,90 156,106 169,120 159,134"/>
|
|
45
|
+
<polyline points="82,150 92,160 86,172"/>
|
|
46
|
+
</g>
|
|
47
|
+
<g fill="#7FE9F7" font-family="monospace" font-size="11" font-weight="700" opacity="0.85">
|
|
48
|
+
<text x="170" y="128" transform="rotate(12 170 128)">1</text>
|
|
49
|
+
<text x="176" y="140" transform="rotate(12 176 140)">0</text>
|
|
50
|
+
<text x="168" y="150" transform="rotate(12 168 150)">1</text>
|
|
51
|
+
<text x="78" y="184" transform="rotate(-14 78 184)">0</text>
|
|
52
|
+
</g>
|
|
53
|
+
</svg>
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 256 256" width="256" height="256" role="img" aria-label="Evilution — evil ruby mutant mascot">
|
|
2
|
+
<title>Evilution</title>
|
|
3
|
+
<defs>
|
|
4
|
+
<radialGradient id="bg" cx="50%" cy="40%" r="78%">
|
|
5
|
+
<stop offset="0%" stop-color="#27364D"/>
|
|
6
|
+
<stop offset="100%" stop-color="#1E293B"/>
|
|
7
|
+
</radialGradient>
|
|
8
|
+
<pattern id="hex" width="56" height="100" patternUnits="userSpaceOnUse" patternTransform="scale(0.42)">
|
|
9
|
+
<path d="M28 66L0 50L0 16L28 0L56 16L56 50L28 66L28 100M28 0L28 34L0 50M28 34L56 50"
|
|
10
|
+
fill="none" stroke="#3B4D68" stroke-width="2" opacity="0.30"/>
|
|
11
|
+
</pattern>
|
|
12
|
+
<filter id="glow" x="-60%" y="-60%" width="220%" height="220%">
|
|
13
|
+
<feGaussianBlur stdDeviation="2.4" result="b"/>
|
|
14
|
+
<feMerge><feMergeNode in="b"/><feMergeNode in="SourceGraphic"/></feMerge>
|
|
15
|
+
</filter>
|
|
16
|
+
</defs>
|
|
17
|
+
|
|
18
|
+
<!-- backdrop -->
|
|
19
|
+
<rect width="256" height="256" rx="52" fill="url(#bg)"/>
|
|
20
|
+
<rect width="256" height="256" rx="52" fill="url(#hex)"/>
|
|
21
|
+
|
|
22
|
+
<!-- clawed arms (behind gem) -->
|
|
23
|
+
<g stroke="#470B13" stroke-width="2.5" stroke-linejoin="round">
|
|
24
|
+
<path d="M60 124 L40 132 L46 138 L36 142 L48 147 L40 153 L56 150 L64 136 Z" fill="#9A1826"/>
|
|
25
|
+
<path d="M196 124 L216 132 L210 138 L220 142 L208 147 L216 153 L200 150 L192 136 Z" fill="#7F1420"/>
|
|
26
|
+
</g>
|
|
27
|
+
|
|
28
|
+
<!-- faceted ruby body -->
|
|
29
|
+
<g stroke="#470B13" stroke-width="2.5" stroke-linejoin="round">
|
|
30
|
+
<!-- table (top) -->
|
|
31
|
+
<polygon points="98,82 128,82 128,120 88,120" fill="#D5384A"/>
|
|
32
|
+
<polygon points="128,82 158,82 168,120 128,120" fill="#B91C2C"/>
|
|
33
|
+
<!-- crown shoulders -->
|
|
34
|
+
<polygon points="54,120 88,120 98,82" fill="#E2566B"/>
|
|
35
|
+
<polygon points="168,120 202,120 158,82" fill="#911A28"/>
|
|
36
|
+
<!-- pavilion: main facet ridges run from the table corners (88,120 / 168,120)
|
|
37
|
+
down to the culet, continuing the crown's inclined edges -->
|
|
38
|
+
<polygon points="54,120 88,120 128,214" fill="#A81B2A"/>
|
|
39
|
+
<polygon points="88,120 128,120 128,214" fill="#911A28"/>
|
|
40
|
+
<polygon points="128,120 168,120 128,214" fill="#7C1320"/>
|
|
41
|
+
<polygon points="168,120 202,120 128,214" fill="#5E0F18"/>
|
|
42
|
+
</g>
|
|
43
|
+
<!-- crisp outer silhouette -->
|
|
44
|
+
<polygon points="98,82 158,82 202,120 128,214 54,120" fill="none" stroke="#3A0910" stroke-width="3" stroke-linejoin="round"/>
|
|
45
|
+
|
|
46
|
+
<!-- glowing mutant eyes -->
|
|
47
|
+
<g filter="url(#glow)">
|
|
48
|
+
<polygon points="100,99 123,106 118,117 96,110" fill="#22D3EE"/>
|
|
49
|
+
<polygon points="156,99 133,106 138,117 160,110" fill="#22D3EE"/>
|
|
50
|
+
<polygon points="110,105 118,107 116,113 108,111" fill="#0E1A2B"/>
|
|
51
|
+
<polygon points="146,105 138,107 140,113 148,111" fill="#0E1A2B"/>
|
|
52
|
+
</g>
|
|
53
|
+
|
|
54
|
+
<!-- fanged grin -->
|
|
55
|
+
<path d="M106 138 L150 138 L142 149 L134 144 L128 150 L122 144 L114 149 Z" fill="#1B060C"/>
|
|
56
|
+
<polygon points="118,138 125,138 122,148" fill="#EAF2F8"/>
|
|
57
|
+
<polygon points="131,138 138,138 134,148" fill="#EAF2F8"/>
|
|
58
|
+
|
|
59
|
+
<!-- mutation cracks leaking binary -->
|
|
60
|
+
<g filter="url(#glow)" fill="none" stroke="#67E8F9" stroke-width="1.7" stroke-linecap="round" opacity="0.92">
|
|
61
|
+
<polyline points="166,90 156,106 169,120 159,134"/>
|
|
62
|
+
<polyline points="82,150 92,160 86,172"/>
|
|
63
|
+
</g>
|
|
64
|
+
<g fill="#7FE9F7" font-family="monospace" font-size="11" font-weight="700" opacity="0.85">
|
|
65
|
+
<text x="170" y="128" transform="rotate(12 170 128)">1</text>
|
|
66
|
+
<text x="176" y="140" transform="rotate(12 176 140)">0</text>
|
|
67
|
+
<text x="168" y="150" transform="rotate(12 168 150)">1</text>
|
|
68
|
+
<text x="78" y="184" transform="rotate(-14 78 184)">0</text>
|
|
69
|
+
</g>
|
|
70
|
+
</svg>
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
# Architecture (contributor guide)
|
|
2
|
+
|
|
3
|
+
How evilution turns a source file into a mutation-testing report, and where to
|
|
4
|
+
plug in new behavior. This is an internals map for contributors — none of the
|
|
5
|
+
classes named here are a public API (see [public_api.md](public_api.md)); they
|
|
6
|
+
can move or change in any release.
|
|
7
|
+
|
|
8
|
+
For deeper dives on two subsystems that have their own docs, see
|
|
9
|
+
[isolation.md](isolation.md) (fork vs in-process, sandboxing, signal safety) and
|
|
10
|
+
[integrations.md](integrations.md) (RSpec / Minitest / Test::Unit specifics).
|
|
11
|
+
|
|
12
|
+
## The one-paragraph version
|
|
13
|
+
|
|
14
|
+
Evilution parses each target file with [Prism](https://github.com/ruby/prism)
|
|
15
|
+
into an AST with exact byte offsets. Every method becomes a *subject*. Each
|
|
16
|
+
mutation *operator* walks a subject's AST and emits *mutations* — byte-range
|
|
17
|
+
edits applied by source-level surgery (no AST unparsing). For every mutation,
|
|
18
|
+
evilution copies the file, applies the edit, runs the covering specs in an
|
|
19
|
+
isolated worker, and records whether the tests noticed (`killed`) or not
|
|
20
|
+
(`survived`). Results are aggregated into a `Summary`, scored, and handed to a
|
|
21
|
+
reporter.
|
|
22
|
+
|
|
23
|
+
## High-level flow
|
|
24
|
+
|
|
25
|
+
```mermaid
|
|
26
|
+
flowchart TD
|
|
27
|
+
CLI["exe/evilution → Evilution::CLI<br/>Commands::Run"] --> CFG["Config<br/>(YAML + CLI + env, frozen)"]
|
|
28
|
+
CFG --> RUN["Runner#call<br/>(orchestrator)"]
|
|
29
|
+
RUN --> SUB["SubjectPipeline<br/>AST::Parser → Subject per method"]
|
|
30
|
+
SUB --> BASE["BaselineRunner<br/>record already-failing specs"]
|
|
31
|
+
SUB --> PLAN["MutationPlanner<br/>Mutator::Registry → Mutation objects<br/>dedupe · disable · sig · equivalent"]
|
|
32
|
+
PLAN --> EXEC["MutationExecutor<br/>jobs=1 Sequential · jobs>1 Parallel::Pool"]
|
|
33
|
+
BASE --> EXEC
|
|
34
|
+
EXEC --> ISO["Isolation::Fork / InProcess<br/>apply mutation → Integration runs specs"]
|
|
35
|
+
ISO --> STAT["classify_status<br/>killed / survived / error / timeout / unresolved"]
|
|
36
|
+
STAT --> SUM["Result::Summary<br/>counts + score"]
|
|
37
|
+
SUM --> REP["ReportPublisher → Reporter::{CLI,JSON,HTML}"]
|
|
38
|
+
REP --> OUT["stdout / evilution-report.html<br/>+ optional Session::Store"]
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
## Module map
|
|
42
|
+
|
|
43
|
+
Everything lives under `lib/evilution/`.
|
|
44
|
+
|
|
45
|
+
| Namespace | Responsibility | Key entry points |
|
|
46
|
+
|---|---|---|
|
|
47
|
+
| `CLI`, `CLI::Commands::*` | Parse argv, build `Config`, dispatch subcommands (`run`, `session`, `compare`), map result to exit code. | `cli.rb`, `cli/commands/run.rb` |
|
|
48
|
+
| `Config`, `Config::*` | Merge `.evilution.yml` + CLI flags + env, validate, freeze. | `config.rb`, `config/sources.rb`, `config/validators/*` |
|
|
49
|
+
| `Runner`, `Runner::*` | Orchestrate the whole run. Each stage is its own collaborator. | `runner.rb`, `runner/*` |
|
|
50
|
+
| `AST`, `Subject` | Prism parse, find method subjects, source surgery, pattern matching, heredoc spans. | `ast/parser.rb`, `ast/source_surgeon.rb`, `subject.rb` |
|
|
51
|
+
| `Mutator`, `Mutator::Operator::*` | 74 operators (default profile) that emit byte-edits; registry + profiles. | `mutator/base.rb`, `mutator/registry.rb`, `mutator/operator/*` |
|
|
52
|
+
| `Mutation` | An immutable mutation record (original/mutated sources, slice, location, parse status). | `mutation.rb` |
|
|
53
|
+
| `SpecResolver`, `SpecSelector` | Map a source file to its covering spec files (layout heuristics + explicit mappings). | `spec_resolver.rb`, `spec_selector.rb` |
|
|
54
|
+
| `Isolation::{Fork,InProcess}`, `ProcessSupervisor` | Run one mutation's tests in isolation; process-group lifecycle, sandboxing, TERM/KILL ladder. | `isolation/fork.rb`, `process_supervisor.rb` |
|
|
55
|
+
| `Integration::{RSpec,Minitest,TestUnit}` | Apply a mutation and run the configured test framework; report the raw outcome. | `integration/base.rb`, `integration/rspec.rb` |
|
|
56
|
+
| `Parallel::{Pool,WorkQueue}` | Fan mutations across worker processes for `jobs > 1`. | `parallel/pool.rb`, `parallel/work_queue.rb` |
|
|
57
|
+
| `Result::{MutationResult,Summary}` | Per-mutation result + aggregated, scored summary. | `result/mutation_result.rb`, `result/summary.rb` |
|
|
58
|
+
| `Reporter::{CLI,JSON,HTML,Suggestion}` | Render a `Summary` to text / JSON / HTML. | `reporter/*` |
|
|
59
|
+
| `Session`, `Compare` | Persist runs to `.evilution/results/*.json`; diff two sessions. | `session/store.rb`, `compare.rb` |
|
|
60
|
+
| `Coverage`, `Equivalent`, `Baseline`, `Cache`, `Hooks`, `MCP` | Coverage-based example targeting, equivalent-mutation detection, baseline capture, incremental cache, lifecycle hooks, MCP server. | respective dirs |
|
|
61
|
+
|
|
62
|
+
## Data flow, source → result
|
|
63
|
+
|
|
64
|
+
Driven entirely by `Evilution::Runner#call` (`runner.rb:24`). Each step names the
|
|
65
|
+
class that owns it.
|
|
66
|
+
|
|
67
|
+
1. **Config** — `Config#initialize` merges YAML + CLI + env (`Config::Sources.merge`),
|
|
68
|
+
applies `DEFAULTS`, freezes. `Runner#initialize` builds the `AST::Parser`,
|
|
69
|
+
`Mutator::Registry.for_profile(config.profile)`, and (if `incremental`) a `Cache`.
|
|
70
|
+
2. **Subjects** — `Runner::SubjectPipeline#call` resolves target files (explicit,
|
|
71
|
+
`source:<glob>`, or `Git::ChangedFiles`), then `AST::Parser#call` runs
|
|
72
|
+
`Prism.parse` and `AST::SubjectFinder` (a `Prism::Visitor`) emits one
|
|
73
|
+
`Evilution::Subject` per `def` node. Optional descendant/target/line-range filters follow.
|
|
74
|
+
3. **Baseline** — `Runner::BaselineRunner#call` builds the integration from
|
|
75
|
+
`Runner::INTEGRATIONS` (`rspec`/`minitest`/`test_unit`) and records spec files
|
|
76
|
+
that already fail *before* any mutation, so their mutations aren't miscounted.
|
|
77
|
+
An optional `Runner::Canary` proves the pipeline can observe a known mutation.
|
|
78
|
+
4. **Mutations** — `Runner::MutationPlanner#call` flat-maps subjects through
|
|
79
|
+
`Mutator::Registry#mutations_for`. The registry instantiates each operator and
|
|
80
|
+
runs `operator.call(subject, filter:)`; each operator subclasses
|
|
81
|
+
`Mutator::Base` and calls `add_mutation`, which runs `AST::SourceSurgeon` and
|
|
82
|
+
builds an immutable `Evilution::Mutation`. The planner then **deduplicates**
|
|
83
|
+
(by `file_path` + mutated source), and filters **disabled** (`DisableComment`),
|
|
84
|
+
**Sorbet `sig`** (`AST::SorbetSigDetector`), and **equivalent**
|
|
85
|
+
(`Equivalent::Detector`) mutations. Equivalent ones become `:equivalent`
|
|
86
|
+
results directly.
|
|
87
|
+
5. **Spec resolution** — per subject, `SpecSelector#call(source_path)` picks specs
|
|
88
|
+
(explicit `spec_files` → `spec_mappings` → `SpecResolver#resolve_specs` layout
|
|
89
|
+
heuristics). Example-level targeting narrows to examples that reference the
|
|
90
|
+
mutated token (`ExampleFilter` / `CoverageExampleFilter`).
|
|
91
|
+
6. **Execute** — `Runner::MutationExecutor#call` picks a strategy by `config.jobs`:
|
|
92
|
+
`Strategy::Sequential` for `jobs == 1`, `Strategy::Parallel` (via
|
|
93
|
+
`Parallel::Pool` / `WorkQueue`) for `jobs > 1`. Either way each mutation reaches
|
|
94
|
+
`@isolator.call(mutation:, test_command:, timeout:)`.
|
|
95
|
+
7. **Isolate + test** — `Isolation::Fork#call` (default for Rails/gems) spawns a
|
|
96
|
+
sandboxed worker through `ProcessSupervisor#spawn` (own process group, TERM →
|
|
97
|
+
grace → KILL ladder, sandbox reap), applies the mutation
|
|
98
|
+
(`Integration::Loading::MutationApplier`), and runs `Integration::Base#call` →
|
|
99
|
+
`run_tests`. The child's result is marshaled back over a length-prefixed pipe.
|
|
100
|
+
`Isolation::InProcess` is the lighter path for plain-Ruby projects.
|
|
101
|
+
8. **Status** — outcome flags come from the integration
|
|
102
|
+
(`Integration::RSpec::ResultBuilder`: passed / test_crashed / examples-loaded
|
|
103
|
+
guard / unresolved); the final symbol is chosen by
|
|
104
|
+
`Isolation::Fork#classify_status`: `:timeout` → `:killed` (crash) →
|
|
105
|
+
`:unresolved` → `:error` → `:survived` (tests passed) → default `:killed`.
|
|
106
|
+
A `NeutralizationPipeline` can reclassify results whose covering spec already
|
|
107
|
+
failed at baseline into `:neutral`.
|
|
108
|
+
9. **Aggregate + report** — `Result::Summary` counts each status and computes
|
|
109
|
+
`score = killed / score_denominator` (total minus error/neutral/equivalent/
|
|
110
|
+
unresolved/unparseable). `Runner::ReportPublisher#publish` selects a reporter by
|
|
111
|
+
`config.format` and writes it; `Session::Store` optionally persists the run.
|
|
112
|
+
`Commands::Run` maps `summary.success?(min_score:)` to exit code `0`/`1` (`2` on error).
|
|
113
|
+
|
|
114
|
+
## How to add a new mutator
|
|
115
|
+
|
|
116
|
+
A mutator is a `Prism::Visitor` subclass that emits byte-range edits.
|
|
117
|
+
|
|
118
|
+
1. **Create the operator** in `lib/evilution/mutator/operator/<name>.rb`:
|
|
119
|
+
|
|
120
|
+
```ruby
|
|
121
|
+
# frozen_string_literal: true
|
|
122
|
+
|
|
123
|
+
require_relative "../operator"
|
|
124
|
+
|
|
125
|
+
class Evilution::Mutator::Operator::MyThing < Evilution::Mutator::Base
|
|
126
|
+
def visit_integer_node(node) # a Prism visit_*_node hook
|
|
127
|
+
add_mutation(
|
|
128
|
+
offset: node.location.start_offset,
|
|
129
|
+
length: node.location.length,
|
|
130
|
+
replacement: "42", # the source text to splice in
|
|
131
|
+
node: node
|
|
132
|
+
)
|
|
133
|
+
super # ALWAYS call super to keep walking
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
- Implement one or more `visit_<node_type>_node(node)` methods (the Prism node
|
|
139
|
+
names — `visit_call_node`, `visit_if_node`, …). Call `super` so child nodes
|
|
140
|
+
are still visited.
|
|
141
|
+
- Emit edits only through `add_mutation(offset:, length:, replacement:, node:)`.
|
|
142
|
+
It handles heredoc-span extension, the mutation filter, source surgery, and
|
|
143
|
+
builds the immutable `Mutation`. Never edit source strings directly.
|
|
144
|
+
- The operator's registered name is auto-derived from the class name
|
|
145
|
+
(`MyThing` → `my_thing`); that string is the `operator` field in JSON output
|
|
146
|
+
and is part of the public contract, so name it deliberately.
|
|
147
|
+
|
|
148
|
+
2. **Require it** in `lib/evilution.rb` alongside the other
|
|
149
|
+
`require_relative "evilution/mutator/operator/..."` lines.
|
|
150
|
+
|
|
151
|
+
3. **Register it** in `lib/evilution/mutator/registry.rb`: add the class to the
|
|
152
|
+
`default` list (runs in every profile) — or, for an aggressive operator meant
|
|
153
|
+
only for `profile: strict`, to `STRICT_EXTRA_OPERATORS` instead.
|
|
154
|
+
|
|
155
|
+
4. **Test it** under `spec/evilution/mutator/operator/<name>_spec.rb`, mirroring an
|
|
156
|
+
existing operator spec. Adding an operator to the `default` profile shifts
|
|
157
|
+
mutation scores and is a **MINOR** version bump (see [versioning.md](versioning.md)).
|
|
158
|
+
|
|
159
|
+
## How to add a new reporter
|
|
160
|
+
|
|
161
|
+
A reporter is `.new(**opts)` + `#call(summary) -> String`. It consumes a frozen
|
|
162
|
+
`Evilution::Result::Summary` and returns the rendered report body.
|
|
163
|
+
|
|
164
|
+
1. **Create the reporter** in `lib/evilution/reporter/<format>.rb`:
|
|
165
|
+
|
|
166
|
+
```ruby
|
|
167
|
+
# frozen_string_literal: true
|
|
168
|
+
|
|
169
|
+
require_relative "../reporter"
|
|
170
|
+
|
|
171
|
+
class Evilution::Reporter::Csv
|
|
172
|
+
def initialize(integration: :rspec) # keyword args only; keep them optional
|
|
173
|
+
@integration = integration
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def call(summary) # returns a String
|
|
177
|
+
# read summary.results, summary.score, summary.killed, summary.survived,
|
|
178
|
+
# summary.survived_results, etc. — see Result::Summary for the full API
|
|
179
|
+
"..."
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
The `Summary` API a reporter reads includes: counts (`total`, `killed`,
|
|
185
|
+
`survived`, `errors`, `neutral`, `equivalent`, `unresolved`, `unparseable`,
|
|
186
|
+
`timed_out`), metrics (`score`, `score_denominator`, `success?(min_score:)`,
|
|
187
|
+
`efficiency`, `peak_memory_mb`), the raw `results`, and filtered lists
|
|
188
|
+
(`survived_results`, `killed_results`, …). Each result exposes `mutation`
|
|
189
|
+
(`operator_name`, `file_path`, `line`, `diff`), `status`, `duration`, and
|
|
190
|
+
status predicates (`killed?`, `survived?`, …).
|
|
191
|
+
|
|
192
|
+
2. **Require it** in `lib/evilution.rb` (next to the other `reporter/*` requires)
|
|
193
|
+
**and** in `lib/evilution/runner.rb` (the runner requires reporters
|
|
194
|
+
independently, near the bottom of the file).
|
|
195
|
+
|
|
196
|
+
3. **Wire the dispatch** in `Runner::ReportPublisher#build_reporter`
|
|
197
|
+
(`runner/report_publisher.rb`): add a `when :csv` branch returning your class,
|
|
198
|
+
and a matching `require_relative "../reporter/csv"` at the top. If the output
|
|
199
|
+
should go to a file rather than stdout, extend the `#publish` write branch
|
|
200
|
+
(only `:html` writes a file today; every other format is `$stdout.puts`ed).
|
|
201
|
+
|
|
202
|
+
4. **Advertise the value.** There is no strict format allowlist to update —
|
|
203
|
+
`config.format` is only `to_sym`'d, not validated — but update the
|
|
204
|
+
`--format` help string in `cli/parser/options_builder.rb` so `csv` is
|
|
205
|
+
discoverable. (The `compare` subcommand keeps its *own* `SUPPORTED_FORMATS`
|
|
206
|
+
list; touch it only if `compare` should support the new format too.)
|
|
207
|
+
|
|
208
|
+
Adding a new format is additive — a **MINOR** bump.
|
|
209
|
+
|
|
210
|
+
## How to add a new test integration (bonus)
|
|
211
|
+
|
|
212
|
+
Integrations subclass `Evilution::Integration::Base` (`integration/base.rb`),
|
|
213
|
+
which defines the contract: class methods `baseline_runner` / `baseline_options`,
|
|
214
|
+
and instance methods `run_tests(mutation)`, `ensure_framework_loaded`,
|
|
215
|
+
`build_args(mutation)`, `reset_state` (each `raise NotImplementedError` until
|
|
216
|
+
overridden). `Base#call` already handles applying the mutation and firing
|
|
217
|
+
`mutation_insert_pre/post` hooks. Register the new class in
|
|
218
|
+
`Runner::INTEGRATIONS` (`runner/baseline_runner.rb`) and allow its symbol in the
|
|
219
|
+
integration config validator. See [integrations.md](integrations.md) for the
|
|
220
|
+
framework-specific details.
|
data/docs/isolation.md
CHANGED
|
@@ -87,6 +87,7 @@ order, falling back to the gem's library entry point (`lib/<gem>.rb`):
|
|
|
87
87
|
1. `spec/spec_helper.rb` (RSpec)
|
|
88
88
|
2. `test/test_helper.rb` (Minitest / Test::Unit)
|
|
89
89
|
3. `test/helper.rb` (flat-layout convention)
|
|
90
|
+
4. `test/<gem>/helper.rb` (namespaced suites, e.g. ruby/csv's `test/csv/helper.rb`; the namespace is derived from the gem name, with the dotted form for dash-named gems) (PR #1386)
|
|
90
91
|
|
|
91
92
|
When a gem is detected but none of those helpers exist, evilution prints a
|
|
92
93
|
warning naming the locations it looked in and pointing at `--preload`, so a
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
# Migrating from `mutant` to Evilution
|
|
2
|
+
|
|
3
|
+
This guide is for teams moving off [`mutant`](https://github.com/mbj/mutant) — whose
|
|
4
|
+
runtime is under a commercial license — to Evilution, which is MIT-licensed with no
|
|
5
|
+
commercial restrictions. It maps the commands, config, and output you already know
|
|
6
|
+
onto their Evilution equivalents, and is honest about where the two tools differ.
|
|
7
|
+
|
|
8
|
+
Evilution is not a drop-in fork of mutant. It produces its own mutations with its own
|
|
9
|
+
operator set and selects work by **file path** rather than by subject expression. The
|
|
10
|
+
result is the same kind of signal — surviving mutations expose weak tests — but the
|
|
11
|
+
numbers and the workflow are not identical. The `compare` command (below) exists
|
|
12
|
+
precisely so you can line the two tools up and see the difference for yourself.
|
|
13
|
+
|
|
14
|
+
## TL;DR
|
|
15
|
+
|
|
16
|
+
| mutant | Evilution |
|
|
17
|
+
| --- | --- |
|
|
18
|
+
| `mutant run --use rspec -- 'MyApp::Foo*'` | `evilution run lib/my_app/ --target 'MyApp::Foo*'` |
|
|
19
|
+
| `mutant.yml` | `.evilution.yml` (run `evilution init` to scaffold) |
|
|
20
|
+
| `mutant-rspec` / `mutant-minitest` gems | built in: `--integration rspec\|minitest\|test-unit` |
|
|
21
|
+
| "mutation coverage" %, "alive" | "mutation score" 0.0–1.0, "survived" |
|
|
22
|
+
| commercial license for the runtime | MIT |
|
|
23
|
+
|
|
24
|
+
## 1. Install
|
|
25
|
+
|
|
26
|
+
Drop the mutant gems and add Evilution:
|
|
27
|
+
|
|
28
|
+
```ruby
|
|
29
|
+
# Gemfile — remove: gem "mutant", "mutant-rspec"
|
|
30
|
+
gem "evilution", group: :test
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
```sh
|
|
34
|
+
bundle install
|
|
35
|
+
bundle exec evilution init # writes a commented .evilution.yml
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
There is no separate integration gem to install — RSpec, Minitest, and Test::Unit
|
|
39
|
+
support ships in the core gem.
|
|
40
|
+
|
|
41
|
+
## 2. The core workflow shift: files, not subjects
|
|
42
|
+
|
|
43
|
+
mutant is **subject-driven**: you pass a match expression and it finds the code.
|
|
44
|
+
Evilution is **file-driven**: you pass file paths, and optionally narrow with
|
|
45
|
+
`--target`.
|
|
46
|
+
|
|
47
|
+
```sh
|
|
48
|
+
# mutant
|
|
49
|
+
bundle exec mutant run --use rspec -- 'MyApp::Calculator#add'
|
|
50
|
+
|
|
51
|
+
# evilution — point at the file, optionally filter to the method
|
|
52
|
+
bundle exec evilution run lib/my_app/calculator.rb --target 'MyApp::Calculator#add'
|
|
53
|
+
|
|
54
|
+
# whole directory, then filter by expression
|
|
55
|
+
bundle exec evilution run lib/ --target 'MyApp::Calculator*'
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
`--target` accepts the expression shapes you are used to, plus a source glob:
|
|
59
|
+
|
|
60
|
+
| Intent | mutant match | Evilution `--target` |
|
|
61
|
+
| --- | --- | --- |
|
|
62
|
+
| One method | `MyApp::Foo#bar` | `MyApp::Foo#bar` |
|
|
63
|
+
| One class | `MyApp::Foo` | `MyApp::Foo` |
|
|
64
|
+
| Namespace (recursive) | `MyApp::Foo*` | `MyApp::Foo*` |
|
|
65
|
+
| All instance methods | `MyApp::Foo#` | `MyApp::Foo#` |
|
|
66
|
+
| All singleton methods | `MyApp::Foo.` | `MyApp::Foo.` |
|
|
67
|
+
| Subclasses | _(n/a)_ | `descendants:MyApp::Foo` |
|
|
68
|
+
| By file glob | _(use file args)_ | `source:lib/**/*.rb` |
|
|
69
|
+
|
|
70
|
+
## 3. Command mapping
|
|
71
|
+
|
|
72
|
+
| Task | mutant | Evilution |
|
|
73
|
+
| --- | --- | --- |
|
|
74
|
+
| Run mutation testing | `mutant run -- 'Foo'` | `evilution run lib/foo.rb` (alias `mutate`; binary alias `evil`) |
|
|
75
|
+
| Choose framework | `--use rspec` / `mutant-minitest` | `--integration rspec\|minitest\|test-unit` |
|
|
76
|
+
| Parallel workers | `-j N` / `--jobs N` | `-j N` / `--jobs N` |
|
|
77
|
+
| Fail fast | `--fail-fast` | `--fail-fast [N]` (off by default; N defaults to 1 when the flag is given without a value) |
|
|
78
|
+
| Coverage gate | `--score` (default `1.0` / 100%) | `--min-score` (default `0.0`) |
|
|
79
|
+
| Preview mutations | `mutant util mutation` / `mutant environment subject list` | `evilution util mutation -e 'a + b'` / `evilution subjects lib/foo.rb` |
|
|
80
|
+
| Set load path / requires | `--include lib --require my_app` | handled by `--preload` (auto-detects `spec/spec_helper.rb` etc.) + Bundler |
|
|
81
|
+
| JSON report | session JSON (always written) | `--format json` (`--save-session` to persist) |
|
|
82
|
+
| Inline opt-out | `# mutant:disable` | `# evilution:disable` |
|
|
83
|
+
| Aggressive operators | _(always on)_ | `--profile strict` (or `--strict`) |
|
|
84
|
+
|
|
85
|
+
### Things mutant has that Evilution does not (yet)
|
|
86
|
+
|
|
87
|
+
- **`--since <git-ref>`** — mutant can restrict subjects to lines changed since a git
|
|
88
|
+
ref. Evilution has no diff-aware selection. The closest tools are `--incremental`
|
|
89
|
+
(caches `killed`/`timeout` results and skips unchanged mutations on re-runs — a
|
|
90
|
+
different model) and `--target source:<glob>` / explicit file args to scope a run.
|
|
91
|
+
|
|
92
|
+
### Things Evilution adds
|
|
93
|
+
|
|
94
|
+
- `--format html` interactive report, and `evilution compare` (see §6).
|
|
95
|
+
- Per-mutation **example targeting** (runs only the examples that exercise the mutated
|
|
96
|
+
code) — on by default, tune with `example_targeting*` keys.
|
|
97
|
+
- `--suggest-tests` emits concrete test code for survivors.
|
|
98
|
+
- Explicit `:unresolved` status when no spec maps to a source file (a coverage-gap
|
|
99
|
+
signal rather than a silent skip).
|
|
100
|
+
|
|
101
|
+
## 4. Config translation (`mutant.yml` → `.evilution.yml`)
|
|
102
|
+
|
|
103
|
+
```yaml
|
|
104
|
+
# mutant.yml
|
|
105
|
+
integration:
|
|
106
|
+
name: rspec
|
|
107
|
+
jobs: 8
|
|
108
|
+
includes:
|
|
109
|
+
- lib
|
|
110
|
+
requires:
|
|
111
|
+
- my_app
|
|
112
|
+
matcher:
|
|
113
|
+
subjects:
|
|
114
|
+
- MyApp::Billing*
|
|
115
|
+
ignore:
|
|
116
|
+
- MyApp::Billing::Legacy#deprecated
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
```yaml
|
|
120
|
+
# .evilution.yml
|
|
121
|
+
integration: rspec
|
|
122
|
+
jobs: 8
|
|
123
|
+
# `includes` / `requires` are usually unnecessary — Evilution preloads the test
|
|
124
|
+
# helper (and the gem entry) automatically. Override only if needed:
|
|
125
|
+
preload: spec/spec_helper.rb
|
|
126
|
+
# `matcher.subjects` -> run file args + --target; expression below is illustrative.
|
|
127
|
+
target: "MyApp::Billing*"
|
|
128
|
+
# `matcher.ignore` -> AST ignore patterns (see docs/ast_pattern_syntax.md) or an
|
|
129
|
+
# inline `# evilution:disable` comment on the method.
|
|
130
|
+
ignore_patterns: []
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Key-by-key:
|
|
134
|
+
|
|
135
|
+
| `mutant.yml` | `.evilution.yml` | Notes |
|
|
136
|
+
| --- | --- | --- |
|
|
137
|
+
| `integration.name: rspec` | `integration: rspec` | string, not a mapping |
|
|
138
|
+
| `jobs` | `jobs` | same |
|
|
139
|
+
| `includes` | _(usually omit)_ | Bundler + `preload` cover the load path |
|
|
140
|
+
| `requires` | `preload` | a single preload file (autodetected by default) |
|
|
141
|
+
| `matcher.subjects` | `target` + file args | one expression; combine with positional paths |
|
|
142
|
+
| `matcher.ignore` | `ignore_patterns` / `# evilution:disable` | AST patterns or inline comments |
|
|
143
|
+
| `fail_fast` | `fail_fast` | integer N or `null` |
|
|
144
|
+
| `--score` gate (CLI; default `1.0`) | `min_score` | evilution defaults to `0.0` — set your own gate |
|
|
145
|
+
| _(n/a)_ | `profile: strict` | opt into aggressive truthiness mutators |
|
|
146
|
+
|
|
147
|
+
Run `evilution init` for a fully commented template, and point your editor at
|
|
148
|
+
`schema/evilution.config.schema.json` for autocomplete/validation.
|
|
149
|
+
|
|
150
|
+
> **Note:** the `integration` enum in that JSON schema currently lists only `rspec`
|
|
151
|
+
> and `minitest`. `test-unit` (normalized internally to `test_unit`) is fully
|
|
152
|
+
> supported by the runtime, but a schema-aware editor may flag it as invalid until
|
|
153
|
+
> the schema catches up — it will still run.
|
|
154
|
+
|
|
155
|
+
## 5. Output differences
|
|
156
|
+
|
|
157
|
+
The vocabulary and the math differ — read this before comparing dashboards.
|
|
158
|
+
|
|
159
|
+
| Concept | mutant | Evilution |
|
|
160
|
+
| --- | --- | --- |
|
|
161
|
+
| Detected mutation | `killed` | `killed` |
|
|
162
|
+
| Undetected mutation | `alive` | `survived` |
|
|
163
|
+
| Headline metric | "mutation coverage" (%) | "mutation score" (0.0–1.0) |
|
|
164
|
+
| Pre-existing test failure | `neutral` / `noop` | `neutral` |
|
|
165
|
+
| Operator name in report | **not emitted** | emitted, e.g. `Arithmetic::Swap` |
|
|
166
|
+
| Per-mutation diff | unified diff with `@@` header + context | `- old` / `+ new` pair (and a full `unified_diff` for survivors) |
|
|
167
|
+
| Report shape | nested: session → subject → coverage results | flat per-status buckets |
|
|
168
|
+
|
|
169
|
+
**Score formula.** mutant's coverage is roughly `killed / (total - neutral)` and the
|
|
170
|
+
culture is to gate at 100%. Evilution's score is:
|
|
171
|
+
|
|
172
|
+
```
|
|
173
|
+
score = killed / (total - errors - neutral - equivalent - unresolved - unparseable)
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
i.e. only `killed / (killed + survived + timed_out)`. It defaults to a `min_score` of
|
|
177
|
+
`0.0` (no gate) — set your own threshold in CI with `--min-score`.
|
|
178
|
+
|
|
179
|
+
**Extra statuses** Evilution reports that mutant has no equivalent for:
|
|
180
|
+
|
|
181
|
+
- `unresolved` — no spec file mapped to the mutated source (coverage gap, excluded from score)
|
|
182
|
+
- `unparseable` — the mutated source did not parse, so it never ran (excluded)
|
|
183
|
+
- `equivalent` — proven behaviorally identical to the original (excluded)
|
|
184
|
+
- `timeout` / `error` — surfaced explicitly per mutation
|
|
185
|
+
|
|
186
|
+
**Exit codes:** `0` pass · `1` score below `--min-score` (survivors to address) · `2` error.
|
|
187
|
+
|
|
188
|
+
## 6. Verify the migration with `compare`
|
|
189
|
+
|
|
190
|
+
`evilution compare` ingests **both** a mutant session JSON and an Evilution JSON
|
|
191
|
+
report, normalizes them to a common shape, and buckets the mutations so you can see
|
|
192
|
+
whether the same code is being killed:
|
|
193
|
+
|
|
194
|
+
```sh
|
|
195
|
+
# produce an evilution report
|
|
196
|
+
bundle exec evilution run lib/ --format json --save-session
|
|
197
|
+
|
|
198
|
+
# line it up against your last mutant session JSON
|
|
199
|
+
bundle exec evilution compare \
|
|
200
|
+
--against .mutant/results/<session-id>.json \
|
|
201
|
+
--current .evilution/results/<timestamp>.json \
|
|
202
|
+
--format text
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
The tool auto-detects which file came from which tool. Because the two tools use
|
|
206
|
+
different operator sets, expect the mutation **counts** to differ — the useful signal
|
|
207
|
+
is which subjects lose or gain coverage, not an exact 1:1 match. Operator names are
|
|
208
|
+
absent from mutant's JSON, so comparison keys on file + line + normalized diff, not on
|
|
209
|
+
operator.
|
|
210
|
+
|
|
211
|
+
## 7. Known gaps & parity notes
|
|
212
|
+
|
|
213
|
+
- **Different mutations.** Evilution ships its own operator registry (`--profile
|
|
214
|
+
default` / `strict`); it is not mutant's AST-handler set. Counts and specific
|
|
215
|
+
survivors will differ. Use `compare` to quantify.
|
|
216
|
+
- **No diff-based selection** (`--since <ref>`). Use file args, `--target source:`,
|
|
217
|
+
or `--incremental`.
|
|
218
|
+
- **File-driven, not subject-driven.** Always pass paths; narrow with `--target`.
|
|
219
|
+
- **100%-coverage culture.** Evilution does not assume a 100% gate; choose `min_score`
|
|
220
|
+
deliberately, and note that `:unresolved` mutations are excluded from the score
|
|
221
|
+
rather than counted as failures.
|
|
222
|
+
- **Operator names** appear in Evilution output but not mutant's, so cross-tool diffs
|
|
223
|
+
match on location + change, not operator identity.
|
|
224
|
+
|
|
225
|
+
If something you relied on in mutant has no clear equivalent here, please open an issue
|
|
226
|
+
— parity feedback directly shapes the 1.0 roadmap.
|
data/docs/public_api.md
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Public API
|
|
2
|
+
|
|
3
|
+
This document defines evilution's public, SemVer-governed surface for `1.x`.
|
|
4
|
+
|
|
5
|
+
## There is no public Ruby API
|
|
6
|
+
|
|
7
|
+
**The entire `Evilution::` Ruby namespace is internal.** Every class, module,
|
|
8
|
+
method, and constant under `Evilution::` may change, move, or disappear in any
|
|
9
|
+
release — including patch releases — with no deprecation cycle. Do not `require`
|
|
10
|
+
evilution and call into it programmatically expecting stability; internal
|
|
11
|
+
refactors will break you without warning.
|
|
12
|
+
|
|
13
|
+
Evilution is consumed through its CLI, its configuration file, its session
|
|
14
|
+
output, and its MCP tools — not as a library you embed. The top-level
|
|
15
|
+
`Evilution` module and the primary entry points (`Evilution::Runner`,
|
|
16
|
+
`Evilution::CLI`, `Evilution::Config`) carry an `@api private` YARD marker; the
|
|
17
|
+
namespace as a whole is internal by this declaration, whether or not an
|
|
18
|
+
individual class is tagged.
|
|
19
|
+
|
|
20
|
+
## What *is* the public contract
|
|
21
|
+
|
|
22
|
+
These surfaces are stable under the [versioning policy](versioning.md). They are
|
|
23
|
+
the supported way to drive evilution:
|
|
24
|
+
|
|
25
|
+
| Surface | Authoritative reference |
|
|
26
|
+
|---|---|
|
|
27
|
+
| **CLI commands and flags** | README "Command Reference" |
|
|
28
|
+
| **`.evilution.yml` configuration keys** | README "Configuration" |
|
|
29
|
+
| **Session JSON files** (`.evilution/results/*.json`) | README "JSON Output Schema" |
|
|
30
|
+
| **MCP tool schemas** (`evilution-mutate`, `evilution-session`, `evilution-info`) | README "MCP Server" → "Contract stability" |
|
|
31
|
+
| **Process exit codes** (`0` pass, `1` fail, `2` error) | README "Exit Codes" |
|
|
32
|
+
| **Hook events and payload keys** (configured via the `hooks:` config key) | README "Configuration" / hooks docs |
|
|
33
|
+
|
|
34
|
+
Anything not in that table — and in particular anything reachable only by calling
|
|
35
|
+
Ruby methods on `Evilution::` objects — is internal.
|
|
36
|
+
|
|
37
|
+
## Why no Ruby API at 1.0
|
|
38
|
+
|
|
39
|
+
Evilution's job is to run mutation testing from the command line (and from an MCP
|
|
40
|
+
agent). The value users depend on is the behaviour of `evilution run`, the shape
|
|
41
|
+
of the report it writes, and the config that tunes it — none of which requires a
|
|
42
|
+
frozen object graph. Keeping the Ruby internals unfrozen lets the isolation,
|
|
43
|
+
parallelism, and mutation-operator internals evolve freely across `1.x` without
|
|
44
|
+
spending major-version budget. If a genuine embedding use case emerges, a small
|
|
45
|
+
public facade can be introduced additively in a MINOR release.
|
|
46
|
+
|
|
47
|
+
## See also
|
|
48
|
+
|
|
49
|
+
- [Versioning & Upgrade Policy](versioning.md) — what each SemVer bump covers and
|
|
50
|
+
how deprecations work.
|
data/docs/versioning.md
CHANGED
|
@@ -7,14 +7,14 @@ This document defines what `evilution` promises across releases.
|
|
|
7
7
|
| Bump | Triggered by |
|
|
8
8
|
|---------------|-------------------------------------------------------------------------------|
|
|
9
9
|
| MAJOR (`2.0`) | Removing or renaming anything in the public contract; changing semantics; tightening input validation. |
|
|
10
|
-
| MINOR (`1.X`) | Adding a new CLI flag, config key, mutation operator,
|
|
10
|
+
| MINOR (`1.X`) | Adding a new CLI flag, config key, mutation operator, or session/MCP field; introducing a public Ruby facade where none exists today; relaxing validation; adding an operator to the `default` profile (whether brand-new or promoted from `strict`). |
|
|
11
11
|
| PATCH (`1.X.Y`) | Bug fix, performance improvement, documentation, internal refactor with no observable contract effect. |
|
|
12
12
|
|
|
13
13
|
## Public contract surface
|
|
14
14
|
|
|
15
15
|
The following surfaces are covered by the SemVer guarantees above:
|
|
16
16
|
|
|
17
|
-
- **Public Ruby API** —
|
|
17
|
+
- **Public Ruby API** — there is none. The entire `Evilution::` namespace is internal and may change in any release. See [docs/public_api.md](public_api.md).
|
|
18
18
|
- **CLI flags and commands** — the README "Command Reference" tables are the authoritative list.
|
|
19
19
|
- **`.evilution.yml` configuration keys** — see the README "Configuration" section.
|
|
20
20
|
- **Session JSON files** (`.evilution/results/*.json`) — see the README "JSON Output Schema" section.
|
|
@@ -27,7 +27,7 @@ Anything not on this list is internal. It can change in any release without a de
|
|
|
27
27
|
|
|
28
28
|
When a feature on the public contract surface is deprecated:
|
|
29
29
|
|
|
30
|
-
1. It is marked with
|
|
30
|
+
1. It is marked with a deprecation note in the relevant doc table (CLI flags, config keys, session/MCP fields).
|
|
31
31
|
2. Where the call site is reachable at runtime, a one-line warning is emitted to stderr.
|
|
32
32
|
3. The deprecated form remains functional for the entire `1.x` line. A feature deprecated in any `1.X` release continues to work in every subsequent `1.X+N` release.
|
|
33
33
|
4. The earliest release that may remove the feature is the next major (`2.0`), per the SemVer table above.
|
data/lib/evilution/cli.rb
CHANGED
|
@@ -6,7 +6,9 @@ require_relative "../../spec_resolver"
|
|
|
6
6
|
class Evilution::Config::Builders::SpecResolver
|
|
7
7
|
def self.call(integration:)
|
|
8
8
|
case integration
|
|
9
|
-
when :minitest
|
|
9
|
+
when :minitest, :test_unit
|
|
10
|
+
# test-unit gems mirror minitest's layout (test/ root, _test.rb suffix);
|
|
11
|
+
# without this they default to spec/_spec.rb and resolve to nothing.
|
|
10
12
|
Evilution::SpecResolver.new(test_dir: "test", test_suffix: "_test.rb", request_dir: "integration")
|
|
11
13
|
else
|
|
12
14
|
Evilution::SpecResolver.new
|
data/lib/evilution/config.rb
CHANGED
|
@@ -4,6 +4,7 @@ require "yaml"
|
|
|
4
4
|
require_relative "spec_resolver"
|
|
5
5
|
require_relative "spec_selector"
|
|
6
6
|
|
|
7
|
+
# @api private
|
|
7
8
|
class Evilution::Config
|
|
8
9
|
CONFIG_FILES = %w[.evilution.yml config/evilution.yml].freeze
|
|
9
10
|
CURRENT_SCHEMA_VERSION = 1
|
|
@@ -117,7 +118,6 @@ class Evilution::Config
|
|
|
117
118
|
FileLoader.load
|
|
118
119
|
end
|
|
119
120
|
|
|
120
|
-
# Generates a default config file template.
|
|
121
121
|
def self.default_template
|
|
122
122
|
<<~YAML
|
|
123
123
|
# Evilution configuration
|
|
@@ -43,6 +43,13 @@ module Evilution::GemDetector
|
|
|
43
43
|
@mutex.synchronize { @cache.clear }
|
|
44
44
|
end
|
|
45
45
|
|
|
46
|
+
# Public accessor for the resolved gem name (gemspec basename, disambiguated
|
|
47
|
+
# by target paths for multi-gemspec roots). Callers use it to derive
|
|
48
|
+
# conventional paths such as a namespaced test helper (test/<gem>/helper.rb).
|
|
49
|
+
def gem_name(root, target_paths: nil)
|
|
50
|
+
gem_name_for(root, target_paths: target_paths)
|
|
51
|
+
end
|
|
52
|
+
|
|
46
53
|
private
|
|
47
54
|
|
|
48
55
|
def starting_dir(path)
|
|
@@ -63,7 +63,9 @@ class Evilution::Integration::Loading::RedefinitionRecovery
|
|
|
63
63
|
|
|
64
64
|
def idempotency_violation?(error)
|
|
65
65
|
msg = error.message
|
|
66
|
-
|
|
66
|
+
# `msg` is a String and patterns are substrings, so `include?` is a
|
|
67
|
+
# substring test — `intersect?` would compare array elements, not text.
|
|
68
|
+
IDEMPOTENCY_PATTERNS.any? { |pat| msg.include?(pat) } # rubocop:disable Style/ArrayIntersect
|
|
67
69
|
end
|
|
68
70
|
|
|
69
71
|
def superclass_mismatch?(error)
|
|
@@ -276,8 +276,19 @@ class Evilution::Integration::Minitest < Evilution::Integration::Base
|
|
|
276
276
|
return if @warned_files.include?(file_path)
|
|
277
277
|
|
|
278
278
|
@warned_files << file_path
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
279
|
+
warn unresolved_test_message(file_path)
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
# Name both recovery paths when skipping :unresolved; when already falling
|
|
283
|
+
# back the suite is running, so only the explicit-spec hint applies.
|
|
284
|
+
# Behaviour-named layouts never auto-resolve (EV-ajby / GH #1376).
|
|
285
|
+
def unresolved_test_message(file_path)
|
|
286
|
+
if @fallback_to_full_suite
|
|
287
|
+
"[evilution] No matching test found for #{file_path}, running full suite. " \
|
|
288
|
+
"Use --spec to specify the test file."
|
|
289
|
+
else
|
|
290
|
+
"[evilution] No matching test found for #{file_path}, marking mutation unresolved. " \
|
|
291
|
+
"Use --spec to specify the test file, or --fallback-full-suite to run the whole suite."
|
|
292
|
+
end
|
|
282
293
|
end
|
|
283
294
|
end
|
|
@@ -11,8 +11,22 @@ class Evilution::Integration::RSpec::UnresolvedSpecWarner
|
|
|
11
11
|
return if @warned.include?(file_path)
|
|
12
12
|
|
|
13
13
|
@warned << file_path
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
warn message(file_path, fallback_to_full_suite)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
private
|
|
18
|
+
|
|
19
|
+
# When already falling back the suite is running, so only the explicit-spec
|
|
20
|
+
# hint is useful. Otherwise the mutation is skipped :unresolved — name BOTH
|
|
21
|
+
# recovery paths, since behaviour-named layouts (specs not mirroring the lib
|
|
22
|
+
# path, e.g. aasm's base.rb) never auto-resolve (EV-ajby / GH #1376).
|
|
23
|
+
def message(file_path, fallback_to_full_suite)
|
|
24
|
+
if fallback_to_full_suite
|
|
25
|
+
"[evilution] No matching spec found for #{file_path}, running full suite. " \
|
|
26
|
+
"Use --spec to specify the spec file."
|
|
27
|
+
else
|
|
28
|
+
"[evilution] No matching spec found for #{file_path}, marking mutation unresolved. " \
|
|
29
|
+
"Use --spec to specify the spec file, or --fallback-full-suite to run the whole suite."
|
|
30
|
+
end
|
|
17
31
|
end
|
|
18
32
|
end
|
|
@@ -41,8 +41,19 @@ class Evilution::Integration::TestUnit::TestFileResolver
|
|
|
41
41
|
return if @warned_files.include?(file_path)
|
|
42
42
|
|
|
43
43
|
@warned_files << file_path
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
44
|
+
warn unresolved_message(file_path)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Name both recovery paths when skipping :unresolved; when already falling
|
|
48
|
+
# back the suite is running, so only the explicit-spec hint applies.
|
|
49
|
+
# Behaviour-named layouts never auto-resolve (EV-ajby / GH #1376).
|
|
50
|
+
def unresolved_message(file_path)
|
|
51
|
+
if @fallback_to_full_suite
|
|
52
|
+
"[evilution] No matching test found for #{file_path}, running full suite. " \
|
|
53
|
+
"Use --spec to specify the test file."
|
|
54
|
+
else
|
|
55
|
+
"[evilution] No matching test found for #{file_path}, marking mutation unresolved. " \
|
|
56
|
+
"Use --spec to specify the test file, or --fallback-full-suite to run the whole suite."
|
|
57
|
+
end
|
|
47
58
|
end
|
|
48
59
|
end
|
|
@@ -31,14 +31,12 @@ class Evilution::RelatedSpecHeuristic
|
|
|
31
31
|
def extract_domain(file_path)
|
|
32
32
|
normalized = normalize_path(file_path)
|
|
33
33
|
|
|
34
|
-
# Strip common prefixes and get the relative path under app/ or lib/
|
|
35
34
|
relative = normalized
|
|
36
35
|
.delete_prefix("app/controllers/")
|
|
37
36
|
.delete_prefix("app/models/")
|
|
38
37
|
.delete_prefix("app/")
|
|
39
38
|
.delete_prefix("lib/")
|
|
40
39
|
|
|
41
|
-
# Remove .rb extension and _controller suffix
|
|
42
40
|
basename = relative.sub(/\.rb\z/, "")
|
|
43
41
|
basename = basename.sub(/_controller\z/, "")
|
|
44
42
|
|
|
@@ -76,13 +76,30 @@ class Evilution::Runner::Canary
|
|
|
76
76
|
end
|
|
77
77
|
|
|
78
78
|
def write_spec(dir)
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
path = File.join(dir, name)
|
|
82
|
-
File.write(path, minitest ? minitest_spec_source : rspec_spec_source)
|
|
79
|
+
path = File.join(dir, spec_filename)
|
|
80
|
+
File.write(path, spec_source)
|
|
83
81
|
path
|
|
84
82
|
end
|
|
85
83
|
|
|
84
|
+
# minitest and test-unit both live under a `_test.rb` file; rspec uses
|
|
85
|
+
# `_spec.rb`. Picking the wrong shape makes the integration load nothing (or
|
|
86
|
+
# raise), so the synthetic mutation scores :error and aborts the run.
|
|
87
|
+
def spec_filename
|
|
88
|
+
test_framework? ? "canary_#{suffix}_test.rb" : "canary_#{suffix}_spec.rb"
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def test_framework?
|
|
92
|
+
%i[minitest test_unit].include?(@config.integration)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def spec_source
|
|
96
|
+
case @config.integration
|
|
97
|
+
when :minitest then minitest_spec_source
|
|
98
|
+
when :test_unit then test_unit_spec_source
|
|
99
|
+
else rspec_spec_source
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
86
103
|
def rspec_spec_source
|
|
87
104
|
<<~RUBY
|
|
88
105
|
RSpec.describe("evilution proof-of-life canary") do
|
|
@@ -103,6 +120,16 @@ class Evilution::Runner::Canary
|
|
|
103
120
|
RUBY
|
|
104
121
|
end
|
|
105
122
|
|
|
123
|
+
def test_unit_spec_source
|
|
124
|
+
<<~RUBY
|
|
125
|
+
class EvilutionCanaryTest_#{suffix} < Test::Unit::TestCase
|
|
126
|
+
def test_pipeline_is_alive
|
|
127
|
+
assert(true)
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
RUBY
|
|
131
|
+
end
|
|
132
|
+
|
|
106
133
|
def build_mutation(class_path)
|
|
107
134
|
Evilution::Mutation.new(
|
|
108
135
|
subject: Evilution::Subject.new(
|
|
@@ -222,7 +222,21 @@ class Evilution::Runner::IsolationResolver
|
|
|
222
222
|
end
|
|
223
223
|
|
|
224
224
|
def find_first_existing_gem_helper
|
|
225
|
-
find_first_existing_under(detected_gem_root, GEM_PRELOAD_CANDIDATES)
|
|
225
|
+
find_first_existing_under(detected_gem_root, GEM_PRELOAD_CANDIDATES) ||
|
|
226
|
+
find_first_existing_under(detected_gem_root, nested_gem_helper_candidates)
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Gems that namespace their suite keep the helper at test/<gem>/helper.rb
|
|
230
|
+
# (ruby/csv: test/csv/helper.rb) rather than the flat test/helper.rb, so the
|
|
231
|
+
# conventional chain misses it and autodetect would fall back to the bare gem
|
|
232
|
+
# entry — which never sets up the test framework, aborting the canary.
|
|
233
|
+
# Derive the namespace dir from the gem name (dotted for dash-named gems, plus
|
|
234
|
+
# the flat form).
|
|
235
|
+
def nested_gem_helper_candidates
|
|
236
|
+
name = detected_gem_root && Evilution::GemDetector.gem_name(detected_gem_root, target_paths: target_files)
|
|
237
|
+
return [] unless name
|
|
238
|
+
|
|
239
|
+
[name.tr("-", "/"), name].uniq.map { |ns| File.join("test", ns, "helper.rb") }
|
|
226
240
|
end
|
|
227
241
|
|
|
228
242
|
def find_first_existing_under(root, candidates)
|
data/lib/evilution/runner.rb
CHANGED
|
@@ -161,7 +161,14 @@ class Evilution::SpecResolver
|
|
|
161
161
|
|
|
162
162
|
fallbacks = primary.flat_map { |c| parent_fallback_candidates(c) }
|
|
163
163
|
|
|
164
|
-
(primary + fallbacks +
|
|
164
|
+
(primary + fallbacks + convention_candidates(stripped)).uniq
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# Non-mirrored Test::Unit/minitest filename conventions: the `test_<name>.rb`
|
|
168
|
+
# prefix and the flat namespace-prefixed form. Both rank below the mirrored
|
|
169
|
+
# layouts assembled in #candidate_test_paths.
|
|
170
|
+
def convention_candidates(stripped)
|
|
171
|
+
prefix_convention_candidates(stripped) + flat_prefixed_candidates(stripped)
|
|
165
172
|
end
|
|
166
173
|
|
|
167
174
|
# Conventional roots that may hold tests: the mirrored root plus the common
|
|
@@ -204,6 +211,25 @@ class Evilution::SpecResolver
|
|
|
204
211
|
end
|
|
205
212
|
end
|
|
206
213
|
|
|
214
|
+
# Test::Unit / minitest gems sometimes flatten a nested source's path into a
|
|
215
|
+
# single `test_`-prefixed file at the test root: lib/connection_pool/timed_stack.rb
|
|
216
|
+
# -> test/test_connection_pool_timed_stack.rb (no test/connection_pool/ subdir).
|
|
217
|
+
# Built from the FULL lib-relative path only (every segment joined with `_`),
|
|
218
|
+
# NOT from namespace-dropped variants — dropping segments would yield bare
|
|
219
|
+
# basename forms (test_timed_stack.rb) that collide across namespaces and are
|
|
220
|
+
# already covered by #prefix_convention_candidates. Top-level sources have no
|
|
221
|
+
# namespace to flatten, so they produce nothing here. Ranked below the mirrored
|
|
222
|
+
# layouts (appended last) so a 1:1 file always wins. Minitest suffix only.
|
|
223
|
+
def flat_prefixed_candidates(stripped)
|
|
224
|
+
return [] unless @test_suffix == MINITEST_SUFFIX
|
|
225
|
+
return [] unless stripped.include?("/")
|
|
226
|
+
|
|
227
|
+
name = stripped.delete_suffix(@test_suffix).tr("/", "_")
|
|
228
|
+
return [] if name.empty?
|
|
229
|
+
|
|
230
|
+
roots.map { |root| "#{root}/test_#{name}.rb" }
|
|
231
|
+
end
|
|
232
|
+
|
|
207
233
|
def controller_to_request_test(stripped_path)
|
|
208
234
|
return nil unless stripped_path.start_with?(CONTROLLER_PREFIX)
|
|
209
235
|
|
data/lib/evilution/version.rb
CHANGED
data/lib/evilution.rb
CHANGED
|
@@ -129,6 +129,12 @@ require_relative "evilution/cli"
|
|
|
129
129
|
require_relative "evilution/disable_comment"
|
|
130
130
|
require_relative "evilution/runner"
|
|
131
131
|
|
|
132
|
+
# Evilution has no public Ruby API. The entire `Evilution::` namespace is
|
|
133
|
+
# internal and may change in any release, including patches. The stable,
|
|
134
|
+
# SemVer-governed contract is the CLI, `.evilution.yml` config keys, session
|
|
135
|
+
# JSON files, the MCP tool schemas, and process exit codes. See
|
|
136
|
+
# docs/public_api.md and docs/versioning.md.
|
|
137
|
+
# @api private
|
|
132
138
|
module Evilution
|
|
133
139
|
# Captured at load time, before any isolator can chdir into a per-mutation
|
|
134
140
|
# sandbox. Used as the anchor for resolving project-relative paths (spec
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: evilution
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 1.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Denis Kiselev
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-07-15 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: diff-lcs
|
|
@@ -99,14 +99,22 @@ files:
|
|
|
99
99
|
- LICENSE.txt
|
|
100
100
|
- README.md
|
|
101
101
|
- Rakefile
|
|
102
|
+
- assets/logo/evilution-mark-1024.png
|
|
103
|
+
- assets/logo/evilution-mark-64.png
|
|
104
|
+
- assets/logo/evilution-mark-transparent-1024.png
|
|
105
|
+
- assets/logo/evilution-mark-transparent.svg
|
|
106
|
+
- assets/logo/evilution-mark.svg
|
|
102
107
|
- claude-swarm.yml
|
|
103
108
|
- comparison_results/baseline_2026-04-09.md
|
|
104
109
|
- comparison_results/operator_classification.md
|
|
105
110
|
- comparison_results/operator_prioritization.md
|
|
111
|
+
- docs/architecture.md
|
|
106
112
|
- docs/ast_pattern_syntax.md
|
|
107
113
|
- docs/integrations.md
|
|
108
114
|
- docs/isolation.md
|
|
115
|
+
- docs/migration-from-mutant.md
|
|
109
116
|
- docs/mutation_density_benchmark.md
|
|
117
|
+
- docs/public_api.md
|
|
110
118
|
- docs/versioning.md
|
|
111
119
|
- exe/evil
|
|
112
120
|
- exe/evilution
|