evilution 0.24.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +210 -0
- data/.claude/prompts/architect.md +14 -1
- data/.claude/skills/create-issue/SKILL.md +55 -0
- data/CHANGELOG.md +51 -0
- data/README.md +80 -4
- data/exe/evil +6 -0
- data/lib/evilution/ast/constant_names.rb +34 -0
- data/lib/evilution/ast/source_surgeon.rb +15 -1
- data/lib/evilution/cli/commands/compare.rb +68 -0
- data/lib/evilution/cli/parser/command_extractor.rb +2 -1
- data/lib/evilution/cli/parser/options_builder.rb +21 -1
- data/lib/evilution/cli/printers/compare.rb +159 -0
- data/lib/evilution/cli.rb +1 -0
- data/lib/evilution/compare/categorizer.rb +109 -0
- data/lib/evilution/compare/detector.rb +21 -0
- data/lib/evilution/compare/fingerprint.rb +83 -0
- data/lib/evilution/compare/invalid_input.rb +12 -0
- data/lib/evilution/compare/normalizer.rb +106 -0
- data/lib/evilution/compare/record.rb +16 -0
- data/lib/evilution/compare.rb +6 -0
- data/lib/evilution/config.rb +165 -3
- data/lib/evilution/example_filter.rb +143 -0
- data/lib/evilution/integration/base.rb +4 -155
- data/lib/evilution/integration/crash_detector.rb +5 -2
- data/lib/evilution/integration/loading/concern_state_cleaner.rb +49 -0
- data/lib/evilution/integration/loading/constant_pinner.rb +24 -0
- data/lib/evilution/integration/loading/mutation_applier.rb +52 -0
- data/lib/evilution/integration/loading/redefinition_recovery.rb +54 -0
- data/lib/evilution/integration/loading/source_evaluator.rb +15 -0
- data/lib/evilution/integration/loading/syntax_validator.rb +19 -0
- data/lib/evilution/integration/loading.rb +6 -0
- data/lib/evilution/integration/minitest.rb +10 -5
- data/lib/evilution/integration/minitest_crash_detector.rb +5 -2
- data/lib/evilution/integration/rspec.rb +82 -7
- data/lib/evilution/isolation/fork.rb +25 -0
- data/lib/evilution/load_path/subpath_resolver.rb +25 -0
- data/lib/evilution/load_path.rb +4 -0
- data/lib/evilution/mcp/info_tool.rb +77 -5
- data/lib/evilution/mcp/mutate_tool/config_builder.rb +20 -0
- data/lib/evilution/mcp/mutate_tool/error_payload.rb +17 -0
- data/lib/evilution/mcp/mutate_tool/option_parser.rb +54 -0
- data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +37 -0
- data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +31 -0
- data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +52 -0
- data/lib/evilution/mcp/mutate_tool.rb +34 -186
- data/lib/evilution/mutation.rb +43 -3
- data/lib/evilution/mutator/base.rb +39 -1
- data/lib/evilution/mutator/operator/argument_nil_substitution.rb +5 -1
- data/lib/evilution/mutator/operator/argument_removal.rb +5 -1
- data/lib/evilution/parallel/work_queue.rb +149 -31
- data/lib/evilution/parallel_db_warning.rb +68 -0
- data/lib/evilution/reporter/cli.rb +37 -11
- data/lib/evilution/reporter/html/assets/style.css +17 -0
- data/lib/evilution/reporter/html/sections/file_section.rb +15 -0
- data/lib/evilution/reporter/html/sections/neutral_details.rb +25 -0
- data/lib/evilution/reporter/html/sections/unparseable_details.rb +25 -0
- data/lib/evilution/reporter/html/sections/unresolved_details.rb +25 -0
- data/lib/evilution/reporter/html/templates/file_section.html.erb +3 -0
- data/lib/evilution/reporter/html/templates/neutral_details.html.erb +14 -0
- data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
- data/lib/evilution/reporter/html/templates/unparseable_details.html.erb +11 -0
- data/lib/evilution/reporter/html/templates/unresolved_details.html.erb +11 -0
- data/lib/evilution/reporter/json.rb +8 -2
- data/lib/evilution/reporter/suggestion/diff_helpers.rb +28 -0
- data/lib/evilution/reporter/suggestion/registry.rb +64 -0
- data/lib/evilution/reporter/suggestion/templates/generic.rb +55 -0
- data/lib/evilution/reporter/suggestion/templates/minitest.rb +659 -0
- data/lib/evilution/reporter/suggestion/templates/rspec.rb +613 -0
- data/lib/evilution/reporter/suggestion.rb +8 -1327
- data/lib/evilution/result/mutation_result.rb +5 -1
- data/lib/evilution/result/summary.rb +13 -1
- data/lib/evilution/runner/baseline_runner.rb +23 -2
- data/lib/evilution/runner/isolation_resolver.rb +12 -1
- data/lib/evilution/runner/mutation_executor.rb +83 -13
- data/lib/evilution/runner/subject_pipeline.rb +18 -8
- data/lib/evilution/runner.rb +6 -0
- data/lib/evilution/source_ast_cache.rb +39 -0
- data/lib/evilution/spec_ast_cache.rb +166 -0
- data/lib/evilution/spec_resolver.rb +6 -1
- data/lib/evilution/spec_selector.rb +39 -0
- data/lib/evilution/temp_dir_tracker.rb +23 -3
- data/lib/evilution/version.rb +1 -1
- data/script/memory_check +7 -5
- metadata +46 -5
- data/lib/evilution/mcp/session_diff_tool.rb +0 -63
- data/lib/evilution/mcp/session_list_tool.rb +0 -50
- data/lib/evilution/mcp/session_show_tool.rb +0 -57
data/README.md
CHANGED
|
@@ -27,6 +27,8 @@ Or standalone: `gem install evilution`
|
|
|
27
27
|
evilution [command] [options] [files...]
|
|
28
28
|
```
|
|
29
29
|
|
|
30
|
+
The shorter alias `evil` ships alongside `evilution` and accepts identical arguments (handy with `alias be='bundle exec'` → `be evil run ...`).
|
|
31
|
+
|
|
30
32
|
### Commands
|
|
31
33
|
|
|
32
34
|
| Command | Description | Default |
|
|
@@ -42,6 +44,7 @@ evilution [command] [options] [files...]
|
|
|
42
44
|
| `session gc --older-than D` | Garbage-collect sessions older than D (e.g. 30d) | |
|
|
43
45
|
| `util mutation` | Preview mutations for a file or inline code | |
|
|
44
46
|
| `environment show` | Display runtime environment and settings | |
|
|
47
|
+
| `compare --against A --current B` | Compare two saved session JSON files into fixed / new / persistent / flaky / reintroduced buckets | |
|
|
45
48
|
|
|
46
49
|
### Options (for `run` command)
|
|
47
50
|
|
|
@@ -53,6 +56,9 @@ evilution [command] [options] [files...]
|
|
|
53
56
|
| `--min-score FLOAT` | Float | 0.0 | Minimum mutation score (0.0–1.0) to pass. |
|
|
54
57
|
| `--spec FILES` | Array | _(none)_ | Spec files to run (comma-separated). Defaults to auto-detection via `SpecResolver`. |
|
|
55
58
|
| `--spec-dir DIR` | String | _(none)_ | Include all `*_spec.rb` files in DIR recursively. Composable with `--spec`. |
|
|
59
|
+
| `--spec-pattern GLOB` | String | _(none)_ | Restrict resolved spec candidates to files matching GLOB (e.g. `spec/models/**/*_spec.rb`). |
|
|
60
|
+
| `--no-example-targeting` | Boolean | _(enabled)_ | Disable per-mutation example targeting (always run every example in the resolved spec file). Example targeting scans each example body for symbols from the mutated method and runs only the matching subset. |
|
|
61
|
+
| `--example-targeting-fallback MODE` | String | `full_file` | Behavior when no example matches: `full_file` (run the whole spec file) or `unresolved` (skip the mutation as `:unresolved`). |
|
|
56
62
|
| `-j`, `--jobs N` | Integer | 1 | Number of parallel workers. Uses demand-driven work distribution with pipe-based IPC. |
|
|
57
63
|
| `--no-baseline` | Boolean | _(enabled)_ | Skip baseline test suite check. By default, a baseline run detects pre-existing failures and marks those mutations as `neutral`. |
|
|
58
64
|
| `--fail-fast [N]` | Integer | _(none)_ | Stop after N surviving mutants (default 1 if no value given). |
|
|
@@ -101,6 +107,9 @@ Creates `.evilution.yml`:
|
|
|
101
107
|
# baseline_session: null # path to session file for HTML comparison
|
|
102
108
|
# ignore_patterns: [] # AST patterns to exclude (see docs/ast_pattern_syntax.md)
|
|
103
109
|
# progress: true # TTY progress bar
|
|
110
|
+
# example_targeting: true # per-mutation example targeting via body-token scan
|
|
111
|
+
# example_targeting_fallback: full_file # full_file | unresolved
|
|
112
|
+
# spec_pattern: null # restrict resolved spec candidates to files matching GLOB
|
|
104
113
|
```
|
|
105
114
|
|
|
106
115
|
**Precedence**: CLI flags override `.evilution.yml` values.
|
|
@@ -142,7 +151,11 @@ Use `--format json` for machine-readable output. Schema:
|
|
|
142
151
|
"survived": "integer — mutations NOT detected (test passed = gap in coverage)",
|
|
143
152
|
"timed_out": "integer — mutations that exceeded timeout",
|
|
144
153
|
"errors": "integer — mutations that caused unexpected errors",
|
|
145
|
-
"
|
|
154
|
+
"neutral": "integer — mutations whose tests already failed before mutation (baseline failure)",
|
|
155
|
+
"equivalent": "integer — mutations proven to have identical behavior to the original",
|
|
156
|
+
"unresolved": "integer — mutations where no spec file resolved (coverage gap, not a failure)",
|
|
157
|
+
"unparseable": "integer — mutations whose mutated source did not parse (short-circuited, never executed)",
|
|
158
|
+
"score": "float — killed / (total - errors - neutral - equivalent - unresolved - unparseable), range 0.0-1.0, rounded to 4 decimals",
|
|
146
159
|
"duration": "float — total wall-clock seconds, rounded to 4 decimals",
|
|
147
160
|
"peak_memory_mb": "float (optional) — peak RSS across all mutation child processes, in MB"
|
|
148
161
|
},
|
|
@@ -151,9 +164,10 @@ Use `--format json` for machine-readable output. Schema:
|
|
|
151
164
|
"operator": "string — mutation operator name (see Operators table)",
|
|
152
165
|
"file": "string — relative path to mutated file",
|
|
153
166
|
"line": "integer — line number of the mutation",
|
|
154
|
-
"status": "string — result status: 'survived', 'killed', 'timeout', or '
|
|
167
|
+
"status": "string — result status: 'survived', 'killed', 'timeout', 'error', 'neutral', 'equivalent', 'unresolved', or 'unparseable'",
|
|
155
168
|
"duration": "float — seconds this mutation took, rounded to 4 decimals",
|
|
156
|
-
"diff": "string —
|
|
169
|
+
"diff": "string — legacy +/- diff snippet",
|
|
170
|
+
"unified_diff": "string (optional, survived only) — git-style unified diff with `--- a/file`, `+++ b/file`, `@@` hunk header and sdiff body; omitted when source slices are unavailable",
|
|
157
171
|
"suggestion": "string — actionable hint for surviving mutants (survived only)"
|
|
158
172
|
}
|
|
159
173
|
],
|
|
@@ -168,6 +182,10 @@ Use `--format json` for machine-readable output. Schema:
|
|
|
168
182
|
}
|
|
169
183
|
],
|
|
170
184
|
"killed": ["... same shape as survived entries ..."],
|
|
185
|
+
"neutral": ["... same shape as survived entries ..."],
|
|
186
|
+
"equivalent": ["... same shape as survived entries ..."],
|
|
187
|
+
"unresolved": ["... same shape as survived entries — coverage gap: no spec file resolved for these mutations"],
|
|
188
|
+
"unparseable": ["... same shape as survived entries — mutated source failed to parse and was never executed"],
|
|
171
189
|
"timed_out": ["... same shape as survived entries ..."],
|
|
172
190
|
"errors": [
|
|
173
191
|
{
|
|
@@ -182,6 +200,21 @@ Use `--format json` for machine-readable output. Schema:
|
|
|
182
200
|
|
|
183
201
|
**Key metric**: `summary.score` — the mutation score. Higher is better. 1.0 means all mutations were caught.
|
|
184
202
|
|
|
203
|
+
### Mutation Statuses
|
|
204
|
+
|
|
205
|
+
| Status | Meaning | Counted in score? |
|
|
206
|
+
|--------------|-----------------------------------------------------------------------|-------------------|
|
|
207
|
+
| `killed` | A test failed when the mutation was applied — test suite caught it | numerator + denominator |
|
|
208
|
+
| `survived` | No test failed — gap in coverage | denominator only |
|
|
209
|
+
| `timeout` | Test run exceeded `--timeout` — treated like survived for scoring | denominator only |
|
|
210
|
+
| `error` | Mutation caused an unexpected error (syntax error, boot failure, etc.) | excluded from denominator |
|
|
211
|
+
| `neutral` | Baseline tests already failed before mutation — not a meaningful signal | excluded |
|
|
212
|
+
| `equivalent` | Mutation is provably identical to the original (e.g. no-op replacement) | excluded |
|
|
213
|
+
| `unresolved` | No spec file resolved for the mutated source — **coverage gap, not a failure**. Use `--fallback-full-suite` to run the full suite instead. | excluded |
|
|
214
|
+
| `unparseable` | Mutated source failed to parse (e.g. dangling heredoc opener after `method_body_replacement`). Short-circuited — never executed. | excluded |
|
|
215
|
+
|
|
216
|
+
Unresolved mutations indicate a missing test mapping — the file has no corresponding test file that the resolver could find (for example, an RSpec `_spec.rb` file or a Minitest `_test.rb` file, depending on configuration). They are reported separately so you can act on them (add a test, adjust test naming, or opt in to the full-suite fallback) without inflating the error count.
|
|
217
|
+
|
|
185
218
|
## Mutation Operators (72 total)
|
|
186
219
|
|
|
187
220
|
Each operator name is stable and appears in JSON output under `survived[].operator`.
|
|
@@ -314,7 +347,7 @@ Unlike `evilution --format json`, every survived entry returned by `evilution-mu
|
|
|
314
347
|
| `spec_file` | Resolved spec/test path (when one exists) — e.g. an RSpec spec file or Minitest test file, so you can drop new tests straight into it |
|
|
315
348
|
| `next_step` | Concrete natural-language hint — "add a test in X that fails against this mutation at Y:line" |
|
|
316
349
|
|
|
317
|
-
These fields are added in addition to the existing `operator`, `file`, `line`, `diff`, `suggestion`, and `test_command` so agents can triage survivors in one pass.
|
|
350
|
+
These fields are added in addition to the existing `operator`, `file`, `line`, `diff`, `unified_diff`, `suggestion`, and `test_command` so agents can triage survivors in one pass.
|
|
318
351
|
|
|
319
352
|
### Concrete Test Suggestions
|
|
320
353
|
|
|
@@ -419,6 +452,49 @@ bundle exec evilution run lib/ --format json --min-score 0.8 --quiet
|
|
|
419
452
|
|
|
420
453
|
Note: `--quiet` suppresses all stdout output (including JSON). Use it in CI only when you care about the exit code and do not need JSON output.
|
|
421
454
|
|
|
455
|
+
### 9. Regression tracking across runs (`compare`)
|
|
456
|
+
|
|
457
|
+
```bash
|
|
458
|
+
bundle exec evilution run lib/ --format json --save-session
|
|
459
|
+
# later, after edits:
|
|
460
|
+
bundle exec evilution run lib/ --format json --save-session
|
|
461
|
+
|
|
462
|
+
bundle exec evilution compare \
|
|
463
|
+
--against .evilution/results/<earlier>.json \
|
|
464
|
+
--current .evilution/results/<later>.json \
|
|
465
|
+
--format json
|
|
466
|
+
```
|
|
467
|
+
|
|
468
|
+
Output buckets:
|
|
469
|
+
|
|
470
|
+
| Bucket | Meaning |
|
|
471
|
+
|-----------------|---------|
|
|
472
|
+
| `fixed` | Survived previously, killed now — the new test actually landed |
|
|
473
|
+
| `new` | Did not exist previously, surviving now — a fresh gap |
|
|
474
|
+
| `persistent` | Survived in both runs — carry-over debt |
|
|
475
|
+
| `reintroduced` | Killed previously, survived now — regression |
|
|
476
|
+
| `flaky` | Status flipped and back — unstable test |
|
|
477
|
+
|
|
478
|
+
Use in CI to gate merges on `reintroduced` being empty, or to surface `new` survivors for reviewer attention without failing the build on `persistent` debt.
|
|
479
|
+
|
|
480
|
+
## Parallel Runs with SQLite
|
|
481
|
+
|
|
482
|
+
Running with `-j N` forks worker processes. If your Rails app uses SQLite, every worker opens the same `db/test.sqlite3` file, and concurrent writers collide on the database-level lock. Symptoms: `ActiveRecord::StatementTimeout`, `SQLite3::BusyException`, and slow runs. Evilution classifies these crashes as `:neutral` (see [EV-toid / #814](https://github.com/taxdome/evilution/issues/814)) so the mutation score is not polluted, but the wall-clock penalty remains.
|
|
483
|
+
|
|
484
|
+
Evilution follows the [`parallel_tests`](https://github.com/grosser/parallel_tests) convention: each worker receives a `TEST_ENV_NUMBER` environment variable (`""` for worker 1, `"2"` for worker 2, `"3"` for worker 3, …). Interpolate it into `config/database.yml` so each worker gets its own SQLite file:
|
|
485
|
+
|
|
486
|
+
```yaml
|
|
487
|
+
test:
|
|
488
|
+
adapter: sqlite3
|
|
489
|
+
database: db/test<%= ENV['TEST_ENV_NUMBER'] %>.sqlite3
|
|
490
|
+
pool: 5
|
|
491
|
+
timeout: 5000
|
|
492
|
+
```
|
|
493
|
+
|
|
494
|
+
After the first `jobs > 1` run, each worker creates its own file (`db/test.sqlite3`, `db/test2.sqlite3`, …). Seed each with `rake db:test:prepare` before running Evilution, or ensure your preload sets up schema on connect.
|
|
495
|
+
|
|
496
|
+
When Evilution detects a parallel run against a SQLite-backed `config/database.yml`, it prints a one-time startup warning pointing to this section.
|
|
497
|
+
|
|
422
498
|
## Development
|
|
423
499
|
|
|
424
500
|
### Memory leak check
|
data/exe/evil
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "prism"
|
|
4
|
+
require_relative "../ast"
|
|
5
|
+
|
|
6
|
+
# Walks a Prism AST and returns every class/module constant declared, nested
|
|
7
|
+
# names rendered fully-qualified (e.g. "Foo::Bar"). Order is source order:
|
|
8
|
+
# outer declarations precede their nested children.
|
|
9
|
+
class Evilution::AST::ConstantNames
|
|
10
|
+
def call(source)
|
|
11
|
+
result = Prism.parse(source)
|
|
12
|
+
return [] if result.failure?
|
|
13
|
+
|
|
14
|
+
collect(result.value)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
private
|
|
18
|
+
|
|
19
|
+
def collect(node, nesting = [])
|
|
20
|
+
names = []
|
|
21
|
+
case node
|
|
22
|
+
when Prism::ModuleNode, Prism::ClassNode
|
|
23
|
+
const = node.constant_path.full_name
|
|
24
|
+
qualified = nesting.any? && !const.include?("::") ? "#{nesting.join("::")}::#{const}" : const
|
|
25
|
+
names << qualified
|
|
26
|
+
names.concat(collect(node.body, nesting + [const])) if node.body
|
|
27
|
+
when Prism::ProgramNode
|
|
28
|
+
names.concat(collect(node.statements, nesting)) if node.statements
|
|
29
|
+
when Prism::StatementsNode
|
|
30
|
+
node.body.each { |child| names.concat(collect(child, nesting)) }
|
|
31
|
+
end
|
|
32
|
+
names
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -1,11 +1,25 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "prism"
|
|
4
|
+
|
|
3
5
|
require_relative "../ast"
|
|
4
6
|
|
|
5
7
|
module Evilution::AST::SourceSurgeon
|
|
8
|
+
Result = Struct.new(:source, :status, keyword_init: true) do
|
|
9
|
+
def ok?
|
|
10
|
+
status == :ok
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def unparseable?
|
|
14
|
+
status == :unparseable
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
6
18
|
def self.apply(source, offset:, length:, replacement:)
|
|
7
19
|
binary = source.b
|
|
8
20
|
binary[offset, length] = replacement.b
|
|
9
|
-
binary.force_encoding(source.encoding)
|
|
21
|
+
mutated = binary.force_encoding(source.encoding)
|
|
22
|
+
status = Prism.parse(mutated).success? ? :ok : :unparseable
|
|
23
|
+
Result.new(source: mutated, status: status).freeze
|
|
10
24
|
end
|
|
11
25
|
end
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "../commands"
|
|
5
|
+
require_relative "../command"
|
|
6
|
+
require_relative "../dispatcher"
|
|
7
|
+
require_relative "../printers/compare"
|
|
8
|
+
require_relative "../../compare"
|
|
9
|
+
require_relative "../../compare/categorizer"
|
|
10
|
+
require_relative "../../compare/detector"
|
|
11
|
+
require_relative "../../compare/normalizer"
|
|
12
|
+
|
|
13
|
+
class Evilution::CLI::Commands::Compare < Evilution::CLI::Command
|
|
14
|
+
SUPPORTED_FORMATS = %i[json text].freeze
|
|
15
|
+
|
|
16
|
+
private
|
|
17
|
+
|
|
18
|
+
def perform
|
|
19
|
+
paths = resolve_paths
|
|
20
|
+
raise Evilution::ConfigError, "exactly two file paths required for compare" unless paths.length == 2
|
|
21
|
+
|
|
22
|
+
fmt = @options[:format] || :json
|
|
23
|
+
raise Evilution::ConfigError, "compare supports --format text or json, got #{fmt.inspect}" unless SUPPORTED_FORMATS.include?(fmt)
|
|
24
|
+
|
|
25
|
+
against = load_and_normalize(paths[0])
|
|
26
|
+
current = load_and_normalize(paths[1])
|
|
27
|
+
buckets = Evilution::Compare::Categorizer.call(against, current)
|
|
28
|
+
Evilution::CLI::Printers::Compare.new(buckets, format: fmt).render(@stdout)
|
|
29
|
+
0
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Flags bind to roles (--against -> slot 0, --current -> slot 1);
|
|
33
|
+
# positional @files fill whatever role the flags didn't claim, in order.
|
|
34
|
+
# Extra positional args after both slots are filled are a user error.
|
|
35
|
+
def resolve_paths
|
|
36
|
+
positional = @files.dup
|
|
37
|
+
against = @options[:against] || positional.shift
|
|
38
|
+
current = @options[:current] || positional.shift
|
|
39
|
+
|
|
40
|
+
raise Evilution::ConfigError, "exactly two file paths required for compare" unless positional.empty?
|
|
41
|
+
|
|
42
|
+
[against, current].compact
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def load_and_normalize(path)
|
|
46
|
+
raise Evilution::Error, "file not found: #{path}" unless File.exist?(path)
|
|
47
|
+
|
|
48
|
+
json = JSON.parse(File.read(path))
|
|
49
|
+
tool = Evilution::Compare::Detector.call(json)
|
|
50
|
+
normalize(json, tool)
|
|
51
|
+
rescue ::JSON::ParserError => e
|
|
52
|
+
raise Evilution::Error, "invalid JSON in #{path}: #{e.message}"
|
|
53
|
+
rescue Evilution::Compare::InvalidInput => e
|
|
54
|
+
raise Evilution::Error, "#{path}: #{e.message}"
|
|
55
|
+
rescue SystemCallError => e
|
|
56
|
+
raise Evilution::Error, e.message
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def normalize(json, tool)
|
|
60
|
+
normalizer = Evilution::Compare::Normalizer.new
|
|
61
|
+
case tool
|
|
62
|
+
when :mutant then normalizer.from_mutant(json)
|
|
63
|
+
when :evilution then normalizer.from_evilution(json)
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
Evilution::CLI::Dispatcher.register(:compare, Evilution::CLI::Commands::Compare)
|
|
@@ -23,6 +23,7 @@ class Evilution::CLI::Parser::OptionsBuilder
|
|
|
23
23
|
add_flag_options(opts)
|
|
24
24
|
add_extra_flag_options(opts)
|
|
25
25
|
add_session_options(opts)
|
|
26
|
+
add_compare_options(opts)
|
|
26
27
|
end
|
|
27
28
|
end
|
|
28
29
|
|
|
@@ -33,7 +34,7 @@ class Evilution::CLI::Parser::OptionsBuilder
|
|
|
33
34
|
opts.separator "Line-range targeting: lib/foo.rb:15-30, lib/foo.rb:15, lib/foo.rb:15-"
|
|
34
35
|
opts.separator ""
|
|
35
36
|
opts.separator "Commands: run (default), init, session {list,show,diff,gc}, subjects, tests {list},"
|
|
36
|
-
opts.separator " util {mutation}, environment {show}, mcp, version"
|
|
37
|
+
opts.separator " util {mutation}, environment {show}, compare, mcp, version"
|
|
37
38
|
opts.separator ""
|
|
38
39
|
opts.separator "Options:"
|
|
39
40
|
end
|
|
@@ -48,6 +49,16 @@ class Evilution::CLI::Parser::OptionsBuilder
|
|
|
48
49
|
opts.on("--min-score FLOAT", Float, "Minimum mutation score to pass") { |s| @options[:min_score] = s }
|
|
49
50
|
opts.on("--spec FILES", Array, "Spec files to run (comma-separated)") { |f| @options[:spec_files] = f }
|
|
50
51
|
opts.on("--spec-dir DIR", "Include all specs in DIR") { |d| expand_spec_dir(d) }
|
|
52
|
+
opts.on("--spec-pattern GLOB",
|
|
53
|
+
"Restrict resolved spec candidates to files matching GLOB") { |p| @options[:spec_pattern] = p }
|
|
54
|
+
opts.on("--no-example-targeting",
|
|
55
|
+
"Disable per-mutation example targeting (run all examples in resolved spec files)") do
|
|
56
|
+
@options[:example_targeting] = false
|
|
57
|
+
end
|
|
58
|
+
opts.on("--example-targeting-fallback MODE", %w[full_file unresolved],
|
|
59
|
+
"Fallback when example targeting finds no match: full_file (default) or unresolved") do |m|
|
|
60
|
+
@options[:example_targeting_fallback] = m
|
|
61
|
+
end
|
|
51
62
|
opts.on("--target EXPR",
|
|
52
63
|
"Filter: method (Foo#bar), type (Foo#/Foo.), namespace (Foo*),",
|
|
53
64
|
"class (Foo), glob (source:**/*.rb), hierarchy (descendants:Foo)") do |m|
|
|
@@ -96,6 +107,15 @@ class Evilution::CLI::Parser::OptionsBuilder
|
|
|
96
107
|
end
|
|
97
108
|
end
|
|
98
109
|
|
|
110
|
+
def add_compare_options(opts)
|
|
111
|
+
opts.on("--against PATH", "Prior mutation run to compare against (used with `compare` command)") do |p|
|
|
112
|
+
@options[:against] = p
|
|
113
|
+
end
|
|
114
|
+
opts.on("--current PATH", "Current mutation run to compare (used with `compare` command)") do |p|
|
|
115
|
+
@options[:current] = p
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
99
119
|
def expand_spec_dir(dir)
|
|
100
120
|
specs = Evilution::CLI::Parser::FileArgs.expand_spec_dir(dir)
|
|
101
121
|
@options[:spec_files] = Array(@options[:spec_files]) + specs
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "../printers"
|
|
5
|
+
|
|
6
|
+
class Evilution::CLI::Printers::Compare
|
|
7
|
+
SCHEMA = {
|
|
8
|
+
"shared" => %w[file line operator fp],
|
|
9
|
+
"alive_only" => %w[file line operator fp other_status]
|
|
10
|
+
}.freeze
|
|
11
|
+
|
|
12
|
+
FILE_LINE_WIDTH = 40
|
|
13
|
+
OPERATOR_WIDTH = 22
|
|
14
|
+
FP_LENGTH = 7
|
|
15
|
+
MUTANT_OPERATOR = "(mutant)"
|
|
16
|
+
ABSENT_STATUS = "absent"
|
|
17
|
+
|
|
18
|
+
def initialize(buckets, format: :json)
|
|
19
|
+
@buckets = buckets
|
|
20
|
+
@format = format || :json
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def render(io)
|
|
24
|
+
case @format
|
|
25
|
+
when :json then render_json(io)
|
|
26
|
+
when :text then render_text(io)
|
|
27
|
+
else raise Evilution::Error, "unknown compare format: #{@format.inspect}"
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def render_json(io)
|
|
34
|
+
payload = {
|
|
35
|
+
"schema" => SCHEMA,
|
|
36
|
+
"summary" => summary_hash,
|
|
37
|
+
"alive_only_against" => @buckets[:alive_only_against].map { |e| alive_entry_array(e) },
|
|
38
|
+
"alive_only_current" => @buckets[:alive_only_current].map { |e| alive_entry_array(e) },
|
|
39
|
+
"shared_alive" => @buckets[:shared_alive].map { |e| shared_entry_array(e) },
|
|
40
|
+
"shared_dead" => @buckets[:shared_dead].map { |e| shared_entry_array(e) }
|
|
41
|
+
}
|
|
42
|
+
io.puts(JSON.generate(payload))
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def render_text(io)
|
|
46
|
+
io.puts("Compare results")
|
|
47
|
+
io.puts("-" * 15)
|
|
48
|
+
io.puts(summary_line)
|
|
49
|
+
|
|
50
|
+
if fully_empty?
|
|
51
|
+
io.puts("No mutations to compare.")
|
|
52
|
+
return
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
print_alive_block(io, :alive_only_against, "current")
|
|
56
|
+
print_alive_block(io, :alive_only_current, "against")
|
|
57
|
+
print_shared_block(io, :shared_alive)
|
|
58
|
+
print_shared_block(io, :shared_dead)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def summary_hash
|
|
62
|
+
against_count = @buckets[:alive_only_against].length
|
|
63
|
+
current_count = @buckets[:alive_only_current].length
|
|
64
|
+
{
|
|
65
|
+
"alive_only_against" => against_count,
|
|
66
|
+
"alive_only_current" => current_count,
|
|
67
|
+
"shared_alive" => @buckets[:shared_alive].length,
|
|
68
|
+
"shared_dead" => @buckets[:shared_dead].length,
|
|
69
|
+
"excluded_against" => @buckets[:excluded_against],
|
|
70
|
+
"excluded_current" => @buckets[:excluded_current],
|
|
71
|
+
"delta" => current_count - against_count
|
|
72
|
+
}
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def summary_line
|
|
76
|
+
s = summary_hash
|
|
77
|
+
parts = [
|
|
78
|
+
"summary:",
|
|
79
|
+
"alive_only_against=#{s["alive_only_against"]}",
|
|
80
|
+
"alive_only_current=#{s["alive_only_current"]}",
|
|
81
|
+
"shared_alive=#{s["shared_alive"]}",
|
|
82
|
+
"shared_dead=#{s["shared_dead"]}",
|
|
83
|
+
"excluded=#{s["excluded_against"]}/#{s["excluded_current"]}",
|
|
84
|
+
"delta=#{format_delta(s["delta"])}"
|
|
85
|
+
]
|
|
86
|
+
parts.join(" ")
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def format_delta(delta)
|
|
90
|
+
return "\u00B10" if delta.zero?
|
|
91
|
+
|
|
92
|
+
format("%+d", delta)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def fully_empty?
|
|
96
|
+
@buckets[:alive_only_against].empty? &&
|
|
97
|
+
@buckets[:alive_only_current].empty? &&
|
|
98
|
+
@buckets[:shared_alive].empty? &&
|
|
99
|
+
@buckets[:shared_dead].empty? &&
|
|
100
|
+
@buckets[:excluded_against].zero? &&
|
|
101
|
+
@buckets[:excluded_current].zero?
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def alive_entry_array(entry)
|
|
105
|
+
r = entry[:record]
|
|
106
|
+
peer = entry[:peer_status]
|
|
107
|
+
peer_str = peer.nil? ? ABSENT_STATUS : peer.to_s
|
|
108
|
+
[r.file_path, r.line, r.operator, r.fingerprint, peer_str]
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def shared_entry_array(entry)
|
|
112
|
+
r = entry[:against]
|
|
113
|
+
[r.file_path, r.line, shared_operator(entry), r.fingerprint]
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Mutant-sourced records always have operator=nil. When comparing mutant
|
|
117
|
+
# vs evilution, prefer whichever side has an operator so the shared row
|
|
118
|
+
# stays informative.
|
|
119
|
+
def shared_operator(entry)
|
|
120
|
+
entry[:against].operator || entry[:current].operator
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def print_alive_block(io, bucket_key, peer_side_label)
|
|
124
|
+
entries = @buckets[bucket_key]
|
|
125
|
+
return if entries.empty?
|
|
126
|
+
|
|
127
|
+
io.puts("")
|
|
128
|
+
io.puts("#{bucket_key} (#{entries.length}):")
|
|
129
|
+
entries.each { |entry| io.puts(format_alive_row(entry, peer_side_label)) }
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def print_shared_block(io, bucket_key)
|
|
133
|
+
entries = @buckets[bucket_key]
|
|
134
|
+
return if entries.empty?
|
|
135
|
+
|
|
136
|
+
io.puts("")
|
|
137
|
+
io.puts("#{bucket_key} (#{entries.length}):")
|
|
138
|
+
entries.each { |entry| io.puts(format_shared_row(entry)) }
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def format_alive_row(entry, peer_side_label)
|
|
142
|
+
r = entry[:record]
|
|
143
|
+
peer = entry[:peer_status]
|
|
144
|
+
peer_str = peer.nil? ? ABSENT_STATUS : peer.to_s
|
|
145
|
+
" #{row_prefix(r)} (#{peer_side_label}: #{peer_str})"
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def format_shared_row(entry)
|
|
149
|
+
r = entry[:against]
|
|
150
|
+
" #{row_prefix(r, operator: shared_operator(entry))}"
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def row_prefix(record, operator: record.operator)
|
|
154
|
+
file_line = "#{record.file_path}:#{record.line}"
|
|
155
|
+
op_label = operator || MUTANT_OPERATOR
|
|
156
|
+
fp = record.fingerprint.to_s[0, FP_LENGTH]
|
|
157
|
+
"#{file_line.ljust(FILE_LINE_WIDTH)}#{op_label.ljust(OPERATOR_WIDTH)}#{fp}"
|
|
158
|
+
end
|
|
159
|
+
end
|
data/lib/evilution/cli.rb
CHANGED
|
@@ -16,6 +16,7 @@ require_relative "cli/commands/session_list"
|
|
|
16
16
|
require_relative "cli/commands/session_show"
|
|
17
17
|
require_relative "cli/commands/session_diff"
|
|
18
18
|
require_relative "cli/commands/session_gc"
|
|
19
|
+
require_relative "cli/commands/compare"
|
|
19
20
|
require_relative "cli/commands/run"
|
|
20
21
|
|
|
21
22
|
class Evilution::CLI
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../compare"
|
|
4
|
+
require_relative "record"
|
|
5
|
+
|
|
6
|
+
module Evilution::Compare::Categorizer
|
|
7
|
+
ALIVE = %i[survived].freeze
|
|
8
|
+
DEAD = %i[killed timeout error].freeze
|
|
9
|
+
# neutral, equivalent, unresolved, unparseable are non-actionable signals
|
|
10
|
+
# — excluded from alive/dead buckets, counted in summary.
|
|
11
|
+
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
# @param against [Array<Record>] prior run (baseline)
|
|
15
|
+
# @param current [Array<Record>] current run
|
|
16
|
+
# @return [Hash] bucketed comparison result with keys:
|
|
17
|
+
# - `:alive_only_against` => `Array<{record: Record, peer_status: Symbol|nil}>`
|
|
18
|
+
# records that survived in against but not in current (or absent in current).
|
|
19
|
+
# `peer_status` is the current-side record's status symbol, or `nil` when
|
|
20
|
+
# no current-side record exists for that fingerprint.
|
|
21
|
+
# - `:alive_only_current` => `Array<{record: Record, peer_status: Symbol|nil}>`
|
|
22
|
+
# mirror of the above from the current side.
|
|
23
|
+
# - `:shared_alive` => `Array<{against: Record, current: Record}>`
|
|
24
|
+
# mutations that survived in both runs.
|
|
25
|
+
# - `:shared_dead` => `Array<{against: Record, current: Record}>`
|
|
26
|
+
# mutations killed/timed-out/errored in both runs.
|
|
27
|
+
# - `:excluded_against` => `Integer`
|
|
28
|
+
# count of against records with non-actionable statuses (neutral,
|
|
29
|
+
# equivalent, unresolved, unparseable).
|
|
30
|
+
# - `:excluded_current` => `Integer` mirror for the current side.
|
|
31
|
+
def call(against, current)
|
|
32
|
+
# Duplicate fingerprints within one side should not happen (Normalizer
|
|
33
|
+
# invariant). If they do, last write wins — we do not dedupe proactively.
|
|
34
|
+
against_by_fp = index_by_fingerprint(against)
|
|
35
|
+
current_by_fp = index_by_fingerprint(current)
|
|
36
|
+
|
|
37
|
+
buckets = {
|
|
38
|
+
alive_only_against: [],
|
|
39
|
+
alive_only_current: [],
|
|
40
|
+
shared_alive: [],
|
|
41
|
+
shared_dead: [],
|
|
42
|
+
excluded_against: 0,
|
|
43
|
+
excluded_current: 0
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
(against_by_fp.keys | current_by_fp.keys).each do |fp|
|
|
47
|
+
classify(against_by_fp[fp], current_by_fp[fp], buckets)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
sort_buckets!(buckets)
|
|
51
|
+
buckets
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Dispatches one fingerprint pair into buckets.
|
|
55
|
+
# Either record may be nil (fingerprint present on only one side).
|
|
56
|
+
def classify(against_record, current_record, buckets)
|
|
57
|
+
count_excluded(against_record, current_record, buckets)
|
|
58
|
+
a_kind = kind_of(against_record)
|
|
59
|
+
c_kind = kind_of(current_record)
|
|
60
|
+
|
|
61
|
+
if a_kind == :alive && c_kind == :alive
|
|
62
|
+
buckets[:shared_alive] << { against: against_record, current: current_record }
|
|
63
|
+
elsif a_kind == :dead && c_kind == :dead
|
|
64
|
+
buckets[:shared_dead] << { against: against_record, current: current_record }
|
|
65
|
+
else
|
|
66
|
+
bucket_single_sided(against_record, current_record, a_kind, c_kind, buckets)
|
|
67
|
+
end
|
|
68
|
+
# A dead-only fingerprint (dead on one side, absent on the other) is
|
|
69
|
+
# intentionally not bucketed and not counted as excluded.
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def count_excluded(against_record, current_record, buckets)
|
|
73
|
+
buckets[:excluded_against] += 1 if against_record && kind_of(against_record) == :excluded
|
|
74
|
+
buckets[:excluded_current] += 1 if current_record && kind_of(current_record) == :excluded
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def bucket_single_sided(against_record, current_record, a_kind, c_kind, buckets)
|
|
78
|
+
# peer_status is the peer record's status symbol, or nil if peer absent.
|
|
79
|
+
# When the peer is excluded, its status symbol (e.g. :neutral) flows through.
|
|
80
|
+
a_peer = current_record && current_record.status
|
|
81
|
+
c_peer = against_record && against_record.status
|
|
82
|
+
buckets[:alive_only_against] << { record: against_record, peer_status: a_peer } if a_kind == :alive
|
|
83
|
+
buckets[:alive_only_current] << { record: current_record, peer_status: c_peer } if c_kind == :alive
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Returns :alive, :dead, :excluded, or nil (for nil records).
|
|
87
|
+
def kind_of(record)
|
|
88
|
+
return nil if record.nil?
|
|
89
|
+
return :alive if ALIVE.include?(record.status)
|
|
90
|
+
return :dead if DEAD.include?(record.status)
|
|
91
|
+
|
|
92
|
+
:excluded
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def sort_buckets!(buckets)
|
|
96
|
+
buckets[:alive_only_against].sort_by! { |e| sort_key(e[:record]) }
|
|
97
|
+
buckets[:alive_only_current].sort_by! { |e| sort_key(e[:record]) }
|
|
98
|
+
buckets[:shared_alive].sort_by! { |e| sort_key(e[:against]) }
|
|
99
|
+
buckets[:shared_dead].sort_by! { |e| sort_key(e[:against]) }
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def sort_key(record)
|
|
103
|
+
[record.file_path, record.line, record.fingerprint]
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def index_by_fingerprint(records)
|
|
107
|
+
records.to_h { |r| [r.fingerprint, r] }
|
|
108
|
+
end
|
|
109
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../compare"
|
|
4
|
+
require_relative "normalizer"
|
|
5
|
+
|
|
6
|
+
module Evilution::Compare::Detector
|
|
7
|
+
module_function
|
|
8
|
+
|
|
9
|
+
def call(json)
|
|
10
|
+
raise Evilution::Compare::InvalidInput, "expected Hash, got #{json.class}" unless json.is_a?(Hash)
|
|
11
|
+
|
|
12
|
+
mutant = json.key?("subject_results")
|
|
13
|
+
evilution = json.key?("summary") && Evilution::Compare::Normalizer::EVILUTION_BUCKETS.any? { |k| json.key?(k) }
|
|
14
|
+
|
|
15
|
+
raise Evilution::Compare::InvalidInput, "ambiguous JSON shape - both mutant and evilution markers present" if mutant && evilution
|
|
16
|
+
return :mutant if mutant
|
|
17
|
+
return :evilution if evilution
|
|
18
|
+
|
|
19
|
+
raise Evilution::Compare::InvalidInput, "cannot detect tool from JSON shape"
|
|
20
|
+
end
|
|
21
|
+
end
|