evilution 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +210 -0
- data/CHANGELOG.md +51 -0
- data/README.md +81 -4
- data/exe/evil +6 -0
- data/lib/evilution/ast/source_surgeon.rb +15 -1
- data/lib/evilution/cli/commands/compare.rb +68 -0
- data/lib/evilution/cli/parser/command_extractor.rb +78 -0
- data/lib/evilution/cli/parser/file_args.rb +41 -0
- data/lib/evilution/cli/parser/options_builder.rb +123 -0
- data/lib/evilution/cli/parser/stdin_reader.rb +28 -0
- data/lib/evilution/cli/parser.rb +27 -196
- data/lib/evilution/cli/printers/compare.rb +159 -0
- data/lib/evilution/cli.rb +1 -0
- data/lib/evilution/compare/categorizer.rb +109 -0
- data/lib/evilution/compare/detector.rb +21 -0
- data/lib/evilution/compare/fingerprint.rb +83 -0
- data/lib/evilution/compare/normalizer.rb +106 -0
- data/lib/evilution/compare/record.rb +16 -0
- data/lib/evilution/compare.rb +15 -0
- data/lib/evilution/config.rb +178 -3
- data/lib/evilution/example_filter.rb +143 -0
- data/lib/evilution/integration/base.rb +11 -57
- data/lib/evilution/integration/crash_detector.rb +5 -2
- data/lib/evilution/integration/minitest.rb +25 -7
- data/lib/evilution/integration/minitest_crash_detector.rb +5 -2
- data/lib/evilution/integration/rspec.rb +99 -12
- data/lib/evilution/isolation/fork.rb +26 -0
- data/lib/evilution/isolation/in_process.rb +1 -0
- data/lib/evilution/mcp/info_tool.rb +77 -5
- data/lib/evilution/mcp/mutate_tool/config_builder.rb +20 -0
- data/lib/evilution/mcp/mutate_tool/error_payload.rb +17 -0
- data/lib/evilution/mcp/mutate_tool/option_parser.rb +54 -0
- data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +37 -0
- data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +31 -0
- data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +52 -0
- data/lib/evilution/mcp/mutate_tool.rb +34 -186
- data/lib/evilution/mutation.rb +43 -3
- data/lib/evilution/mutator/base.rb +39 -1
- data/lib/evilution/mutator/operator/argument_nil_substitution.rb +5 -1
- data/lib/evilution/mutator/operator/argument_removal.rb +5 -1
- data/lib/evilution/parallel/work_queue.rb +149 -31
- data/lib/evilution/parallel_db_warning.rb +68 -0
- data/lib/evilution/reporter/cli.rb +38 -11
- data/lib/evilution/reporter/html/assets/style.css +85 -0
- data/lib/evilution/reporter/html/baseline_keys.rb +28 -0
- data/lib/evilution/reporter/html/diff_formatter.rb +27 -0
- data/lib/evilution/reporter/html/escape.rb +12 -0
- data/lib/evilution/reporter/html/namespace.rb +11 -0
- data/lib/evilution/reporter/html/report.rb +68 -0
- data/lib/evilution/reporter/html/section.rb +21 -0
- data/lib/evilution/reporter/html/sections/baseline_comparison.rb +46 -0
- data/lib/evilution/reporter/html/sections/error_details.rb +30 -0
- data/lib/evilution/reporter/html/sections/error_entry.rb +22 -0
- data/lib/evilution/reporter/html/sections/file_section.rb +62 -0
- data/lib/evilution/reporter/html/sections/header.rb +29 -0
- data/lib/evilution/reporter/html/sections/mutation_map.rb +32 -0
- data/lib/evilution/reporter/html/sections/neutral_details.rb +25 -0
- data/lib/evilution/reporter/html/sections/summary_cards.rb +11 -0
- data/lib/evilution/reporter/html/sections/survived_details.rb +35 -0
- data/lib/evilution/reporter/html/sections/survived_entry.rb +36 -0
- data/lib/evilution/reporter/html/sections/truncation_notice.rb +17 -0
- data/lib/evilution/reporter/html/sections/unparseable_details.rb +25 -0
- data/lib/evilution/reporter/html/sections/unresolved_details.rb +25 -0
- data/lib/evilution/reporter/html/sections.rb +4 -0
- data/lib/evilution/reporter/html/stylesheet.rb +14 -0
- data/lib/evilution/reporter/html/templates/baseline_comparison.html.erb +8 -0
- data/lib/evilution/reporter/html/templates/error_details.html.erb +6 -0
- data/lib/evilution/reporter/html/templates/error_entry.html.erb +10 -0
- data/lib/evilution/reporter/html/templates/file_section.html.erb +12 -0
- data/lib/evilution/reporter/html/templates/header.html.erb +4 -0
- data/lib/evilution/reporter/html/templates/mutation_map.html.erb +6 -0
- data/lib/evilution/reporter/html/templates/neutral_details.html.erb +14 -0
- data/lib/evilution/reporter/html/templates/report.html.erb +17 -0
- data/lib/evilution/reporter/html/templates/summary_cards.html.erb +26 -0
- data/lib/evilution/reporter/html/templates/survived_details.html.erb +21 -0
- data/lib/evilution/reporter/html/templates/survived_entry.html.erb +8 -0
- data/lib/evilution/reporter/html/templates/truncation_notice.html.erb +1 -0
- data/lib/evilution/reporter/html/templates/unparseable_details.html.erb +11 -0
- data/lib/evilution/reporter/html/templates/unresolved_details.html.erb +11 -0
- data/lib/evilution/reporter/html.rb +11 -390
- data/lib/evilution/reporter/json.rb +19 -9
- data/lib/evilution/reporter/suggestion/diff_helpers.rb +28 -0
- data/lib/evilution/reporter/suggestion/registry.rb +64 -0
- data/lib/evilution/reporter/suggestion/templates/generic.rb +55 -0
- data/lib/evilution/reporter/suggestion/templates/minitest.rb +659 -0
- data/lib/evilution/reporter/suggestion/templates/rspec.rb +613 -0
- data/lib/evilution/reporter/suggestion.rb +8 -1327
- data/lib/evilution/result/mutation_result.rb +9 -1
- data/lib/evilution/result/summary.rb +21 -1
- data/lib/evilution/runner/baseline_runner.rb +92 -0
- data/lib/evilution/runner/diagnostics.rb +105 -0
- data/lib/evilution/runner/isolation_resolver.rb +134 -0
- data/lib/evilution/runner/mutation_executor.rb +325 -0
- data/lib/evilution/runner/mutation_planner.rb +126 -0
- data/lib/evilution/runner/report_publisher.rb +60 -0
- data/lib/evilution/runner/subject_pipeline.rb +121 -0
- data/lib/evilution/runner.rb +61 -692
- data/lib/evilution/source_ast_cache.rb +39 -0
- data/lib/evilution/spec_ast_cache.rb +166 -0
- data/lib/evilution/spec_resolver.rb +6 -1
- data/lib/evilution/spec_selector.rb +39 -0
- data/lib/evilution/temp_dir_tracker.rb +23 -3
- data/lib/evilution/version.rb +1 -1
- data/script/memory_check +7 -5
- metadata +75 -2
data/README.md
CHANGED
|
@@ -27,6 +27,8 @@ Or standalone: `gem install evilution`
|
|
|
27
27
|
evilution [command] [options] [files...]
|
|
28
28
|
```
|
|
29
29
|
|
|
30
|
+
The shorter alias `evil` ships alongside `evilution` and accepts identical arguments (handy with `alias be='bundle exec'` → `be evil run ...`).
|
|
31
|
+
|
|
30
32
|
### Commands
|
|
31
33
|
|
|
32
34
|
| Command | Description | Default |
|
|
@@ -42,6 +44,7 @@ evilution [command] [options] [files...]
|
|
|
42
44
|
| `session gc --older-than D` | Garbage-collect sessions older than D (e.g. 30d) | |
|
|
43
45
|
| `util mutation` | Preview mutations for a file or inline code | |
|
|
44
46
|
| `environment show` | Display runtime environment and settings | |
|
|
47
|
+
| `compare --against A --current B` | Compare two saved session JSON files into fixed / new / persistent / flaky / reintroduced buckets | |
|
|
45
48
|
|
|
46
49
|
### Options (for `run` command)
|
|
47
50
|
|
|
@@ -53,6 +56,9 @@ evilution [command] [options] [files...]
|
|
|
53
56
|
| `--min-score FLOAT` | Float | 0.0 | Minimum mutation score (0.0–1.0) to pass. |
|
|
54
57
|
| `--spec FILES` | Array | _(none)_ | Spec files to run (comma-separated). Defaults to auto-detection via `SpecResolver`. |
|
|
55
58
|
| `--spec-dir DIR` | String | _(none)_ | Include all `*_spec.rb` files in DIR recursively. Composable with `--spec`. |
|
|
59
|
+
| `--spec-pattern GLOB` | String | _(none)_ | Restrict resolved spec candidates to files matching GLOB (e.g. `spec/models/**/*_spec.rb`). |
|
|
60
|
+
| `--no-example-targeting` | Boolean | _(enabled)_ | Disable per-mutation example targeting (always run every example in the resolved spec file). Example targeting scans each example body for symbols from the mutated method and runs only the matching subset. |
|
|
61
|
+
| `--example-targeting-fallback MODE` | String | `full_file` | Behavior when no example matches: `full_file` (run the whole spec file) or `unresolved` (skip the mutation as `:unresolved`). |
|
|
56
62
|
| `-j`, `--jobs N` | Integer | 1 | Number of parallel workers. Uses demand-driven work distribution with pipe-based IPC. |
|
|
57
63
|
| `--no-baseline` | Boolean | _(enabled)_ | Skip baseline test suite check. By default, a baseline run detects pre-existing failures and marks those mutations as `neutral`. |
|
|
58
64
|
| `--fail-fast [N]` | Integer | _(none)_ | Stop after N surviving mutants (default 1 if no value given). |
|
|
@@ -69,6 +75,7 @@ evilution [command] [options] [files...]
|
|
|
69
75
|
| `--no-preload` | Boolean | _(enabled)_ | Disable parent-process preload. |
|
|
70
76
|
| `--skip-heredoc-literals` | Boolean | false | Skip all string literal mutations inside heredocs. |
|
|
71
77
|
| `--show-disabled` | Boolean | false | Report mutations skipped by `# evilution:disable` comments. |
|
|
78
|
+
| `--fallback-full-suite` | Boolean | false | When no matching spec/test resolves for a mutation, run the whole test suite instead of marking it `:unresolved` and skipping. |
|
|
72
79
|
| `--baseline-session PATH` | String | _(none)_ | Saved session file for HTML report comparison. |
|
|
73
80
|
| `-e CODE`, `--eval CODE` | String | _(none)_ | Inline Ruby code for `util mutation` command. |
|
|
74
81
|
|
|
@@ -100,6 +107,9 @@ Creates `.evilution.yml`:
|
|
|
100
107
|
# baseline_session: null # path to session file for HTML comparison
|
|
101
108
|
# ignore_patterns: [] # AST patterns to exclude (see docs/ast_pattern_syntax.md)
|
|
102
109
|
# progress: true # TTY progress bar
|
|
110
|
+
# example_targeting: true # per-mutation example targeting via body-token scan
|
|
111
|
+
# example_targeting_fallback: full_file # full_file | unresolved
|
|
112
|
+
# spec_pattern: null # restrict resolved spec candidates to files matching GLOB
|
|
103
113
|
```
|
|
104
114
|
|
|
105
115
|
**Precedence**: CLI flags override `.evilution.yml` values.
|
|
@@ -141,7 +151,11 @@ Use `--format json` for machine-readable output. Schema:
|
|
|
141
151
|
"survived": "integer — mutations NOT detected (test passed = gap in coverage)",
|
|
142
152
|
"timed_out": "integer — mutations that exceeded timeout",
|
|
143
153
|
"errors": "integer — mutations that caused unexpected errors",
|
|
144
|
-
"
|
|
154
|
+
"neutral": "integer — mutations whose tests already failed before mutation (baseline failure)",
|
|
155
|
+
"equivalent": "integer — mutations proven to have identical behavior to the original",
|
|
156
|
+
"unresolved": "integer — mutations where no spec file resolved (coverage gap, not a failure)",
|
|
157
|
+
"unparseable": "integer — mutations whose mutated source did not parse (short-circuited, never executed)",
|
|
158
|
+
"score": "float — killed / (total - errors - neutral - equivalent - unresolved - unparseable), range 0.0-1.0, rounded to 4 decimals",
|
|
145
159
|
"duration": "float — total wall-clock seconds, rounded to 4 decimals",
|
|
146
160
|
"peak_memory_mb": "float (optional) — peak RSS across all mutation child processes, in MB"
|
|
147
161
|
},
|
|
@@ -150,9 +164,10 @@ Use `--format json` for machine-readable output. Schema:
|
|
|
150
164
|
"operator": "string — mutation operator name (see Operators table)",
|
|
151
165
|
"file": "string — relative path to mutated file",
|
|
152
166
|
"line": "integer — line number of the mutation",
|
|
153
|
-
"status": "string — result status: 'survived', 'killed', 'timeout', or '
|
|
167
|
+
"status": "string — result status: 'survived', 'killed', 'timeout', 'error', 'neutral', 'equivalent', 'unresolved', or 'unparseable'",
|
|
154
168
|
"duration": "float — seconds this mutation took, rounded to 4 decimals",
|
|
155
|
-
"diff": "string —
|
|
169
|
+
"diff": "string — legacy +/- diff snippet",
|
|
170
|
+
"unified_diff": "string (optional, survived only) — git-style unified diff with `--- a/file`, `+++ b/file`, `@@` hunk header and sdiff body; omitted when source slices are unavailable",
|
|
156
171
|
"suggestion": "string — actionable hint for surviving mutants (survived only)"
|
|
157
172
|
}
|
|
158
173
|
],
|
|
@@ -167,6 +182,10 @@ Use `--format json` for machine-readable output. Schema:
|
|
|
167
182
|
}
|
|
168
183
|
],
|
|
169
184
|
"killed": ["... same shape as survived entries ..."],
|
|
185
|
+
"neutral": ["... same shape as survived entries ..."],
|
|
186
|
+
"equivalent": ["... same shape as survived entries ..."],
|
|
187
|
+
"unresolved": ["... same shape as survived entries — coverage gap: no spec file resolved for these mutations"],
|
|
188
|
+
"unparseable": ["... same shape as survived entries — mutated source failed to parse and was never executed"],
|
|
170
189
|
"timed_out": ["... same shape as survived entries ..."],
|
|
171
190
|
"errors": [
|
|
172
191
|
{
|
|
@@ -181,6 +200,21 @@ Use `--format json` for machine-readable output. Schema:
|
|
|
181
200
|
|
|
182
201
|
**Key metric**: `summary.score` — the mutation score. Higher is better. 1.0 means all mutations were caught.
|
|
183
202
|
|
|
203
|
+
### Mutation Statuses
|
|
204
|
+
|
|
205
|
+
| Status | Meaning | Counted in score? |
|
|
206
|
+
|--------------|-----------------------------------------------------------------------|-------------------|
|
|
207
|
+
| `killed` | A test failed when the mutation was applied — test suite caught it | numerator + denominator |
|
|
208
|
+
| `survived` | No test failed — gap in coverage | denominator only |
|
|
209
|
+
| `timeout` | Test run exceeded `--timeout` — treated like survived for scoring | denominator only |
|
|
210
|
+
| `error` | Mutation caused an unexpected error (syntax error, boot failure, etc.) | excluded from denominator |
|
|
211
|
+
| `neutral` | Baseline tests already failed before mutation — not a meaningful signal | excluded |
|
|
212
|
+
| `equivalent` | Mutation is provably identical to the original (e.g. no-op replacement) | excluded |
|
|
213
|
+
| `unresolved` | No spec file resolved for the mutated source — **coverage gap, not a failure**. Use `--fallback-full-suite` to run the full suite instead. | excluded |
|
|
214
|
+
| `unparseable` | Mutated source failed to parse (e.g. dangling heredoc opener after `method_body_replacement`). Short-circuited — never executed. | excluded |
|
|
215
|
+
|
|
216
|
+
Unresolved mutations indicate a missing test mapping — the file has no corresponding test file that the resolver could find (for example, an RSpec `_spec.rb` file or a Minitest `_test.rb` file, depending on configuration). They are reported separately so you can act on them (add a test, adjust test naming, or opt in to the full-suite fallback) without inflating the error count.
|
|
217
|
+
|
|
184
218
|
## Mutation Operators (72 total)
|
|
185
219
|
|
|
186
220
|
Each operator name is stable and appears in JSON output under `survived[].operator`.
|
|
@@ -313,7 +347,7 @@ Unlike `evilution --format json`, every survived entry returned by `evilution-mu
|
|
|
313
347
|
| `spec_file` | Resolved spec/test path (when one exists) — e.g. an RSpec spec file or Minitest test file, so you can drop new tests straight into it |
|
|
314
348
|
| `next_step` | Concrete natural-language hint — "add a test in X that fails against this mutation at Y:line" |
|
|
315
349
|
|
|
316
|
-
These fields are added in addition to the existing `operator`, `file`, `line`, `diff`, `suggestion`, and `test_command` so agents can triage survivors in one pass.
|
|
350
|
+
These fields are added in addition to the existing `operator`, `file`, `line`, `diff`, `unified_diff`, `suggestion`, and `test_command` so agents can triage survivors in one pass.
|
|
317
351
|
|
|
318
352
|
### Concrete Test Suggestions
|
|
319
353
|
|
|
@@ -418,6 +452,49 @@ bundle exec evilution run lib/ --format json --min-score 0.8 --quiet
|
|
|
418
452
|
|
|
419
453
|
Note: `--quiet` suppresses all stdout output (including JSON). Use it in CI only when you care about the exit code and do not need JSON output.
|
|
420
454
|
|
|
455
|
+
### 9. Regression tracking across runs (`compare`)
|
|
456
|
+
|
|
457
|
+
```bash
|
|
458
|
+
bundle exec evilution run lib/ --format json --save-session
|
|
459
|
+
# later, after edits:
|
|
460
|
+
bundle exec evilution run lib/ --format json --save-session
|
|
461
|
+
|
|
462
|
+
bundle exec evilution compare \
|
|
463
|
+
--against .evilution/results/<earlier>.json \
|
|
464
|
+
--current .evilution/results/<later>.json \
|
|
465
|
+
--format json
|
|
466
|
+
```
|
|
467
|
+
|
|
468
|
+
Output buckets:
|
|
469
|
+
|
|
470
|
+
| Bucket | Meaning |
|
|
471
|
+
|-----------------|---------|
|
|
472
|
+
| `fixed` | Survived previously, killed now — the new test actually landed |
|
|
473
|
+
| `new` | Did not exist previously, surviving now — a fresh gap |
|
|
474
|
+
| `persistent` | Survived in both runs — carry-over debt |
|
|
475
|
+
| `reintroduced` | Killed previously, survived now — regression |
|
|
476
|
+
| `flaky` | Status flipped and back — unstable test |
|
|
477
|
+
|
|
478
|
+
Use in CI to gate merges on `reintroduced` being empty, or to surface `new` survivors for reviewer attention without failing the build on `persistent` debt.
|
|
479
|
+
|
|
480
|
+
## Parallel Runs with SQLite
|
|
481
|
+
|
|
482
|
+
Running with `-j N` forks worker processes. If your Rails app uses SQLite, every worker opens the same `db/test.sqlite3` file, and concurrent writers collide on the database-level lock. Symptoms: `ActiveRecord::StatementTimeout`, `SQLite3::BusyException`, and slow runs. Evilution classifies these crashes as `:neutral` (see [EV-toid / #814](https://github.com/taxdome/evilution/issues/814)) so the mutation score is not polluted, but the wall-clock penalty remains.
|
|
483
|
+
|
|
484
|
+
Evilution follows the [`parallel_tests`](https://github.com/grosser/parallel_tests) convention: each worker receives a `TEST_ENV_NUMBER` environment variable (`""` for worker 1, `"2"` for worker 2, `"3"` for worker 3, …). Interpolate it into `config/database.yml` so each worker gets its own SQLite file:
|
|
485
|
+
|
|
486
|
+
```yaml
|
|
487
|
+
test:
|
|
488
|
+
adapter: sqlite3
|
|
489
|
+
database: db/test<%= ENV['TEST_ENV_NUMBER'] %>.sqlite3
|
|
490
|
+
pool: 5
|
|
491
|
+
timeout: 5000
|
|
492
|
+
```
|
|
493
|
+
|
|
494
|
+
After the first `jobs > 1` run, each worker creates its own file (`db/test.sqlite3`, `db/test2.sqlite3`, …). Seed each with `rake db:test:prepare` before running Evilution, or ensure your preload sets up schema on connect.
|
|
495
|
+
|
|
496
|
+
When Evilution detects a parallel run against a SQLite-backed `config/database.yml`, it prints a one-time startup warning pointing to this section.
|
|
497
|
+
|
|
421
498
|
## Development
|
|
422
499
|
|
|
423
500
|
### Memory leak check
|
data/exe/evil
ADDED
|
@@ -1,11 +1,25 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "prism"
|
|
4
|
+
|
|
3
5
|
require_relative "../ast"
|
|
4
6
|
|
|
5
7
|
module Evilution::AST::SourceSurgeon
|
|
8
|
+
Result = Struct.new(:source, :status, keyword_init: true) do
|
|
9
|
+
def ok?
|
|
10
|
+
status == :ok
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def unparseable?
|
|
14
|
+
status == :unparseable
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
6
18
|
def self.apply(source, offset:, length:, replacement:)
|
|
7
19
|
binary = source.b
|
|
8
20
|
binary[offset, length] = replacement.b
|
|
9
|
-
binary.force_encoding(source.encoding)
|
|
21
|
+
mutated = binary.force_encoding(source.encoding)
|
|
22
|
+
status = Prism.parse(mutated).success? ? :ok : :unparseable
|
|
23
|
+
Result.new(source: mutated, status: status).freeze
|
|
10
24
|
end
|
|
11
25
|
end
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "../commands"
|
|
5
|
+
require_relative "../command"
|
|
6
|
+
require_relative "../dispatcher"
|
|
7
|
+
require_relative "../printers/compare"
|
|
8
|
+
require_relative "../../compare"
|
|
9
|
+
require_relative "../../compare/categorizer"
|
|
10
|
+
require_relative "../../compare/detector"
|
|
11
|
+
require_relative "../../compare/normalizer"
|
|
12
|
+
|
|
13
|
+
class Evilution::CLI::Commands::Compare < Evilution::CLI::Command
|
|
14
|
+
SUPPORTED_FORMATS = %i[json text].freeze
|
|
15
|
+
|
|
16
|
+
private
|
|
17
|
+
|
|
18
|
+
def perform
|
|
19
|
+
paths = resolve_paths
|
|
20
|
+
raise Evilution::ConfigError, "exactly two file paths required for compare" unless paths.length == 2
|
|
21
|
+
|
|
22
|
+
fmt = @options[:format] || :json
|
|
23
|
+
raise Evilution::ConfigError, "compare supports --format text or json, got #{fmt.inspect}" unless SUPPORTED_FORMATS.include?(fmt)
|
|
24
|
+
|
|
25
|
+
against = load_and_normalize(paths[0])
|
|
26
|
+
current = load_and_normalize(paths[1])
|
|
27
|
+
buckets = Evilution::Compare::Categorizer.call(against, current)
|
|
28
|
+
Evilution::CLI::Printers::Compare.new(buckets, format: fmt).render(@stdout)
|
|
29
|
+
0
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Flags bind to roles (--against -> slot 0, --current -> slot 1);
|
|
33
|
+
# positional @files fill whatever role the flags didn't claim, in order.
|
|
34
|
+
# Extra positional args after both slots are filled are a user error.
|
|
35
|
+
def resolve_paths
|
|
36
|
+
positional = @files.dup
|
|
37
|
+
against = @options[:against] || positional.shift
|
|
38
|
+
current = @options[:current] || positional.shift
|
|
39
|
+
|
|
40
|
+
raise Evilution::ConfigError, "exactly two file paths required for compare" unless positional.empty?
|
|
41
|
+
|
|
42
|
+
[against, current].compact
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def load_and_normalize(path)
|
|
46
|
+
raise Evilution::Error, "file not found: #{path}" unless File.exist?(path)
|
|
47
|
+
|
|
48
|
+
json = JSON.parse(File.read(path))
|
|
49
|
+
tool = Evilution::Compare::Detector.call(json)
|
|
50
|
+
normalize(json, tool)
|
|
51
|
+
rescue ::JSON::ParserError => e
|
|
52
|
+
raise Evilution::Error, "invalid JSON in #{path}: #{e.message}"
|
|
53
|
+
rescue Evilution::Compare::InvalidInput => e
|
|
54
|
+
raise Evilution::Error, "#{path}: #{e.message}"
|
|
55
|
+
rescue SystemCallError => e
|
|
56
|
+
raise Evilution::Error, e.message
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def normalize(json, tool)
|
|
60
|
+
normalizer = Evilution::Compare::Normalizer.new
|
|
61
|
+
case tool
|
|
62
|
+
when :mutant then normalizer.from_mutant(json)
|
|
63
|
+
when :evilution then normalizer.from_evilution(json)
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
Evilution::CLI::Dispatcher.register(:compare, Evilution::CLI::Commands::Compare)
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class Evilution::CLI::Parser::CommandExtractor
|
|
4
|
+
SIMPLE_COMMANDS = {
|
|
5
|
+
"version" => :version,
|
|
6
|
+
"init" => :init,
|
|
7
|
+
"mcp" => :mcp,
|
|
8
|
+
"subjects" => :subjects,
|
|
9
|
+
"compare" => :compare
|
|
10
|
+
}.freeze
|
|
11
|
+
|
|
12
|
+
SESSION_SUBCOMMANDS = {
|
|
13
|
+
"list" => :session_list,
|
|
14
|
+
"show" => :session_show,
|
|
15
|
+
"diff" => :session_diff,
|
|
16
|
+
"gc" => :session_gc
|
|
17
|
+
}.freeze
|
|
18
|
+
|
|
19
|
+
TESTS_SUBCOMMANDS = { "list" => :tests_list }.freeze
|
|
20
|
+
ENVIRONMENT_SUBCOMMANDS = { "show" => :environment_show }.freeze
|
|
21
|
+
UTIL_SUBCOMMANDS = { "mutation" => :util_mutation }.freeze
|
|
22
|
+
|
|
23
|
+
Result = Struct.new(:command, :remaining_argv, :parse_error)
|
|
24
|
+
|
|
25
|
+
def self.call(argv)
|
|
26
|
+
new(argv).call
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def initialize(argv)
|
|
30
|
+
@argv = argv.dup
|
|
31
|
+
@command = :run
|
|
32
|
+
@parse_error = nil
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def call
|
|
36
|
+
extract
|
|
37
|
+
Result.new(@command, @argv, @parse_error)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def extract
|
|
43
|
+
first = @argv.first
|
|
44
|
+
if SIMPLE_COMMANDS.key?(first)
|
|
45
|
+
@command = SIMPLE_COMMANDS[first]
|
|
46
|
+
@argv.shift
|
|
47
|
+
elsif first == "run"
|
|
48
|
+
@argv.shift
|
|
49
|
+
elsif first == "session"
|
|
50
|
+
@argv.shift
|
|
51
|
+
extract_subcommand(SESSION_SUBCOMMANDS, "session", "list, show, diff, gc")
|
|
52
|
+
elsif first == "tests"
|
|
53
|
+
@argv.shift
|
|
54
|
+
extract_subcommand(TESTS_SUBCOMMANDS, "tests", "list")
|
|
55
|
+
elsif first == "environment"
|
|
56
|
+
@argv.shift
|
|
57
|
+
extract_subcommand(ENVIRONMENT_SUBCOMMANDS, "environment", "show")
|
|
58
|
+
elsif first == "util"
|
|
59
|
+
@argv.shift
|
|
60
|
+
extract_subcommand(UTIL_SUBCOMMANDS, "util", "mutation")
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def extract_subcommand(table, family, available)
|
|
65
|
+
sub = @argv.first
|
|
66
|
+
if table.key?(sub)
|
|
67
|
+
@command = table[sub]
|
|
68
|
+
@argv.shift
|
|
69
|
+
elsif sub.nil?
|
|
70
|
+
@command = :parse_error
|
|
71
|
+
@parse_error = "Missing #{family} subcommand. Available subcommands: #{available}"
|
|
72
|
+
else
|
|
73
|
+
@command = :parse_error
|
|
74
|
+
@parse_error = "Unknown #{family} subcommand: #{sub}. Available subcommands: #{available}"
|
|
75
|
+
@argv.shift
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Evilution::CLI::Parser::FileArgs
|
|
4
|
+
module_function
|
|
5
|
+
|
|
6
|
+
def parse(raw_args)
|
|
7
|
+
files = []
|
|
8
|
+
ranges = {}
|
|
9
|
+
|
|
10
|
+
raw_args.each do |arg|
|
|
11
|
+
file, range_str = arg.split(":", 2)
|
|
12
|
+
files << file
|
|
13
|
+
next unless range_str
|
|
14
|
+
|
|
15
|
+
ranges[file] = parse_line_range(range_str)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
[files, ranges]
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def expand_spec_dir(dir)
|
|
22
|
+
unless File.directory?(dir)
|
|
23
|
+
warn("Error: #{dir} is not a directory")
|
|
24
|
+
return []
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
Dir.glob(File.join(dir, "**/*_spec.rb"))
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def parse_line_range(str)
|
|
31
|
+
if str.include?("-")
|
|
32
|
+
start_str, end_str = str.split("-", 2)
|
|
33
|
+
start_line = Integer(start_str)
|
|
34
|
+
end_line = end_str.empty? ? Float::INFINITY : Integer(end_str)
|
|
35
|
+
start_line..end_line
|
|
36
|
+
else
|
|
37
|
+
line = Integer(str)
|
|
38
|
+
line..line
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "optparse"
|
|
4
|
+
require_relative "../../version"
|
|
5
|
+
require_relative "file_args"
|
|
6
|
+
|
|
7
|
+
class Evilution::CLI::Parser::OptionsBuilder
|
|
8
|
+
def self.build(options)
|
|
9
|
+
new(options).build
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def initialize(options)
|
|
13
|
+
@options = options
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def build
|
|
17
|
+
OptionParser.new do |opts|
|
|
18
|
+
opts.banner = "Usage: evilution [command] [options] [files...]"
|
|
19
|
+
opts.version = Evilution::VERSION
|
|
20
|
+
add_separators(opts)
|
|
21
|
+
add_core_options(opts)
|
|
22
|
+
add_filter_options(opts)
|
|
23
|
+
add_flag_options(opts)
|
|
24
|
+
add_extra_flag_options(opts)
|
|
25
|
+
add_session_options(opts)
|
|
26
|
+
add_compare_options(opts)
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def add_separators(opts)
|
|
33
|
+
opts.separator ""
|
|
34
|
+
opts.separator "Line-range targeting: lib/foo.rb:15-30, lib/foo.rb:15, lib/foo.rb:15-"
|
|
35
|
+
opts.separator ""
|
|
36
|
+
opts.separator "Commands: run (default), init, session {list,show,diff,gc}, subjects, tests {list},"
|
|
37
|
+
opts.separator " util {mutation}, environment {show}, compare, mcp, version"
|
|
38
|
+
opts.separator ""
|
|
39
|
+
opts.separator "Options:"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def add_core_options(opts)
|
|
43
|
+
opts.on("-j", "--jobs N", Integer, "Number of parallel workers (default: 1)") { |n| @options[:jobs] = n }
|
|
44
|
+
opts.on("-t", "--timeout N", Integer, "Per-mutation timeout in seconds") { |n| @options[:timeout] = n }
|
|
45
|
+
opts.on("-f", "--format FORMAT", "Output format: text, json, html") { |f| @options[:format] = f.to_sym }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def add_filter_options(opts)
|
|
49
|
+
opts.on("--min-score FLOAT", Float, "Minimum mutation score to pass") { |s| @options[:min_score] = s }
|
|
50
|
+
opts.on("--spec FILES", Array, "Spec files to run (comma-separated)") { |f| @options[:spec_files] = f }
|
|
51
|
+
opts.on("--spec-dir DIR", "Include all specs in DIR") { |d| expand_spec_dir(d) }
|
|
52
|
+
opts.on("--spec-pattern GLOB",
|
|
53
|
+
"Restrict resolved spec candidates to files matching GLOB") { |p| @options[:spec_pattern] = p }
|
|
54
|
+
opts.on("--no-example-targeting",
|
|
55
|
+
"Disable per-mutation example targeting (run all examples in resolved spec files)") do
|
|
56
|
+
@options[:example_targeting] = false
|
|
57
|
+
end
|
|
58
|
+
opts.on("--example-targeting-fallback MODE", %w[full_file unresolved],
|
|
59
|
+
"Fallback when example targeting finds no match: full_file (default) or unresolved") do |m|
|
|
60
|
+
@options[:example_targeting_fallback] = m
|
|
61
|
+
end
|
|
62
|
+
opts.on("--target EXPR",
|
|
63
|
+
"Filter: method (Foo#bar), type (Foo#/Foo.), namespace (Foo*),",
|
|
64
|
+
"class (Foo), glob (source:**/*.rb), hierarchy (descendants:Foo)") do |m|
|
|
65
|
+
@options[:target] = m
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def add_flag_options(opts)
|
|
70
|
+
opts.on("--fail-fast", "Stop after N surviving mutants " \
|
|
71
|
+
"(default: disabled; if provided without N, uses 1; use --fail-fast=N)") { @options[:fail_fast] ||= 1 }
|
|
72
|
+
opts.on("--no-baseline", "Skip baseline test suite check") { @options[:baseline] = false }
|
|
73
|
+
opts.on("--incremental", "Cache killed/timeout results; skip re-running them on unchanged files") { @options[:incremental] = true }
|
|
74
|
+
opts.on("--integration NAME", "Test integration: rspec, minitest (default: rspec)") { |i| @options[:integration] = i }
|
|
75
|
+
opts.on("--isolation STRATEGY", "Isolation: auto, fork, in_process (default: auto)") { |s| @options[:isolation] = s }
|
|
76
|
+
opts.on("--preload FILE", "Preload FILE in the parent process before forking " \
|
|
77
|
+
"(default: auto-detect spec/rails_helper.rb for Rails projects)") { |f| @options[:preload] = f }
|
|
78
|
+
opts.on("--no-preload", "Disable parent-process preload even for Rails projects") { @options[:preload] = false }
|
|
79
|
+
opts.on("--stdin", "Read target file paths from stdin (one per line)") { @options[:stdin] = true }
|
|
80
|
+
opts.on("--suggest-tests", "Generate concrete test code in suggestions (RSpec or Minitest)") { @options[:suggest_tests] = true }
|
|
81
|
+
opts.on("--no-progress", "Disable progress bar") { @options[:progress] = false }
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def add_extra_flag_options(opts)
|
|
85
|
+
opts.on("--skip-heredoc-literals", "Skip all string literal mutations inside heredocs") { @options[:skip_heredoc_literals] = true }
|
|
86
|
+
opts.on("--related-specs-heuristic", "Append related request/integration/feature/system specs for includes() mutations") do
|
|
87
|
+
@options[:related_specs_heuristic] = true
|
|
88
|
+
end
|
|
89
|
+
opts.on("--fallback-full-suite", "Run the whole test suite when no matching spec/test resolves " \
|
|
90
|
+
"for a mutation (default: mark the mutation :unresolved and skip)") do
|
|
91
|
+
@options[:fallback_to_full_suite] = true
|
|
92
|
+
end
|
|
93
|
+
opts.on("--show-disabled", "Report mutations skipped by # evilution:disable") { @options[:show_disabled] = true }
|
|
94
|
+
opts.on("--baseline-session PATH", "Compare against a baseline session in HTML report") { |p| @options[:baseline_session] = p }
|
|
95
|
+
opts.on("--save-session", "Save session results to .evilution/results/") { @options[:save_session] = true }
|
|
96
|
+
opts.on("-e", "--eval CODE", "Evaluate code snippet (for util mutation)") { |c| @options[:eval] = c }
|
|
97
|
+
opts.on("-v", "--verbose", "Verbose output") { @options[:verbose] = true }
|
|
98
|
+
opts.on("-q", "--quiet", "Suppress output") { @options[:quiet] = true }
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def add_session_options(opts)
|
|
102
|
+
opts.on("--results-dir DIR", "Session results directory") { |d| @options[:results_dir] = d }
|
|
103
|
+
opts.on("--limit N", Integer, "Show only the N most recent sessions") { |n| @options[:limit] = n }
|
|
104
|
+
opts.on("--since DATE", "Show sessions since DATE (YYYY-MM-DD)") { |d| @options[:since] = d }
|
|
105
|
+
opts.on("--older-than DURATION", "Delete sessions older than DURATION (e.g., 30d, 24h, 1w)") do |d|
|
|
106
|
+
@options[:older_than] = d
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def add_compare_options(opts)
|
|
111
|
+
opts.on("--against PATH", "Prior mutation run to compare against (used with `compare` command)") do |p|
|
|
112
|
+
@options[:against] = p
|
|
113
|
+
end
|
|
114
|
+
opts.on("--current PATH", "Current mutation run to compare (used with `compare` command)") do |p|
|
|
115
|
+
@options[:current] = p
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def expand_spec_dir(dir)
|
|
120
|
+
specs = Evilution::CLI::Parser::FileArgs.expand_spec_dir(dir)
|
|
121
|
+
@options[:spec_files] = Array(@options[:spec_files]) + specs
|
|
122
|
+
end
|
|
123
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "file_args"
|
|
4
|
+
|
|
5
|
+
class Evilution::CLI::Parser::StdinReader
|
|
6
|
+
Result = Struct.new(:files, :ranges, :error)
|
|
7
|
+
|
|
8
|
+
def self.call(io, existing_files:)
|
|
9
|
+
new(io, existing_files: existing_files).call
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def initialize(io, existing_files:)
|
|
13
|
+
@io = io
|
|
14
|
+
@existing_files = existing_files
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def call
|
|
18
|
+
return Result.new([], {}, "--stdin cannot be combined with positional file arguments") if @existing_files.any?
|
|
19
|
+
|
|
20
|
+
lines = []
|
|
21
|
+
@io.each_line do |line|
|
|
22
|
+
line = line.strip
|
|
23
|
+
lines << line unless line.empty?
|
|
24
|
+
end
|
|
25
|
+
files, ranges = Evilution::CLI::Parser::FileArgs.parse(lines)
|
|
26
|
+
Result.new(files, ranges, nil)
|
|
27
|
+
end
|
|
28
|
+
end
|