evilution 0.24.0 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. checksums.yaml +4 -4
  2. data/.beads/interactions.jsonl +210 -0
  3. data/.claude/prompts/architect.md +14 -1
  4. data/.claude/skills/create-issue/SKILL.md +55 -0
  5. data/CHANGELOG.md +51 -0
  6. data/README.md +80 -4
  7. data/exe/evil +6 -0
  8. data/lib/evilution/ast/constant_names.rb +34 -0
  9. data/lib/evilution/ast/source_surgeon.rb +15 -1
  10. data/lib/evilution/cli/commands/compare.rb +68 -0
  11. data/lib/evilution/cli/parser/command_extractor.rb +2 -1
  12. data/lib/evilution/cli/parser/options_builder.rb +21 -1
  13. data/lib/evilution/cli/printers/compare.rb +159 -0
  14. data/lib/evilution/cli.rb +1 -0
  15. data/lib/evilution/compare/categorizer.rb +109 -0
  16. data/lib/evilution/compare/detector.rb +21 -0
  17. data/lib/evilution/compare/fingerprint.rb +83 -0
  18. data/lib/evilution/compare/invalid_input.rb +12 -0
  19. data/lib/evilution/compare/normalizer.rb +106 -0
  20. data/lib/evilution/compare/record.rb +16 -0
  21. data/lib/evilution/compare.rb +6 -0
  22. data/lib/evilution/config.rb +165 -3
  23. data/lib/evilution/example_filter.rb +143 -0
  24. data/lib/evilution/integration/base.rb +4 -155
  25. data/lib/evilution/integration/crash_detector.rb +5 -2
  26. data/lib/evilution/integration/loading/concern_state_cleaner.rb +49 -0
  27. data/lib/evilution/integration/loading/constant_pinner.rb +24 -0
  28. data/lib/evilution/integration/loading/mutation_applier.rb +52 -0
  29. data/lib/evilution/integration/loading/redefinition_recovery.rb +54 -0
  30. data/lib/evilution/integration/loading/source_evaluator.rb +15 -0
  31. data/lib/evilution/integration/loading/syntax_validator.rb +19 -0
  32. data/lib/evilution/integration/loading.rb +6 -0
  33. data/lib/evilution/integration/minitest.rb +10 -5
  34. data/lib/evilution/integration/minitest_crash_detector.rb +5 -2
  35. data/lib/evilution/integration/rspec.rb +82 -7
  36. data/lib/evilution/isolation/fork.rb +25 -0
  37. data/lib/evilution/load_path/subpath_resolver.rb +25 -0
  38. data/lib/evilution/load_path.rb +4 -0
  39. data/lib/evilution/mcp/info_tool.rb +77 -5
  40. data/lib/evilution/mcp/mutate_tool/config_builder.rb +20 -0
  41. data/lib/evilution/mcp/mutate_tool/error_payload.rb +17 -0
  42. data/lib/evilution/mcp/mutate_tool/option_parser.rb +54 -0
  43. data/lib/evilution/mcp/mutate_tool/progress_streamer.rb +37 -0
  44. data/lib/evilution/mcp/mutate_tool/report_trimmer.rb +31 -0
  45. data/lib/evilution/mcp/mutate_tool/survived_enricher.rb +52 -0
  46. data/lib/evilution/mcp/mutate_tool.rb +34 -186
  47. data/lib/evilution/mutation.rb +43 -3
  48. data/lib/evilution/mutator/base.rb +39 -1
  49. data/lib/evilution/mutator/operator/argument_nil_substitution.rb +5 -1
  50. data/lib/evilution/mutator/operator/argument_removal.rb +5 -1
  51. data/lib/evilution/parallel/work_queue.rb +149 -31
  52. data/lib/evilution/parallel_db_warning.rb +68 -0
  53. data/lib/evilution/reporter/cli.rb +37 -11
  54. data/lib/evilution/reporter/html/assets/style.css +17 -0
  55. data/lib/evilution/reporter/html/sections/file_section.rb +15 -0
  56. data/lib/evilution/reporter/html/sections/neutral_details.rb +25 -0
  57. data/lib/evilution/reporter/html/sections/unparseable_details.rb +25 -0
  58. data/lib/evilution/reporter/html/sections/unresolved_details.rb +25 -0
  59. data/lib/evilution/reporter/html/templates/file_section.html.erb +3 -0
  60. data/lib/evilution/reporter/html/templates/neutral_details.html.erb +14 -0
  61. data/lib/evilution/reporter/html/templates/summary_cards.html.erb +3 -0
  62. data/lib/evilution/reporter/html/templates/unparseable_details.html.erb +11 -0
  63. data/lib/evilution/reporter/html/templates/unresolved_details.html.erb +11 -0
  64. data/lib/evilution/reporter/json.rb +8 -2
  65. data/lib/evilution/reporter/suggestion/diff_helpers.rb +28 -0
  66. data/lib/evilution/reporter/suggestion/registry.rb +64 -0
  67. data/lib/evilution/reporter/suggestion/templates/generic.rb +55 -0
  68. data/lib/evilution/reporter/suggestion/templates/minitest.rb +659 -0
  69. data/lib/evilution/reporter/suggestion/templates/rspec.rb +613 -0
  70. data/lib/evilution/reporter/suggestion.rb +8 -1327
  71. data/lib/evilution/result/mutation_result.rb +5 -1
  72. data/lib/evilution/result/summary.rb +13 -1
  73. data/lib/evilution/runner/baseline_runner.rb +23 -2
  74. data/lib/evilution/runner/isolation_resolver.rb +12 -1
  75. data/lib/evilution/runner/mutation_executor.rb +83 -13
  76. data/lib/evilution/runner/subject_pipeline.rb +18 -8
  77. data/lib/evilution/runner.rb +6 -0
  78. data/lib/evilution/source_ast_cache.rb +39 -0
  79. data/lib/evilution/spec_ast_cache.rb +166 -0
  80. data/lib/evilution/spec_resolver.rb +6 -1
  81. data/lib/evilution/spec_selector.rb +39 -0
  82. data/lib/evilution/temp_dir_tracker.rb +23 -3
  83. data/lib/evilution/version.rb +1 -1
  84. data/script/memory_check +7 -5
  85. metadata +46 -5
  86. data/lib/evilution/mcp/session_diff_tool.rb +0 -63
  87. data/lib/evilution/mcp/session_list_tool.rb +0 -50
  88. data/lib/evilution/mcp/session_show_tool.rb +0 -57
data/README.md CHANGED
@@ -27,6 +27,8 @@ Or standalone: `gem install evilution`
27
27
  evilution [command] [options] [files...]
28
28
  ```
29
29
 
30
+ The shorter alias `evil` ships alongside `evilution` and accepts identical arguments (handy with `alias be='bundle exec'` → `be evil run ...`).
31
+
30
32
  ### Commands
31
33
 
32
34
  | Command | Description | Default |
@@ -42,6 +44,7 @@ evilution [command] [options] [files...]
42
44
  | `session gc --older-than D` | Garbage-collect sessions older than D (e.g. 30d) | |
43
45
  | `util mutation` | Preview mutations for a file or inline code | |
44
46
  | `environment show` | Display runtime environment and settings | |
47
+ | `compare --against A --current B` | Compare two saved session JSON files into fixed / new / persistent / flaky / reintroduced buckets | |
45
48
 
46
49
  ### Options (for `run` command)
47
50
 
@@ -53,6 +56,9 @@ evilution [command] [options] [files...]
53
56
  | `--min-score FLOAT` | Float | 0.0 | Minimum mutation score (0.0–1.0) to pass. |
54
57
  | `--spec FILES` | Array | _(none)_ | Spec files to run (comma-separated). Defaults to auto-detection via `SpecResolver`. |
55
58
  | `--spec-dir DIR` | String | _(none)_ | Include all `*_spec.rb` files in DIR recursively. Composable with `--spec`. |
59
+ | `--spec-pattern GLOB` | String | _(none)_ | Restrict resolved spec candidates to files matching GLOB (e.g. `spec/models/**/*_spec.rb`). |
60
+ | `--no-example-targeting` | Boolean | _(enabled)_ | Disable per-mutation example targeting (always run every example in the resolved spec file). Example targeting scans each example body for symbols from the mutated method and runs only the matching subset. |
61
+ | `--example-targeting-fallback MODE` | String | `full_file` | Behavior when no example matches: `full_file` (run the whole spec file) or `unresolved` (skip the mutation as `:unresolved`). |
56
62
  | `-j`, `--jobs N` | Integer | 1 | Number of parallel workers. Uses demand-driven work distribution with pipe-based IPC. |
57
63
  | `--no-baseline` | Boolean | _(enabled)_ | Skip baseline test suite check. By default, a baseline run detects pre-existing failures and marks those mutations as `neutral`. |
58
64
  | `--fail-fast [N]` | Integer | _(none)_ | Stop after N surviving mutants (default 1 if no value given). |
@@ -101,6 +107,9 @@ Creates `.evilution.yml`:
101
107
  # baseline_session: null # path to session file for HTML comparison
102
108
  # ignore_patterns: [] # AST patterns to exclude (see docs/ast_pattern_syntax.md)
103
109
  # progress: true # TTY progress bar
110
+ # example_targeting: true # per-mutation example targeting via body-token scan
111
+ # example_targeting_fallback: full_file # full_file | unresolved
112
+ # spec_pattern: null # restrict resolved spec candidates to files matching GLOB
104
113
  ```
105
114
 
106
115
  **Precedence**: CLI flags override `.evilution.yml` values.
@@ -142,7 +151,11 @@ Use `--format json` for machine-readable output. Schema:
142
151
  "survived": "integer — mutations NOT detected (test passed = gap in coverage)",
143
152
  "timed_out": "integer — mutations that exceeded timeout",
144
153
  "errors": "integer — mutations that caused unexpected errors",
145
- "score": "float killed / (total - errors), range 0.0-1.0, rounded to 4 decimals",
154
+ "neutral": "integer mutations whose tests already failed before mutation (baseline failure)",
155
+ "equivalent": "integer — mutations proven to have identical behavior to the original",
156
+ "unresolved": "integer — mutations where no spec file resolved (coverage gap, not a failure)",
157
+ "unparseable": "integer — mutations whose mutated source did not parse (short-circuited, never executed)",
158
+ "score": "float — killed / (total - errors - neutral - equivalent - unresolved - unparseable), range 0.0-1.0, rounded to 4 decimals",
146
159
  "duration": "float — total wall-clock seconds, rounded to 4 decimals",
147
160
  "peak_memory_mb": "float (optional) — peak RSS across all mutation child processes, in MB"
148
161
  },
@@ -151,9 +164,10 @@ Use `--format json` for machine-readable output. Schema:
151
164
  "operator": "string — mutation operator name (see Operators table)",
152
165
  "file": "string — relative path to mutated file",
153
166
  "line": "integer — line number of the mutation",
154
- "status": "string — result status: 'survived', 'killed', 'timeout', or 'error'",
167
+ "status": "string — result status: 'survived', 'killed', 'timeout', 'error', 'neutral', 'equivalent', 'unresolved', or 'unparseable'",
155
168
  "duration": "float — seconds this mutation took, rounded to 4 decimals",
156
- "diff": "string — unified diff snippet",
169
+ "diff": "string — legacy +/- diff snippet",
170
+ "unified_diff": "string (optional, survived only) — git-style unified diff with `--- a/file`, `+++ b/file`, `@@` hunk header and sdiff body; omitted when source slices are unavailable",
157
171
  "suggestion": "string — actionable hint for surviving mutants (survived only)"
158
172
  }
159
173
  ],
@@ -168,6 +182,10 @@ Use `--format json` for machine-readable output. Schema:
168
182
  }
169
183
  ],
170
184
  "killed": ["... same shape as survived entries ..."],
185
+ "neutral": ["... same shape as survived entries ..."],
186
+ "equivalent": ["... same shape as survived entries ..."],
187
+ "unresolved": ["... same shape as survived entries — coverage gap: no spec file resolved for these mutations"],
188
+ "unparseable": ["... same shape as survived entries — mutated source failed to parse and was never executed"],
171
189
  "timed_out": ["... same shape as survived entries ..."],
172
190
  "errors": [
173
191
  {
@@ -182,6 +200,21 @@ Use `--format json` for machine-readable output. Schema:
182
200
 
183
201
  **Key metric**: `summary.score` — the mutation score. Higher is better. 1.0 means all mutations were caught.
184
202
 
203
+ ### Mutation Statuses
204
+
205
+ | Status | Meaning | Counted in score? |
206
+ |--------------|-----------------------------------------------------------------------|-------------------|
207
+ | `killed` | A test failed when the mutation was applied — test suite caught it | numerator + denominator |
208
+ | `survived` | No test failed — gap in coverage | denominator only |
209
+ | `timeout` | Test run exceeded `--timeout` — treated like survived for scoring | denominator only |
210
+ | `error` | Mutation caused an unexpected error (syntax error, boot failure, etc.) | excluded from denominator |
211
+ | `neutral` | Baseline tests already failed before mutation — not a meaningful signal | excluded |
212
+ | `equivalent` | Mutation is provably identical to the original (e.g. no-op replacement) | excluded |
213
+ | `unresolved` | No spec file resolved for the mutated source — **coverage gap, not a failure**. Use `--fallback-full-suite` to run the full suite instead. | excluded |
214
+ | `unparseable` | Mutated source failed to parse (e.g. dangling heredoc opener after `method_body_replacement`). Short-circuited — never executed. | excluded |
215
+
216
+ Unresolved mutations indicate a missing test mapping — the file has no corresponding test file that the resolver could find (for example, an RSpec `_spec.rb` file or a Minitest `_test.rb` file, depending on configuration). They are reported separately so you can act on them (add a test, adjust test naming, or opt in to the full-suite fallback) without inflating the error count.
217
+
185
218
  ## Mutation Operators (72 total)
186
219
 
187
220
  Each operator name is stable and appears in JSON output under `survived[].operator`.
@@ -314,7 +347,7 @@ Unlike `evilution --format json`, every survived entry returned by `evilution-mu
314
347
  | `spec_file` | Resolved spec/test path (when one exists) — e.g. an RSpec spec file or Minitest test file, so you can drop new tests straight into it |
315
348
  | `next_step` | Concrete natural-language hint — "add a test in X that fails against this mutation at Y:line" |
316
349
 
317
- These fields are added in addition to the existing `operator`, `file`, `line`, `diff`, `suggestion`, and `test_command` so agents can triage survivors in one pass.
350
+ These fields are added in addition to the existing `operator`, `file`, `line`, `diff`, `unified_diff`, `suggestion`, and `test_command` so agents can triage survivors in one pass.
318
351
 
319
352
  ### Concrete Test Suggestions
320
353
 
@@ -419,6 +452,49 @@ bundle exec evilution run lib/ --format json --min-score 0.8 --quiet
419
452
 
420
453
  Note: `--quiet` suppresses all stdout output (including JSON). Use it in CI only when you care about the exit code and do not need JSON output.
421
454
 
455
+ ### 9. Regression tracking across runs (`compare`)
456
+
457
+ ```bash
458
+ bundle exec evilution run lib/ --format json --save-session
459
+ # later, after edits:
460
+ bundle exec evilution run lib/ --format json --save-session
461
+
462
+ bundle exec evilution compare \
463
+ --against .evilution/results/<earlier>.json \
464
+ --current .evilution/results/<later>.json \
465
+ --format json
466
+ ```
467
+
468
+ Output buckets:
469
+
470
+ | Bucket | Meaning |
471
+ |-----------------|---------|
472
+ | `fixed` | Survived previously, killed now — the new test actually landed |
473
+ | `new` | Did not exist previously, surviving now — a fresh gap |
474
+ | `persistent` | Survived in both runs — carry-over debt |
475
+ | `reintroduced` | Killed previously, survived now — regression |
476
+ | `flaky` | Status flipped and back — unstable test |
477
+
478
+ Use in CI to gate merges on `reintroduced` being empty, or to surface `new` survivors for reviewer attention without failing the build on `persistent` debt.
479
+
480
+ ## Parallel Runs with SQLite
481
+
482
+ Running with `-j N` forks worker processes. If your Rails app uses SQLite, every worker opens the same `db/test.sqlite3` file, and concurrent writers collide on the database-level lock. Symptoms: `ActiveRecord::StatementTimeout`, `SQLite3::BusyException`, and slow runs. Evilution classifies these crashes as `:neutral` (see [EV-toid / #814](https://github.com/taxdome/evilution/issues/814)) so the mutation score is not polluted, but the wall-clock penalty remains.
483
+
484
+ Evilution follows the [`parallel_tests`](https://github.com/grosser/parallel_tests) convention: each worker receives a `TEST_ENV_NUMBER` environment variable (`""` for worker 1, `"2"` for worker 2, `"3"` for worker 3, …). Interpolate it into `config/database.yml` so each worker gets its own SQLite file:
485
+
486
+ ```yaml
487
+ test:
488
+ adapter: sqlite3
489
+ database: db/test<%= ENV['TEST_ENV_NUMBER'] %>.sqlite3
490
+ pool: 5
491
+ timeout: 5000
492
+ ```
493
+
494
+ After the first `jobs > 1` run, each worker creates its own file (`db/test.sqlite3`, `db/test2.sqlite3`, …). Seed each with `rake db:test:prepare` before running Evilution, or ensure your preload sets up schema on connect.
495
+
496
+ When Evilution detects a parallel run against a SQLite-backed `config/database.yml`, it prints a one-time startup warning pointing to this section.
497
+
422
498
  ## Development
423
499
 
424
500
  ### Memory leak check
data/exe/evil ADDED
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require "evilution"
5
+
6
+ exit Evilution::CLI.new(ARGV).call
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "prism"
4
+ require_relative "../ast"
5
+
6
+ # Walks a Prism AST and returns every class/module constant declared, nested
7
+ # names rendered fully-qualified (e.g. "Foo::Bar"). Order is source order:
8
+ # outer declarations precede their nested children.
9
+ class Evilution::AST::ConstantNames
10
+ def call(source)
11
+ result = Prism.parse(source)
12
+ return [] if result.failure?
13
+
14
+ collect(result.value)
15
+ end
16
+
17
+ private
18
+
19
+ def collect(node, nesting = [])
20
+ names = []
21
+ case node
22
+ when Prism::ModuleNode, Prism::ClassNode
23
+ const = node.constant_path.full_name
24
+ qualified = nesting.any? && !const.include?("::") ? "#{nesting.join("::")}::#{const}" : const
25
+ names << qualified
26
+ names.concat(collect(node.body, nesting + [const])) if node.body
27
+ when Prism::ProgramNode
28
+ names.concat(collect(node.statements, nesting)) if node.statements
29
+ when Prism::StatementsNode
30
+ node.body.each { |child| names.concat(collect(child, nesting)) }
31
+ end
32
+ names
33
+ end
34
+ end
@@ -1,11 +1,25 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "prism"
4
+
3
5
  require_relative "../ast"
4
6
 
5
7
  module Evilution::AST::SourceSurgeon
8
+ Result = Struct.new(:source, :status, keyword_init: true) do
9
+ def ok?
10
+ status == :ok
11
+ end
12
+
13
+ def unparseable?
14
+ status == :unparseable
15
+ end
16
+ end
17
+
6
18
  def self.apply(source, offset:, length:, replacement:)
7
19
  binary = source.b
8
20
  binary[offset, length] = replacement.b
9
- binary.force_encoding(source.encoding)
21
+ mutated = binary.force_encoding(source.encoding)
22
+ status = Prism.parse(mutated).success? ? :ok : :unparseable
23
+ Result.new(source: mutated, status: status).freeze
10
24
  end
11
25
  end
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require_relative "../commands"
5
+ require_relative "../command"
6
+ require_relative "../dispatcher"
7
+ require_relative "../printers/compare"
8
+ require_relative "../../compare"
9
+ require_relative "../../compare/categorizer"
10
+ require_relative "../../compare/detector"
11
+ require_relative "../../compare/normalizer"
12
+
13
+ class Evilution::CLI::Commands::Compare < Evilution::CLI::Command
14
+ SUPPORTED_FORMATS = %i[json text].freeze
15
+
16
+ private
17
+
18
+ def perform
19
+ paths = resolve_paths
20
+ raise Evilution::ConfigError, "exactly two file paths required for compare" unless paths.length == 2
21
+
22
+ fmt = @options[:format] || :json
23
+ raise Evilution::ConfigError, "compare supports --format text or json, got #{fmt.inspect}" unless SUPPORTED_FORMATS.include?(fmt)
24
+
25
+ against = load_and_normalize(paths[0])
26
+ current = load_and_normalize(paths[1])
27
+ buckets = Evilution::Compare::Categorizer.call(against, current)
28
+ Evilution::CLI::Printers::Compare.new(buckets, format: fmt).render(@stdout)
29
+ 0
30
+ end
31
+
32
+ # Flags bind to roles (--against -> slot 0, --current -> slot 1);
33
+ # positional @files fill whatever role the flags didn't claim, in order.
34
+ # Extra positional args after both slots are filled are a user error.
35
+ def resolve_paths
36
+ positional = @files.dup
37
+ against = @options[:against] || positional.shift
38
+ current = @options[:current] || positional.shift
39
+
40
+ raise Evilution::ConfigError, "exactly two file paths required for compare" unless positional.empty?
41
+
42
+ [against, current].compact
43
+ end
44
+
45
+ def load_and_normalize(path)
46
+ raise Evilution::Error, "file not found: #{path}" unless File.exist?(path)
47
+
48
+ json = JSON.parse(File.read(path))
49
+ tool = Evilution::Compare::Detector.call(json)
50
+ normalize(json, tool)
51
+ rescue ::JSON::ParserError => e
52
+ raise Evilution::Error, "invalid JSON in #{path}: #{e.message}"
53
+ rescue Evilution::Compare::InvalidInput => e
54
+ raise Evilution::Error, "#{path}: #{e.message}"
55
+ rescue SystemCallError => e
56
+ raise Evilution::Error, e.message
57
+ end
58
+
59
+ def normalize(json, tool)
60
+ normalizer = Evilution::Compare::Normalizer.new
61
+ case tool
62
+ when :mutant then normalizer.from_mutant(json)
63
+ when :evilution then normalizer.from_evilution(json)
64
+ end
65
+ end
66
+ end
67
+
68
+ Evilution::CLI::Dispatcher.register(:compare, Evilution::CLI::Commands::Compare)
@@ -5,7 +5,8 @@ class Evilution::CLI::Parser::CommandExtractor
5
5
  "version" => :version,
6
6
  "init" => :init,
7
7
  "mcp" => :mcp,
8
- "subjects" => :subjects
8
+ "subjects" => :subjects,
9
+ "compare" => :compare
9
10
  }.freeze
10
11
 
11
12
  SESSION_SUBCOMMANDS = {
@@ -23,6 +23,7 @@ class Evilution::CLI::Parser::OptionsBuilder
23
23
  add_flag_options(opts)
24
24
  add_extra_flag_options(opts)
25
25
  add_session_options(opts)
26
+ add_compare_options(opts)
26
27
  end
27
28
  end
28
29
 
@@ -33,7 +34,7 @@ class Evilution::CLI::Parser::OptionsBuilder
33
34
  opts.separator "Line-range targeting: lib/foo.rb:15-30, lib/foo.rb:15, lib/foo.rb:15-"
34
35
  opts.separator ""
35
36
  opts.separator "Commands: run (default), init, session {list,show,diff,gc}, subjects, tests {list},"
36
- opts.separator " util {mutation}, environment {show}, mcp, version"
37
+ opts.separator " util {mutation}, environment {show}, compare, mcp, version"
37
38
  opts.separator ""
38
39
  opts.separator "Options:"
39
40
  end
@@ -48,6 +49,16 @@ class Evilution::CLI::Parser::OptionsBuilder
48
49
  opts.on("--min-score FLOAT", Float, "Minimum mutation score to pass") { |s| @options[:min_score] = s }
49
50
  opts.on("--spec FILES", Array, "Spec files to run (comma-separated)") { |f| @options[:spec_files] = f }
50
51
  opts.on("--spec-dir DIR", "Include all specs in DIR") { |d| expand_spec_dir(d) }
52
+ opts.on("--spec-pattern GLOB",
53
+ "Restrict resolved spec candidates to files matching GLOB") { |p| @options[:spec_pattern] = p }
54
+ opts.on("--no-example-targeting",
55
+ "Disable per-mutation example targeting (run all examples in resolved spec files)") do
56
+ @options[:example_targeting] = false
57
+ end
58
+ opts.on("--example-targeting-fallback MODE", %w[full_file unresolved],
59
+ "Fallback when example targeting finds no match: full_file (default) or unresolved") do |m|
60
+ @options[:example_targeting_fallback] = m
61
+ end
51
62
  opts.on("--target EXPR",
52
63
  "Filter: method (Foo#bar), type (Foo#/Foo.), namespace (Foo*),",
53
64
  "class (Foo), glob (source:**/*.rb), hierarchy (descendants:Foo)") do |m|
@@ -96,6 +107,15 @@ class Evilution::CLI::Parser::OptionsBuilder
96
107
  end
97
108
  end
98
109
 
110
+ def add_compare_options(opts)
111
+ opts.on("--against PATH", "Prior mutation run to compare against (used with `compare` command)") do |p|
112
+ @options[:against] = p
113
+ end
114
+ opts.on("--current PATH", "Current mutation run to compare (used with `compare` command)") do |p|
115
+ @options[:current] = p
116
+ end
117
+ end
118
+
99
119
  def expand_spec_dir(dir)
100
120
  specs = Evilution::CLI::Parser::FileArgs.expand_spec_dir(dir)
101
121
  @options[:spec_files] = Array(@options[:spec_files]) + specs
@@ -0,0 +1,159 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require_relative "../printers"
5
+
6
+ class Evilution::CLI::Printers::Compare
7
+ SCHEMA = {
8
+ "shared" => %w[file line operator fp],
9
+ "alive_only" => %w[file line operator fp other_status]
10
+ }.freeze
11
+
12
+ FILE_LINE_WIDTH = 40
13
+ OPERATOR_WIDTH = 22
14
+ FP_LENGTH = 7
15
+ MUTANT_OPERATOR = "(mutant)"
16
+ ABSENT_STATUS = "absent"
17
+
18
+ def initialize(buckets, format: :json)
19
+ @buckets = buckets
20
+ @format = format || :json
21
+ end
22
+
23
+ def render(io)
24
+ case @format
25
+ when :json then render_json(io)
26
+ when :text then render_text(io)
27
+ else raise Evilution::Error, "unknown compare format: #{@format.inspect}"
28
+ end
29
+ end
30
+
31
+ private
32
+
33
+ def render_json(io)
34
+ payload = {
35
+ "schema" => SCHEMA,
36
+ "summary" => summary_hash,
37
+ "alive_only_against" => @buckets[:alive_only_against].map { |e| alive_entry_array(e) },
38
+ "alive_only_current" => @buckets[:alive_only_current].map { |e| alive_entry_array(e) },
39
+ "shared_alive" => @buckets[:shared_alive].map { |e| shared_entry_array(e) },
40
+ "shared_dead" => @buckets[:shared_dead].map { |e| shared_entry_array(e) }
41
+ }
42
+ io.puts(JSON.generate(payload))
43
+ end
44
+
45
+ def render_text(io)
46
+ io.puts("Compare results")
47
+ io.puts("-" * 15)
48
+ io.puts(summary_line)
49
+
50
+ if fully_empty?
51
+ io.puts("No mutations to compare.")
52
+ return
53
+ end
54
+
55
+ print_alive_block(io, :alive_only_against, "current")
56
+ print_alive_block(io, :alive_only_current, "against")
57
+ print_shared_block(io, :shared_alive)
58
+ print_shared_block(io, :shared_dead)
59
+ end
60
+
61
+ def summary_hash
62
+ against_count = @buckets[:alive_only_against].length
63
+ current_count = @buckets[:alive_only_current].length
64
+ {
65
+ "alive_only_against" => against_count,
66
+ "alive_only_current" => current_count,
67
+ "shared_alive" => @buckets[:shared_alive].length,
68
+ "shared_dead" => @buckets[:shared_dead].length,
69
+ "excluded_against" => @buckets[:excluded_against],
70
+ "excluded_current" => @buckets[:excluded_current],
71
+ "delta" => current_count - against_count
72
+ }
73
+ end
74
+
75
+ def summary_line
76
+ s = summary_hash
77
+ parts = [
78
+ "summary:",
79
+ "alive_only_against=#{s["alive_only_against"]}",
80
+ "alive_only_current=#{s["alive_only_current"]}",
81
+ "shared_alive=#{s["shared_alive"]}",
82
+ "shared_dead=#{s["shared_dead"]}",
83
+ "excluded=#{s["excluded_against"]}/#{s["excluded_current"]}",
84
+ "delta=#{format_delta(s["delta"])}"
85
+ ]
86
+ parts.join(" ")
87
+ end
88
+
89
+ def format_delta(delta)
90
+ return "\u00B10" if delta.zero?
91
+
92
+ format("%+d", delta)
93
+ end
94
+
95
+ def fully_empty?
96
+ @buckets[:alive_only_against].empty? &&
97
+ @buckets[:alive_only_current].empty? &&
98
+ @buckets[:shared_alive].empty? &&
99
+ @buckets[:shared_dead].empty? &&
100
+ @buckets[:excluded_against].zero? &&
101
+ @buckets[:excluded_current].zero?
102
+ end
103
+
104
+ def alive_entry_array(entry)
105
+ r = entry[:record]
106
+ peer = entry[:peer_status]
107
+ peer_str = peer.nil? ? ABSENT_STATUS : peer.to_s
108
+ [r.file_path, r.line, r.operator, r.fingerprint, peer_str]
109
+ end
110
+
111
+ def shared_entry_array(entry)
112
+ r = entry[:against]
113
+ [r.file_path, r.line, shared_operator(entry), r.fingerprint]
114
+ end
115
+
116
+ # Mutant-sourced records always have operator=nil. When comparing mutant
117
+ # vs evilution, prefer whichever side has an operator so the shared row
118
+ # stays informative.
119
+ def shared_operator(entry)
120
+ entry[:against].operator || entry[:current].operator
121
+ end
122
+
123
+ def print_alive_block(io, bucket_key, peer_side_label)
124
+ entries = @buckets[bucket_key]
125
+ return if entries.empty?
126
+
127
+ io.puts("")
128
+ io.puts("#{bucket_key} (#{entries.length}):")
129
+ entries.each { |entry| io.puts(format_alive_row(entry, peer_side_label)) }
130
+ end
131
+
132
+ def print_shared_block(io, bucket_key)
133
+ entries = @buckets[bucket_key]
134
+ return if entries.empty?
135
+
136
+ io.puts("")
137
+ io.puts("#{bucket_key} (#{entries.length}):")
138
+ entries.each { |entry| io.puts(format_shared_row(entry)) }
139
+ end
140
+
141
+ def format_alive_row(entry, peer_side_label)
142
+ r = entry[:record]
143
+ peer = entry[:peer_status]
144
+ peer_str = peer.nil? ? ABSENT_STATUS : peer.to_s
145
+ " #{row_prefix(r)} (#{peer_side_label}: #{peer_str})"
146
+ end
147
+
148
+ def format_shared_row(entry)
149
+ r = entry[:against]
150
+ " #{row_prefix(r, operator: shared_operator(entry))}"
151
+ end
152
+
153
+ def row_prefix(record, operator: record.operator)
154
+ file_line = "#{record.file_path}:#{record.line}"
155
+ op_label = operator || MUTANT_OPERATOR
156
+ fp = record.fingerprint.to_s[0, FP_LENGTH]
157
+ "#{file_line.ljust(FILE_LINE_WIDTH)}#{op_label.ljust(OPERATOR_WIDTH)}#{fp}"
158
+ end
159
+ end
data/lib/evilution/cli.rb CHANGED
@@ -16,6 +16,7 @@ require_relative "cli/commands/session_list"
16
16
  require_relative "cli/commands/session_show"
17
17
  require_relative "cli/commands/session_diff"
18
18
  require_relative "cli/commands/session_gc"
19
+ require_relative "cli/commands/compare"
19
20
  require_relative "cli/commands/run"
20
21
 
21
22
  class Evilution::CLI
@@ -0,0 +1,109 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../compare"
4
+ require_relative "record"
5
+
6
+ module Evilution::Compare::Categorizer
7
+ ALIVE = %i[survived].freeze
8
+ DEAD = %i[killed timeout error].freeze
9
+ # neutral, equivalent, unresolved, unparseable are non-actionable signals
10
+ # — excluded from alive/dead buckets, counted in summary.
11
+
12
+ module_function
13
+
14
+ # @param against [Array<Record>] prior run (baseline)
15
+ # @param current [Array<Record>] current run
16
+ # @return [Hash] bucketed comparison result with keys:
17
+ # - `:alive_only_against` => `Array<{record: Record, peer_status: Symbol|nil}>`
18
+ # records that survived in against but not in current (or absent in current).
19
+ # `peer_status` is the current-side record's status symbol, or `nil` when
20
+ # no current-side record exists for that fingerprint.
21
+ # - `:alive_only_current` => `Array<{record: Record, peer_status: Symbol|nil}>`
22
+ # mirror of the above from the current side.
23
+ # - `:shared_alive` => `Array<{against: Record, current: Record}>`
24
+ # mutations that survived in both runs.
25
+ # - `:shared_dead` => `Array<{against: Record, current: Record}>`
26
+ # mutations killed/timed-out/errored in both runs.
27
+ # - `:excluded_against` => `Integer`
28
+ # count of against records with non-actionable statuses (neutral,
29
+ # equivalent, unresolved, unparseable).
30
+ # - `:excluded_current` => `Integer` mirror for the current side.
31
+ def call(against, current)
32
+ # Duplicate fingerprints within one side should not happen (Normalizer
33
+ # invariant). If they do, last write wins — we do not dedupe proactively.
34
+ against_by_fp = index_by_fingerprint(against)
35
+ current_by_fp = index_by_fingerprint(current)
36
+
37
+ buckets = {
38
+ alive_only_against: [],
39
+ alive_only_current: [],
40
+ shared_alive: [],
41
+ shared_dead: [],
42
+ excluded_against: 0,
43
+ excluded_current: 0
44
+ }
45
+
46
+ (against_by_fp.keys | current_by_fp.keys).each do |fp|
47
+ classify(against_by_fp[fp], current_by_fp[fp], buckets)
48
+ end
49
+
50
+ sort_buckets!(buckets)
51
+ buckets
52
+ end
53
+
54
+ # Dispatches one fingerprint pair into buckets.
55
+ # Either record may be nil (fingerprint present on only one side).
56
+ def classify(against_record, current_record, buckets)
57
+ count_excluded(against_record, current_record, buckets)
58
+ a_kind = kind_of(against_record)
59
+ c_kind = kind_of(current_record)
60
+
61
+ if a_kind == :alive && c_kind == :alive
62
+ buckets[:shared_alive] << { against: against_record, current: current_record }
63
+ elsif a_kind == :dead && c_kind == :dead
64
+ buckets[:shared_dead] << { against: against_record, current: current_record }
65
+ else
66
+ bucket_single_sided(against_record, current_record, a_kind, c_kind, buckets)
67
+ end
68
+ # A dead-only fingerprint (dead on one side, absent on the other) is
69
+ # intentionally not bucketed and not counted as excluded.
70
+ end
71
+
72
+ def count_excluded(against_record, current_record, buckets)
73
+ buckets[:excluded_against] += 1 if against_record && kind_of(against_record) == :excluded
74
+ buckets[:excluded_current] += 1 if current_record && kind_of(current_record) == :excluded
75
+ end
76
+
77
+ def bucket_single_sided(against_record, current_record, a_kind, c_kind, buckets)
78
+ # peer_status is the peer record's status symbol, or nil if peer absent.
79
+ # When the peer is excluded, its status symbol (e.g. :neutral) flows through.
80
+ a_peer = current_record && current_record.status
81
+ c_peer = against_record && against_record.status
82
+ buckets[:alive_only_against] << { record: against_record, peer_status: a_peer } if a_kind == :alive
83
+ buckets[:alive_only_current] << { record: current_record, peer_status: c_peer } if c_kind == :alive
84
+ end
85
+
86
+ # Returns :alive, :dead, :excluded, or nil (for nil records).
87
+ def kind_of(record)
88
+ return nil if record.nil?
89
+ return :alive if ALIVE.include?(record.status)
90
+ return :dead if DEAD.include?(record.status)
91
+
92
+ :excluded
93
+ end
94
+
95
+ def sort_buckets!(buckets)
96
+ buckets[:alive_only_against].sort_by! { |e| sort_key(e[:record]) }
97
+ buckets[:alive_only_current].sort_by! { |e| sort_key(e[:record]) }
98
+ buckets[:shared_alive].sort_by! { |e| sort_key(e[:against]) }
99
+ buckets[:shared_dead].sort_by! { |e| sort_key(e[:against]) }
100
+ end
101
+
102
+ def sort_key(record)
103
+ [record.file_path, record.line, record.fingerprint]
104
+ end
105
+
106
+ def index_by_fingerprint(records)
107
+ records.to_h { |r| [r.fingerprint, r] }
108
+ end
109
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../compare"
4
+ require_relative "normalizer"
5
+
6
+ module Evilution::Compare::Detector
7
+ module_function
8
+
9
+ def call(json)
10
+ raise Evilution::Compare::InvalidInput, "expected Hash, got #{json.class}" unless json.is_a?(Hash)
11
+
12
+ mutant = json.key?("subject_results")
13
+ evilution = json.key?("summary") && Evilution::Compare::Normalizer::EVILUTION_BUCKETS.any? { |k| json.key?(k) }
14
+
15
+ raise Evilution::Compare::InvalidInput, "ambiguous JSON shape - both mutant and evilution markers present" if mutant && evilution
16
+ return :mutant if mutant
17
+ return :evilution if evilution
18
+
19
+ raise Evilution::Compare::InvalidInput, "cannot detect tool from JSON shape"
20
+ end
21
+ end