evilution 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/.gitignore +4 -0
- data/.beads/interactions.jsonl +12 -0
- data/.beads/issues.jsonl +9 -6
- data/CHANGELOG.md +17 -0
- data/README.md +14 -10
- data/comparison_results/baseline_2026-04-09.md +35 -0
- data/comparison_results/operator_classification.md +79 -0
- data/comparison_results/operator_prioritization.md +68 -0
- data/docs/mutation_density_benchmark.md +91 -0
- data/lib/evilution/ast/parser.rb +2 -1
- data/lib/evilution/baseline.rb +14 -11
- data/lib/evilution/cli.rb +2 -1
- data/lib/evilution/config.rb +15 -3
- data/lib/evilution/disable_comment.rb +2 -1
- data/lib/evilution/integration/base.rb +98 -1
- data/lib/evilution/integration/minitest.rb +145 -0
- data/lib/evilution/integration/minitest_crash_detector.rb +55 -0
- data/lib/evilution/integration/rspec.rb +33 -100
- data/lib/evilution/mcp/mutate_tool.rb +6 -6
- data/lib/evilution/mutator/base.rb +4 -0
- data/lib/evilution/mutator/operator/bitwise_complement.rb +1 -1
- data/lib/evilution/mutator/operator/block_pass_removal.rb +30 -0
- data/lib/evilution/mutator/operator/ensure_removal.rb +1 -1
- data/lib/evilution/mutator/operator/index_to_at.rb +30 -0
- data/lib/evilution/mutator/operator/index_to_dig.rb +2 -2
- data/lib/evilution/mutator/operator/index_to_fetch.rb +2 -2
- data/lib/evilution/mutator/operator/keyword_argument.rb +1 -1
- data/lib/evilution/mutator/operator/regex_simplification.rb +169 -0
- data/lib/evilution/mutator/operator/rescue_body_replacement.rb +1 -1
- data/lib/evilution/mutator/operator/rescue_removal.rb +1 -1
- data/lib/evilution/mutator/registry.rb +3 -0
- data/lib/evilution/reporter/html.rb +2 -2
- data/lib/evilution/reporter/json.rb +2 -2
- data/lib/evilution/reporter/suggestion.rb +659 -2
- data/lib/evilution/runner.rb +31 -12
- data/lib/evilution/spec_resolver.rb +24 -16
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution.rb +4 -0
- data/scripts/benchmark_density +261 -0
- data/scripts/benchmark_density.yml +19 -0
- data/scripts/compare_mutations +404 -0
- data/scripts/compare_mutations.yml +24 -0
- data/scripts/mutant_json_adapter +224 -0
- metadata +16 -2
data/lib/evilution/ast/parser.rb
CHANGED
|
@@ -70,7 +70,8 @@ module Evilution::AST
|
|
|
70
70
|
end
|
|
71
71
|
|
|
72
72
|
loc = node.location
|
|
73
|
-
method_source = @source
|
|
73
|
+
method_source = @source.byteslice(loc.start_offset, loc.end_offset - loc.start_offset)
|
|
74
|
+
.force_encoding(@source.encoding)
|
|
74
75
|
|
|
75
76
|
@subjects << Evilution::Subject.new(
|
|
76
77
|
name: name,
|
data/lib/evilution/baseline.rb
CHANGED
|
@@ -14,9 +14,11 @@ class Evilution::Baseline
|
|
|
14
14
|
end
|
|
15
15
|
end
|
|
16
16
|
|
|
17
|
-
def initialize(spec_resolver: Evilution::SpecResolver.new, timeout: 30)
|
|
17
|
+
def initialize(spec_resolver: Evilution::SpecResolver.new, timeout: 30, runner: nil, fallback_dir: "spec")
|
|
18
18
|
@spec_resolver = spec_resolver
|
|
19
19
|
@timeout = timeout
|
|
20
|
+
@runner = runner
|
|
21
|
+
@fallback_dir = fallback_dir
|
|
20
22
|
end
|
|
21
23
|
|
|
22
24
|
def call(subjects)
|
|
@@ -33,10 +35,14 @@ class Evilution::Baseline
|
|
|
33
35
|
end
|
|
34
36
|
|
|
35
37
|
def run_spec_file(spec_file)
|
|
38
|
+
raise Evilution::Error, "no baseline runner configured" unless @runner
|
|
39
|
+
|
|
36
40
|
read_io, write_io = IO.pipe
|
|
37
41
|
pid = fork_spec_runner(spec_file, read_io, write_io)
|
|
38
42
|
write_io.close
|
|
39
43
|
read_result(read_io, pid)
|
|
44
|
+
rescue Evilution::Error
|
|
45
|
+
raise
|
|
40
46
|
rescue StandardError
|
|
41
47
|
false
|
|
42
48
|
ensure
|
|
@@ -45,19 +51,16 @@ class Evilution::Baseline
|
|
|
45
51
|
end
|
|
46
52
|
|
|
47
53
|
def fork_spec_runner(spec_file, read_io, write_io)
|
|
54
|
+
runner = @runner
|
|
48
55
|
Process.fork do
|
|
49
56
|
read_io.close
|
|
50
57
|
$stdout.reopen(File::NULL, "w")
|
|
51
58
|
$stderr.reopen(File::NULL, "w")
|
|
52
59
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
status = RSpec::Core::Runner.run(
|
|
56
|
-
["--format", "progress", "--no-color", "--order", "defined", spec_file]
|
|
57
|
-
)
|
|
58
|
-
Marshal.dump({ passed: status.zero? }, write_io)
|
|
60
|
+
passed = runner.call(spec_file)
|
|
61
|
+
Marshal.dump({ passed: passed }, write_io)
|
|
59
62
|
write_io.close
|
|
60
|
-
exit!(
|
|
63
|
+
exit!(passed ? 0 : 1)
|
|
61
64
|
end
|
|
62
65
|
end
|
|
63
66
|
|
|
@@ -97,10 +100,10 @@ class Evilution::Baseline
|
|
|
97
100
|
subjects.map do |s|
|
|
98
101
|
resolved = @spec_resolver.call(s.file_path)
|
|
99
102
|
if resolved.nil? && warned.add?(s.file_path)
|
|
100
|
-
warn "[evilution] No matching
|
|
101
|
-
"Use --spec to specify the
|
|
103
|
+
warn "[evilution] No matching test found for #{s.file_path}, running full suite. " \
|
|
104
|
+
"Use --spec to specify the test file."
|
|
102
105
|
end
|
|
103
|
-
resolved ||
|
|
106
|
+
resolved || @fallback_dir
|
|
104
107
|
end.uniq
|
|
105
108
|
end
|
|
106
109
|
end
|
data/lib/evilution/cli.rb
CHANGED
|
@@ -247,9 +247,10 @@ class Evilution::CLI
|
|
|
247
247
|
"(default: disabled; if provided without N, uses 1; use --fail-fast=N)") { @options[:fail_fast] ||= 1 }
|
|
248
248
|
opts.on("--no-baseline", "Skip baseline test suite check") { @options[:baseline] = false }
|
|
249
249
|
opts.on("--incremental", "Cache killed/timeout results; skip re-running them on unchanged files") { @options[:incremental] = true }
|
|
250
|
+
opts.on("--integration NAME", "Test integration: rspec, minitest (default: rspec)") { |i| @options[:integration] = i }
|
|
250
251
|
opts.on("--isolation STRATEGY", "Isolation: auto, fork, in_process (default: auto)") { |s| @options[:isolation] = s }
|
|
251
252
|
opts.on("--stdin", "Read target file paths from stdin (one per line)") { @options[:stdin] = true }
|
|
252
|
-
opts.on("--suggest-tests", "Generate concrete
|
|
253
|
+
opts.on("--suggest-tests", "Generate concrete test code in suggestions (RSpec or Minitest)") { @options[:suggest_tests] = true }
|
|
253
254
|
opts.on("--no-progress", "Disable progress bar") { @options[:progress] = false }
|
|
254
255
|
add_extra_flag_options(opts)
|
|
255
256
|
end
|
data/lib/evilution/config.rb
CHANGED
|
@@ -125,7 +125,7 @@ class Evilution::Config
|
|
|
125
125
|
# Minimum mutation score to pass (0.0 to 1.0, default: 0.0)
|
|
126
126
|
# min_score: 0.0
|
|
127
127
|
|
|
128
|
-
# Test integration: rspec (default: rspec)
|
|
128
|
+
# Test integration: rspec, minitest (default: rspec)
|
|
129
129
|
# integration: rspec
|
|
130
130
|
|
|
131
131
|
# Number of parallel workers (default: 1)
|
|
@@ -134,7 +134,7 @@ class Evilution::Config
|
|
|
134
134
|
# Stop after N surviving mutants (default: disabled)
|
|
135
135
|
# fail_fast: 1
|
|
136
136
|
|
|
137
|
-
# Generate concrete
|
|
137
|
+
# Generate concrete test code in suggestions, matching integration (default: false)
|
|
138
138
|
# suggest_tests: false
|
|
139
139
|
|
|
140
140
|
# Skip all string literal mutations inside heredocs (default: false)
|
|
@@ -172,7 +172,7 @@ class Evilution::Config
|
|
|
172
172
|
@format = merged[:format].to_sym
|
|
173
173
|
@target = merged[:target]
|
|
174
174
|
@min_score = merged[:min_score].to_f
|
|
175
|
-
@integration = merged[:integration]
|
|
175
|
+
@integration = validate_integration(merged[:integration])
|
|
176
176
|
@verbose = merged[:verbose]
|
|
177
177
|
@quiet = merged[:quiet]
|
|
178
178
|
@jobs = validate_jobs(merged[:jobs])
|
|
@@ -192,6 +192,18 @@ class Evilution::Config
|
|
|
192
192
|
@hooks = validate_hooks(merged[:hooks])
|
|
193
193
|
end
|
|
194
194
|
|
|
195
|
+
def validate_integration(value)
|
|
196
|
+
raise Evilution::ConfigError, "integration must be rspec or minitest, got nil" if value.nil?
|
|
197
|
+
|
|
198
|
+
value = value.to_sym
|
|
199
|
+
unless %i[rspec minitest].include?(value)
|
|
200
|
+
raise Evilution::ConfigError,
|
|
201
|
+
"integration must be rspec or minitest, got #{value.inspect}"
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
value
|
|
205
|
+
end
|
|
206
|
+
|
|
195
207
|
def validate_isolation(value)
|
|
196
208
|
raise Evilution::ConfigError, "isolation must be auto, fork, or in_process, got nil" if value.nil?
|
|
197
209
|
|
|
@@ -22,7 +22,8 @@ class Evilution::DisableComment
|
|
|
22
22
|
def classify_comments(parse_result, source)
|
|
23
23
|
parse_result.comments.filter_map do |comment|
|
|
24
24
|
loc = comment.location
|
|
25
|
-
text = source
|
|
25
|
+
text = source.byteslice(loc.start_offset, loc.end_offset - loc.start_offset)
|
|
26
|
+
.force_encoding(source.encoding)
|
|
26
27
|
|
|
27
28
|
if text.match?(DISABLE_MARKER)
|
|
28
29
|
line = source.lines[loc.start_line - 1]
|
|
@@ -1,13 +1,110 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "tmpdir"
|
|
3
5
|
require_relative "../integration"
|
|
6
|
+
require_relative "../temp_dir_tracker"
|
|
4
7
|
|
|
5
8
|
class Evilution::Integration::Base
|
|
9
|
+
def self.baseline_runner
|
|
10
|
+
raise NotImplementedError, "#{name}.baseline_runner must be implemented"
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def self.baseline_options
|
|
14
|
+
raise NotImplementedError, "#{name}.baseline_options must be implemented"
|
|
15
|
+
end
|
|
16
|
+
|
|
6
17
|
def initialize(hooks: nil)
|
|
7
18
|
@hooks = hooks
|
|
8
19
|
end
|
|
9
20
|
|
|
10
21
|
def call(mutation)
|
|
11
|
-
|
|
22
|
+
@temp_dir = nil
|
|
23
|
+
ensure_framework_loaded
|
|
24
|
+
fire_hook(:mutation_insert_pre, mutation: mutation, file_path: mutation.file_path)
|
|
25
|
+
apply_mutation(mutation)
|
|
26
|
+
fire_hook(:mutation_insert_post, mutation: mutation, file_path: mutation.file_path)
|
|
27
|
+
run_tests(mutation)
|
|
28
|
+
ensure
|
|
29
|
+
restore_original(mutation)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
private
|
|
33
|
+
|
|
34
|
+
def ensure_framework_loaded
|
|
35
|
+
raise NotImplementedError, "#{self.class}#ensure_framework_loaded must be implemented"
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def run_tests(_mutation)
|
|
39
|
+
raise NotImplementedError, "#{self.class}#run_tests must be implemented"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def build_args(_mutation)
|
|
43
|
+
raise NotImplementedError, "#{self.class}#build_args must be implemented"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def reset_state
|
|
47
|
+
raise NotImplementedError, "#{self.class}#reset_state must be implemented"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def fire_hook(event, **payload)
|
|
51
|
+
@hooks.fire(event, **payload) if @hooks
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def apply_mutation(mutation)
|
|
55
|
+
@temp_dir = Dir.mktmpdir("evilution")
|
|
56
|
+
Evilution::TempDirTracker.register(@temp_dir)
|
|
57
|
+
@displaced_feature = nil
|
|
58
|
+
subpath = resolve_require_subpath(mutation.file_path)
|
|
59
|
+
|
|
60
|
+
if subpath
|
|
61
|
+
dest = File.join(@temp_dir, subpath)
|
|
62
|
+
FileUtils.mkdir_p(File.dirname(dest))
|
|
63
|
+
File.write(dest, mutation.mutated_source)
|
|
64
|
+
$LOAD_PATH.unshift(@temp_dir)
|
|
65
|
+
displace_loaded_feature(mutation.file_path)
|
|
66
|
+
else
|
|
67
|
+
absolute = File.expand_path(mutation.file_path)
|
|
68
|
+
dest = File.join(@temp_dir, absolute)
|
|
69
|
+
FileUtils.mkdir_p(File.dirname(dest))
|
|
70
|
+
File.write(dest, mutation.mutated_source)
|
|
71
|
+
load(dest)
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def restore_original(_mutation)
|
|
76
|
+
return unless @temp_dir
|
|
77
|
+
|
|
78
|
+
$LOAD_PATH.delete(@temp_dir)
|
|
79
|
+
$LOADED_FEATURES.reject! { |f| f.start_with?(@temp_dir) }
|
|
80
|
+
$LOADED_FEATURES << @displaced_feature if @displaced_feature && !$LOADED_FEATURES.include?(@displaced_feature)
|
|
81
|
+
@displaced_feature = nil
|
|
82
|
+
FileUtils.rm_rf(@temp_dir)
|
|
83
|
+
Evilution::TempDirTracker.unregister(@temp_dir)
|
|
84
|
+
@temp_dir = nil
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def resolve_require_subpath(file_path)
|
|
88
|
+
absolute = File.expand_path(file_path)
|
|
89
|
+
best_subpath = nil
|
|
90
|
+
|
|
91
|
+
$LOAD_PATH.each do |entry|
|
|
92
|
+
dir = File.expand_path(entry)
|
|
93
|
+
prefix = dir.end_with?("/") ? dir : "#{dir}/"
|
|
94
|
+
next unless absolute.start_with?(prefix)
|
|
95
|
+
|
|
96
|
+
candidate = absolute.delete_prefix(prefix)
|
|
97
|
+
best_subpath = candidate if best_subpath.nil? || candidate.length < best_subpath.length
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
best_subpath
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def displace_loaded_feature(file_path)
|
|
104
|
+
absolute = File.expand_path(file_path)
|
|
105
|
+
return unless $LOADED_FEATURES.include?(absolute)
|
|
106
|
+
|
|
107
|
+
@displaced_feature = absolute
|
|
108
|
+
$LOADED_FEATURES.delete(absolute)
|
|
12
109
|
end
|
|
13
110
|
end
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "stringio"
|
|
4
|
+
require_relative "base"
|
|
5
|
+
require_relative "minitest_crash_detector"
|
|
6
|
+
require_relative "../spec_resolver"
|
|
7
|
+
|
|
8
|
+
require_relative "../integration"
|
|
9
|
+
|
|
10
|
+
class Evilution::Integration::Minitest < Evilution::Integration::Base
|
|
11
|
+
def self.baseline_runner
|
|
12
|
+
lambda { |test_file|
|
|
13
|
+
require "minitest"
|
|
14
|
+
require "stringio"
|
|
15
|
+
::Minitest::Runnable.runnables.clear
|
|
16
|
+
files = File.directory?(test_file) ? Dir.glob(File.join(test_file, "**/*_test.rb")) : [test_file]
|
|
17
|
+
files.each { |f| load(File.expand_path(f)) }
|
|
18
|
+
out = StringIO.new
|
|
19
|
+
options = ::Minitest.process_args(["--seed", "0"])
|
|
20
|
+
options[:io] = out
|
|
21
|
+
reporter = ::Minitest::CompositeReporter.new
|
|
22
|
+
reporter << ::Minitest::SummaryReporter.new(out, options)
|
|
23
|
+
reporter.start
|
|
24
|
+
::Minitest.__run(reporter, options)
|
|
25
|
+
reporter.report
|
|
26
|
+
reporter.passed?
|
|
27
|
+
}
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def self.baseline_options
|
|
31
|
+
{
|
|
32
|
+
runner: baseline_runner,
|
|
33
|
+
spec_resolver: Evilution::SpecResolver.new(test_dir: "test", test_suffix: "_test.rb", request_dir: "integration"),
|
|
34
|
+
fallback_dir: "test"
|
|
35
|
+
}
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def initialize(test_files: nil, hooks: nil)
|
|
39
|
+
@test_files = test_files
|
|
40
|
+
@minitest_loaded = false
|
|
41
|
+
@spec_resolver = Evilution::SpecResolver.new(test_dir: "test", test_suffix: "_test.rb", request_dir: "integration")
|
|
42
|
+
@crash_detector = nil
|
|
43
|
+
@warned_files = Set.new
|
|
44
|
+
super(hooks: hooks)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
attr_reader :test_files
|
|
50
|
+
|
|
51
|
+
def ensure_framework_loaded
|
|
52
|
+
return if @minitest_loaded
|
|
53
|
+
|
|
54
|
+
fire_hook(:setup_integration_pre, integration: :minitest)
|
|
55
|
+
require "minitest"
|
|
56
|
+
@minitest_loaded = true
|
|
57
|
+
fire_hook(:setup_integration_post, integration: :minitest)
|
|
58
|
+
rescue LoadError => e
|
|
59
|
+
raise Evilution::Error, "minitest is required but not available: #{e.message}"
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def run_tests(mutation)
|
|
63
|
+
reset_state
|
|
64
|
+
files = resolve_test_files(mutation)
|
|
65
|
+
command = "ruby -Itest #{files.join(" ")}"
|
|
66
|
+
|
|
67
|
+
files.each { |f| load(File.expand_path(f)) }
|
|
68
|
+
|
|
69
|
+
args = build_args(mutation)
|
|
70
|
+
detector = reset_crash_detector
|
|
71
|
+
passed = run_minitest(args, detector)
|
|
72
|
+
|
|
73
|
+
build_minitest_result(passed, command, detector)
|
|
74
|
+
rescue StandardError => e
|
|
75
|
+
{ passed: false, error: e.message, test_command: command }
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def build_args(_mutation)
|
|
79
|
+
["--seed", "0"]
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def reset_state
|
|
83
|
+
::Minitest::Runnable.runnables.clear
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def run_minitest(args, detector)
|
|
87
|
+
out = StringIO.new
|
|
88
|
+
options = ::Minitest.process_args(args)
|
|
89
|
+
options[:io] = out
|
|
90
|
+
|
|
91
|
+
reporter = ::Minitest::CompositeReporter.new
|
|
92
|
+
reporter << ::Minitest::SummaryReporter.new(out, options)
|
|
93
|
+
reporter << detector
|
|
94
|
+
|
|
95
|
+
reporter.start
|
|
96
|
+
::Minitest.__run(reporter, options)
|
|
97
|
+
reporter.report
|
|
98
|
+
|
|
99
|
+
reporter.passed?
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def reset_crash_detector
|
|
103
|
+
if @crash_detector
|
|
104
|
+
@crash_detector.reset
|
|
105
|
+
else
|
|
106
|
+
@crash_detector = Evilution::Integration::MinitestCrashDetector.new
|
|
107
|
+
end
|
|
108
|
+
@crash_detector
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def build_minitest_result(passed, command, detector)
|
|
112
|
+
if passed
|
|
113
|
+
{ passed: true, test_command: command }
|
|
114
|
+
elsif detector.only_crashes?
|
|
115
|
+
{ passed: false, error: "test crashes: #{detector.crash_summary}", test_command: command }
|
|
116
|
+
else
|
|
117
|
+
{ passed: false, test_command: command }
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def resolve_test_files(mutation)
|
|
122
|
+
return test_files if test_files
|
|
123
|
+
|
|
124
|
+
resolved = @spec_resolver.call(mutation.file_path)
|
|
125
|
+
unless resolved
|
|
126
|
+
warn_unresolved_test(mutation.file_path)
|
|
127
|
+
return glob_test_files
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
[resolved]
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def glob_test_files
|
|
134
|
+
files = Dir.glob("test/**/*_test.rb")
|
|
135
|
+
files.empty? ? ["test"] : files
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def warn_unresolved_test(file_path)
|
|
139
|
+
return if @warned_files.include?(file_path)
|
|
140
|
+
|
|
141
|
+
@warned_files << file_path
|
|
142
|
+
warn "[evilution] No matching test found for #{file_path}, running full suite. " \
|
|
143
|
+
"Use --spec to specify the test file."
|
|
144
|
+
end
|
|
145
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../integration"
|
|
4
|
+
|
|
5
|
+
class Evilution::Integration::MinitestCrashDetector
|
|
6
|
+
def initialize
|
|
7
|
+
reset
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def start
|
|
11
|
+
# Required by Minitest reporter interface
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def report
|
|
15
|
+
# Required by Minitest reporter interface
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def passed?
|
|
19
|
+
@crashes.empty?
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def reset
|
|
23
|
+
@assertion_failures = 0
|
|
24
|
+
@crashes = []
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def record(result)
|
|
28
|
+
result.failures.each do |failure|
|
|
29
|
+
if failure.is_a?(::Minitest::UnexpectedError)
|
|
30
|
+
@crashes << failure.error
|
|
31
|
+
elsif failure.is_a?(::Minitest::Assertion)
|
|
32
|
+
@assertion_failures += 1
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def has_assertion_failure? # rubocop:disable Naming/PredicatePrefix
|
|
38
|
+
@assertion_failures.positive?
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def has_crash? # rubocop:disable Naming/PredicatePrefix
|
|
42
|
+
@crashes.any?
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def only_crashes?
|
|
46
|
+
@crashes.any? && @assertion_failures.zero?
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def crash_summary
|
|
50
|
+
return nil if @crashes.empty?
|
|
51
|
+
|
|
52
|
+
types = @crashes.map { |e| e.class.name }.uniq
|
|
53
|
+
"#{types.join(", ")} (#{@crashes.length} crash#{"es" unless @crashes.length == 1})"
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -1,17 +1,29 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "fileutils"
|
|
4
3
|
require "stringio"
|
|
5
|
-
require "tmpdir"
|
|
6
4
|
require_relative "base"
|
|
7
5
|
require_relative "crash_detector"
|
|
8
6
|
require_relative "../spec_resolver"
|
|
9
7
|
require_relative "../related_spec_heuristic"
|
|
10
|
-
require_relative "../temp_dir_tracker"
|
|
11
8
|
|
|
12
9
|
require_relative "../integration"
|
|
13
10
|
|
|
14
11
|
class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
12
|
+
def self.baseline_runner
|
|
13
|
+
lambda { |spec_file|
|
|
14
|
+
require "rspec/core"
|
|
15
|
+
::RSpec.reset
|
|
16
|
+
status = ::RSpec::Core::Runner.run(
|
|
17
|
+
["--format", "progress", "--no-color", "--order", "defined", spec_file]
|
|
18
|
+
)
|
|
19
|
+
status.zero?
|
|
20
|
+
}
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def self.baseline_options
|
|
24
|
+
{ runner: baseline_runner }
|
|
25
|
+
end
|
|
26
|
+
|
|
15
27
|
def initialize(test_files: nil, hooks: nil)
|
|
16
28
|
@test_files = test_files
|
|
17
29
|
@rspec_loaded = false
|
|
@@ -22,105 +34,24 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
|
22
34
|
super(hooks: hooks)
|
|
23
35
|
end
|
|
24
36
|
|
|
25
|
-
def call(mutation)
|
|
26
|
-
@temp_dir = nil
|
|
27
|
-
ensure_rspec_loaded
|
|
28
|
-
@hooks.fire(:mutation_insert_pre, mutation: mutation, file_path: mutation.file_path) if @hooks
|
|
29
|
-
apply_mutation(mutation)
|
|
30
|
-
@hooks.fire(:mutation_insert_post, mutation: mutation, file_path: mutation.file_path) if @hooks
|
|
31
|
-
run_rspec(mutation)
|
|
32
|
-
ensure
|
|
33
|
-
restore_original(mutation)
|
|
34
|
-
end
|
|
35
|
-
|
|
36
37
|
private
|
|
37
38
|
|
|
38
39
|
attr_reader :test_files
|
|
39
40
|
|
|
40
|
-
def
|
|
41
|
+
def ensure_framework_loaded
|
|
41
42
|
return if @rspec_loaded
|
|
42
43
|
|
|
43
|
-
|
|
44
|
+
fire_hook(:setup_integration_pre, integration: :rspec)
|
|
44
45
|
require "rspec/core"
|
|
45
46
|
Evilution::Integration::CrashDetector.register_with_rspec
|
|
46
47
|
@rspec_loaded = true
|
|
47
|
-
|
|
48
|
+
fire_hook(:setup_integration_post, integration: :rspec)
|
|
48
49
|
rescue LoadError => e
|
|
49
50
|
raise Evilution::Error, "rspec-core is required but not available: #{e.message}"
|
|
50
51
|
end
|
|
51
52
|
|
|
52
|
-
def
|
|
53
|
-
|
|
54
|
-
Evilution::TempDirTracker.register(@temp_dir)
|
|
55
|
-
@displaced_feature = nil
|
|
56
|
-
subpath = resolve_require_subpath(mutation.file_path)
|
|
57
|
-
|
|
58
|
-
if subpath
|
|
59
|
-
dest = File.join(@temp_dir, subpath)
|
|
60
|
-
FileUtils.mkdir_p(File.dirname(dest))
|
|
61
|
-
File.write(dest, mutation.mutated_source)
|
|
62
|
-
$LOAD_PATH.unshift(@temp_dir)
|
|
63
|
-
displace_loaded_feature(mutation.file_path)
|
|
64
|
-
else
|
|
65
|
-
absolute = File.expand_path(mutation.file_path)
|
|
66
|
-
dest = File.join(@temp_dir, absolute)
|
|
67
|
-
FileUtils.mkdir_p(File.dirname(dest))
|
|
68
|
-
File.write(dest, mutation.mutated_source)
|
|
69
|
-
load(dest)
|
|
70
|
-
end
|
|
71
|
-
end
|
|
72
|
-
|
|
73
|
-
def restore_original(mutation) # rubocop:disable Lint/UnusedMethodArgument
|
|
74
|
-
return unless @temp_dir
|
|
75
|
-
|
|
76
|
-
$LOAD_PATH.delete(@temp_dir)
|
|
77
|
-
$LOADED_FEATURES.reject! { |f| f.start_with?(@temp_dir) }
|
|
78
|
-
$LOADED_FEATURES << @displaced_feature if @displaced_feature && !$LOADED_FEATURES.include?(@displaced_feature)
|
|
79
|
-
@displaced_feature = nil
|
|
80
|
-
FileUtils.rm_rf(@temp_dir)
|
|
81
|
-
Evilution::TempDirTracker.unregister(@temp_dir)
|
|
82
|
-
@temp_dir = nil
|
|
83
|
-
end
|
|
84
|
-
|
|
85
|
-
def resolve_require_subpath(file_path)
|
|
86
|
-
absolute = File.expand_path(file_path)
|
|
87
|
-
best_subpath = nil
|
|
88
|
-
|
|
89
|
-
$LOAD_PATH.each do |entry|
|
|
90
|
-
dir = File.expand_path(entry)
|
|
91
|
-
prefix = dir.end_with?("/") ? dir : "#{dir}/"
|
|
92
|
-
next unless absolute.start_with?(prefix)
|
|
93
|
-
|
|
94
|
-
candidate = absolute.delete_prefix(prefix)
|
|
95
|
-
best_subpath = candidate if best_subpath.nil? || candidate.length < best_subpath.length
|
|
96
|
-
end
|
|
97
|
-
|
|
98
|
-
best_subpath
|
|
99
|
-
end
|
|
100
|
-
|
|
101
|
-
def displace_loaded_feature(file_path)
|
|
102
|
-
absolute = File.expand_path(file_path)
|
|
103
|
-
return unless $LOADED_FEATURES.include?(absolute)
|
|
104
|
-
|
|
105
|
-
@displaced_feature = absolute
|
|
106
|
-
$LOADED_FEATURES.delete(absolute)
|
|
107
|
-
end
|
|
108
|
-
|
|
109
|
-
def run_rspec(mutation)
|
|
110
|
-
# When used via the Runner with Isolation::Fork, each mutation is executed
|
|
111
|
-
# in its own forked child process, so RSpec state (loaded example groups,
|
|
112
|
-
# world, configuration) cannot accumulate across mutation runs — the child
|
|
113
|
-
# process exits after each run.
|
|
114
|
-
#
|
|
115
|
-
# This integration can also be invoked directly (e.g. in specs or alternative
|
|
116
|
-
# runners) without fork isolation. clear_examples reuses the existing World
|
|
117
|
-
# and Configuration (avoiding per-run instance growth) while clearing loaded
|
|
118
|
-
# example groups, constants, and configuration state.
|
|
119
|
-
if ::RSpec.respond_to?(:clear_examples)
|
|
120
|
-
::RSpec.clear_examples
|
|
121
|
-
else
|
|
122
|
-
::RSpec.reset
|
|
123
|
-
end
|
|
53
|
+
def run_tests(mutation)
|
|
54
|
+
reset_state
|
|
124
55
|
|
|
125
56
|
out = StringIO.new
|
|
126
57
|
err = StringIO.new
|
|
@@ -139,6 +70,19 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
|
139
70
|
release_rspec_state(eg_before)
|
|
140
71
|
end
|
|
141
72
|
|
|
73
|
+
def build_args(mutation)
|
|
74
|
+
files = resolve_test_files(mutation)
|
|
75
|
+
["--format", "progress", "--no-color", "--order", "defined", *files]
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def reset_state
|
|
79
|
+
if ::RSpec.respond_to?(:clear_examples)
|
|
80
|
+
::RSpec.clear_examples
|
|
81
|
+
else
|
|
82
|
+
::RSpec.reset
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
142
86
|
def snapshot_example_groups
|
|
143
87
|
groups = Set.new
|
|
144
88
|
ObjectSpace.each_object(Class) do |klass|
|
|
@@ -150,11 +94,6 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
|
150
94
|
|
|
151
95
|
def release_rspec_state(eg_before)
|
|
152
96
|
release_example_groups(eg_before)
|
|
153
|
-
# Remove ExampleGroups constants so the named reference is dropped.
|
|
154
|
-
# We avoid a full RSpec.reset here because it creates new World and
|
|
155
|
-
# Configuration instances each call; the pre-run reset already handles
|
|
156
|
-
# that. Instead, clear the world's example_groups array (which holds
|
|
157
|
-
# direct class references) and the source cache.
|
|
158
97
|
::RSpec::ExampleGroups.remove_all_constants if defined?(::RSpec::ExampleGroups)
|
|
159
98
|
release_world_example_groups
|
|
160
99
|
end
|
|
@@ -166,7 +105,6 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
|
166
105
|
next unless klass < ::RSpec::Core::ExampleGroup
|
|
167
106
|
next if eg_before.include?(klass.object_id)
|
|
168
107
|
|
|
169
|
-
# Remove nested module constants (LetDefinitions, NamedSubjectPreventSuper)
|
|
170
108
|
klass.constants(false).each do |const|
|
|
171
109
|
klass.send(:remove_const, const)
|
|
172
110
|
rescue NameError # rubocop:disable Lint/SuppressedException
|
|
@@ -205,11 +143,6 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
|
|
|
205
143
|
end
|
|
206
144
|
end
|
|
207
145
|
|
|
208
|
-
def build_args(mutation)
|
|
209
|
-
files = resolve_test_files(mutation)
|
|
210
|
-
["--format", "progress", "--no-color", "--order", "defined", *files]
|
|
211
|
-
end
|
|
212
|
-
|
|
213
146
|
def resolve_test_files(mutation)
|
|
214
147
|
return test_files if test_files
|
|
215
148
|
|
|
@@ -12,7 +12,7 @@ require_relative "../mcp"
|
|
|
12
12
|
class Evilution::MCP::MutateTool < MCP::Tool
|
|
13
13
|
tool_name "evilution-mutate"
|
|
14
14
|
description "Run mutation testing on Ruby source files. " \
|
|
15
|
-
"Use suggest_tests: true to get concrete
|
|
15
|
+
"Use suggest_tests: true to get concrete test code (RSpec or Minitest) for surviving mutants."
|
|
16
16
|
input_schema(
|
|
17
17
|
properties: {
|
|
18
18
|
files: {
|
|
@@ -43,7 +43,7 @@ class Evilution::MCP::MutateTool < MCP::Tool
|
|
|
43
43
|
},
|
|
44
44
|
suggest_tests: {
|
|
45
45
|
type: "boolean",
|
|
46
|
-
description: "When true, suggestions for survived mutants include concrete
|
|
46
|
+
description: "When true, suggestions for survived mutants include concrete test code " \
|
|
47
47
|
"instead of static description text (default: false)"
|
|
48
48
|
},
|
|
49
49
|
verbosity: {
|
|
@@ -64,10 +64,10 @@ class Evilution::MCP::MutateTool < MCP::Tool
|
|
|
64
64
|
config_opts = build_config_opts(parsed_files, line_ranges, target, timeout, jobs, fail_fast, spec,
|
|
65
65
|
suggest_tests)
|
|
66
66
|
config = Evilution::Config.new(**config_opts)
|
|
67
|
-
on_result = build_streaming_callback(server_context, suggest_tests)
|
|
67
|
+
on_result = build_streaming_callback(server_context, suggest_tests, config.integration)
|
|
68
68
|
runner = Evilution::Runner.new(config: config, on_result: on_result)
|
|
69
69
|
summary = runner.call
|
|
70
|
-
report = Evilution::Reporter::JSON.new(suggest_tests: suggest_tests == true).call(summary)
|
|
70
|
+
report = Evilution::Reporter::JSON.new(suggest_tests: suggest_tests == true, integration: config.integration).call(summary)
|
|
71
71
|
compact = trim_report(report, normalize_verbosity(verbosity))
|
|
72
72
|
|
|
73
73
|
::MCP::Tool::Response.new([{ type: "text", text: compact }])
|
|
@@ -156,10 +156,10 @@ class Evilution::MCP::MutateTool < MCP::Tool
|
|
|
156
156
|
data[key].each { |entry| entry.delete("diff") }
|
|
157
157
|
end
|
|
158
158
|
|
|
159
|
-
def build_streaming_callback(server_context, suggest_tests)
|
|
159
|
+
def build_streaming_callback(server_context, suggest_tests, integration)
|
|
160
160
|
return nil unless suggest_tests && server_context.respond_to?(:report_progress)
|
|
161
161
|
|
|
162
|
-
suggestion = Evilution::Reporter::Suggestion.new(suggest_tests: true)
|
|
162
|
+
suggestion = Evilution::Reporter::Suggestion.new(suggest_tests: true, integration: integration)
|
|
163
163
|
survivor_index = 0
|
|
164
164
|
|
|
165
165
|
proc do |result|
|