evilution 0.16.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/.migration-hint-ts +1 -1
- data/.beads/issues.jsonl +47 -46
- data/CHANGELOG.md +48 -0
- data/README.md +143 -50
- data/docs/ast_pattern_syntax.md +210 -0
- data/lib/evilution/ast/pattern/filter.rb +25 -0
- data/lib/evilution/ast/pattern/matcher.rb +107 -0
- data/lib/evilution/ast/pattern/parser.rb +185 -0
- data/lib/evilution/ast/pattern.rb +4 -0
- data/lib/evilution/ast/sorbet_sig_detector.rb +52 -0
- data/lib/evilution/cli.rb +400 -24
- data/lib/evilution/config.rb +43 -2
- data/lib/evilution/disable_comment.rb +90 -0
- data/lib/evilution/hooks/loader.rb +35 -0
- data/lib/evilution/hooks/registry.rb +60 -0
- data/lib/evilution/hooks.rb +58 -0
- data/lib/evilution/integration/base.rb +4 -0
- data/lib/evilution/integration/rspec.rb +6 -2
- data/lib/evilution/isolation/fork.rb +5 -0
- data/lib/evilution/mcp/session_diff_tool.rb +5 -35
- data/lib/evilution/mutator/base.rb +4 -1
- data/lib/evilution/mutator/operator/collection_return.rb +33 -0
- data/lib/evilution/mutator/operator/defined_check.rb +16 -0
- data/lib/evilution/mutator/operator/index_assignment_removal.rb +18 -0
- data/lib/evilution/mutator/operator/index_to_dig.rb +58 -0
- data/lib/evilution/mutator/operator/index_to_fetch.rb +30 -0
- data/lib/evilution/mutator/operator/keyword_argument.rb +91 -0
- data/lib/evilution/mutator/operator/mixin_removal.rb +2 -1
- data/lib/evilution/mutator/operator/multiple_assignment.rb +47 -0
- data/lib/evilution/mutator/operator/pattern_matching_alternative.rb +46 -0
- data/lib/evilution/mutator/operator/pattern_matching_array.rb +97 -0
- data/lib/evilution/mutator/operator/pattern_matching_guard.rb +44 -0
- data/lib/evilution/mutator/operator/regex_capture.rb +43 -0
- data/lib/evilution/mutator/operator/scalar_return.rb +37 -0
- data/lib/evilution/mutator/operator/splat_operator.rb +46 -0
- data/lib/evilution/mutator/operator/superclass_removal.rb +2 -1
- data/lib/evilution/mutator/operator/yield_statement.rb +51 -0
- data/lib/evilution/mutator/registry.rb +17 -3
- data/lib/evilution/parallel/pool.rb +7 -51
- data/lib/evilution/parallel/work_queue.rb +224 -0
- data/lib/evilution/reporter/cli.rb +22 -1
- data/lib/evilution/reporter/html.rb +76 -3
- data/lib/evilution/reporter/json.rb +23 -2
- data/lib/evilution/reporter/suggestion.rb +115 -1
- data/lib/evilution/result/summary.rb +20 -2
- data/lib/evilution/runner.rb +133 -13
- data/lib/evilution/session/diff.rb +85 -0
- data/lib/evilution/session/store.rb +5 -2
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution.rb +23 -0
- metadata +28 -2
|
@@ -13,11 +13,13 @@ class Evilution::Reporter::CLI
|
|
|
13
13
|
lines << mutations_line(summary)
|
|
14
14
|
lines << score_line(summary)
|
|
15
15
|
lines << duration_line(summary)
|
|
16
|
+
lines << efficiency_line(summary) if summary.duration.positive?
|
|
16
17
|
peak = summary.peak_memory_mb
|
|
17
18
|
lines << peak_memory_line(peak) if peak
|
|
18
19
|
append_survived(lines, summary)
|
|
19
20
|
append_neutral(lines, summary)
|
|
20
21
|
append_equivalent(lines, summary)
|
|
22
|
+
append_disabled(lines, summary)
|
|
21
23
|
lines << ""
|
|
22
24
|
lines << "[TRUNCATED] Stopped early due to --fail-fast" if summary.truncated?
|
|
23
25
|
lines << result_line(summary)
|
|
@@ -51,6 +53,14 @@ class Evilution::Reporter::CLI
|
|
|
51
53
|
summary.equivalent_results.each { |result| lines << format_neutral(result) }
|
|
52
54
|
end
|
|
53
55
|
|
|
56
|
+
def append_disabled(lines, summary)
|
|
57
|
+
return unless summary.disabled_mutations.any?
|
|
58
|
+
|
|
59
|
+
lines << ""
|
|
60
|
+
lines << "Disabled mutations (skipped by # evilution:disable):"
|
|
61
|
+
summary.disabled_mutations.each { |mutation| lines << format_disabled(mutation) }
|
|
62
|
+
end
|
|
63
|
+
|
|
54
64
|
def header
|
|
55
65
|
"Evilution v#{Evilution::VERSION} — Mutation Testing Results"
|
|
56
66
|
end
|
|
@@ -60,6 +70,7 @@ class Evilution::Reporter::CLI
|
|
|
60
70
|
"#{summary.survived} survived, #{summary.timed_out} timed out"
|
|
61
71
|
parts += ", #{summary.neutral} neutral" if summary.neutral.positive?
|
|
62
72
|
parts += ", #{summary.equivalent} equivalent" if summary.equivalent.positive?
|
|
73
|
+
parts += ", #{summary.skipped} skipped" if summary.skipped.positive?
|
|
63
74
|
parts
|
|
64
75
|
end
|
|
65
76
|
|
|
@@ -73,11 +84,17 @@ class Evilution::Reporter::CLI
|
|
|
73
84
|
"Duration: #{format("%.2f", summary.duration)}s"
|
|
74
85
|
end
|
|
75
86
|
|
|
87
|
+
def efficiency_line(summary)
|
|
88
|
+
pct = format("%.2f%%", summary.efficiency * 100)
|
|
89
|
+
rate = format("%.2f", summary.mutations_per_second)
|
|
90
|
+
"Efficiency: #{pct} killtime, #{rate} mutations/s"
|
|
91
|
+
end
|
|
92
|
+
|
|
76
93
|
def format_survived(result)
|
|
77
94
|
mutation = result.mutation
|
|
78
95
|
location = "#{mutation.file_path}:#{mutation.line}"
|
|
79
96
|
diff_lines = mutation.diff.split("\n").map { |l| " #{l}" }.join("\n")
|
|
80
|
-
" #{mutation.operator_name}: #{location}\n#{diff_lines}"
|
|
97
|
+
" #{mutation.operator_name}: #{location} (#{mutation.subject.name})\n#{diff_lines}"
|
|
81
98
|
end
|
|
82
99
|
|
|
83
100
|
def format_neutral(result)
|
|
@@ -85,6 +102,10 @@ class Evilution::Reporter::CLI
|
|
|
85
102
|
" #{mutation.operator_name}: #{mutation.file_path}:#{mutation.line}"
|
|
86
103
|
end
|
|
87
104
|
|
|
105
|
+
def format_disabled(mutation)
|
|
106
|
+
" #{mutation.operator_name}: #{mutation.file_path}:#{mutation.line}"
|
|
107
|
+
end
|
|
108
|
+
|
|
88
109
|
def result_line(summary)
|
|
89
110
|
min_score = 0.8
|
|
90
111
|
pass_fail = summary.success?(min_score: min_score) ? "PASS" : "FAIL"
|
|
@@ -6,8 +6,10 @@ require_relative "suggestion"
|
|
|
6
6
|
require_relative "../reporter"
|
|
7
7
|
|
|
8
8
|
class Evilution::Reporter::HTML
|
|
9
|
-
def initialize
|
|
9
|
+
def initialize(baseline: nil)
|
|
10
10
|
@suggestion = Evilution::Reporter::Suggestion.new
|
|
11
|
+
@baseline = baseline
|
|
12
|
+
@baseline_keys = build_baseline_keys(baseline)
|
|
11
13
|
end
|
|
12
14
|
|
|
13
15
|
def call(summary)
|
|
@@ -40,6 +42,7 @@ class Evilution::Reporter::HTML
|
|
|
40
42
|
<body>
|
|
41
43
|
#{build_header(summary)}
|
|
42
44
|
#{build_summary_cards(summary)}
|
|
45
|
+
#{build_baseline_comparison(summary)}
|
|
43
46
|
#{build_truncation_notice(summary)}
|
|
44
47
|
#{build_file_sections(files)}
|
|
45
48
|
#{build_footer}
|
|
@@ -77,12 +80,29 @@ class Evilution::Reporter::HTML
|
|
|
77
80
|
<div class="card"><span class="card-value">#{summary.errors}</span><span class="card-label">Errors</span></div>
|
|
78
81
|
<div class="card"><span class="card-value">#{summary.neutral}</span><span class="card-label">Neutral</span></div>
|
|
79
82
|
<div class="card"><span class="card-value">#{summary.equivalent}</span><span class="card-label">Equivalent</span></div>
|
|
83
|
+
#{build_skipped_card(summary)}
|
|
80
84
|
<div class="card"><span class="card-value">#{format("%.2f", summary.duration)}s</span><span class="card-label">Duration</span></div>
|
|
85
|
+
#{build_efficiency_cards(summary)}
|
|
81
86
|
#{peak_html}
|
|
82
87
|
</section>
|
|
83
88
|
HTML
|
|
84
89
|
end
|
|
85
90
|
|
|
91
|
+
def build_efficiency_cards(summary)
|
|
92
|
+
return "" unless summary.duration.positive?
|
|
93
|
+
|
|
94
|
+
pct = format("%.1f%%", summary.efficiency * 100)
|
|
95
|
+
rate = format("%.2f", summary.mutations_per_second)
|
|
96
|
+
%(<div class="card"><span class="card-value">#{pct}</span><span class="card-label">Efficiency</span></div>) +
|
|
97
|
+
%(<div class="card"><span class="card-value">#{rate}/s</span><span class="card-label">Rate</span></div>)
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def build_skipped_card(summary)
|
|
101
|
+
return "" unless summary.skipped.positive?
|
|
102
|
+
|
|
103
|
+
%(<div class="card"><span class="card-value">#{summary.skipped}</span><span class="card-label">Skipped</span></div>)
|
|
104
|
+
end
|
|
105
|
+
|
|
86
106
|
def build_truncation_notice(summary)
|
|
87
107
|
return "" unless summary.truncated?
|
|
88
108
|
|
|
@@ -149,10 +169,13 @@ class Evilution::Reporter::HTML
|
|
|
149
169
|
mutation = result.mutation
|
|
150
170
|
suggestion_text = @suggestion.suggestion_for(mutation)
|
|
151
171
|
diff_html = format_diff(mutation.diff)
|
|
172
|
+
regression = regression?(mutation)
|
|
173
|
+
entry_class = regression ? "survived-entry regression" : "survived-entry"
|
|
174
|
+
regression_badge = regression ? ' <span class="regression-badge">NEW REGRESSION</span>' : ""
|
|
152
175
|
<<~HTML
|
|
153
|
-
<div class="
|
|
176
|
+
<div class="#{entry_class}">
|
|
154
177
|
<div class="survived-header">
|
|
155
|
-
<span class="operator">#{h(mutation.operator_name)}</span>
|
|
178
|
+
<span class="operator">#{h(mutation.operator_name)}#{regression_badge}</span>
|
|
156
179
|
<span class="location">#{h(mutation.file_path)}:#{mutation.line}</span>
|
|
157
180
|
</div>
|
|
158
181
|
<pre class="diff">#{diff_html}</pre>
|
|
@@ -189,6 +212,48 @@ class Evilution::Reporter::HTML
|
|
|
189
212
|
end
|
|
190
213
|
end
|
|
191
214
|
|
|
215
|
+
def build_baseline_comparison(summary)
|
|
216
|
+
return "" unless @baseline
|
|
217
|
+
|
|
218
|
+
base_summary = @baseline["summary"] || {}
|
|
219
|
+
base_score = base_summary["score"] || 0.0
|
|
220
|
+
head_score = summary.score
|
|
221
|
+
delta = head_score - base_score
|
|
222
|
+
delta_str = format("%+.2f%%", delta * 100)
|
|
223
|
+
delta_class = if delta.positive?
|
|
224
|
+
"delta-positive"
|
|
225
|
+
elsif delta.negative?
|
|
226
|
+
"delta-negative"
|
|
227
|
+
else
|
|
228
|
+
"delta-neutral"
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
<<~HTML
|
|
232
|
+
<section class="baseline-comparison">
|
|
233
|
+
<h2>Baseline Comparison</h2>
|
|
234
|
+
<div class="comparison-scores">
|
|
235
|
+
<span>Baseline: #{format("%.2f%%", base_score * 100)}</span>
|
|
236
|
+
<span>Current: #{format("%.2f%%", head_score * 100)}</span>
|
|
237
|
+
<span class="#{delta_class}">Delta: #{delta_str}</span>
|
|
238
|
+
</div>
|
|
239
|
+
</section>
|
|
240
|
+
HTML
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
def regression?(mutation)
|
|
244
|
+
return false if @baseline_keys.nil?
|
|
245
|
+
|
|
246
|
+
key = [mutation.operator_name, mutation.file_path, mutation.line, mutation.subject.name]
|
|
247
|
+
!@baseline_keys.include?(key)
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
def build_baseline_keys(baseline)
|
|
251
|
+
return nil unless baseline
|
|
252
|
+
|
|
253
|
+
survived = baseline["survived"] || []
|
|
254
|
+
survived.to_set { |m| [m["operator"], m["file"], m["line"], m["subject"]] }
|
|
255
|
+
end
|
|
256
|
+
|
|
192
257
|
def h(text)
|
|
193
258
|
CGI.escapeHTML(text.to_s)
|
|
194
259
|
end
|
|
@@ -243,6 +308,14 @@ class Evilution::Reporter::HTML
|
|
|
243
308
|
.diff-added { color: #3fb950; display: block; }
|
|
244
309
|
.suggestion { color: #d29922; font-size: 0.8rem; margin-top: 0.5rem; font-style: italic; }
|
|
245
310
|
.empty { color: #8b949e; text-align: center; padding: 2rem; }
|
|
311
|
+
.baseline-comparison { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 1rem; margin-bottom: 2rem; }
|
|
312
|
+
.baseline-comparison h2 { font-size: 1rem; color: #f0f6fc; margin-bottom: 0.75rem; }
|
|
313
|
+
.comparison-scores { display: flex; gap: 2rem; font-size: 0.9rem; }
|
|
314
|
+
.delta-positive { color: #3fb950; font-weight: bold; }
|
|
315
|
+
.delta-negative { color: #f85149; font-weight: bold; }
|
|
316
|
+
.delta-neutral { color: #8b949e; font-weight: bold; }
|
|
317
|
+
.survived-entry.regression { border-color: #f85149; background: #2a1010; }
|
|
318
|
+
.regression-badge { background: #da3633; color: #fff; font-size: 0.65rem; padding: 0.1rem 0.4rem; border-radius: 4px; margin-left: 0.5rem; text-transform: uppercase; font-weight: bold; }
|
|
246
319
|
footer { text-align: center; color: #484f58; font-size: 0.75rem; margin-top: 2rem; padding-top: 1rem; border-top: 1px solid #21262d; }
|
|
247
320
|
</style>
|
|
248
321
|
HTML
|
|
@@ -19,7 +19,7 @@ class Evilution::Reporter::JSON
|
|
|
19
19
|
|
|
20
20
|
# rubocop:disable Metrics/PerceivedComplexity
|
|
21
21
|
def build_report(summary)
|
|
22
|
-
{
|
|
22
|
+
report = {
|
|
23
23
|
version: Evilution::VERSION,
|
|
24
24
|
timestamp: Time.now.iso8601,
|
|
25
25
|
summary: build_summary(summary),
|
|
@@ -30,9 +30,17 @@ class Evilution::Reporter::JSON
|
|
|
30
30
|
errors: summary.results.select(&:error?).map { |r| build_mutation_detail(r) },
|
|
31
31
|
equivalent: summary.equivalent_results.map { |r| build_mutation_detail(r) }
|
|
32
32
|
}
|
|
33
|
+
append_disabled_to_report(report, summary)
|
|
34
|
+
report
|
|
33
35
|
end
|
|
34
36
|
# rubocop:enable Metrics/PerceivedComplexity
|
|
35
37
|
|
|
38
|
+
def append_disabled_to_report(report, summary)
|
|
39
|
+
return unless summary.disabled_mutations.any?
|
|
40
|
+
|
|
41
|
+
report[:disabled] = summary.disabled_mutations.map { |m| build_disabled_detail(m) }
|
|
42
|
+
end
|
|
43
|
+
|
|
36
44
|
def build_summary(summary)
|
|
37
45
|
data = {
|
|
38
46
|
total: summary.total,
|
|
@@ -43,9 +51,13 @@ class Evilution::Reporter::JSON
|
|
|
43
51
|
neutral: summary.neutral,
|
|
44
52
|
equivalent: summary.equivalent,
|
|
45
53
|
score: summary.score.round(4),
|
|
46
|
-
duration: summary.duration.round(4)
|
|
54
|
+
duration: summary.duration.round(4),
|
|
55
|
+
killtime: summary.killtime.round(4),
|
|
56
|
+
efficiency: summary.efficiency.round(4),
|
|
57
|
+
mutations_per_second: summary.mutations_per_second.round(2)
|
|
47
58
|
}
|
|
48
59
|
data[:truncated] = true if summary.truncated?
|
|
60
|
+
data[:skipped] = summary.skipped if summary.skipped.positive?
|
|
49
61
|
peak = summary.peak_memory_mb
|
|
50
62
|
data[:peak_memory_mb] = peak.round(1) if peak
|
|
51
63
|
data
|
|
@@ -67,4 +79,13 @@ class Evilution::Reporter::JSON
|
|
|
67
79
|
detail[:memory_delta_kb] = result.memory_delta_kb if result.memory_delta_kb
|
|
68
80
|
detail
|
|
69
81
|
end
|
|
82
|
+
|
|
83
|
+
def build_disabled_detail(mutation)
|
|
84
|
+
{
|
|
85
|
+
operator: mutation.operator_name,
|
|
86
|
+
file: mutation.file_path,
|
|
87
|
+
line: mutation.line,
|
|
88
|
+
diff: mutation.diff
|
|
89
|
+
}
|
|
90
|
+
end
|
|
70
91
|
end
|
|
@@ -42,7 +42,15 @@ class Evilution::Reporter::Suggestion
|
|
|
42
42
|
"bitwise_replacement" => "Add a test that checks the exact bitwise result to distinguish &, |, and ^ operators",
|
|
43
43
|
"bitwise_complement" => "Add a test that verifies the bitwise complement (~) result, not just the sign or magnitude",
|
|
44
44
|
"zsuper_removal" => "Add a test that verifies inherited behavior from super is needed, not just the subclass logic",
|
|
45
|
-
"explicit_super_mutation" => "Add a test that verifies the correct arguments are passed to super and the inherited result matters"
|
|
45
|
+
"explicit_super_mutation" => "Add a test that verifies the correct arguments are passed to super and the inherited result matters",
|
|
46
|
+
"index_to_fetch" => "Add a test that distinguishes [] (returns nil for missing keys) from .fetch (raises KeyError)",
|
|
47
|
+
"index_to_dig" => "Add a test that verifies chained [] access returns the correct nested value",
|
|
48
|
+
"index_assignment_removal" => "Add a test that verifies the []= assignment side effect is observable (the collection is modified)",
|
|
49
|
+
"pattern_matching_guard" => "Add a test with input that matches the pattern but fails the guard to verify filtering",
|
|
50
|
+
"pattern_matching_alternative" => "Add a test with input that matches only one specific alternative to verify each branch is reachable",
|
|
51
|
+
"pattern_matching_array" => "Add a test that verifies each element position in the array pattern matches the expected type or value",
|
|
52
|
+
"collection_return" => "Add a test that verifies the method returns a non-empty collection, not just any array or hash",
|
|
53
|
+
"scalar_return" => "Add a test that verifies the method returns a non-zero/non-empty scalar value, not just any type"
|
|
46
54
|
}.freeze
|
|
47
55
|
|
|
48
56
|
CONCRETE_TEMPLATES = {
|
|
@@ -541,6 +549,112 @@ class Evilution::Reporter::Suggestion
|
|
|
541
549
|
expect(result).to eq(expected)
|
|
542
550
|
end
|
|
543
551
|
RSPEC
|
|
552
|
+
},
|
|
553
|
+
"index_to_fetch" => lambda { |mutation|
|
|
554
|
+
method_name = parse_method_name(mutation.subject.name)
|
|
555
|
+
original_line, mutated_line = extract_diff_lines(mutation.diff)
|
|
556
|
+
<<~RSPEC.strip
|
|
557
|
+
# Mutation: changed `#{original_line}` to `#{mutated_line}` in #{mutation.subject.name}
|
|
558
|
+
# #{mutation.file_path}:#{mutation.line}
|
|
559
|
+
it 'distinguishes [] from .fetch for missing keys in ##{method_name}' do
|
|
560
|
+
# Access a missing key: [] returns nil, .fetch raises KeyError
|
|
561
|
+
expect { subject.#{method_name}(collection_with_missing_key) }.to raise_error(KeyError)
|
|
562
|
+
end
|
|
563
|
+
RSPEC
|
|
564
|
+
},
|
|
565
|
+
"index_to_dig" => lambda { |mutation|
|
|
566
|
+
method_name = parse_method_name(mutation.subject.name)
|
|
567
|
+
original_line, mutated_line = extract_diff_lines(mutation.diff)
|
|
568
|
+
<<~RSPEC.strip
|
|
569
|
+
# Mutation: changed `#{original_line}` to `#{mutated_line}` in #{mutation.subject.name}
|
|
570
|
+
# #{mutation.file_path}:#{mutation.line}
|
|
571
|
+
it 'verifies the chained [] access returns the correct nested value in ##{method_name}' do
|
|
572
|
+
# Assert the nested lookup produces the expected value
|
|
573
|
+
result = subject.#{method_name}(nested_collection)
|
|
574
|
+
expect(result).to eq(expected)
|
|
575
|
+
end
|
|
576
|
+
RSPEC
|
|
577
|
+
},
|
|
578
|
+
"index_assignment_removal" => lambda { |mutation|
|
|
579
|
+
method_name = parse_method_name(mutation.subject.name)
|
|
580
|
+
original_line, mutated_line = extract_diff_lines(mutation.diff)
|
|
581
|
+
<<~RSPEC.strip
|
|
582
|
+
# Mutation: changed `#{original_line}` to `#{mutated_line}` in #{mutation.subject.name}
|
|
583
|
+
# #{mutation.file_path}:#{mutation.line}
|
|
584
|
+
it 'verifies the []= assignment modifies the collection in ##{method_name}' do
|
|
585
|
+
# Assert the collection contains the assigned value after the method runs
|
|
586
|
+
result = subject.#{method_name}(collection)
|
|
587
|
+
expect(result).to include(expected_key => expected_value)
|
|
588
|
+
end
|
|
589
|
+
RSPEC
|
|
590
|
+
},
|
|
591
|
+
"pattern_matching_guard" => lambda { |mutation|
|
|
592
|
+
method_name = parse_method_name(mutation.subject.name)
|
|
593
|
+
original_line, mutated_line = extract_diff_lines(mutation.diff)
|
|
594
|
+
<<~RSPEC.strip
|
|
595
|
+
# Mutation: changed `#{original_line}` to `#{mutated_line}` in #{mutation.subject.name}
|
|
596
|
+
# #{mutation.file_path}:#{mutation.line}
|
|
597
|
+
it 'verifies the pattern guard filters correctly in ##{method_name}' do
|
|
598
|
+
# Test with input that matches the pattern but fails the guard condition
|
|
599
|
+
# The guard should prevent matching, routing to a different branch
|
|
600
|
+
result = subject.#{method_name}(input_matching_pattern_but_failing_guard)
|
|
601
|
+
expect(result).to eq(expected)
|
|
602
|
+
end
|
|
603
|
+
RSPEC
|
|
604
|
+
},
|
|
605
|
+
"pattern_matching_alternative" => lambda { |mutation|
|
|
606
|
+
method_name = parse_method_name(mutation.subject.name)
|
|
607
|
+
original_line, mutated_line = extract_diff_lines(mutation.diff)
|
|
608
|
+
<<~RSPEC.strip
|
|
609
|
+
# Mutation: changed `#{original_line}` to `#{mutated_line}` in #{mutation.subject.name}
|
|
610
|
+
# #{mutation.file_path}:#{mutation.line}
|
|
611
|
+
it 'verifies each pattern alternative is reachable in ##{method_name}' do
|
|
612
|
+
# Test with input that matches only one specific alternative
|
|
613
|
+
# Each alternative should have a dedicated test case
|
|
614
|
+
result = subject.#{method_name}(input_for_specific_alternative)
|
|
615
|
+
expect(result).to eq(expected)
|
|
616
|
+
end
|
|
617
|
+
RSPEC
|
|
618
|
+
},
|
|
619
|
+
"collection_return" => lambda { |mutation|
|
|
620
|
+
method_name = parse_method_name(mutation.subject.name)
|
|
621
|
+
original_line, mutated_line = extract_diff_lines(mutation.diff)
|
|
622
|
+
<<~RSPEC.strip
|
|
623
|
+
# Mutation: changed `#{original_line}` to `#{mutated_line}` in #{mutation.subject.name}
|
|
624
|
+
# #{mutation.file_path}:#{mutation.line}
|
|
625
|
+
it 'returns a non-empty collection from ##{method_name}' do
|
|
626
|
+
# Assert the collection has the expected elements, not just non-empty
|
|
627
|
+
result = subject.#{method_name}(input_value)
|
|
628
|
+
expect(result).to eq(expected)
|
|
629
|
+
end
|
|
630
|
+
RSPEC
|
|
631
|
+
},
|
|
632
|
+
"scalar_return" => lambda { |mutation|
|
|
633
|
+
method_name = parse_method_name(mutation.subject.name)
|
|
634
|
+
original_line, mutated_line = extract_diff_lines(mutation.diff)
|
|
635
|
+
<<~RSPEC.strip
|
|
636
|
+
# Mutation: changed `#{original_line}` to `#{mutated_line}` in #{mutation.subject.name}
|
|
637
|
+
# #{mutation.file_path}:#{mutation.line}
|
|
638
|
+
it 'returns a non-zero/non-empty value from ##{method_name}' do
|
|
639
|
+
# Assert the exact scalar value, not just presence or type
|
|
640
|
+
result = subject.#{method_name}(input_value)
|
|
641
|
+
expect(result).to eq(expected)
|
|
642
|
+
end
|
|
643
|
+
RSPEC
|
|
644
|
+
},
|
|
645
|
+
"pattern_matching_array" => lambda { |mutation|
|
|
646
|
+
method_name = parse_method_name(mutation.subject.name)
|
|
647
|
+
original_line, mutated_line = extract_diff_lines(mutation.diff)
|
|
648
|
+
<<~RSPEC.strip
|
|
649
|
+
# Mutation: changed `#{original_line}` to `#{mutated_line}` in #{mutation.subject.name}
|
|
650
|
+
# #{mutation.file_path}:#{mutation.line}
|
|
651
|
+
it 'verifies each array pattern element matters in ##{method_name}' do
|
|
652
|
+
# Test with input where changing one element type causes a different match
|
|
653
|
+
# Each position in the array pattern should be validated
|
|
654
|
+
result = subject.#{method_name}(input_with_wrong_element_type)
|
|
655
|
+
expect(result).to eq(expected)
|
|
656
|
+
end
|
|
657
|
+
RSPEC
|
|
544
658
|
}
|
|
545
659
|
}.freeze
|
|
546
660
|
|
|
@@ -3,12 +3,14 @@
|
|
|
3
3
|
require_relative "../result"
|
|
4
4
|
|
|
5
5
|
class Evilution::Result::Summary
|
|
6
|
-
attr_reader :results, :duration
|
|
6
|
+
attr_reader :results, :duration, :skipped, :disabled_mutations
|
|
7
7
|
|
|
8
|
-
def initialize(results:, duration: 0.0, truncated: false)
|
|
8
|
+
def initialize(results:, duration: 0.0, truncated: false, skipped: 0, disabled_mutations: [])
|
|
9
9
|
@results = results
|
|
10
10
|
@duration = duration
|
|
11
11
|
@truncated = truncated
|
|
12
|
+
@skipped = skipped
|
|
13
|
+
@disabled_mutations = disabled_mutations
|
|
12
14
|
freeze
|
|
13
15
|
end
|
|
14
16
|
|
|
@@ -71,6 +73,22 @@ class Evilution::Result::Summary
|
|
|
71
73
|
results.select(&:equivalent?)
|
|
72
74
|
end
|
|
73
75
|
|
|
76
|
+
def killtime
|
|
77
|
+
results.sum(0.0, &:duration)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def efficiency
|
|
81
|
+
return 0.0 if duration.zero?
|
|
82
|
+
|
|
83
|
+
killtime / duration
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def mutations_per_second
|
|
87
|
+
return 0.0 if duration.zero?
|
|
88
|
+
|
|
89
|
+
total.to_f / duration
|
|
90
|
+
end
|
|
91
|
+
|
|
74
92
|
def peak_memory_mb
|
|
75
93
|
max_rss = nil
|
|
76
94
|
results.each do |result|
|
data/lib/evilution/runner.rb
CHANGED
|
@@ -20,17 +20,25 @@ require_relative "baseline"
|
|
|
20
20
|
require_relative "cache"
|
|
21
21
|
require_relative "parallel/pool"
|
|
22
22
|
require_relative "session/store"
|
|
23
|
+
require_relative "ast/pattern/filter"
|
|
24
|
+
require_relative "disable_comment"
|
|
25
|
+
require_relative "ast/sorbet_sig_detector"
|
|
23
26
|
|
|
24
27
|
class Evilution::Runner
|
|
25
28
|
attr_reader :config
|
|
26
29
|
|
|
27
|
-
def initialize(config: Evilution::Config.new, on_result: nil)
|
|
30
|
+
def initialize(config: Evilution::Config.new, on_result: nil, hooks: nil)
|
|
28
31
|
@config = config
|
|
29
32
|
@on_result = on_result
|
|
33
|
+
@hooks = hooks
|
|
30
34
|
@parser = Evilution::AST::Parser.new
|
|
31
35
|
@registry = Evilution::Mutator::Registry.default
|
|
32
36
|
@isolator = build_isolator
|
|
33
37
|
@cache = config.incremental? ? Evilution::Cache.new : nil
|
|
38
|
+
@disable_detector = Evilution::DisableComment.new
|
|
39
|
+
@disabled_ranges_cache = {}
|
|
40
|
+
@sig_detector = Evilution::AST::SorbetSigDetector.new
|
|
41
|
+
@sig_ranges_cache = {}
|
|
34
42
|
end
|
|
35
43
|
|
|
36
44
|
def call
|
|
@@ -41,7 +49,7 @@ class Evilution::Runner
|
|
|
41
49
|
|
|
42
50
|
baseline_result = run_baseline(subjects)
|
|
43
51
|
|
|
44
|
-
mutations = generate_mutations(subjects)
|
|
52
|
+
mutations, skipped_count, disabled_mutations = generate_mutations(subjects)
|
|
45
53
|
equivalent_mutations, mutations = filter_equivalent(mutations)
|
|
46
54
|
release_subject_nodes(subjects)
|
|
47
55
|
clear_operator_caches
|
|
@@ -54,17 +62,15 @@ class Evilution::Runner
|
|
|
54
62
|
|
|
55
63
|
duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
|
|
56
64
|
|
|
57
|
-
summary = Evilution::Result::Summary.new(results: results, duration: duration, truncated: truncated
|
|
65
|
+
summary = Evilution::Result::Summary.new(results: results, duration: duration, truncated: truncated,
|
|
66
|
+
skipped: skipped_count,
|
|
67
|
+
disabled_mutations: disabled_mutations)
|
|
58
68
|
output_report(summary)
|
|
59
69
|
save_session(summary)
|
|
60
70
|
|
|
61
71
|
summary
|
|
62
72
|
end
|
|
63
73
|
|
|
64
|
-
private
|
|
65
|
-
|
|
66
|
-
attr_reader :parser, :registry, :isolator, :cache, :on_result
|
|
67
|
-
|
|
68
74
|
def parse_and_filter_subjects
|
|
69
75
|
subjects = parse_subjects
|
|
70
76
|
subjects = filter_by_descendants(subjects) if descendants_target?
|
|
@@ -73,6 +79,10 @@ class Evilution::Runner
|
|
|
73
79
|
subjects
|
|
74
80
|
end
|
|
75
81
|
|
|
82
|
+
private
|
|
83
|
+
|
|
84
|
+
attr_reader :parser, :registry, :isolator, :cache, :on_result, :hooks, :disable_detector, :sig_detector
|
|
85
|
+
|
|
76
86
|
def parse_subjects
|
|
77
87
|
files = resolve_target_files
|
|
78
88
|
files.flat_map { |file| parser.call(file) }
|
|
@@ -169,9 +179,84 @@ class Evilution::Runner
|
|
|
169
179
|
end
|
|
170
180
|
|
|
171
181
|
def generate_mutations(subjects)
|
|
172
|
-
|
|
173
|
-
|
|
182
|
+
filter = build_ignore_filter
|
|
183
|
+
mutations = subjects.flat_map do |subject|
|
|
184
|
+
registry.mutations_for(subject, filter: filter)
|
|
185
|
+
end
|
|
186
|
+
skipped_count = filter ? filter.skipped_count : 0
|
|
187
|
+
|
|
188
|
+
mutations, disabled = filter_disabled(mutations)
|
|
189
|
+
disabled.each(&:strip_sources!) if config.show_disabled?
|
|
190
|
+
disabled_mutations = config.show_disabled? ? disabled : []
|
|
191
|
+
|
|
192
|
+
mutations, sig_skipped = filter_sig_blocks(mutations)
|
|
193
|
+
|
|
194
|
+
[mutations, skipped_count + disabled.length + sig_skipped, disabled_mutations]
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def filter_disabled(mutations)
|
|
198
|
+
enabled = []
|
|
199
|
+
disabled = []
|
|
200
|
+
|
|
201
|
+
mutations.each do |mutation|
|
|
202
|
+
if mutation_disabled?(mutation)
|
|
203
|
+
disabled << mutation
|
|
204
|
+
else
|
|
205
|
+
enabled << mutation
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
[enabled, disabled]
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def mutation_disabled?(mutation)
|
|
213
|
+
ranges = disabled_ranges_for(mutation.file_path)
|
|
214
|
+
ranges.any? { |range| range.cover?(mutation.line) }
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def disabled_ranges_for(file_path)
|
|
218
|
+
@disabled_ranges_cache[file_path] ||= begin
|
|
219
|
+
source = File.read(file_path)
|
|
220
|
+
@disable_detector.call(source)
|
|
221
|
+
rescue SystemCallError
|
|
222
|
+
[]
|
|
223
|
+
end
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
def filter_sig_blocks(mutations)
|
|
227
|
+
enabled = []
|
|
228
|
+
skipped = 0
|
|
229
|
+
|
|
230
|
+
mutations.each do |mutation|
|
|
231
|
+
if mutation_in_sig_block?(mutation)
|
|
232
|
+
skipped += 1
|
|
233
|
+
else
|
|
234
|
+
enabled << mutation
|
|
235
|
+
end
|
|
174
236
|
end
|
|
237
|
+
|
|
238
|
+
[enabled, skipped]
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
def mutation_in_sig_block?(mutation)
|
|
242
|
+
ranges = sig_line_ranges_for(mutation.file_path)
|
|
243
|
+
ranges.any? { |range| range.cover?(mutation.line) }
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
def sig_line_ranges_for(file_path)
|
|
247
|
+
@sig_ranges_cache[file_path] ||= begin
|
|
248
|
+
source = File.read(file_path)
|
|
249
|
+
@sig_detector.line_ranges(source)
|
|
250
|
+
rescue SystemCallError
|
|
251
|
+
[]
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
def build_ignore_filter
|
|
256
|
+
patterns = config.ignore_patterns
|
|
257
|
+
return nil if patterns.nil? || patterns.empty?
|
|
258
|
+
|
|
259
|
+
Evilution::AST::Pattern::Filter.new(patterns)
|
|
175
260
|
end
|
|
176
261
|
|
|
177
262
|
def filter_equivalent(mutations)
|
|
@@ -240,18 +325,23 @@ class Evilution::Runner
|
|
|
240
325
|
|
|
241
326
|
def run_mutations_parallel(mutations, baseline_result = nil)
|
|
242
327
|
integration = build_integration
|
|
243
|
-
pool = Evilution::Parallel::Pool.new(size: config.jobs)
|
|
328
|
+
pool = Evilution::Parallel::Pool.new(size: config.jobs, hooks: @hooks)
|
|
244
329
|
worker_isolator = Evilution::Isolation::InProcess.new
|
|
245
330
|
spec_resolver = baseline_result&.failed? ? Evilution::SpecResolver.new : nil
|
|
246
331
|
state = { results: [], survived_count: 0, truncated: false, completed: 0 }
|
|
247
332
|
|
|
333
|
+
all_worker_stats = []
|
|
334
|
+
|
|
248
335
|
mutations.each_slice(config.jobs) do |batch|
|
|
249
336
|
break if state[:truncated]
|
|
250
337
|
|
|
251
338
|
batch_results = run_parallel_batch(batch, pool, worker_isolator, integration)
|
|
339
|
+
all_worker_stats.concat(pool.worker_stats)
|
|
252
340
|
process_batch(batch_results, baseline_result, spec_resolver, state)
|
|
253
341
|
end
|
|
254
342
|
|
|
343
|
+
log_worker_stats(aggregate_worker_stats(all_worker_stats))
|
|
344
|
+
|
|
255
345
|
[state[:results], state[:truncated]]
|
|
256
346
|
end
|
|
257
347
|
|
|
@@ -341,7 +431,7 @@ class Evilution::Runner
|
|
|
341
431
|
|
|
342
432
|
def build_isolator
|
|
343
433
|
case resolve_isolation
|
|
344
|
-
when :fork then Evilution::Isolation::Fork.new
|
|
434
|
+
when :fork then Evilution::Isolation::Fork.new(hooks: @hooks)
|
|
345
435
|
when :in_process then Evilution::Isolation::InProcess.new
|
|
346
436
|
end
|
|
347
437
|
end
|
|
@@ -356,7 +446,7 @@ class Evilution::Runner
|
|
|
356
446
|
case config.integration
|
|
357
447
|
when :rspec
|
|
358
448
|
test_files = config.spec_files.empty? ? nil : config.spec_files
|
|
359
|
-
Evilution::Integration::RSpec.new(test_files: test_files)
|
|
449
|
+
Evilution::Integration::RSpec.new(test_files: test_files, hooks: @hooks)
|
|
360
450
|
else
|
|
361
451
|
raise Evilution::Error, "unknown integration: #{config.integration}"
|
|
362
452
|
end
|
|
@@ -446,6 +536,28 @@ class Evilution::Runner
|
|
|
446
536
|
warn "[evilution] failed to save session: #{e.message}" unless config.quiet
|
|
447
537
|
end
|
|
448
538
|
|
|
539
|
+
def log_worker_stats(stats)
|
|
540
|
+
return unless config.verbose && !config.quiet && stats.any?
|
|
541
|
+
|
|
542
|
+
stats.each do |stat|
|
|
543
|
+
pct = format("%.1f", stat.utilization * 100)
|
|
544
|
+
$stderr.write("[verbose] worker #{stat.pid}: #{stat.items_completed} items, utilization #{pct}%\n")
|
|
545
|
+
end
|
|
546
|
+
end
|
|
547
|
+
|
|
548
|
+
def aggregate_worker_stats(stats)
|
|
549
|
+
return stats if stats.empty?
|
|
550
|
+
|
|
551
|
+
stats.group_by(&:pid).map do |pid, entries|
|
|
552
|
+
Evilution::Parallel::WorkQueue::WorkerStat.new(
|
|
553
|
+
pid,
|
|
554
|
+
entries.sum(&:items_completed),
|
|
555
|
+
entries.sum(&:busy_time),
|
|
556
|
+
entries.sum(&:wall_time)
|
|
557
|
+
)
|
|
558
|
+
end
|
|
559
|
+
end
|
|
560
|
+
|
|
449
561
|
def notify_result(result, index)
|
|
450
562
|
on_result&.call(result)
|
|
451
563
|
@progress_bar&.tick(status: result.status)
|
|
@@ -466,10 +578,18 @@ class Evilution::Runner
|
|
|
466
578
|
when :text
|
|
467
579
|
Evilution::Reporter::CLI.new
|
|
468
580
|
when :html
|
|
469
|
-
Evilution::Reporter::HTML.new
|
|
581
|
+
Evilution::Reporter::HTML.new(baseline: load_baseline_session)
|
|
470
582
|
end
|
|
471
583
|
end
|
|
472
584
|
|
|
585
|
+
def load_baseline_session
|
|
586
|
+
path = config.baseline_session
|
|
587
|
+
return nil unless path
|
|
588
|
+
|
|
589
|
+
store = Evilution::Session::Store.new
|
|
590
|
+
store.load(path)
|
|
591
|
+
end
|
|
592
|
+
|
|
473
593
|
def partition_cached(batch)
|
|
474
594
|
uncached_indices = []
|
|
475
595
|
cached_results = {}
|