evilution 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.beads/interactions.jsonl +41 -0
- data/CHANGELOG.md +65 -0
- data/README.md +149 -13
- data/docs/architecture.md +20 -5
- data/docs/isolation.md +3 -4
- data/exe/evil +8 -1
- data/exe/evilution +8 -1
- data/lib/evilution/ast/local_reads.rb +43 -0
- data/lib/evilution/cli/exit_guard.rb +59 -0
- data/lib/evilution/cli/parser/options_builder.rb +1 -0
- data/lib/evilution/cli.rb +1 -0
- data/lib/evilution/config.rb +3 -2
- data/lib/evilution/diagnostic.rb +22 -0
- data/lib/evilution/integration/loading/concern_state_cleaner.rb +20 -4
- data/lib/evilution/integration/loading/redefinition_recovery.rb +1 -1
- data/lib/evilution/integration/loading/reeval_warning_filter.rb +72 -0
- data/lib/evilution/integration/loading/source_evaluator.rb +4 -1
- data/lib/evilution/integration/minitest.rb +2 -1
- data/lib/evilution/integration/rspec/crash_detector_lifecycle.rb +9 -1
- data/lib/evilution/integration/rspec/state_guard/configuration_streams.rb +4 -1
- data/lib/evilution/integration/rspec/unresolved_spec_warner.rb +2 -1
- data/lib/evilution/integration/rspec.rb +48 -2
- data/lib/evilution/integration/test_unit/test_file_resolver.rb +2 -1
- data/lib/evilution/isolation/fork.rb +29 -8
- data/lib/evilution/mcp/complete_result_server.rb +41 -0
- data/lib/evilution/mcp/server.rb +2 -1
- data/lib/evilution/mutator/base.rb +14 -2
- data/lib/evilution/mutator/operator/block_destructuring_expansion.rb +85 -0
- data/lib/evilution/mutator/operator/block_parameter_drop.rb +96 -0
- data/lib/evilution/mutator/operator/boolean_expression_to_nil.rb +22 -0
- data/lib/evilution/mutator/operator/boolean_operand_promotion.rb +31 -0
- data/lib/evilution/mutator/operator/case_in.rb +64 -0
- data/lib/evilution/mutator/operator/case_when.rb +72 -1
- data/lib/evilution/mutator/operator/conditional_branch.rb +20 -7
- data/lib/evilution/mutator/operator/forwarding_super_to_explicit.rb +71 -0
- data/lib/evilution/mutator/operator/if_branch_swap.rb +48 -0
- data/lib/evilution/mutator/operator/loop_body_to_raise.rb +68 -0
- data/lib/evilution/mutator/operator/method_body_replacement.rb +10 -1
- data/lib/evilution/mutator/operator/method_body_to_raise.rb +59 -0
- data/lib/evilution/mutator/operator/method_body_to_super.rb +150 -0
- data/lib/evilution/mutator/operator/optional_default_injection.rb +71 -0
- data/lib/evilution/mutator/operator/optional_parameter_to_required.rb +41 -0
- data/lib/evilution/mutator/operator/pattern_predicate.rb +28 -0
- data/lib/evilution/mutator/operator/typed_default_return.rb +84 -0
- data/lib/evilution/mutator/primitives.rb +52 -0
- data/lib/evilution/mutator/registry.rb +14 -0
- data/lib/evilution/process_supervisor.rb +20 -8
- data/lib/evilution/reporter/cli/item_formatters/neutral_group.rb +23 -0
- data/lib/evilution/reporter/cli/item_formatters/subject_score.rb +42 -0
- data/lib/evilution/reporter/cli/item_formatters/subject_score_group.rb +16 -0
- data/lib/evilution/reporter/cli/line_formatters/infra_retry_notice.rb +19 -0
- data/lib/evilution/reporter/cli/line_formatters/result_line.rb +27 -3
- data/lib/evilution/reporter/cli/line_formatters/score.rb +19 -1
- data/lib/evilution/reporter/cli/line_formatters/unresolved_targets.rb +35 -0
- data/lib/evilution/reporter/cli/metrics_block.rb +4 -0
- data/lib/evilution/reporter/cli/trailer.rb +11 -7
- data/lib/evilution/reporter/cli.rb +20 -4
- data/lib/evilution/reporter/json/subjects.rb +29 -0
- data/lib/evilution/reporter/json.rb +23 -1
- data/lib/evilution/result/mutation_result.rb +3 -2
- data/lib/evilution/result/neutral_reason.rb +35 -0
- data/lib/evilution/result/subject_score.rb +28 -0
- data/lib/evilution/result/subject_scorer.rb +37 -0
- data/lib/evilution/result/summary.rb +44 -2
- data/lib/evilution/runner/canary.rb +52 -5
- data/lib/evilution/runner/mutation_executor/infra_retry.rb +54 -0
- data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb +9 -3
- data/lib/evilution/runner/mutation_executor/neutralizer/infra_error.rb +18 -1
- data/lib/evilution/runner/mutation_executor/result_cache.rb +11 -0
- data/lib/evilution/runner/mutation_executor/strategy/parallel.rb +19 -1
- data/lib/evilution/runner/mutation_executor.rb +32 -4
- data/lib/evilution/runner/report_publisher.rb +31 -9
- data/lib/evilution/runner/target_spec_audit.rb +43 -0
- data/lib/evilution/runner.rb +10 -1
- data/lib/evilution/version.rb +1 -1
- data/lib/evilution.rb +18 -0
- data/scripts/compare_targeting +4 -2
- metadata +33 -2
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../line_formatters"
|
|
4
|
+
|
|
5
|
+
# EV-j0bv / GH #1607: parallel workers contending on shared infrastructure — a
|
|
6
|
+
# database file, a lock — crash the test process in a way that says nothing
|
|
7
|
+
# about the mutation. Those mutations are re-run serially once the pool is
|
|
8
|
+
# done, and the run says so, because the alternative is a neutral count that
|
|
9
|
+
# moves with --jobs on identical input with no explanation.
|
|
10
|
+
class Evilution::Reporter::CLI::LineFormatters::InfraRetryNotice
|
|
11
|
+
def format(summary)
|
|
12
|
+
count = summary.infra_retried
|
|
13
|
+
return nil if count.zero?
|
|
14
|
+
|
|
15
|
+
noun = count == 1 ? "mutation" : "mutations"
|
|
16
|
+
pronoun = count == 1 ? "it" : "them"
|
|
17
|
+
"! #{count} #{noun} hit infrastructure errors under parallel workers; re-ran #{pronoun} serially."
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -3,18 +3,42 @@
|
|
|
3
3
|
require_relative "../line_formatters"
|
|
4
4
|
require_relative "../pct"
|
|
5
5
|
|
|
6
|
+
# The threshold comes from the run's own configuration. Printing a verdict
|
|
7
|
+
# against a default the exit code did not share is what made a 0% score report
|
|
8
|
+
# `FAIL` and exit 0 (EV-39t1 / GH #1604); with no minimum configured the line
|
|
9
|
+
# states the score and says so, rather than implying a gate that is not armed.
|
|
6
10
|
class Evilution::Reporter::CLI::LineFormatters::ResultLine
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
def initialize(pct: Evilution::Reporter::CLI::Pct.new, min_score: DEFAULT_MIN_SCORE)
|
|
11
|
+
def initialize(pct: Evilution::Reporter::CLI::Pct.new, min_score: nil)
|
|
10
12
|
@pct = pct
|
|
11
13
|
@min_score = min_score
|
|
12
14
|
end
|
|
13
15
|
|
|
14
16
|
def format(summary)
|
|
17
|
+
return unresolved_targets_line(summary) if summary.unresolved_targets?
|
|
18
|
+
return no_threshold_line(summary) unless gate?
|
|
19
|
+
|
|
15
20
|
pass_fail = summary.success?(min_score: @min_score) ? "PASS" : "FAIL"
|
|
16
21
|
score_pct = @pct.format(summary.score)
|
|
17
22
|
threshold_pct = @pct.format(@min_score)
|
|
18
23
|
"Result: #{pass_fail} (score #{score_pct} #{pass_fail == "PASS" ? ">=" : "<"} #{threshold_pct})"
|
|
19
24
|
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
# A minimum of zero passes every score, so it is not a gate either.
|
|
29
|
+
def gate?
|
|
30
|
+
!@min_score.nil? && @min_score.positive?
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def no_threshold_line(summary)
|
|
34
|
+
"Result: #{@pct.format(summary.score)} (no minimum score set)"
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# The score covers only the files that resolved to a spec, so reporting it
|
|
38
|
+
# against the threshold here would name the wrong problem.
|
|
39
|
+
def unresolved_targets_line(summary)
|
|
40
|
+
count = summary.unresolved_target_files.length
|
|
41
|
+
subject = count == 1 ? "target file has" : "target files have"
|
|
42
|
+
"Result: FAIL (#{count} #{subject} no resolvable spec)"
|
|
43
|
+
end
|
|
20
44
|
end
|
|
@@ -3,12 +3,30 @@
|
|
|
3
3
|
require_relative "../line_formatters"
|
|
4
4
|
require_relative "../pct"
|
|
5
5
|
|
|
6
|
+
# The score covers only the mutations that got a verdict. Where a run left some
|
|
7
|
+
# out — neutral, unresolved, equivalent, errored — full marks over a fraction of
|
|
8
|
+
# it reads as a verdict on the whole, so the line says how much it covered
|
|
9
|
+
# (EV-5pob / GH #1606).
|
|
6
10
|
class Evilution::Reporter::CLI::LineFormatters::Score
|
|
7
11
|
def initialize(pct: Evilution::Reporter::CLI::Pct.new)
|
|
8
12
|
@pct = pct
|
|
9
13
|
end
|
|
10
14
|
|
|
11
15
|
def format(summary)
|
|
12
|
-
"Score: #{@pct.format(summary.score)} (#{summary
|
|
16
|
+
"Score: #{@pct.format(summary.score)} (#{counts(summary)})"
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
private
|
|
20
|
+
|
|
21
|
+
def counts(summary)
|
|
22
|
+
verified = summary.score_denominator
|
|
23
|
+
pair = "#{summary.killed}/#{verified}"
|
|
24
|
+
return pair if verified == summary.total
|
|
25
|
+
|
|
26
|
+
"#{pair} verified of #{summary.total} mutations#{neutral_note(summary)}"
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def neutral_note(summary)
|
|
30
|
+
summary.neutral.positive? ? ", #{summary.neutral} neutral" : ""
|
|
13
31
|
end
|
|
14
32
|
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../line_formatters"
|
|
4
|
+
|
|
5
|
+
# EV-p4sm / GH #1603: a target file that resolves to no test file contributes
|
|
6
|
+
# only a handful of `unresolved` mutations, which UnresolvedRateWarning dilutes
|
|
7
|
+
# against every other file in the run — 14 unresolved out of 98 sits under its
|
|
8
|
+
# threshold and says nothing at all. This formatter reports the stronger,
|
|
9
|
+
# per-file fact instead, and names the files so the reader knows which of the
|
|
10
|
+
# paths they passed was never tested.
|
|
11
|
+
class Evilution::Reporter::CLI::LineFormatters::UnresolvedTargets
|
|
12
|
+
def format(summary)
|
|
13
|
+
return nil unless summary.unresolved_targets?
|
|
14
|
+
|
|
15
|
+
files = summary.unresolved_target_files
|
|
16
|
+
listing = files.map { |path| " #{path}" }.join("\n")
|
|
17
|
+
"! #{headline(files.length, summary.target_file_count)}:\n#{listing}"
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
# The ratio is dropped rather than guessed when the run's target count is
|
|
23
|
+
# unknown, which is how a summary restored from a saved session arrives. The
|
|
24
|
+
# noun agrees with whichever number it follows ("1 of 2 target files has",
|
|
25
|
+
# "1 target file has"), the verb with how many were unresolved.
|
|
26
|
+
def headline(unresolved_count, target_file_count)
|
|
27
|
+
noun_count = target_file_count.nil? ? unresolved_count : target_file_count
|
|
28
|
+
noun = noun_count == 1 ? "target file" : "target files"
|
|
29
|
+
verb = unresolved_count == 1 ? "has" : "have"
|
|
30
|
+
tail = unresolved_count == 1 ? "it was never tested" : "they were never tested"
|
|
31
|
+
scope = target_file_count.nil? ? unresolved_count.to_s : "#{unresolved_count} of #{target_file_count}"
|
|
32
|
+
|
|
33
|
+
"#{scope} #{noun} #{verb} no resolvable spec — #{tail}"
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -5,6 +5,8 @@ require_relative "line_formatters/mutations"
|
|
|
5
5
|
require_relative "line_formatters/score"
|
|
6
6
|
require_relative "line_formatters/error_rate_warning"
|
|
7
7
|
require_relative "line_formatters/unresolved_rate_warning"
|
|
8
|
+
require_relative "line_formatters/unresolved_targets"
|
|
9
|
+
require_relative "line_formatters/infra_retry_notice"
|
|
8
10
|
require_relative "line_formatters/duration"
|
|
9
11
|
require_relative "line_formatters/efficiency"
|
|
10
12
|
require_relative "line_formatters/peak_memory"
|
|
@@ -15,6 +17,8 @@ class Evilution::Reporter::CLI::MetricsBlock
|
|
|
15
17
|
Evilution::Reporter::CLI::LineFormatters::Score.new,
|
|
16
18
|
Evilution::Reporter::CLI::LineFormatters::ErrorRateWarning.new,
|
|
17
19
|
Evilution::Reporter::CLI::LineFormatters::UnresolvedRateWarning.new,
|
|
20
|
+
Evilution::Reporter::CLI::LineFormatters::UnresolvedTargets.new,
|
|
21
|
+
Evilution::Reporter::CLI::LineFormatters::InfraRetryNotice.new,
|
|
18
22
|
Evilution::Reporter::CLI::LineFormatters::Duration.new,
|
|
19
23
|
Evilution::Reporter::CLI::LineFormatters::Efficiency.new,
|
|
20
24
|
Evilution::Reporter::CLI::LineFormatters::PeakMemory.new
|
|
@@ -6,14 +6,18 @@ require_relative "line_formatters/result_line"
|
|
|
6
6
|
require_relative "line_formatters/feedback_footer"
|
|
7
7
|
|
|
8
8
|
class Evilution::Reporter::CLI::Trailer
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
9
|
+
def self.default_lines(min_score: nil)
|
|
10
|
+
[
|
|
11
|
+
Evilution::Reporter::CLI::LineFormatters::TruncationNotice.new,
|
|
12
|
+
Evilution::Reporter::CLI::LineFormatters::ResultLine.new(min_score: min_score),
|
|
13
|
+
Evilution::Reporter::CLI::LineFormatters::FeedbackFooter.new
|
|
14
|
+
]
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
DEFAULT_LINES = default_lines.freeze
|
|
14
18
|
|
|
15
|
-
def initialize(lines:
|
|
16
|
-
@lines = lines
|
|
19
|
+
def initialize(lines: nil, min_score: nil)
|
|
20
|
+
@lines = lines || self.class.default_lines(min_score: min_score)
|
|
17
21
|
end
|
|
18
22
|
|
|
19
23
|
def call(summary)
|
|
@@ -6,11 +6,12 @@ class Evilution::Reporter::CLI
|
|
|
6
6
|
SEPARATOR = "=" * 44
|
|
7
7
|
|
|
8
8
|
def initialize(
|
|
9
|
+
min_score: nil,
|
|
9
10
|
header: LineFormatters::Header.new,
|
|
10
11
|
metrics_block: MetricsBlock.new,
|
|
11
12
|
section_renderer: SectionRenderer.new,
|
|
12
13
|
sections: DEFAULT_SECTIONS,
|
|
13
|
-
trailer: Trailer.new
|
|
14
|
+
trailer: Trailer.new(min_score: min_score)
|
|
14
15
|
)
|
|
15
16
|
@header = header
|
|
16
17
|
@metrics_block = metrics_block
|
|
@@ -46,6 +47,9 @@ require_relative "cli/line_formatters/result_line"
|
|
|
46
47
|
require_relative "cli/line_formatters/feedback_footer"
|
|
47
48
|
require_relative "cli/item_formatters/coverage_gap"
|
|
48
49
|
require_relative "cli/item_formatters/result_location"
|
|
50
|
+
require_relative "cli/item_formatters/neutral_group"
|
|
51
|
+
require_relative "cli/item_formatters/subject_score"
|
|
52
|
+
require_relative "cli/item_formatters/subject_score_group"
|
|
49
53
|
require_relative "cli/item_formatters/error"
|
|
50
54
|
require_relative "cli/item_formatters/disabled"
|
|
51
55
|
require_relative "cli/metrics_block"
|
|
@@ -60,9 +64,21 @@ Evilution::Reporter::CLI.const_set(
|
|
|
60
64
|
formatter: Evilution::Reporter::CLI::ItemFormatters::CoverageGap.new
|
|
61
65
|
),
|
|
62
66
|
Evilution::Reporter::CLI::Section.new(
|
|
63
|
-
title:
|
|
64
|
-
|
|
65
|
-
|
|
67
|
+
title: lambda { |groups|
|
|
68
|
+
subjects = groups.sum(&:length)
|
|
69
|
+
"Subjects needing attention (#{subjects} subject#{"s" unless subjects == 1} " \
|
|
70
|
+
"in #{groups.length} file#{"s" unless groups.length == 1}):"
|
|
71
|
+
},
|
|
72
|
+
fetcher: lambda(&:subjects_needing_attention_by_file),
|
|
73
|
+
formatter: Evilution::Reporter::CLI::ItemFormatters::SubjectScoreGroup.new
|
|
74
|
+
),
|
|
75
|
+
Evilution::Reporter::CLI::Section.new(
|
|
76
|
+
title: lambda { |groups|
|
|
77
|
+
count = groups.sum { |(_reason, results)| results.length }
|
|
78
|
+
"Neutral mutations (#{count}, not verified):"
|
|
79
|
+
},
|
|
80
|
+
fetcher: lambda(&:neutral_results_by_reason),
|
|
81
|
+
formatter: Evilution::Reporter::CLI::ItemFormatters::NeutralGroup.new
|
|
66
82
|
),
|
|
67
83
|
Evilution::Reporter::CLI::Section.new(
|
|
68
84
|
title: "Equivalent mutations (provably identical behavior):",
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../json"
|
|
4
|
+
|
|
5
|
+
# The per-subject rows of a JSON report.
|
|
6
|
+
#
|
|
7
|
+
# Every subject is listed, whether or not it needs attention: a consumer that
|
|
8
|
+
# wants only the gaps filters on `reached` and `score`, while one asserting that
|
|
9
|
+
# a method is covered at all needs to see the rest (EV-nlx1 / GH #1605).
|
|
10
|
+
class Evilution::Reporter::JSON::Subjects
|
|
11
|
+
def call(summary)
|
|
12
|
+
summary.subject_scores.map { |score| row(score) }
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
private
|
|
16
|
+
|
|
17
|
+
def row(score)
|
|
18
|
+
{
|
|
19
|
+
name: score.name,
|
|
20
|
+
file: score.file_path,
|
|
21
|
+
total: score.total,
|
|
22
|
+
killed: score.killed,
|
|
23
|
+
verified: score.verified,
|
|
24
|
+
survived: score.survived,
|
|
25
|
+
score: score.score.round(4),
|
|
26
|
+
reached: score.reached?
|
|
27
|
+
}
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -8,8 +8,9 @@ require_relative "../reporter"
|
|
|
8
8
|
require_relative "../session/schema"
|
|
9
9
|
|
|
10
10
|
class Evilution::Reporter::JSON
|
|
11
|
-
def initialize(suggest_tests: false, integration: :rspec)
|
|
11
|
+
def initialize(suggest_tests: false, integration: :rspec, subjects: Subjects.new)
|
|
12
12
|
@suggestion = Evilution::Reporter::Suggestion.new(suggest_tests: suggest_tests, integration: integration)
|
|
13
|
+
@subjects = subjects
|
|
13
14
|
end
|
|
14
15
|
|
|
15
16
|
def call(summary)
|
|
@@ -25,6 +26,7 @@ class Evilution::Reporter::JSON
|
|
|
25
26
|
timestamp: Time.now.iso8601,
|
|
26
27
|
summary: build_summary(summary),
|
|
27
28
|
coverage_gaps: build_coverage_gaps(summary),
|
|
29
|
+
subjects: @subjects.call(summary),
|
|
28
30
|
**result_categories(summary)
|
|
29
31
|
}
|
|
30
32
|
append_disabled_to_report(report, summary)
|
|
@@ -94,12 +96,20 @@ class Evilution::Reporter::JSON
|
|
|
94
96
|
end
|
|
95
97
|
|
|
96
98
|
def append_optional_summary_fields(data, summary)
|
|
99
|
+
append_diagnostic_summary_fields(data, summary)
|
|
97
100
|
data[:truncated] = true if summary.truncated?
|
|
98
101
|
data[:skipped] = summary.skipped if summary.skipped.positive?
|
|
99
102
|
peak = summary.peak_memory_mb
|
|
100
103
|
data[:peak_memory_mb] = peak.round(1) if peak
|
|
101
104
|
end
|
|
102
105
|
|
|
106
|
+
# What the run had to say about itself rather than about the mutations: a
|
|
107
|
+
# target that was never tested, work that had to be redone serially.
|
|
108
|
+
def append_diagnostic_summary_fields(data, summary)
|
|
109
|
+
data[:unresolved_target_files] = summary.unresolved_target_files if summary.unresolved_targets?
|
|
110
|
+
data[:infra_retried] = summary.infra_retried if summary.infra_retried.positive?
|
|
111
|
+
end
|
|
112
|
+
|
|
103
113
|
def build_mutation_detail(result)
|
|
104
114
|
mutation = result.mutation
|
|
105
115
|
detail = base_mutation_fields(mutation, result)
|
|
@@ -107,9 +117,19 @@ class Evilution::Reporter::JSON
|
|
|
107
117
|
detail[:test_command] = result.test_command if result.test_command
|
|
108
118
|
append_memory_fields(detail, result)
|
|
109
119
|
append_error_fields(detail, result)
|
|
120
|
+
append_neutral_reason(detail, result)
|
|
110
121
|
detail
|
|
111
122
|
end
|
|
112
123
|
|
|
124
|
+
# Why a neutral was recorded — a red baseline and an infrastructure crash want
|
|
125
|
+
# opposite responses, and both land in the same bucket (EV-5pob / GH #1606).
|
|
126
|
+
def append_neutral_reason(detail, result)
|
|
127
|
+
reason = result.neutral_reason
|
|
128
|
+
return unless reason
|
|
129
|
+
|
|
130
|
+
detail[:neutral_reason] = { kind: reason.kind.to_s, detail: reason.detail }
|
|
131
|
+
end
|
|
132
|
+
|
|
113
133
|
def base_mutation_fields(mutation, result)
|
|
114
134
|
{
|
|
115
135
|
operator: mutation.operator_name,
|
|
@@ -161,3 +181,5 @@ class Evilution::Reporter::JSON
|
|
|
161
181
|
}
|
|
162
182
|
end
|
|
163
183
|
end
|
|
184
|
+
|
|
185
|
+
require_relative "json/subjects"
|
|
@@ -7,10 +7,10 @@ require_relative "memory_stats"
|
|
|
7
7
|
class Evilution::Result::MutationResult
|
|
8
8
|
STATUSES = %i[killed survived timeout error neutral equivalent unresolved unparseable].freeze
|
|
9
9
|
|
|
10
|
-
attr_reader :mutation, :status, :duration, :killing_test, :test_command, :memory, :error
|
|
10
|
+
attr_reader :mutation, :status, :duration, :killing_test, :test_command, :memory, :error, :neutral_reason
|
|
11
11
|
|
|
12
12
|
def initialize(mutation:, status:, duration: 0.0, killing_test: nil,
|
|
13
|
-
test_command: nil, memory: nil, error: nil)
|
|
13
|
+
test_command: nil, memory: nil, error: nil, neutral_reason: nil)
|
|
14
14
|
raise ArgumentError, "invalid status: #{status}" unless STATUSES.include?(status)
|
|
15
15
|
|
|
16
16
|
@mutation = mutation
|
|
@@ -20,6 +20,7 @@ class Evilution::Result::MutationResult
|
|
|
20
20
|
@test_command = test_command
|
|
21
21
|
@memory = memory
|
|
22
22
|
@error = error
|
|
23
|
+
@neutral_reason = neutral_reason
|
|
23
24
|
freeze
|
|
24
25
|
end
|
|
25
26
|
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../result"
|
|
4
|
+
|
|
5
|
+
# Why a mutation was recorded neutral.
|
|
6
|
+
#
|
|
7
|
+
# Neutral covers two unrelated situations, and they call for opposite responses:
|
|
8
|
+
# a spec file that was already red before any mutation ran, and a test process
|
|
9
|
+
# that died on infrastructure rather than on the mutation. Without the reason
|
|
10
|
+
# they are indistinguishable in a report, and a run can print full marks while
|
|
11
|
+
# the neutral bucket quietly holds what would otherwise be survivors
|
|
12
|
+
# (EV-5pob / GH #1606).
|
|
13
|
+
Evilution::Result::NeutralReason = Data.define(:kind, :detail) do
|
|
14
|
+
def self.baseline_failure(spec_file)
|
|
15
|
+
new(kind: :baseline_failure, detail: spec_file)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def self.infra_error(error_class)
|
|
19
|
+
new(kind: :infra_error, detail: error_class)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def to_s
|
|
23
|
+
detail ? "#{label} (#{detail})" : label
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def label
|
|
29
|
+
case kind
|
|
30
|
+
when :baseline_failure then "baseline already failing"
|
|
31
|
+
when :infra_error then "infrastructure error"
|
|
32
|
+
else kind.to_s
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../result"
|
|
4
|
+
|
|
5
|
+
# What one subject — a single method — scored, alongside the counts behind it.
|
|
6
|
+
#
|
|
7
|
+
# The run's own score is computed per file, so a well-tested file that gains new
|
|
8
|
+
# untested methods still reports 100%: whatever the resolved spec reaches is all
|
|
9
|
+
# the number ever describes (EV-nlx1 / GH #1605). Per subject the picture
|
|
10
|
+
# separates: a method whose mutations were never reached scores nothing and says
|
|
11
|
+
# so, instead of disappearing into the file's total.
|
|
12
|
+
Evilution::Result::SubjectScore = Data.define(:name, :file_path, :total, :killed, :verified, :survived) do
|
|
13
|
+
# Killed over what actually got a verdict, the same denominator the run uses.
|
|
14
|
+
def score
|
|
15
|
+
return 0.0 if verified.zero?
|
|
16
|
+
|
|
17
|
+
killed.to_f / verified
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Whether any mutation of this subject got a verdict at all.
|
|
21
|
+
def reached?
|
|
22
|
+
verified.positive?
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def fully_verified?
|
|
26
|
+
reached? && killed == verified
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "subject_score"
|
|
4
|
+
|
|
5
|
+
# Groups mutation results by the subject they belong to and scores each one.
|
|
6
|
+
#
|
|
7
|
+
# Same denominator as the run's own score: statuses that carry no verdict
|
|
8
|
+
# (unresolved, neutral, equivalent, errored, unparseable) are left out of it, so
|
|
9
|
+
# a subject those account for entirely reports as unreached rather than as a
|
|
10
|
+
# perfect score over nothing.
|
|
11
|
+
class Evilution::Result::SubjectScorer
|
|
12
|
+
VERDICT_STATUSES = %i[killed survived timeout].freeze
|
|
13
|
+
private_constant :VERDICT_STATUSES
|
|
14
|
+
|
|
15
|
+
def call(results)
|
|
16
|
+
grouped = results.group_by { |result| [result.mutation.file_path, result.mutation.subject.name] }
|
|
17
|
+
|
|
18
|
+
grouped
|
|
19
|
+
.map { |(file_path, name), subject_results| score_for(name, file_path, subject_results) }
|
|
20
|
+
.sort_by { |score| [score.file_path, score.name] }
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
def score_for(name, file_path, results)
|
|
26
|
+
verdicts = results.count { |result| VERDICT_STATUSES.include?(result.status) }
|
|
27
|
+
|
|
28
|
+
Evilution::Result::SubjectScore.new(
|
|
29
|
+
name: name,
|
|
30
|
+
file_path: file_path,
|
|
31
|
+
total: results.length,
|
|
32
|
+
killed: results.count(&:killed?),
|
|
33
|
+
verified: verdicts,
|
|
34
|
+
survived: results.count(&:survived?)
|
|
35
|
+
)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -2,19 +2,57 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "../result"
|
|
4
4
|
require_relative "coverage_gap_grouper"
|
|
5
|
+
require_relative "subject_scorer"
|
|
5
6
|
|
|
6
7
|
class Evilution::Result::Summary
|
|
7
|
-
attr_reader :results, :duration, :skipped, :disabled_mutations
|
|
8
|
+
attr_reader :results, :duration, :skipped, :disabled_mutations, :unresolved_target_files,
|
|
9
|
+
:target_file_count, :infra_retried
|
|
8
10
|
|
|
9
|
-
def initialize(results:, duration: 0.0, truncated: false, skipped: 0, disabled_mutations: []
|
|
11
|
+
def initialize(results:, duration: 0.0, truncated: false, skipped: 0, disabled_mutations: [],
|
|
12
|
+
unresolved_target_files: [], target_file_count: nil, infra_retried: 0)
|
|
10
13
|
@results = results
|
|
11
14
|
@duration = duration
|
|
12
15
|
@truncated = truncated
|
|
13
16
|
@skipped = skipped
|
|
14
17
|
@disabled_mutations = disabled_mutations
|
|
18
|
+
@unresolved_target_files = unresolved_target_files.freeze
|
|
19
|
+
@target_file_count = target_file_count
|
|
20
|
+
@infra_retried = infra_retried
|
|
15
21
|
freeze
|
|
16
22
|
end
|
|
17
23
|
|
|
24
|
+
# Neutral results gathered by the reason they were recorded, which is what the
|
|
25
|
+
# report needs to say which kind of neutral a reader is looking at
|
|
26
|
+
# (EV-5pob / GH #1606).
|
|
27
|
+
def neutral_results_by_reason
|
|
28
|
+
neutral_results.group_by(&:neutral_reason).to_a
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# What each subject — each method — scored on its own. The run's score is
|
|
32
|
+
# computed per file, which says nothing about a method inside it that no
|
|
33
|
+
# example reaches (EV-nlx1 / GH #1605).
|
|
34
|
+
def subject_scores
|
|
35
|
+
Evilution::Result::SubjectScorer.new.call(results)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# The subjects the file-level score does not speak for: something survived, or
|
|
39
|
+
# nothing reached them at all.
|
|
40
|
+
def subjects_needing_attention
|
|
41
|
+
subject_scores.reject(&:fully_verified?)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# The same subjects, gathered under the file they live in, which is how the
|
|
45
|
+
# report lists them.
|
|
46
|
+
def subjects_needing_attention_by_file
|
|
47
|
+
subjects_needing_attention.group_by(&:file_path).values
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Files evilution was pointed at that resolved to no test file, so nothing
|
|
51
|
+
# about them was ever measured (EV-p4sm / GH #1603).
|
|
52
|
+
def unresolved_targets?
|
|
53
|
+
!unresolved_target_files.empty?
|
|
54
|
+
end
|
|
55
|
+
|
|
18
56
|
def truncated?
|
|
19
57
|
@truncated
|
|
20
58
|
end
|
|
@@ -66,7 +104,11 @@ class Evilution::Result::Summary
|
|
|
66
104
|
killed.to_f / denominator
|
|
67
105
|
end
|
|
68
106
|
|
|
107
|
+
# A target file that was never tested fails the run on its own: the score
|
|
108
|
+
# only speaks for the files that did resolve to a spec.
|
|
69
109
|
def success?(min_score: 1.0)
|
|
110
|
+
return false if unresolved_targets?
|
|
111
|
+
|
|
70
112
|
score >= min_score
|
|
71
113
|
end
|
|
72
114
|
|
|
@@ -34,7 +34,7 @@ class Evilution::Runner::Canary
|
|
|
34
34
|
test_command: ->(mutation) { build_integration(spec_path).call(mutation) },
|
|
35
35
|
timeout: @config.timeout
|
|
36
36
|
)
|
|
37
|
-
raise Failed, failure_message(result
|
|
37
|
+
raise Failed, failure_message(result) unless result.status == :survived
|
|
38
38
|
|
|
39
39
|
nil
|
|
40
40
|
ensure
|
|
@@ -146,12 +146,59 @@ class Evilution::Runner::Canary
|
|
|
146
146
|
@integration_class.new(test_files: [spec_path], hooks: @hooks)
|
|
147
147
|
end
|
|
148
148
|
|
|
149
|
-
|
|
149
|
+
# When the child reported an error, that error IS the diagnosis -- naming it
|
|
150
|
+
# beats guessing. The speculative list stays only for the cases that carry no
|
|
151
|
+
# error at all (:killed, :timeout), where guesses are the only help there is.
|
|
152
|
+
# Diagnosing GH #1581 meant rebuilding this canary by hand to read the field
|
|
153
|
+
# this message used to drop, and none of the four guesses was the cause.
|
|
154
|
+
# EV-65nf / GH #1586.
|
|
155
|
+
def failure_message(result)
|
|
156
|
+
"#{failure_preamble(result.status)} #{diagnosis(result)} " \
|
|
157
|
+
"Re-run with --no-canary to bypass this check."
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def failure_preamble(status)
|
|
150
161
|
"evilution proof-of-life canary failed: a guaranteed-unobservable synthetic " \
|
|
151
162
|
"mutation was scored #{status.inspect} instead of :survived. The mutation " \
|
|
152
|
-
"pipeline is misreporting — every score this run would produce is unreliable.
|
|
153
|
-
|
|
163
|
+
"pipeline is misreporting — every score this run would produce is unreliable."
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def diagnosis(result)
|
|
167
|
+
reported = reported_error(result)
|
|
168
|
+
return speculative_causes if reported.nil?
|
|
169
|
+
|
|
170
|
+
"The child reported: #{reported}"
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def reported_error(result)
|
|
174
|
+
message = result.error_message
|
|
175
|
+
return nil if message.nil? || message.empty?
|
|
176
|
+
|
|
177
|
+
described = with_class_prefix(message, result.error_class)
|
|
178
|
+
frame = first_frame(result)
|
|
179
|
+
frame.nil? ? described : "#{described} (at #{frame})"
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# MutationApplier already prefixes the class onto the message it packs
|
|
183
|
+
# ("#{e.class}: #{e.message}"), while other paths pack the bare message and
|
|
184
|
+
# leave the class in its own field. Prefixing unconditionally would print
|
|
185
|
+
# "NameError: NameError: ..." for the former.
|
|
186
|
+
def with_class_prefix(message, klass)
|
|
187
|
+
return message if klass.nil? || klass.empty? || message.start_with?("#{klass}:")
|
|
188
|
+
|
|
189
|
+
"#{klass}: #{message}"
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def first_frame(result)
|
|
193
|
+
backtrace = result.error_backtrace
|
|
194
|
+
return nil if backtrace.nil? || backtrace.empty?
|
|
195
|
+
|
|
196
|
+
backtrace.first
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def speculative_causes
|
|
200
|
+
"Likely causes: Rails/Zeitwerk autoloading breaking child eval; an env-specific " \
|
|
154
201
|
"RSpec config (e.g. fail_if_no_examples); a classify_status fallback defect; or " \
|
|
155
|
-
"an isolation-mode defect.
|
|
202
|
+
"an isolation-mode defect."
|
|
156
203
|
end
|
|
157
204
|
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../mutation_executor"
|
|
4
|
+
require_relative "neutralizer/infra_error"
|
|
5
|
+
|
|
6
|
+
# Re-runs, one at a time, the mutations a parallel pass could not judge because
|
|
7
|
+
# the test process crashed on infrastructure — a database lock, a statement
|
|
8
|
+
# timeout — rather than on the mutation itself.
|
|
9
|
+
#
|
|
10
|
+
# Those crashes are demoted to `:neutral` so they do not inflate the kill count
|
|
11
|
+
# (EV-toid / GH #814), which leaves the neutral bucket swinging with `--jobs`
|
|
12
|
+
# on identical input: the contention that causes them only exists while several
|
|
13
|
+
# workers run at once. Re-running them after the pool is done removes the
|
|
14
|
+
# contention, so the verdict matches what a serial run would have produced
|
|
15
|
+
# (EV-j0bv / GH #1607).
|
|
16
|
+
#
|
|
17
|
+
# Only mutations neutralised by an infra crash are re-run. A neutral from a
|
|
18
|
+
# failing baseline is a real statement about the spec, and re-running it would
|
|
19
|
+
# say the same thing again.
|
|
20
|
+
class Evilution::Runner::MutationExecutor::InfraRetry
|
|
21
|
+
attr_reader :retried_count
|
|
22
|
+
|
|
23
|
+
def initialize(runner:, pipeline:)
|
|
24
|
+
@runner = runner
|
|
25
|
+
@pipeline = pipeline
|
|
26
|
+
@retried_count = 0
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def call(results, baseline_result:, integration:)
|
|
30
|
+
@retried_count = 0
|
|
31
|
+
|
|
32
|
+
results.map do |result|
|
|
33
|
+
next result unless retryable?(result)
|
|
34
|
+
|
|
35
|
+
@retried_count += 1
|
|
36
|
+
rerun(result, baseline_result: baseline_result, integration: integration)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def retryable?(result)
|
|
43
|
+
Evilution::Runner::MutationExecutor::Neutralizer::InfraError.infra_neutral?(result)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# The parallel pass keeps the sources of a retryable mutation alive so it can
|
|
47
|
+
# be applied again; they are released here, once it has been.
|
|
48
|
+
def rerun(result, baseline_result:, integration:)
|
|
49
|
+
mutation = result.mutation
|
|
50
|
+
rerun_result = @runner.call(mutation, integration: integration)
|
|
51
|
+
mutation.strip_sources!
|
|
52
|
+
@pipeline.call(rerun_result, baseline_result: baseline_result)
|
|
53
|
+
end
|
|
54
|
+
end
|