saboteur 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +153 -1
- data/README.md +526 -73
- data/lib/generators/saboteur/install/install_generator.rb +21 -0
- data/lib/saboteur/agent_skill.rb +40 -0
- data/lib/saboteur/baseline_health.rb +116 -0
- data/lib/saboteur/cli.rb +305 -30
- data/lib/saboteur/config.rb +65 -19
- data/lib/saboteur/config_file.rb +97 -0
- data/lib/saboteur/custom_map.rb +170 -0
- data/lib/saboteur/doctor.rb +88 -0
- data/lib/saboteur/engine/baseline_guard.rb +116 -0
- data/lib/saboteur/engine/boot.rb +9 -0
- data/lib/saboteur/engine/candidate_selection.rb +35 -0
- data/lib/saboteur/engine/coverage_store.rb +40 -6
- data/lib/saboteur/engine/data_inputs.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +28 -0
- data/lib/saboteur/engine/diff_scope.rb +8 -0
- data/lib/saboteur/engine/distribution.rb +4 -2
- data/lib/saboteur/engine/run_fingerprint.rb +344 -0
- data/lib/saboteur/engine/scheduler.rb +7 -16
- data/lib/saboteur/engine/suite_files.rb +8 -1
- data/lib/saboteur/engine/verdict_check.rb +4 -2
- data/lib/saboteur/engine.rb +125 -25
- data/lib/saboteur/execution/fork_readiness.rb +46 -0
- data/lib/saboteur/execution/reevaluator.rb +7 -4
- data/lib/saboteur/execution/supervisor.rb +23 -11
- data/lib/saboteur/frameworks/rails.rb +329 -13
- data/lib/saboteur/help.rb +210 -0
- data/lib/saboteur/ignore_list.rb +71 -17
- data/lib/saboteur/install.rb +132 -0
- data/lib/saboteur/integration/base.rb +8 -0
- data/lib/saboteur/integration/rspec.rb +69 -4
- data/lib/saboteur/mutation_source/argument_removal.rb +86 -0
- data/lib/saboteur/mutation_source/builtin.rb +14 -3
- data/lib/saboteur/mutation_source/filter_replacement.rb +60 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +74 -0
- data/lib/saboteur/mutation_source/pair_removal.rb +101 -0
- data/lib/saboteur/replay.rb +33 -0
- data/lib/saboteur/reporter/agent.rb +87 -0
- data/lib/saboteur/reporter/agent_clipping.rb +29 -0
- data/lib/saboteur/reporter/agent_commands.rb +61 -0
- data/lib/saboteur/reporter/agent_incomplete.rb +37 -0
- data/lib/saboteur/reporter/agent_summary.rb +131 -0
- data/lib/saboteur/reporter/findings.rb +131 -0
- data/lib/saboteur/reporter/html.rb +213 -0
- data/lib/saboteur/reporter/replay.rb +23 -0
- data/lib/saboteur/reporter/stryker.rb +21 -4
- data/lib/saboteur/reporter/terminal.rb +76 -2
- data/lib/saboteur/result.rb +61 -9
- data/lib/saboteur/score.rb +89 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +48 -0
- data/lib/saboteur.rb +7 -0
- metadata +30 -2
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "cgi"
|
|
5
|
+
require "fileutils"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
module Reporter
|
|
9
|
+
# One self-contained HTML page: inline style, no script, no network. It is
|
|
10
|
+
# rendered from the same hash the JSON reporter writes (Stryker.build), so
|
|
11
|
+
# both reports list the same mutants with the same statuses. The scores can
|
|
12
|
+
# differ: the JSON schema counts NoCoverage as undetected, while this page
|
|
13
|
+
# shows Report#score, which leaves it out (see the "Covered mutation score"
|
|
14
|
+
# label on the page). The score note says what is and is not counted.
|
|
15
|
+
class Html
|
|
16
|
+
STYLE = <<~CSS
|
|
17
|
+
body{font:14px/1.5 system-ui,sans-serif;margin:1rem auto;max-width:72rem;padding:0 1rem;color:#1b1b1b}
|
|
18
|
+
h1,h2{margin:1.2em 0 .4em}table{border-collapse:collapse;width:100%;min-width:60rem}
|
|
19
|
+
th,td{border-bottom:1px solid #ddd;padding:.25rem .5rem;text-align:left;vertical-align:top;overflow-wrap:break-word}
|
|
20
|
+
td.fit{white-space:nowrap}td.wide{min-width:14rem}
|
|
21
|
+
th{background:#f4f4f4;font-size:12px;text-transform:uppercase}
|
|
22
|
+
.scroll{overflow-x:auto}.note{color:#555;font-size:12px}
|
|
23
|
+
.status{border-radius:3px;padding:0 .4rem;color:#fff;white-space:nowrap}
|
|
24
|
+
.Killed{background:#2e7d32}.Survived{background:#c62828}.Timeout{background:#b45309}
|
|
25
|
+
.RuntimeError{background:#6a1b9a}.NoCoverage{background:#455a64}.Ignored{background:#616161}
|
|
26
|
+
.Accepted{background:#8d6e63}
|
|
27
|
+
.warning{background:#fff8e1;border-left:4px solid #f9a825;margin:.3rem 0;padding:.3rem .6rem;overflow-wrap:anywhere}
|
|
28
|
+
code,pre{font-family:ui-monospace,monospace;font-size:13px}
|
|
29
|
+
pre{display:inline-block;vertical-align:top;margin:0;white-space:pre-wrap;overflow-wrap:anywhere}summary{cursor:pointer;font-weight:600;overflow-wrap:anywhere}
|
|
30
|
+
CSS
|
|
31
|
+
|
|
32
|
+
SCORE_FORMULA = "(killed + timeout) / (killed + survived + timeout)"
|
|
33
|
+
|
|
34
|
+
def initialize(path)
|
|
35
|
+
@path = path
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def call(report)
|
|
39
|
+
FileUtils.mkdir_p(File.dirname(@path))
|
|
40
|
+
File.write(@path, self.class.build(report))
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def self.build(report)
|
|
44
|
+
new(nil).build(report)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def build(report)
|
|
48
|
+
data = Stryker.build(report)
|
|
49
|
+
names = test_names(data)
|
|
50
|
+
results = report.results.to_h do |result|
|
|
51
|
+
[[Stryker.relative(result.subject.file, report.root), result.digest], result]
|
|
52
|
+
end
|
|
53
|
+
<<~HTML
|
|
54
|
+
<!DOCTYPE html>
|
|
55
|
+
<html lang="en"><head><meta charset="utf-8">
|
|
56
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
57
|
+
<title>saboteur report</title>
|
|
58
|
+
<style>#{STYLE}</style></head><body>
|
|
59
|
+
<h1>saboteur report</h1>
|
|
60
|
+
#{summary(report, data)}#{warnings(data[:warnings])}#{skipped_subjects(data[:skippedSubjects])}
|
|
61
|
+
<h2>Files</h2>
|
|
62
|
+
#{data[:files].map { |file, entry| file_section(file, entry, names, results) }.join("\n")}
|
|
63
|
+
</body></html>
|
|
64
|
+
HTML
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
private
|
|
68
|
+
|
|
69
|
+
def summary(report, data)
|
|
70
|
+
seed = data.dig(:config, :seed)
|
|
71
|
+
seed_note = seed ? ", seed #{seed}" : ""
|
|
72
|
+
counts = data[:files].values.flat_map { |entry| entry[:mutants] }.map { |mutant| mutant[:status] }.tally
|
|
73
|
+
<<~HTML
|
|
74
|
+
<p><strong>Covered mutation score: #{score_text(report.score)}</strong>
|
|
75
|
+
(#{report.subjects.size} subjects, #{report.results.size} mutants, #{format('%.1f', report.wall_time)}s#{seed_note})</p>
|
|
76
|
+
<p class="note">Covered mutation score = #{SCORE_FORMULA}. NoCoverage, errors, resource limits and skipped
|
|
77
|
+
mutants are not counted. Survivors accepted on the ignore list stay in the denominator. It covers only the subjects of this run, not the whole project. A tool that reads the
|
|
78
|
+
Stryker JSON may show a different score, because the schema counts NoCoverage as undetected.</p>
|
|
79
|
+
#{score_split(report)}
|
|
80
|
+
<p class="note">Wide tables scroll sideways.</p>
|
|
81
|
+
<p>#{counts.map { |status, count| "#{status_label(status)} #{count}" }.join(' ')}</p>
|
|
82
|
+
<p>Survivors: #{report.unignored_survivors.size} to fix, #{report.survivors.size - report.unignored_survivors.size}
|
|
83
|
+
accepted on the ignore list (still shown as Survived, with the reason).</p>
|
|
84
|
+
HTML
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# How the score splits into assertion and crash kills, and the score
|
|
88
|
+
# without the whole-body mutants, which almost any test that runs the
|
|
89
|
+
# method kills. Each part shows only when it has something to say.
|
|
90
|
+
def score_split(report)
|
|
91
|
+
breakdown = report.score_breakdown
|
|
92
|
+
lines = []
|
|
93
|
+
lines << kill_split_line(breakdown) if breakdown.split_known?
|
|
94
|
+
lines << whole_body_line(report)
|
|
95
|
+
lines.compact.join("\n")
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def kill_split_line(breakdown)
|
|
99
|
+
"<p>Of the tested mutants: #{parts_text(breakdown)}. A crash shows the code ran and broke, " \
|
|
100
|
+
"not that a test checked a value.</p>"
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def parts_text(breakdown)
|
|
104
|
+
breakdown.parts.map { |label, share| "#{score_text(share)} #{label}" }.join(", ")
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def whole_body_line(report)
|
|
108
|
+
whole = report.coarse_score_breakdown
|
|
109
|
+
fine = report.fine_score_breakdown
|
|
110
|
+
return if whole.tested.zero? || fine.tested.zero?
|
|
111
|
+
|
|
112
|
+
split = fine.split_known? ? " (#{parts_text(fine)})" : ""
|
|
113
|
+
"<p>Without whole-body mutants (body_nil, body_true, body_false, body_raise): #{score_text(fine.value)}" \
|
|
114
|
+
"#{split} over #{fine.tested} mutants. Whole-body mutants alone: #{score_text(whole.value)} " \
|
|
115
|
+
"over #{whole.tested}.</p>"
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def score_text(score)
|
|
119
|
+
score ? format("%.1f%%", score * 100) : "n/a"
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def warnings(messages)
|
|
123
|
+
return "" if messages.empty?
|
|
124
|
+
|
|
125
|
+
items = messages.map { |message| %(<div class="warning">#{escape(message)}</div>) }
|
|
126
|
+
"<h2>Warnings</h2>\n#{items.join("\n")}"
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def skipped_subjects(entries)
|
|
130
|
+
return "" if entries.empty?
|
|
131
|
+
|
|
132
|
+
rows = entries.map do |entry|
|
|
133
|
+
"<tr><td>#{escape(entry[:subject])}</td><td>#{escape(entry[:file])}:#{entry[:line]}</td>" \
|
|
134
|
+
"<td>#{escape(entry[:reason])}</td></tr>"
|
|
135
|
+
end
|
|
136
|
+
"<h2>Skipped subjects</h2>\n<div class=\"scroll\"><table><thead><tr><th>Subject</th><th>Location</th>" \
|
|
137
|
+
"<th>Reason</th></tr></thead>#{rows.join}</table></div>"
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def test_names(data)
|
|
141
|
+
data[:testFiles].values.flat_map { |group| group[:tests] }.to_h { |test| [test[:id], test[:name]] }
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def file_section(file, entry, names, results)
|
|
145
|
+
open = entry[:mutants].any? { |mutant| mutant[:status] == "Survived" } ? " open" : ""
|
|
146
|
+
ordered = entry[:mutants].sort_by { |mutant| position_key(mutant) }
|
|
147
|
+
rows = ordered.map { |mutant| mutant_row(mutant, names, results[[file, mutant[:id]]]) }
|
|
148
|
+
"<details#{open}><summary>#{escape(file)} (#{entry[:mutants].size} mutants)</summary>\n" \
|
|
149
|
+
"<div class=\"scroll\"><table><thead><tr><th>Location</th><th>Status</th><th>Operator</th><th>Change</th>" \
|
|
150
|
+
"<th>Id</th><th>Notes</th></tr></thead>\n#{rows.join("\n")}</table></div></details>"
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def position_key(mutant)
|
|
154
|
+
start = mutant[:location][:start]
|
|
155
|
+
[start[:line], start[:column], mutant[:id]]
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def mutant_row(mutant, names, result)
|
|
159
|
+
start = mutant[:location][:start]
|
|
160
|
+
tests = Array(mutant[:killedBy]).map { |id| names.fetch(id) }.join(", ")
|
|
161
|
+
notes = [mutant[:statusReason], tests.empty? ? nil : "killed by #{tests}",
|
|
162
|
+
passing_tests_note(mutant, names, result)].compact.join("; ")
|
|
163
|
+
"<tr><td class=\"fit\">line #{start[:line]}, column #{start[:column]}</td><td>#{row_status(mutant,
|
|
164
|
+
result)}</td>" \
|
|
165
|
+
"<td>#{escape(mutant[:mutatorName])}</td><td class=\"wide\">#{change(mutant, result)}</td>" \
|
|
166
|
+
"<td class=\"fit\"><code>#{escape(mutant[:id])}</code></td><td class=\"wide\">#{escape(notes)}</td></tr>"
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# A survivor's covering tests all passed with the mutant applied, or, for
|
|
170
|
+
# a reused verdict, in the earlier run it came from.
|
|
171
|
+
def passing_tests_note(mutant, names, result)
|
|
172
|
+
return unless mutant[:status] == "Survived"
|
|
173
|
+
|
|
174
|
+
passed = Array(mutant[:coveredBy]).map { |id| names.fetch(id) }
|
|
175
|
+
return if passed.empty?
|
|
176
|
+
|
|
177
|
+
shown = passed.first(Terminal::MAX_PASSING_TESTS).join(", ")
|
|
178
|
+
more = passed.size - Terminal::MAX_PASSING_TESTS
|
|
179
|
+
extra = more.positive? ? " (+#{more} more)" : ""
|
|
180
|
+
when_passed = result.reused? ? "passed in an earlier run" : "passed"
|
|
181
|
+
"covered by #{passed.size} #{passed.one? ? 'test' : 'tests'} that #{when_passed}: #{shown}#{extra}"
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
def row_status(mutant, result)
|
|
185
|
+
badge = status_label(mutant[:status])
|
|
186
|
+
# Only survivors are ever put on the ignore list.
|
|
187
|
+
result.ignored ? "#{badge} #{status_label('Accepted')}" : badge
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
# One edit shows the original text and its replacement. A mutant with
|
|
191
|
+
# several edits shows only the replacements, because one location cannot
|
|
192
|
+
# stand for all of them.
|
|
193
|
+
def change(mutant, result)
|
|
194
|
+
edits = result.mutation.edits
|
|
195
|
+
return "<pre>#{escape(mutant[:replacement])}</pre>" unless edits.size == 1
|
|
196
|
+
|
|
197
|
+
source = result.subject.file_source
|
|
198
|
+
edit = edits.first
|
|
199
|
+
original = source.byteslice(edit.start_byte, edit.end_byte - edit.start_byte).to_s
|
|
200
|
+
"<pre>#{escape(original.dup.force_encoding(Encoding::UTF_8).scrub)}</pre> becomes " \
|
|
201
|
+
"<pre>#{escape(edit.replacement)}</pre>"
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def status_label(status)
|
|
205
|
+
%(<span class="status #{status}">#{status}</span>)
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def escape(text)
|
|
209
|
+
CGI.escapeHTML(text.to_s)
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
end
|
|
213
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module Reporter
|
|
6
|
+
# A replay is an inspection too, regardless of the resulting verdict.
|
|
7
|
+
class Replay
|
|
8
|
+
def initialize(out)
|
|
9
|
+
@out = out
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def call(report)
|
|
13
|
+
report.results.each do |result|
|
|
14
|
+
@out.puts "\nReplay: #{result.subject.id} #{result.mutation.operator} #{result.digest} (#{result.status})"
|
|
15
|
+
@out.puts result.diff(root: report.root)
|
|
16
|
+
@out.puts "Selected tests:"
|
|
17
|
+
result.covered_by.each { |name| @out.puts " #{name}" }
|
|
18
|
+
@out.puts " none" if result.covered_by.empty?
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -16,14 +16,14 @@ module Saboteur
|
|
|
16
16
|
@path = path
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
-
def call(report)
|
|
19
|
+
def call(report, exit_cause: nil)
|
|
20
20
|
FileUtils.mkdir_p(File.dirname(@path))
|
|
21
|
-
File.write(@path, JSON.generate(self.class.build(report)))
|
|
21
|
+
File.write(@path, JSON.generate(self.class.build(report, exit_cause: exit_cause)))
|
|
22
22
|
end
|
|
23
23
|
|
|
24
24
|
# Tests are listed once under testFiles with short ids; mutants refer to
|
|
25
25
|
# them by id in killedBy/coveredBy instead of repeating full test names.
|
|
26
|
-
def self.build(report)
|
|
26
|
+
def self.build(report, exit_cause: nil)
|
|
27
27
|
test_ids = test_index(report.results)
|
|
28
28
|
files = report.results.group_by { |result| result.subject.file }.to_h do |file, results|
|
|
29
29
|
[relative(file, report.root), file_entry(results, test_ids)]
|
|
@@ -34,10 +34,27 @@ module Saboteur
|
|
|
34
34
|
projectRoot: report.root,
|
|
35
35
|
files: files,
|
|
36
36
|
testFiles: test_files(test_ids),
|
|
37
|
-
|
|
37
|
+
skippedSubjects: skipped_subjects(report),
|
|
38
|
+
rejectedKills: report.rejected_kills,
|
|
39
|
+
warnings: report.warnings,
|
|
40
|
+
framework: { name: "saboteur", version: VERSION },
|
|
41
|
+
saboteur: { findings: Findings.build(report), scores: scores(report) }
|
|
42
|
+
.merge(exit_cause ? { exitCause: exit_cause } : {})
|
|
38
43
|
}.merge(report.seed ? { config: { seed: report.seed } } : {})
|
|
39
44
|
end
|
|
40
45
|
|
|
46
|
+
# Score with its assertion and crash shares, for all mutants and without
|
|
47
|
+
# the whole-body ones (body_nil, body_true, body_false, body_raise).
|
|
48
|
+
def self.scores(report)
|
|
49
|
+
{ all: report.score_breakdown.to_h, withoutWholeBodyMutants: report.fine_score_breakdown.to_h }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def self.skipped_subjects(report)
|
|
53
|
+
report.skipped_subjects.map do |subject, reason|
|
|
54
|
+
{ subject: subject.id, file: relative(subject.file, report.root), line: subject.start_line, reason: reason }
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
41
58
|
def self.test_index(results)
|
|
42
59
|
names = results.flat_map { |result| result.covered_by + result.killed_by }.uniq
|
|
43
60
|
names.each_with_index.to_h { |name, index| [name, index.to_s] }
|
|
@@ -7,6 +7,8 @@ module Saboteur
|
|
|
7
7
|
# errors, then a unified diff for each survivor.
|
|
8
8
|
class Terminal
|
|
9
9
|
MAX_ERRORS = 10
|
|
10
|
+
MAX_PASSING_TESTS = 5
|
|
11
|
+
MAX_CHANGED_LINES = 3
|
|
10
12
|
|
|
11
13
|
def initialize(out)
|
|
12
14
|
@out = out
|
|
@@ -14,6 +16,7 @@ module Saboteur
|
|
|
14
16
|
|
|
15
17
|
def call(report)
|
|
16
18
|
summary(report)
|
|
19
|
+
skipped_subjects(report)
|
|
17
20
|
warnings(report)
|
|
18
21
|
errors(report)
|
|
19
22
|
survivors(report)
|
|
@@ -39,9 +42,10 @@ module Saboteur
|
|
|
39
42
|
end
|
|
40
43
|
|
|
41
44
|
def details(report)
|
|
42
|
-
|
|
43
|
-
@out.puts "score #{
|
|
45
|
+
breakdown = report.score_breakdown
|
|
46
|
+
@out.puts "score #{percent(breakdown.value)}#{score_split(breakdown)} " \
|
|
44
47
|
"(killed+timeout)/(killed+survived+timeout); errors, resource limits and skips excluded"
|
|
48
|
+
whole_body_score(report)
|
|
45
49
|
mean = report.mean_runtime
|
|
46
50
|
@out.puts "mean time per executed mutant #{format('%.3f', mean)} s" if mean
|
|
47
51
|
@out.puts "invalid mutations dropped (did not parse): #{report.invalid_mutations}" if
|
|
@@ -50,9 +54,35 @@ module Saboteur
|
|
|
50
54
|
reused = report.results.count(&:reused?)
|
|
51
55
|
@out.puts "reused #{reused} of #{report.results.size} verdicts from earlier runs (--incremental)" if
|
|
52
56
|
reused.positive?
|
|
57
|
+
@out.puts "kills rejected by unchanged-method check: #{report.rejected_kills}"
|
|
53
58
|
kill_causes(report)
|
|
54
59
|
end
|
|
55
60
|
|
|
61
|
+
# " (assertion 50.0 %, crash 50.0 %)": how much of the score is a failed
|
|
62
|
+
# check and how much is a crash. Timeouts and kills with no recorded
|
|
63
|
+
# cause are named only when there are some, so the parts add up.
|
|
64
|
+
def score_split(breakdown)
|
|
65
|
+
return "" unless breakdown.split_known?
|
|
66
|
+
|
|
67
|
+
" (#{breakdown.parts.map { |label, share| "#{label} #{percent(share)}" }.join(', ')})"
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Whole-body mutants (body_nil, body_true, body_false, body_raise) are
|
|
71
|
+
# killed by almost any test that runs the method, so they get their own
|
|
72
|
+
# line.
|
|
73
|
+
def whole_body_score(report)
|
|
74
|
+
whole = report.coarse_score_breakdown
|
|
75
|
+
fine = report.fine_score_breakdown
|
|
76
|
+
return if whole.tested.zero? || fine.tested.zero?
|
|
77
|
+
|
|
78
|
+
@out.puts "score without whole-body mutants #{percent(fine.value)}#{score_split(fine)} " \
|
|
79
|
+
"(#{fine.tested} mutants); whole-body mutants alone #{percent(whole.value)} (#{whole.tested})"
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def percent(share)
|
|
83
|
+
share ? format("%.1f %%", share * 100) : "n/a"
|
|
84
|
+
end
|
|
85
|
+
|
|
56
86
|
# How many kills came from a failed check and how many from a crash, and
|
|
57
87
|
# which operators mostly kill by crashing (their score says less).
|
|
58
88
|
def kill_causes(report)
|
|
@@ -80,6 +110,15 @@ module Saboteur
|
|
|
80
110
|
parts.join(", ")
|
|
81
111
|
end
|
|
82
112
|
|
|
113
|
+
def skipped_subjects(report)
|
|
114
|
+
report.skipped_subjects.values.tally.each do |reason, count|
|
|
115
|
+
@out.puts "skipped subjects: #{reason}: #{count}"
|
|
116
|
+
end
|
|
117
|
+
report.skipped_subjects.each do |subject, reason|
|
|
118
|
+
@out.puts " #{subject.id} #{relative(subject.file, report.root)}:#{subject.start_line}: #{reason}"
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
83
122
|
def warnings(report)
|
|
84
123
|
report.warnings.each { |warning| @out.puts "warning: #{warning}" }
|
|
85
124
|
end
|
|
@@ -103,12 +142,47 @@ module Saboteur
|
|
|
103
142
|
shown(report).each do |result, hidden|
|
|
104
143
|
location = "#{relative(result.subject.file, report.root)}:#{result.subject.start_line}"
|
|
105
144
|
@out.puts "\n#{result.subject.id} #{result.mutation.operator} #{result.digest} #{location}"
|
|
145
|
+
changed_lines(report, result)
|
|
106
146
|
@out.puts "ignored: #{result.ignored}" if result.ignored
|
|
107
147
|
@out.puts result.diff(root: report.root)
|
|
148
|
+
passing_tests(result)
|
|
108
149
|
@out.puts "#{hidden} more survivor#{'s' if hidden > 1} on this line in the JSON report" if hidden.positive?
|
|
109
150
|
end
|
|
110
151
|
end
|
|
111
152
|
|
|
153
|
+
# The tests that ran with the mutant applied and still passed: the
|
|
154
|
+
# missing assertion belongs in one of them. They are listed likeliest
|
|
155
|
+
# killers first, the order the run used. Line coverage can overstate which
|
|
156
|
+
# tests exercise the line, so the list is a lead, not proof.
|
|
157
|
+
def passing_tests(result)
|
|
158
|
+
tests = result.covered_by
|
|
159
|
+
return if tests.empty?
|
|
160
|
+
|
|
161
|
+
@out.puts "#{tests.size} #{tests.one? ? 'test' : 'tests'} covered this line and #{passed_phrase(result)} " \
|
|
162
|
+
"(the missing check may belong in one of them):"
|
|
163
|
+
tests.first(MAX_PASSING_TESTS).each { |name| @out.puts " #{name}" }
|
|
164
|
+
more = tests.size - MAX_PASSING_TESTS
|
|
165
|
+
@out.puts " ... #{more} more in the JSON report (coveredBy)" if more.positive?
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
# A reused verdict was not run again: its tests passed in the earlier run.
|
|
169
|
+
def passed_phrase(result)
|
|
170
|
+
result.reused? ? "passed in an earlier run (--incremental)" : "still passed"
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# In diff mode: the changed lines this mutant touches. The location above
|
|
174
|
+
# is where the mutated statement starts, which can be a line or two above
|
|
175
|
+
# the line that changed.
|
|
176
|
+
def changed_lines(report, result)
|
|
177
|
+
lines = report.changed_lines_for(result)
|
|
178
|
+
return if lines.empty?
|
|
179
|
+
|
|
180
|
+
path = relative(result.subject.file, report.root)
|
|
181
|
+
shown = lines.first(MAX_CHANGED_LINES).map { |line| "#{path}:#{line}" }.join(", ")
|
|
182
|
+
more = lines.size - MAX_CHANGED_LINES
|
|
183
|
+
@out.puts "changed #{lines.one? ? 'line' : 'lines'}: #{shown}#{" (+#{more} more)" if more.positive?}"
|
|
184
|
+
end
|
|
185
|
+
|
|
112
186
|
# [[result, how many more on the same line]]. In review mode survivors
|
|
113
187
|
# on one line usually share a fate, so one is shown: live before
|
|
114
188
|
# ignored, then coarse before fine, then the earliest edit.
|
data/lib/saboteur/result.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
require_relative "reporter/diff"
|
|
5
|
+
require_relative "score"
|
|
5
6
|
|
|
6
7
|
module Saboteur
|
|
7
8
|
# resource_limit: the mutant's process tree hit the memory or process cap.
|
|
@@ -17,7 +18,7 @@ module Saboteur
|
|
|
17
18
|
attr_accessor :ignored
|
|
18
19
|
|
|
19
20
|
def initialize(subject:, mutation:, status:, mutated_source:, reason: nil, runtime: nil, killed_by: [],
|
|
20
|
-
covered_by: [], kill_cause: nil, reused: false)
|
|
21
|
+
covered_by: [], kill_cause: nil, reused: false, rejected_kill: false)
|
|
21
22
|
raise ArgumentError, "unknown status #{status.inspect}" unless STATUSES.include?(status)
|
|
22
23
|
|
|
23
24
|
@subject = subject
|
|
@@ -29,14 +30,20 @@ module Saboteur
|
|
|
29
30
|
@covered_by = covered_by
|
|
30
31
|
@mutated_source = mutated_source
|
|
31
32
|
@kill_cause = kill_cause
|
|
33
|
+
@rejected_kill = rejected_kill
|
|
32
34
|
@reused = reused
|
|
33
35
|
@ignored = nil
|
|
34
36
|
end
|
|
35
37
|
|
|
36
38
|
# The same mutant, marked as an error: its verdict did not hold up.
|
|
37
|
-
def as_error(reason)
|
|
39
|
+
def as_error(reason, rejected_kill: false)
|
|
38
40
|
MutantResult.new(subject: subject, mutation: mutation, status: :error, mutated_source: mutated_source,
|
|
39
|
-
reason: reason, runtime: runtime, killed_by: killed_by, covered_by: covered_by
|
|
41
|
+
reason: reason, runtime: runtime, killed_by: killed_by, covered_by: covered_by,
|
|
42
|
+
rejected_kill: rejected_kill)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def rejected_kill?
|
|
46
|
+
@rejected_kill
|
|
40
47
|
end
|
|
41
48
|
|
|
42
49
|
def digest
|
|
@@ -53,6 +60,11 @@ module Saboteur
|
|
|
53
60
|
status == :killed && !kill_cause.nil? && kill_cause != "assertion"
|
|
54
61
|
end
|
|
55
62
|
|
|
63
|
+
# Killed because a test's check failed: the test looked at a value.
|
|
64
|
+
def assertion_kill?
|
|
65
|
+
status == :killed && kill_cause == "assertion"
|
|
66
|
+
end
|
|
67
|
+
|
|
56
68
|
# Taken from an earlier run (--incremental) instead of run again.
|
|
57
69
|
def reused?
|
|
58
70
|
@reused
|
|
@@ -78,12 +90,16 @@ module Saboteur
|
|
|
78
90
|
|
|
79
91
|
# Everything one run produced.
|
|
80
92
|
class Report
|
|
81
|
-
|
|
93
|
+
# More than half of the executed mutants errored: the run tested
|
|
94
|
+
# almost nothing and must not pass a gate.
|
|
95
|
+
MAX_ERROR_FRACTION = 0.5
|
|
96
|
+
|
|
97
|
+
attr_reader :results, :subjects, :wall_time, :invalid_mutations, :warnings, :root, :seed, :skipped_subjects
|
|
82
98
|
|
|
83
99
|
# REVIEW: diff mode; the terminal then shows one survivor per line.
|
|
84
100
|
# seed: the random seed every test run started from (nil when unknown).
|
|
85
101
|
def initialize(results:, subjects:, wall_time:, root:, invalid_mutations: 0, warnings: [], review: false,
|
|
86
|
-
seed: nil)
|
|
102
|
+
seed: nil, skipped_subjects: {}, diff_scope: nil)
|
|
87
103
|
@results = results
|
|
88
104
|
@subjects = subjects
|
|
89
105
|
@wall_time = wall_time
|
|
@@ -92,6 +108,29 @@ module Saboteur
|
|
|
92
108
|
@warnings = warnings
|
|
93
109
|
@review = review
|
|
94
110
|
@seed = seed
|
|
111
|
+
@skipped_subjects = skipped_subjects
|
|
112
|
+
@diff_scope = diff_scope
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# In diff mode, the changed lines (sorted) that a mutant touches; empty
|
|
116
|
+
# outside diff mode, or when the mutant touches none (a whole-body mutant
|
|
117
|
+
# of a method whose only change is its signature).
|
|
118
|
+
def changed_lines_for(result)
|
|
119
|
+
@diff_scope ? @diff_scope.changed_lines_for(result.subject, result.mutation) : []
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def rejected_kills
|
|
123
|
+
results.count(&:rejected_kill?)
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Errored mutants (a crash, a neutral check that failed) are outside
|
|
127
|
+
# the score, so a run whose executed mutants mostly errored tested
|
|
128
|
+
# almost nothing. Skipped mutants never ran: they are neither the
|
|
129
|
+
# numerator nor the denominator.
|
|
130
|
+
def majority_errored?
|
|
131
|
+
return false if executed.empty?
|
|
132
|
+
|
|
133
|
+
executed.count { |result| result.status == :error }.fdiv(executed.size) > MAX_ERROR_FRACTION
|
|
95
134
|
end
|
|
96
135
|
|
|
97
136
|
def review?
|
|
@@ -104,10 +143,23 @@ module Saboteur
|
|
|
104
143
|
|
|
105
144
|
# (killed + timeout) / (killed + survived + timeout); nil with nothing tested.
|
|
106
145
|
def score
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
146
|
+
score_breakdown.value
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# The score with its split into assertion kills and crash kills.
|
|
150
|
+
def score_breakdown
|
|
151
|
+
@score_breakdown ||= Score.of(results)
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# The same breakdown without the whole-body mutants (every
|
|
155
|
+
# mutation marked coarse: body_nil, body_true, body_false, body_raise). Almost any test that runs the method kills
|
|
156
|
+
# those, so they flatter the score.
|
|
157
|
+
def fine_score_breakdown
|
|
158
|
+
@fine_score_breakdown ||= Score.of(results.reject { |result| result.mutation.coarse })
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def coarse_score_breakdown
|
|
162
|
+
@coarse_score_breakdown ||= Score.of(results.select { |result| result.mutation.coarse })
|
|
111
163
|
end
|
|
112
164
|
|
|
113
165
|
def survivors
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
# The mutation score of a group of results, with the share of it that came
|
|
6
|
+
# from failed assertions and the share that came from crashes.
|
|
7
|
+
#
|
|
8
|
+
# value is (killed + timeout) / (killed + survived + timeout). A crash kill
|
|
9
|
+
# shows the code ran and broke, not that a test checked a value, so the
|
|
10
|
+
# split lets a reader see how much of the score is checked behavior.
|
|
11
|
+
# Errors, resource limits and skips are outside every number here.
|
|
12
|
+
class Score
|
|
13
|
+
def self.of(results)
|
|
14
|
+
new(results)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
attr_reader :assertion_kills, :crash_kills, :unclassified_kills, :timeouts, :survivors
|
|
18
|
+
|
|
19
|
+
def initialize(results)
|
|
20
|
+
killed = results.select { |result| result.status == :killed }
|
|
21
|
+
@assertion_kills = killed.count(&:assertion_kill?)
|
|
22
|
+
@crash_kills = killed.count(&:crash_kill?)
|
|
23
|
+
@unclassified_kills = killed.size - assertion_kills - crash_kills
|
|
24
|
+
@timeouts = results.count { |result| result.status == :timeout }
|
|
25
|
+
@survivors = results.count(&:survived?)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def detected
|
|
29
|
+
assertion_kills + crash_kills + unclassified_kills + timeouts
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def tested
|
|
33
|
+
detected + survivors
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# nil with nothing tested.
|
|
37
|
+
def value
|
|
38
|
+
share(detected)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def assertion_share
|
|
42
|
+
share(assertion_kills)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def crash_share
|
|
46
|
+
share(crash_kills)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def unclassified_share
|
|
50
|
+
share(unclassified_kills)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def timeout_share
|
|
54
|
+
share(timeouts)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# False when no kill has a recorded cause: a split would then say
|
|
58
|
+
# nothing. A cause is missing for verdicts reused from a run made before
|
|
59
|
+
# causes were recorded, for a test that called exit, and for a failure
|
|
60
|
+
# outside any example (suite level).
|
|
61
|
+
def split_known?
|
|
62
|
+
assertion_kills.positive? || crash_kills.positive?
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# [[label, share]]: assertion and crash always, then timeout and
|
|
66
|
+
# "cause not recorded" only when there are some, so the parts add up to
|
|
67
|
+
# value.
|
|
68
|
+
def parts
|
|
69
|
+
list = [["assertion", assertion_share], ["crash", crash_share]]
|
|
70
|
+
list << ["timeout", timeout_share] if timeouts.positive?
|
|
71
|
+
list << ["cause not recorded", unclassified_share] if unclassified_kills.positive?
|
|
72
|
+
list
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def to_h
|
|
76
|
+
{
|
|
77
|
+
score: value, tested: tested, assertionKills: assertion_kills, crashKills: crash_kills,
|
|
78
|
+
unclassifiedKills: unclassified_kills, timeouts: timeouts, survivors: survivors,
|
|
79
|
+
assertionShare: assertion_share, crashShare: crash_share
|
|
80
|
+
}
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
private
|
|
84
|
+
|
|
85
|
+
def share(count)
|
|
86
|
+
tested.zero? ? nil : count.fdiv(tested)
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
data/lib/saboteur/version.rb
CHANGED