saboteur 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +66 -0
- data/README.md +109 -247
- data/exe/saboteur +2 -2
- data/lib/saboteur/baseline/loader.rb +41 -0
- data/lib/saboteur/baseline/run_evidence.rb +1 -1
- data/lib/saboteur/baseline/run_gate.rb +51 -0
- data/lib/saboteur/baseline/workflow.rb +2 -15
- data/lib/saboteur/cli/baseline_gate.rb +58 -0
- data/lib/saboteur/cli/error_cause.rb +16 -0
- data/lib/saboteur/cli/exit_status.rb +14 -13
- data/lib/saboteur/cli/failure_output.rb +23 -0
- data/lib/saboteur/cli/output_options.rb +24 -2
- data/lib/saboteur/cli/run_budget_options.rb +6 -0
- data/lib/saboteur/cli/run_command.rb +14 -1
- data/lib/saboteur/cli/run_reporting.rb +34 -6
- data/lib/saboteur/cli/run_validation.rb +36 -3
- data/lib/saboteur/cli/selection_options.rb +2 -0
- data/lib/saboteur/cli.rb +6 -1
- data/lib/saboteur/config.rb +23 -2
- data/lib/saboteur/crash_report.rb +16 -7
- data/lib/saboteur/engine/baseline_guard.rb +2 -2
- data/lib/saboteur/engine/coverage_partial.rb +8 -1
- data/lib/saboteur/engine/coverage_store.rb +4 -5
- data/lib/saboteur/engine/diagnostics.rb +120 -15
- data/lib/saboteur/engine/run_fingerprint.rb +1 -1
- data/lib/saboteur/engine/scheduler.rb +20 -0
- data/lib/saboteur/engine/selection_announcement.rb +8 -1
- data/lib/saboteur/engine/survivor_confirmation.rb +159 -0
- data/lib/saboteur/engine.rb +43 -6
- data/lib/saboteur/help.rb +4 -1
- data/lib/saboteur/mutation_source/builtin.rb +9 -3
- data/lib/saboteur/mutation_source/copy_replacement.rb +72 -0
- data/lib/saboteur/mutation_source/ensure_removal.rb +49 -0
- data/lib/saboteur/mutation_source/operator_examples.rb +2 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +3 -1
- data/lib/saboteur/reporter/agent_summary.rb +3 -1
- data/lib/saboteur/reporter/assets/mutation_report.LICENSE +201 -0
- data/lib/saboteur/reporter/assets/mutation_report.js +331 -0
- data/lib/saboteur/reporter/baseline_tail.rb +93 -0
- data/lib/saboteur/reporter/diff.rb +10 -0
- data/lib/saboteur/reporter/html.rb +69 -14
- data/lib/saboteur/reporter/kill_causes.rb +53 -0
- data/lib/saboteur/reporter/next_steps.rb +4 -3
- data/lib/saboteur/reporter/palette.rb +64 -13
- data/lib/saboteur/reporter/safe_out.rb +1 -1
- data/lib/saboteur/reporter/score_table.rb +53 -18
- data/lib/saboteur/reporter/stryker.rb +4 -1
- data/lib/saboteur/reporter/survivor_classification.rb +12 -3
- data/lib/saboteur/reporter/survivor_hints.rb +72 -0
- data/lib/saboteur/reporter/survivor_list.rb +207 -75
- data/lib/saboteur/reporter/terminal.rb +315 -69
- data/lib/saboteur/result.rb +28 -1
- data/lib/saboteur/score.rb +6 -0
- data/lib/saboteur/version.rb +1 -1
- metadata +12 -2
|
@@ -2,11 +2,14 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
require_relative "safe_out"
|
|
5
|
+
require_relative "baseline_tail"
|
|
5
6
|
require_relative "next_steps"
|
|
7
|
+
require_relative "kill_causes"
|
|
6
8
|
require_relative "survivor_ranking"
|
|
7
9
|
require_relative "palette"
|
|
8
10
|
require_relative "score_table"
|
|
9
11
|
require_relative "survivor_list"
|
|
12
|
+
require_relative "survivor_classification"
|
|
10
13
|
require_relative "test_matrix"
|
|
11
14
|
|
|
12
15
|
module Saboteur
|
|
@@ -16,20 +19,43 @@ module Saboteur
|
|
|
16
19
|
# Debug statistics (score split, timings, kill causes) need verbose.
|
|
17
20
|
class Terminal
|
|
18
21
|
MAX_ERRORS = 10
|
|
19
|
-
|
|
22
|
+
MAX_NOT_SCORED = 10
|
|
23
|
+
MAX_PASSING_TESTS = 5
|
|
20
24
|
|
|
21
|
-
def initialize(out, verbose: false, color: :auto,
|
|
25
|
+
def initialize(out, verbose: false, quiet: false, color: :auto, overrides: nil,
|
|
26
|
+
replay_command: nil, all_mutants: false, exit_code: nil,
|
|
27
|
+
threshold: nil, previous_score: nil, baseline_gate: nil,
|
|
28
|
+
scope: "whole project", html_report: nil, baseline: nil, coverage: nil, mutant_phase: nil)
|
|
22
29
|
@out = SafeOut.new(out)
|
|
30
|
+
@all_mutants = all_mutants
|
|
23
31
|
@verbose = verbose
|
|
24
|
-
@
|
|
32
|
+
@quiet = quiet
|
|
33
|
+
# Clean-screen context: scope and report path from the CLI; PR4a plumbs the phase stats.
|
|
34
|
+
@scope = scope
|
|
35
|
+
@html_report = html_report
|
|
36
|
+
@baseline = baseline
|
|
37
|
+
@coverage = coverage
|
|
38
|
+
@mutant_phase = mutant_phase
|
|
39
|
+
@palette = Palette.for(out, mode: color, overrides: overrides)
|
|
25
40
|
@score_table = ScoreTable.new(@out, @palette)
|
|
26
41
|
@next_steps = NextSteps.new(@out, @palette, replay_command: replay_command)
|
|
42
|
+
# The CLI classifies the run: the closing verdict line names the
|
|
43
|
+
# exit and its reason; the threshold gives the score its colour.
|
|
44
|
+
@exit_code = exit_code
|
|
45
|
+
@threshold = threshold
|
|
46
|
+
# Last recorded score from the score history, for the headline delta.
|
|
47
|
+
@previous_score = previous_score
|
|
48
|
+
# The --baseline run gate, when the run was gated: its comparison
|
|
49
|
+
# replaces the canonical tally and owns the exit-1 verdict.
|
|
50
|
+
@baseline_gate = baseline_gate
|
|
27
51
|
end
|
|
28
52
|
|
|
29
53
|
def call(report)
|
|
54
|
+
return quiet_call(report) if @quiet
|
|
55
|
+
|
|
30
56
|
survivors(report)
|
|
31
57
|
errors(report)
|
|
32
|
-
skipped_subjects(report)
|
|
58
|
+
skipped_subjects(report) if @verbose
|
|
33
59
|
test_command_gaps(report)
|
|
34
60
|
outside_the_score(report)
|
|
35
61
|
inline_suppressions(report)
|
|
@@ -42,6 +68,28 @@ module Saboteur
|
|
|
42
68
|
|
|
43
69
|
private
|
|
44
70
|
|
|
71
|
+
def middot = @palette.glyph(:middot)
|
|
72
|
+
|
|
73
|
+
# One finding per statement, live survivors in ranked order (#1065).
|
|
74
|
+
def quiet_call(report)
|
|
75
|
+
groups = SurvivorRanking.ordered(report, report.unignored_survivors).group_by do |result|
|
|
76
|
+
[relative(result.subject.file, report.root), MutationText.line(result), result.subject.id]
|
|
77
|
+
end
|
|
78
|
+
@out.puts(groups.map { |(path, line, id), rs| quiet_survivor_line(path, line, id, rs) }) if groups.any?
|
|
79
|
+
untested_subjects(report).each do |subject|
|
|
80
|
+
@out.puts "#{relative(subject.file, report.root)}:#{subject.start_line} #{subject.id} [untested]"
|
|
81
|
+
end
|
|
82
|
+
@baseline_gate ? BaselineTail.new(@out, @palette).call(@baseline_gate, @exit_code) : canonical_line(report)
|
|
83
|
+
verdict_line(report)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def quiet_survivor_line(path, line, subject_id, results)
|
|
87
|
+
first, *rest = results
|
|
88
|
+
label = SurvivorClassification.label(first)
|
|
89
|
+
text = "#{path}:#{line} #{subject_id}#{" [#{label}]" if label} - #{first.digest}"
|
|
90
|
+
rest.any? ? "#{text} (+#{rest.size} more)" : text
|
|
91
|
+
end
|
|
92
|
+
|
|
45
93
|
def operator_policy(report)
|
|
46
94
|
policy = report.operator_policy
|
|
47
95
|
return unless policy && policy["excluded"].any?
|
|
@@ -99,32 +147,76 @@ module Saboteur
|
|
|
99
147
|
end
|
|
100
148
|
|
|
101
149
|
def summary(report)
|
|
150
|
+
# A gated run's clean screen is PR8's baseline tail, not this one.
|
|
151
|
+
return clean_screen(report) if !@baseline_gate && clean_run?(report)
|
|
152
|
+
|
|
102
153
|
seed = report.seed && ", seed #{report.seed} (repeat with --seed #{report.seed})"
|
|
103
|
-
|
|
104
|
-
|
|
154
|
+
tally = "saboteur #{VERSION}: #{report.subjects.size} subjects, #{mutant_counts(report)}, " \
|
|
155
|
+
"wall #{format('%.2f', report.wall_time)} s#{seed}"
|
|
156
|
+
@out.puts @palette.paint(:dim, tally)
|
|
105
157
|
@out.puts report.shard.line if report.shard
|
|
106
|
-
|
|
158
|
+
score_headline(report)
|
|
107
159
|
details(report)
|
|
108
160
|
@score_table.call(report)
|
|
109
|
-
|
|
161
|
+
not_scored(report)
|
|
162
|
+
@baseline_gate ? BaselineTail.new(@out, @palette).call(@baseline_gate, @exit_code) : canonical_line(report)
|
|
163
|
+
verdict_line(report)
|
|
110
164
|
@next_steps.call(report)
|
|
111
165
|
end
|
|
112
166
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
167
|
+
# The deliberate success screen (PR6): a fully-killed run says the win
|
|
168
|
+
# out loud. Replaces the whole closing block on a clean run.
|
|
169
|
+
def clean_run?(report)
|
|
170
|
+
counts = report.counts
|
|
171
|
+
@exit_code&.zero? && report.executed.any? && report.unignored_survivors.empty? && !report.early_stop &&
|
|
172
|
+
(counts[:error] + counts[:timeout] + counts[:unverified] + counts[:resource_limit]).zero?
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def clean_screen(report)
|
|
176
|
+
header = "saboteur #{VERSION} #{middot} #{@scope}"
|
|
177
|
+
header += " #{middot} seed #{report.seed}" if report.seed
|
|
178
|
+
@out.puts @palette.paint(:dim, header)
|
|
179
|
+
clean_phases(report)
|
|
180
|
+
@out.puts
|
|
181
|
+
win = "#{@palette.glyph(:check)} #{win_tally(report)} #{middot} " \
|
|
182
|
+
"score #{format('%.1f%%', report.score * 100)}"
|
|
183
|
+
total = @palette.paint(:dim, "#{middot} #{format('%.1f', report.wall_time)}s total")
|
|
184
|
+
@out.puts "#{@palette.paint(:pass, win)} #{total}"
|
|
185
|
+
untested = untested_subjects(report)
|
|
186
|
+
@out.puts " #{@palette.paint(:warn, untested_win_note(untested))}" if untested.any?
|
|
187
|
+
clean_reports_line
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
# The phase lines and the progress-dots line print only when their data
|
|
191
|
+
# is present: PR4a plumbs baseline/coverage/phase stats, PR5 owns the
|
|
192
|
+
# live dots this echoes.
|
|
193
|
+
def clean_phases(report)
|
|
194
|
+
return unless @baseline && @coverage && @mutant_phase
|
|
195
|
+
|
|
196
|
+
check = @palette.paint(:pass, @palette.glyph(:check))
|
|
197
|
+
@out.puts "#{check} baseline green (#{@baseline[:examples]} examples, #{@baseline[:seconds]}s) " \
|
|
198
|
+
"#{check} coverage map: #{@coverage[:tests]} tests (#{@coverage[:seconds]}s)"
|
|
199
|
+
total = report.executed.size
|
|
200
|
+
dots = "." * total
|
|
201
|
+
suffix = @palette.paint(:dim, "(#{total}/#{total} #{middot} #{@mutant_phase[:seconds]}s)")
|
|
202
|
+
@out.puts "#{dots} #{suffix}"
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def win_tally(report)
|
|
206
|
+
"#{report.counts[:killed]}/#{report.executed.size} mutants killed"
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def untested_win_note(untested)
|
|
210
|
+
ids = untested.map(&:id).join(", ")
|
|
211
|
+
"#{untested.size} subject#{'s' unless untested.one?} not scored - no tests exist: #{ids}"
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
def clean_reports_line
|
|
215
|
+
return unless @html_report
|
|
216
|
+
|
|
217
|
+
line = "reports: #{@html_report}"
|
|
218
|
+
line += " #{middot} next: saboteur run --since origin/main" if @scope == "whole project"
|
|
219
|
+
@out.puts " #{@palette.paint(:dim, line)}"
|
|
128
220
|
end
|
|
129
221
|
|
|
130
222
|
def hazard_suppressions(report)
|
|
@@ -134,22 +226,190 @@ module Saboteur
|
|
|
134
226
|
"(--allow-hazardous runs them anyway; the canary still watches the tree)"
|
|
135
227
|
end
|
|
136
228
|
|
|
137
|
-
|
|
229
|
+
# Subjects skipped for "no coverage" (CandidateSelection's reason).
|
|
230
|
+
def untested_subjects(report)
|
|
231
|
+
report.skipped_subjects.select { |_, reason| reason == ScoreTable::NO_COVERAGE }
|
|
232
|
+
.keys.sort_by { |subject| [subject.file, subject.start_line] }
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# The score covers tested subjects only; the untested count stands
|
|
236
|
+
# next to it, never folded in.
|
|
237
|
+
def score_headline(report)
|
|
138
238
|
breakdown = report.score_breakdown
|
|
239
|
+
untested = untested_subjects(report)
|
|
139
240
|
# The share covers scored mutants only, so an error-dominated run
|
|
140
|
-
# must not print it as a bare score: "
|
|
241
|
+
# must not print it as a bare score: "100.0 %" above "the run
|
|
141
242
|
# tested almost nothing" reads as a contradiction (#557).
|
|
142
243
|
if report.majority_errored?
|
|
143
|
-
@out.puts "
|
|
244
|
+
@out.puts "Score: n/a (#{breakdown.tested} of #{report.executed.size} mutants scored, " \
|
|
144
245
|
"#{percent(breakdown.value)} of scored)"
|
|
246
|
+
return
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
line = "Score: #{painted_score(breakdown.value)}#{headline_context(breakdown.value)}"
|
|
250
|
+
line << if breakdown.tested.zero?
|
|
251
|
+
" - nothing tested"
|
|
252
|
+
else
|
|
253
|
+
" (#{breakdown.detected}/#{breakdown.tested} killed, tested subjects)"
|
|
254
|
+
end
|
|
255
|
+
if @verbose && breakdown.tested.positive?
|
|
256
|
+
line << "#{score_split(breakdown)} (killed+timeout)/(killed+survived+timeout); " \
|
|
257
|
+
"errors, resource limits and skips excluded"
|
|
258
|
+
end
|
|
259
|
+
if untested.any?
|
|
260
|
+
line << " #{@palette.paint(:warn, "#{untested.size} subject#{'s' unless untested.one?} " \
|
|
261
|
+
'not scored - no tests exist')}"
|
|
262
|
+
end
|
|
263
|
+
@out.puts line
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
# What the score is judged against and where it came from: the
|
|
267
|
+
# threshold it must meet, and the last score the history recorded.
|
|
268
|
+
def headline_context(value)
|
|
269
|
+
text = +""
|
|
270
|
+
text << threshold_echo(value) if @threshold
|
|
271
|
+
text << @palette.paint(:dim, " #{middot} prev #{format('%.1f%%', @previous_score * 100)}") if @previous_score
|
|
272
|
+
text
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
# The headline echoes the threshold it is judged against; a score
|
|
276
|
+
# below it is a break, painted with the fail colour.
|
|
277
|
+
def threshold_echo(value)
|
|
278
|
+
broken = value && value * 100 < @threshold
|
|
279
|
+
text = " #{middot} threshold #{format('%g', @threshold)}#{' (break)' if broken}"
|
|
280
|
+
broken ? @palette.paint(:fail, text) : @palette.paint(:dim, text)
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
# No threshold, no colour. A single --threshold T is the band (T, T):
|
|
284
|
+
# red below, green at or above, no yellow state.
|
|
285
|
+
def painted_score(value)
|
|
286
|
+
text = value ? format("%.1f%%", value * 100) : "n/a"
|
|
287
|
+
return text unless @threshold && value
|
|
288
|
+
|
|
289
|
+
@palette.paint(band_color(value * 100, @threshold, @threshold), text)
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
# Stryker band: red below low, green at or above high, yellow inside.
|
|
293
|
+
# A single --threshold T passes (T, T), so yellow is unreachable there.
|
|
294
|
+
def band_color(score, low, high)
|
|
295
|
+
return :fail if score < low
|
|
296
|
+
return :pass if score >= high
|
|
297
|
+
|
|
298
|
+
:warn
|
|
299
|
+
end
|
|
300
|
+
|
|
301
|
+
# The gaps the score cannot see, named with their location.
|
|
302
|
+
def not_scored(report)
|
|
303
|
+
untested = untested_subjects(report)
|
|
304
|
+
return if untested.empty?
|
|
305
|
+
|
|
306
|
+
@out.puts
|
|
307
|
+
@out.puts @palette.paint(:warn, "Not scored - no tests exist (treat these as your biggest gaps):")
|
|
308
|
+
entries = untested.map { |subject| "#{subject.id} (#{subject.location(report.root)})" }
|
|
309
|
+
line = " #{@palette.paint(:warn, '!')} #{entries.first(MAX_NOT_SCORED).join(" #{middot} ")}"
|
|
310
|
+
line << " #{middot} ... #{entries.size - MAX_NOT_SCORED} more" if entries.size > MAX_NOT_SCORED
|
|
311
|
+
@out.puts line
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
# One greppable line, pytest-style.
|
|
315
|
+
def canonical_line(report)
|
|
316
|
+
parts = state_parts(report)
|
|
317
|
+
parts << "no mutants" if parts.empty?
|
|
318
|
+
line = "saboteur: #{parts.join(', ')} in #{duration(report.wall_time)}"
|
|
319
|
+
line += " - score #{painted_score(report.score)}"
|
|
320
|
+
line += " (threshold #{format('%g', @threshold)})" if @threshold
|
|
321
|
+
@out.puts line
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
# The states that say how the run went: nonzero counts only, except
|
|
325
|
+
# the untested count, which is always named when there is any.
|
|
326
|
+
def state_parts(report)
|
|
327
|
+
counts = report.counts
|
|
328
|
+
live = report.unignored_survivors.size
|
|
329
|
+
untested = untested_subjects(report).size
|
|
330
|
+
[counted(counts[:killed], "killed", plural: false, color: :pass),
|
|
331
|
+
counted(live, "survived", plural: false, color: :fail),
|
|
332
|
+
counted(report.survivors.size - live, "accepted", plural: false),
|
|
333
|
+
counted(counts[:timeout], "timeout", color: :warn), counted(counts[:resource_limit], "resource limit"),
|
|
334
|
+
error_part(report, counts[:error]), counted(counts[:unverified], "unverified", plural: false),
|
|
335
|
+
("#{untested} subject#{'s' unless untested == 1} untested" if untested.positive?)].compact
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
def counted(count, word, plural: true, color: nil)
|
|
339
|
+
return unless count.positive?
|
|
340
|
+
|
|
341
|
+
text = "#{count} #{word}#{'s' if plural && count != 1}"
|
|
342
|
+
color ? @palette.paint(color, text) : text
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
def error_part(report, errors)
|
|
346
|
+
return if errors.zero?
|
|
347
|
+
|
|
348
|
+
rejected = report.rejected_kills
|
|
349
|
+
note = " (#{rejected} rejected kill#{'s' if rejected > 1})" if rejected.positive?
|
|
350
|
+
"#{errors} error#{'s' unless errors == 1}#{note}"
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
def duration(seconds)
|
|
354
|
+
minutes, rest = seconds.round.divmod(60)
|
|
355
|
+
minutes.positive? ? "#{minutes}m#{rest.to_s.rjust(2, '0')}s" : "#{rest}s"
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
# Why the run exits the way it does, echoing the thresholds. Exit 0
|
|
359
|
+
# prints no verdict: PR6 owns the clean screen.
|
|
360
|
+
# The exit codes are the CLI's (CLI::EXIT_*): literals here, because
|
|
361
|
+
# the CLI requires this file, not the other way round.
|
|
362
|
+
def verdict_line(report)
|
|
363
|
+
return if @exit_code.nil? || @exit_code.zero?
|
|
364
|
+
return if @baseline_gate && @exit_code == 1 # BaselineTail renders the gate's verdict
|
|
365
|
+
|
|
366
|
+
case @exit_code
|
|
367
|
+
when 1 then @out.puts @palette.paint(:fail, failure_verdict(report))
|
|
368
|
+
when 4
|
|
369
|
+
@out.puts @palette.paint(:fail, "#{@palette.glyph(:cross)} exit 4 - --max-duration budget spent; " \
|
|
370
|
+
"the score covers only completed work")
|
|
145
371
|
else
|
|
146
|
-
@out.puts
|
|
147
|
-
"#{if @verbose
|
|
148
|
-
"#{score_split(breakdown)} (killed+timeout)/(killed+survived+timeout); " \
|
|
149
|
-
'errors, resource limits and skips excluded'
|
|
150
|
-
end}"
|
|
372
|
+
@out.puts @palette.paint(:fail, incomplete_verdict(report))
|
|
151
373
|
end
|
|
152
|
-
|
|
374
|
+
end
|
|
375
|
+
|
|
376
|
+
def failure_verdict(report)
|
|
377
|
+
reasons = []
|
|
378
|
+
score = report.score
|
|
379
|
+
if @threshold && score && score * 100 < @threshold
|
|
380
|
+
reasons << "score #{format('%.1f%%', score * 100)} below threshold #{format('%g', @threshold)}"
|
|
381
|
+
end
|
|
382
|
+
live = report.unignored_survivors.size
|
|
383
|
+
reasons << "#{live} survivor#{'s' unless live == 1}" if live.positive?
|
|
384
|
+
reasons << "gate limits exceeded" if reasons.empty?
|
|
385
|
+
"#{@palette.glyph(:cross)} exit 1 - #{reasons.join(', ')}"
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
def incomplete_verdict(report)
|
|
389
|
+
if report.majority_errored?
|
|
390
|
+
"#{@palette.glyph(:cross)} exit 3 - most executed mutants errored; the run tested almost nothing"
|
|
391
|
+
elsif unverifiable_change?(report)
|
|
392
|
+
count = report.subjects.size
|
|
393
|
+
reasons = report.skipped_subjects.values.uniq.join(", ")
|
|
394
|
+
"#{@palette.glyph(:cross)} exit 3 - #{count} change#{'s' unless count == 1} could not be " \
|
|
395
|
+
"verified (#{reasons})"
|
|
396
|
+
elsif report.executed.any? && report.executed.all? { |result| result.status == :unreachable }
|
|
397
|
+
"#{@palette.glyph(:cross)} exit 3 - the change could not be verified (all mutants unreachable)"
|
|
398
|
+
elsif report.early_stop && report.score_breakdown.tested.zero?
|
|
399
|
+
"#{@palette.glyph(:cross)} exit 3 - stopped before any mutant was tested; there is no score to judge"
|
|
400
|
+
else
|
|
401
|
+
"#{@palette.glyph(:cross)} exit 3 - run incomplete; the score is not a verified result"
|
|
402
|
+
end
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
# A scoped run whose every subject was skipped verified nothing (#574).
|
|
406
|
+
def unverifiable_change?(report)
|
|
407
|
+
report.review? && report.subjects.any? &&
|
|
408
|
+
report.subjects.all? { |subject| report.skipped_subjects.key?(subject) }
|
|
409
|
+
end
|
|
410
|
+
|
|
411
|
+
def details(report)
|
|
412
|
+
report.skip_reasons.each { |reason, count| @out.puts skip_reason_line(reason, count) } if @verbose
|
|
153
413
|
reused = report.results.count(&:reused?)
|
|
154
414
|
@out.puts "reused #{reused} of #{report.results.size} verdicts from earlier runs (--incremental)" if
|
|
155
415
|
reused.positive?
|
|
@@ -164,7 +424,7 @@ module Saboteur
|
|
|
164
424
|
mean = report.mean_runtime
|
|
165
425
|
@out.puts "mean time per executed mutant #{format('%.3f', mean)} s" if mean
|
|
166
426
|
@out.puts "kills rejected by unchanged-method check: #{report.rejected_kills}"
|
|
167
|
-
|
|
427
|
+
KillCauses.new(@out).print(report)
|
|
168
428
|
end
|
|
169
429
|
|
|
170
430
|
# " (assertion 50.0 %, crash 50.0 %)": how much of the score is a failed
|
|
@@ -192,33 +452,6 @@ module Saboteur
|
|
|
192
452
|
share ? format("%.1f %%", share * 100) : "n/a"
|
|
193
453
|
end
|
|
194
454
|
|
|
195
|
-
# How many kills came from a failed check and how many from a crash, and
|
|
196
|
-
# which operators mostly kill by crashing (their score says less).
|
|
197
|
-
def kill_causes(report)
|
|
198
|
-
killed = report.results.select { |result| result.status == :killed && result.kill_cause }
|
|
199
|
-
return if killed.empty?
|
|
200
|
-
|
|
201
|
-
crashes = killed.select(&:crash_kill?)
|
|
202
|
-
@out.puts "kills: #{killed.size - crashes.size} by a failed assertion, #{crashes.size} by a crash" \
|
|
203
|
-
"#{crash_classes(crashes)}"
|
|
204
|
-
@out.puts "crash kills by operator: #{crashes_by_operator(killed)}" if crashes.any?
|
|
205
|
-
end
|
|
206
|
-
|
|
207
|
-
def crash_classes(crashes)
|
|
208
|
-
return "" if crashes.empty?
|
|
209
|
-
|
|
210
|
-
counts = crashes.map(&:kill_cause).tally.sort_by { |_, count| -count }
|
|
211
|
-
" (#{counts.map { |name, count| "#{name} #{count}" }.join(', ')})"
|
|
212
|
-
end
|
|
213
|
-
|
|
214
|
-
def crashes_by_operator(killed)
|
|
215
|
-
parts = killed.group_by { |result| result.mutation.operator }.filter_map do |operator, results|
|
|
216
|
-
count = results.count(&:crash_kill?)
|
|
217
|
-
"#{operator} #{count} of #{results.size}" if count.positive?
|
|
218
|
-
end
|
|
219
|
-
parts.join(", ")
|
|
220
|
-
end
|
|
221
|
-
|
|
222
455
|
# #641: a --test-command run has no coverage map, so a subject the
|
|
223
456
|
# command never loads reads as ordinary survivors. When a subject's
|
|
224
457
|
# mutants all came back without a single kill or error and at least
|
|
@@ -237,11 +470,24 @@ module Saboteur
|
|
|
237
470
|
end
|
|
238
471
|
end
|
|
239
472
|
|
|
473
|
+
# "no coverage" names two different things (#1070): subjects no test
|
|
474
|
+
# runs at all, and mutant locations inside covered methods that no
|
|
475
|
+
# test reaches. Say which is which; every other reason prints as-is.
|
|
476
|
+
def skip_reason_line(reason, count)
|
|
477
|
+
reason == "no coverage" ? "mutant locations not reached: #{count}" : "skipped: #{reason}: #{count}"
|
|
478
|
+
end
|
|
479
|
+
|
|
480
|
+
def skipped_subject_line(reason, count)
|
|
481
|
+
reason == "no coverage" ? "methods never run by tests: #{count}" : "skipped subjects: #{reason}: #{count}"
|
|
482
|
+
end
|
|
483
|
+
|
|
484
|
+
# Verbose-only: the Not scored block names the no-coverage subjects;
|
|
485
|
+
# other skips are announced live in a scoped run (#574).
|
|
240
486
|
def skipped_subjects(report)
|
|
241
|
-
report.skipped_subjects.values.tally.each
|
|
242
|
-
@out.puts "skipped subjects: #{reason}: #{count}"
|
|
243
|
-
end
|
|
487
|
+
report.skipped_subjects.values.tally.each { |reason, count| @out.puts skipped_subject_line(reason, count) }
|
|
244
488
|
report.skipped_subjects.each do |subject, reason|
|
|
489
|
+
next if reason == ScoreTable::NO_COVERAGE
|
|
490
|
+
|
|
245
491
|
@out.puts " #{subject.id} #{relative(subject.file, report.root)}:#{subject.start_line}: #{reason}"
|
|
246
492
|
end
|
|
247
493
|
end
|
|
@@ -271,8 +517,8 @@ module Saboteur
|
|
|
271
517
|
return unless stop
|
|
272
518
|
|
|
273
519
|
label = stop.reason == "fail_fast" ? "--fail-fast" : "--max-duration spent"
|
|
274
|
-
@out.puts @palette.paint(:
|
|
275
|
-
|
|
520
|
+
@out.puts @palette.paint(:warn, "stopped early (#{label}): #{early_stop_summary(stop)}; the score below " \
|
|
521
|
+
"covers only completed work")
|
|
276
522
|
budget_shortfall(stop, report.results.size)
|
|
277
523
|
list(stop.unreached) { |id| " not run: #{id}" }
|
|
278
524
|
list(stop.truncated) { |entry| " stopped mid-way: #{entry[:subject]} (#{entry[:omitted]} mutants omitted)" }
|
|
@@ -290,15 +536,15 @@ module Saboteur
|
|
|
290
536
|
# Exit 4 alone does not say whether anything was judged: headline
|
|
291
537
|
# it when the baselines and coverage map ate the whole budget (#637).
|
|
292
538
|
if completed.zero?
|
|
293
|
-
@out.puts @palette.paint(:
|
|
294
|
-
|
|
539
|
+
@out.puts @palette.paint(:warn, "no mutants ran before the --max-duration budget was spent; " \
|
|
540
|
+
"there is no score to judge")
|
|
295
541
|
end
|
|
296
542
|
line = if stop.planned_mutants
|
|
297
543
|
"partial: #{stop.planned_mutants - completed} of #{stop.planned_mutants} mutants not run"
|
|
298
544
|
else
|
|
299
545
|
"partial: mutant count unknown; the budget was spent before planning finished"
|
|
300
546
|
end
|
|
301
|
-
@out.puts @palette.paint(:
|
|
547
|
+
@out.puts @palette.paint(:warn, line)
|
|
302
548
|
end
|
|
303
549
|
|
|
304
550
|
def early_stop_summary(stop)
|
|
@@ -350,7 +596,7 @@ module Saboteur
|
|
|
350
596
|
end
|
|
351
597
|
|
|
352
598
|
def survivors(report)
|
|
353
|
-
SurvivorList.new(@out, @palette).call(report, ranked_survivors(report))
|
|
599
|
+
SurvivorList.new(@out, @palette, all_mutants: @all_mutants).call(report, ranked_survivors(report))
|
|
354
600
|
@out.puts unless report.survivors.empty?
|
|
355
601
|
end
|
|
356
602
|
|
data/lib/saboteur/result.rb
CHANGED
|
@@ -29,8 +29,15 @@ module Saboteur
|
|
|
29
29
|
# todo survivor keeps counting as an unignored survivor.
|
|
30
30
|
attr_accessor :triage_state, :triage_reason
|
|
31
31
|
|
|
32
|
+
# Post-run survivor confirmation (#1063): "confirmed" once the survivor's
|
|
33
|
+
# unrun covering specs had their budgeted say, "unconfirmed(reason)" when
|
|
34
|
+
# they could not; nil when confirmation never applied. Additive - the
|
|
35
|
+
# status keeps its meaning either way.
|
|
36
|
+
attr_reader :confirmation
|
|
37
|
+
|
|
32
38
|
def initialize(subject:, mutation:, status:, mutated_source:, reason: nil, runtime: nil, killed_by: [],
|
|
33
|
-
covered_by: [], kill_cause: nil, reused: false, rejected_kill: false, executed: nil
|
|
39
|
+
covered_by: [], kill_cause: nil, reused: false, rejected_kill: false, executed: nil,
|
|
40
|
+
confirmation: nil)
|
|
34
41
|
raise ArgumentError, "unknown status #{status.inspect}" unless STATUSES.include?(status)
|
|
35
42
|
|
|
36
43
|
@subject = subject
|
|
@@ -45,12 +52,26 @@ module Saboteur
|
|
|
45
52
|
@executed = executed
|
|
46
53
|
@rejected_kill = rejected_kill
|
|
47
54
|
@reused = reused
|
|
55
|
+
@confirmation = confirmation
|
|
48
56
|
@ignored = nil
|
|
49
57
|
# triage_state/triage_reason stay unset until the ignore list writes
|
|
50
58
|
# them: the accessors read as nil before that, and there is no
|
|
51
59
|
# statement for a mutation to remove (#304 dogfood).
|
|
52
60
|
end
|
|
53
61
|
|
|
62
|
+
# The same mutant with its survivor confirmation recorded (#1063); a
|
|
63
|
+
# confirmation kill flips the status and carries the confirming tests.
|
|
64
|
+
# The copy is built without @ignored/triage_state, so it must run before
|
|
65
|
+
# ignore marking (IgnoreList#apply) writes those.
|
|
66
|
+
def with_confirmation(confirmation, status: @status, reason: @reason, runtime: @runtime,
|
|
67
|
+
killed_by: @killed_by, covered_by: @covered_by, kill_cause: @kill_cause,
|
|
68
|
+
executed: @executed)
|
|
69
|
+
MutantResult.new(subject: subject, mutation: mutation, status: status, mutated_source: mutated_source,
|
|
70
|
+
reason: reason, runtime: runtime, killed_by: killed_by, covered_by: covered_by,
|
|
71
|
+
kill_cause: kill_cause, reused: reused?, rejected_kill: rejected_kill?,
|
|
72
|
+
executed: executed, confirmation: confirmation)
|
|
73
|
+
end
|
|
74
|
+
|
|
54
75
|
# The same mutant, marked unverified: the budget ran out before its
|
|
55
76
|
# verdict could be checked, so it must not count as detected (#296).
|
|
56
77
|
def as_unverified(reason)
|
|
@@ -187,6 +208,12 @@ module Saboteur
|
|
|
187
208
|
@matrix
|
|
188
209
|
end
|
|
189
210
|
|
|
211
|
+
# True when the run was scoped to a diff (--since): the expand command
|
|
212
|
+
# echoes that scope.
|
|
213
|
+
def diff?
|
|
214
|
+
!@diff_scope.nil?
|
|
215
|
+
end
|
|
216
|
+
|
|
190
217
|
# In diff mode, the changed lines (sorted) that a mutant touches; empty
|
|
191
218
|
# outside diff mode, or when the mutant touches none (a whole-body mutant
|
|
192
219
|
# of a method whose only change is its signature).
|
data/lib/saboteur/score.rb
CHANGED
|
@@ -72,6 +72,12 @@ module Saboteur
|
|
|
72
72
|
list
|
|
73
73
|
end
|
|
74
74
|
|
|
75
|
+
# The kill split the reports print: assertion, crash and unclassified
|
|
76
|
+
# counts, with the to_h key names.
|
|
77
|
+
def kill_causes_h
|
|
78
|
+
to_h.slice(:assertionKills, :crashKills, :unclassifiedKills)
|
|
79
|
+
end
|
|
80
|
+
|
|
75
81
|
def to_h
|
|
76
82
|
{
|
|
77
83
|
score: value, tested: tested, assertionKills: assertion_kills, crashKills: crash_kills,
|
data/lib/saboteur/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: saboteur
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.11.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Vamsi Pavan Mahesh Gunturu
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-10-
|
|
11
|
+
date: 2026-10-21 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: prism
|
|
@@ -54,12 +54,15 @@ files:
|
|
|
54
54
|
- lib/saboteur/baseline/completeness.rb
|
|
55
55
|
- lib/saboteur/baseline/evidence.rb
|
|
56
56
|
- lib/saboteur/baseline/identity.rb
|
|
57
|
+
- lib/saboteur/baseline/loader.rb
|
|
57
58
|
- lib/saboteur/baseline/manifest.rb
|
|
58
59
|
- lib/saboteur/baseline/run_evidence.rb
|
|
60
|
+
- lib/saboteur/baseline/run_gate.rb
|
|
59
61
|
- lib/saboteur/baseline/workflow.rb
|
|
60
62
|
- lib/saboteur/baseline_health.rb
|
|
61
63
|
- lib/saboteur/cli.rb
|
|
62
64
|
- lib/saboteur/cli/agent_provenance.rb
|
|
65
|
+
- lib/saboteur/cli/baseline_gate.rb
|
|
63
66
|
- lib/saboteur/cli/dirty_tree_notice.rb
|
|
64
67
|
- lib/saboteur/cli/doctor_command.rb
|
|
65
68
|
- lib/saboteur/cli/error_cause.rb
|
|
@@ -136,6 +139,7 @@ files:
|
|
|
136
139
|
- lib/saboteur/engine/subject_selection.rb
|
|
137
140
|
- lib/saboteur/engine/suite_files.rb
|
|
138
141
|
- lib/saboteur/engine/suite_partition.rb
|
|
142
|
+
- lib/saboteur/engine/survivor_confirmation.rb
|
|
139
143
|
- lib/saboteur/engine/test_guess.rb
|
|
140
144
|
- lib/saboteur/engine/thread_warnings.rb
|
|
141
145
|
- lib/saboteur/engine/unchanged_syntax.rb
|
|
@@ -183,9 +187,11 @@ files:
|
|
|
183
187
|
- lib/saboteur/mutation_source/chain_removal.rb
|
|
184
188
|
- lib/saboteur/mutation_source/condition_negation.rb
|
|
185
189
|
- lib/saboteur/mutation_source/constant_literals.rb
|
|
190
|
+
- lib/saboteur/mutation_source/copy_replacement.rb
|
|
186
191
|
- lib/saboteur/mutation_source/edit.rb
|
|
187
192
|
- lib/saboteur/mutation_source/element_removal.rb
|
|
188
193
|
- lib/saboteur/mutation_source/endpoint_replacement.rb
|
|
194
|
+
- lib/saboteur/mutation_source/ensure_removal.rb
|
|
189
195
|
- lib/saboteur/mutation_source/entry_edits.rb
|
|
190
196
|
- lib/saboteur/mutation_source/filter_replacement.rb
|
|
191
197
|
- lib/saboteur/mutation_source/hazard_guard.rb
|
|
@@ -213,6 +219,9 @@ files:
|
|
|
213
219
|
- lib/saboteur/reporter/agent_commands.rb
|
|
214
220
|
- lib/saboteur/reporter/agent_incomplete.rb
|
|
215
221
|
- lib/saboteur/reporter/agent_summary.rb
|
|
222
|
+
- lib/saboteur/reporter/assets/mutation_report.LICENSE
|
|
223
|
+
- lib/saboteur/reporter/assets/mutation_report.js
|
|
224
|
+
- lib/saboteur/reporter/baseline_tail.rb
|
|
216
225
|
- lib/saboteur/reporter/diff.rb
|
|
217
226
|
- lib/saboteur/reporter/findings.rb
|
|
218
227
|
- lib/saboteur/reporter/history_paths.rb
|
|
@@ -220,6 +229,7 @@ files:
|
|
|
220
229
|
- lib/saboteur/reporter/history_reader.rb
|
|
221
230
|
- lib/saboteur/reporter/history_record.rb
|
|
222
231
|
- lib/saboteur/reporter/html.rb
|
|
232
|
+
- lib/saboteur/reporter/kill_causes.rb
|
|
223
233
|
- lib/saboteur/reporter/mutation_text.rb
|
|
224
234
|
- lib/saboteur/reporter/next_steps.rb
|
|
225
235
|
- lib/saboteur/reporter/palette.rb
|