saboteur 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +66 -0
- data/README.md +109 -247
- data/exe/saboteur +2 -2
- data/lib/saboteur/baseline/loader.rb +41 -0
- data/lib/saboteur/baseline/run_evidence.rb +1 -1
- data/lib/saboteur/baseline/run_gate.rb +51 -0
- data/lib/saboteur/baseline/workflow.rb +2 -15
- data/lib/saboteur/cli/baseline_gate.rb +58 -0
- data/lib/saboteur/cli/error_cause.rb +16 -0
- data/lib/saboteur/cli/exit_status.rb +14 -13
- data/lib/saboteur/cli/failure_output.rb +23 -0
- data/lib/saboteur/cli/output_options.rb +24 -2
- data/lib/saboteur/cli/run_budget_options.rb +6 -0
- data/lib/saboteur/cli/run_command.rb +14 -1
- data/lib/saboteur/cli/run_reporting.rb +34 -6
- data/lib/saboteur/cli/run_validation.rb +36 -3
- data/lib/saboteur/cli/selection_options.rb +2 -0
- data/lib/saboteur/cli.rb +6 -1
- data/lib/saboteur/config.rb +23 -2
- data/lib/saboteur/crash_report.rb +16 -7
- data/lib/saboteur/engine/baseline_guard.rb +2 -2
- data/lib/saboteur/engine/coverage_partial.rb +8 -1
- data/lib/saboteur/engine/coverage_store.rb +4 -5
- data/lib/saboteur/engine/diagnostics.rb +120 -15
- data/lib/saboteur/engine/run_fingerprint.rb +1 -1
- data/lib/saboteur/engine/scheduler.rb +20 -0
- data/lib/saboteur/engine/selection_announcement.rb +8 -1
- data/lib/saboteur/engine/survivor_confirmation.rb +159 -0
- data/lib/saboteur/engine.rb +43 -6
- data/lib/saboteur/help.rb +4 -1
- data/lib/saboteur/mutation_source/builtin.rb +9 -3
- data/lib/saboteur/mutation_source/copy_replacement.rb +72 -0
- data/lib/saboteur/mutation_source/ensure_removal.rb +49 -0
- data/lib/saboteur/mutation_source/operator_examples.rb +2 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +3 -1
- data/lib/saboteur/reporter/agent_summary.rb +3 -1
- data/lib/saboteur/reporter/assets/mutation_report.LICENSE +201 -0
- data/lib/saboteur/reporter/assets/mutation_report.js +331 -0
- data/lib/saboteur/reporter/baseline_tail.rb +93 -0
- data/lib/saboteur/reporter/diff.rb +10 -0
- data/lib/saboteur/reporter/html.rb +69 -14
- data/lib/saboteur/reporter/kill_causes.rb +53 -0
- data/lib/saboteur/reporter/next_steps.rb +4 -3
- data/lib/saboteur/reporter/palette.rb +64 -13
- data/lib/saboteur/reporter/safe_out.rb +1 -1
- data/lib/saboteur/reporter/score_table.rb +53 -18
- data/lib/saboteur/reporter/stryker.rb +4 -1
- data/lib/saboteur/reporter/survivor_classification.rb +12 -3
- data/lib/saboteur/reporter/survivor_hints.rb +72 -0
- data/lib/saboteur/reporter/survivor_list.rb +207 -75
- data/lib/saboteur/reporter/terminal.rb +315 -69
- data/lib/saboteur/result.rb +28 -1
- data/lib/saboteur/score.rb +6 -0
- data/lib/saboteur/version.rb +1 -1
- metadata +12 -2
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require "io/console"
|
|
4
5
|
require "tempfile"
|
|
5
6
|
|
|
7
|
+
require_relative "../reporter/palette"
|
|
8
|
+
|
|
6
9
|
module Saboteur
|
|
7
10
|
class Engine
|
|
8
11
|
# What a run has to say besides results: warnings for the report, the
|
|
@@ -66,19 +69,32 @@ module Saboteur
|
|
|
66
69
|
end
|
|
67
70
|
|
|
68
71
|
MARKS = { killed: ".", survived: "S", timeout: "T", resource_limit: "L", error: "E" }.freeze
|
|
69
|
-
|
|
70
|
-
|
|
72
|
+
MARK_TOKENS = { "." => :dim, "S" => :fail, "T" => :warn, "E" => :fail, "L" => :fail }.freeze
|
|
73
|
+
# Readback whitelist: mutant children inherit the progress fd and can
|
|
74
|
+
# write arbitrary bytes, so only known mark letters come back out.
|
|
75
|
+
READBACK_GLYPHS = (MARK_TOKENS.keys + ["?"]).freeze
|
|
76
|
+
# s:skipped never prints a glyph: skipped mutants never reach
|
|
77
|
+
# progress (see start_progress); the letter is legend-only.
|
|
78
|
+
LEGEND = ".:killed S:survived T:timeout E:error s:skipped L:resource limit"
|
|
79
|
+
|
|
80
|
+
# Chunk sizing: leave room for the " (n/total ...)" suffix, never
|
|
81
|
+
# below the floor.
|
|
82
|
+
CHUNK_MARGIN = 10
|
|
83
|
+
CHUNK_FLOOR = 40
|
|
84
|
+
DEFAULT_WIDTH = 80
|
|
71
85
|
|
|
72
86
|
GenerationEvent = Data.define(:file, :subject, :source, :operator, :reason)
|
|
73
87
|
Omission = Data.define(:subject, :count, :limit)
|
|
74
88
|
DiscoveryFailure = Data.define(:file, :reason)
|
|
75
89
|
|
|
76
90
|
attr_reader :warnings, :live_announced, :invalid_mutations, :generation_events, :omissions, :discovery_failures,
|
|
77
|
-
:hazard_suppressions, :inline_suppressions, :verbose
|
|
91
|
+
:hazard_suppressions, :inline_suppressions, :verbose, :quiet_start
|
|
78
92
|
|
|
79
|
-
def initialize(err, verbose: false)
|
|
93
|
+
def initialize(err, verbose: false, palette: nil, quiet_start: true)
|
|
80
94
|
@err = err
|
|
81
95
|
@verbose = verbose
|
|
96
|
+
@quiet_start = quiet_start
|
|
97
|
+
@palette = palette || Reporter::Palette.for(err)
|
|
82
98
|
@warnings = []
|
|
83
99
|
@invalid_mutations = 0
|
|
84
100
|
@generation_events = []
|
|
@@ -163,6 +179,51 @@ module Saboteur
|
|
|
163
179
|
@err.puts("saboteur: #{Text.display_safe_lines(message)}")
|
|
164
180
|
end
|
|
165
181
|
|
|
182
|
+
# Chatter the default view folds into the quiet-start lines;
|
|
183
|
+
# --verbose prints it byte-for-byte as before.
|
|
184
|
+
def detail(message)
|
|
185
|
+
inform(message) if @verbose || !@quiet_start
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# A selection explanation the default view carries as one dim
|
|
189
|
+
# summary line: the full text stays in report.warnings for machine
|
|
190
|
+
# consumers and counts as announced, so it is not reprinted at the
|
|
191
|
+
# end of the run.
|
|
192
|
+
def announce_demoted(message)
|
|
193
|
+
safe = Text.display_safe_lines(message)
|
|
194
|
+
warn(safe)
|
|
195
|
+
@live_announced << safe
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
# The quiet start: three lines, then progress. --verbose keeps
|
|
199
|
+
# today's chatter instead and never prints these.
|
|
200
|
+
def quiet_start_header(scope:, seed:, operators:)
|
|
201
|
+
return if @verbose || !@quiet_start
|
|
202
|
+
|
|
203
|
+
dot = @palette.glyph(:middot)
|
|
204
|
+
line = "saboteur #{Saboteur::VERSION} #{dot} #{scope} #{dot} seed #{seed} #{dot} " \
|
|
205
|
+
"#{operators} operators (--list-operators)"
|
|
206
|
+
@err.puts(@palette.paint(:dim, line))
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def quiet_baseline(passes:, elapsed:, tests:, cached:)
|
|
210
|
+
return if @verbose || !@quiet_start
|
|
211
|
+
|
|
212
|
+
check = @palette.paint(:pass, @palette.glyph(:check))
|
|
213
|
+
suffix = cached ? " (cached)" : ""
|
|
214
|
+
line = "#{check} baseline green (#{passes} #{passes == 1 ? 'pass' : 'passes'}, " \
|
|
215
|
+
"#{format('%.1f', elapsed)}s) #{check} coverage map: #{tests} " \
|
|
216
|
+
"#{tests == 1 ? 'test' : 'tests'}#{suffix}"
|
|
217
|
+
@err.puts(line)
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
def quiet_inference(spec_count)
|
|
221
|
+
return if spec_count.nil? || @verbose || !@quiet_start
|
|
222
|
+
|
|
223
|
+
files = spec_count == 1 ? "spec file" : "spec files"
|
|
224
|
+
@err.puts(@palette.paint(:dim, "spec inference: #{spec_count} #{files} (details: --verbose)"))
|
|
225
|
+
end
|
|
226
|
+
|
|
166
227
|
# --verbose: one line per finished subject with its result marks, so a
|
|
167
228
|
# survivor is attributable to a method as soon as the subject
|
|
168
229
|
# completes, before the whole run ends (#262).
|
|
@@ -170,7 +231,7 @@ module Saboteur
|
|
|
170
231
|
return unless @verbose
|
|
171
232
|
return if results.empty?
|
|
172
233
|
|
|
173
|
-
marks = results.map { |result| MARKS.fetch(result.status, "?") }.join
|
|
234
|
+
marks = results.map { |result| painted_mark(MARKS.fetch(result.status, "?")) }.join
|
|
174
235
|
inform("#{subject.id} #{marks}")
|
|
175
236
|
end
|
|
176
237
|
|
|
@@ -204,21 +265,36 @@ module Saboteur
|
|
|
204
265
|
@progress_file = Tempfile.new("saboteur-progress")
|
|
205
266
|
@progress_file.binmode
|
|
206
267
|
@progress_file.sync = true
|
|
207
|
-
@
|
|
268
|
+
@glyphs = +""
|
|
269
|
+
@err.puts(@palette.paint(:dim, LEGEND))
|
|
208
270
|
end
|
|
209
271
|
|
|
210
272
|
def progress(result)
|
|
211
273
|
mark = MARKS.fetch(result.status, "?")
|
|
212
|
-
return plain_progress(mark) unless @progress_file
|
|
274
|
+
return plain_progress(painted_mark(mark)) unless @progress_file
|
|
213
275
|
|
|
214
276
|
done = record_progress(mark)
|
|
215
|
-
progress_on_terminal?
|
|
277
|
+
if progress_on_terminal?
|
|
278
|
+
@glyphs << mark
|
|
279
|
+
@glyphs.size >= chunk_width ? commit_chunk(done) : redraw_line(done)
|
|
280
|
+
elsif (done % chunk_width).zero?
|
|
281
|
+
commit_file_chunk(done - chunk_width, chunk_width, done)
|
|
282
|
+
end
|
|
216
283
|
@err.flush
|
|
217
284
|
end
|
|
218
285
|
|
|
286
|
+
def painted_mark(mark)
|
|
287
|
+
token = MARK_TOKENS[mark]
|
|
288
|
+
token ? @palette.paint(token, mark) : mark
|
|
289
|
+
end
|
|
290
|
+
|
|
219
291
|
def finish_progress
|
|
220
|
-
|
|
221
|
-
|
|
292
|
+
return unless @progress_file
|
|
293
|
+
|
|
294
|
+
size = @progress_file.size
|
|
295
|
+
pending = size % chunk_width
|
|
296
|
+
commit_file_chunk(size - pending, pending, size) if pending.positive?
|
|
297
|
+
@progress_file.close!
|
|
222
298
|
@progress_file = nil
|
|
223
299
|
end
|
|
224
300
|
|
|
@@ -245,13 +321,42 @@ module Saboteur
|
|
|
245
321
|
@err.respond_to?(:tty?) && @err.tty?
|
|
246
322
|
end
|
|
247
323
|
|
|
248
|
-
def
|
|
249
|
-
|
|
324
|
+
def chunk_width
|
|
325
|
+
columns = begin
|
|
326
|
+
@err.winsize[1] if @err.respond_to?(:winsize)
|
|
327
|
+
rescue SystemCallError
|
|
328
|
+
nil
|
|
329
|
+
end
|
|
330
|
+
[(columns || DEFAULT_WIDTH) - CHUNK_MARGIN, CHUNK_FLOOR].max
|
|
250
331
|
end
|
|
251
332
|
|
|
252
|
-
def
|
|
253
|
-
@
|
|
254
|
-
|
|
333
|
+
def painted_glyphs
|
|
334
|
+
@glyphs.each_char.map { |glyph| painted_mark(glyph) }.join
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
def redraw_line(done)
|
|
338
|
+
@line_open = true
|
|
339
|
+
@err.print("\r\e[K#{painted_glyphs} #{@palette.paint(:dim, progress_summary(done))}")
|
|
340
|
+
end
|
|
341
|
+
|
|
342
|
+
def commit_chunk(done)
|
|
343
|
+
@err.print("\r\e[K#{painted_glyphs} #{@palette.paint(:dim, progress_summary(done))}\n")
|
|
344
|
+
@glyphs.clear
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
# Piped chunks and the final partial chunk come from the progress
|
|
348
|
+
# file, not the buffer: workers each buffer only their own marks,
|
|
349
|
+
# while the shared file the workers inherit holds every mark.
|
|
350
|
+
def commit_file_chunk(start, length, done)
|
|
351
|
+
@err.print("\r\e[K") if @line_open
|
|
352
|
+
@err.print("#{painted_file_glyphs(start, length)} #{@palette.paint(:dim, progress_summary(done))}\n")
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
# binread caps at both +length+ (the recorded mark count) and EOF,
|
|
356
|
+
# so a short or padded file cannot leak extra glyphs.
|
|
357
|
+
def painted_file_glyphs(start, length)
|
|
358
|
+
bytes = File.binread(@progress_file.path, length, start)
|
|
359
|
+
bytes.each_char.filter_map { |glyph| painted_mark(glyph) if READBACK_GLYPHS.include?(glyph) }.join
|
|
255
360
|
end
|
|
256
361
|
|
|
257
362
|
def eta_text(done)
|
|
@@ -14,7 +14,7 @@ module Saboteur
|
|
|
14
14
|
class RunFingerprint
|
|
15
15
|
FORMAT_VERSION = 1
|
|
16
16
|
MANIFESTS = %w[Gemfile gems.rb Gemfile.lock gems.locked .rspec .rspec-local].freeze
|
|
17
|
-
LOCATION_SETTINGS = Config::OUTPUT_KEYS
|
|
17
|
+
LOCATION_SETTINGS = Config::OUTPUT_KEYS + Config::PRESENTATION_KEYS
|
|
18
18
|
ENTRY_KINDS = %w[ruby manifest boot hook source_plugin extra data].freeze
|
|
19
19
|
EXCLUDED_DIRECTORIES = %w[.git vendor node_modules].freeze
|
|
20
20
|
Entry = Data.define(:path, :kind, :state, :sha256, :required)
|
|
@@ -232,6 +232,26 @@ module Saboteur
|
|
|
232
232
|
result
|
|
233
233
|
end
|
|
234
234
|
|
|
235
|
+
attr_reader :runner, :map
|
|
236
|
+
|
|
237
|
+
# The eval prefix a post-run confirmation run needs for this subject
|
|
238
|
+
# (#1063): the same wrapper detection load_problem applies, reduced to
|
|
239
|
+
# the sorbet sig prefix a survivor could have run with.
|
|
240
|
+
def confirmation_prefix(subject)
|
|
241
|
+
return nil unless @verify_loaded
|
|
242
|
+
return nil if @reevaluator.whole_file?(subject)
|
|
243
|
+
|
|
244
|
+
owner = @reevaluator.resolve_owner(subject)
|
|
245
|
+
target = subject.singleton ? owner.singleton_class : owner
|
|
246
|
+
method = own_method(target, subject.name)
|
|
247
|
+
return nil unless method
|
|
248
|
+
|
|
249
|
+
file, = method.source_location
|
|
250
|
+
Execution::Wrapper.around(method, file) == :sorbet ? subject.sig_source : nil
|
|
251
|
+
rescue Error
|
|
252
|
+
nil
|
|
253
|
+
end
|
|
254
|
+
|
|
235
255
|
# The tests that ran the edited code (a coarse edit spans the whole
|
|
236
256
|
# body, so it gets every test of the method), prior killers first, then
|
|
237
257
|
# fastest-first.
|
|
@@ -27,8 +27,15 @@ module Saboteur
|
|
|
27
27
|
end
|
|
28
28
|
|
|
29
29
|
def announce_selection(guess)
|
|
30
|
-
if guess.changed.zero?
|
|
30
|
+
if guess.changed.zero?
|
|
31
31
|
@diagnostics.announce(selection_message(guess))
|
|
32
|
+
elsif guess.specs.any?
|
|
33
|
+
@quiet_spec_count = guess.specs.size
|
|
34
|
+
if @diagnostics.verbose || !@diagnostics.quiet_start
|
|
35
|
+
@diagnostics.announce(selection_message(guess))
|
|
36
|
+
else
|
|
37
|
+
@diagnostics.announce_demoted(selection_message(guess))
|
|
38
|
+
end
|
|
32
39
|
else
|
|
33
40
|
@diagnostics.inform(selection_message(guess))
|
|
34
41
|
end
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# Post-run survivor confirmation (#1063): with partial_coverage=use the
|
|
7
|
+
# coverage map can be a partial rebuild, so every open survivor's
|
|
8
|
+
# covering specs the mutant never ran get one budgeted confirmation run
|
|
9
|
+
# through the same Runner invocation the scheduler makes. A failing run
|
|
10
|
+
# kills the mutant; a passing run marks the result confirmed; anything
|
|
11
|
+
# that could not run is marked unconfirmed with the reason. The field is
|
|
12
|
+
# additive - the status keeps its meaning - and no outcome is silent.
|
|
13
|
+
class SurvivorConfirmation
|
|
14
|
+
# The most covering specs one survivor re-runs; the config's
|
|
15
|
+
# confirmation_budget caps the whole pass.
|
|
16
|
+
MAX_SPECS_PER_SURVIVOR = 8
|
|
17
|
+
|
|
18
|
+
def initialize(config:, map:, scheduler:, diagnostics:, budget_exceeded:, cache: nil)
|
|
19
|
+
@config = config
|
|
20
|
+
@map = map
|
|
21
|
+
@scheduler = scheduler
|
|
22
|
+
@diagnostics = diagnostics
|
|
23
|
+
@budget_exceeded = budget_exceeded
|
|
24
|
+
@cache = cache
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def call(results, ignore_list:, early_stop: nil)
|
|
28
|
+
survivors = results.select { |result| result.survived? && !ignore_list.accepts?(result) }
|
|
29
|
+
return results if survivors.empty?
|
|
30
|
+
|
|
31
|
+
if early_stop
|
|
32
|
+
@diagnostics.warn("survivor confirmation covers only the survivors reached before the run " \
|
|
33
|
+
"stopped early (#{early_stop.reason})")
|
|
34
|
+
end
|
|
35
|
+
@remaining = @config["confirmation_budget"]
|
|
36
|
+
results.map { |result| survivors.include?(result) ? confirm(result) : result }
|
|
37
|
+
.tap { |confirmed| summarize(confirmed) }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def confirm(result)
|
|
43
|
+
return unconfirmed(result, "the confirmation budget is spent") if @remaining <= 0
|
|
44
|
+
return unconfirmed(result, "the --max-duration budget is spent") if @budget_exceeded.call
|
|
45
|
+
|
|
46
|
+
tests = unrun_covering_tests(result)
|
|
47
|
+
return result if tests.empty?
|
|
48
|
+
|
|
49
|
+
outcome = run_confirmation(result, tests)
|
|
50
|
+
@remaining -= outcome.runtime || 0
|
|
51
|
+
case outcome.status
|
|
52
|
+
when :killed then checked_kill(result, tests, outcome)
|
|
53
|
+
when :survived then result.with_confirmation("confirmed")
|
|
54
|
+
else unconfirmed(result, "the confirmation run #{outcome.status}: #{outcome.message}")
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# The whole-suite map's covering tests for the mutant's edited lines
|
|
59
|
+
# (branch evidence intersected, mirroring the scheduler), minus the
|
|
60
|
+
# tests the mutant already ran, fastest first, capped.
|
|
61
|
+
def unrun_covering_tests(result)
|
|
62
|
+
fastest_first(covering_tests(result) - (result.executed || result.covered_by)).first(MAX_SPECS_PER_SURVIVOR)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def covering_tests(result)
|
|
66
|
+
line_numbers = result.mutation.edits.flat_map { |edit| result.subject.coverage_lines_for(edit) }.uniq
|
|
67
|
+
tests = @map.tests_for_edit(result.subject, line_numbers)
|
|
68
|
+
branch_tests = @map.tests_for_branches(result.subject, result.mutation.edits)
|
|
69
|
+
branch_tests ? tests & branch_tests : tests
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def fastest_first(tests)
|
|
73
|
+
tests.sort_by { |id| @map.recorded_runtime([id]) || Float::INFINITY }
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def run_confirmation(result, tests)
|
|
77
|
+
@scheduler.runner.run(subject: result.subject, text: result.mutated_source, test_ids: tests,
|
|
78
|
+
timeout: timeout_for(tests, result), mutation: result.mutation,
|
|
79
|
+
eval_prefix: @scheduler.confirmation_prefix(result.subject))
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# A confirmation kill stands only after the same VerdictCheck the
|
|
83
|
+
# scheduler runs: the kill sequence, in order, must pass with the
|
|
84
|
+
# method unchanged, so an order-dependent spec cannot convert a true
|
|
85
|
+
# survivor into a credited kill (security review). A rejected kill
|
|
86
|
+
# becomes an error; a check the budgets never let run comes back
|
|
87
|
+
# unverified - outside the score and never cached.
|
|
88
|
+
def checked_kill(result, tests, outcome)
|
|
89
|
+
kill = result.with_confirmation("confirmed", status: :killed, reason: outcome.message,
|
|
90
|
+
killed_by: outcome.failed_ids, covered_by: tests,
|
|
91
|
+
kill_cause: outcome.kill_cause,
|
|
92
|
+
runtime: (result.runtime || 0) + (outcome.runtime || 0),
|
|
93
|
+
executed: (result.executed || []) + outcome.executed_ids)
|
|
94
|
+
checked = VerdictCheck.new(run_unchanged: unchanged_runner(result.subject),
|
|
95
|
+
baseline_timeout: @config["baseline_timeout"]).call([kill]).first
|
|
96
|
+
return checked unless checked.status == :killed
|
|
97
|
+
|
|
98
|
+
@diagnostics.inform("#{result.subject.id} #{result.mutation.operator} #{result.digest}: " \
|
|
99
|
+
"killed on confirmation by #{outcome.failed_ids.first}")
|
|
100
|
+
record_kill(result, kill)
|
|
101
|
+
checked
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# The standing kill goes into the verdict cache with the mutant's full
|
|
105
|
+
# covering test set, not the capped confirmation subset: the cache
|
|
106
|
+
# fingerprint digests that set, so only the full set overwrites the
|
|
107
|
+
# survivor verdict the run cached and lets the next run reuse the kill
|
|
108
|
+
# (#1079). ResultCache#record replaces the subject+mutation entry, so
|
|
109
|
+
# the stale survivor verdict cannot win a later lookup.
|
|
110
|
+
def record_kill(result, kill)
|
|
111
|
+
@cache&.record(kill.with_confirmation("confirmed", covered_by: covering_tests(result)))
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# The unchanged-method re-run VerdictCheck checks a kill against, in
|
|
115
|
+
# the unchanged_runner shape (scheduler.rb), against the confirmation
|
|
116
|
+
# and --max-duration budgets instead of the scheduler's plan state.
|
|
117
|
+
def unchanged_runner(subject)
|
|
118
|
+
lambda do |tests, timeout|
|
|
119
|
+
if @remaining <= 0 || @budget_exceeded.call
|
|
120
|
+
Execution::Runner::Outcome.new(status: :unverified, runtime: 0.0,
|
|
121
|
+
message: "the confirmation budget is spent")
|
|
122
|
+
else
|
|
123
|
+
outcome = @scheduler.runner.run(subject: subject, text: subject.source, test_ids: tests,
|
|
124
|
+
timeout: timeout,
|
|
125
|
+
eval_prefix: @scheduler.confirmation_prefix(subject))
|
|
126
|
+
@remaining -= outcome.runtime || 0
|
|
127
|
+
outcome
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def unconfirmed(result, reason)
|
|
133
|
+
result.with_confirmation("unconfirmed(#{reason})")
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
# The scheduler's timeout_for shape with the post-run slowdown of 1.0.
|
|
137
|
+
# recorded_runtime is nil without arm durations: fall back to the
|
|
138
|
+
# mutant's own first-run runtime, then to the suite's mean.
|
|
139
|
+
def timeout_for(tests, result)
|
|
140
|
+
recorded = @map.recorded_runtime(tests) || result.runtime || suite_mean_runtime
|
|
141
|
+
return @config["baseline_timeout"] unless recorded
|
|
142
|
+
|
|
143
|
+
[@config["timeout_factor"] * recorded, @config["timeout_min"]].max
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def suite_mean_runtime
|
|
147
|
+
durations = @map.tests.map { |id| @map.recorded_runtime([id]) }.compact
|
|
148
|
+
durations.empty? ? nil : durations.sum / durations.size
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def summarize(results)
|
|
152
|
+
counts = results.filter_map(&:confirmation).tally
|
|
153
|
+
return if counts.empty?
|
|
154
|
+
|
|
155
|
+
@diagnostics.inform("survivor confirmation: #{counts.map { |mark, n| "#{n} #{mark}" }.join(', ')}")
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
end
|
data/lib/saboteur/engine.rb
CHANGED
|
@@ -12,6 +12,7 @@ require_relative "engine/load_coverage"
|
|
|
12
12
|
require_relative "engine/out_of_scope_guard"
|
|
13
13
|
require_relative "engine/run_budget"
|
|
14
14
|
require_relative "engine/selection_announcement"
|
|
15
|
+
require_relative "engine/survivor_confirmation"
|
|
15
16
|
require_relative "engine/thread_warnings"
|
|
16
17
|
|
|
17
18
|
require_relative "engine/pipelines"
|
|
@@ -38,10 +39,13 @@ module Saboteur
|
|
|
38
39
|
# The specs inference narrowed the run to, or nil when it did not.
|
|
39
40
|
attr_reader :guessed_tests
|
|
40
41
|
|
|
41
|
-
def initialize(config, err: $stderr, digest: nil, capture_evidence: false, evidence_files: [], verbose: false
|
|
42
|
+
def initialize(config, err: $stderr, digest: nil, capture_evidence: false, evidence_files: [], verbose: false,
|
|
43
|
+
palette: nil, quiet_start: true)
|
|
42
44
|
@config = config
|
|
43
45
|
@hooks = Hooks.new
|
|
44
|
-
@diagnostics = Diagnostics.new(err, verbose: verbose)
|
|
46
|
+
@diagnostics = Diagnostics.new(err, verbose: verbose, palette: palette, quiet_start: quiet_start)
|
|
47
|
+
@completed_baseline_passes = 0
|
|
48
|
+
@baseline_elapsed = 0.0
|
|
45
49
|
@root = File.realpath(config.root)
|
|
46
50
|
@skipped_subjects = {}
|
|
47
51
|
@replay = Replay.new(digest: digest) if digest
|
|
@@ -158,7 +162,7 @@ module Saboteur
|
|
|
158
162
|
end
|
|
159
163
|
|
|
160
164
|
# The run's immutable report, built once the last result is in.
|
|
161
|
-
def announce_operators = @diagnostics.
|
|
165
|
+
def announce_operators = @diagnostics.detail("Operators: #{config.operator_policy['effective'].join(', ')}")
|
|
162
166
|
|
|
163
167
|
def build_report(results, subjects, scope, started)
|
|
164
168
|
Report.new(results: results, subjects: subjects, wall_time: monotonic - started, root: @root,
|
|
@@ -200,6 +204,7 @@ module Saboteur
|
|
|
200
204
|
store = coverage_store(integration, load_lines, load_methods)
|
|
201
205
|
full_passes = config["baseline_passes"] + (store.reusable?(files) ? 0 : 1)
|
|
202
206
|
warn_unscoped(integration, full_passes)
|
|
207
|
+
announce_quiet_start(store, files)
|
|
203
208
|
check_baseline(integration, full_passes)
|
|
204
209
|
refuse_unaffordable_coverage_build(store, files)
|
|
205
210
|
return [] if abandon_for_budget?(subjects)
|
|
@@ -217,13 +222,31 @@ module Saboteur
|
|
|
217
222
|
# mutant when serial. Pack the heap now that the map and candidates
|
|
218
223
|
# are retained, to reduce later copying of shared pages.
|
|
219
224
|
Execution::ForkReadiness.prepare_heap
|
|
220
|
-
results =
|
|
225
|
+
results = run_and_confirm(runnable_subjects, runnable_lists, scheduler, cache, ignore_list)
|
|
221
226
|
cache&.save
|
|
222
227
|
results
|
|
223
228
|
rescue BudgetPhaseTimeout => e
|
|
224
229
|
budget_phase_stop(e, subjects)
|
|
225
230
|
end
|
|
226
231
|
|
|
232
|
+
# The candidate run plus the opt-in post-run confirmation pass (#1063):
|
|
233
|
+
# confirmation runs after the last result, before the verdict cache is
|
|
234
|
+
# written, so a confirmed kill overwrites the survivor verdict the run
|
|
235
|
+
# cached (#1079). BudgetPhaseTimeout is not raised from the pass:
|
|
236
|
+
# post-run its rescue would discard every result, so a spent budget
|
|
237
|
+
# marks the remaining survivors unconfirmed instead.
|
|
238
|
+
def run_and_confirm(subjects, lists, scheduler, cache, ignore_list)
|
|
239
|
+
confirm_survivors(run_candidates(subjects, lists, scheduler, cache, ignore_list), scheduler, cache, ignore_list)
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
def confirm_survivors(results, scheduler, cache, ignore_list)
|
|
243
|
+
return results unless CoveragePartial.use?(config, config["coverage_parts"])
|
|
244
|
+
|
|
245
|
+
SurvivorConfirmation.new(config: config, map: scheduler.map, scheduler: scheduler, diagnostics: @diagnostics,
|
|
246
|
+
budget_exceeded: method(:budget_exceeded?), cache: cache)
|
|
247
|
+
.call(results, ignore_list: ignore_list, early_stop: @early_stop)
|
|
248
|
+
end
|
|
249
|
+
|
|
227
250
|
def run_candidates(subjects, lists, scheduler, cache, ignore_list)
|
|
228
251
|
@diagnostics.start_progress(lists.sum(&:size))
|
|
229
252
|
distribution = Distribution.new(jobs: external_command? ? 1 : config["jobs"], scheduler: scheduler,
|
|
@@ -266,8 +289,22 @@ module Saboteur
|
|
|
266
289
|
end
|
|
267
290
|
end
|
|
268
291
|
|
|
292
|
+
# The default view's first line: the run's identity, dim, before the
|
|
293
|
+
# baseline's own chatter (demoted to --verbose) would start.
|
|
294
|
+
def announce_quiet_start(store, files)
|
|
295
|
+
@map_cached = store.reusable?(files)
|
|
296
|
+
@diagnostics.quiet_start_header(scope: config["since"] ? "--since #{config['since']}" : "whole project",
|
|
297
|
+
seed: config["seed"], operators: config.operator_policy["effective"].size)
|
|
298
|
+
end
|
|
299
|
+
|
|
269
300
|
def inform_about_run(subjects, map)
|
|
270
|
-
@diagnostics.
|
|
301
|
+
if @diagnostics.quiet_start && !@diagnostics.verbose
|
|
302
|
+
@diagnostics.quiet_baseline(passes: @completed_baseline_passes, elapsed: @baseline_elapsed,
|
|
303
|
+
tests: map.tests.size, cached: @map_cached)
|
|
304
|
+
@diagnostics.quiet_inference(@quiet_spec_count)
|
|
305
|
+
else
|
|
306
|
+
@diagnostics.inform("#{subjects.size} subjects, #{map.tests.size} tests in coverage map")
|
|
307
|
+
end
|
|
271
308
|
ThreadWarnings.warn(@diagnostics)
|
|
272
309
|
end
|
|
273
310
|
|
|
@@ -437,7 +474,7 @@ module Saboteur
|
|
|
437
474
|
# Loads the target and stops coverage, returning what the rest of the
|
|
438
475
|
# run needs from that boot.
|
|
439
476
|
def boot_target(boot)
|
|
440
|
-
@diagnostics.
|
|
477
|
+
@diagnostics.detail("loading the application")
|
|
441
478
|
integration = boot.load_target
|
|
442
479
|
files = config.included_files
|
|
443
480
|
load_lines, load_methods = suspend_coverage(files)
|
data/lib/saboteur/help.rb
CHANGED
|
@@ -30,7 +30,10 @@ module Saboteur
|
|
|
30
30
|
"resource-limited mutant.",
|
|
31
31
|
4 => "Time budget spent: --max-duration stopped the run before the next subject, so the score is " \
|
|
32
32
|
"partial. --max-duration-ok leaves the exit to the plain verdict: 0, 1, or 3 when the budget " \
|
|
33
|
-
"stopped the run before any mutant was tested."
|
|
33
|
+
"stopped the run before any mutant was tested.",
|
|
34
|
+
5 => "Baseline suite red: the unchanged suite failed its check, so no mutant ran. Fix your tests first.",
|
|
35
|
+
70 => "Internal crash: a saboteur bug, never your code. The error line names a log with the full " \
|
|
36
|
+
"backtrace; please report it."
|
|
34
37
|
}.freeze
|
|
35
38
|
|
|
36
39
|
DOCTOR_EXIT_CODE_MEANINGS = {
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
require "prism"
|
|
5
5
|
require_relative "base"
|
|
6
6
|
require_relative "filter_replacement"
|
|
7
|
+
require_relative "copy_replacement"
|
|
7
8
|
require_relative "literal_replacement"
|
|
8
9
|
require_relative "argument_removal"
|
|
9
10
|
require_relative "argument_swap"
|
|
@@ -21,6 +22,7 @@ require_relative "interpolation_removal"
|
|
|
21
22
|
require_relative "jump_value"
|
|
22
23
|
require_relative "regex_replacement"
|
|
23
24
|
require_relative "rescue_removal"
|
|
25
|
+
require_relative "ensure_removal"
|
|
24
26
|
require_relative "when_removal"
|
|
25
27
|
require_relative "unary_removal"
|
|
26
28
|
|
|
@@ -31,6 +33,10 @@ module Saboteur
|
|
|
31
33
|
class Builtin < Base
|
|
32
34
|
DEFAULT_ARID = [
|
|
33
35
|
'\A(puts|pp|p|print|warn)\b',
|
|
36
|
+
# Exception message arguments read as diagnostics too (#1066): a
|
|
37
|
+
# raise/fail string edit sits inside the call, so the line and the
|
|
38
|
+
# call head both have to match.
|
|
39
|
+
'\A(raise|fail)\b',
|
|
34
40
|
'\A(@|self\.|::)?(Rails\.)?logger\b',
|
|
35
41
|
# A logger at the end of a receiver chain: Sentry.sdk_logger.warn
|
|
36
42
|
# (#665). The match is case-sensitive, so SDK_LOGGER stays
|
|
@@ -76,10 +82,10 @@ module Saboteur
|
|
|
76
82
|
SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
|
|
77
83
|
# The single-purpose sources, in the order they run after the shared walk.
|
|
78
84
|
DELEGATED = [
|
|
79
|
-
FilterReplacement, LiteralReplacement, ArgumentRemoval, PairRemoval, ElementRemoval,
|
|
80
|
-
EndpointReplacement, RangeReplacement, OperatorAssignment, ConditionNegation, MethodSwap,
|
|
85
|
+
FilterReplacement, CopyReplacement, LiteralReplacement, ArgumentRemoval, PairRemoval, ElementRemoval,
|
|
86
|
+
ChainRemoval, EndpointReplacement, RangeReplacement, OperatorAssignment, ConditionNegation, MethodSwap,
|
|
81
87
|
OperandReplacement, InterpolationRemoval, RegexReplacement, ArgumentSwap, JumpValue, RescueRemoval,
|
|
82
|
-
WhenRemoval, UnaryRemoval
|
|
88
|
+
EnsureRemoval, WhenRemoval, UnaryRemoval
|
|
83
89
|
].freeze
|
|
84
90
|
|
|
85
91
|
def initialize(arid: DEFAULT_ARID, operators: [], exclude_operators: [])
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "prism"
|
|
5
|
+
require_relative "base"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
module MutationSource
|
|
9
|
+
# Drops a defensive copy (x.dup, x.clone becomes x) or makes a deep copy
|
|
10
|
+
# shallow (x.deep_dup becomes x.dup). Both let a caller and a callee share
|
|
11
|
+
# state that the original code kept apart.
|
|
12
|
+
class CopyReplacement < Base
|
|
13
|
+
DROPPED = %i[dup clone].freeze
|
|
14
|
+
SHALLOWER = { deep_dup: "dup" }.freeze
|
|
15
|
+
SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
|
|
16
|
+
|
|
17
|
+
# Base inherits Object's empty initializer; there is no parent state to set.
|
|
18
|
+
def initialize(arid: []) # rubocop:disable Lint/MissingSuper
|
|
19
|
+
@arid = arid.map { |pattern| Regexp.new(pattern) }
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def name
|
|
23
|
+
"copy_replacement"
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def mutations_for(subject)
|
|
27
|
+
found = []
|
|
28
|
+
visit(subject.body_node, subject, found)
|
|
29
|
+
found
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
private
|
|
33
|
+
|
|
34
|
+
def visit(node, subject, found)
|
|
35
|
+
return if node.nil? || SCOPE_NODES.any? { |scope| node.is_a?(scope) }
|
|
36
|
+
|
|
37
|
+
if node.is_a?(Prism::StatementsNode)
|
|
38
|
+
node.body.each { |statement| visit(statement, subject, found) unless arid?(statement) }
|
|
39
|
+
return
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
replace_copy(node, subject, found) if node.is_a?(Prism::CallNode)
|
|
43
|
+
node.compact_child_nodes.each { |child| visit(child, subject, found) }
|
|
44
|
+
nil
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def replace_copy(node, subject, found)
|
|
48
|
+
return unless node.receiver && node.block.nil? && node.arguments.nil?
|
|
49
|
+
|
|
50
|
+
edit = DROPPED.include?(node.name) ? drop(node, subject) : shallow(node, subject)
|
|
51
|
+
found << Mutation.new(subject_id: subject.id, operator: name, edits: [edit], source: name) if edit
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Receiver stays; the dot (or &.) through the end of the call goes.
|
|
55
|
+
def drop(node, subject)
|
|
56
|
+
Edit.new(file: subject.file, start_byte: node.call_operator_loc.start_offset,
|
|
57
|
+
end_byte: node.end_offset, replacement: "")
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def shallow(node, subject)
|
|
61
|
+
replacement = SHALLOWER[node.name] or return
|
|
62
|
+
location = node.message_loc
|
|
63
|
+
Edit.new(file: subject.file, start_byte: location.start_offset, end_byte: location.end_offset,
|
|
64
|
+
replacement: replacement)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def arid?(node)
|
|
68
|
+
@arid.any? { |pattern| pattern.match?(node.slice) }
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|