saboteur 0.9.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +66 -0
  3. data/README.md +109 -247
  4. data/exe/saboteur +2 -2
  5. data/lib/saboteur/baseline/loader.rb +41 -0
  6. data/lib/saboteur/baseline/run_evidence.rb +1 -1
  7. data/lib/saboteur/baseline/run_gate.rb +51 -0
  8. data/lib/saboteur/baseline/workflow.rb +2 -15
  9. data/lib/saboteur/cli/baseline_gate.rb +58 -0
  10. data/lib/saboteur/cli/error_cause.rb +16 -0
  11. data/lib/saboteur/cli/exit_status.rb +14 -13
  12. data/lib/saboteur/cli/failure_output.rb +23 -0
  13. data/lib/saboteur/cli/output_options.rb +24 -2
  14. data/lib/saboteur/cli/run_budget_options.rb +6 -0
  15. data/lib/saboteur/cli/run_command.rb +14 -1
  16. data/lib/saboteur/cli/run_reporting.rb +34 -6
  17. data/lib/saboteur/cli/run_validation.rb +36 -3
  18. data/lib/saboteur/cli/selection_options.rb +2 -0
  19. data/lib/saboteur/cli.rb +6 -1
  20. data/lib/saboteur/config.rb +23 -2
  21. data/lib/saboteur/crash_report.rb +16 -7
  22. data/lib/saboteur/engine/baseline_guard.rb +2 -2
  23. data/lib/saboteur/engine/coverage_partial.rb +8 -1
  24. data/lib/saboteur/engine/coverage_store.rb +4 -5
  25. data/lib/saboteur/engine/diagnostics.rb +120 -15
  26. data/lib/saboteur/engine/run_fingerprint.rb +1 -1
  27. data/lib/saboteur/engine/scheduler.rb +20 -0
  28. data/lib/saboteur/engine/selection_announcement.rb +8 -1
  29. data/lib/saboteur/engine/survivor_confirmation.rb +159 -0
  30. data/lib/saboteur/engine.rb +43 -6
  31. data/lib/saboteur/help.rb +4 -1
  32. data/lib/saboteur/mutation_source/builtin.rb +9 -3
  33. data/lib/saboteur/mutation_source/copy_replacement.rb +72 -0
  34. data/lib/saboteur/mutation_source/ensure_removal.rb +49 -0
  35. data/lib/saboteur/mutation_source/operator_examples.rb +2 -0
  36. data/lib/saboteur/mutation_source/operator_registry.rb +3 -1
  37. data/lib/saboteur/reporter/agent_summary.rb +3 -1
  38. data/lib/saboteur/reporter/assets/mutation_report.LICENSE +201 -0
  39. data/lib/saboteur/reporter/assets/mutation_report.js +331 -0
  40. data/lib/saboteur/reporter/baseline_tail.rb +93 -0
  41. data/lib/saboteur/reporter/diff.rb +10 -0
  42. data/lib/saboteur/reporter/html.rb +69 -14
  43. data/lib/saboteur/reporter/kill_causes.rb +53 -0
  44. data/lib/saboteur/reporter/next_steps.rb +4 -3
  45. data/lib/saboteur/reporter/palette.rb +64 -13
  46. data/lib/saboteur/reporter/safe_out.rb +1 -1
  47. data/lib/saboteur/reporter/score_table.rb +53 -18
  48. data/lib/saboteur/reporter/stryker.rb +4 -1
  49. data/lib/saboteur/reporter/survivor_classification.rb +12 -3
  50. data/lib/saboteur/reporter/survivor_hints.rb +72 -0
  51. data/lib/saboteur/reporter/survivor_list.rb +207 -75
  52. data/lib/saboteur/reporter/terminal.rb +315 -69
  53. data/lib/saboteur/result.rb +28 -1
  54. data/lib/saboteur/score.rb +6 -0
  55. data/lib/saboteur/version.rb +1 -1
  56. metadata +12 -2
@@ -1,8 +1,11 @@
1
1
  # SPDX-License-Identifier: MIT
2
2
  # frozen_string_literal: true
3
3
 
4
+ require "io/console"
4
5
  require "tempfile"
5
6
 
7
+ require_relative "../reporter/palette"
8
+
6
9
  module Saboteur
7
10
  class Engine
8
11
  # What a run has to say besides results: warnings for the report, the
@@ -66,19 +69,32 @@ module Saboteur
66
69
  end
67
70
 
68
71
  MARKS = { killed: ".", survived: "S", timeout: "T", resource_limit: "L", error: "E" }.freeze
69
- LEGEND = ". killed S survived T timeout L resource limit E error"
70
- LINE_WIDTH = 50
72
+ MARK_TOKENS = { "." => :dim, "S" => :fail, "T" => :warn, "E" => :fail, "L" => :fail }.freeze
73
+ # Readback whitelist: mutant children inherit the progress fd and can
74
+ # write arbitrary bytes, so only known mark letters come back out.
75
+ READBACK_GLYPHS = (MARK_TOKENS.keys + ["?"]).freeze
76
+ # s:skipped never prints a glyph: skipped mutants never reach
77
+ # progress (see start_progress); the letter is legend-only.
78
+ LEGEND = ".:killed S:survived T:timeout E:error s:skipped L:resource limit"
79
+
80
+ # Chunk sizing: leave room for the " (n/total ...)" suffix, never
81
+ # below the floor.
82
+ CHUNK_MARGIN = 10
83
+ CHUNK_FLOOR = 40
84
+ DEFAULT_WIDTH = 80
71
85
 
72
86
  GenerationEvent = Data.define(:file, :subject, :source, :operator, :reason)
73
87
  Omission = Data.define(:subject, :count, :limit)
74
88
  DiscoveryFailure = Data.define(:file, :reason)
75
89
 
76
90
  attr_reader :warnings, :live_announced, :invalid_mutations, :generation_events, :omissions, :discovery_failures,
77
- :hazard_suppressions, :inline_suppressions, :verbose
91
+ :hazard_suppressions, :inline_suppressions, :verbose, :quiet_start
78
92
 
79
- def initialize(err, verbose: false)
93
+ def initialize(err, verbose: false, palette: nil, quiet_start: true)
80
94
  @err = err
81
95
  @verbose = verbose
96
+ @quiet_start = quiet_start
97
+ @palette = palette || Reporter::Palette.for(err)
82
98
  @warnings = []
83
99
  @invalid_mutations = 0
84
100
  @generation_events = []
@@ -163,6 +179,51 @@ module Saboteur
163
179
  @err.puts("saboteur: #{Text.display_safe_lines(message)}")
164
180
  end
165
181
 
182
+ # Chatter the default view folds into the quiet-start lines;
183
+ # --verbose prints it byte-for-byte as before.
184
+ def detail(message)
185
+ inform(message) if @verbose || !@quiet_start
186
+ end
187
+
188
+ # A selection explanation the default view carries as one dim
189
+ # summary line: the full text stays in report.warnings for machine
190
+ # consumers and counts as announced, so it is not reprinted at the
191
+ # end of the run.
192
+ def announce_demoted(message)
193
+ safe = Text.display_safe_lines(message)
194
+ warn(safe)
195
+ @live_announced << safe
196
+ end
197
+
198
+ # The quiet start: three lines, then progress. --verbose keeps
199
+ # today's chatter instead and never prints these.
200
+ def quiet_start_header(scope:, seed:, operators:)
201
+ return if @verbose || !@quiet_start
202
+
203
+ dot = @palette.glyph(:middot)
204
+ line = "saboteur #{Saboteur::VERSION} #{dot} #{scope} #{dot} seed #{seed} #{dot} " \
205
+ "#{operators} operators (--list-operators)"
206
+ @err.puts(@palette.paint(:dim, line))
207
+ end
208
+
209
+ def quiet_baseline(passes:, elapsed:, tests:, cached:)
210
+ return if @verbose || !@quiet_start
211
+
212
+ check = @palette.paint(:pass, @palette.glyph(:check))
213
+ suffix = cached ? " (cached)" : ""
214
+ line = "#{check} baseline green (#{passes} #{passes == 1 ? 'pass' : 'passes'}, " \
215
+ "#{format('%.1f', elapsed)}s) #{check} coverage map: #{tests} " \
216
+ "#{tests == 1 ? 'test' : 'tests'}#{suffix}"
217
+ @err.puts(line)
218
+ end
219
+
220
+ def quiet_inference(spec_count)
221
+ return if spec_count.nil? || @verbose || !@quiet_start
222
+
223
+ files = spec_count == 1 ? "spec file" : "spec files"
224
+ @err.puts(@palette.paint(:dim, "spec inference: #{spec_count} #{files} (details: --verbose)"))
225
+ end
226
+
166
227
  # --verbose: one line per finished subject with its result marks, so a
167
228
  # survivor is attributable to a method as soon as the subject
168
229
  # completes, before the whole run ends (#262).
@@ -170,7 +231,7 @@ module Saboteur
170
231
  return unless @verbose
171
232
  return if results.empty?
172
233
 
173
- marks = results.map { |result| MARKS.fetch(result.status, "?") }.join
234
+ marks = results.map { |result| painted_mark(MARKS.fetch(result.status, "?")) }.join
174
235
  inform("#{subject.id} #{marks}")
175
236
  end
176
237
 
@@ -204,21 +265,36 @@ module Saboteur
204
265
  @progress_file = Tempfile.new("saboteur-progress")
205
266
  @progress_file.binmode
206
267
  @progress_file.sync = true
207
- @err.puts("saboteur: progress #{LEGEND}") unless progress_on_terminal?
268
+ @glyphs = +""
269
+ @err.puts(@palette.paint(:dim, LEGEND))
208
270
  end
209
271
 
210
272
  def progress(result)
211
273
  mark = MARKS.fetch(result.status, "?")
212
- return plain_progress(mark) unless @progress_file
274
+ return plain_progress(painted_mark(mark)) unless @progress_file
213
275
 
214
276
  done = record_progress(mark)
215
- progress_on_terminal? ? terminal_progress(done) : ci_progress(mark, done)
277
+ if progress_on_terminal?
278
+ @glyphs << mark
279
+ @glyphs.size >= chunk_width ? commit_chunk(done) : redraw_line(done)
280
+ elsif (done % chunk_width).zero?
281
+ commit_file_chunk(done - chunk_width, chunk_width, done)
282
+ end
216
283
  @err.flush
217
284
  end
218
285
 
286
+ def painted_mark(mark)
287
+ token = MARK_TOKENS[mark]
288
+ token ? @palette.paint(token, mark) : mark
289
+ end
290
+
219
291
  def finish_progress
220
- @err.puts if @progress_file && (progress_on_terminal? || @progress_file.size % LINE_WIDTH != 0)
221
- @progress_file&.close!
292
+ return unless @progress_file
293
+
294
+ size = @progress_file.size
295
+ pending = size % chunk_width
296
+ commit_file_chunk(size - pending, pending, size) if pending.positive?
297
+ @progress_file.close!
222
298
  @progress_file = nil
223
299
  end
224
300
 
@@ -245,13 +321,42 @@ module Saboteur
245
321
  @err.respond_to?(:tty?) && @err.tty?
246
322
  end
247
323
 
248
- def terminal_progress(done)
249
- @err.print("\r\e[Ksaboteur: #{progress_summary(done)}")
324
+ def chunk_width
325
+ columns = begin
326
+ @err.winsize[1] if @err.respond_to?(:winsize)
327
+ rescue SystemCallError
328
+ nil
329
+ end
330
+ [(columns || DEFAULT_WIDTH) - CHUNK_MARGIN, CHUNK_FLOOR].max
250
331
  end
251
332
 
252
- def ci_progress(mark, done)
253
- @err.print(mark)
254
- @err.print(" #{progress_summary(done)}\n") if (done % LINE_WIDTH).zero?
333
+ def painted_glyphs
334
+ @glyphs.each_char.map { |glyph| painted_mark(glyph) }.join
335
+ end
336
+
337
+ def redraw_line(done)
338
+ @line_open = true
339
+ @err.print("\r\e[K#{painted_glyphs} #{@palette.paint(:dim, progress_summary(done))}")
340
+ end
341
+
342
+ def commit_chunk(done)
343
+ @err.print("\r\e[K#{painted_glyphs} #{@palette.paint(:dim, progress_summary(done))}\n")
344
+ @glyphs.clear
345
+ end
346
+
347
+ # Piped chunks and the final partial chunk come from the progress
348
+ # file, not the buffer: workers each buffer only their own marks,
349
+ # while the shared file the workers inherit holds every mark.
350
+ def commit_file_chunk(start, length, done)
351
+ @err.print("\r\e[K") if @line_open
352
+ @err.print("#{painted_file_glyphs(start, length)} #{@palette.paint(:dim, progress_summary(done))}\n")
353
+ end
354
+
355
+ # binread caps at both +length+ (the recorded mark count) and EOF,
356
+ # so a short or padded file cannot leak extra glyphs.
357
+ def painted_file_glyphs(start, length)
358
+ bytes = File.binread(@progress_file.path, length, start)
359
+ bytes.each_char.filter_map { |glyph| painted_mark(glyph) if READBACK_GLYPHS.include?(glyph) }.join
255
360
  end
256
361
 
257
362
  def eta_text(done)
@@ -14,7 +14,7 @@ module Saboteur
14
14
  class RunFingerprint
15
15
  FORMAT_VERSION = 1
16
16
  MANIFESTS = %w[Gemfile gems.rb Gemfile.lock gems.locked .rspec .rspec-local].freeze
17
- LOCATION_SETTINGS = Config::OUTPUT_KEYS
17
+ LOCATION_SETTINGS = Config::OUTPUT_KEYS + Config::PRESENTATION_KEYS
18
18
  ENTRY_KINDS = %w[ruby manifest boot hook source_plugin extra data].freeze
19
19
  EXCLUDED_DIRECTORIES = %w[.git vendor node_modules].freeze
20
20
  Entry = Data.define(:path, :kind, :state, :sha256, :required)
@@ -232,6 +232,26 @@ module Saboteur
232
232
  result
233
233
  end
234
234
 
235
+ attr_reader :runner, :map
236
+
237
+ # The eval prefix a post-run confirmation run needs for this subject
238
+ # (#1063): the same wrapper detection load_problem applies, reduced to
239
+ # the sorbet sig prefix a survivor could have run with.
240
+ def confirmation_prefix(subject)
241
+ return nil unless @verify_loaded
242
+ return nil if @reevaluator.whole_file?(subject)
243
+
244
+ owner = @reevaluator.resolve_owner(subject)
245
+ target = subject.singleton ? owner.singleton_class : owner
246
+ method = own_method(target, subject.name)
247
+ return nil unless method
248
+
249
+ file, = method.source_location
250
+ Execution::Wrapper.around(method, file) == :sorbet ? subject.sig_source : nil
251
+ rescue Error
252
+ nil
253
+ end
254
+
235
255
  # The tests that ran the edited code (a coarse edit spans the whole
236
256
  # body, so it gets every test of the method), prior killers first, then
237
257
  # fastest-first.
@@ -27,8 +27,15 @@ module Saboteur
27
27
  end
28
28
 
29
29
  def announce_selection(guess)
30
- if guess.changed.zero? || guess.specs.any?
30
+ if guess.changed.zero?
31
31
  @diagnostics.announce(selection_message(guess))
32
+ elsif guess.specs.any?
33
+ @quiet_spec_count = guess.specs.size
34
+ if @diagnostics.verbose || !@diagnostics.quiet_start
35
+ @diagnostics.announce(selection_message(guess))
36
+ else
37
+ @diagnostics.announce_demoted(selection_message(guess))
38
+ end
32
39
  else
33
40
  @diagnostics.inform(selection_message(guess))
34
41
  end
@@ -0,0 +1,159 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class Engine
6
+ # Post-run survivor confirmation (#1063): with partial_coverage=use the
7
+ # coverage map can be a partial rebuild, so every open survivor's
8
+ # covering specs the mutant never ran get one budgeted confirmation run
9
+ # through the same Runner invocation the scheduler makes. A failing run
10
+ # kills the mutant; a passing run marks the result confirmed; anything
11
+ # that could not run is marked unconfirmed with the reason. The field is
12
+ # additive - the status keeps its meaning - and no outcome is silent.
13
+ class SurvivorConfirmation
14
+ # The most covering specs one survivor re-runs; the config's
15
+ # confirmation_budget caps the whole pass.
16
+ MAX_SPECS_PER_SURVIVOR = 8
17
+
18
+ def initialize(config:, map:, scheduler:, diagnostics:, budget_exceeded:, cache: nil)
19
+ @config = config
20
+ @map = map
21
+ @scheduler = scheduler
22
+ @diagnostics = diagnostics
23
+ @budget_exceeded = budget_exceeded
24
+ @cache = cache
25
+ end
26
+
27
+ def call(results, ignore_list:, early_stop: nil)
28
+ survivors = results.select { |result| result.survived? && !ignore_list.accepts?(result) }
29
+ return results if survivors.empty?
30
+
31
+ if early_stop
32
+ @diagnostics.warn("survivor confirmation covers only the survivors reached before the run " \
33
+ "stopped early (#{early_stop.reason})")
34
+ end
35
+ @remaining = @config["confirmation_budget"]
36
+ results.map { |result| survivors.include?(result) ? confirm(result) : result }
37
+ .tap { |confirmed| summarize(confirmed) }
38
+ end
39
+
40
+ private
41
+
42
+ def confirm(result)
43
+ return unconfirmed(result, "the confirmation budget is spent") if @remaining <= 0
44
+ return unconfirmed(result, "the --max-duration budget is spent") if @budget_exceeded.call
45
+
46
+ tests = unrun_covering_tests(result)
47
+ return result if tests.empty?
48
+
49
+ outcome = run_confirmation(result, tests)
50
+ @remaining -= outcome.runtime || 0
51
+ case outcome.status
52
+ when :killed then checked_kill(result, tests, outcome)
53
+ when :survived then result.with_confirmation("confirmed")
54
+ else unconfirmed(result, "the confirmation run #{outcome.status}: #{outcome.message}")
55
+ end
56
+ end
57
+
58
+ # The whole-suite map's covering tests for the mutant's edited lines
59
+ # (branch evidence intersected, mirroring the scheduler), minus the
60
+ # tests the mutant already ran, fastest first, capped.
61
+ def unrun_covering_tests(result)
62
+ fastest_first(covering_tests(result) - (result.executed || result.covered_by)).first(MAX_SPECS_PER_SURVIVOR)
63
+ end
64
+
65
+ def covering_tests(result)
66
+ line_numbers = result.mutation.edits.flat_map { |edit| result.subject.coverage_lines_for(edit) }.uniq
67
+ tests = @map.tests_for_edit(result.subject, line_numbers)
68
+ branch_tests = @map.tests_for_branches(result.subject, result.mutation.edits)
69
+ branch_tests ? tests & branch_tests : tests
70
+ end
71
+
72
+ def fastest_first(tests)
73
+ tests.sort_by { |id| @map.recorded_runtime([id]) || Float::INFINITY }
74
+ end
75
+
76
+ def run_confirmation(result, tests)
77
+ @scheduler.runner.run(subject: result.subject, text: result.mutated_source, test_ids: tests,
78
+ timeout: timeout_for(tests, result), mutation: result.mutation,
79
+ eval_prefix: @scheduler.confirmation_prefix(result.subject))
80
+ end
81
+
82
+ # A confirmation kill stands only after the same VerdictCheck the
83
+ # scheduler runs: the kill sequence, in order, must pass with the
84
+ # method unchanged, so an order-dependent spec cannot convert a true
85
+ # survivor into a credited kill (security review). A rejected kill
86
+ # becomes an error; a check the budgets never let run comes back
87
+ # unverified - outside the score and never cached.
88
+ def checked_kill(result, tests, outcome)
89
+ kill = result.with_confirmation("confirmed", status: :killed, reason: outcome.message,
90
+ killed_by: outcome.failed_ids, covered_by: tests,
91
+ kill_cause: outcome.kill_cause,
92
+ runtime: (result.runtime || 0) + (outcome.runtime || 0),
93
+ executed: (result.executed || []) + outcome.executed_ids)
94
+ checked = VerdictCheck.new(run_unchanged: unchanged_runner(result.subject),
95
+ baseline_timeout: @config["baseline_timeout"]).call([kill]).first
96
+ return checked unless checked.status == :killed
97
+
98
+ @diagnostics.inform("#{result.subject.id} #{result.mutation.operator} #{result.digest}: " \
99
+ "killed on confirmation by #{outcome.failed_ids.first}")
100
+ record_kill(result, kill)
101
+ checked
102
+ end
103
+
104
+ # The standing kill goes into the verdict cache with the mutant's full
105
+ # covering test set, not the capped confirmation subset: the cache
106
+ # fingerprint digests that set, so only the full set overwrites the
107
+ # survivor verdict the run cached and lets the next run reuse the kill
108
+ # (#1079). ResultCache#record replaces the subject+mutation entry, so
109
+ # the stale survivor verdict cannot win a later lookup.
110
+ def record_kill(result, kill)
111
+ @cache&.record(kill.with_confirmation("confirmed", covered_by: covering_tests(result)))
112
+ end
113
+
114
+ # The unchanged-method re-run VerdictCheck checks a kill against, in
115
+ # the unchanged_runner shape (scheduler.rb), against the confirmation
116
+ # and --max-duration budgets instead of the scheduler's plan state.
117
+ def unchanged_runner(subject)
118
+ lambda do |tests, timeout|
119
+ if @remaining <= 0 || @budget_exceeded.call
120
+ Execution::Runner::Outcome.new(status: :unverified, runtime: 0.0,
121
+ message: "the confirmation budget is spent")
122
+ else
123
+ outcome = @scheduler.runner.run(subject: subject, text: subject.source, test_ids: tests,
124
+ timeout: timeout,
125
+ eval_prefix: @scheduler.confirmation_prefix(subject))
126
+ @remaining -= outcome.runtime || 0
127
+ outcome
128
+ end
129
+ end
130
+ end
131
+
132
+ def unconfirmed(result, reason)
133
+ result.with_confirmation("unconfirmed(#{reason})")
134
+ end
135
+
136
+ # The scheduler's timeout_for shape with the post-run slowdown of 1.0.
137
+ # recorded_runtime is nil without arm durations: fall back to the
138
+ # mutant's own first-run runtime, then to the suite's mean.
139
+ def timeout_for(tests, result)
140
+ recorded = @map.recorded_runtime(tests) || result.runtime || suite_mean_runtime
141
+ return @config["baseline_timeout"] unless recorded
142
+
143
+ [@config["timeout_factor"] * recorded, @config["timeout_min"]].max
144
+ end
145
+
146
+ def suite_mean_runtime
147
+ durations = @map.tests.map { |id| @map.recorded_runtime([id]) }.compact
148
+ durations.empty? ? nil : durations.sum / durations.size
149
+ end
150
+
151
+ def summarize(results)
152
+ counts = results.filter_map(&:confirmation).tally
153
+ return if counts.empty?
154
+
155
+ @diagnostics.inform("survivor confirmation: #{counts.map { |mark, n| "#{n} #{mark}" }.join(', ')}")
156
+ end
157
+ end
158
+ end
159
+ end
@@ -12,6 +12,7 @@ require_relative "engine/load_coverage"
12
12
  require_relative "engine/out_of_scope_guard"
13
13
  require_relative "engine/run_budget"
14
14
  require_relative "engine/selection_announcement"
15
+ require_relative "engine/survivor_confirmation"
15
16
  require_relative "engine/thread_warnings"
16
17
 
17
18
  require_relative "engine/pipelines"
@@ -38,10 +39,13 @@ module Saboteur
38
39
  # The specs inference narrowed the run to, or nil when it did not.
39
40
  attr_reader :guessed_tests
40
41
 
41
- def initialize(config, err: $stderr, digest: nil, capture_evidence: false, evidence_files: [], verbose: false)
42
+ def initialize(config, err: $stderr, digest: nil, capture_evidence: false, evidence_files: [], verbose: false,
43
+ palette: nil, quiet_start: true)
42
44
  @config = config
43
45
  @hooks = Hooks.new
44
- @diagnostics = Diagnostics.new(err, verbose: verbose)
46
+ @diagnostics = Diagnostics.new(err, verbose: verbose, palette: palette, quiet_start: quiet_start)
47
+ @completed_baseline_passes = 0
48
+ @baseline_elapsed = 0.0
45
49
  @root = File.realpath(config.root)
46
50
  @skipped_subjects = {}
47
51
  @replay = Replay.new(digest: digest) if digest
@@ -158,7 +162,7 @@ module Saboteur
158
162
  end
159
163
 
160
164
  # The run's immutable report, built once the last result is in.
161
- def announce_operators = @diagnostics.inform("Operators: #{config.operator_policy['effective'].join(', ')}")
165
+ def announce_operators = @diagnostics.detail("Operators: #{config.operator_policy['effective'].join(', ')}")
162
166
 
163
167
  def build_report(results, subjects, scope, started)
164
168
  Report.new(results: results, subjects: subjects, wall_time: monotonic - started, root: @root,
@@ -200,6 +204,7 @@ module Saboteur
200
204
  store = coverage_store(integration, load_lines, load_methods)
201
205
  full_passes = config["baseline_passes"] + (store.reusable?(files) ? 0 : 1)
202
206
  warn_unscoped(integration, full_passes)
207
+ announce_quiet_start(store, files)
203
208
  check_baseline(integration, full_passes)
204
209
  refuse_unaffordable_coverage_build(store, files)
205
210
  return [] if abandon_for_budget?(subjects)
@@ -217,13 +222,31 @@ module Saboteur
217
222
  # mutant when serial. Pack the heap now that the map and candidates
218
223
  # are retained, to reduce later copying of shared pages.
219
224
  Execution::ForkReadiness.prepare_heap
220
- results = run_candidates(runnable_subjects, runnable_lists, scheduler, cache, ignore_list)
225
+ results = run_and_confirm(runnable_subjects, runnable_lists, scheduler, cache, ignore_list)
221
226
  cache&.save
222
227
  results
223
228
  rescue BudgetPhaseTimeout => e
224
229
  budget_phase_stop(e, subjects)
225
230
  end
226
231
 
232
+ # The candidate run plus the opt-in post-run confirmation pass (#1063):
233
+ # confirmation runs after the last result, before the verdict cache is
234
+ # written, so a confirmed kill overwrites the survivor verdict the run
235
+ # cached (#1079). BudgetPhaseTimeout is not raised from the pass:
236
+ # post-run its rescue would discard every result, so a spent budget
237
+ # marks the remaining survivors unconfirmed instead.
238
+ def run_and_confirm(subjects, lists, scheduler, cache, ignore_list)
239
+ confirm_survivors(run_candidates(subjects, lists, scheduler, cache, ignore_list), scheduler, cache, ignore_list)
240
+ end
241
+
242
+ def confirm_survivors(results, scheduler, cache, ignore_list)
243
+ return results unless CoveragePartial.use?(config, config["coverage_parts"])
244
+
245
+ SurvivorConfirmation.new(config: config, map: scheduler.map, scheduler: scheduler, diagnostics: @diagnostics,
246
+ budget_exceeded: method(:budget_exceeded?), cache: cache)
247
+ .call(results, ignore_list: ignore_list, early_stop: @early_stop)
248
+ end
249
+
227
250
  def run_candidates(subjects, lists, scheduler, cache, ignore_list)
228
251
  @diagnostics.start_progress(lists.sum(&:size))
229
252
  distribution = Distribution.new(jobs: external_command? ? 1 : config["jobs"], scheduler: scheduler,
@@ -266,8 +289,22 @@ module Saboteur
266
289
  end
267
290
  end
268
291
 
292
+ # The default view's first line: the run's identity, dim, before the
293
+ # baseline's own chatter (demoted to --verbose) would start.
294
+ def announce_quiet_start(store, files)
295
+ @map_cached = store.reusable?(files)
296
+ @diagnostics.quiet_start_header(scope: config["since"] ? "--since #{config['since']}" : "whole project",
297
+ seed: config["seed"], operators: config.operator_policy["effective"].size)
298
+ end
299
+
269
300
  def inform_about_run(subjects, map)
270
- @diagnostics.inform("#{subjects.size} subjects, #{map.tests.size} tests in coverage map")
301
+ if @diagnostics.quiet_start && !@diagnostics.verbose
302
+ @diagnostics.quiet_baseline(passes: @completed_baseline_passes, elapsed: @baseline_elapsed,
303
+ tests: map.tests.size, cached: @map_cached)
304
+ @diagnostics.quiet_inference(@quiet_spec_count)
305
+ else
306
+ @diagnostics.inform("#{subjects.size} subjects, #{map.tests.size} tests in coverage map")
307
+ end
271
308
  ThreadWarnings.warn(@diagnostics)
272
309
  end
273
310
 
@@ -437,7 +474,7 @@ module Saboteur
437
474
  # Loads the target and stops coverage, returning what the rest of the
438
475
  # run needs from that boot.
439
476
  def boot_target(boot)
440
- @diagnostics.inform("loading the application")
477
+ @diagnostics.detail("loading the application")
441
478
  integration = boot.load_target
442
479
  files = config.included_files
443
480
  load_lines, load_methods = suspend_coverage(files)
data/lib/saboteur/help.rb CHANGED
@@ -30,7 +30,10 @@ module Saboteur
30
30
  "resource-limited mutant.",
31
31
  4 => "Time budget spent: --max-duration stopped the run before the next subject, so the score is " \
32
32
  "partial. --max-duration-ok leaves the exit to the plain verdict: 0, 1, or 3 when the budget " \
33
- "stopped the run before any mutant was tested."
33
+ "stopped the run before any mutant was tested.",
34
+ 5 => "Baseline suite red: the unchanged suite failed its check, so no mutant ran. Fix your tests first.",
35
+ 70 => "Internal crash: a saboteur bug, never your code. The error line names a log with the full " \
36
+ "backtrace; please report it."
34
37
  }.freeze
35
38
 
36
39
  DOCTOR_EXIT_CODE_MEANINGS = {
@@ -4,6 +4,7 @@
4
4
  require "prism"
5
5
  require_relative "base"
6
6
  require_relative "filter_replacement"
7
+ require_relative "copy_replacement"
7
8
  require_relative "literal_replacement"
8
9
  require_relative "argument_removal"
9
10
  require_relative "argument_swap"
@@ -21,6 +22,7 @@ require_relative "interpolation_removal"
21
22
  require_relative "jump_value"
22
23
  require_relative "regex_replacement"
23
24
  require_relative "rescue_removal"
25
+ require_relative "ensure_removal"
24
26
  require_relative "when_removal"
25
27
  require_relative "unary_removal"
26
28
 
@@ -31,6 +33,10 @@ module Saboteur
31
33
  class Builtin < Base
32
34
  DEFAULT_ARID = [
33
35
  '\A(puts|pp|p|print|warn)\b',
36
+ # Exception message arguments read as diagnostics too (#1066): a
37
+ # raise/fail string edit sits inside the call, so the line and the
38
+ # call head both have to match.
39
+ '\A(raise|fail)\b',
34
40
  '\A(@|self\.|::)?(Rails\.)?logger\b',
35
41
  # A logger at the end of a receiver chain: Sentry.sdk_logger.warn
36
42
  # (#665). The match is case-sensitive, so SDK_LOGGER stays
@@ -76,10 +82,10 @@ module Saboteur
76
82
  SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
77
83
  # The single-purpose sources, in the order they run after the shared walk.
78
84
  DELEGATED = [
79
- FilterReplacement, LiteralReplacement, ArgumentRemoval, PairRemoval, ElementRemoval, ChainRemoval,
80
- EndpointReplacement, RangeReplacement, OperatorAssignment, ConditionNegation, MethodSwap,
85
+ FilterReplacement, CopyReplacement, LiteralReplacement, ArgumentRemoval, PairRemoval, ElementRemoval,
86
+ ChainRemoval, EndpointReplacement, RangeReplacement, OperatorAssignment, ConditionNegation, MethodSwap,
81
87
  OperandReplacement, InterpolationRemoval, RegexReplacement, ArgumentSwap, JumpValue, RescueRemoval,
82
- WhenRemoval, UnaryRemoval
88
+ EnsureRemoval, WhenRemoval, UnaryRemoval
83
89
  ].freeze
84
90
 
85
91
  def initialize(arid: DEFAULT_ARID, operators: [], exclude_operators: [])
@@ -0,0 +1,72 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "prism"
5
+ require_relative "base"
6
+
7
+ module Saboteur
8
+ module MutationSource
9
+ # Drops a defensive copy (x.dup, x.clone becomes x) or makes a deep copy
10
+ # shallow (x.deep_dup becomes x.dup). Both let a caller and a callee share
11
+ # state that the original code kept apart.
12
+ class CopyReplacement < Base
13
+ DROPPED = %i[dup clone].freeze
14
+ SHALLOWER = { deep_dup: "dup" }.freeze
15
+ SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
16
+
17
+ # Base inherits Object's empty initializer; there is no parent state to set.
18
+ def initialize(arid: []) # rubocop:disable Lint/MissingSuper
19
+ @arid = arid.map { |pattern| Regexp.new(pattern) }
20
+ end
21
+
22
+ def name
23
+ "copy_replacement"
24
+ end
25
+
26
+ def mutations_for(subject)
27
+ found = []
28
+ visit(subject.body_node, subject, found)
29
+ found
30
+ end
31
+
32
+ private
33
+
34
+ def visit(node, subject, found)
35
+ return if node.nil? || SCOPE_NODES.any? { |scope| node.is_a?(scope) }
36
+
37
+ if node.is_a?(Prism::StatementsNode)
38
+ node.body.each { |statement| visit(statement, subject, found) unless arid?(statement) }
39
+ return
40
+ end
41
+
42
+ replace_copy(node, subject, found) if node.is_a?(Prism::CallNode)
43
+ node.compact_child_nodes.each { |child| visit(child, subject, found) }
44
+ nil
45
+ end
46
+
47
+ def replace_copy(node, subject, found)
48
+ return unless node.receiver && node.block.nil? && node.arguments.nil?
49
+
50
+ edit = DROPPED.include?(node.name) ? drop(node, subject) : shallow(node, subject)
51
+ found << Mutation.new(subject_id: subject.id, operator: name, edits: [edit], source: name) if edit
52
+ end
53
+
54
+ # Receiver stays; the dot (or &.) through the end of the call goes.
55
+ def drop(node, subject)
56
+ Edit.new(file: subject.file, start_byte: node.call_operator_loc.start_offset,
57
+ end_byte: node.end_offset, replacement: "")
58
+ end
59
+
60
+ def shallow(node, subject)
61
+ replacement = SHALLOWER[node.name] or return
62
+ location = node.message_loc
63
+ Edit.new(file: subject.file, start_byte: location.start_offset, end_byte: location.end_offset,
64
+ replacement: replacement)
65
+ end
66
+
67
+ def arid?(node)
68
+ @arid.any? { |pattern| pattern.match?(node.slice) }
69
+ end
70
+ end
71
+ end
72
+ end