saboteur 0.12.0 → 0.12.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +97 -0
  3. data/README.md +16 -3
  4. data/lib/saboteur/baseline/command.rb +6 -1
  5. data/lib/saboteur/baseline/comparison.rb +8 -0
  6. data/lib/saboteur/baseline/completeness.rb +14 -4
  7. data/lib/saboteur/baseline/evidence.rb +8 -1
  8. data/lib/saboteur/baseline/run_evidence.rb +10 -1
  9. data/lib/saboteur/baseline/workflow.rb +17 -5
  10. data/lib/saboteur/cli/color_option.rb +26 -0
  11. data/lib/saboteur/cli/doctor_command.rb +1 -0
  12. data/lib/saboteur/cli/exit_status.rb +12 -7
  13. data/lib/saboteur/cli/explain_command.rb +2 -1
  14. data/lib/saboteur/cli/failure_output.rb +13 -2
  15. data/lib/saboteur/cli/help_command.rb +5 -0
  16. data/lib/saboteur/cli/history_command.rb +2 -0
  17. data/lib/saboteur/cli/output_options.rb +1 -5
  18. data/lib/saboteur/cli/run_command.rb +1 -1
  19. data/lib/saboteur/cli/run_reporting.rb +2 -10
  20. data/lib/saboteur/cli/run_validation.rb +0 -1
  21. data/lib/saboteur/cli/selection_options.rb +3 -2
  22. data/lib/saboteur/cli/survivors_command.rb +81 -24
  23. data/lib/saboteur/cli/whole_suite_map_lookup.rb +33 -0
  24. data/lib/saboteur/cli.rb +6 -2
  25. data/lib/saboteur/engine/coverage_store.rb +7 -4
  26. data/lib/saboteur/engine/diagnostics.rb +27 -5
  27. data/lib/saboteur/engine/pipelines.rb +9 -2
  28. data/lib/saboteur/engine/survivor_confirmation.rb +75 -11
  29. data/lib/saboteur/engine/test_guess.rb +2 -1
  30. data/lib/saboteur/engine.rb +37 -18
  31. data/lib/saboteur/execution/runner.rb +3 -0
  32. data/lib/saboteur/gates.rb +1 -1
  33. data/lib/saboteur/help.rb +5 -5
  34. data/lib/saboteur/integration/rspec.rb +30 -0
  35. data/lib/saboteur/reporter/default_findings.rb +1 -1
  36. data/lib/saboteur/reporter/findings.rb +10 -1
  37. data/lib/saboteur/reporter/gap_kinds.rb +6 -0
  38. data/lib/saboteur/reporter/html.rb +12 -1
  39. data/lib/saboteur/reporter/next_steps.rb +3 -1
  40. data/lib/saboteur/reporter/score_table.rb +6 -11
  41. data/lib/saboteur/reporter/stryker.rb +7 -2
  42. data/lib/saboteur/reporter/survivor_classification.rb +15 -0
  43. data/lib/saboteur/reporter/survivor_list.rb +6 -6
  44. data/lib/saboteur/reporter/terminal.rb +6 -3
  45. data/lib/saboteur/reporter/unconfirmed_section.rb +44 -0
  46. data/lib/saboteur/result.rb +39 -2
  47. data/lib/saboteur/selection/coverage_map.rb +11 -1
  48. data/lib/saboteur/version.rb +1 -1
  49. data/lib/saboteur.rb +4 -0
  50. metadata +4 -1
@@ -2,8 +2,10 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  require "json"
5
+ require "stringio"
5
6
 
6
7
  require_relative "../survivor_guidance"
8
+ require_relative "../reporter/palette"
7
9
  require_relative "../reporter/survivor_hints"
8
10
 
9
11
  module Saboteur
@@ -17,6 +19,11 @@ module Saboteur
17
19
  # Where `saboteur run` writes the JSON report unless configured otherwise.
18
20
  DEFAULT_REPORT = Config::DEFAULTS["json_report"]
19
21
 
22
+ # survivors owns its usage line (#1150): the report path is optional
23
+ # because the run always writes the same default.
24
+ SURVIVORS_USAGE = "Usage: saboteur survivors [options] [REPORT] - REPORT defaults to " \
25
+ ".saboteur/report.json"
26
+
20
27
  private
21
28
 
22
29
  def survivors_command
@@ -27,28 +34,53 @@ module Saboteur
27
34
  end
28
35
 
29
36
  def survivors_command_inner
30
- parser = OptionParser.new do |opts|
31
- opts.on("--ranked", "Review-priority order: the report's ranked findings first")
32
- opts.on("--file SUBSTR", "Only survivors whose path contains SUBSTR")
33
- opts.on("--operator NAME", "Only survivors of operator NAME")
34
- opts.on("--triage LABEL", "Only survivors classified LABEL (for example boundary)")
35
- opts.on("--show DIGEST", "Diff, replay command and ignore template for one survivor")
36
- end
37
37
  options = {}
38
- args = parser.parse!(@argv, into: options)
38
+ parser = survivors_parser(options)
39
+ args = parse_survivors_args(parser, options)
40
+ if options[:help]
41
+ @out.puts(parser)
42
+ return EXIT_OK
43
+ end
39
44
  path = args.shift || DEFAULT_REPORT
40
- raise UsageError, "unexpected arguments: #{args.join(' ')}" if args.any?
45
+ raise UsageError, "unexpected arguments: #{args.join(' ')}\n#{parser}" if args.any?
41
46
 
42
47
  report = load_survivor_report(path)
43
- browser = SurvivorsBrowser.new(report)
48
+ browser = SurvivorsBrowser.new(report, palette: Reporter::Palette.for(@out, mode: options.fetch(:color, :auto)))
44
49
  if options[:show]
45
50
  @out.puts(browser.show(options[:show]))
46
51
  else
47
- @out.puts(browser.list(**options.slice(:ranked, :file, :operator, :triage)))
52
+ # Explicit keywords, and list takes no catch-all (#1267): a new
53
+ # parser option that never reaches the browser raises here instead
54
+ # of being swallowed. --ranked is the documented compat no-op.
55
+ @out.puts(browser.list(file: options[:file], operator: options[:operator], triage: options[:triage]))
48
56
  end
49
57
  EXIT_OK
50
58
  end
51
59
 
60
+ # The same colour flags as run (#1150), plus the browser filters.
61
+ def survivors_parser(options)
62
+ OptionParser.new do |opts|
63
+ opts.banner = SURVIVORS_USAGE
64
+ opts.on("--ranked", "Review-priority order (the default): the terminal report's ordering")
65
+ opts.on("--file SUBSTR", "Only survivors whose path contains SUBSTR")
66
+ opts.on("--operator NAME", "Only survivors of operator NAME")
67
+ opts.on("--triage LABEL", "Only survivors classified LABEL (for example boundary)")
68
+ opts.on("--show DIGEST", "Diff, replay command and ignore template for one survivor")
69
+ opts.on("--color [MODE]", %w[always never auto],
70
+ "Colour the output: always, never, or auto when MODE is omitted") do |mode|
71
+ options[:color] = (mode || :always).to_sym
72
+ end
73
+ opts.on("--no-color", "Alias for --color=never") { options[:color] = :never }
74
+ opts.on("--help", "Show this help") { options[:help] = true }
75
+ end
76
+ end
77
+
78
+ def parse_survivors_args(parser, options)
79
+ parser.parse!(@argv, into: options)
80
+ rescue OptionParser::ParseError => e
81
+ raise UsageError, "#{e.message}\n#{parser}"
82
+ end
83
+
52
84
  def load_survivor_report(path)
53
85
  unless File.file?(path)
54
86
  hint = path == DEFAULT_REPORT ? "; run saboteur first, or pass a report path" : ""
@@ -64,14 +96,21 @@ module Saboteur
64
96
  # Reads the report's files section (the complete mutant list), not the
65
97
  # capped findings, so survivors beyond the findings limit still list.
66
98
  class SurvivorsBrowser
67
- def initialize(report)
99
+ # Sibling digests named on a collapse line; the rest are a count - the
100
+ # same shape the terminal report's collapse uses (#1150).
101
+ SIBLING_DIGEST_CAP = 3
102
+
103
+ def initialize(report, palette: Reporter::Palette.for(StringIO.new))
68
104
  @report = report
69
105
  @findings = report.dig("saboteur", "findings") || {}
106
+ @palette = palette
70
107
  end
71
108
 
72
- def list(ranked: false, file: nil, operator: nil, triage: nil)
73
- survivors = filtered(file, operator, triage)
74
- survivors = ranked_order(survivors) if ranked
109
+ # One ordering everywhere (#1150): the terminal report's ranked order
110
+ # is the default, exactly as the findings carry it; --ranked stays
111
+ # accepted for compatibility.
112
+ def list(file: nil, operator: nil, triage: nil)
113
+ survivors = ranked_order(filtered(file, operator, triage))
75
114
  return "no survivors#{filters_note(file, operator, triage)}" if survivors.empty?
76
115
 
77
116
  [header(survivors.size, file, operator, triage), *rows(survivors)]
@@ -160,19 +199,32 @@ module Saboteur
160
199
  def header(size, file, operator, triage)
161
200
  text = "#{size} survivor#{'s' unless size == 1}#{filters_note(file, operator, triage)}"
162
201
  truncated = @findings["survivorsTruncated"].to_i
163
- return text unless truncated.positive?
164
-
165
- "#{text} (the report's ranked findings name only the first #{@findings['limit'] || 10})"
202
+ if truncated.positive?
203
+ text = "#{text} (the report's ranked findings name only the first #{@findings['limit'] || 10})"
204
+ end
205
+ @palette.paint(:head, text)
166
206
  end
167
207
 
208
+ # The terminal collapse: sibling mutants on one statement fold into the
209
+ # first row, which names up to SIBLING_DIGEST_CAP of their digests.
168
210
  def rows(survivors)
169
- survivors.each_with_index.map do |s, i|
170
- format("%3<i>d. %<path>s:%<line>d %-28<operator>s %<digest>s %<classification>s",
171
- i: i + 1, path: s[:path], line: s[:line], operator: s[:operator], digest: s[:digest],
172
- classification: s[:classification] || "-")
211
+ survivors.group_by { |s| [s[:path], s[:line]] }.values.each_with_index.map do |group, i|
212
+ first, *siblings = group
213
+ format("%3<i>d. %<path>s:%<line>d %-28<operator>s %<digest>s %<classification>s%<suffix>s",
214
+ i: i + 1, path: first[:path], line: first[:line], operator: first[:operator],
215
+ digest: first[:digest], classification: first[:classification] || "-",
216
+ suffix: siblings_suffix(siblings))
173
217
  end
174
218
  end
175
219
 
220
+ def siblings_suffix(siblings)
221
+ return "" if siblings.empty?
222
+
223
+ named = siblings.first(SIBLING_DIGEST_CAP).map { |s| s[:digest] }.join(", ")
224
+ rest = siblings.size - SIBLING_DIGEST_CAP
225
+ " (+#{siblings.size} here: #{named}#{", +#{rest} more" if rest.positive?})"
226
+ end
227
+
176
228
  def ignore_entry(survivor, subject)
177
229
  block = if subject
178
230
  SurvivorGuidance.ignore_block(survivor[:digest], subject)
@@ -192,12 +244,17 @@ module Saboteur
192
244
  subject: m["subject"], covered: m["coveredBy"] || [], source: entry["source"] }
193
245
  end
194
246
  end
195
- survivors.sort_by { |s| [s[:path], s[:line], s[:digest]] }
247
+ survivors
196
248
  end
197
249
  end
198
250
 
251
+ # Ranked order straight from the report: the uncapped findings "order"
252
+ # list first, with the capped findings as the fallback for older
253
+ # reports; anything unknown trails in file order.
199
254
  def ranked_order(survivors)
200
- rank = @findings.fetch("survivors", []).each_with_index.to_h { |f, i| [f["digest"], i] }
255
+ order = @findings["order"] ||
256
+ (@findings.fetch("survivors", []) + @findings.fetch("accepted", [])).map { |f| f["digest"] }
257
+ rank = order.each_with_index.to_h
201
258
  survivors.sort_by { |s| [rank[s[:digest]] || Float::INFINITY, -s[:covered].size, s[:path], s[:line], s[:digest]] }
202
259
  end
203
260
 
@@ -0,0 +1,33 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # The whole-suite coverage map a finished run hands to survivor
7
+ # presentation, and the notice for a scoped run that has none.
8
+ module WholeSuiteMapLookup
9
+ private
10
+
11
+ # No engine or no cache: no whole-suite map can exist, so the survivor
12
+ # block keeps its legacy covered-by line over an impossible build (review C1).
13
+ def whole_suite_map_for(engine, report)
14
+ warn_survivors_unconfirmed(report, engine)
15
+ return engine.whole_suite_coverage_map if engine&.whole_suite_map_lookup?
16
+
17
+ Reporter::SurvivorList::NO_WHOLE_SUITE_LOOKUP
18
+ end
19
+
20
+ # A scoped run's survivors may be pinned by a spec it never loaded.
21
+ # With no whole-suite map to check, say so once, even under --quiet
22
+ # (#1286). Verdicts, score and exit code are unchanged.
23
+ def warn_survivors_unconfirmed(report, engine)
24
+ return if report.unignored_survivors.empty?
25
+ return unless engine&.whole_suite_map_lookup? && engine.scoped_selection?
26
+ return if engine.whole_suite_coverage_map
27
+
28
+ @err.puts "saboteur: survivors unconfirmed: no whole-suite coverage map; " \
29
+ "run `saboteur run --all` to build one"
30
+ end
31
+ end
32
+ end
33
+ end
data/lib/saboteur/cli.rb CHANGED
@@ -13,7 +13,9 @@ require_relative "cli/error_cause"
13
13
  require_relative "cli/exit_status"
14
14
  require_relative "cli/baseline_gate"
15
15
  require_relative "preview"
16
+ require_relative "cli/color_option"
16
17
  require_relative "cli/run_reporting"
18
+ require_relative "cli/whole_suite_map_lookup"
17
19
  require_relative "cli/survivors_command"
18
20
  require_relative "cli/explain_command"
19
21
  require_relative "cli/source_set"
@@ -43,7 +45,9 @@ module Saboteur
43
45
  include Baseline::Command
44
46
  include ExitStatus
45
47
  include BaselineGate
48
+ include ColorOption
46
49
  include RunReporting
50
+ include WholeSuiteMapLookup
47
51
  include SurvivorsCommand
48
52
  include ExplainCommand
49
53
  include SourceSet
@@ -100,7 +104,7 @@ module Saboteur
100
104
  end
101
105
 
102
106
  def call
103
- command = @argv.shift
107
+ command = @command = @argv.shift
104
108
  case command
105
109
  when "run" then run_command
106
110
  when "baseline" then baseline
@@ -109,7 +113,7 @@ module Saboteur
109
113
  when "doctor" then doctor_command
110
114
  when "badge", "trend" then history_entry(command)
111
115
  when "version", "--version" then @out.puts(VERSION) || EXIT_OK
112
- when "help", "--help", "-h" then show_help(@argv.shift)
116
+ when "help", "--help", "-h" then help_command
113
117
  when "agent-skill" then @out.print(AgentSkill::TEXT) || EXIT_OK
114
118
  else preset_or_usage_error(command)
115
119
  end
@@ -18,7 +18,10 @@ module Saboteur
18
18
  CACHE_INPUTS = %w[integration require rspec_args rails tests coverage_recorder].freeze
19
19
  # Names the freshest whole-suite map build; read as an untrusted hint:
20
20
  # only a bare 64-hex key joins the cache dir (#427).
21
- WHOLE_SUITE_POINTER = "whole-suite-map"
21
+ # Renamed from "whole-suite-map" in 0.12.2: older versions also wrote
22
+ # the pointer for maps built from inferred specs, which are not whole
23
+ # suites, so their pointers are never read (#1286).
24
+ WHOLE_SUITE_POINTER = "whole-suite-map-v2"
22
25
  LOCKFILES = %w[Gemfile.lock gems.locked].freeze
23
26
  LOG = "coverage-build.log"
24
27
  # RSpec rewrites this on every run (example_status_persistence_file_path),
@@ -65,7 +68,7 @@ module Saboteur
65
68
  return serve_cached(path, cached, files, preloaded) if cached
66
69
 
67
70
  note = miss_note(files)
68
- @diagnostics.inform(note.message(scoped: !@config["tests"].empty?)) if @config["cache"]
71
+ @diagnostics.notice(note.message(scoped: !@config["tests"].empty?)) if @config["cache"]
69
72
  started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
70
73
  map = fresh_map(files, note, preloaded)
71
74
  build_seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
@@ -237,7 +240,7 @@ module Saboteur
237
240
  return build_parts(files, parts) if parts.size > 1
238
241
 
239
242
  scope = @config["tests"].empty? ? "the whole suite" : "the selected specs"
240
- @diagnostics.inform("building coverage map (runs #{scope} once)")
243
+ @diagnostics.notice("building coverage map (runs #{scope} once)")
241
244
  # The suite child forked next inherits this heap; pack it first, to
242
245
  # reduce later copying of shared pages.
243
246
  Execution::ForkReadiness.prepare_heap
@@ -267,7 +270,7 @@ module Saboteur
267
270
  total, counts = usage_checked { check_parts_cover_suite(parts) }
268
271
  saved = saved_parts(files, parts.size, counts)
269
272
  todo = saved.each_index.select { |index| saved[index].nil? }
270
- @diagnostics.inform(parts_note(parts.size, todo.size))
273
+ @diagnostics.notice(parts_note(parts.size, todo.size))
271
274
  built = todo.empty? ? [] : run_parts(files, parts, todo)
272
275
  todo.zip(built) { |index, map| saved[index] = map }
273
276
  map = combine_saved(saved, total, files, parts.size)
@@ -17,6 +17,12 @@ module Saboteur
17
17
  # be watched: the heartbeat then adds the count, the percentage and an
18
18
  # estimate of the time left to its elapsed-time line.
19
19
  class ProgressHeartbeat
20
+ # --quiet substitute: responds to the supervisor's calls, prints nothing.
21
+ NULL = Object.new.tap do |noop|
22
+ def noop.watch(*) = self
23
+ def noop.tick = nil
24
+ end
25
+
20
26
  COUNT = %r{(\d+)/([1-9]\d*) examples finished}
21
27
  TAIL_BYTES = 4096
22
28
 
@@ -91,9 +97,10 @@ module Saboteur
91
97
  attr_reader :warnings, :live_announced, :invalid_mutations, :generation_events, :omissions, :discovery_failures,
92
98
  :hazard_suppressions, :inline_suppressions, :verbose, :quiet_start
93
99
 
94
- def initialize(err, verbose: false, palette: nil, quiet_start: true)
100
+ def initialize(err, verbose: false, palette: nil, quiet_start: true, quiet: false)
95
101
  @err = err
96
102
  @verbose = verbose
103
+ @quiet = quiet
97
104
  @quiet_start = quiet_start
98
105
  @palette = palette || Reporter::Palette.for(err)
99
106
  @warnings = []
@@ -180,10 +187,17 @@ module Saboteur
180
187
  @err.puts("saboteur: #{Text.display_safe_lines(message)}")
181
188
  end
182
189
 
190
+ # First-run explanations (#1264): the --quiet contract is findings,
191
+ # score and verdict, so these stay off stderr under it and print
192
+ # exactly as before otherwise.
193
+ def notice(message)
194
+ inform(message) unless @quiet
195
+ end
196
+
183
197
  # Chatter the default view folds into the quiet-start lines;
184
198
  # --verbose prints it byte-for-byte as before.
185
199
  def detail(message)
186
- inform(message) if @verbose || !@quiet_start
200
+ inform(message) if !@quiet && (@verbose || !@quiet_start)
187
201
  end
188
202
 
189
203
  # A selection explanation the default view carries as one dim
@@ -199,7 +213,7 @@ module Saboteur
199
213
  # The quiet start: three lines, then progress. --verbose keeps
200
214
  # today's chatter instead and never prints these.
201
215
  def quiet_start_header(scope:, seed:, operators:)
202
- return if @verbose || !@quiet_start
216
+ return if @quiet || @verbose || !@quiet_start
203
217
 
204
218
  dot = @palette.glyph(:middot)
205
219
  line = "saboteur #{Saboteur::VERSION} #{dot} #{scope} #{dot} seed #{seed} #{dot} " \
@@ -208,7 +222,7 @@ module Saboteur
208
222
  end
209
223
 
210
224
  def quiet_baseline(passes:, elapsed:, tests:, cached:)
211
- return if @verbose || !@quiet_start
225
+ return if @quiet || @verbose || !@quiet_start
212
226
 
213
227
  check = @palette.paint(:pass, @palette.glyph(:check))
214
228
  suffix = cached ? " (cached)" : ""
@@ -219,7 +233,7 @@ module Saboteur
219
233
  end
220
234
 
221
235
  def quiet_inference(spec_count)
222
- return if spec_count.nil? || @verbose || !@quiet_start
236
+ return if spec_count.nil? || @quiet || @verbose || !@quiet_start
223
237
 
224
238
  files = spec_count == 1 ? "spec file" : "spec files"
225
239
  @err.puts(@palette.paint(:dim, "spec inference: #{spec_count} #{files} (details: --verbose)"))
@@ -252,6 +266,10 @@ module Saboteur
252
266
  # fork time. Prints "<label>: N s elapsed" about every interval
253
267
  # seconds, so a long phase does not look stuck.
254
268
  def progress_heartbeat(label, interval: 30)
269
+ # --quiet: the contract is findings, score and verdict - no elapsed
270
+ # chatter; the heartbeat's ticks become no-ops.
271
+ return ProgressHeartbeat::NULL if @quiet
272
+
255
273
  ProgressHeartbeat.new(self, label, interval: interval)
256
274
  end
257
275
 
@@ -261,6 +279,8 @@ module Saboteur
261
279
  # The count lives in one file the workers inherit, so every worker adds
262
280
  # to the same number.
263
281
  def start_progress(total)
282
+ return if @quiet
283
+
264
284
  @progress_total = total
265
285
  @progress_started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
266
286
  @progress_file = Tempfile.new("saboteur-progress")
@@ -271,6 +291,8 @@ module Saboteur
271
291
  end
272
292
 
273
293
  def progress(result)
294
+ return if @quiet
295
+
274
296
  mark = MARKS.fetch(result.status, "?")
275
297
  return plain_progress(painted_mark(mark)) unless @progress_file
276
298
 
@@ -104,9 +104,16 @@ module Saboteur
104
104
  # "Full run" for ignore staleness (#1206): no --since diff, no --subject
105
105
  # filter, and no narrowed include (explicit file arguments replace the
106
106
  # configured include) - only then was every in-scope subject examined, so
107
- # an entry whose subject never appears really points at gone code.
107
+ # an entry whose subject never appears really points at gone code. The
108
+ # include nobody set counts as unscoped whether it is the generic
109
+ # default or the Rails one a config-less app gets (#1255).
108
110
  def unscoped_full_run?
109
- !config["since"] && config["subjects"].empty? && config["include"] == Config::DEFAULTS["include"]
111
+ !config["since"] && config["subjects"].empty? && default_include?
112
+ end
113
+
114
+ def default_include?
115
+ config["include"] == Config::DEFAULTS["include"] ||
116
+ (config["include"] == Config::RAILS_INCLUDE && !config.explicit?("include"))
110
117
  end
111
118
 
112
119
  def external_command? = !config["test_command"].to_s.empty?
@@ -15,13 +15,14 @@ module Saboteur
15
15
  # confirmation_budget caps the whole pass.
16
16
  MAX_SPECS_PER_SURVIVOR = 8
17
17
 
18
- def initialize(config:, map:, scheduler:, diagnostics:, budget_exceeded:, cache: nil)
18
+ def initialize(config:, map:, scheduler:, diagnostics:, budget_exceeded:, cache: nil, load_specs: ->(_files) {})
19
19
  @config = config
20
20
  @map = map
21
21
  @scheduler = scheduler
22
22
  @diagnostics = diagnostics
23
23
  @budget_exceeded = budget_exceeded
24
24
  @cache = cache
25
+ @load_specs = load_specs
25
26
  end
26
27
 
27
28
  def call(results, ignore_list:, early_stop: nil)
@@ -33,19 +34,60 @@ module Saboteur
33
34
  "stopped early (#{early_stop.reason})")
34
35
  end
35
36
  @remaining = @config["confirmation_budget"]
37
+ @map_silent = 0
36
38
  results.map { |result| survivors.include?(result) ? confirm(result) : result }
37
39
  .tap { |confirmed| summarize(confirmed) }
40
+ .tap { warn_map_silent }
38
41
  end
39
42
 
40
43
  private
41
44
 
45
+ # A crash while confirming one survivor downgrades it to unconfirmed
46
+ # with one warning; it never ends the run (#1140).
42
47
  def confirm(result)
48
+ confirm_checked(result)
49
+ rescue RefusedSpecPath => e
50
+ warn_refused(e)
51
+ result.with_confirmation("unconfirmed_map(a covering spec from the whole-suite map was refused)")
52
+ rescue StandardError => e
53
+ warn_failed(result, e)
54
+ unconfirmed(result, "confirmation failed: #{e.class}")
55
+ end
56
+
57
+ # The map named a spec that is not a spec file inside the project: the
58
+ # map cannot be trusted, so its survivors stay counted (#1286).
59
+ def warn_refused(error)
60
+ return if @warned_refused
61
+
62
+ @warned_refused = true
63
+ @diagnostics.warn("survivor confirmation refused the whole-suite coverage map: " \
64
+ "#{error.message.lines.first.strip}; its survivors stay counted; " \
65
+ "run `saboteur run --all` to rebuild it")
66
+ end
67
+
68
+ # One line per run, however many survivors hit the same failure.
69
+ def warn_failed(result, error)
70
+ return if @warned_failure
71
+
72
+ @warned_failure = true
73
+ @diagnostics.warn("survivor confirmation failed for #{result.subject.id} #{result.digest} " \
74
+ "(#{error.class}: #{error.message.lines.map(&:strip).reject(&:empty?).first}); " \
75
+ "its survivors stay unconfirmed, outside the score")
76
+ end
77
+
78
+ def confirm_checked(result)
43
79
  return unconfirmed(result, "the confirmation budget is spent") if @remaining <= 0
44
80
  return unconfirmed(result, "the --max-duration budget is spent") if @budget_exceeded.call
45
81
 
46
- tests = unrun_covering_tests(result)
47
- return unconfirmed(result, "no covering tests") if tests.empty?
82
+ # Nothing in the whole-suite map covers the edited lines (the tree
83
+ # drifted since the map was built): the map cannot confirm anything.
84
+ covering = covering_tests(result)
85
+ return map_silent(result) if covering.empty?
48
86
 
87
+ tests = unrun_covering_tests(result, covering)
88
+ return result.with_confirmation("confirmed") if tests.empty?
89
+
90
+ load_unselected_specs(tests)
49
91
  outcome = run_confirmation(result, tests)
50
92
  @remaining -= outcome.runtime || 0
51
93
  case outcome.status
@@ -58,8 +100,14 @@ module Saboteur
58
100
  # The whole-suite map's covering tests for the mutant's edited lines
59
101
  # (branch evidence intersected, mirroring the scheduler), minus the
60
102
  # tests the mutant already ran, fastest first, capped.
61
- def unrun_covering_tests(result)
62
- fastest_first(covering_tests(result) - (result.executed || result.covered_by)).first(MAX_SPECS_PER_SURVIVOR)
103
+ def unrun_covering_tests(result, covering)
104
+ fastest_first(covering - (result.executed || result.covered_by)).first(MAX_SPECS_PER_SURVIVOR)
105
+ end
106
+
107
+ # The covering specs may live in files this run never loaded; the
108
+ # forked confirmation child can only run examples already in its world.
109
+ def load_unselected_specs(tests)
110
+ @load_specs.call(tests.map { |id| id.sub(/\[[\d:]+\]\z/, "") })
63
111
  end
64
112
 
65
113
  def covering_tests(result)
@@ -86,11 +134,12 @@ module Saboteur
86
134
  # becomes an error; a check the budgets never let run comes back
87
135
  # unverified - outside the score and never cached.
88
136
  def checked_kill(result, tests, outcome)
89
- kill = result.with_confirmation("confirmed", status: :killed, reason: outcome.message,
90
- killed_by: outcome.failed_ids, covered_by: tests,
91
- kill_cause: outcome.kill_cause,
92
- runtime: (result.runtime || 0) + (outcome.runtime || 0),
93
- executed: (result.executed || []) + outcome.executed_ids)
137
+ kill = result.with_confirmation(
138
+ "killed_by_confirmation", status: :killed, reason: outcome.message, killed_by: outcome.failed_ids,
139
+ covered_by: tests, kill_cause: outcome.kill_cause,
140
+ runtime: (result.runtime || 0) + (outcome.runtime || 0),
141
+ executed: (result.executed || []) + outcome.executed_ids
142
+ )
94
143
  checked = VerdictCheck.new(run_unchanged: unchanged_runner(result.subject),
95
144
  baseline_timeout: @config["baseline_timeout"]).call([kill]).first
96
145
  return checked unless checked.status == :killed
@@ -108,7 +157,7 @@ module Saboteur
108
157
  # (#1079). ResultCache#record replaces the subject+mutation entry, so
109
158
  # the stale survivor verdict cannot win a later lookup.
110
159
  def record_kill(result, kill)
111
- @cache&.record(kill.with_confirmation("confirmed", covered_by: covering_tests(result)))
160
+ @cache&.record(kill.with_confirmation("killed_by_confirmation", covered_by: covering_tests(result)))
112
161
  end
113
162
 
114
163
  # The unchanged-method re-run VerdictCheck checks a kill against, in
@@ -129,6 +178,13 @@ module Saboteur
129
178
  end
130
179
  end
131
180
 
181
+ # The map says nothing about the edited lines (the tree drifted since it
182
+ # was built): the survivor stays counted and the pass says so once.
183
+ def map_silent(result)
184
+ @map_silent += 1
185
+ result.with_confirmation("unconfirmed_map(no covering tests in the whole-suite map)")
186
+ end
187
+
132
188
  def unconfirmed(result, reason)
133
189
  result.with_confirmation("unconfirmed(#{reason})")
134
190
  end
@@ -148,6 +204,14 @@ module Saboteur
148
204
  durations.empty? ? nil : durations.sum / durations.size
149
205
  end
150
206
 
207
+ def warn_map_silent
208
+ return if @map_silent.zero?
209
+
210
+ @diagnostics.warn("survivor confirmation: the whole-suite coverage map covers none of the edited lines of " \
211
+ "#{@map_silent} survivor#{'s' unless @map_silent == 1}, so they stay counted; " \
212
+ "run `saboteur run --all` to rebuild it")
213
+ end
214
+
151
215
  def summarize(results)
152
216
  counts = results.filter_map(&:confirmation).tally
153
217
  return if counts.empty?
@@ -116,7 +116,8 @@ module Saboteur
116
116
  # Why the subjectless files needed no spec.
117
117
  def subjectless_message(since:)
118
118
  detail = since ? "no changed mutation subjects" : "no mutation subjects"
119
- "inferred specs: no spec is required for #{names(subjectless)}: they have #{detail}"
119
+ who = subjectless.size == 1 ? "it has" : "they have"
120
+ "inferred specs: no spec is required for #{names(subjectless)}: #{who} #{detail}"
120
121
  end
121
122
 
122
123
  # The per-file warning when the run proceeds with the mapped