saboteur 0.8.0 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +115 -0
  3. data/README.md +34 -6
  4. data/lib/saboteur/attribute_finder.rb +113 -0
  5. data/lib/saboteur/cli/presets.rb +41 -1
  6. data/lib/saboteur/cli/run_budget_options.rb +3 -1
  7. data/lib/saboteur/cli/source_set.rb +0 -18
  8. data/lib/saboteur/cli.rb +4 -2
  9. data/lib/saboteur/config.rb +4 -1
  10. data/lib/saboteur/config_file.rb +12 -8
  11. data/lib/saboteur/custom_map.rb +12 -1
  12. data/lib/saboteur/dirty_tree_warning.rb +5 -3
  13. data/lib/saboteur/doctor.rb +4 -0
  14. data/lib/saboteur/engine/baseline_guard.rb +76 -10
  15. data/lib/saboteur/engine/boot.rb +7 -3
  16. data/lib/saboteur/engine/bytecode_equivalence.rb +17 -1
  17. data/lib/saboteur/engine/candidate_selection.rb +1 -0
  18. data/lib/saboteur/engine/coverage_store.rb +25 -3
  19. data/lib/saboteur/engine/diagnostics.rb +1 -1
  20. data/lib/saboteur/engine/gated_run.rb +3 -1
  21. data/lib/saboteur/engine/load_coverage.rb +18 -0
  22. data/lib/saboteur/engine/pipelines.rb +8 -0
  23. data/lib/saboteur/engine/run_budget.rb +5 -2
  24. data/lib/saboteur/engine/sandbox_policy.rb +21 -3
  25. data/lib/saboteur/engine/scheduler.rb +32 -24
  26. data/lib/saboteur/engine/selection_announcement.rb +8 -3
  27. data/lib/saboteur/engine/subject_selection.rb +115 -31
  28. data/lib/saboteur/engine/test_guess.rb +106 -16
  29. data/lib/saboteur/engine.rb +3 -2
  30. data/lib/saboteur/execution/external_runner.rb +18 -0
  31. data/lib/saboteur/execution/fiddle_loader.rb +34 -0
  32. data/lib/saboteur/execution/marker_sweep.rb +2 -1
  33. data/lib/saboteur/execution/process_table.rb +7 -3
  34. data/lib/saboteur/execution/reevaluator.rb +7 -0
  35. data/lib/saboteur/execution/sandbox.rb +10 -4
  36. data/lib/saboteur/execution/seatbelt.rb +10 -3
  37. data/lib/saboteur/execution/supervisor.rb +2 -1
  38. data/lib/saboteur/execution/test_command_shape.rb +149 -0
  39. data/lib/saboteur/frameworks/sqlite_memory_database.rb +32 -2
  40. data/lib/saboteur/integration/rspec.rb +99 -11
  41. data/lib/saboteur/mutation_source/argument_swap.rb +54 -0
  42. data/lib/saboteur/mutation_source/builtin.rb +3 -1
  43. data/lib/saboteur/mutation_source/hazard_guard.rb +94 -13
  44. data/lib/saboteur/mutation_source/operator_examples.rb +1 -0
  45. data/lib/saboteur/mutation_source/operator_registry.rb +1 -0
  46. data/lib/saboteur/preview.rb +5 -0
  47. data/lib/saboteur/reporter/agent_incomplete.rb +20 -5
  48. data/lib/saboteur/reporter/terminal.rb +16 -1
  49. data/lib/saboteur/selection/coverage_map.rb +17 -2
  50. data/lib/saboteur/version.rb +1 -1
  51. data/lib/saboteur.rb +1 -0
  52. metadata +5 -15
@@ -24,7 +24,7 @@ module Saboteur
24
24
  # coverage build when the cache is cold, so the first timed pass can
25
25
  # price the rest (#260).
26
26
  def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, full_passes: 2, between_passes: nil,
27
- timeout_cap: nil)
27
+ timeout_cap: nil, run_started: nil)
28
28
  @config = config
29
29
  @integration = integration
30
30
  @hooks = hooks
@@ -33,6 +33,7 @@ module Saboteur
33
33
  @full_passes = full_passes
34
34
  @between_passes = between_passes
35
35
  @timeout_cap = timeout_cap
36
+ @run_started = run_started
36
37
  @completed_passes = 0
37
38
  end
38
39
 
@@ -51,10 +52,7 @@ module Saboteur
51
52
  started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
52
53
  data = check(number)
53
54
  announce_first_pass_cost(number, started)
54
- unless data["passed"] && data["failed"].empty?
55
- refuse(number,
56
- with_hint("the suite failed: #{Array(data['failed']).join(', ')}", data))
57
- end
55
+ refuse_suite_failure(number, data) unless data["passed"] && data["failed"].empty?
58
56
  executed = data["executed"]
59
57
  refuse(number, "executed tests do not match the loaded suite") unless
60
58
  executed.sort == expected.sort
@@ -67,6 +65,16 @@ module Saboteur
67
65
 
68
66
  private
69
67
 
68
+ def refuse_failed_suite(number, data)
69
+ outside = data["failed"].empty?
70
+ reason = if outside
71
+ "the suite errored outside any example; see the log tail"
72
+ else
73
+ "the suite failed: #{data['failed'].join(', ')}"
74
+ end
75
+ refuse(number, with_hint(reason, data), tail: outside)
76
+ end
77
+
70
78
  # A spent --max-duration budget skips the remaining passes - including
71
79
  # the first, when boot alone already spent it (#245).
72
80
  def budget_spent?(_index)
@@ -95,9 +103,10 @@ module Saboteur
95
103
 
96
104
  coverage_pending = @full_passes > @config["baseline_passes"]
97
105
  remaining = @full_passes - number - (coverage_pending ? 1 : 0)
106
+ elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).round
107
+ warn_budget_projection(elapsed, remaining + (coverage_pending ? 1 : 0))
98
108
  return if remaining.zero? && !coverage_pending
99
109
 
100
- elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).round
101
110
  work = []
102
111
  work << "the remaining #{remaining} full #{remaining == 1 ? 'pass' : 'passes'}" if remaining.positive?
103
112
  work << "the coverage map build" if coverage_pending
@@ -106,6 +115,28 @@ module Saboteur
106
115
  "#{work.join(' and ')} at that rate")
107
116
  end
108
117
 
118
+ # The first timed pass also prices the run against --max-duration:
119
+ # boot and this pass are spent, the remaining passes and a cold
120
+ # coverage build cost about that much again at that rate, and only
121
+ # what is left feeds mutants. Warn while a Ctrl-C is still cheap -
122
+ # the refusal only fires once the coverage build is unaffordable
123
+ # outright (#637).
124
+ def warn_budget_projection(pass_elapsed, remaining_passes)
125
+ budget = @config["max_duration"]
126
+ return unless budget && @run_started
127
+
128
+ spent = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @run_started
129
+ projected = spent + (pass_elapsed * remaining_passes)
130
+ mutants_get = budget - projected
131
+ return if mutants_get >= pass_elapsed
132
+
133
+ verdict = mutants_get <= 0 ? "exceeds" : "barely fits"
134
+ @diagnostics.warn("projected baseline+coverage ~#{projected.round}s #{verdict} " \
135
+ "--max-duration #{budget.round}s; mutants would get " \
136
+ "~#{format('%g', [mutants_get, 0].max.round(1))}s. Raise --max-duration, pass fewer " \
137
+ "--baseline-passes, or narrow the scope")
138
+ end
139
+
109
140
  # An admitted pass still gets only the time left in the run budget:
110
141
  # without the cap it would overrun a spent --max-duration by its
111
142
  # whole coverage_timeout.
@@ -209,6 +240,41 @@ module Saboteur
209
240
  writer.close
210
241
  end
211
242
 
243
+ # A failed unchanged suite never runs mutants: warn first when the
244
+ # failure carries the Minitest signature (#626), then refuse.
245
+ def refuse_suite_failure(number, data)
246
+ warn_minitest_signature(number)
247
+ refuse_failed_suite(number, data)
248
+ end
249
+
250
+ # Minitest::Spec suites in *_spec.rb clothing load under the RSpec
251
+ # integration and then fail the baseline with NoMethodError for
252
+ # Minitest-only methods (#626): the refusal still reads "fix the
253
+ # suite", so warn what the signature actually points at. It stays a
254
+ # warning: an RSpec suite can require minitest for a helper and
255
+ # fail for its own reason, and a heuristic refusal would
256
+ # false-positive there.
257
+ def warn_minitest_signature(number)
258
+ lines = log_lines(number)
259
+ errors = lines.grep(/NoMethodError/)
260
+ return if errors.empty? || !spec_helper_requires_minitest?
261
+
262
+ symptom = (lines.find { |line| line.include?("undefined method") } || errors.first).strip
263
+ @diagnostics.warn("the baseline failure looks like a Minitest suite running under RSpec, not a " \
264
+ "broken RSpec suite: the log shows #{errors.size} " \
265
+ "NoMethodError#{'s' if errors.size > 1} (for example: #{symptom.slice(0, EXCERPT_WIDTH)}) " \
266
+ "and spec/spec_helper.rb requires minitest. saboteur is RSpec-only, so a " \
267
+ "Minitest suite cannot be mutation tested with it.")
268
+ end
269
+
270
+ # The framework check behind the warning: spec_helper is the
271
+ # suite's own declaration of what it runs on.
272
+ def spec_helper_requires_minitest?
273
+ File.read(File.join(@config.root, "spec", "spec_helper.rb")).match?(/^\s*require\s+["']minitest/)
274
+ rescue SystemCallError
275
+ false
276
+ end
277
+
212
278
  # A child that recognises its own failure shape (an empty sqlite
213
279
  # :memory: database after the fork, #387) says how to fix it.
214
280
  def with_hint(reason, data)
@@ -230,9 +296,9 @@ module Saboteur
230
296
  "the suite passed but left #{count} child #{count == 1 ? 'process' : 'processes'} behind: #{detail}"
231
297
  end
232
298
 
233
- def refuse(number, reason, excerpt: true)
299
+ def refuse(number, reason, excerpt: true, tail: false)
234
300
  raise Error, "baseline check #{number} failed at seed #{@config['seed']}: #{reason}. " \
235
- "#{failure_excerpt(number) if excerpt}See the full log at #{log_path(number)}. " \
301
+ "#{failure_excerpt(number, tail: tail) if excerpt}See the full log at #{log_path(number)}. " \
236
302
  "Fix the suite or its setup, then run saboteur again with the same seed. " \
237
303
  "No mutants were started."
238
304
  end
@@ -240,11 +306,11 @@ module Saboteur
240
306
  # The first lines of the failure, inline, so the cause is readable
241
307
  # without opening the log: RSpec's "Failures:" section when the log has
242
308
  # one, otherwise the last lines of the log.
243
- def failure_excerpt(number)
309
+ def failure_excerpt(number, tail: false)
244
310
  lines = log_lines(number)
245
311
  return "" if lines.empty?
246
312
 
247
- start = lines.index { |line| line.start_with?("Failures:") }
313
+ start = lines.index { |line| line.start_with?("Failures:") } unless tail
248
314
  shown = start ? lines[start, EXCERPT_LINES] : lines.last(EXCERPT_LINES)
249
315
  "The log says:\n#{shown.map { |line| " #{line.slice(0, EXCERPT_WIDTH)}" }.join("\n")}\n"
250
316
  end
@@ -66,7 +66,9 @@ module Saboteur
66
66
  @rails = rails = Frameworks::Rails.new(@config, @hooks) if @config["rails"]
67
67
  rails&.before_boot
68
68
  @hooks.run(:before_boot)
69
- add_spec_to_load_path if @config["rails"]
69
+ add_spec_to_load_path
70
+ # Helpers may configure RSpec before the integration loads specs.
71
+ Integration::RSpec.new(@config).require_framework if @config["integration"] == "rspec"
70
72
  @config["require"].each { |file| require File.expand_path(file, @root) }
71
73
  integration = Integration.fetch(@config["integration"]).new(@config)
72
74
  integration.setup
@@ -78,8 +80,10 @@ module Saboteur
78
80
  private
79
81
 
80
82
  # rspec puts spec/ on the load path, so a boot file under spec/ (a
81
- # dummy app's rails_helper, say) can require its siblings. Match
82
- # that, or the same require list fails to load.
83
+ # dummy app's rails_helper, or a non-Rails gem's spec_helper that
84
+ # requires its siblings) can require its siblings. Match that for
85
+ # every project - a non-Rails gem does not add spec/ itself (#615) -
86
+ # or the same require list fails to load.
83
87
  def add_spec_to_load_path
84
88
  path = File.join(@root, "spec")
85
89
  $LOAD_PATH.unshift(path) unless $LOAD_PATH.include?(path)
@@ -42,11 +42,27 @@ module Saboteur
42
42
 
43
43
  def fingerprint(text, frozen)
44
44
  options = { frozen_string_literal: frozen }
45
- strip_positions(RubyVM::InstructionSequence.compile(text, nil, nil, 1, options).to_a)
45
+ iseq = warnings_silenced do
46
+ RubyVM::InstructionSequence.compile(text, "saboteur/bytecode-equivalence", nil, 1, options)
47
+ end
48
+ strip_positions(iseq.to_a)
46
49
  rescue SyntaxError, StandardError
47
50
  nil
48
51
  end
49
52
 
53
+ # Mutants compile one by one, and generated bodies (body_nil,
54
+ # statement_removal) trip Ruby's own void-context warnings, so a host
55
+ # suite running with -w gets one line per mutant flooding stderr
56
+ # (#592, #656). The check is internal: silence the compiler. A mutant
57
+ # that does not parse still fails the compile and runs.
58
+ def warnings_silenced
59
+ saved = $VERBOSE
60
+ $VERBOSE = nil
61
+ yield
62
+ ensure
63
+ $VERBOSE = saved
64
+ end
65
+
50
66
  # Structural equality that never coerces: an Integer is not a Float,
51
67
  # and +0.0 is not -0.0 (1.0 / +0.0 and 1.0 / -0.0 are different
52
68
  # infinities), which eql? alone would not tell apart.
@@ -66,6 +66,7 @@ module Saboteur
66
66
  # Load-only code is not "no coverage": it ran, just never inside a
67
67
  # test, and no forked test can re-run it (see CoverageMap::RAN_AT_LOAD).
68
68
  def skip_reason_for(subject)
69
+ return Selection::CoverageMap::PRELOADED if @map.preloaded?(subject.file)
69
70
  return "no coverage" unless @map.ran_at_load?(subject.file, subject.coverage_lines)
70
71
 
71
72
  Selection::CoverageMap::RAN_AT_LOAD
@@ -40,10 +40,11 @@ module Saboteur
40
40
 
41
41
  def fetch(files)
42
42
  path = cache_path(files)
43
+ preloaded = LoadCoverage.preloaded_uninstrumented(files)
43
44
  cached = cached_map(files)
44
- return refuse_empty_map(cached, files) if cached
45
+ return refuse_empty_map(cached.with_preloaded(preloaded), files, preloaded) if cached
45
46
 
46
- map = refuse_empty_map(build(files), files)
47
+ map = refuse_empty_map(build(files).with_preloaded(preloaded), files, preloaded)
47
48
  map.save(path)
48
49
  File.write(File.join(@config.path("cache_dir"), "preview.json"),
49
50
  JSON.generate(path: path, fingerprint: RunFingerprint.new(config: @config).snapshot.digest))
@@ -157,9 +158,20 @@ module Saboteur
157
158
  # these sources, so the map passes with a warning. Applies to a cached
158
159
  # map too, so a map cached before this check cannot keep a run green,
159
160
  # and the guard runs before saving, so an empty map is never cached.
160
- def refuse_empty_map(map, files)
161
+ def refuse_empty_map(map, files, preloaded)
161
162
  return map if files.empty? || !map.empty_for?(files)
162
163
 
164
+ # A file loaded before the executable's Coverage.start can never be
165
+ # instrumented, so emptiness there is a setup defect, not untested
166
+ # code: name it apart from the generic no-coverage refusal.
167
+ preloaded_empty = preloaded & files.map { |file| Selection::CoverageMap.canonical(file) }
168
+ if preloaded_empty.any?
169
+ message = preloaded_message(preloaded_empty)
170
+ raise Error, message if @config["tests"].empty?
171
+
172
+ @diagnostics.warn(message)
173
+ return map
174
+ end
163
175
  if @config["tests"].empty?
164
176
  raise Error, "Coverage recorded nothing in #{files.size} included files during a whole-suite " \
165
177
  "build. No test in the suite may exercise these files, or the setup hid the app " \
@@ -170,6 +182,16 @@ module Saboteur
170
182
  map
171
183
  end
172
184
 
185
+ def preloaded_message(files)
186
+ names = files.first(5).map { |file| file.delete_prefix("#{@root}/") }
187
+ more = files.size > 5 ? ", ..." : ""
188
+ "#{names.join(', ')}#{more} loaded before saboteur started coverage (a require ran before the " \
189
+ "executable's Coverage.start - commonly the gemspec requiring the library while bundler/setup " \
190
+ "evaluates it, or RUBYOPT -r), so no test can ever cover #{files.size > 1 ? 'them' : 'it'}. " \
191
+ "Let the test suite load these " \
192
+ "files; a gemspec should require only its version file."
193
+ end
194
+
173
195
  # Bootsnap is named only when it is loaded: blaming a library the app
174
196
  # does not use sends the reader to the wrong place.
175
197
  def bootsnap_note
@@ -74,7 +74,7 @@ module Saboteur
74
74
  DiscoveryFailure = Data.define(:file, :reason)
75
75
 
76
76
  attr_reader :warnings, :invalid_mutations, :generation_events, :omissions, :discovery_failures,
77
- :hazard_suppressions, :inline_suppressions
77
+ :hazard_suppressions, :inline_suppressions, :verbose
78
78
 
79
79
  def initialize(err, verbose: false)
80
80
  @err = err
@@ -26,7 +26,9 @@ module Saboteur
26
26
  # a gated run keep the hard stop.
27
27
  def stop_or_warn_unmapped(guess)
28
28
  return if guess.unmatched.empty?
29
- raise ConfigError, guess.unmatched_message if !config["since"] || guess.specs.empty? || gated?
29
+ if !config["since"] || guess.specs.empty? || gated?
30
+ raise ConfigError, guess.unmatched_message(verbose: @diagnostics.verbose)
31
+ end
30
32
 
31
33
  guess.unmatched.each { |file| @diagnostics.announce(guess.unmatched_warning(file)) }
32
34
  end
@@ -32,6 +32,24 @@ module Saboteur
32
32
  end
33
33
  end
34
34
 
35
+ # Include-listed files that are already loaded yet carry no coverage
36
+ # instrumentation: something required them before the executable's
37
+ # Coverage.start (a gemspec requiring the library while bundler/setup
38
+ # evaluates it, or RUBYOPT -r), so no test can ever be attributed to
39
+ # them. Read in the parent, where the snapshot still reflects the boot.
40
+ def self.preloaded_uninstrumented(files, loaded: $LOADED_FEATURES)
41
+ return [] unless Coverage.state == :suspended
42
+
43
+ instrumented = CoverageReads.result(stop: false, clear: false)
44
+ .each_key
45
+ .to_set { |path| Selection::CoverageMap.canonical(path) }
46
+ loaded_set = loaded.to_set { |path| Selection::CoverageMap.canonical(path) }
47
+ files.map { |file| Selection::CoverageMap.canonical(file) }
48
+ .uniq
49
+ .reject { |file| instrumented.include?(file) }
50
+ .select { |file| loaded_set.include?(file) }
51
+ end
52
+
35
53
  # A one-line or endless definition executes its def line even when its
36
54
  # body is never called. If any sibling on that physical line is uncalled,
37
55
  # discard its ambiguous line evidence rather than crediting every body.
@@ -1,6 +1,8 @@
1
1
  # SPDX-License-Identifier: MIT
2
2
  # frozen_string_literal: true
3
3
 
4
+ require_relative "../execution/test_command_shape"
5
+
4
6
  module Saboteur
5
7
  class Engine
6
8
  # The two run pipelines: in-process (boot, coverage, workers) and the
@@ -43,6 +45,12 @@ module Saboteur
43
45
  def call_external(ignore_list, started)
44
46
  @diagnostics.inform("external test command: #{config['test_command']} " \
45
47
  "(serial, no coverage, unsandboxed, mutants applied in a disposable copy)")
48
+ if Execution::TestCommandShape.multi_file_ruby?(config["test_command"])
49
+ @diagnostics.warn("the test command hands several test files to one ruby: ruby executes only " \
50
+ "the first, so the rest never run and their mutants silently survive (#611) - " \
51
+ "use the framework's multi-file form (bin/rails test f1 f2, rspec f1 f2) or a " \
52
+ "shell loop (for f in ...; do ruby \"$f\" || exit 1; done)")
53
+ end
46
54
  files = config.included_files
47
55
  subjects, scope = select_subjects(files)
48
56
  results = subjects.empty? ? [] : mutate_external(subjects, scope, ignore_list)
@@ -35,8 +35,11 @@ module Saboteur
35
35
 
36
36
  # Absolute monotonic deadline shared with the schedulers (forked
37
37
  # workers inherit it): a subject's remaining mutants stop once it
38
- # passes, so the overshoot after a spent budget stays under one
39
- # mutant deadline.
38
+ # passes. New mutation work never overshoots more than the fork in
39
+ # flight; a subject the deadline cut mid-list still confirms the
40
+ # kills it already measured (#588), and those checks and retries
41
+ # fork uncapped - the overshoot is bounded by the subject's
42
+ # distinct killing sequences times their full timeout.
40
43
  def budget_deadline
41
44
  config["max_duration"] && (@run_started + config["max_duration"])
42
45
  end
@@ -39,8 +39,7 @@ module Saboteur
39
39
  # same line twice (D's review, #178).
40
40
  unless @sandbox_note_given
41
41
  @sandbox_note_given = true
42
- @diagnostics.warn("sandbox is on but #{self.class.sandbox_gap(abi)}; children run with the hazard guard " \
43
- "and canary only")
42
+ @diagnostics.warn(degraded_note(abi))
44
43
  end
45
44
  nil
46
45
  end
@@ -55,11 +54,30 @@ module Saboteur
55
54
  end
56
55
 
57
56
  def self.sandbox_gap(abi)
58
- return "this kernel lacks Landlock (needs Linux 6.2+)" if abi.zero?
57
+ if abi.zero?
58
+ # Landlock is Linux-only: on macOS there is no kernel to upgrade,
59
+ # so name the platform instead of a Linux version (#667).
60
+ return "macOS has no Landlock" if Execution::Seatbelt.darwin?
61
+
62
+ return "this kernel lacks Landlock (needs Linux 6.2+)"
63
+ end
59
64
 
60
65
  "this kernel's Landlock ABI is #{abi} but ABI 3 (Linux 6.2+) is needed to restrict truncate()"
61
66
  end
62
67
 
68
+ # The degraded-run note: on macOS the gap is permanent, so point at
69
+ # the Seatbelt opt-in; on Linux the kernel wording stands alone
70
+ # (#667). Severity stays a warning - degrading confinement is never
71
+ # routine.
72
+ def degraded_note(abi)
73
+ gap = self.class.sandbox_gap(abi)
74
+ tail = "children run with the hazard guard and canary only"
75
+ return "sandbox is on but #{gap}; #{tail}" unless Execution::Seatbelt.darwin?
76
+
77
+ "sandbox is on but #{gap}; #{tail} - opt into the macOS sandbox with c.sandbox = \"seatbelt\" " \
78
+ "(docs/guides/macos-seatbelt-sandbox.md), or set c.sandbox = false to silence this"
79
+ end
80
+
63
81
  def restrict_hook
64
82
  writable = sandbox_writable_dirs
65
83
  ->(keep_fds) { Execution::Sandbox.restrict!(writable: writable, keep_fds: keep_fds) }
@@ -64,7 +64,7 @@ module Saboteur
64
64
  results = run_candidates(candidates) do |mutation, text|
65
65
  reuse(plan, mutation, text) || execute(plan, mutation, text)
66
66
  end
67
- checked(plan, results)
67
+ checked(plan, results, truncated: results.size < candidates.size)
68
68
  end
69
69
 
70
70
  # The per-mutant timeout: timeout_factor times what the tests took
@@ -81,14 +81,21 @@ module Saboteur
81
81
 
82
82
  # Verdicts that did not hold up are warned about once per subject;
83
83
  # only checked verdicts are stored for --incremental.
84
- # A spent budget admits no check: forking the unchanged run with no
85
- # time left would turn genuine kills into fabricated errors and cache
86
- # them. Fresh kills stand unverified instead - outside the score,
87
- # never cached - and the distribution marks the subject so the run
88
- # reports budget-partial.
89
- def checked(plan, results)
90
- return results.map { |result| unverifiable(result) } if budget_spent?
91
-
84
+ # A spent budget admits no check for a subject that finished its
85
+ # mutants: forking the unchanged run with no time left would turn
86
+ # genuine kills into fabricated errors and cache them. Fresh kills
87
+ # stand unverified instead - outside the score, never cached - and
88
+ # the distribution marks the subject so the run reports
89
+ # budget-partial.
90
+ # A subject the budget stopped mid-list is different (#588): its
91
+ # completed mutants were already measured, so their checks still
92
+ # run, uncapped - the budget stops new mutation work, not the
93
+ # confirmation of work already done. The overshoot stays bounded by
94
+ # the subject's distinct killing sequences.
95
+ def checked(plan, results, truncated: false)
96
+ return results.map { |result| unverifiable(result) } if budget_spent? && !truncated
97
+
98
+ @confirming_truncation = truncated
92
99
  subject = plan.subject
93
100
  check = VerdictCheck.new(run_unchanged: unchanged_runner(subject, plan),
94
101
  baseline_timeout: @config["baseline_timeout"])
@@ -101,6 +108,8 @@ module Saboteur
101
108
  @diagnostics.warn("#{subject.id}: #{overturned} verdict(s) failed with the method unchanged")
102
109
  end
103
110
  final
111
+ ensure
112
+ @confirming_truncation = false
104
113
  end
105
114
 
106
115
  # A kill rejected because its covering examples fail with the method
@@ -111,10 +120,12 @@ module Saboteur
111
120
  # mutant's run on the same examples. Otherwise the error stays.
112
121
  #
113
122
  # Cost bound: at most two extra forks per rejected kill, the unchanged
114
- # prefix run and one mutant run. Both spend --max-duration like every
115
- # other fork: neither starts once the budget is spent, and neither gets
116
- # more than the time left. A retry the budget stops is unverified, not
117
- # an error or a kill. Keep this bound if the retry ever grows.
123
+ # prefix run and one mutant run. Ordinarily both spend --max-duration
124
+ # like every other fork: neither starts once the budget is spent, and
125
+ # neither gets more than the time left. A retry the budget stops is
126
+ # unverified, not an error or a kill. The exception is a truncated
127
+ # subject (#588): its completed kills still retry, at the full
128
+ # timeout. Keep this bound if the retry ever grows.
118
129
  def retried_in_defined_order(plan, result)
119
130
  return result unless result.rejected_kill? && !result.reused? && result.covered_by.any?
120
131
 
@@ -129,10 +140,10 @@ module Saboteur
129
140
  end
130
141
 
131
142
  def rerun_in_prefix(plan, result, tests)
132
- return result.as_unverified(BUDGET_RETRY_REASON) if budget_spent?
143
+ return result.as_unverified(BUDGET_RETRY_REASON) if budget_spent? && !@confirming_truncation
133
144
 
134
145
  full = timeout_for(tests, plan.slowdown)
135
- capped = budget_capped(full)
146
+ capped = @confirming_truncation ? full : budget_capped(full)
136
147
  outcome = @runner.run(subject: plan.subject, text: result.mutated_source, test_ids: tests,
137
148
  timeout: capped, mutation: result.mutation, eval_prefix: eval_prefix(plan))
138
149
  return result.as_unverified(BUDGET_RETRY_REASON) if outcome.status == :timeout && capped < full
@@ -150,11 +161,11 @@ module Saboteur
150
161
  # the check never got its answer (#296).
151
162
  def unchanged_runner(subject, plan)
152
163
  lambda do |tests, timeout|
153
- if budget_spent?
164
+ if budget_spent? && !@confirming_truncation
154
165
  Execution::Runner::Outcome.new(status: :unverified, runtime: 0.0,
155
166
  message: "the --max-duration budget is spent")
156
167
  else
157
- capped = budget_capped(timeout)
168
+ capped = @confirming_truncation ? timeout : budget_capped(timeout)
158
169
  outcome = @runner.run(subject: subject, text: subject.source, test_ids: tests, timeout: capped,
159
170
  eval_prefix: eval_prefix(plan))
160
171
  note_leftovers(subject, "verdict check", outcome)
@@ -238,17 +249,14 @@ module Saboteur
238
249
  [@config["timeout_factor"] * runtime, @config["timeout_min"]].max
239
250
  end
240
251
 
241
- # What a subject no test ran gets. Code that only ran while the app
242
- # loaded is not "no coverage" - it ran - but no forked test can re-run
243
- # it, so it gets its own status, out of the score, with a hint.
252
+ # What a subject no test ran gets. Load-only code is not "no coverage":
253
+ # it ran, but no forked test can re-run it, so it gets its own status.
244
254
  def unloadable(candidates, subject, line_numbers = nil)
255
+ return skip_all(candidates, subject, Selection::CoverageMap::PRELOADED) if @map.preloaded?(subject.file)
245
256
  return skip_all(candidates, subject, "no coverage") unless
246
257
  @map.ran_at_load?(subject.file, line_numbers || subject.coverage_lines)
247
258
 
248
- candidates.map do |mutation, text|
249
- MutantResult.new(subject: subject, mutation: mutation, status: :ran_at_load, mutated_source: text,
250
- reason: Selection::CoverageMap::RAN_AT_LOAD)
251
- end
259
+ load_time_results(candidates, subject)
252
260
  end
253
261
 
254
262
  # A def-time macro's only callers ran while the app loaded: the
@@ -10,10 +10,15 @@ module Saboteur
10
10
  module SelectionAnnouncement
11
11
  private
12
12
 
13
+ # A --since diff with no changed source file (a spec-only or docs-only
14
+ # change) yields zero subjects, so the suite never runs: say there is
15
+ # nothing to test rather than promise a whole-suite run that does not
16
+ # happen (#607).
13
17
  def selection_message(guess)
14
- if guess.changed.zero?
15
- target = config["since"] ? "no changed source file" : "no source file"
16
- "inferred specs: #{target} to infer a spec for, so the whole suite runs"
18
+ if guess.changed.zero? && config["since"]
19
+ "inferred specs: no changed source file to infer a spec for, so there is nothing to test"
20
+ elsif guess.changed.zero?
21
+ "inferred specs: no source file to infer a spec for, so the whole suite runs"
17
22
  elsif guess.specs.empty?
18
23
  guess.no_specs_message(since: config["since"])
19
24
  else