saboteur 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +399 -0
  3. data/README.md +202 -42
  4. data/exe/saboteur +34 -7
  5. data/lib/saboteur/baseline/comparison.rb +1 -0
  6. data/lib/saboteur/baseline/evidence.rb +11 -1
  7. data/lib/saboteur/baseline/run_evidence.rb +1 -1
  8. data/lib/saboteur/baseline/workflow.rb +2 -1
  9. data/lib/saboteur/baseline_health.rb +40 -6
  10. data/lib/saboteur/cli/exit_status.rb +49 -4
  11. data/lib/saboteur/cli/help_options.rb +16 -0
  12. data/lib/saboteur/cli/presets.rb +36 -0
  13. data/lib/saboteur/cli/run_budget_options.rb +37 -0
  14. data/lib/saboteur/cli/run_reporting.rb +75 -6
  15. data/lib/saboteur/cli.rb +80 -48
  16. data/lib/saboteur/config.rb +88 -13
  17. data/lib/saboteur/crash_report.rb +154 -0
  18. data/lib/saboteur/custom_map.rb +3 -1
  19. data/lib/saboteur/doctor.rb +64 -2
  20. data/lib/saboteur/engine/baseline_guard.rb +79 -5
  21. data/lib/saboteur/engine/boot.rb +12 -1
  22. data/lib/saboteur/engine/canary.rb +41 -10
  23. data/lib/saboteur/engine/candidate_selection.rb +9 -1
  24. data/lib/saboteur/engine/candidates.rb +4 -1
  25. data/lib/saboteur/engine/coverage_guard.rb +29 -0
  26. data/lib/saboteur/engine/coverage_store.rb +48 -6
  27. data/lib/saboteur/engine/diagnostics.rb +23 -2
  28. data/lib/saboteur/engine/diff_scope.rb +24 -5
  29. data/lib/saboteur/engine/distribution.rb +124 -8
  30. data/lib/saboteur/engine/early_stop_recording.rb +46 -0
  31. data/lib/saboteur/engine/result_cache.rb +2 -1
  32. data/lib/saboteur/engine/run_budget.rb +63 -0
  33. data/lib/saboteur/engine/run_fingerprint.rb +5 -1
  34. data/lib/saboteur/engine/sandbox_policy.rb +18 -11
  35. data/lib/saboteur/engine/scheduler.rb +114 -18
  36. data/lib/saboteur/engine/subject_selection.rb +84 -5
  37. data/lib/saboteur/engine/suite_files.rb +3 -1
  38. data/lib/saboteur/engine/test_guess.rb +183 -23
  39. data/lib/saboteur/engine/thread_warnings.rb +53 -0
  40. data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
  41. data/lib/saboteur/engine/verdict_check.rb +54 -18
  42. data/lib/saboteur/engine.rb +175 -96
  43. data/lib/saboteur/execution/protocol.rb +2 -1
  44. data/lib/saboteur/execution/runner.rb +179 -17
  45. data/lib/saboteur/execution/sandbox.rb +6 -1
  46. data/lib/saboteur/execution/supervisor.rb +18 -3
  47. data/lib/saboteur/execution/worker_pool.rb +87 -7
  48. data/lib/saboteur/frameworks/database_hosts.rb +30 -0
  49. data/lib/saboteur/frameworks/rails.rb +8 -10
  50. data/lib/saboteur/help.rb +55 -10
  51. data/lib/saboteur/ignore_list.rb +11 -1
  52. data/lib/saboteur/install.rb +4 -3
  53. data/lib/saboteur/integration/base.rb +7 -2
  54. data/lib/saboteur/integration/rspec.rb +40 -4
  55. data/lib/saboteur/mutation_source/builtin.rb +22 -13
  56. data/lib/saboteur/mutation_source/hazard_guard.rb +21 -3
  57. data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
  58. data/lib/saboteur/mutation_source/method_swap.rb +45 -0
  59. data/lib/saboteur/mutation_source/mutation.rb +5 -3
  60. data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
  61. data/lib/saboteur/mutation_source/operator_examples.rb +38 -0
  62. data/lib/saboteur/mutation_source/operator_registry.rb +36 -8
  63. data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
  64. data/lib/saboteur/preview.rb +12 -2
  65. data/lib/saboteur/reporter/agent_commands.rb +49 -16
  66. data/lib/saboteur/reporter/agent_incomplete.rb +27 -2
  67. data/lib/saboteur/reporter/agent_summary.rb +1 -1
  68. data/lib/saboteur/reporter/history_policy.rb +35 -0
  69. data/lib/saboteur/reporter/history_record.rb +4 -5
  70. data/lib/saboteur/reporter/next_steps.rb +14 -4
  71. data/lib/saboteur/reporter/score_history.rb +19 -3
  72. data/lib/saboteur/reporter/stryker.rb +19 -8
  73. data/lib/saboteur/reporter/survivor_hints.rb +3 -0
  74. data/lib/saboteur/reporter/terminal.rb +43 -10
  75. data/lib/saboteur/result.rb +25 -7
  76. data/lib/saboteur/selection/coverage_map.rb +2 -1
  77. data/lib/saboteur/selection/git_diff.rb +15 -5
  78. data/lib/saboteur/selection/untracked_changes.rb +126 -0
  79. data/lib/saboteur/subject.rb +11 -0
  80. data/lib/saboteur/text.rb +42 -0
  81. data/lib/saboteur/version.rb +1 -1
  82. data/lib/saboteur.rb +16 -1
  83. metadata +20 -2
@@ -2,6 +2,7 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  require "open3"
5
+ require_relative "../text"
5
6
 
6
7
  module Saboteur
7
8
  class Engine
@@ -15,7 +16,9 @@ module Saboteur
15
16
  GIT_DIR = ".git"
16
17
 
17
18
  def initialize(root, cache_dir: ".saboteur")
18
- @root = root
19
+ # Retag losslessly: path bytes are identity here, and later
20
+ # File.join calls must not mix encodings.
21
+ @root = Text.utf8_identity(root.to_s)
19
22
  @cache_dir = cache_dir.to_s
20
23
  @snapshot = inventory
21
24
  end
@@ -29,16 +32,40 @@ module Saboteur
29
32
 
30
33
  of = " of #{subject.id}" if subject
31
34
  culprit = digest ? " The most recent mutant, #{digest}#{of}, is the likely cause." : ""
32
- raise Saboteur::CanaryTripped,
33
- "canary: #{missing.size} watched file(s) vanished from the working tree while mutants ran " \
34
- "(first: #{missing.first}).#{culprit} The canary watches tracked files and .git presence " \
35
- "and only detects the loss after the fact; it cannot restore anything. Recover from your " \
36
- "own backup, a fresh checkout, or a reclone - a deleted .git or uncommitted work is gone " \
37
- "unless you have another copy."
35
+ raise Saboteur::CanaryTripped, message(missing, culprit)
36
+ end
37
+
38
+ # The cheap per-mutant probe (#235): stats the snapshot paths only -
39
+ # no git, no glob - so it can run after every mutant.
40
+ def probe
41
+ @snapshot.reject { |rel| present?(rel) }
42
+ end
43
+
44
+ # Like check!, but for the per-mutant probe: the mutant whose run just
45
+ # ended is named, never the subject's most recent mutant.
46
+ def probe!(subject: nil, digest: nil)
47
+ missing = probe
48
+ return if missing.empty?
49
+
50
+ of = " of #{subject.id}" if subject
51
+ culprit = digest ? " The mutant that just ran, #{digest}#{of}, is the likely cause." : ""
52
+ raise Saboteur::CanaryTripped.new(message(missing, culprit), culprit_digest: digest)
38
53
  end
39
54
 
40
55
  private
41
56
 
57
+ def present?(rel)
58
+ File.exist?(File.join(@root, rel)) || File.symlink?(File.join(@root, rel))
59
+ end
60
+
61
+ def message(missing, culprit)
62
+ "canary: #{missing.size} watched file(s) vanished from the working tree while mutants ran " \
63
+ "(first: #{Text.utf8(missing.first)}).#{culprit} The canary watches tracked files and .git presence " \
64
+ "and only detects the loss after the fact; it cannot restore anything. Recover from your " \
65
+ "own backup, a fresh checkout, or a reclone - a deleted .git or uncommitted work is gone " \
66
+ "unless you have another copy."
67
+ end
68
+
42
69
  def inventory
43
70
  files = git? ? git_tracked_files : walk_files
44
71
  files << GIT_DIR if File.exist?(File.join(@root, GIT_DIR))
@@ -54,15 +81,19 @@ module Saboteur
54
81
  return walk_files unless status.success?
55
82
 
56
83
  # ls-files reads the index, which still names a deleted file; keep
57
- # only paths present in the worktree so a deletion shows up.
58
- out.split("\0").reject { |rel| cache_path?(rel) }
84
+ # only paths present in the worktree so a deletion shows up. Split
85
+ # the raw bytes first and retag each path losslessly: scrubbing a
86
+ # raw-byte name would make the File.exist? probe fail and drop the
87
+ # file from the watch, so its deletion would go unseen.
88
+ out.b.split("\0".b).map { |rel| Text.utf8_identity(rel) }
89
+ .reject { |rel| cache_path?(rel) }
59
90
  .select { |rel| File.exist?(File.join(@root, rel)) || File.symlink?(File.join(@root, rel)) }
60
91
  end
61
92
 
62
93
  def walk_files
63
94
  Dir.glob(File.join(@root, "**", "*"), File::FNM_DOTMATCH)
64
95
  .select { |path| File.file?(path) }
65
- .map { |path| path.delete_prefix("#{@root}/") }
96
+ .map { |path| Text.utf8_identity(path).delete_prefix("#{@root}/") }
66
97
  .reject { |rel| rel.start_with?("#{GIT_DIR}/") || cache_path?(rel) }
67
98
  end
68
99
 
@@ -6,7 +6,7 @@ module Saboteur
6
6
  # Selects a bounded list for each covered subject before workers fork.
7
7
  # Subject skips are not mutants and do not enter the mutation score.
8
8
  class CandidateSelection
9
- attr_reader :skipped_subjects, :selected
9
+ attr_reader :skipped_subjects, :selected, :unscoped_digests
10
10
 
11
11
  def initialize(candidates:, map:, limit:, diagnostics:)
12
12
  @candidates = candidates
@@ -15,6 +15,7 @@ module Saboteur
15
15
  @diagnostics = diagnostics
16
16
  @skipped_subjects = {}
17
17
  @selected = {}
18
+ @unscoped_digests = {}
18
19
  end
19
20
 
20
21
  def for(subject, scope: nil)
@@ -25,6 +26,13 @@ module Saboteur
25
26
  end
26
27
 
27
28
  list = @candidates.for(subject)
29
+ # The full pre-scope digests let IgnoreList#apply tell "mutation gone"
30
+ # from "mutation scoped out of this run" (#284).
31
+ if scope
32
+ @unscoped_digests[subject.id] = list.map do |mutation, _|
33
+ mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)
34
+ end
35
+ end
28
36
  list = scope.restrict(subject, list) if scope
29
37
  omitted = list.size - @limit
30
38
  if omitted.positive?
@@ -13,7 +13,10 @@ module Saboteur
13
13
  def self.build_sources(config)
14
14
  root = File.realpath(config.root)
15
15
  config["sources"].map do |entry|
16
- next MutationSource::Builtin.new(arid: config["arid"], operators: config["operators"]) if entry == "builtin"
16
+ if entry == "builtin"
17
+ next MutationSource::Builtin.new(arid: config["arid"], operators: config["operators"],
18
+ exclude_operators: config["exclude_operators"])
19
+ end
17
20
 
18
21
  file, const = entry.split(":", 2)
19
22
  raise ConfigError, "mutation source #{entry.inspect} must be builtin or path.rb:ClassName" unless const
@@ -0,0 +1,29 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class Engine
6
+ # Owns the host process's Coverage state across a run: suspends after
7
+ # boot, and at the end resumes only its own suspension. A CLI process
8
+ # exits right after, so resume matters to hosts that embed the engine -
9
+ # a spec, or the dogfood's own coverage-map build (#254).
10
+ class CoverageGuard
11
+ def initialize(diagnostics)
12
+ @diagnostics = diagnostics
13
+ end
14
+
15
+ def suspend
16
+ @was_running = Coverage.state == :running
17
+ Coverage.suspend if @was_running
18
+ end
19
+
20
+ # Best-effort: a host that disturbed Coverage mid-run gets a warning,
21
+ # never a masked run outcome.
22
+ def resume
23
+ Coverage.resume if @was_running && Coverage.state == :suspended
24
+ rescue StandardError => e
25
+ @diagnostics.warn("coverage resume failed: #{e.class}: #{e.message}")
26
+ end
27
+ end
28
+ end
29
+ end
@@ -15,19 +15,29 @@ module Saboteur
15
15
  LOCKFILES = %w[Gemfile.lock gems.locked].freeze
16
16
  LOG = "coverage-build.log"
17
17
 
18
- def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {})
18
+ def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {}, timeout_cap: nil)
19
19
  @config = config
20
20
  @integration = integration
21
21
  @hooks = hooks
22
22
  @supervisor = supervisor
23
23
  @diagnostics = diagnostics
24
+ @timeout_cap = timeout_cap
24
25
  @load_lines = load_lines
25
26
  @root = File.realpath(config.root)
26
27
  end
27
28
 
29
+ # True when fetch would reuse the cached map instead of running the
30
+ # suite pass that builds it: lets the run announce the real number of
31
+ # full passes up front (#260). Loads the map with the same validation
32
+ # fetch uses and retains it, so a corrupt or truncated cache announces
33
+ # a build pass, not a reuse it cannot keep.
34
+ def reusable?(files)
35
+ !cached_map(files).nil?
36
+ end
37
+
28
38
  def fetch(files)
29
- path = File.join(@config.path("cache_dir"), "coverage-#{cache_key(files)}.json")
30
- cached = @config["cache"] ? Selection::CoverageMap.load(path) : nil
39
+ path = cache_path(files)
40
+ cached = cached_map(files)
31
41
  return refuse_empty_map(cached, files) if cached
32
42
 
33
43
  map = refuse_empty_map(build(files), files)
@@ -39,6 +49,16 @@ module Saboteur
39
49
 
40
50
  private
41
51
 
52
+ def cache_path(files)
53
+ File.join(@config.path("cache_dir"), "coverage-#{cache_key(files)}.json")
54
+ end
55
+
56
+ def cached_map(files)
57
+ return nil unless @config["cache"]
58
+
59
+ @cached_map ||= Selection::CoverageMap.load(cache_path(files))
60
+ end
61
+
42
62
  def cache_key(files)
43
63
  inputs = JSON.generate(CACHE_INPUTS.to_h { |key| [key, @config[key]] })
44
64
  all = files + @integration.test_files + SuiteFiles.new(@config).files + lockfiles
@@ -49,6 +69,12 @@ module Saboteur
49
69
  LOCKFILES.map { |file| File.join(@root, file) }.select { |file| File.file?(file) }
50
70
  end
51
71
 
72
+ # An admitted build still gets only the time left in the run budget
73
+ # (see BaselineGuard#phase_timeout).
74
+ def build_timeout
75
+ @timeout_cap ? @timeout_cap.call(@config["coverage_timeout"]) : @config["coverage_timeout"]
76
+ end
77
+
52
78
  def build(files)
53
79
  unless Coverage.respond_to?(:state) && Coverage.state == :suspended
54
80
  raise Error, "Coverage is #{Coverage.state}; saboteur must set it up before the app loads. " \
@@ -61,13 +87,16 @@ module Saboteur
61
87
  # reduce later copying of shared pages.
62
88
  Execution::ForkReadiness.prepare_heap
63
89
  heartbeat = @diagnostics.progress_heartbeat("building coverage map").watch(log_path)
64
- result = @supervisor.run(timeout: @config["coverage_timeout"], on_tick: heartbeat.method(:tick)) do |writer|
90
+ admitted = build_timeout
91
+ result = @supervisor.run(timeout: admitted, on_tick: heartbeat.method(:tick),
92
+ label: "coverage child died during the map build") do |writer|
65
93
  record(writer, files)
66
94
  end
67
- map_from(result)
95
+ map_from(result, admitted)
68
96
  end
69
97
 
70
- def map_from(result)
98
+ def map_from(result, admitted = @config["coverage_timeout"])
99
+ raise_budget_stop(admitted) if budget_stopped?(result, admitted)
71
100
  raise Error, "coverage map build stopped: #{result.detail}" if result.stopped_by
72
101
  raise Error, "coverage map build failed: #{died(result.exit_status)}" if result.payload.empty?
73
102
 
@@ -77,6 +106,19 @@ module Saboteur
77
106
  with_parent_load_lines(Selection::CoverageMap.from_json(result.payload))
78
107
  end
79
108
 
109
+ # A build that hit its deadline only because --max-duration shortened
110
+ # the admission below the configured phase timeout: the budget ended
111
+ # the run, the suite is not broken (see BaselineGuard#budget_stopped?).
112
+ def budget_stopped?(result, admitted)
113
+ result.stopped_by == :timeout && admitted < @config["coverage_timeout"]
114
+ end
115
+
116
+ def raise_budget_stop(admitted)
117
+ raise BudgetPhaseTimeout, "the --max-duration budget stopped the coverage map build " \
118
+ "(admitted #{admitted.round(1)}s of the #{@config['coverage_timeout']}s " \
119
+ "phase timeout)"
120
+ end
121
+
80
122
  # The parent's load lines merge into the freshly built map: the forked
81
123
  # build child cannot inherit the parent's Coverage counters.
82
124
  def with_parent_load_lines(map)
@@ -76,8 +76,9 @@ module Saboteur
76
76
  attr_reader :warnings, :invalid_mutations, :generation_events, :omissions, :discovery_failures,
77
77
  :hazard_suppressions, :inline_suppressions
78
78
 
79
- def initialize(err)
79
+ def initialize(err, verbose: false)
80
80
  @err = err
81
+ @verbose = verbose
81
82
  @warnings = []
82
83
  @invalid_mutations = 0
83
84
  @generation_events = []
@@ -102,8 +103,11 @@ module Saboteur
102
103
  @generation_events << GenerationEvent.new(*values)
103
104
  end
104
105
 
106
+ # The selection and the test inference each walk the same changed
107
+ # files in one run; a failure either of them finds is recorded once.
105
108
  def record_discovery_failure(file, reason:)
106
- @discovery_failures << DiscoveryFailure.new(file: file.to_s.dup.freeze, reason: reason.to_s.dup.freeze)
109
+ failure = DiscoveryFailure.new(file: file.to_s.dup.freeze, reason: reason.to_s.dup.freeze)
110
+ @discovery_failures << failure unless @discovery_failures.include?(failure)
107
111
  end
108
112
 
109
113
  # A mutant the hazard guard refused to generate (issue #102): counted
@@ -129,6 +133,23 @@ module Saboteur
129
133
  @err.puts("saboteur: #{message}")
130
134
  end
131
135
 
136
+ # --verbose: one line per finished subject with its result marks, so a
137
+ # survivor is attributable to a method as soon as the subject
138
+ # completes, before the whole run ends (#262).
139
+ def subject_progress(subject, results)
140
+ return unless @verbose
141
+ return if results.empty?
142
+
143
+ marks = results.map { |result| MARKS.fetch(result.status, "?") }.join
144
+ inform("#{subject.id} #{marks}")
145
+ end
146
+
147
+ # A message that is both a report warning and a stderr note.
148
+ def announce(message)
149
+ warn(message)
150
+ inform(message)
151
+ end
152
+
132
153
  # A heartbeat for a long phase: the supervisor's wait loop calls tick
133
154
  # in the parent after the fork, so no extra thread is ever alive at
134
155
  # fork time. Prints "<label>: N s elapsed" about every interval
@@ -1,6 +1,8 @@
1
1
  # SPDX-License-Identifier: MIT
2
2
  # frozen_string_literal: true
3
3
 
4
+ require_relative "../text"
5
+
4
6
  module Saboteur
5
7
  class Engine
6
8
  # Review-shaped diff mode (--since). A reviewer asks "do the tests check
@@ -14,7 +16,7 @@ module Saboteur
14
16
  end
15
17
 
16
18
  def restrict(subject, candidates)
17
- changed = @changed_lines.fetch(subject.file, Set.new)
19
+ changed = lines_for(subject)
18
20
  candidates.select do |mutation, _|
19
21
  mutation.coarse || mutation.edits.any? { |edit| edited_lines(subject, edit).any? { changed.include?(_1) } }
20
22
  end
@@ -24,22 +26,39 @@ module Saboteur
24
26
  # several lines is mutated as one piece and reported at its first line,
25
27
  # which can sit above the line that changed; this names the changed line.
26
28
  def changed_lines_for(subject, mutation)
27
- changed = @changed_lines.fetch(subject.file, Set.new)
29
+ changed = lines_for(subject)
28
30
  mutation.edits.flat_map { |edit| edited_lines(subject, edit).to_a }.uniq.select { changed.include?(_1) }.sort
29
31
  end
30
32
 
31
33
  # Distance to the nearest changed line in this file. Unknown outside
32
- # the diff is nil, not a made-up nearby location.
34
+ # the diff is nil, not a made-up nearby location. A range answers
35
+ # geometrically - 0 when the edit overlaps, else the gap to the nearer
36
+ # endpoint - so a whole-file range never materializes one entry per
37
+ # line.
33
38
  def distance_for(subject, mutation)
34
- changed = @changed_lines.fetch(subject.file, Set.new)
39
+ changed = lines_for(subject)
35
40
  mutation.edits.map do |edit|
36
41
  first, last = edited_lines(subject, edit).minmax
37
- changed.map { |line| (line - line.clamp(first, last)).abs }.min
42
+ if changed.is_a?(Range)
43
+ if last < changed.begin then changed.begin - last
44
+ elsif first > changed.end then first - changed.end
45
+ else 0
46
+ end
47
+ else
48
+ changed.map { |line| (line - line.clamp(first, last)).abs }.min
49
+ end
38
50
  end.min
39
51
  end
40
52
 
41
53
  private
42
54
 
55
+ # subject.file carries the locale tag under a non-UTF-8 locale
56
+ # while the map keys are UTF-8; retag losslessly or a non-ASCII
57
+ # path never matches and every fine mutant is silently dropped.
58
+ def lines_for(subject)
59
+ @changed_lines.fetch(Text.utf8_identity(subject.file), Set.new)
60
+ end
61
+
43
62
  def edited_lines(subject, edit)
44
63
  first = subject.line_of(edit.start_byte)
45
64
  last = subject.line_of([edit.end_byte - 1, edit.start_byte].max)
@@ -9,23 +9,31 @@ module Saboteur
9
9
  # candidate list, so results look the same either way.
10
10
  class Distribution
11
11
  # After run: the subjects a stop_after callable left unrun (always empty
12
- # without one).
13
- attr_reader :unreached
12
+ # without one), and the subjects that started but stopped mid-list as
13
+ # [subject, omitted_candidate_count] pairs (a spent --max-duration
14
+ # breaking a subject's scheduler loop). unverified lists the subjects
15
+ # whose fresh kills the spent budget left unchecked (#296).
16
+ attr_reader :unreached, :truncated, :unverified
14
17
 
15
18
  # stop_after: nil, or a callable that gets one subject's results and
16
19
  # returns true to stop before the next subject (--fail-fast). It only
17
20
  # works on one process: parallel workers have already started the
18
21
  # other subjects.
19
22
  def initialize(jobs:, scheduler:, diagnostics:, cache:, on_start: nil, after_subject: nil,
20
- stop_after: nil)
23
+ stop_after: nil, halt_check: nil,
24
+ drain_timeout: 30) # seconds; two minimum-strength mutant deadlines at default settings
21
25
  @jobs = jobs
22
26
  @stop_after = stop_after
27
+ @halt_check = halt_check
23
28
  @unreached = []
29
+ @truncated = []
30
+ @unverified = []
24
31
  @on_start = on_start
25
32
  @scheduler = scheduler
26
33
  @diagnostics = diagnostics
27
34
  @cache = cache
28
35
  @after_subject = after_subject
36
+ @drain_timeout = drain_timeout
29
37
  end
30
38
 
31
39
  # lists: each subject's [[mutation, text]] candidates, in subject order.
@@ -35,23 +43,116 @@ module Saboteur
35
43
  # The watcher runs as each subject's row arrives, before the pool
36
44
  # hands out more work, so a tripped canary stops the queue (jobs>1
37
45
  # used to check only after the whole batch returned, issue #102).
38
- arrival = lambda do |i, row|
39
- @after_subject&.call(subjects[i], decode(row, subjects[i], lists[i]))
46
+ # A tool error no longer raises on sight: one wiped log dir hits
47
+ # every worker, and a bystander whose current job ran no mutant
48
+ # reports "an unknown mutant", which could land before the report
49
+ # of the worker whose mutant did the damage (macOS CI on #285).
50
+ # The pool is halted instead - no new work, and the drain is
51
+ # bounded by drain_timeout so a wedged worker cannot stall the
52
+ # abort - and the abort raises the report with the best
53
+ # attribution.
54
+ # The abort reports and the pool are run state: a Distribution
55
+ # runs once. @pool is only read from arrivals, which fire inside
56
+ # @pool.map after the assignment.
57
+ # A budget already spent by boot, baselines or the coverage map
58
+ # launches nothing: every subject comes back unreached (#245).
59
+ if @halt_check&.call
60
+ @unreached = subjects
61
+ return []
40
62
  end
41
- pool = Execution::WorkerPool.new(jobs: @jobs, on_start: @on_start, on_result: arrival)
42
- rows = pool.map(subjects.each_index.to_a, weights: lists.map(&:size)) do |i|
63
+
64
+ @abort_reports = []
65
+ arrival = arrival_for(subjects, lists)
66
+ @pool = Execution::WorkerPool.new(jobs: @jobs, on_start: @on_start, on_result: arrival,
67
+ drain_timeout: @drain_timeout)
68
+ rows = @pool.map(subjects.each_index.to_a, weights: lists.map(&:size)) do |i|
43
69
  work(subjects[i], lists[i])
44
70
  end
45
- rows.each_with_index.flat_map { |row, i| rebuild(row, subjects[i], lists[i]) }
71
+ raise_preferred_tool_error(@abort_reports) if @abort_reports.any?
72
+ record_unreached(rows, subjects, lists)
73
+ rows.each_with_index.flat_map do |row, i|
74
+ next [] if row.nil?
75
+
76
+ raise_if_tool_error(row)
77
+ rebuild(row, subjects[i], lists[i])
78
+ end
46
79
  end
47
80
 
48
81
  private
49
82
 
83
+ # @pool is only read here from arrivals, which fire inside @pool.map
84
+ # after the assignment. A spent --max-duration budget stops the queue
85
+ # without the safety-abort drain: in-flight subjects finish and
86
+ # report, and only never-started ones come back as nil holes, so
87
+ # unreached never mislabels an interrupted subject as unrun (#245).
88
+ def arrival_for(subjects, lists)
89
+ lambda do |i, row|
90
+ if row["tool_error"]
91
+ collect_abort_report(row)
92
+ else
93
+ check_subject(subjects, lists, i, row)
94
+ @pool.halt!(drain: false) if @halt_check&.call
95
+ end
96
+ end
97
+ end
98
+
99
+ # nil rows never started; a short results list is a subject whose
100
+ # scheduler stopped mid-list at the budget deadline. Worker-level error
101
+ # rows synthesize an error per candidate, so they never look truncated.
102
+ def unverified_result?(result)
103
+ result.is_a?(MutantResult) && result.status == :unverified
104
+ end
105
+
106
+ def record_unreached(rows, subjects, lists)
107
+ @unreached = []
108
+ @truncated = []
109
+ @unverified = []
110
+ rows.each_with_index do |row, i|
111
+ if row.nil?
112
+ @unreached << subjects[i]
113
+ next
114
+ end
115
+ next unless row["results"] && !row["error"]
116
+
117
+ @truncated << [subjects[i], lists[i].size - row["results"].size] if row["results"].size < lists[i].size
118
+ @unverified << subjects[i] if row["results"].any? { |result| result["status"] == "unverified" }
119
+ end
120
+ end
121
+
122
+ # @pool is always assigned when an arrival fires: arrivals only fire
123
+ # from inside @pool.map, which runs after the assignment.
124
+ def collect_abort_report(row)
125
+ @abort_reports << row
126
+ @pool.halt!
127
+ end
128
+
129
+ # A canary trip from the end-of-subject check folds into the same
130
+ # record + halt + preferred-report path as a worker's own report:
131
+ # escaping the pool here would override a collected report that
132
+ # names the real culprit (#285 review).
133
+ def check_subject(subjects, lists, index, row)
134
+ @after_subject&.call(subjects[index], decode(row, subjects[index], lists[index]))
135
+ rescue Error => e
136
+ collect_abort_report(Execution::WorkerPool.tool_error_row(e))
137
+ end
138
+
50
139
  def run_serially(subjects, lists)
51
140
  results = []
52
141
  subjects.zip(lists).each_with_index do |(subject, list), index|
142
+ # Same pre-dispatch budget check as the parallel first wave: an
143
+ # already-spent baseline starts no further subjects (#245).
144
+ if @halt_check&.call
145
+ @unreached = subjects.drop(index)
146
+ break
147
+ end
148
+
53
149
  subject_results = @scheduler.call(subject, list)
54
150
  results.concat(subject_results)
151
+ # A scheduler that stopped mid-list at the budget deadline: the
152
+ # completed results stay, the omitted candidates are accounted.
153
+ omitted = list.size - subject_results.size
154
+ @truncated << [subject, omitted] if omitted.positive?
155
+ @unverified << subject if subject_results.any? { |result| unverified_result?(result) }
55
156
  @after_subject&.call(subject, subject_results)
56
157
  next unless @stop_after&.call(subject_results)
57
158
 
@@ -91,6 +192,21 @@ module Saboteur
91
192
 
92
193
  # Turns a worker row back into results, without diagnostics or caching:
93
194
  # used both at result arrival (the canary) and at rebuild time.
195
+ # A tool error from the worker (a vanished cache dir, ...) aborts
196
+ # the whole run with exit 3; it is not a per-mutant outcome (#283).
197
+ def raise_if_tool_error(row)
198
+ raise Error, row["tool_error"] if row["tool_error"]
199
+ end
200
+
201
+ # The abort names a culprit when any report can, ranked by
202
+ # attribution quality (#285 review): a report carrying its mutant's
203
+ # per-mutant digest first, then a canary trip (the hazard report
204
+ # still beats a bare report, #283), then the first bare report.
205
+ def raise_preferred_tool_error(rows)
206
+ row = rows.find { |r| r["culprit"] } || rows.find { |r| r["canary"] } || rows.first
207
+ raise Error, row["tool_error"]
208
+ end
209
+
94
210
  def decode(row, subject, list)
95
211
  return failed(row["error"], subject, list) if row["error"]
96
212
 
@@ -0,0 +1,46 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class Engine
6
+ # Records the subjects an early stop left behind - whole subjects that
7
+ # never started and candidates skipped mid-way through a subject - so
8
+ # the report says the verdict covers less than the selection.
9
+ module EarlyStopRecording
10
+ private
11
+
12
+ # The subjects a stop left unrun or mid-way: the verdict covers less than the selection.
13
+ def record_early_stop(distribution)
14
+ return if distribution.unreached.empty? && distribution.truncated.empty? && distribution.unverified.empty?
15
+
16
+ truncated = distribution.truncated.map { |subject, omitted| { subject: subject.id, omitted: omitted } }
17
+ @early_stop = EarlyStop.new(reason: stop_reason(distribution), unreached: distribution.unreached.map(&:id),
18
+ truncated: truncated, unverified: distribution.unverified.map(&:id))
19
+ end
20
+
21
+ # Mid-subject omissions and skipped checks are budget work: only a
22
+ # stop that left whole subjects unreached can be fail_fast's.
23
+ def stop_reason(distribution)
24
+ return "max_duration" if distribution.truncated.any? || distribution.unverified.any?
25
+
26
+ run_early_stop_reason
27
+ end
28
+
29
+ # max_duration was announced before fail_fast could stop the run: a
30
+ # budget stop overrides a failure stop announced later.
31
+ def run_early_stop_reason
32
+ @budget_announced ? "max_duration" : "fail_fast"
33
+ end
34
+
35
+ # A baseline or coverage phase stopped because --max-duration ate its
36
+ # allowance: budget exhaustion, not a broken suite. The run reports
37
+ # partial with every selected subject unreached (exit 4) (#296).
38
+ def budget_phase_stop(error, subjects)
39
+ @diagnostics.warn(error.message)
40
+ @early_stop = EarlyStop.new(reason: "max_duration", unreached: subjects.map(&:id), truncated: [],
41
+ unverified: [])
42
+ []
43
+ end
44
+ end
45
+ end
46
+ end
@@ -4,6 +4,7 @@
4
4
  require "digest"
5
5
  require "fileutils"
6
6
  require "json"
7
+ require_relative "../text"
7
8
 
8
9
  module Saboteur
9
10
  class Engine
@@ -65,7 +66,7 @@ module Saboteur
65
66
  private
66
67
 
67
68
  def load
68
- data = JSON.parse(File.read(@path))
69
+ data = JSON.parse(Text.utf8_strict(File.read(@path)))
69
70
  data["format"] == FORMAT ? data["entries"] : {}
70
71
  rescue SystemCallError, JSON::ParserError
71
72
  {}