saboteur 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1024 -0
  3. data/README.md +199 -952
  4. data/exe/saboteur +34 -7
  5. data/lib/saboteur/baseline/workflow.rb +2 -1
  6. data/lib/saboteur/baseline_health.rb +40 -6
  7. data/lib/saboteur/cli/exit_status.rb +80 -4
  8. data/lib/saboteur/cli/help_options.rb +17 -0
  9. data/lib/saboteur/cli/history_command.rb +135 -0
  10. data/lib/saboteur/cli/presets.rb +40 -0
  11. data/lib/saboteur/cli/run_budget_options.rb +37 -0
  12. data/lib/saboteur/cli/run_reporting.rb +74 -5
  13. data/lib/saboteur/cli/selection_options.rb +31 -0
  14. data/lib/saboteur/cli/shard_options.rb +17 -0
  15. data/lib/saboteur/cli/source_set.rb +53 -0
  16. data/lib/saboteur/cli/survivors_command.rb +157 -0
  17. data/lib/saboteur/cli.rb +136 -98
  18. data/lib/saboteur/config.rb +163 -15
  19. data/lib/saboteur/config_file.rb +8 -0
  20. data/lib/saboteur/constant_reads.rb +182 -0
  21. data/lib/saboteur/constant_subject.rb +70 -0
  22. data/lib/saboteur/constant_subject_finder.rb +281 -0
  23. data/lib/saboteur/coverage_reads.rb +29 -0
  24. data/lib/saboteur/crash_report.rb +154 -0
  25. data/lib/saboteur/custom_map.rb +3 -1
  26. data/lib/saboteur/dirty_tree_warning.rb +128 -0
  27. data/lib/saboteur/doctor.rb +49 -1
  28. data/lib/saboteur/engine/baseline_guard.rb +136 -14
  29. data/lib/saboteur/engine/boot.rb +16 -1
  30. data/lib/saboteur/engine/canary.rb +41 -10
  31. data/lib/saboteur/engine/candidate_selection.rb +28 -1
  32. data/lib/saboteur/engine/candidates.rb +43 -3
  33. data/lib/saboteur/engine/coverage_guard.rb +29 -0
  34. data/lib/saboteur/engine/coverage_store.rb +79 -15
  35. data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
  36. data/lib/saboteur/engine/diagnostics.rb +41 -2
  37. data/lib/saboteur/engine/diff_scope.rb +24 -5
  38. data/lib/saboteur/engine/distribution.rb +134 -8
  39. data/lib/saboteur/engine/early_stop_recording.rb +47 -0
  40. data/lib/saboteur/engine/gated_run.rb +35 -0
  41. data/lib/saboteur/engine/load_coverage.rb +45 -0
  42. data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
  43. data/lib/saboteur/engine/pipelines.rb +107 -0
  44. data/lib/saboteur/engine/result_cache.rb +67 -9
  45. data/lib/saboteur/engine/run_budget.rb +90 -0
  46. data/lib/saboteur/engine/run_fingerprint.rb +4 -0
  47. data/lib/saboteur/engine/sandbox_policy.rb +56 -11
  48. data/lib/saboteur/engine/scheduler.rb +222 -20
  49. data/lib/saboteur/engine/selection_announcement.rb +54 -0
  50. data/lib/saboteur/engine/shard.rb +69 -0
  51. data/lib/saboteur/engine/subject_selection.rb +373 -19
  52. data/lib/saboteur/engine/suite_files.rb +3 -1
  53. data/lib/saboteur/engine/test_guess.rb +438 -25
  54. data/lib/saboteur/engine/thread_warnings.rb +53 -0
  55. data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
  56. data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
  57. data/lib/saboteur/engine/verdict_check.rb +54 -18
  58. data/lib/saboteur/engine.rb +202 -158
  59. data/lib/saboteur/execution/external_runner.rb +471 -0
  60. data/lib/saboteur/execution/marker_sweep.rb +79 -0
  61. data/lib/saboteur/execution/peak_memory.rb +28 -0
  62. data/lib/saboteur/execution/process_table.rb +12 -0
  63. data/lib/saboteur/execution/process_tree.rb +65 -5
  64. data/lib/saboteur/execution/protocol.rb +2 -1
  65. data/lib/saboteur/execution/reevaluator.rb +68 -12
  66. data/lib/saboteur/execution/runner.rb +225 -24
  67. data/lib/saboteur/execution/sandbox.rb +21 -3
  68. data/lib/saboteur/execution/seatbelt.rb +226 -0
  69. data/lib/saboteur/execution/supervisor.rb +45 -13
  70. data/lib/saboteur/execution/worker_pool.rb +110 -8
  71. data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
  72. data/lib/saboteur/gates.rb +43 -2
  73. data/lib/saboteur/help.rb +108 -13
  74. data/lib/saboteur/ignore_list.rb +40 -7
  75. data/lib/saboteur/install.rb +4 -3
  76. data/lib/saboteur/integration/base.rb +7 -2
  77. data/lib/saboteur/integration/rspec.rb +175 -9
  78. data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
  79. data/lib/saboteur/mutation_source/builtin.rb +44 -14
  80. data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
  81. data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
  82. data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
  83. data/lib/saboteur/mutation_source/element_removal.rb +73 -0
  84. data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
  85. data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
  86. data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
  87. data/lib/saboteur/mutation_source/hazard_guard.rb +129 -3
  88. data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
  89. data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
  90. data/lib/saboteur/mutation_source/method_swap.rb +193 -0
  91. data/lib/saboteur/mutation_source/mutation.rb +5 -3
  92. data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
  93. data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
  94. data/lib/saboteur/mutation_source/operator_examples.rb +6 -1
  95. data/lib/saboteur/mutation_source/operator_registry.rb +7 -2
  96. data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
  97. data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
  98. data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
  99. data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
  100. data/lib/saboteur/preview.rb +122 -13
  101. data/lib/saboteur/refinement_activations.rb +60 -0
  102. data/lib/saboteur/reporter/agent.rb +9 -1
  103. data/lib/saboteur/reporter/agent_commands.rb +47 -15
  104. data/lib/saboteur/reporter/agent_incomplete.rb +38 -2
  105. data/lib/saboteur/reporter/agent_summary.rb +4 -3
  106. data/lib/saboteur/reporter/findings.rb +5 -2
  107. data/lib/saboteur/reporter/history_policy.rb +11 -0
  108. data/lib/saboteur/reporter/history_reader.rb +47 -0
  109. data/lib/saboteur/reporter/html.rb +26 -4
  110. data/lib/saboteur/reporter/mutation_text.rb +6 -0
  111. data/lib/saboteur/reporter/next_steps.rb +14 -4
  112. data/lib/saboteur/reporter/score_history.rb +5 -1
  113. data/lib/saboteur/reporter/stryker.rb +40 -7
  114. data/lib/saboteur/reporter/survivor_classification.rb +101 -0
  115. data/lib/saboteur/reporter/survivor_hints.rb +3 -0
  116. data/lib/saboteur/reporter/survivor_list.rb +15 -9
  117. data/lib/saboteur/reporter/terminal.rb +74 -17
  118. data/lib/saboteur/result.rb +38 -7
  119. data/lib/saboteur/selection/coverage_map.rb +107 -11
  120. data/lib/saboteur/selection/dependency_shield.rb +80 -0
  121. data/lib/saboteur/selection/external_test_map.rb +27 -0
  122. data/lib/saboteur/selection/git_diff.rb +105 -13
  123. data/lib/saboteur/selection/layout_only_lines.rb +80 -0
  124. data/lib/saboteur/selection/recorder.rb +22 -4
  125. data/lib/saboteur/selection/source_scope.rb +22 -0
  126. data/lib/saboteur/selection/untracked_changes.rb +126 -0
  127. data/lib/saboteur/subject.rb +53 -7
  128. data/lib/saboteur/subject_finder.rb +173 -29
  129. data/lib/saboteur/text.rb +42 -0
  130. data/lib/saboteur/version.rb +1 -1
  131. data/lib/saboteur/working_tree.rb +38 -15
  132. data/lib/saboteur.rb +29 -1
  133. metadata +50 -2
@@ -0,0 +1,226 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "fiddle"
5
+
6
+ module Saboteur
7
+ module Execution
8
+ # The macOS counterpart of Sandbox (Landlock): a Seatbelt profile
9
+ # applied with sandbox_init inside the mutant child. sandbox-exec and
10
+ # sandbox_init are deprecated by Apple with no published replacement
11
+ # and no removal timeline (#436 tracks how Apple, Chrome, Codex,
12
+ # Claude Code, Bazel and Nix respond), so Seatbelt is opt-in and the
13
+ # probe below fails closed: a child that cannot be confined runs with
14
+ # the hazard guard and canary only, never silently "sandboxed".
15
+ #
16
+ # Like Sandbox, requiring this file is inert off macOS: libsandbox is
17
+ # dlopened lazily by the first probe, and the probe forks so the
18
+ # irreversible sandbox_init never touches the run's own process.
19
+ module Seatbelt
20
+ LIBSANDBOX = "/usr/lib/system/libsystem_sandbox.dylib"
21
+ LIBSYSTEM = "/usr/lib/libSystem.B.dylib"
22
+ F_GETPATH = 50
23
+ PATH_MAX = 1024
24
+
25
+ # The probe profile must deny the write it then attempts.
26
+ PROBE_PROFILE = "(version 1)(deny default)"
27
+
28
+ # sandbox_init error texts that say the facility is genuinely gone
29
+ # (a future macOS without Seatbelt): degrade, noted once. Anything
30
+ # else (a profile syntax tightening, EPERM, a lost child) fails
31
+ # closed: SandboxPolicy#seatbelt_hook refuses the run rather than
32
+ # confining nothing, exactly like the Landlock probe (#178) - never
33
+ # silently unsandboxed.
34
+ UNSUPPORTED_ERRORS = ["operation not supported", "function not implemented"].freeze
35
+
36
+ module_function
37
+
38
+ def darwin?
39
+ RUBY_PLATFORM.include?("darwin")
40
+ end
41
+
42
+ # One forked behavioral probe, memoized per process:
43
+ # :unsupported_platform off macOS, :ok when a deny-default profile
44
+ # really blocks a write, :unavailable when sandbox_init reports the
45
+ # facility missing, :unenforced when the API claims success but the
46
+ # write goes through, :error for anything else. Behavioral, not
47
+ # symbol presence: a macOS update that rejects the profile language
48
+ # shows up here, not in a dlopen.
49
+ def probe
50
+ return @probe if defined?(@probe)
51
+ return @probe = :unsupported_platform unless darwin?
52
+
53
+ @probe = run_probe
54
+ rescue Fiddle::DLError
55
+ @probe = :unavailable
56
+ end
57
+
58
+ def available?
59
+ probe == :ok
60
+ end
61
+
62
+ # Called at the top of a mutant child, mirroring Sandbox.restrict!:
63
+ # irreversible and fail-closed - a child whose profile does not apply
64
+ # must not run the mutant. keep_fds is Sandbox's hook contract
65
+ # (#234): Landlock gates opens, so its children keep writing through
66
+ # descriptors the parent opened (their logs); Seatbelt checks
67
+ # file-write* on EVERY write, so those inherited files join the
68
+ # profile as literal grants instead. Pipes and sockets among them
69
+ # have no path and need no grant.
70
+ def restrict!(writable:, keep_fds: [])
71
+ profile = Profile.build(writable: writable.map { |dir| resolved_grant(dir) },
72
+ inherited_files: fd_paths(keep_fds))
73
+ rc, message = sandbox_init(profile)
74
+ unless rc.zero?
75
+ raise Error, "the Seatbelt sandbox refused to engage (#{message}); the mutant does not run unconfined"
76
+ end
77
+
78
+ nil
79
+ end
80
+
81
+ # Seatbelt compares the kernel's resolved path, so a grant spelled
82
+ # through a symlink (macOS /var is /private/var) never matches and the
83
+ # "granted" write is denied. Resolve what exists; a dir not yet
84
+ # created grants nothing either way, so keep its spelling rather than
85
+ # fail the child on a dead grant. fcntl(F_GETPATH) already returns
86
+ # the resolved path for inherited files, so they need no such pass.
87
+ def resolved_grant(dir)
88
+ File.exist?(dir) ? File.realpath(dir) : dir
89
+ end
90
+
91
+ # fcntl(F_GETPATH) maps an inherited descriptor back to its path; a
92
+ # descriptor without one (pipe, socket) is skipped.
93
+ def fd_paths(keep_fds)
94
+ keep_fds.filter_map do |fd|
95
+ buffer = Fiddle::Pointer.malloc(PATH_MAX)
96
+ LibSystem.fcntl_fn.call(fd, F_GETPATH, buffer).negative? ? nil : buffer.to_s
97
+ end
98
+ end
99
+
100
+ def classify(verdict)
101
+ case verdict
102
+ when "ok" then :ok
103
+ when "unenforced" then :unenforced
104
+ when /\Ainit_failed:(?<message>.*)\z/m
105
+ message = Regexp.last_match[:message].downcase
106
+ UNSUPPORTED_ERRORS.any? { |text| message.include?(text) } ? :unavailable : :error
107
+ else
108
+ :error
109
+ end
110
+ end
111
+
112
+ def run_probe
113
+ reader, writer = IO.pipe
114
+ pid = fork { probe_child(writer) }
115
+ writer.close
116
+ verdict = reader.read
117
+ reader.close
118
+ Process.waitpid(pid)
119
+ classify(verdict)
120
+ ensure
121
+ reader.close unless reader.closed?
122
+ writer.close unless writer.closed?
123
+ end
124
+
125
+ # Child only: confine, then attempt the write the profile must block.
126
+ # The parent reads a single verdict word; exit! so no at_exit hook or
127
+ # finalizer runs under the sandbox.
128
+ def probe_child(writer)
129
+ rc, message = sandbox_init(PROBE_PROFILE)
130
+ verdict =
131
+ if rc.zero?
132
+ begin
133
+ File.open(File::NULL, "w") { true }
134
+ "unenforced"
135
+ rescue SystemCallError, IOError
136
+ "ok"
137
+ end
138
+ else
139
+ "init_failed:#{message}"
140
+ end
141
+ writer.write(verdict)
142
+ writer.close
143
+ exit!(0)
144
+ end
145
+
146
+ # int sandbox_init(const char *profile, uint64_t flags, char **errorbuf)
147
+ def sandbox_init(profile)
148
+ errbuf = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP)
149
+ rc = LibSandbox.sandbox_init_fn.call(profile, 0, errbuf)
150
+ return [0, nil] if rc.zero?
151
+
152
+ # errorbuf is a char**: dereference once to reach the message.
153
+ message = begin
154
+ address = errbuf[0, Fiddle::SIZEOF_VOIDP].unpack1("J")
155
+ address&.positive? ? Fiddle::Pointer.new(address).to_s : "unknown"
156
+ rescue StandardError
157
+ "unknown"
158
+ end
159
+ [rc, "sandbox_init: #{message}"]
160
+ end
161
+
162
+ # fcntl for the F_GETPATH lookup; lazy like LibSandbox.
163
+ module LibSystem
164
+ module_function
165
+
166
+ def fcntl_fn
167
+ @fcntl_fn ||= Fiddle::Function.new(
168
+ Fiddle.dlopen(Seatbelt::LIBSYSTEM)["fcntl"],
169
+ [Fiddle::TYPE_INT, Fiddle::TYPE_INT, Fiddle::TYPE_VOIDP],
170
+ Fiddle::TYPE_INT
171
+ )
172
+ end
173
+ end
174
+
175
+ # Lazy, so requiring the file on Linux or a stripped macOS stays inert
176
+ # (mirrors Sandbox::LibC, #178).
177
+ module LibSandbox
178
+ module_function
179
+
180
+ def handle
181
+ @handle ||= Fiddle.dlopen(Seatbelt::LIBSANDBOX)
182
+ end
183
+
184
+ def sandbox_init_fn
185
+ @sandbox_init_fn ||= Fiddle::Function.new(
186
+ handle["sandbox_init"],
187
+ [Fiddle::TYPE_VOIDP, Fiddle::TYPE_LONG_LONG, Fiddle::TYPE_VOIDP],
188
+ Fiddle::TYPE_INT
189
+ )
190
+ end
191
+ end
192
+
193
+ # A Landlock-shaped Seatbelt profile: reads, execution, network and
194
+ # IPC stay allowed everywhere (allow default); only writes are
195
+ # denied, then re-allowed under the granted dirs and /dev/null -
196
+ # exactly the Sandbox grant semantics. allow-default is deliberate:
197
+ # deny-default profiles break on macOS updates whenever Apple adds a
198
+ # mach service the profile did not name, the failure mode behind the
199
+ # macOS 26 sandbox breakage reports (#436).
200
+ module Profile
201
+ HEADER = "(version 1)\n"
202
+
203
+ module_function
204
+
205
+ # inherited_files: already-open logs the child must keep writing
206
+ # (#234) - granted per file, never the cache dir that holds them.
207
+ def build(writable:, inherited_files: [])
208
+ lines = [HEADER, "(allow default)\n", "(deny file-write*)\n"]
209
+ writable.each do |dir|
210
+ lines << %((allow file-write* (subpath "#{escape(dir)}"))\n)
211
+ end
212
+ (inherited_files + [File::NULL]).uniq.each do |file|
213
+ lines << %((allow file-write* (literal "#{escape(file)}"))\n)
214
+ end
215
+ lines.join
216
+ end
217
+
218
+ # SBPL string literals are double-quoted with backslash escapes; a
219
+ # grant path must never inject profile source.
220
+ def escape(path)
221
+ path.gsub(/["\\]/) { |char| "\\#{char}" }
222
+ end
223
+ end
224
+ end
225
+ end
226
+ end
@@ -1,6 +1,8 @@
1
1
  # SPDX-License-Identifier: MIT
2
2
  # frozen_string_literal: true
3
3
 
4
+ require "securerandom"
5
+
4
6
  require_relative "process_tree"
5
7
  require_relative "protocol"
6
8
 
@@ -21,13 +23,23 @@ module Saboteur
21
23
  # 4. Refusing recursion: the child is marked with CHILD_ENV, and Engine
22
24
  # refuses to start a run inside a marked process.
23
25
  # 5. On Linux, the supervisor is a child subreaper, so escaped orphans are
24
- # re-parented to it instead of init and stay in its tree.
26
+ # re-parented to it instead of init and stay reaped by it.
27
+ # 6. Every child carries a unique marker in its environment; the tree
28
+ # poll can miss a descendant that leaves the group and loses its
29
+ # intermediate parent between polls (#365), so kill_all also sweeps
30
+ # the process table for the marker, which survives both escapes.
25
31
  class Supervisor
26
32
  # Set in every child to its nesting depth (1 for a child of a top-level
27
33
  # run). Engine refuses to start when it is set; the supervisor itself
28
34
  # refuses to fork past MAX_DEPTH, so even a run with that guard removed
29
35
  # (a mutant of the guard, say) cannot recurse without bound.
30
36
  CHILD_ENV = "SABOTEUR_CHILD"
37
+ # Colon-joined marker list in a child's environment: the run's own
38
+ # token last, its ancestors' before it, so any ancestor's sweep still
39
+ # names a leaked nested run. See ProcessTree's marker sweep.
40
+ TREE_ENV = "SABOTEUR_TREE"
41
+ # The comm prefix of a supervised child; comm holds 15 bytes.
42
+ COMM_PREFIX = "sbt"
31
43
  MAX_DEPTH = 4
32
44
  # The hard CPU limit, as a multiple of the budget (see limit_cpu).
33
45
  HARD_CPU_FACTOR = 20
@@ -47,8 +59,8 @@ module Saboteur
47
59
  # least half of the memory limit, the evidence that a SIGKILL may be
48
60
  # an out-of-memory kill.
49
61
  Result = Data.define(:payload, :exit_status, :stopped_by, :detail, :runtime, :leftover_pids,
50
- :supervisor_killed, :memory_pressure) do
51
- def initialize(supervisor_killed: false, memory_pressure: false, **fields)
62
+ :leftover_notes, :supervisor_killed, :memory_pressure) do
63
+ def initialize(leftover_notes: [], supervisor_killed: false, memory_pressure: false, **fields)
52
64
  super
53
65
  end
54
66
 
@@ -119,7 +131,7 @@ module Saboteur
119
131
  # on_tick is called from the parent's wait loop after the fork, about
120
132
  # every poll, so callers can report progress without a thread alive
121
133
  # at fork time.
122
- def run(timeout:, cpu_seconds: nil, on_tick: nil, &block)
134
+ def run(timeout:, cpu_seconds: nil, on_tick: nil, label: "supervised child died", &block)
123
135
  depth = ENV.fetch(CHILD_ENV, "0").to_i
124
136
  raise Error, "refusing to fork: saboteur children nested #{depth} deep (limit #{MAX_DEPTH})" if
125
137
  depth >= MAX_DEPTH
@@ -131,6 +143,10 @@ module Saboteur
131
143
  started = monotonic
132
144
  @peak_growth_kb = 0
133
145
  supervisor = Process.pid
146
+ @child_label = label
147
+ # Read by the forked child through its copy of this process, like
148
+ # @child_label above.
149
+ @marker = SecureRandom.hex(6)
134
150
  pid = fork { child(reader, writer, supervisor, cpu_seconds, depth + 1, &block) }
135
151
  writer.close
136
152
  contain(pid, reader, started, timeout, on_tick)
@@ -146,13 +162,29 @@ module Saboteur
146
162
  reader.close
147
163
  self.class.close_in_child.each { |io| io.close unless io.closed? }
148
164
  ENV[CHILD_ENV] = depth.to_s
149
- Process.setpgid(0, 0)
165
+ ENV[TREE_ENV] = [ENV.fetch(TREE_ENV, nil), @marker].compact.join(":")
166
+ MarkerSweep.write_comm(@marker)
167
+ # Best-effort like the parent-side own_group: macOS can refuse the
168
+ # group change (EPERM), and the child must still run (#296).
169
+ own_group(0)
150
170
  limit_cpu(cpu_seconds)
151
171
  trap("TERM") { exit!(143) }
152
172
  watch_parent(supervisor)
153
173
  yield writer
154
174
  exit!(0)
155
- rescue Exception # rubocop:disable Lint/RescueException
175
+ rescue Exception => e # rubocop:disable Lint/RescueException
176
+ # The pipe stays empty when the child dies before answering, so this
177
+ # is the only place the cause survives (#255). The warning is
178
+ # best-effort and the exit is ensured: a broken stderr must not let
179
+ # the exception unwind past the fork block into CLI code in the
180
+ # child, where exit handlers could emit duplicate output (#289).
181
+ begin
182
+ frames = e.backtrace ? e.backtrace.first(8).join("\n from ") : "(no backtrace)"
183
+ warn("saboteur: #{@child_label}: #{e.class}: #{e.message}\n from #{frames}")
184
+ rescue Exception # rubocop:disable Lint/RescueException
185
+ nil
186
+ end
187
+ ensure
156
188
  exit!(1)
157
189
  end
158
190
 
@@ -195,7 +227,7 @@ module Saboteur
195
227
  # --- parent ---------------------------------------------------------
196
228
 
197
229
  def contain(pid, reader, started, timeout, on_tick)
198
- tree = ProcessTree.new(pid)
230
+ tree = ProcessTree.new(pid, marker: @marker)
199
231
  self.class.live[pid] = tree
200
232
  own_group(pid)
201
233
  deadline = started + timeout
@@ -211,27 +243,27 @@ module Saboteur
211
243
  # (progress IO failed) or any other error must not skip the
212
244
  # kill/reap, or the child and its tree would run unsupervised
213
245
  # until this process exits.
214
- exit_status, supervisor_killed, leftover_pids = release(pid, tree, exit_status, stop, grace_expired)
246
+ exit_status, supervisor_killed, leftover_pids, notes = release(pid, tree, exit_status, stop, grace_expired)
215
247
  end
216
248
  Result.new(payload: payload, exit_status: exit_status, stopped_by: stop, detail: detail,
217
- runtime: monotonic - started, leftover_pids: leftover_pids,
249
+ runtime: monotonic - started, leftover_pids: leftover_pids, leftover_notes: notes,
218
250
  supervisor_killed: supervisor_killed, memory_pressure: memory_pressure?)
219
251
  end
220
252
 
221
- # Kills and reaps the child and everything it started. Returns
222
- # [exit_status, supervisor_killed, leftover_pids].
253
+ # Kills and reaps the child and its tree; returns [exit_status, supervisor_killed, leftover_pids, notes].
223
254
  def release(pid, tree, exit_status, stop, grace_expired)
224
255
  exit_status ||= terminate(pid) if stop == :timeout
225
256
  supervisor_killed = false
226
257
  # The grace ran out with the child seemingly alive. What ends it is
227
258
  # only ours to claim if we see it alive: stop it first.
228
259
  exit_status, supervisor_killed = freeze_child(pid) if grace_expired
229
- leftover_pids = tree.kill_all - [pid]
260
+ # Notes name the leftovers before the kill erases their command lines (#486).
261
+ leftover_pids, notes = tree.kill_all_except(pid)
230
262
  exit_status ||= reap(pid)
231
263
  self.class.live.delete(pid)
232
264
  reap_adopted(leftover_pids)
233
265
  reap_group(pid)
234
- [exit_status, supervisor_killed, leftover_pids]
266
+ [exit_status, supervisor_killed, leftover_pids, notes]
235
267
  end
236
268
 
237
269
  # After EOF, waits for the child to exit by itself while still checking
@@ -2,6 +2,8 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  require "json"
5
+ require_relative "../text"
6
+ require_relative "peak_memory"
5
7
  require_relative "process_tree"
6
8
  require_relative "supervisor"
7
9
 
@@ -25,16 +27,54 @@ module Saboteur
25
27
  class WorkerPool
26
28
  # done: items finished; status: the Process::Status once reaped;
27
29
  # last_words: what the worker reported before dying, if anything.
28
- Worker = Struct.new(:number, :pid, :commands, :results, :current, :done, :status, :last_words)
30
+ Worker = Struct.new(:number, :pid, :commands, :results, :current, :done, :status, :last_words, :peak_rss_mb)
29
31
  START_FAILURES = 3
30
32
 
31
33
  # on_start: called in each worker with its number before its first
32
34
  # item (per-worker database connections, for example).
33
- def initialize(jobs:, on_start: nil, on_result: nil)
35
+ # worker_peaks: worker number => peak RSS in MB, filled as workers
36
+ # retire (#202). Empty for an inline (jobs 1) run.
37
+ attr_reader :worker_peaks
38
+
39
+ def initialize(jobs:, on_start: nil, on_result: nil, drain_timeout: 30)
40
+ @worker_peaks = {}
34
41
  @jobs = jobs
35
42
  @on_start = on_start
36
43
  @on_result = on_result
37
44
  @start_failures = 0
45
+ @halted = false
46
+ @drain_timeout = drain_timeout
47
+ end
48
+
49
+ # Stop handing queued items out while letting in-flight items finish
50
+ # and report: a worker that merely noticed the damage can report
51
+ # before the worker whose mutant caused it, and the later report is
52
+ # the one that names the culprit (#285). Unlike raising from
53
+ # on_result, nothing in flight is killed. Unstarted items come back
54
+ # as nil holes in the result array.
55
+ #
56
+ # drain: true bounds the wait for a safety abort (tool error, canary):
57
+ # a wedged worker must not stall it, so drain_timeout seconds after
58
+ # the first halt the pool stops waiting, kills whatever is still
59
+ # running and leaves nil holes (#285 review). drain: false is the
60
+ # ordinary --max-duration halt: in-flight subjects always finish and
61
+ # report, so a nil hole only ever means "never started"; the
62
+ # overshoot stays bounded by each worker's own mutant deadlines.
63
+ def halt!(drain: true)
64
+ @halted = true
65
+ return if @drain_deadline || !drain
66
+
67
+ @drain_deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @drain_timeout
68
+ end
69
+
70
+ # The serialized shape of a tool error row: the message plus the
71
+ # attribution flags the parent needs to rank reports - a per-mutant
72
+ # culprit digest first, then a canary trip (#283, #285).
73
+ def self.tool_error_row(error)
74
+ row = { "tool_error" => error.message }
75
+ row["canary"] = true if error.is_a?(CanaryTripped)
76
+ row["culprit"] = error.culprit_digest if error.respond_to?(:culprit_digest) && error.culprit_digest
77
+ row
38
78
  end
39
79
 
40
80
  def map(items, weights: nil, &work)
@@ -64,8 +104,20 @@ module Saboteur
64
104
 
65
105
  # Waits up to a second for results, then checks for workers that died
66
106
  # without closing their pipe (a stray grandchild may still hold it).
107
+ # While halted the wait is bounded by the drain deadline: past it the
108
+ # remaining workers are retired (stop kills a wedged one) and their
109
+ # items come back as nil holes.
67
110
  def collect(workers, queue, results, items, &work)
68
- ready = IO.select(workers.map(&:results), nil, nil, 1)&.first || []
111
+ timeout = 1
112
+ if @drain_deadline
113
+ remaining = @drain_deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
114
+ if remaining <= 0
115
+ abandon_all(workers)
116
+ return
117
+ end
118
+ timeout = [timeout, remaining].min
119
+ end
120
+ ready = IO.select(workers.map(&:results), nil, nil, timeout)&.first || []
69
121
  ready.each do |io|
70
122
  worker = workers.find { |candidate| candidate.results == io }
71
123
  receive(worker, workers, queue, results, items, &work) if worker
@@ -82,7 +134,7 @@ module Saboteur
82
134
  take_and_report(worker, line, results) if line
83
135
  return replace(worker, workers, queue, results, items, &) if line.nil? || worker.last_words
84
136
 
85
- assign(worker, queue)
137
+ assign(worker, queue) unless @halted
86
138
  retire(worker, workers) unless worker.current
87
139
  end
88
140
 
@@ -96,7 +148,7 @@ module Saboteur
96
148
  end
97
149
 
98
150
  def parse(line)
99
- JSON.parse(line)
151
+ JSON.parse(Text.utf8_strict(line))
100
152
  rescue JSON::ParserError => e
101
153
  { "error" => "unreadable result from worker: #{e.message}" }
102
154
  end
@@ -108,6 +160,10 @@ module Saboteur
108
160
  # Stores a line from the worker: its item's result, or why it crashed.
109
161
  def take(worker, line, results)
110
162
  data = parse(line)
163
+ if data.key?("peak_rss_mb")
164
+ @worker_peaks[data.delete("worker") || worker.number] = worker.peak_rss_mb = data["peak_rss_mb"]
165
+ return
166
+ end
111
167
  return worker.last_words = data["fatal"] if data.key?("fatal")
112
168
  return unless worker.current
113
169
 
@@ -127,6 +183,28 @@ module Saboteur
127
183
  nil
128
184
  end
129
185
 
186
+ # Past the drain deadline nothing more is coming: kill every wedged
187
+ # worker's tree first, then reap, so the cleanup costs one round
188
+ # instead of one stop grace per worker (#285 review). Items in
189
+ # flight come back as nil holes.
190
+ def abandon_all(workers)
191
+ wedged = workers.dup
192
+ wedged.each do |worker|
193
+ worker.commands.close unless worker.commands.closed?
194
+ Supervisor.live.delete(worker.pid)&.kill_all
195
+ end
196
+ # Every wedged tree dies before any wait: one cleanup round.
197
+ # rubocop:disable-next Style/CombinableLoops
198
+ wedged.each do |worker|
199
+ worker.status ||= Process.wait2(worker.pid).last unless reaped?(worker)
200
+ worker.results.close unless worker.results.closed?
201
+ rescue Errno::ECHILD
202
+ nil
203
+ ensure
204
+ workers.delete(worker)
205
+ end
206
+ end
207
+
130
208
  def replace(worker, workers, queue, results, items, &)
131
209
  drain(worker, results)
132
210
  retire(worker, workers)
@@ -138,8 +216,11 @@ module Saboteur
138
216
  results[worker.current] = { "error" => "#{reason}, running this item" }
139
217
  @on_result&.call(worker.current, results[worker.current])
140
218
  end
141
- give_up(reason) if worker.done.zero?
142
- return if queue.empty?
219
+ # Halted pools start nothing, so a start-failure storm cannot be the
220
+ # run's problem then; raising it would mask the rows that carry the
221
+ # real abort cause (#286).
222
+ give_up(reason) if worker.done.zero? && !@halted
223
+ return if @halted || queue.empty?
143
224
 
144
225
  fresh = start(worker.number, items, workers, &)
145
226
  workers << fresh
@@ -201,7 +282,10 @@ module Saboteur
201
282
  # Runs in the worker until the parent closes its command pipe. If it
202
283
  # crashes, its last line says why.
203
284
  def serve(number, items, commands, results, &)
204
- Process.setpgid(0, 0)
285
+ # Grouping helps tree cleanup find the worker's descendants; macOS
286
+ # can refuse it (EPERM), and a refused setpgid must not crash the
287
+ # worker - parent-link tracking still finds the tree (#296).
288
+ own_group(0)
205
289
  Supervisor.reset_in_worker!
206
290
  Supervisor.close_in_child.push(commands, results)
207
291
  ENV["SABOTEUR_WORKER"] = number.to_s
@@ -211,6 +295,7 @@ module Saboteur
211
295
  results.puts(JSON.generate(run_one(items[Integer(line)], &)))
212
296
  results.flush
213
297
  end
298
+ report_peak(results, number)
214
299
  exit!(0)
215
300
  rescue Exception => e # rubocop:disable Lint/RescueException
216
301
  last_words(results, "worker #{number} crashed: #{e.class}: #{e.message} (#{e.backtrace&.first})")
@@ -224,8 +309,24 @@ module Saboteur
224
309
  nil
225
310
  end
226
311
 
312
+ # Last word before exit!: the kernel's exact peak RSS of this worker,
313
+ # for the run's memory report (#202). Skipped where procfs is absent.
314
+ def report_peak(results, number)
315
+ peak = PeakMemory.peak_mb
316
+ return unless peak
317
+
318
+ results.puts(JSON.generate("worker" => number, "peak_rss_mb" => peak))
319
+ results.flush
320
+ end
321
+
227
322
  def run_one(item, &work)
228
323
  work.call(item)
324
+ rescue Error => e
325
+ # A tool error (a vanished cache dir, ...) aborts the run with
326
+ # exit 3; it is not a per-mutant outcome. Serialize it so the
327
+ # parent can re-raise it as itself (#283), with the flags the
328
+ # parent needs to rank reports by attribution quality (#285).
329
+ self.class.tool_error_row(e)
229
330
  rescue StandardError => e
230
331
  { "error" => "#{e.class}: #{e.message}" }
231
332
  end
@@ -239,6 +340,7 @@ module Saboteur
239
340
  sleep 0.01 until reaped?(worker) || Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
240
341
  tree&.kill_all
241
342
  worker.status ||= Process.wait2(worker.pid).last unless reaped?(worker)
343
+ drain(worker, {})
242
344
  worker.results.close unless worker.results.closed?
243
345
  rescue Errno::ECHILD
244
346
  nil
@@ -0,0 +1,49 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ module Frameworks
6
+ # The #387 shape: a suite connects ActiveRecord to a sqlite ":memory:"
7
+ # database and loads its schema at require time. Every saboteur child is
8
+ # a fork, ActiveRecord's fork-safe pool reconnects there, and a fresh
9
+ # in-memory database is EMPTY - the schema is gone, and every example
10
+ # fails with "Could not find table" while plain rspec stays green. A
11
+ # child that detects the shape says so and names the remedies instead of
12
+ # sending the user debugging a healthy suite.
13
+ module SqliteMemoryDatabase
14
+ REMEDY = "a sqlite :memory: database is empty in this forked child: in-memory databases do not " \
15
+ "survive saboteur's fork, so a schema loaded at require time is lost. Load the schema in " \
16
+ "a before(:suite) hook, or in an after_fork hook (config/saboteur_hooks.rb via c.hooks)"
17
+
18
+ module_function
19
+
20
+ # The remedy text when a current pool points at an empty sqlite
21
+ # :memory: database, else nil. Diagnostic only: any error is no hint,
22
+ # never a failure of the run itself.
23
+ def fork_loss_hint
24
+ return unless defined?(ActiveRecord::Base)
25
+
26
+ pools = ActiveRecord::Base.connection_handler.connection_pool_list
27
+ pools.any? { |pool| empty_memory_database?(pool) } ? REMEDY : nil
28
+ rescue StandardError, ScriptError
29
+ nil
30
+ end
31
+
32
+ def empty_memory_database?(pool)
33
+ config = pool.db_config
34
+ return false unless config.adapter.to_s == "sqlite3" && config.database.to_s == ":memory:"
35
+
36
+ pool.with_connection { |connection| user_tables(connection).empty? }
37
+ rescue StandardError
38
+ false
39
+ end
40
+
41
+ # Migration bookkeeping is not user schema: a database with only these
42
+ # tables still has no schema for the suite's models.
43
+ def user_tables(connection)
44
+ connection.select_values("SELECT name FROM sqlite_master WHERE type = 'table'") -
45
+ %w[sqlite_sequence schema_migrations ar_internal_metadata]
46
+ end
47
+ end
48
+ end
49
+ end