saboteur 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1024 -0
- data/README.md +199 -952
- data/exe/saboteur +34 -7
- data/lib/saboteur/baseline/workflow.rb +2 -1
- data/lib/saboteur/baseline_health.rb +40 -6
- data/lib/saboteur/cli/exit_status.rb +80 -4
- data/lib/saboteur/cli/help_options.rb +17 -0
- data/lib/saboteur/cli/history_command.rb +135 -0
- data/lib/saboteur/cli/presets.rb +40 -0
- data/lib/saboteur/cli/run_budget_options.rb +37 -0
- data/lib/saboteur/cli/run_reporting.rb +74 -5
- data/lib/saboteur/cli/selection_options.rb +31 -0
- data/lib/saboteur/cli/shard_options.rb +17 -0
- data/lib/saboteur/cli/source_set.rb +53 -0
- data/lib/saboteur/cli/survivors_command.rb +157 -0
- data/lib/saboteur/cli.rb +136 -98
- data/lib/saboteur/config.rb +163 -15
- data/lib/saboteur/config_file.rb +8 -0
- data/lib/saboteur/constant_reads.rb +182 -0
- data/lib/saboteur/constant_subject.rb +70 -0
- data/lib/saboteur/constant_subject_finder.rb +281 -0
- data/lib/saboteur/coverage_reads.rb +29 -0
- data/lib/saboteur/crash_report.rb +154 -0
- data/lib/saboteur/custom_map.rb +3 -1
- data/lib/saboteur/dirty_tree_warning.rb +128 -0
- data/lib/saboteur/doctor.rb +49 -1
- data/lib/saboteur/engine/baseline_guard.rb +136 -14
- data/lib/saboteur/engine/boot.rb +16 -1
- data/lib/saboteur/engine/canary.rb +41 -10
- data/lib/saboteur/engine/candidate_selection.rb +28 -1
- data/lib/saboteur/engine/candidates.rb +43 -3
- data/lib/saboteur/engine/coverage_guard.rb +29 -0
- data/lib/saboteur/engine/coverage_store.rb +79 -15
- data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +41 -2
- data/lib/saboteur/engine/diff_scope.rb +24 -5
- data/lib/saboteur/engine/distribution.rb +134 -8
- data/lib/saboteur/engine/early_stop_recording.rb +47 -0
- data/lib/saboteur/engine/gated_run.rb +35 -0
- data/lib/saboteur/engine/load_coverage.rb +45 -0
- data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
- data/lib/saboteur/engine/pipelines.rb +107 -0
- data/lib/saboteur/engine/result_cache.rb +67 -9
- data/lib/saboteur/engine/run_budget.rb +90 -0
- data/lib/saboteur/engine/run_fingerprint.rb +4 -0
- data/lib/saboteur/engine/sandbox_policy.rb +56 -11
- data/lib/saboteur/engine/scheduler.rb +222 -20
- data/lib/saboteur/engine/selection_announcement.rb +54 -0
- data/lib/saboteur/engine/shard.rb +69 -0
- data/lib/saboteur/engine/subject_selection.rb +373 -19
- data/lib/saboteur/engine/suite_files.rb +3 -1
- data/lib/saboteur/engine/test_guess.rb +438 -25
- data/lib/saboteur/engine/thread_warnings.rb +53 -0
- data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
- data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
- data/lib/saboteur/engine/verdict_check.rb +54 -18
- data/lib/saboteur/engine.rb +202 -158
- data/lib/saboteur/execution/external_runner.rb +471 -0
- data/lib/saboteur/execution/marker_sweep.rb +79 -0
- data/lib/saboteur/execution/peak_memory.rb +28 -0
- data/lib/saboteur/execution/process_table.rb +12 -0
- data/lib/saboteur/execution/process_tree.rb +65 -5
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/reevaluator.rb +68 -12
- data/lib/saboteur/execution/runner.rb +225 -24
- data/lib/saboteur/execution/sandbox.rb +21 -3
- data/lib/saboteur/execution/seatbelt.rb +226 -0
- data/lib/saboteur/execution/supervisor.rb +45 -13
- data/lib/saboteur/execution/worker_pool.rb +110 -8
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
- data/lib/saboteur/gates.rb +43 -2
- data/lib/saboteur/help.rb +108 -13
- data/lib/saboteur/ignore_list.rb +40 -7
- data/lib/saboteur/install.rb +4 -3
- data/lib/saboteur/integration/base.rb +7 -2
- data/lib/saboteur/integration/rspec.rb +175 -9
- data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
- data/lib/saboteur/mutation_source/builtin.rb +44 -14
- data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
- data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
- data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
- data/lib/saboteur/mutation_source/element_removal.rb +73 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
- data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +129 -3
- data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/method_swap.rb +193 -0
- data/lib/saboteur/mutation_source/mutation.rb +5 -3
- data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
- data/lib/saboteur/mutation_source/operator_examples.rb +6 -1
- data/lib/saboteur/mutation_source/operator_registry.rb +7 -2
- data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
- data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
- data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
- data/lib/saboteur/preview.rb +122 -13
- data/lib/saboteur/refinement_activations.rb +60 -0
- data/lib/saboteur/reporter/agent.rb +9 -1
- data/lib/saboteur/reporter/agent_commands.rb +47 -15
- data/lib/saboteur/reporter/agent_incomplete.rb +38 -2
- data/lib/saboteur/reporter/agent_summary.rb +4 -3
- data/lib/saboteur/reporter/findings.rb +5 -2
- data/lib/saboteur/reporter/history_policy.rb +11 -0
- data/lib/saboteur/reporter/history_reader.rb +47 -0
- data/lib/saboteur/reporter/html.rb +26 -4
- data/lib/saboteur/reporter/mutation_text.rb +6 -0
- data/lib/saboteur/reporter/next_steps.rb +14 -4
- data/lib/saboteur/reporter/score_history.rb +5 -1
- data/lib/saboteur/reporter/stryker.rb +40 -7
- data/lib/saboteur/reporter/survivor_classification.rb +101 -0
- data/lib/saboteur/reporter/survivor_hints.rb +3 -0
- data/lib/saboteur/reporter/survivor_list.rb +15 -9
- data/lib/saboteur/reporter/terminal.rb +74 -17
- data/lib/saboteur/result.rb +38 -7
- data/lib/saboteur/selection/coverage_map.rb +107 -11
- data/lib/saboteur/selection/dependency_shield.rb +80 -0
- data/lib/saboteur/selection/external_test_map.rb +27 -0
- data/lib/saboteur/selection/git_diff.rb +105 -13
- data/lib/saboteur/selection/layout_only_lines.rb +80 -0
- data/lib/saboteur/selection/recorder.rb +22 -4
- data/lib/saboteur/selection/source_scope.rb +22 -0
- data/lib/saboteur/selection/untracked_changes.rb +126 -0
- data/lib/saboteur/subject.rb +53 -7
- data/lib/saboteur/subject_finder.rb +173 -29
- data/lib/saboteur/text.rb +42 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +38 -15
- data/lib/saboteur.rb +29 -1
- metadata +50 -2
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "fiddle"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
module Execution
|
|
8
|
+
# The macOS counterpart of Sandbox (Landlock): a Seatbelt profile
|
|
9
|
+
# applied with sandbox_init inside the mutant child. sandbox-exec and
|
|
10
|
+
# sandbox_init are deprecated by Apple with no published replacement
|
|
11
|
+
# and no removal timeline (#436 tracks how Apple, Chrome, Codex,
|
|
12
|
+
# Claude Code, Bazel and Nix respond), so Seatbelt is opt-in and the
|
|
13
|
+
# probe below fails closed: a child that cannot be confined runs with
|
|
14
|
+
# the hazard guard and canary only, never silently "sandboxed".
|
|
15
|
+
#
|
|
16
|
+
# Like Sandbox, requiring this file is inert off macOS: libsandbox is
|
|
17
|
+
# dlopened lazily by the first probe, and the probe forks so the
|
|
18
|
+
# irreversible sandbox_init never touches the run's own process.
|
|
19
|
+
module Seatbelt
|
|
20
|
+
LIBSANDBOX = "/usr/lib/system/libsystem_sandbox.dylib"
|
|
21
|
+
LIBSYSTEM = "/usr/lib/libSystem.B.dylib"
|
|
22
|
+
F_GETPATH = 50
|
|
23
|
+
PATH_MAX = 1024
|
|
24
|
+
|
|
25
|
+
# The probe profile must deny the write it then attempts.
|
|
26
|
+
PROBE_PROFILE = "(version 1)(deny default)"
|
|
27
|
+
|
|
28
|
+
# sandbox_init error texts that say the facility is genuinely gone
|
|
29
|
+
# (a future macOS without Seatbelt): degrade, noted once. Anything
|
|
30
|
+
# else (a profile syntax tightening, EPERM, a lost child) fails
|
|
31
|
+
# closed: SandboxPolicy#seatbelt_hook refuses the run rather than
|
|
32
|
+
# confining nothing, exactly like the Landlock probe (#178) - never
|
|
33
|
+
# silently unsandboxed.
|
|
34
|
+
UNSUPPORTED_ERRORS = ["operation not supported", "function not implemented"].freeze
|
|
35
|
+
|
|
36
|
+
module_function
|
|
37
|
+
|
|
38
|
+
def darwin?
|
|
39
|
+
RUBY_PLATFORM.include?("darwin")
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# One forked behavioral probe, memoized per process:
|
|
43
|
+
# :unsupported_platform off macOS, :ok when a deny-default profile
|
|
44
|
+
# really blocks a write, :unavailable when sandbox_init reports the
|
|
45
|
+
# facility missing, :unenforced when the API claims success but the
|
|
46
|
+
# write goes through, :error for anything else. Behavioral, not
|
|
47
|
+
# symbol presence: a macOS update that rejects the profile language
|
|
48
|
+
# shows up here, not in a dlopen.
|
|
49
|
+
def probe
|
|
50
|
+
return @probe if defined?(@probe)
|
|
51
|
+
return @probe = :unsupported_platform unless darwin?
|
|
52
|
+
|
|
53
|
+
@probe = run_probe
|
|
54
|
+
rescue Fiddle::DLError
|
|
55
|
+
@probe = :unavailable
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def available?
|
|
59
|
+
probe == :ok
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Called at the top of a mutant child, mirroring Sandbox.restrict!:
|
|
63
|
+
# irreversible and fail-closed - a child whose profile does not apply
|
|
64
|
+
# must not run the mutant. keep_fds is Sandbox's hook contract
|
|
65
|
+
# (#234): Landlock gates opens, so its children keep writing through
|
|
66
|
+
# descriptors the parent opened (their logs); Seatbelt checks
|
|
67
|
+
# file-write* on EVERY write, so those inherited files join the
|
|
68
|
+
# profile as literal grants instead. Pipes and sockets among them
|
|
69
|
+
# have no path and need no grant.
|
|
70
|
+
def restrict!(writable:, keep_fds: [])
|
|
71
|
+
profile = Profile.build(writable: writable.map { |dir| resolved_grant(dir) },
|
|
72
|
+
inherited_files: fd_paths(keep_fds))
|
|
73
|
+
rc, message = sandbox_init(profile)
|
|
74
|
+
unless rc.zero?
|
|
75
|
+
raise Error, "the Seatbelt sandbox refused to engage (#{message}); the mutant does not run unconfined"
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
nil
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Seatbelt compares the kernel's resolved path, so a grant spelled
|
|
82
|
+
# through a symlink (macOS /var is /private/var) never matches and the
|
|
83
|
+
# "granted" write is denied. Resolve what exists; a dir not yet
|
|
84
|
+
# created grants nothing either way, so keep its spelling rather than
|
|
85
|
+
# fail the child on a dead grant. fcntl(F_GETPATH) already returns
|
|
86
|
+
# the resolved path for inherited files, so they need no such pass.
|
|
87
|
+
def resolved_grant(dir)
|
|
88
|
+
File.exist?(dir) ? File.realpath(dir) : dir
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# fcntl(F_GETPATH) maps an inherited descriptor back to its path; a
|
|
92
|
+
# descriptor without one (pipe, socket) is skipped.
|
|
93
|
+
def fd_paths(keep_fds)
|
|
94
|
+
keep_fds.filter_map do |fd|
|
|
95
|
+
buffer = Fiddle::Pointer.malloc(PATH_MAX)
|
|
96
|
+
LibSystem.fcntl_fn.call(fd, F_GETPATH, buffer).negative? ? nil : buffer.to_s
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def classify(verdict)
|
|
101
|
+
case verdict
|
|
102
|
+
when "ok" then :ok
|
|
103
|
+
when "unenforced" then :unenforced
|
|
104
|
+
when /\Ainit_failed:(?<message>.*)\z/m
|
|
105
|
+
message = Regexp.last_match[:message].downcase
|
|
106
|
+
UNSUPPORTED_ERRORS.any? { |text| message.include?(text) } ? :unavailable : :error
|
|
107
|
+
else
|
|
108
|
+
:error
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def run_probe
|
|
113
|
+
reader, writer = IO.pipe
|
|
114
|
+
pid = fork { probe_child(writer) }
|
|
115
|
+
writer.close
|
|
116
|
+
verdict = reader.read
|
|
117
|
+
reader.close
|
|
118
|
+
Process.waitpid(pid)
|
|
119
|
+
classify(verdict)
|
|
120
|
+
ensure
|
|
121
|
+
reader.close unless reader.closed?
|
|
122
|
+
writer.close unless writer.closed?
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Child only: confine, then attempt the write the profile must block.
|
|
126
|
+
# The parent reads a single verdict word; exit! so no at_exit hook or
|
|
127
|
+
# finalizer runs under the sandbox.
|
|
128
|
+
def probe_child(writer)
|
|
129
|
+
rc, message = sandbox_init(PROBE_PROFILE)
|
|
130
|
+
verdict =
|
|
131
|
+
if rc.zero?
|
|
132
|
+
begin
|
|
133
|
+
File.open(File::NULL, "w") { true }
|
|
134
|
+
"unenforced"
|
|
135
|
+
rescue SystemCallError, IOError
|
|
136
|
+
"ok"
|
|
137
|
+
end
|
|
138
|
+
else
|
|
139
|
+
"init_failed:#{message}"
|
|
140
|
+
end
|
|
141
|
+
writer.write(verdict)
|
|
142
|
+
writer.close
|
|
143
|
+
exit!(0)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# int sandbox_init(const char *profile, uint64_t flags, char **errorbuf)
|
|
147
|
+
def sandbox_init(profile)
|
|
148
|
+
errbuf = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP)
|
|
149
|
+
rc = LibSandbox.sandbox_init_fn.call(profile, 0, errbuf)
|
|
150
|
+
return [0, nil] if rc.zero?
|
|
151
|
+
|
|
152
|
+
# errorbuf is a char**: dereference once to reach the message.
|
|
153
|
+
message = begin
|
|
154
|
+
address = errbuf[0, Fiddle::SIZEOF_VOIDP].unpack1("J")
|
|
155
|
+
address&.positive? ? Fiddle::Pointer.new(address).to_s : "unknown"
|
|
156
|
+
rescue StandardError
|
|
157
|
+
"unknown"
|
|
158
|
+
end
|
|
159
|
+
[rc, "sandbox_init: #{message}"]
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# fcntl for the F_GETPATH lookup; lazy like LibSandbox.
|
|
163
|
+
module LibSystem
|
|
164
|
+
module_function
|
|
165
|
+
|
|
166
|
+
def fcntl_fn
|
|
167
|
+
@fcntl_fn ||= Fiddle::Function.new(
|
|
168
|
+
Fiddle.dlopen(Seatbelt::LIBSYSTEM)["fcntl"],
|
|
169
|
+
[Fiddle::TYPE_INT, Fiddle::TYPE_INT, Fiddle::TYPE_VOIDP],
|
|
170
|
+
Fiddle::TYPE_INT
|
|
171
|
+
)
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# Lazy, so requiring the file on Linux or a stripped macOS stays inert
|
|
176
|
+
# (mirrors Sandbox::LibC, #178).
|
|
177
|
+
module LibSandbox
|
|
178
|
+
module_function
|
|
179
|
+
|
|
180
|
+
def handle
|
|
181
|
+
@handle ||= Fiddle.dlopen(Seatbelt::LIBSANDBOX)
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
def sandbox_init_fn
|
|
185
|
+
@sandbox_init_fn ||= Fiddle::Function.new(
|
|
186
|
+
handle["sandbox_init"],
|
|
187
|
+
[Fiddle::TYPE_VOIDP, Fiddle::TYPE_LONG_LONG, Fiddle::TYPE_VOIDP],
|
|
188
|
+
Fiddle::TYPE_INT
|
|
189
|
+
)
|
|
190
|
+
end
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# A Landlock-shaped Seatbelt profile: reads, execution, network and
|
|
194
|
+
# IPC stay allowed everywhere (allow default); only writes are
|
|
195
|
+
# denied, then re-allowed under the granted dirs and /dev/null -
|
|
196
|
+
# exactly the Sandbox grant semantics. allow-default is deliberate:
|
|
197
|
+
# deny-default profiles break on macOS updates whenever Apple adds a
|
|
198
|
+
# mach service the profile did not name, the failure mode behind the
|
|
199
|
+
# macOS 26 sandbox breakage reports (#436).
|
|
200
|
+
module Profile
|
|
201
|
+
HEADER = "(version 1)\n"
|
|
202
|
+
|
|
203
|
+
module_function
|
|
204
|
+
|
|
205
|
+
# inherited_files: already-open logs the child must keep writing
|
|
206
|
+
# (#234) - granted per file, never the cache dir that holds them.
|
|
207
|
+
def build(writable:, inherited_files: [])
|
|
208
|
+
lines = [HEADER, "(allow default)\n", "(deny file-write*)\n"]
|
|
209
|
+
writable.each do |dir|
|
|
210
|
+
lines << %((allow file-write* (subpath "#{escape(dir)}"))\n)
|
|
211
|
+
end
|
|
212
|
+
(inherited_files + [File::NULL]).uniq.each do |file|
|
|
213
|
+
lines << %((allow file-write* (literal "#{escape(file)}"))\n)
|
|
214
|
+
end
|
|
215
|
+
lines.join
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# SBPL string literals are double-quoted with backslash escapes; a
|
|
219
|
+
# grant path must never inject profile source.
|
|
220
|
+
def escape(path)
|
|
221
|
+
path.gsub(/["\\]/) { |char| "\\#{char}" }
|
|
222
|
+
end
|
|
223
|
+
end
|
|
224
|
+
end
|
|
225
|
+
end
|
|
226
|
+
end
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require "securerandom"
|
|
5
|
+
|
|
4
6
|
require_relative "process_tree"
|
|
5
7
|
require_relative "protocol"
|
|
6
8
|
|
|
@@ -21,13 +23,23 @@ module Saboteur
|
|
|
21
23
|
# 4. Refusing recursion: the child is marked with CHILD_ENV, and Engine
|
|
22
24
|
# refuses to start a run inside a marked process.
|
|
23
25
|
# 5. On Linux, the supervisor is a child subreaper, so escaped orphans are
|
|
24
|
-
# re-parented to it instead of init and stay
|
|
26
|
+
# re-parented to it instead of init and stay reaped by it.
|
|
27
|
+
# 6. Every child carries a unique marker in its environment; the tree
|
|
28
|
+
# poll can miss a descendant that leaves the group and loses its
|
|
29
|
+
# intermediate parent between polls (#365), so kill_all also sweeps
|
|
30
|
+
# the process table for the marker, which survives both escapes.
|
|
25
31
|
class Supervisor
|
|
26
32
|
# Set in every child to its nesting depth (1 for a child of a top-level
|
|
27
33
|
# run). Engine refuses to start when it is set; the supervisor itself
|
|
28
34
|
# refuses to fork past MAX_DEPTH, so even a run with that guard removed
|
|
29
35
|
# (a mutant of the guard, say) cannot recurse without bound.
|
|
30
36
|
CHILD_ENV = "SABOTEUR_CHILD"
|
|
37
|
+
# Colon-joined marker list in a child's environment: the run's own
|
|
38
|
+
# token last, its ancestors' before it, so any ancestor's sweep still
|
|
39
|
+
# names a leaked nested run. See ProcessTree's marker sweep.
|
|
40
|
+
TREE_ENV = "SABOTEUR_TREE"
|
|
41
|
+
# The comm prefix of a supervised child; comm holds 15 bytes.
|
|
42
|
+
COMM_PREFIX = "sbt"
|
|
31
43
|
MAX_DEPTH = 4
|
|
32
44
|
# The hard CPU limit, as a multiple of the budget (see limit_cpu).
|
|
33
45
|
HARD_CPU_FACTOR = 20
|
|
@@ -47,8 +59,8 @@ module Saboteur
|
|
|
47
59
|
# least half of the memory limit, the evidence that a SIGKILL may be
|
|
48
60
|
# an out-of-memory kill.
|
|
49
61
|
Result = Data.define(:payload, :exit_status, :stopped_by, :detail, :runtime, :leftover_pids,
|
|
50
|
-
:supervisor_killed, :memory_pressure) do
|
|
51
|
-
def initialize(supervisor_killed: false, memory_pressure: false, **fields)
|
|
62
|
+
:leftover_notes, :supervisor_killed, :memory_pressure) do
|
|
63
|
+
def initialize(leftover_notes: [], supervisor_killed: false, memory_pressure: false, **fields)
|
|
52
64
|
super
|
|
53
65
|
end
|
|
54
66
|
|
|
@@ -119,7 +131,7 @@ module Saboteur
|
|
|
119
131
|
# on_tick is called from the parent's wait loop after the fork, about
|
|
120
132
|
# every poll, so callers can report progress without a thread alive
|
|
121
133
|
# at fork time.
|
|
122
|
-
def run(timeout:, cpu_seconds: nil, on_tick: nil, &block)
|
|
134
|
+
def run(timeout:, cpu_seconds: nil, on_tick: nil, label: "supervised child died", &block)
|
|
123
135
|
depth = ENV.fetch(CHILD_ENV, "0").to_i
|
|
124
136
|
raise Error, "refusing to fork: saboteur children nested #{depth} deep (limit #{MAX_DEPTH})" if
|
|
125
137
|
depth >= MAX_DEPTH
|
|
@@ -131,6 +143,10 @@ module Saboteur
|
|
|
131
143
|
started = monotonic
|
|
132
144
|
@peak_growth_kb = 0
|
|
133
145
|
supervisor = Process.pid
|
|
146
|
+
@child_label = label
|
|
147
|
+
# Read by the forked child through its copy of this process, like
|
|
148
|
+
# @child_label above.
|
|
149
|
+
@marker = SecureRandom.hex(6)
|
|
134
150
|
pid = fork { child(reader, writer, supervisor, cpu_seconds, depth + 1, &block) }
|
|
135
151
|
writer.close
|
|
136
152
|
contain(pid, reader, started, timeout, on_tick)
|
|
@@ -146,13 +162,29 @@ module Saboteur
|
|
|
146
162
|
reader.close
|
|
147
163
|
self.class.close_in_child.each { |io| io.close unless io.closed? }
|
|
148
164
|
ENV[CHILD_ENV] = depth.to_s
|
|
149
|
-
|
|
165
|
+
ENV[TREE_ENV] = [ENV.fetch(TREE_ENV, nil), @marker].compact.join(":")
|
|
166
|
+
MarkerSweep.write_comm(@marker)
|
|
167
|
+
# Best-effort like the parent-side own_group: macOS can refuse the
|
|
168
|
+
# group change (EPERM), and the child must still run (#296).
|
|
169
|
+
own_group(0)
|
|
150
170
|
limit_cpu(cpu_seconds)
|
|
151
171
|
trap("TERM") { exit!(143) }
|
|
152
172
|
watch_parent(supervisor)
|
|
153
173
|
yield writer
|
|
154
174
|
exit!(0)
|
|
155
|
-
rescue Exception # rubocop:disable Lint/RescueException
|
|
175
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
176
|
+
# The pipe stays empty when the child dies before answering, so this
|
|
177
|
+
# is the only place the cause survives (#255). The warning is
|
|
178
|
+
# best-effort and the exit is ensured: a broken stderr must not let
|
|
179
|
+
# the exception unwind past the fork block into CLI code in the
|
|
180
|
+
# child, where exit handlers could emit duplicate output (#289).
|
|
181
|
+
begin
|
|
182
|
+
frames = e.backtrace ? e.backtrace.first(8).join("\n from ") : "(no backtrace)"
|
|
183
|
+
warn("saboteur: #{@child_label}: #{e.class}: #{e.message}\n from #{frames}")
|
|
184
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
185
|
+
nil
|
|
186
|
+
end
|
|
187
|
+
ensure
|
|
156
188
|
exit!(1)
|
|
157
189
|
end
|
|
158
190
|
|
|
@@ -195,7 +227,7 @@ module Saboteur
|
|
|
195
227
|
# --- parent ---------------------------------------------------------
|
|
196
228
|
|
|
197
229
|
def contain(pid, reader, started, timeout, on_tick)
|
|
198
|
-
tree = ProcessTree.new(pid)
|
|
230
|
+
tree = ProcessTree.new(pid, marker: @marker)
|
|
199
231
|
self.class.live[pid] = tree
|
|
200
232
|
own_group(pid)
|
|
201
233
|
deadline = started + timeout
|
|
@@ -211,27 +243,27 @@ module Saboteur
|
|
|
211
243
|
# (progress IO failed) or any other error must not skip the
|
|
212
244
|
# kill/reap, or the child and its tree would run unsupervised
|
|
213
245
|
# until this process exits.
|
|
214
|
-
exit_status, supervisor_killed, leftover_pids = release(pid, tree, exit_status, stop, grace_expired)
|
|
246
|
+
exit_status, supervisor_killed, leftover_pids, notes = release(pid, tree, exit_status, stop, grace_expired)
|
|
215
247
|
end
|
|
216
248
|
Result.new(payload: payload, exit_status: exit_status, stopped_by: stop, detail: detail,
|
|
217
|
-
runtime: monotonic - started, leftover_pids: leftover_pids,
|
|
249
|
+
runtime: monotonic - started, leftover_pids: leftover_pids, leftover_notes: notes,
|
|
218
250
|
supervisor_killed: supervisor_killed, memory_pressure: memory_pressure?)
|
|
219
251
|
end
|
|
220
252
|
|
|
221
|
-
# Kills and reaps the child and
|
|
222
|
-
# [exit_status, supervisor_killed, leftover_pids].
|
|
253
|
+
# Kills and reaps the child and its tree; returns [exit_status, supervisor_killed, leftover_pids, notes].
|
|
223
254
|
def release(pid, tree, exit_status, stop, grace_expired)
|
|
224
255
|
exit_status ||= terminate(pid) if stop == :timeout
|
|
225
256
|
supervisor_killed = false
|
|
226
257
|
# The grace ran out with the child seemingly alive. What ends it is
|
|
227
258
|
# only ours to claim if we see it alive: stop it first.
|
|
228
259
|
exit_status, supervisor_killed = freeze_child(pid) if grace_expired
|
|
229
|
-
|
|
260
|
+
# Notes name the leftovers before the kill erases their command lines (#486).
|
|
261
|
+
leftover_pids, notes = tree.kill_all_except(pid)
|
|
230
262
|
exit_status ||= reap(pid)
|
|
231
263
|
self.class.live.delete(pid)
|
|
232
264
|
reap_adopted(leftover_pids)
|
|
233
265
|
reap_group(pid)
|
|
234
|
-
[exit_status, supervisor_killed, leftover_pids]
|
|
266
|
+
[exit_status, supervisor_killed, leftover_pids, notes]
|
|
235
267
|
end
|
|
236
268
|
|
|
237
269
|
# After EOF, waits for the child to exit by itself while still checking
|
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
require "json"
|
|
5
|
+
require_relative "../text"
|
|
6
|
+
require_relative "peak_memory"
|
|
5
7
|
require_relative "process_tree"
|
|
6
8
|
require_relative "supervisor"
|
|
7
9
|
|
|
@@ -25,16 +27,54 @@ module Saboteur
|
|
|
25
27
|
class WorkerPool
|
|
26
28
|
# done: items finished; status: the Process::Status once reaped;
|
|
27
29
|
# last_words: what the worker reported before dying, if anything.
|
|
28
|
-
Worker = Struct.new(:number, :pid, :commands, :results, :current, :done, :status, :last_words)
|
|
30
|
+
Worker = Struct.new(:number, :pid, :commands, :results, :current, :done, :status, :last_words, :peak_rss_mb)
|
|
29
31
|
START_FAILURES = 3
|
|
30
32
|
|
|
31
33
|
# on_start: called in each worker with its number before its first
|
|
32
34
|
# item (per-worker database connections, for example).
|
|
33
|
-
|
|
35
|
+
# worker_peaks: worker number => peak RSS in MB, filled as workers
|
|
36
|
+
# retire (#202). Empty for an inline (jobs 1) run.
|
|
37
|
+
attr_reader :worker_peaks
|
|
38
|
+
|
|
39
|
+
def initialize(jobs:, on_start: nil, on_result: nil, drain_timeout: 30)
|
|
40
|
+
@worker_peaks = {}
|
|
34
41
|
@jobs = jobs
|
|
35
42
|
@on_start = on_start
|
|
36
43
|
@on_result = on_result
|
|
37
44
|
@start_failures = 0
|
|
45
|
+
@halted = false
|
|
46
|
+
@drain_timeout = drain_timeout
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Stop handing queued items out while letting in-flight items finish
|
|
50
|
+
# and report: a worker that merely noticed the damage can report
|
|
51
|
+
# before the worker whose mutant caused it, and the later report is
|
|
52
|
+
# the one that names the culprit (#285). Unlike raising from
|
|
53
|
+
# on_result, nothing in flight is killed. Unstarted items come back
|
|
54
|
+
# as nil holes in the result array.
|
|
55
|
+
#
|
|
56
|
+
# drain: true bounds the wait for a safety abort (tool error, canary):
|
|
57
|
+
# a wedged worker must not stall it, so drain_timeout seconds after
|
|
58
|
+
# the first halt the pool stops waiting, kills whatever is still
|
|
59
|
+
# running and leaves nil holes (#285 review). drain: false is the
|
|
60
|
+
# ordinary --max-duration halt: in-flight subjects always finish and
|
|
61
|
+
# report, so a nil hole only ever means "never started"; the
|
|
62
|
+
# overshoot stays bounded by each worker's own mutant deadlines.
|
|
63
|
+
def halt!(drain: true)
|
|
64
|
+
@halted = true
|
|
65
|
+
return if @drain_deadline || !drain
|
|
66
|
+
|
|
67
|
+
@drain_deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @drain_timeout
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# The serialized shape of a tool error row: the message plus the
|
|
71
|
+
# attribution flags the parent needs to rank reports - a per-mutant
|
|
72
|
+
# culprit digest first, then a canary trip (#283, #285).
|
|
73
|
+
def self.tool_error_row(error)
|
|
74
|
+
row = { "tool_error" => error.message }
|
|
75
|
+
row["canary"] = true if error.is_a?(CanaryTripped)
|
|
76
|
+
row["culprit"] = error.culprit_digest if error.respond_to?(:culprit_digest) && error.culprit_digest
|
|
77
|
+
row
|
|
38
78
|
end
|
|
39
79
|
|
|
40
80
|
def map(items, weights: nil, &work)
|
|
@@ -64,8 +104,20 @@ module Saboteur
|
|
|
64
104
|
|
|
65
105
|
# Waits up to a second for results, then checks for workers that died
|
|
66
106
|
# without closing their pipe (a stray grandchild may still hold it).
|
|
107
|
+
# While halted the wait is bounded by the drain deadline: past it the
|
|
108
|
+
# remaining workers are retired (stop kills a wedged one) and their
|
|
109
|
+
# items come back as nil holes.
|
|
67
110
|
def collect(workers, queue, results, items, &work)
|
|
68
|
-
|
|
111
|
+
timeout = 1
|
|
112
|
+
if @drain_deadline
|
|
113
|
+
remaining = @drain_deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
114
|
+
if remaining <= 0
|
|
115
|
+
abandon_all(workers)
|
|
116
|
+
return
|
|
117
|
+
end
|
|
118
|
+
timeout = [timeout, remaining].min
|
|
119
|
+
end
|
|
120
|
+
ready = IO.select(workers.map(&:results), nil, nil, timeout)&.first || []
|
|
69
121
|
ready.each do |io|
|
|
70
122
|
worker = workers.find { |candidate| candidate.results == io }
|
|
71
123
|
receive(worker, workers, queue, results, items, &work) if worker
|
|
@@ -82,7 +134,7 @@ module Saboteur
|
|
|
82
134
|
take_and_report(worker, line, results) if line
|
|
83
135
|
return replace(worker, workers, queue, results, items, &) if line.nil? || worker.last_words
|
|
84
136
|
|
|
85
|
-
assign(worker, queue)
|
|
137
|
+
assign(worker, queue) unless @halted
|
|
86
138
|
retire(worker, workers) unless worker.current
|
|
87
139
|
end
|
|
88
140
|
|
|
@@ -96,7 +148,7 @@ module Saboteur
|
|
|
96
148
|
end
|
|
97
149
|
|
|
98
150
|
def parse(line)
|
|
99
|
-
JSON.parse(line)
|
|
151
|
+
JSON.parse(Text.utf8_strict(line))
|
|
100
152
|
rescue JSON::ParserError => e
|
|
101
153
|
{ "error" => "unreadable result from worker: #{e.message}" }
|
|
102
154
|
end
|
|
@@ -108,6 +160,10 @@ module Saboteur
|
|
|
108
160
|
# Stores a line from the worker: its item's result, or why it crashed.
|
|
109
161
|
def take(worker, line, results)
|
|
110
162
|
data = parse(line)
|
|
163
|
+
if data.key?("peak_rss_mb")
|
|
164
|
+
@worker_peaks[data.delete("worker") || worker.number] = worker.peak_rss_mb = data["peak_rss_mb"]
|
|
165
|
+
return
|
|
166
|
+
end
|
|
111
167
|
return worker.last_words = data["fatal"] if data.key?("fatal")
|
|
112
168
|
return unless worker.current
|
|
113
169
|
|
|
@@ -127,6 +183,28 @@ module Saboteur
|
|
|
127
183
|
nil
|
|
128
184
|
end
|
|
129
185
|
|
|
186
|
+
# Past the drain deadline nothing more is coming: kill every wedged
|
|
187
|
+
# worker's tree first, then reap, so the cleanup costs one round
|
|
188
|
+
# instead of one stop grace per worker (#285 review). Items in
|
|
189
|
+
# flight come back as nil holes.
|
|
190
|
+
def abandon_all(workers)
|
|
191
|
+
wedged = workers.dup
|
|
192
|
+
wedged.each do |worker|
|
|
193
|
+
worker.commands.close unless worker.commands.closed?
|
|
194
|
+
Supervisor.live.delete(worker.pid)&.kill_all
|
|
195
|
+
end
|
|
196
|
+
# Every wedged tree dies before any wait: one cleanup round.
|
|
197
|
+
# rubocop:disable-next Style/CombinableLoops
|
|
198
|
+
wedged.each do |worker|
|
|
199
|
+
worker.status ||= Process.wait2(worker.pid).last unless reaped?(worker)
|
|
200
|
+
worker.results.close unless worker.results.closed?
|
|
201
|
+
rescue Errno::ECHILD
|
|
202
|
+
nil
|
|
203
|
+
ensure
|
|
204
|
+
workers.delete(worker)
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
|
|
130
208
|
def replace(worker, workers, queue, results, items, &)
|
|
131
209
|
drain(worker, results)
|
|
132
210
|
retire(worker, workers)
|
|
@@ -138,8 +216,11 @@ module Saboteur
|
|
|
138
216
|
results[worker.current] = { "error" => "#{reason}, running this item" }
|
|
139
217
|
@on_result&.call(worker.current, results[worker.current])
|
|
140
218
|
end
|
|
141
|
-
|
|
142
|
-
|
|
219
|
+
# Halted pools start nothing, so a start-failure storm cannot be the
|
|
220
|
+
# run's problem then; raising it would mask the rows that carry the
|
|
221
|
+
# real abort cause (#286).
|
|
222
|
+
give_up(reason) if worker.done.zero? && !@halted
|
|
223
|
+
return if @halted || queue.empty?
|
|
143
224
|
|
|
144
225
|
fresh = start(worker.number, items, workers, &)
|
|
145
226
|
workers << fresh
|
|
@@ -201,7 +282,10 @@ module Saboteur
|
|
|
201
282
|
# Runs in the worker until the parent closes its command pipe. If it
|
|
202
283
|
# crashes, its last line says why.
|
|
203
284
|
def serve(number, items, commands, results, &)
|
|
204
|
-
|
|
285
|
+
# Grouping helps tree cleanup find the worker's descendants; macOS
|
|
286
|
+
# can refuse it (EPERM), and a refused setpgid must not crash the
|
|
287
|
+
# worker - parent-link tracking still finds the tree (#296).
|
|
288
|
+
own_group(0)
|
|
205
289
|
Supervisor.reset_in_worker!
|
|
206
290
|
Supervisor.close_in_child.push(commands, results)
|
|
207
291
|
ENV["SABOTEUR_WORKER"] = number.to_s
|
|
@@ -211,6 +295,7 @@ module Saboteur
|
|
|
211
295
|
results.puts(JSON.generate(run_one(items[Integer(line)], &)))
|
|
212
296
|
results.flush
|
|
213
297
|
end
|
|
298
|
+
report_peak(results, number)
|
|
214
299
|
exit!(0)
|
|
215
300
|
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
216
301
|
last_words(results, "worker #{number} crashed: #{e.class}: #{e.message} (#{e.backtrace&.first})")
|
|
@@ -224,8 +309,24 @@ module Saboteur
|
|
|
224
309
|
nil
|
|
225
310
|
end
|
|
226
311
|
|
|
312
|
+
# Last word before exit!: the kernel's exact peak RSS of this worker,
|
|
313
|
+
# for the run's memory report (#202). Skipped where procfs is absent.
|
|
314
|
+
def report_peak(results, number)
|
|
315
|
+
peak = PeakMemory.peak_mb
|
|
316
|
+
return unless peak
|
|
317
|
+
|
|
318
|
+
results.puts(JSON.generate("worker" => number, "peak_rss_mb" => peak))
|
|
319
|
+
results.flush
|
|
320
|
+
end
|
|
321
|
+
|
|
227
322
|
def run_one(item, &work)
|
|
228
323
|
work.call(item)
|
|
324
|
+
rescue Error => e
|
|
325
|
+
# A tool error (a vanished cache dir, ...) aborts the run with
|
|
326
|
+
# exit 3; it is not a per-mutant outcome. Serialize it so the
|
|
327
|
+
# parent can re-raise it as itself (#283), with the flags the
|
|
328
|
+
# parent needs to rank reports by attribution quality (#285).
|
|
329
|
+
self.class.tool_error_row(e)
|
|
229
330
|
rescue StandardError => e
|
|
230
331
|
{ "error" => "#{e.class}: #{e.message}" }
|
|
231
332
|
end
|
|
@@ -239,6 +340,7 @@ module Saboteur
|
|
|
239
340
|
sleep 0.01 until reaped?(worker) || Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
|
|
240
341
|
tree&.kill_all
|
|
241
342
|
worker.status ||= Process.wait2(worker.pid).last unless reaped?(worker)
|
|
343
|
+
drain(worker, {})
|
|
242
344
|
worker.results.close unless worker.results.closed?
|
|
243
345
|
rescue Errno::ECHILD
|
|
244
346
|
nil
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module Frameworks
|
|
6
|
+
# The #387 shape: a suite connects ActiveRecord to a sqlite ":memory:"
|
|
7
|
+
# database and loads its schema at require time. Every saboteur child is
|
|
8
|
+
# a fork, ActiveRecord's fork-safe pool reconnects there, and a fresh
|
|
9
|
+
# in-memory database is EMPTY - the schema is gone, and every example
|
|
10
|
+
# fails with "Could not find table" while plain rspec stays green. A
|
|
11
|
+
# child that detects the shape says so and names the remedies instead of
|
|
12
|
+
# sending the user debugging a healthy suite.
|
|
13
|
+
module SqliteMemoryDatabase
|
|
14
|
+
REMEDY = "a sqlite :memory: database is empty in this forked child: in-memory databases do not " \
|
|
15
|
+
"survive saboteur's fork, so a schema loaded at require time is lost. Load the schema in " \
|
|
16
|
+
"a before(:suite) hook, or in an after_fork hook (config/saboteur_hooks.rb via c.hooks)"
|
|
17
|
+
|
|
18
|
+
module_function
|
|
19
|
+
|
|
20
|
+
# The remedy text when a current pool points at an empty sqlite
|
|
21
|
+
# :memory: database, else nil. Diagnostic only: any error is no hint,
|
|
22
|
+
# never a failure of the run itself.
|
|
23
|
+
def fork_loss_hint
|
|
24
|
+
return unless defined?(ActiveRecord::Base)
|
|
25
|
+
|
|
26
|
+
pools = ActiveRecord::Base.connection_handler.connection_pool_list
|
|
27
|
+
pools.any? { |pool| empty_memory_database?(pool) } ? REMEDY : nil
|
|
28
|
+
rescue StandardError, ScriptError
|
|
29
|
+
nil
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def empty_memory_database?(pool)
|
|
33
|
+
config = pool.db_config
|
|
34
|
+
return false unless config.adapter.to_s == "sqlite3" && config.database.to_s == ":memory:"
|
|
35
|
+
|
|
36
|
+
pool.with_connection { |connection| user_tables(connection).empty? }
|
|
37
|
+
rescue StandardError
|
|
38
|
+
false
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Migration bookkeeping is not user schema: a database with only these
|
|
42
|
+
# tables still has no schema for the suite's models.
|
|
43
|
+
def user_tables(connection)
|
|
44
|
+
connection.select_values("SELECT name FROM sqlite_master WHERE type = 'table'") -
|
|
45
|
+
%w[sqlite_sequence schema_migrations ar_internal_metadata]
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|