saboteur 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +47 -0
- data/LICENSE +21 -0
- data/README.md +145 -0
- data/exe/saboteur +12 -0
- data/lib/saboteur/cli.rb +112 -0
- data/lib/saboteur/config.rb +133 -0
- data/lib/saboteur/engine/boot.rb +83 -0
- data/lib/saboteur/engine/candidates.rb +63 -0
- data/lib/saboteur/engine/coverage_store.rb +99 -0
- data/lib/saboteur/engine/diagnostics.rb +37 -0
- data/lib/saboteur/engine/diff_scope.rb +32 -0
- data/lib/saboteur/engine/distribution.rb +74 -0
- data/lib/saboteur/engine/result_cache.rb +93 -0
- data/lib/saboteur/engine/scheduler.rb +215 -0
- data/lib/saboteur/engine/subject_selection.rb +63 -0
- data/lib/saboteur/engine/suite_files.rb +82 -0
- data/lib/saboteur/engine/verdict_check.rb +78 -0
- data/lib/saboteur/engine.rb +133 -0
- data/lib/saboteur/execution/process_table.rb +177 -0
- data/lib/saboteur/execution/process_tree.rb +99 -0
- data/lib/saboteur/execution/protocol.rb +41 -0
- data/lib/saboteur/execution/reevaluator.rb +144 -0
- data/lib/saboteur/execution/runner.rb +191 -0
- data/lib/saboteur/execution/supervisor.rb +337 -0
- data/lib/saboteur/execution/worker_pool.rb +249 -0
- data/lib/saboteur/frameworks/rails.rb +166 -0
- data/lib/saboteur/hooks.rb +51 -0
- data/lib/saboteur/ignore_list.rb +62 -0
- data/lib/saboteur/integration/base.rb +72 -0
- data/lib/saboteur/integration/rspec.rb +143 -0
- data/lib/saboteur/mutation_source/base.rb +26 -0
- data/lib/saboteur/mutation_source/builtin.rb +232 -0
- data/lib/saboteur/mutation_source/edit.rb +18 -0
- data/lib/saboteur/mutation_source/mutation.rb +28 -0
- data/lib/saboteur/reporter/diff.rb +97 -0
- data/lib/saboteur/reporter/stryker.rb +116 -0
- data/lib/saboteur/reporter/terminal.rb +136 -0
- data/lib/saboteur/result.rb +134 -0
- data/lib/saboteur/selection/coverage_map.rb +113 -0
- data/lib/saboteur/selection/git_diff.rb +64 -0
- data/lib/saboteur/selection/recorder.rb +72 -0
- data/lib/saboteur/subject.rb +166 -0
- data/lib/saboteur/subject_finder.rb +98 -0
- data/lib/saboteur/version.rb +6 -0
- data/lib/saboteur.rb +51 -0
- metadata +130 -0
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module Execution
|
|
6
|
+
# Redefines a subject's method from (possibly mutated) text, in the running
|
|
7
|
+
# process, without touching the file on disk.
|
|
8
|
+
#
|
|
9
|
+
# Strategies (see docs/design.md 6.1):
|
|
10
|
+
# class_eval - owner.class_eval(text, file, line), as specified. Constants
|
|
11
|
+
# from enclosing modules are not in lexical scope.
|
|
12
|
+
# nesting - rebuild the original `module A; class B` nesting on one line
|
|
13
|
+
# and eval at top level, so constant lookup matches the file.
|
|
14
|
+
#
|
|
15
|
+
# Both keep the original file and line numbers, carry a frozen_string_literal
|
|
16
|
+
# magic comment, and restore the method's visibility afterwards.
|
|
17
|
+
class Reevaluator
|
|
18
|
+
STRATEGIES = %w[class_eval nesting].freeze
|
|
19
|
+
FROZEN_HEADER = "# frozen_string_literal: true\n"
|
|
20
|
+
|
|
21
|
+
def initialize(strategy = "nesting")
|
|
22
|
+
raise Error, "unknown reeval strategy: #{strategy}" unless STRATEGIES.include?(strategy.to_s)
|
|
23
|
+
|
|
24
|
+
@strategy = strategy.to_s
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def call(subject, text)
|
|
28
|
+
owner = resolve_owner(subject)
|
|
29
|
+
target = subject.singleton ? owner.singleton_class : owner
|
|
30
|
+
visibility = visibility_of(target, subject.name)
|
|
31
|
+
module_function = module_function?(owner, subject)
|
|
32
|
+
aliases = aliases_of(target, subject.name)
|
|
33
|
+
|
|
34
|
+
@strategy == "nesting" ? eval_nested(subject, text) : eval_in_owner(owner, subject, text)
|
|
35
|
+
|
|
36
|
+
restore(owner, target, subject.name, visibility, module_function)
|
|
37
|
+
repoint(target, subject.name, aliases)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Resolves the subject's owner the way Ruby would resolve the nesting in
|
|
41
|
+
# the file. const_get also triggers autoload (Zeitwerk) before any reopening.
|
|
42
|
+
def resolve_owner(subject)
|
|
43
|
+
scopes = [Object]
|
|
44
|
+
subject.nesting.reject(&:sclass?).each do |scope|
|
|
45
|
+
scopes << resolve_path(scope.path, scopes)
|
|
46
|
+
end
|
|
47
|
+
owner = scopes.last
|
|
48
|
+
raise Error, "#{subject.owner} is not a class or module" unless owner.is_a?(Module)
|
|
49
|
+
|
|
50
|
+
owner
|
|
51
|
+
rescue NameError => e
|
|
52
|
+
raise Error, "cannot resolve #{subject.owner}: #{e.message}"
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
def resolve_path(path, scopes)
|
|
58
|
+
return Object.const_get(path.delete_prefix("::")) if path.start_with?("::")
|
|
59
|
+
|
|
60
|
+
first, *rest = path.split("::")
|
|
61
|
+
base = if rest.empty?
|
|
62
|
+
# `class B` inside `module A` opens A::B (or ::B at top level).
|
|
63
|
+
scopes.last.const_get(first, false)
|
|
64
|
+
else
|
|
65
|
+
lexical_lookup(first, scopes)
|
|
66
|
+
end
|
|
67
|
+
rest.reduce(base) { |mod, name| mod.const_get(name, false) }
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def lexical_lookup(name, scopes)
|
|
71
|
+
found = scopes.reverse.find { |scope| scope.const_defined?(name, false) }
|
|
72
|
+
(found || Object).const_get(name, false)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def visibility_of(target, name)
|
|
76
|
+
if target.private_method_defined?(name, false) then :private
|
|
77
|
+
elsif target.protected_method_defined?(name, false) then :protected
|
|
78
|
+
else :public
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def module_function?(owner, subject)
|
|
83
|
+
return false if subject.singleton || owner.is_a?(Class)
|
|
84
|
+
return false unless owner.private_method_defined?(subject.name, false)
|
|
85
|
+
return false unless owner.singleton_class.public_method_defined?(subject.name, false)
|
|
86
|
+
|
|
87
|
+
owner.instance_method(subject.name).source_location ==
|
|
88
|
+
owner.singleton_class.instance_method(subject.name).source_location
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def eval_in_owner(owner, subject, text)
|
|
92
|
+
code, line = with_magic_comment(subject, text)
|
|
93
|
+
eval_target = subject.sclass ? owner.singleton_class : owner
|
|
94
|
+
eval_target.class_eval(code, subject.file, line)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def eval_nested(subject, text)
|
|
98
|
+
headers = subject.nesting.map(&:header)
|
|
99
|
+
wrapped = "#{headers.join('; ')}; #{text}; #{(['end'] * headers.size).join('; ')}"
|
|
100
|
+
code, line = with_magic_comment(subject, wrapped)
|
|
101
|
+
TOPLEVEL_BINDING.eval(code, subject.file, line)
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def with_magic_comment(subject, code)
|
|
105
|
+
return [code, subject.start_line] unless subject.frozen_literals
|
|
106
|
+
|
|
107
|
+
[FROZEN_HEADER + code, subject.start_line - 1]
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Methods defined with alias/alias_method copy the old body. Callers
|
|
111
|
+
# that use the alias would never see the mutant, which would then
|
|
112
|
+
# survive for no reason. Only aliases of the current definition count
|
|
113
|
+
# (same source location): an alias taken before the method was
|
|
114
|
+
# redefined keeps the older body on purpose. Returns { alias => visibility }.
|
|
115
|
+
def aliases_of(target, name)
|
|
116
|
+
current = target.instance_method(name).source_location
|
|
117
|
+
names = target.instance_methods(false) + target.private_instance_methods(false)
|
|
118
|
+
names.each_with_object({}) do |other, found|
|
|
119
|
+
next if other == name
|
|
120
|
+
|
|
121
|
+
method = target.instance_method(other)
|
|
122
|
+
next unless method.original_name == name && method.source_location == current
|
|
123
|
+
|
|
124
|
+
found[other] = visibility_of(target, other)
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def repoint(target, name, aliases)
|
|
129
|
+
aliases.each do |other, visibility|
|
|
130
|
+
target.send(:alias_method, other, name)
|
|
131
|
+
target.send(visibility, other)
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def restore(owner, target, name, visibility, module_function)
|
|
136
|
+
if module_function
|
|
137
|
+
owner.send(:module_function, name)
|
|
138
|
+
elsif visibility != :public
|
|
139
|
+
target.send(visibility, name)
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
end
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "json"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
module Execution
|
|
8
|
+
# Runs one subject variant (unchanged or mutated) in a supervised child.
|
|
9
|
+
#
|
|
10
|
+
# The child re-evaluates the method, reports "started", runs the selected
|
|
11
|
+
# tests and reports the result over a pipe. Supervisor owns the child's
|
|
12
|
+
# whole process tree: deadline, memory and process caps, and killing every
|
|
13
|
+
# descendant afterwards, including ones that left the process group.
|
|
14
|
+
class Runner
|
|
15
|
+
# kill_cause: "assertion" when the killing test failed an expectation,
|
|
16
|
+
# else the class of the error it died with (a crash, not a check).
|
|
17
|
+
Outcome = Data.define(:status, :runtime, :failed_ids, :message, :leftover_pids, :kill_cause) do
|
|
18
|
+
def initialize(status:, runtime:, failed_ids: [], message: nil, leftover_pids: [], kill_cause: nil)
|
|
19
|
+
super
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
ASSERTION_FAILURES = %w[
|
|
24
|
+
RSpec::Expectations::ExpectationNotMetError
|
|
25
|
+
RSpec::Expectations::MultipleExpectationsNotMetError
|
|
26
|
+
RSpec::Mocks::MockExpectationError
|
|
27
|
+
].freeze
|
|
28
|
+
|
|
29
|
+
# Everything one child needs. mutation is nil for the verdict check.
|
|
30
|
+
Job = Data.define(:subject, :text, :test_ids, :timeout, :mutation, :log_path)
|
|
31
|
+
|
|
32
|
+
LOG_TAIL_BYTES = 2000
|
|
33
|
+
|
|
34
|
+
# log_dir: where children write their output; a child's log is kept
|
|
35
|
+
# only when it ends in an error.
|
|
36
|
+
# seed: every child seeds Ruby's random numbers with it before its
|
|
37
|
+
# tests, so a test that uses rand sees the same values in every run.
|
|
38
|
+
def initialize(integration:, reevaluator:, log_dir:, hooks: Hooks.new, supervisor: Supervisor.new, seed: 1)
|
|
39
|
+
@integration = integration
|
|
40
|
+
@reevaluator = reevaluator
|
|
41
|
+
@hooks = hooks
|
|
42
|
+
@log_dir = log_dir
|
|
43
|
+
@supervisor = supervisor
|
|
44
|
+
@seed = seed
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# text: method source to evaluate in the child (the unchanged source for
|
|
48
|
+
# the neutral check). mutation: passed to around_mutant hooks (nil for
|
|
49
|
+
# the neutral check). Returns an Outcome with status
|
|
50
|
+
# :killed, :survived, :timeout, :resource_limit (memory or process cap) or
|
|
51
|
+
# :error.
|
|
52
|
+
def run(subject:, text:, test_ids:, timeout:, mutation: nil)
|
|
53
|
+
@hooks.run(:before_fork, mutation)
|
|
54
|
+
@sequence = @sequence.to_i + 1
|
|
55
|
+
job = Job.new(subject:, text:, test_ids:, timeout:, mutation:,
|
|
56
|
+
log_path: File.join(@log_dir, "child-#{Process.pid}-#{@sequence}.log"))
|
|
57
|
+
result = @supervisor.run(timeout: timeout) { |writer| child(writer, job) }
|
|
58
|
+
outcome = outcome_for(result, job)
|
|
59
|
+
File.delete(job.log_path) if outcome.status != :error && File.exist?(job.log_path)
|
|
60
|
+
outcome
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
private
|
|
64
|
+
|
|
65
|
+
# --- child ----------------------------------------------------------
|
|
66
|
+
|
|
67
|
+
def child(writer, job)
|
|
68
|
+
Process.setproctitle(title(job))
|
|
69
|
+
redirect_output(job.log_path)
|
|
70
|
+
@hooks.run(:after_fork)
|
|
71
|
+
@reevaluator.call(job.subject, job.text)
|
|
72
|
+
Protocol.emit(writer, "started")
|
|
73
|
+
Kernel.srand(@seed)
|
|
74
|
+
result = @hooks.around(job.mutation) { run_tests(job.test_ids) }
|
|
75
|
+
if result.nil?
|
|
76
|
+
Protocol.emit(writer, "error", message: "around_mutant hook did not run the tests")
|
|
77
|
+
else
|
|
78
|
+
Protocol.emit(writer, "result", **result)
|
|
79
|
+
end
|
|
80
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
81
|
+
Protocol.emit(writer, "error", message: "#{e.class}: #{e.message}", backtrace: e.backtrace&.first(5))
|
|
82
|
+
exit!(1)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# What `ps` shows for this child: which method, and which mutant.
|
|
86
|
+
def title(job)
|
|
87
|
+
"saboteur: #{job.subject.id} #{job.mutation ? job.mutation.operator : 'neutral check'}"
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def run_tests(test_ids)
|
|
91
|
+
result = @integration.run(test_ids)
|
|
92
|
+
{ passed: result.passed, failed: result.failed_ids, executed: result.executed_ids, runtime: result.runtime,
|
|
93
|
+
failure_classes: result.failure_classes, failure_messages: result.failure_messages }
|
|
94
|
+
rescue SystemExit => e
|
|
95
|
+
# A test called `exit`. The tests noticed something; count it as a failure.
|
|
96
|
+
{ passed: false, failed: [], executed: [], runtime: 0.0, exited: e.status }
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def redirect_output(log_path)
|
|
100
|
+
$stdout.reopen(log_path, "w")
|
|
101
|
+
$stderr.reopen($stdout)
|
|
102
|
+
$stdout.sync = true
|
|
103
|
+
$stderr.sync = true
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# --- parent ---------------------------------------------------------
|
|
107
|
+
|
|
108
|
+
def outcome_for(result, job)
|
|
109
|
+
outcome = case result.stopped_by
|
|
110
|
+
when :timeout
|
|
111
|
+
Outcome.new(status: :timeout, runtime: result.runtime,
|
|
112
|
+
message: "exceeded #{job.timeout.round(2)} s")
|
|
113
|
+
when :memory, :processes
|
|
114
|
+
Outcome.new(status: :resource_limit, runtime: result.runtime, message: result.detail)
|
|
115
|
+
else
|
|
116
|
+
classify(Protocol.parse(result.payload), result.exit_status, result.runtime, job)
|
|
117
|
+
end
|
|
118
|
+
outcome.with(leftover_pids: result.leftover_pids)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def classify(events, exit_status, runtime, job)
|
|
122
|
+
started = events.any? { |event| event["event"] == "started" }
|
|
123
|
+
result = events.find { |event| event["event"] == "result" }
|
|
124
|
+
error = events.find { |event| event["event"] == "error" }
|
|
125
|
+
|
|
126
|
+
if result
|
|
127
|
+
classify_result(result, runtime, job.test_ids)
|
|
128
|
+
else
|
|
129
|
+
message = if error
|
|
130
|
+
error["message"]
|
|
131
|
+
else
|
|
132
|
+
"child #{describe(exit_status)} #{started ? 'during' : 'before'} tests"
|
|
133
|
+
end
|
|
134
|
+
message = "#{message} (before tests)" if error && !started
|
|
135
|
+
Outcome.new(status: :error, runtime: runtime, message: [message, log_tail(job.log_path)].compact.join("\n"))
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def classify_result(result, runtime, test_ids)
|
|
140
|
+
failed = Array(result["failed"])
|
|
141
|
+
executed = Array(result["executed"])
|
|
142
|
+
return Outcome.new(status: :killed, runtime: runtime, message: "test called exit") if result.key?("exited")
|
|
143
|
+
|
|
144
|
+
if failed.any?
|
|
145
|
+
unexpected = failed - test_ids
|
|
146
|
+
return mismatch(runtime, "failed tests not requested: #{unexpected.first(3).join(', ')}") if unexpected.any?
|
|
147
|
+
|
|
148
|
+
return Outcome.new(status: :killed, runtime: runtime, failed_ids: failed,
|
|
149
|
+
message: result.fetch("failure_messages", {})[failed.first],
|
|
150
|
+
kill_cause: kill_cause(result.fetch("failure_classes", {})[failed.first]))
|
|
151
|
+
end
|
|
152
|
+
return Outcome.new(status: :killed, runtime: runtime, message: "suite-level failure") unless result["passed"]
|
|
153
|
+
|
|
154
|
+
if executed.sort != test_ids.sort
|
|
155
|
+
return mismatch(runtime, "executed #{executed.size} of #{test_ids.size} requested tests")
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
Outcome.new(status: :survived, runtime: runtime)
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def kill_cause(failure_class)
|
|
162
|
+
return nil if failure_class.nil?
|
|
163
|
+
|
|
164
|
+
ASSERTION_FAILURES.include?(failure_class) ? "assertion" : failure_class
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def mismatch(runtime, message)
|
|
168
|
+
Outcome.new(status: :error, runtime: runtime, message: message)
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def describe(status)
|
|
172
|
+
return "exited" unless status.is_a?(Process::Status)
|
|
173
|
+
return "killed by signal #{status.termsig}" if status.signaled?
|
|
174
|
+
|
|
175
|
+
"exited with status #{status.exitstatus}"
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def log_tail(path)
|
|
179
|
+
return nil unless File.exist?(path)
|
|
180
|
+
|
|
181
|
+
size = File.size(path)
|
|
182
|
+
return nil if size.zero?
|
|
183
|
+
|
|
184
|
+
File.open(path) do |file|
|
|
185
|
+
file.seek([size - LOG_TAIL_BYTES, 0].max)
|
|
186
|
+
file.read.scrub.strip
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
end
|
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "process_tree"
|
|
5
|
+
require_relative "protocol"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
module Execution
|
|
9
|
+
# Forks one child and owns everything it starts until a verdict is reached.
|
|
10
|
+
#
|
|
11
|
+
# Invariant: no process started on behalf of a child outlives that child's
|
|
12
|
+
# supervision. Enforced by, in order:
|
|
13
|
+
# 1. Tracking the whole descendant tree (ProcessTree), not just the
|
|
14
|
+
# process group, and killing all of it after every child, whether it
|
|
15
|
+
# passed, failed, crashed or timed out.
|
|
16
|
+
# 2. A deadline, a memory cap and a process cap over the tree, checked
|
|
17
|
+
# while the child runs.
|
|
18
|
+
# 3. Kernel backstops the tree tracking cannot miss: RLIMIT_CPU in the
|
|
19
|
+
# child, which every descendant inherits even if it escapes the tree,
|
|
20
|
+
# and a parent-death check in the child.
|
|
21
|
+
# 4. Refusing recursion: the child is marked with CHILD_ENV, and Engine
|
|
22
|
+
# refuses to start a run inside a marked process.
|
|
23
|
+
# 5. On Linux, the supervisor is a child subreaper, so escaped orphans are
|
|
24
|
+
# re-parented to it instead of init and stay in its tree.
|
|
25
|
+
class Supervisor
|
|
26
|
+
# Set in every child to its nesting depth (1 for a child of a top-level
|
|
27
|
+
# run). Engine refuses to start when it is set; the supervisor itself
|
|
28
|
+
# refuses to fork past MAX_DEPTH, so even a run with that guard removed
|
|
29
|
+
# (a mutant of the guard, say) cannot recurse without bound.
|
|
30
|
+
CHILD_ENV = "SABOTEUR_CHILD"
|
|
31
|
+
MAX_DEPTH = 4
|
|
32
|
+
# The hard CPU limit, as a multiple of the budget (see limit_cpu).
|
|
33
|
+
HARD_CPU_FACTOR = 20
|
|
34
|
+
|
|
35
|
+
Limits = Data.define(:max_memory_mb, :max_processes) do
|
|
36
|
+
def self.default
|
|
37
|
+
new(max_memory_mb: 2048, max_processes: 64)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# payload: bytes the child wrote to its pipe. stop: nil when the child
|
|
42
|
+
# finished on its own, else :timeout, :memory or :processes.
|
|
43
|
+
Result = Data.define(:payload, :exit_status, :stopped_by, :detail, :runtime, :leftover_pids)
|
|
44
|
+
|
|
45
|
+
POLL_INTERVAL = 0.5
|
|
46
|
+
PR_SET_CHILD_SUBREAPER = 36
|
|
47
|
+
TERM_GRACE = 1.0
|
|
48
|
+
REAP_DEADLINE = 5.0
|
|
49
|
+
|
|
50
|
+
@live = {}
|
|
51
|
+
@close_in_child = []
|
|
52
|
+
class << self
|
|
53
|
+
# close_in_child: IOs a supervised child must close first (a worker's
|
|
54
|
+
# pipes, so a stray grandchild cannot keep them open).
|
|
55
|
+
attr_reader :live, :close_in_child
|
|
56
|
+
|
|
57
|
+
# In a freshly forked worker: forget the parent's trees (they are
|
|
58
|
+
# siblings, not ours) and handle INT/TERM for this process.
|
|
59
|
+
def reset_in_worker!
|
|
60
|
+
@live = {}
|
|
61
|
+
@close_in_child = []
|
|
62
|
+
@installed = false
|
|
63
|
+
install_parent_handlers
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Kills every tree still under supervision. Used on parent exit and on
|
|
67
|
+
# INT/TERM, so Ctrl-C never leaves mutants running.
|
|
68
|
+
def kill_live_trees
|
|
69
|
+
live.each_value(&:kill_all)
|
|
70
|
+
live.clear
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def install_parent_handlers
|
|
74
|
+
return if @installed
|
|
75
|
+
|
|
76
|
+
@installed = true
|
|
77
|
+
owner = Process.pid
|
|
78
|
+
at_exit { kill_live_trees if Process.pid == owner }
|
|
79
|
+
%w[INT TERM].each do |signal|
|
|
80
|
+
previous = trap(signal) do
|
|
81
|
+
kill_live_trees if Process.pid == owner
|
|
82
|
+
previous.respond_to?(:call) ? previous.call : exit(128 + Signal.list.fetch(signal))
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def initialize(limits: Limits.default)
|
|
89
|
+
@limits = limits
|
|
90
|
+
self.class.install_parent_handlers
|
|
91
|
+
become_subreaper
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Runs the block in a forked child with a pipe writer; the child exits
|
|
95
|
+
# when the block returns. Returns a Result. cpu_seconds bounds every
|
|
96
|
+
# descendant's CPU time (default: twice the deadline, at least 10 s).
|
|
97
|
+
def run(timeout:, cpu_seconds: nil, &block)
|
|
98
|
+
depth = ENV.fetch(CHILD_ENV, "0").to_i
|
|
99
|
+
raise Error, "refusing to fork: saboteur children nested #{depth} deep (limit #{MAX_DEPTH})" if
|
|
100
|
+
depth >= MAX_DEPTH
|
|
101
|
+
|
|
102
|
+
cpu_seconds ||= [(timeout * 2).ceil, 10].max
|
|
103
|
+
reader, writer = IO.pipe
|
|
104
|
+
$stdout.flush
|
|
105
|
+
$stderr.flush
|
|
106
|
+
started = monotonic
|
|
107
|
+
supervisor = Process.pid
|
|
108
|
+
pid = fork { child(reader, writer, supervisor, cpu_seconds, depth + 1, &block) }
|
|
109
|
+
writer.close
|
|
110
|
+
contain(pid, reader, started, timeout)
|
|
111
|
+
ensure
|
|
112
|
+
reader&.close unless reader&.closed?
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
private
|
|
116
|
+
|
|
117
|
+
# --- child ----------------------------------------------------------
|
|
118
|
+
|
|
119
|
+
def child(reader, writer, supervisor, cpu_seconds, depth)
|
|
120
|
+
reader.close
|
|
121
|
+
self.class.close_in_child.each { |io| io.close unless io.closed? }
|
|
122
|
+
ENV[CHILD_ENV] = depth.to_s
|
|
123
|
+
Process.setpgid(0, 0)
|
|
124
|
+
limit_cpu(cpu_seconds)
|
|
125
|
+
trap("TERM") { exit!(143) }
|
|
126
|
+
watch_parent(supervisor)
|
|
127
|
+
yield writer
|
|
128
|
+
exit!(0)
|
|
129
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
130
|
+
exit!(1)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# SIGXCPU at the soft limit, SIGKILL at the hard one. Limits can only be
|
|
134
|
+
# lowered, so a nested supervisor stays within the one it inherited.
|
|
135
|
+
#
|
|
136
|
+
# macOS over-counts CPU time for a process with many waking threads
|
|
137
|
+
# and sends SIGXCPU early (a Rails test child has ~20 threads). So
|
|
138
|
+
# SIGXCPU is checked against the CPU time really used: below the
|
|
139
|
+
# budget it is ignored and the soft limit moves out; at the budget the
|
|
140
|
+
# process dies by SIGXCPU. The hard limit (SIGKILL, cannot be caught)
|
|
141
|
+
# sits far out (HARD_CPU_FACTOR), for a process that ignores SIGXCPU.
|
|
142
|
+
def limit_cpu(seconds)
|
|
143
|
+
_, hard = Process.getrlimit(Process::RLIMIT_CPU)
|
|
144
|
+
hard_cap = [seconds * HARD_CPU_FACTOR, hard].min
|
|
145
|
+
Process.setrlimit(Process::RLIMIT_CPU, [seconds, hard_cap].min, hard_cap)
|
|
146
|
+
trap("XCPU") { cpu_alarm(seconds) }
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def cpu_alarm(budget)
|
|
150
|
+
if Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID) >= budget
|
|
151
|
+
trap("XCPU", "DEFAULT")
|
|
152
|
+
Process.kill("XCPU", Process.pid)
|
|
153
|
+
else
|
|
154
|
+
soft, hard = Process.getrlimit(Process::RLIMIT_CPU)
|
|
155
|
+
Process.setrlimit(Process::RLIMIT_CPU, [soft * 2, hard].min, hard)
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# macOS has no PDEATHSIG: poll, and exit if the supervisor is gone.
|
|
160
|
+
def watch_parent(supervisor)
|
|
161
|
+
Thread.new do
|
|
162
|
+
loop do
|
|
163
|
+
sleep 1
|
|
164
|
+
exit!(1) if Process.ppid != supervisor
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# --- parent ---------------------------------------------------------
|
|
170
|
+
|
|
171
|
+
def contain(pid, reader, started, timeout)
|
|
172
|
+
tree = ProcessTree.new(pid)
|
|
173
|
+
self.class.live[pid] = tree
|
|
174
|
+
own_group(pid)
|
|
175
|
+
payload, stop, detail = watch(tree, reader, started + timeout)
|
|
176
|
+
exit_status = stop == :timeout ? terminate(pid) : nil
|
|
177
|
+
leftover_pids = tree.kill_all - [pid]
|
|
178
|
+
exit_status ||= reap(pid)
|
|
179
|
+
self.class.live.delete(pid)
|
|
180
|
+
reap_adopted(leftover_pids)
|
|
181
|
+
reap_group(pid)
|
|
182
|
+
Result.new(payload: payload, exit_status: exit_status, stopped_by: stop, detail: detail,
|
|
183
|
+
runtime: monotonic - started, leftover_pids: leftover_pids)
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# Reads until EOF, polling the tree's size and memory. Returns
|
|
187
|
+
# [payload, stop, detail].
|
|
188
|
+
def watch(tree, reader, deadline)
|
|
189
|
+
payload = +""
|
|
190
|
+
next_poll = monotonic + POLL_INTERVAL
|
|
191
|
+
tree.live_members
|
|
192
|
+
loop do
|
|
193
|
+
now = monotonic
|
|
194
|
+
return [payload, :timeout, "exceeded deadline"] if now >= deadline
|
|
195
|
+
|
|
196
|
+
if now >= next_poll
|
|
197
|
+
stop = over_limit(tree)
|
|
198
|
+
return [payload, *stop] if stop
|
|
199
|
+
|
|
200
|
+
next_poll = now + POLL_INTERVAL
|
|
201
|
+
end
|
|
202
|
+
next unless reader.wait_readable([deadline, next_poll].min - now)
|
|
203
|
+
|
|
204
|
+
chunk = reader.read_nonblock(65_536, exception: false)
|
|
205
|
+
break if chunk.nil?
|
|
206
|
+
|
|
207
|
+
payload << chunk unless chunk == :wait_readable
|
|
208
|
+
end
|
|
209
|
+
[payload, nil, nil]
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def over_limit(tree)
|
|
213
|
+
count, growth_kb = tree.usage(baseline_rss_kb)
|
|
214
|
+
if growth_kb > @limits.max_memory_mb * 1024
|
|
215
|
+
[:memory, "process tree grew by #{growth_kb / 1024} MB (limit #{@limits.max_memory_mb} MB)"]
|
|
216
|
+
elsif count > @limits.max_processes
|
|
217
|
+
[:processes, "process tree reached #{count} processes (limit #{@limits.max_processes})"]
|
|
218
|
+
end
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# The supervisor's own RSS, refreshed at most every 10 s (forked children
|
|
222
|
+
# share it, so it is the baseline for growth).
|
|
223
|
+
def baseline_rss_kb
|
|
224
|
+
now = monotonic
|
|
225
|
+
if @baseline_at.nil? || now - @baseline_at > 10
|
|
226
|
+
@baseline_rss_kb = ProcessTable.snapshot.fetch(Process.pid).rss_kb
|
|
227
|
+
@baseline_at = now
|
|
228
|
+
end
|
|
229
|
+
@baseline_rss_kb
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
def terminate(pid)
|
|
233
|
+
signal("TERM", -pid)
|
|
234
|
+
deadline = monotonic + TERM_GRACE
|
|
235
|
+
until monotonic > deadline
|
|
236
|
+
status = waitpid(pid)
|
|
237
|
+
return status if status
|
|
238
|
+
|
|
239
|
+
sleep 0.01
|
|
240
|
+
end
|
|
241
|
+
nil
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# Never blocks forever: the child was SIGKILLed by kill_all, so it is
|
|
245
|
+
# reaped within milliseconds; give up after REAP_DEADLINE and say so.
|
|
246
|
+
def reap(pid)
|
|
247
|
+
deadline = monotonic + REAP_DEADLINE
|
|
248
|
+
until monotonic > deadline
|
|
249
|
+
status = waitpid(pid)
|
|
250
|
+
return status if status
|
|
251
|
+
|
|
252
|
+
signal("KILL", pid)
|
|
253
|
+
sleep 0.01
|
|
254
|
+
end
|
|
255
|
+
:unreaped
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def waitpid(pid)
|
|
259
|
+
_, status = Process.waitpid2(pid, Process::WNOHANG)
|
|
260
|
+
status
|
|
261
|
+
rescue Errno::ECHILD
|
|
262
|
+
:reaped
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
# With a subreaper (Linux), killed orphans are re-parented to us; reap
|
|
266
|
+
# exactly those, never another child of the supervisor. SIGKILL takes
|
|
267
|
+
# effect asynchronously and re-parenting happens only once each parent
|
|
268
|
+
# has died, so one look is not enough: keep looking until every leftover
|
|
269
|
+
# is reaped or gone, for at most REAP_DEADLINE.
|
|
270
|
+
def reap_adopted(pids)
|
|
271
|
+
pending = pids.dup
|
|
272
|
+
deadline = monotonic + REAP_DEADLINE
|
|
273
|
+
until pending.empty? || monotonic > deadline
|
|
274
|
+
pending.reject! { |pid| settled?(pid) }
|
|
275
|
+
sleep 0.01 unless pending.empty?
|
|
276
|
+
end
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
# A process the child started just before it was killed may never have
|
|
280
|
+
# been seen by the tree, yet the group signal killed it and, with a
|
|
281
|
+
# subreaper, it is now ours. The child led its own process group, so
|
|
282
|
+
# reap whatever of that group was adopted, for at most REAP_DEADLINE.
|
|
283
|
+
def reap_group(pgid)
|
|
284
|
+
signal("KILL", -pgid)
|
|
285
|
+
deadline = monotonic + REAP_DEADLINE
|
|
286
|
+
until monotonic > deadline
|
|
287
|
+
sleep 0.01 unless Process.waitpid(-pgid, Process::WNOHANG)
|
|
288
|
+
end
|
|
289
|
+
rescue Errno::ECHILD
|
|
290
|
+
nil
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
# True once the process is reaped by us, or gone and not ours to reap.
|
|
294
|
+
def settled?(pid)
|
|
295
|
+
Process.waitpid(pid, Process::WNOHANG) ? true : false
|
|
296
|
+
rescue Errno::ECHILD
|
|
297
|
+
!exists?(pid)
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def exists?(pid)
|
|
301
|
+
Process.kill(0, pid)
|
|
302
|
+
true
|
|
303
|
+
rescue Errno::ESRCH
|
|
304
|
+
false
|
|
305
|
+
rescue Errno::EPERM
|
|
306
|
+
true
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
def own_group(pid)
|
|
310
|
+
Process.setpgid(pid, pid)
|
|
311
|
+
rescue Errno::EACCES, Errno::ESRCH, Errno::EPERM
|
|
312
|
+
nil
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
def signal(name, target)
|
|
316
|
+
Process.kill(name, target)
|
|
317
|
+
rescue Errno::ESRCH, Errno::EPERM
|
|
318
|
+
nil
|
|
319
|
+
end
|
|
320
|
+
|
|
321
|
+
def become_subreaper
|
|
322
|
+
return unless RUBY_PLATFORM.include?("linux")
|
|
323
|
+
|
|
324
|
+
require "fiddle"
|
|
325
|
+
prctl = Fiddle::Function.new(Fiddle::Handle::DEFAULT["prctl"],
|
|
326
|
+
[Fiddle::TYPE_INT, Fiddle::TYPE_LONG], Fiddle::TYPE_INT)
|
|
327
|
+
prctl.call(PR_SET_CHILD_SUBREAPER, 1)
|
|
328
|
+
rescue StandardError, Fiddle::DLError
|
|
329
|
+
nil
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
def monotonic
|
|
333
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
334
|
+
end
|
|
335
|
+
end
|
|
336
|
+
end
|
|
337
|
+
end
|