supervisor_machines 0.1.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +52 -0
- data/Cargo.lock +22 -2
- data/Cargo.toml +3 -0
- data/README.md +101 -6
- data/ext/supervisor_machines_native/core/Cargo.toml +4 -6
- data/ext/supervisor_machines_native/core/src/lib.rs +102 -22
- data/ext/supervisor_machines_native/core/src/supervisor.rs +270 -154
- data/ext/supervisor_machines_native/ffi/Cargo.toml +5 -9
- data/ext/supervisor_machines_native/ffi/src/decider.rs +333 -117
- data/ext/supervisor_machines_native/ffi/src/init.rs +30 -0
- data/ext/supervisor_machines_native/ffi/src/lib.rs +199 -108
- data/lib/supervisor_machines/supervisor.rb +113 -60
- data/lib/supervisor_machines/values.rb +63 -0
- data/lib/supervisor_machines/version.rb +1 -1
- data/lib/supervisor_machines.rb +12 -3
- data/sig/supervisor_machines.rbs +179 -63
- metadata +7 -5
|
@@ -1,135 +1,226 @@
|
|
|
1
1
|
//! Ruby bindings for the `supervisor-machines` decision machine.
|
|
2
2
|
//!
|
|
3
|
-
//! The core decides and nothing else: this crate converts Ruby symbols
|
|
4
|
-
//! its
|
|
5
|
-
//! spawning, joining and killing to
|
|
3
|
+
//! The core decides and nothing else: this crate converts Ruby symbols and
|
|
4
|
+
//! `Data` values into its types, hands back every answer as a symbol or a
|
|
5
|
+
//! `Data` value, and leaves spawning, joining and killing to
|
|
6
|
+
//! `SupervisorMachines::Supervisor`.
|
|
6
7
|
//!
|
|
7
|
-
//!
|
|
8
|
-
//!
|
|
9
|
-
|
|
10
|
-
#![allow(clippy::needless_pass_by_value)]
|
|
8
|
+
//! Every instant and span crosses as Integer nanoseconds: a non-Integer is a
|
|
9
|
+
//! `TypeError`, a negative or oversized one a `RangeError`.
|
|
11
10
|
|
|
12
11
|
mod decider;
|
|
12
|
+
mod init;
|
|
13
|
+
|
|
14
|
+
use magnus::{Error, IntoValue, RArray, RClass, RModule, Ruby, Value, prelude::*};
|
|
15
|
+
use matryona_machines::ruby::{
|
|
16
|
+
Class, SymbolNames, Symbols, class_name, duration, exact_fields_with, integer, nanoseconds,
|
|
17
|
+
};
|
|
18
|
+
use supervisor_machines::{Decision, Exit, Intensity, Policy, Recording, Refusal, Span, Strategy};
|
|
19
|
+
|
|
20
|
+
matryona_machines::symbols! {
|
|
21
|
+
/// Every [`Strategy`], in the order the core declares them.
|
|
22
|
+
pub(crate) Strategies for Strategy as "strategy" {
|
|
23
|
+
OneForOne => "one_for_one",
|
|
24
|
+
OneForAll => "one_for_all",
|
|
25
|
+
RestForOne => "rest_for_one",
|
|
26
|
+
}
|
|
27
|
+
}
|
|
13
28
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
/// `
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
29
|
+
matryona_machines::symbols! {
|
|
30
|
+
/// Every [`Policy`], in the order the core declares them.
|
|
31
|
+
pub(crate) Policies for Policy as "policy" {
|
|
32
|
+
Permanent => "permanent",
|
|
33
|
+
Transient => "transient",
|
|
34
|
+
Temporary => "temporary",
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
matryona_machines::symbols! {
|
|
39
|
+
/// Every [`Exit`], in the order the core declares them.
|
|
40
|
+
pub(crate) Exits for Exit as "exit" {
|
|
41
|
+
Normal => "normal",
|
|
42
|
+
Abnormal => "abnormal",
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
matryona_machines::symbols! {
|
|
47
|
+
/// The core's refusals.
|
|
48
|
+
pub(crate) Refusals for Refusal as "a refusal" {
|
|
49
|
+
UnknownChild => "unknown_child",
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
matryona_machines::symbols! {
|
|
54
|
+
/// Every [`Decision`]; `Restart` leaves as a `Restart`, the rest as
|
|
55
|
+
/// these Symbols.
|
|
56
|
+
pub(crate) Decisions for Decision as "a decision" => names_only {
|
|
57
|
+
Ignore => "ignore",
|
|
58
|
+
Restart(..) => "restart",
|
|
59
|
+
Escalate => "escalate",
|
|
60
|
+
AlreadyEscalated => "already_escalated",
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
matryona_machines::symbols! {
|
|
65
|
+
/// What `record_external_restart` answers.
|
|
66
|
+
pub(crate) Recordings for Recording as "a recording" {
|
|
67
|
+
Recorded => "recorded",
|
|
68
|
+
Outdated => "outdated",
|
|
37
69
|
}
|
|
38
70
|
}
|
|
39
71
|
|
|
40
|
-
fn
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
72
|
+
pub(crate) fn module(ruby: &Ruby) -> Result<RModule, Error> {
|
|
73
|
+
ruby.define_module("SupervisorMachines")
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/// The gem's `Data` classes, resolved once at load (`values.rb` defines them
|
|
77
|
+
/// before the extension is required) and pinned: the exact-class check
|
|
78
|
+
/// compares against these, and no answer looks a class up by name after the
|
|
79
|
+
/// core decided.
|
|
80
|
+
struct Classes {
|
|
81
|
+
intensity: Class,
|
|
82
|
+
span: Class,
|
|
83
|
+
restart: Class,
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
matryona_machines::registry! {
|
|
87
|
+
static CLASSES: Classes = "supervisor_machines";
|
|
88
|
+
fn classes;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
pub(crate) fn resolve_classes(ruby: &Ruby, module: RModule) -> Result<(), Error> {
|
|
92
|
+
let class = |name| Class::resolve(ruby, module, name);
|
|
93
|
+
let classes = Classes {
|
|
94
|
+
intensity: class("Intensity")?,
|
|
95
|
+
span: class("Span")?,
|
|
96
|
+
restart: class("Restart")?,
|
|
97
|
+
};
|
|
98
|
+
CLASSES.set(ruby, classes).map(drop)
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
fn checked_intensity(ruby: &Ruby, restarts: Value, window: Value) -> Result<Intensity, Error> {
|
|
102
|
+
let (restarts, window) = (integer(restarts, "restarts")?, duration(window, "window")?);
|
|
103
|
+
Intensity::new(restarts, window).ok_or_else(|| {
|
|
104
|
+
Error::new(
|
|
105
|
+
ruby.exception_range_error(),
|
|
106
|
+
"an intensity window must be longer than zero",
|
|
107
|
+
)
|
|
108
|
+
})
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
fn checked_span(ruby: &Ruby, first: Value, last: Value) -> Result<Span, Error> {
|
|
112
|
+
let (first, last) = (integer(first, "first")?, integer(last, "last")?);
|
|
113
|
+
Span::new(first, last).ok_or_else(|| {
|
|
114
|
+
Error::new(
|
|
115
|
+
ruby.exception_range_error(),
|
|
116
|
+
format!("a span cannot end at {last} before it starts at {first}"),
|
|
117
|
+
)
|
|
118
|
+
})
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// A value read back is an instance of exactly the gem's class (a subclass's
|
|
122
|
+
// constant leaks past the seal, so `is_a?` is not enough; an opened singleton
|
|
123
|
+
// class is fine), its fields read from the struct itself so a redefined
|
|
124
|
+
// reader is never called, and it is revalidated through the core.
|
|
125
|
+
|
|
126
|
+
/// A `SupervisorMachines::Intensity`, revalidated through the core.
|
|
127
|
+
pub(crate) fn intensity_from(ruby: &Ruby, value: Value) -> Result<Intensity, Error> {
|
|
128
|
+
let class = classes(ruby)?.intensity.get(ruby);
|
|
129
|
+
let [restarts, window] = exact_fields_with(ruby, value, class, || {
|
|
130
|
+
format!(
|
|
131
|
+
"expected a SupervisorMachines::Intensity, not {}",
|
|
132
|
+
class_name(value)
|
|
133
|
+
)
|
|
134
|
+
})?;
|
|
135
|
+
checked_intensity(ruby, restarts, window)
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/// A `SupervisorMachines::Span`, revalidated through the core.
|
|
139
|
+
fn span_from(ruby: &Ruby, value: Value) -> Result<Span, Error> {
|
|
140
|
+
let class = classes(ruby)?.span.get(ruby);
|
|
141
|
+
let [first, last] = exact_fields_with(ruby, value, class, || {
|
|
44
142
|
format!(
|
|
45
|
-
"
|
|
46
|
-
value
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
)
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
(
|
|
55
|
-
("
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
///
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
pub(crate)
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
let name = Symbol::from_value(value).and_then(|symbol| symbol.name().ok());
|
|
73
|
-
name.as_deref()
|
|
74
|
-
.and_then(|name| set.iter().find(|(known, _)| *known == name))
|
|
75
|
-
.map(|&(_, member)| member)
|
|
76
|
-
.ok_or_else(|| {
|
|
77
|
-
Error::new(
|
|
78
|
-
ruby.exception_arg_error(),
|
|
79
|
-
format!("unknown {kind}: {}", value.inspect()),
|
|
80
|
-
)
|
|
143
|
+
"expected a SupervisorMachines::Span, not {}",
|
|
144
|
+
class_name(value)
|
|
145
|
+
)
|
|
146
|
+
})?;
|
|
147
|
+
checked_span(ruby, first, last)
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
pub(crate) fn intensity_value(ruby: &Ruby, intensity: Intensity) -> Result<Value, Error> {
|
|
151
|
+
let window = nanoseconds(ruby, intensity.window());
|
|
152
|
+
let class = classes(ruby)?.intensity.get(ruby);
|
|
153
|
+
class.funcall("new", (intensity.restarts(), window))
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/// The classes a [`Decision`] is built from, taken before the core is asked
|
|
157
|
+
/// so that nothing after it can fail on a lookup.
|
|
158
|
+
#[derive(Clone, Copy)]
|
|
159
|
+
pub(crate) struct DecisionClasses {
|
|
160
|
+
span: RClass,
|
|
161
|
+
restart: RClass,
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
impl DecisionClasses {
|
|
165
|
+
pub(crate) fn fetch(ruby: &Ruby) -> Result<Self, Error> {
|
|
166
|
+
let classes = classes(ruby)?;
|
|
167
|
+
Ok(Self {
|
|
168
|
+
span: classes.span.get(ruby),
|
|
169
|
+
restart: classes.restart.get(ruby),
|
|
81
170
|
})
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
pub(crate) fn span(self, span: Span) -> Result<Value, Error> {
|
|
174
|
+
self.span.funcall("new", (span.first(), span.last()))
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/// `:ignore`, a `Restart`, `:escalate` or `:already_escalated`.
|
|
178
|
+
pub(crate) fn decision(self, ruby: &Ruby, decision: Decision) -> Result<Value, Error> {
|
|
179
|
+
match decision {
|
|
180
|
+
Decision::Restart(span) => self.restart.funcall("new", (self.span(span)?,)),
|
|
181
|
+
unit @ (Decision::Ignore | Decision::Escalate | Decision::AlreadyEscalated) => {
|
|
182
|
+
Ok(Decisions::symbol_of(ruby, unit).as_value())
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
82
186
|
}
|
|
83
187
|
|
|
84
|
-
|
|
85
|
-
|
|
188
|
+
/// What `Intensity.new` stores: the validated restart count and window.
|
|
189
|
+
pub(crate) fn intensity_hook(ruby: &Ruby, restarts: Value, window: Value) -> Result<Value, Error> {
|
|
190
|
+
let intensity = checked_intensity(ruby, restarts, window)?;
|
|
191
|
+
Ok((intensity.restarts(), nanoseconds(ruby, intensity.window())).into_value_with(ruby))
|
|
86
192
|
}
|
|
87
193
|
|
|
88
|
-
|
|
89
|
-
|
|
194
|
+
/// What `Span.new` stores: the validated ends.
|
|
195
|
+
pub(crate) fn span_hook(ruby: &Ruby, first: Value, last: Value) -> Result<(usize, usize), Error> {
|
|
196
|
+
checked_span(ruby, first, last).map(|span| (span.first(), span.last()))
|
|
90
197
|
}
|
|
91
198
|
|
|
92
|
-
|
|
93
|
-
|
|
199
|
+
/// What `Restart.new` stores: a `SupervisorMachines::Span`, checked for its
|
|
200
|
+
/// exact class and revalidated through the core here, not in Ruby.
|
|
201
|
+
pub(crate) fn restart_hook(ruby: &Ruby, span: Value) -> Result<Value, Error> {
|
|
202
|
+
span_from(ruby, span).map(|_| span)
|
|
94
203
|
}
|
|
95
204
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
.find(|&&(_, known)| known == strategy)
|
|
100
|
-
.map_or("", |&(name, _)| name);
|
|
101
|
-
ruby.to_symbol(name)
|
|
205
|
+
/// A policy symbol the core knows, or `TypeError`/`RangeError`.
|
|
206
|
+
pub(crate) fn policy_hook(ruby: &Ruby, value: Value) -> Result<Value, Error> {
|
|
207
|
+
Policies::parse(ruby, value).map(|_| value)
|
|
102
208
|
}
|
|
103
209
|
|
|
104
|
-
/// A
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
match decision {
|
|
108
|
-
Decision::Ignore => Ok(ruby.to_symbol("ignore").as_value()),
|
|
109
|
-
Decision::Escalate => Ok(ruby.to_symbol("escalate").as_value()),
|
|
110
|
-
Decision::Restart(span) => ruby
|
|
111
|
-
.range_new(span.first, span.last, false)
|
|
112
|
-
.map(magnus::value::ReprValue::as_value),
|
|
113
|
-
}
|
|
210
|
+
/// A strategy symbol the core knows, or `TypeError`/`RangeError`.
|
|
211
|
+
pub(crate) fn strategy_hook(ruby: &Ruby, value: Value) -> Result<Value, Error> {
|
|
212
|
+
Strategies::parse(ruby, value).map(|_| value)
|
|
114
213
|
}
|
|
115
214
|
|
|
116
|
-
/// A frozen array of
|
|
117
|
-
fn
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
215
|
+
/// A frozen array of `items`, in order.
|
|
216
|
+
pub(crate) fn frozen<T: IntoValue>(
|
|
217
|
+
ruby: &Ruby,
|
|
218
|
+
items: impl ExactSizeIterator<Item = T>,
|
|
219
|
+
) -> Result<RArray, Error> {
|
|
220
|
+
let array = ruby.ary_new_capa(items.len());
|
|
221
|
+
for item in items {
|
|
222
|
+
array.push(item)?;
|
|
121
223
|
}
|
|
122
224
|
array.freeze();
|
|
123
225
|
Ok(array)
|
|
124
226
|
}
|
|
125
|
-
|
|
126
|
-
#[magnus::init]
|
|
127
|
-
fn init(ruby: &Ruby) -> Result<(), Error> {
|
|
128
|
-
let module = ruby.define_module("SupervisorMachines")?;
|
|
129
|
-
module.const_set("NS_PER_SECOND", NS_PER_SECOND)?;
|
|
130
|
-
module.const_set("DEFAULT_RESTARTS", DEFAULT_RESTARTS)?;
|
|
131
|
-
module.const_set("DEFAULT_WINDOW_NS", DEFAULT_WINDOW_NS)?;
|
|
132
|
-
module.const_set("STRATEGIES", names(ruby, &STRATEGIES)?)?;
|
|
133
|
-
module.const_set("POLICIES", names(ruby, &POLICIES)?)?;
|
|
134
|
-
decider::define(ruby, module)
|
|
135
|
-
}
|
|
@@ -14,39 +14,74 @@ module SupervisorMachines
|
|
|
14
14
|
# are final: {#start} refuses a tree that is not registering, and a +failed+
|
|
15
15
|
# tree answers every {#stop} with {Escalated}.
|
|
16
16
|
#
|
|
17
|
-
#
|
|
17
|
+
# Every instant and span it takes or reports is Integer nanoseconds, as
|
|
18
|
+
# deadline_machines takes them; only Thread#join gets Float seconds.
|
|
19
|
+
#
|
|
20
|
+
# minute = 60 * 1_000_000_000
|
|
21
|
+
# tree = SupervisorMachines::Supervisor.new(:one_for_all, SupervisorMachines::Intensity.new(3, minute))
|
|
18
22
|
# tree.supervise(:cache) { Thread.new { cache.run } }
|
|
19
23
|
# tree.supervise(:warmup, policy: :transient) { Thread.new { warm } }
|
|
20
24
|
# tree.start
|
|
21
|
-
# tree.stop(
|
|
25
|
+
# tree.stop(5 * 1_000_000_000)
|
|
22
26
|
class Supervisor
|
|
23
27
|
# A registered child and whichever thread currently runs it. +epoch+ rises
|
|
24
28
|
# every time the child is taken down, so a thread the tree has already
|
|
25
29
|
# replaced neither reports its exit nor takes the child's place when its
|
|
26
30
|
# starter returns late. +done+ marks a child the tree has dropped, which no
|
|
27
|
-
# group restart brings back.
|
|
28
|
-
|
|
31
|
+
# group restart brings back. A child is replaced, never mutated, and only
|
|
32
|
+
# under the tree's lock.
|
|
33
|
+
Child = Data.define(:name, :policy, :starter, :thread, :epoch, :done) do
|
|
34
|
+
def initialize(name:, policy:, starter:, thread:, epoch:, done:)
|
|
35
|
+
policy = SupervisorMachines.__send__(:__policy, policy)
|
|
36
|
+
super
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
Child.singleton_class.undef_method(:allocate)
|
|
40
|
+
Child.define_singleton_method(:inherited) { |_| raise TypeError, "#{Child} cannot be subclassed" }
|
|
41
|
+
private_constant :Child
|
|
42
|
+
|
|
43
|
+
# Converts a budget in nanoseconds into the seconds Thread#join takes.
|
|
44
|
+
NS_PER_SECOND = 1_000_000_000
|
|
45
|
+
private_constant :NS_PER_SECOND
|
|
46
|
+
|
|
47
|
+
# The respawn delay when none is given: 100 ms doubling to 10 s, full
|
|
48
|
+
# jitter.
|
|
49
|
+
BACKOFF = ChronoMachines::Exponential.new(NS_PER_SECOND / 10, 2.0, 10 * NS_PER_SECOND, 1.0)
|
|
50
|
+
|
|
51
|
+
# Jitter draws are uniform over 0...DRAWS.
|
|
52
|
+
DRAWS = 1 << 64
|
|
53
|
+
private_constant :DRAWS
|
|
29
54
|
|
|
30
55
|
# Wakes the monitor when the tree is being taken down.
|
|
31
56
|
STOP = Object.new
|
|
32
57
|
private_constant :STOP
|
|
33
58
|
|
|
34
|
-
attr_reader :strategy
|
|
59
|
+
attr_reader :strategy, :intensity
|
|
35
60
|
|
|
36
|
-
# +
|
|
37
|
-
#
|
|
38
|
-
#
|
|
39
|
-
#
|
|
40
|
-
#
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
61
|
+
# +strategy+ is one of STRATEGIES and +intensity+ an {Intensity}, as the
|
|
62
|
+
# core's +Supervisor::new+ takes them; past the intensity the tree
|
|
63
|
+
# escalates. TypeError for the wrong types, RangeError for an unknown
|
|
64
|
+
# strategy or more restarts than {Decider::MAX_RESTARTS}. +backoff+ is
|
|
65
|
+
# asked for the delay between a restart decision and the respawn:
|
|
66
|
+
# anything answering +delay(attempt, draw)+ with Integer nanoseconds, a
|
|
67
|
+
# +chrono_machines+ shape ({BACKOFF} unless told otherwise). It has no
|
|
68
|
+
# attempt budget of its own: the core decides when the tree escalates.
|
|
69
|
+
#
|
|
70
|
+
# +backoff:+ and {#supervise}'s +policy: :permanent+ are keyword defaults
|
|
71
|
+
# the core has no counterpart for. They stay because this class is a
|
|
72
|
+
# runtime on top of the core, not a mirror of it; {Decider} is the mirror.
|
|
73
|
+
def initialize(strategy, intensity, backoff: nil)
|
|
74
|
+
strategy = SupervisorMachines.__send__(:__strategy, strategy)
|
|
75
|
+
# Read natively, never through `instance_of?` or the readers, and
|
|
76
|
+
# rebuilt through the core: `Marshal.load` skips `initialize`.
|
|
77
|
+
intensity = SupervisorMachines.__send__(:__supervised_intensity, intensity)
|
|
45
78
|
|
|
46
79
|
@strategy = strategy
|
|
47
|
-
@
|
|
48
|
-
|
|
49
|
-
|
|
80
|
+
@intensity = intensity
|
|
81
|
+
backoff ||= BACKOFF
|
|
82
|
+
raise TypeError, "backoff must answer delay, got #{backoff.inspect}" unless backoff.respond_to?(:delay)
|
|
83
|
+
|
|
84
|
+
@backoff = backoff
|
|
50
85
|
@slots = []
|
|
51
86
|
@events = Thread::Queue.new
|
|
52
87
|
@mutex = Thread::Mutex.new
|
|
@@ -57,13 +92,14 @@ module SupervisorMachines
|
|
|
57
92
|
|
|
58
93
|
# Registers a child under +name+. The block is its starter: it returns the
|
|
59
94
|
# +Thread+ that runs the child, and is called again for every restart.
|
|
95
|
+
# TypeError for a +policy+ that is not a Symbol, RangeError for one outside
|
|
96
|
+
# POLICIES.
|
|
60
97
|
def supervise(name, policy: :permanent, &starter)
|
|
61
|
-
|
|
62
|
-
|
|
98
|
+
child = Child.new(name, policy, starter, nil, 0, false)
|
|
63
99
|
@mutex.synchronize do
|
|
64
100
|
raise Error, "a #{@state} supervisor cannot take on a child" unless @state == :registering
|
|
65
101
|
|
|
66
|
-
@slots <<
|
|
102
|
+
@slots << child
|
|
67
103
|
end
|
|
68
104
|
self
|
|
69
105
|
end
|
|
@@ -73,13 +109,13 @@ module SupervisorMachines
|
|
|
73
109
|
# already up back down, leaves the tree failed, and raises what the starter
|
|
74
110
|
# raised.
|
|
75
111
|
#
|
|
76
|
-
# Raises {Error} for a tree that is past registering
|
|
112
|
+
# Raises {Error} for a tree that is past registering, RangeError for a
|
|
113
|
+
# child count the core has no shape for.
|
|
77
114
|
def start
|
|
78
115
|
@mutex.synchronize do
|
|
79
116
|
raise Error, "a #{@state} supervisor cannot be started" unless @state == :registering
|
|
80
117
|
|
|
81
|
-
@decider = Decider.new(
|
|
82
|
-
restarts: @restarts, window_ns: to_ns(@window))
|
|
118
|
+
@decider = Decider.new(@slots.size, @strategy, @intensity)
|
|
83
119
|
@state = :running
|
|
84
120
|
@monitor = Thread.new { monitor }
|
|
85
121
|
end
|
|
@@ -107,11 +143,14 @@ module SupervisorMachines
|
|
|
107
143
|
end
|
|
108
144
|
|
|
109
145
|
# Takes every running child down in reverse start order, giving the whole
|
|
110
|
-
# shutdown +grace+
|
|
111
|
-
# A tree still registering has nothing to take down and stays
|
|
146
|
+
# shutdown +grace+ nanoseconds, or all the time it needs when +grace+ is
|
|
147
|
+
# nil. A tree still registering has nothing to take down and stays
|
|
148
|
+
# registering. TypeError for a non-Integer +grace+, RangeError for a
|
|
149
|
+
# negative one, both before anything is touched.
|
|
112
150
|
#
|
|
113
151
|
# Raises {Escalated} for a tree that gave up, every time it is called.
|
|
114
|
-
def stop(grace
|
|
152
|
+
def stop(grace = nil)
|
|
153
|
+
budget = grace.nil? ? nil : grace_ns(grace)
|
|
115
154
|
entered = @mutex.synchronize do
|
|
116
155
|
state = @state
|
|
117
156
|
@state = :stopped if state == :running
|
|
@@ -122,9 +161,9 @@ module SupervisorMachines
|
|
|
122
161
|
|
|
123
162
|
@events.push(STOP) if entered == :running
|
|
124
163
|
failures = nil
|
|
125
|
-
DeadlineMachines.with_deadline(
|
|
164
|
+
DeadlineMachines.with_deadline(budget) do |deadline|
|
|
126
165
|
failures = shutdown
|
|
127
|
-
@monitor.kill unless @monitor.join(deadline.remaining)
|
|
166
|
+
@monitor.kill unless @monitor.join(join_seconds(deadline.remaining))
|
|
128
167
|
end
|
|
129
168
|
raise Escalated, escalation if terminated?
|
|
130
169
|
raise failures.first unless failures.empty?
|
|
@@ -132,10 +171,10 @@ module SupervisorMachines
|
|
|
132
171
|
self
|
|
133
172
|
end
|
|
134
173
|
|
|
135
|
-
#
|
|
136
|
-
# neither the monitor thread nor a failing {#start} has a caller
|
|
137
|
-
# read.
|
|
138
|
-
ESCALATION_GRACE = 5
|
|
174
|
+
# Nanoseconds a supervisor taking itself down waits for a killed child,
|
|
175
|
+
# since neither the monitor thread nor a failing {#start} has a caller
|
|
176
|
+
# budget to read.
|
|
177
|
+
ESCALATION_GRACE = 5 * NS_PER_SECOND
|
|
139
178
|
|
|
140
179
|
private
|
|
141
180
|
|
|
@@ -182,14 +221,17 @@ module SupervisorMachines
|
|
|
182
221
|
child = @slots[index]
|
|
183
222
|
next unless running? && child.epoch == epoch
|
|
184
223
|
|
|
185
|
-
child.thread
|
|
186
|
-
|
|
187
|
-
@decider.stopped(index: index, policy: child.policy, exit: ending, now_ns: now)
|
|
224
|
+
@slots[index] = child.with(thread: nil, done: child.done || child.policy == :temporary)
|
|
225
|
+
@decider.stopped(index, child.policy, ending, now)
|
|
188
226
|
end
|
|
189
227
|
|
|
190
228
|
case decision
|
|
191
|
-
when
|
|
229
|
+
when Restart then restart(decision.span)
|
|
192
230
|
when :escalate then escalate
|
|
231
|
+
# Nothing to carry out: a child left stopped, a tree that already gave
|
|
232
|
+
# up, an index the tree does not own (it only asks about its own), or
|
|
233
|
+
# an event from a generation the tree has replaced.
|
|
234
|
+
when :ignore, :already_escalated, :unknown_child, nil then nil
|
|
193
235
|
end
|
|
194
236
|
end
|
|
195
237
|
|
|
@@ -197,7 +239,7 @@ module SupervisorMachines
|
|
|
197
239
|
# waits out the backoff without holding the tree, then brings the span back
|
|
198
240
|
# in index order.
|
|
199
241
|
def restart(span)
|
|
200
|
-
retiring = @mutex.synchronize { span.
|
|
242
|
+
retiring = @mutex.synchronize { span.last.downto(span.first).filter_map { |index| retire(index) } }
|
|
201
243
|
DeadlineMachines.with_deadline(ESCALATION_GRACE) do |deadline|
|
|
202
244
|
retiring.each { |thread| reap(thread, deadline.remaining) }
|
|
203
245
|
end
|
|
@@ -209,9 +251,9 @@ module SupervisorMachines
|
|
|
209
251
|
|
|
210
252
|
# The reason a respawn could not happen, or nil once the span is running.
|
|
211
253
|
def respawn(span)
|
|
212
|
-
delay = @backoff.
|
|
213
|
-
@mutex.synchronize { @halt.wait(@mutex, delay) if delay && running? }
|
|
214
|
-
span.
|
|
254
|
+
delay = @backoff.delay(restarts.clamp(1, 255), Random.rand(DRAWS))
|
|
255
|
+
@mutex.synchronize { @halt.wait(@mutex, delay.fdiv(NS_PER_SECOND)) if delay.positive? && running? }
|
|
256
|
+
span.first.upto(span.last) { |index| spawn(index) }
|
|
215
257
|
nil
|
|
216
258
|
rescue Exception => e
|
|
217
259
|
unstarted(e)
|
|
@@ -227,7 +269,7 @@ module SupervisorMachines
|
|
|
227
269
|
end
|
|
228
270
|
|
|
229
271
|
# Reverse start order, each child killed and then waited for inside one
|
|
230
|
-
# budget shared by the whole tree. Touches no tree state, so a shutdown
|
|
272
|
+
# budget of +grace+ nanoseconds (nil: no limit) shared by the whole tree. Touches no tree state, so a shutdown
|
|
231
273
|
# under way in the monitor never holds up the one in {#stop}. Returns what
|
|
232
274
|
# the children raised on their way out, having stopped all of them.
|
|
233
275
|
def shutdown(grace = DeadlineMachines.remaining)
|
|
@@ -243,12 +285,12 @@ module SupervisorMachines
|
|
|
243
285
|
failures.compact
|
|
244
286
|
end
|
|
245
287
|
|
|
246
|
-
# Kills the thread and waits out the rest of the budget
|
|
247
|
-
#
|
|
248
|
-
# still alive afterwards.
|
|
288
|
+
# Kills the thread and waits out the rest of the budget, +remaining+
|
|
289
|
+
# nanoseconds or nil for no limit, for it, reporting whatever it raised on
|
|
290
|
+
# the way out. A thread that outlives the budget is still alive afterwards.
|
|
249
291
|
def reap(thread, remaining)
|
|
250
292
|
thread.kill
|
|
251
|
-
thread.join(remaining)
|
|
293
|
+
thread.join(join_seconds(remaining))
|
|
252
294
|
nil
|
|
253
295
|
rescue Exception => e
|
|
254
296
|
e
|
|
@@ -258,9 +300,11 @@ module SupervisorMachines
|
|
|
258
300
|
# is in when the call goes out. A child the tree has dropped, and every
|
|
259
301
|
# child of a tree that is no longer running, is left alone.
|
|
260
302
|
def spawn(index)
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
303
|
+
generation, starter = @mutex.synchronize do
|
|
304
|
+
child = @slots[index]
|
|
305
|
+
[child.epoch, child.starter] if running? && !child.done
|
|
306
|
+
end
|
|
307
|
+
install(index, generation, starter.call) if generation
|
|
264
308
|
end
|
|
265
309
|
|
|
266
310
|
# Hands the started thread to the child, unless the tree has moved past the
|
|
@@ -271,20 +315,19 @@ module SupervisorMachines
|
|
|
271
315
|
child = @slots[index]
|
|
272
316
|
next false unless running? && child.epoch == generation
|
|
273
317
|
|
|
274
|
-
child.thread
|
|
318
|
+
@slots[index] = child.with(thread: thread)
|
|
275
319
|
watch(index, generation, thread)
|
|
276
320
|
true
|
|
277
321
|
end
|
|
278
322
|
reap(thread, ESCALATION_GRACE) unless installed
|
|
279
323
|
end
|
|
280
324
|
|
|
281
|
-
# Invalidates the watcher following the child and hands its
|
|
282
|
-
# be reaped outside the lock.
|
|
283
|
-
def retire(
|
|
284
|
-
child
|
|
285
|
-
|
|
286
|
-
child.thread
|
|
287
|
-
thread
|
|
325
|
+
# Invalidates the watcher following the child at +index+ and hands its
|
|
326
|
+
# thread back to be reaped outside the lock.
|
|
327
|
+
def retire(index)
|
|
328
|
+
child = @slots[index]
|
|
329
|
+
@slots[index] = child.with(epoch: child.epoch + 1, thread: nil)
|
|
330
|
+
child.thread
|
|
288
331
|
end
|
|
289
332
|
|
|
290
333
|
# One thread per running child, blocked on it, reporting how it ended.
|
|
@@ -298,11 +341,12 @@ module SupervisorMachines
|
|
|
298
341
|
end
|
|
299
342
|
|
|
300
343
|
def escalation
|
|
301
|
-
@failed || "#{@strategy} supervisor escalated: more than #{@restarts} restarts in
|
|
344
|
+
@failed || "#{@strategy} supervisor escalated: more than #{@intensity.restarts} restarts in " \
|
|
345
|
+
"#{@intensity.window} ns"
|
|
302
346
|
end
|
|
303
347
|
|
|
304
348
|
def abandoned
|
|
305
|
-
"#{@strategy} supervisor escalated: a retired child would not stop within #{ESCALATION_GRACE}
|
|
349
|
+
"#{@strategy} supervisor escalated: a retired child would not stop within #{ESCALATION_GRACE} ns"
|
|
306
350
|
end
|
|
307
351
|
|
|
308
352
|
def unstarted(error)
|
|
@@ -313,8 +357,17 @@ module SupervisorMachines
|
|
|
313
357
|
DeadlineMachines.clock.now_ns
|
|
314
358
|
end
|
|
315
359
|
|
|
316
|
-
|
|
317
|
-
|
|
360
|
+
# A grace checked before anything is touched: Integer nanoseconds.
|
|
361
|
+
def grace_ns(nanos)
|
|
362
|
+
raise TypeError, "a grace is Integer nanoseconds, got #{nanos.inspect}" unless nanos.is_a?(Integer)
|
|
363
|
+
raise RangeError, "a grace cannot be negative: #{nanos}" if nanos.negative?
|
|
364
|
+
|
|
365
|
+
nanos
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
# A budget in nanoseconds, or nil for none, as Thread#join's timeout.
|
|
369
|
+
def join_seconds(nanos)
|
|
370
|
+
nanos&.fdiv(NS_PER_SECOND)
|
|
318
371
|
end
|
|
319
372
|
end
|
|
320
373
|
end
|