supervisor_machines 0.1.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +52 -0
- data/Cargo.lock +22 -2
- data/Cargo.toml +3 -0
- data/README.md +101 -6
- data/ext/supervisor_machines_native/core/Cargo.toml +4 -6
- data/ext/supervisor_machines_native/core/src/lib.rs +102 -22
- data/ext/supervisor_machines_native/core/src/supervisor.rs +270 -154
- data/ext/supervisor_machines_native/ffi/Cargo.toml +5 -9
- data/ext/supervisor_machines_native/ffi/src/decider.rs +333 -117
- data/ext/supervisor_machines_native/ffi/src/init.rs +30 -0
- data/ext/supervisor_machines_native/ffi/src/lib.rs +199 -108
- data/lib/supervisor_machines/supervisor.rb +113 -60
- data/lib/supervisor_machines/values.rb +63 -0
- data/lib/supervisor_machines/version.rb +1 -1
- data/lib/supervisor_machines.rb +12 -3
- data/sig/supervisor_machines.rbs +179 -63
- metadata +7 -5
|
@@ -1,109 +1,133 @@
|
|
|
1
1
|
//! The decision machine itself.
|
|
2
2
|
|
|
3
|
-
use
|
|
3
|
+
use core::time::Duration;
|
|
4
|
+
|
|
5
|
+
use crate::{Decision, Exit, Intensity, Policy, Recording, Refusal, Span, Strategy};
|
|
4
6
|
|
|
5
7
|
/// Holds the strategy, the restart intensity, and the recent restart history.
|
|
6
8
|
///
|
|
7
9
|
/// `CHILDREN` is how many children the supervisor owns, which fixes the
|
|
8
|
-
/// strategy spans. `HISTORY` is the restart
|
|
9
|
-
/// intensity's restart count, or a
|
|
10
|
+
/// strategy spans. `HISTORY` is the restart instants kept; it must exceed the
|
|
11
|
+
/// intensity's restart count, or a burst past the intensity could not be
|
|
12
|
+
/// measured: [`new`](Self::new) answers `None` otherwise.
|
|
13
|
+
///
|
|
14
|
+
/// The supervisor keeps no clock of its own. Its only memory of time is the
|
|
15
|
+
/// history, a sorted multiset of restart instants, so restarts may be reported in
|
|
16
|
+
/// any order (several processes sharing one history seldom agree on one).
|
|
10
17
|
///
|
|
11
18
|
/// ```
|
|
19
|
+
/// use core::time::Duration;
|
|
12
20
|
/// use supervisor_machines::{Decision, Exit, Intensity, Policy, Span, Strategy, Supervisor};
|
|
13
21
|
///
|
|
14
|
-
///
|
|
15
|
-
///
|
|
22
|
+
/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|
16
23
|
/// // Three children that only work as a set, two restarts per minute allowed.
|
|
17
|
-
/// let
|
|
18
|
-
///
|
|
19
|
-
///
|
|
20
|
-
/// );
|
|
24
|
+
/// let intensity = Intensity::new(2, Duration::from_secs(60)).ok_or("zero window")?;
|
|
25
|
+
/// let mut tree: Supervisor<3, 8> =
|
|
26
|
+
/// Supervisor::new(Strategy::OneForAll, intensity).ok_or("shape")?;
|
|
21
27
|
///
|
|
22
28
|
/// // The middle child dies badly, so all three come back.
|
|
23
|
-
/// let decision = tree.stopped(1, Policy::Permanent, Exit::Abnormal,
|
|
24
|
-
/// assert_eq!(decision,
|
|
29
|
+
/// let decision = tree.stopped(1, Policy::Permanent, Exit::Abnormal, Duration::ZERO)?;
|
|
30
|
+
/// assert_eq!(Some(decision), Span::new(0, 2).map(Decision::Restart));
|
|
25
31
|
///
|
|
26
32
|
/// // A temporary child is never worth restarting.
|
|
27
33
|
/// assert_eq!(
|
|
28
|
-
/// tree.stopped(2, Policy::Temporary, Exit::Abnormal,
|
|
29
|
-
/// Decision::Ignore,
|
|
34
|
+
/// tree.stopped(2, Policy::Temporary, Exit::Abnormal, Duration::from_secs(1)),
|
|
35
|
+
/// Ok(Decision::Ignore),
|
|
30
36
|
/// );
|
|
37
|
+
/// # Ok(())
|
|
38
|
+
/// # }
|
|
31
39
|
/// ```
|
|
32
40
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
33
41
|
pub struct Supervisor<const CHILDREN: usize, const HISTORY: usize> {
|
|
34
42
|
strategy: Strategy,
|
|
35
43
|
intensity: Intensity,
|
|
36
|
-
history: [
|
|
44
|
+
history: [Duration; HISTORY],
|
|
37
45
|
recorded: usize,
|
|
38
46
|
terminated: bool,
|
|
39
47
|
}
|
|
40
48
|
|
|
41
49
|
impl<const CHILDREN: usize, const HISTORY: usize> Supervisor<CHILDREN, HISTORY> {
|
|
42
|
-
///
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
CHILDREN > 0,
|
|
46
|
-
"a supervisor with no children has nothing to decide"
|
|
47
|
-
);
|
|
48
|
-
assert!(
|
|
49
|
-
HISTORY > 0,
|
|
50
|
-
"restart history needs room for at least one restart"
|
|
51
|
-
);
|
|
52
|
-
};
|
|
53
|
-
|
|
50
|
+
/// A supervisor with an empty history, or `None` for a shape that could
|
|
51
|
+
/// decide nothing: no children, or a history too short to hold one more
|
|
52
|
+
/// restart than the intensity tolerates.
|
|
54
53
|
#[must_use]
|
|
55
|
-
pub
|
|
56
|
-
|
|
57
|
-
|
|
54
|
+
pub fn new(strategy: Strategy, intensity: Intensity) -> Option<Self> {
|
|
55
|
+
// A restart count past `usize` is past any history.
|
|
56
|
+
let tolerated = usize::try_from(intensity.restarts()).unwrap_or(usize::MAX);
|
|
57
|
+
if CHILDREN == 0 || HISTORY <= tolerated {
|
|
58
|
+
return None;
|
|
59
|
+
}
|
|
60
|
+
Some(Self {
|
|
58
61
|
strategy,
|
|
59
62
|
intensity,
|
|
60
|
-
history: [
|
|
63
|
+
history: [Duration::ZERO; HISTORY],
|
|
61
64
|
recorded: 0,
|
|
62
65
|
terminated: false,
|
|
63
|
-
}
|
|
66
|
+
})
|
|
64
67
|
}
|
|
65
68
|
|
|
66
69
|
/// A supervisor that already holds `history`, as another process left it.
|
|
70
|
+
/// `None` exactly when [`new`](Self::new) would answer `None`.
|
|
67
71
|
///
|
|
68
|
-
///
|
|
69
|
-
///
|
|
72
|
+
/// The saved history is a multiset, as [`history`](Self::history)
|
|
73
|
+
/// reports it: every instant counts, so two restarts at one instant
|
|
74
|
+
/// restore as two ([`absorb`](Self::absorb) instead keeps the larger
|
|
75
|
+
/// count of each instant across both histories). Restarts older than the intensity
|
|
76
|
+
/// window as of `now` are dropped on the way in, and when more remain
|
|
77
|
+
/// than `HISTORY` can hold the oldest go.
|
|
78
|
+
/// Escalation is not part of the saved state: a restored supervisor is
|
|
79
|
+
/// not terminated, and decides for itself at its next
|
|
80
|
+
/// [`stopped`](Self::stopped). Nothing here is an issued id, so there is
|
|
81
|
+
/// no scope to restore under.
|
|
70
82
|
///
|
|
71
83
|
/// ```
|
|
84
|
+
/// use core::time::Duration;
|
|
72
85
|
/// use supervisor_machines::{Intensity, Strategy, Supervisor};
|
|
73
86
|
///
|
|
74
|
-
///
|
|
75
|
-
///
|
|
87
|
+
/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|
88
|
+
/// let seconds = Duration::from_secs;
|
|
89
|
+
/// let intensity = Intensity::new(3, seconds(60)).ok_or("zero window")?;
|
|
76
90
|
/// let mut before: Supervisor<1, 8> =
|
|
77
|
-
/// Supervisor::new(Strategy::OneForOne,
|
|
78
|
-
/// before.record_external_restart(10
|
|
79
|
-
/// before.record_external_restart(20
|
|
91
|
+
/// Supervisor::new(Strategy::OneForOne, intensity).ok_or("shape")?;
|
|
92
|
+
/// before.record_external_restart(seconds(10));
|
|
93
|
+
/// before.record_external_restart(seconds(20));
|
|
80
94
|
///
|
|
81
95
|
/// // Persist `before.history()`, then rebuild it elsewhere.
|
|
82
96
|
/// let after: Supervisor<1, 8> = Supervisor::restore(
|
|
83
97
|
/// before.strategy(),
|
|
84
98
|
/// before.intensity(),
|
|
85
99
|
/// before.history(),
|
|
86
|
-
/// 30
|
|
87
|
-
/// )
|
|
88
|
-
///
|
|
100
|
+
/// seconds(30),
|
|
101
|
+
/// )
|
|
102
|
+
/// .ok_or("shape")?;
|
|
103
|
+
/// assert_eq!(after.recent_restarts(seconds(30)), 2);
|
|
104
|
+
/// # Ok(())
|
|
105
|
+
/// # }
|
|
89
106
|
/// ```
|
|
90
107
|
#[must_use]
|
|
91
108
|
pub fn restore(
|
|
92
109
|
strategy: Strategy,
|
|
93
110
|
intensity: Intensity,
|
|
94
|
-
history: impl IntoIterator<Item =
|
|
95
|
-
|
|
96
|
-
) -> Self {
|
|
97
|
-
let mut supervisor = Self::new(strategy, intensity)
|
|
98
|
-
|
|
99
|
-
|
|
111
|
+
history: impl IntoIterator<Item = Duration>,
|
|
112
|
+
now: Duration,
|
|
113
|
+
) -> Option<Self> {
|
|
114
|
+
let mut supervisor = Self::new(strategy, intensity)?;
|
|
115
|
+
let cutoff = now.saturating_sub(intensity.window());
|
|
116
|
+
for at in history.into_iter().filter(|&at| at >= cutoff) {
|
|
117
|
+
// `false` is an instant older than a full history of newer
|
|
118
|
+
// ones: dropping it is what keeping the newest means.
|
|
119
|
+
supervisor.record(at);
|
|
120
|
+
}
|
|
121
|
+
Some(supervisor)
|
|
100
122
|
}
|
|
101
123
|
|
|
124
|
+
/// The strategy it was built with.
|
|
102
125
|
#[must_use]
|
|
103
126
|
pub const fn strategy(&self) -> Strategy {
|
|
104
127
|
self.strategy
|
|
105
128
|
}
|
|
106
129
|
|
|
130
|
+
/// The intensity it was built with.
|
|
107
131
|
#[must_use]
|
|
108
132
|
pub const fn intensity(&self) -> Intensity {
|
|
109
133
|
self.intensity
|
|
@@ -116,64 +140,99 @@ impl<const CHILDREN: usize, const HISTORY: usize> Supervisor<CHILDREN, HISTORY>
|
|
|
116
140
|
self.terminated
|
|
117
141
|
}
|
|
118
142
|
|
|
119
|
-
/// Every restart the supervisor still holds, oldest first
|
|
143
|
+
/// Every restart instant the supervisor still holds, oldest first.
|
|
120
144
|
///
|
|
121
|
-
///
|
|
122
|
-
///
|
|
123
|
-
|
|
124
|
-
|
|
145
|
+
/// An iterator over the supervisor's own storage, not an output buffer:
|
|
146
|
+
/// it borrows, copies nothing and allocates nothing. This is what to
|
|
147
|
+
/// persist so another process can [`restore`](Self::restore) or
|
|
148
|
+
/// [`absorb`](Self::absorb) it.
|
|
149
|
+
pub fn history(&self) -> impl Iterator<Item = Duration> + '_ {
|
|
150
|
+
self.held().iter().copied()
|
|
125
151
|
}
|
|
126
152
|
|
|
127
|
-
/// Restarts still inside the window as of `
|
|
153
|
+
/// Restarts still inside the window as of `now`. A pure query.
|
|
128
154
|
#[must_use]
|
|
129
|
-
pub
|
|
130
|
-
let cutoff =
|
|
131
|
-
let
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
155
|
+
pub fn recent_restarts(&self, now: Duration) -> u32 {
|
|
156
|
+
let cutoff = now.saturating_sub(self.intensity.window());
|
|
157
|
+
let recent = self
|
|
158
|
+
.held()
|
|
159
|
+
.iter()
|
|
160
|
+
.rev()
|
|
161
|
+
.take_while(|&&at| at >= cutoff)
|
|
162
|
+
.count();
|
|
163
|
+
u32::try_from(recent).unwrap_or(u32::MAX)
|
|
138
164
|
}
|
|
139
165
|
|
|
140
|
-
/// Count a restart some other process performed at `
|
|
166
|
+
/// Count a restart some other process performed at `now`.
|
|
141
167
|
///
|
|
142
168
|
/// It spends the intensity budget exactly as a restart this supervisor
|
|
143
169
|
/// granted would, but decides nothing: escalation waits for the next
|
|
144
|
-
/// [`stopped`](Self::stopped).
|
|
145
|
-
|
|
146
|
-
|
|
170
|
+
/// [`stopped`](Self::stopped). Never refused: any instant is a restart
|
|
171
|
+
/// that happened. Two calls with the same instant are two restarts.
|
|
172
|
+
pub fn record_external_restart(&mut self, now: Duration) -> Recording {
|
|
173
|
+
if self.record(now) {
|
|
174
|
+
Recording::Recorded
|
|
175
|
+
} else {
|
|
176
|
+
Recording::Outdated
|
|
177
|
+
}
|
|
147
178
|
}
|
|
148
179
|
|
|
149
180
|
/// Merge restarts another process holds, as its [`history`](Self::history)
|
|
150
|
-
/// reported them.
|
|
181
|
+
/// reported them (in any order), and answer how many were added.
|
|
182
|
+
///
|
|
183
|
+
/// Both histories are multisets and the merge keeps the larger count of
|
|
184
|
+
/// each instant: an instant held twice here and once in `history` stays
|
|
185
|
+
/// twice, one held once here and twice there ends up twice. So absorbing
|
|
186
|
+
/// the same history twice changes nothing and answers zero, the order of
|
|
187
|
+
/// two absorbs does not matter, and another process's two restarts in one
|
|
188
|
+
/// nanosecond stay two. Restarts in `history` older than the window as of
|
|
189
|
+
/// `now` are dropped. When the merge holds more than `HISTORY` the oldest
|
|
190
|
+
/// go, as for [`record_external_restart`](Self::record_external_restart),
|
|
191
|
+
/// and the answer counts only the instants still held. Never refused.
|
|
151
192
|
///
|
|
152
|
-
///
|
|
153
|
-
///
|
|
154
|
-
///
|
|
155
|
-
pub fn absorb(&mut self, history:
|
|
156
|
-
let cutoff =
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
193
|
+
/// Allocates nothing. A sorted `history` (as [`history`](Self::history)
|
|
194
|
+
/// yields it) is merged in one pass; an unsorted one costs a scan of the
|
|
195
|
+
/// earlier entries per entry.
|
|
196
|
+
pub fn absorb(&mut self, history: &[Duration], now: Duration) -> usize {
|
|
197
|
+
let cutoff = now.saturating_sub(self.intensity.window());
|
|
198
|
+
// The counts to compare against are this supervisor's before the
|
|
199
|
+
// merge: inserting shifts (and, when full, drops) what it holds.
|
|
200
|
+
let snapshot = *self;
|
|
201
|
+
let before = snapshot.held();
|
|
202
|
+
let sorted = history.is_sorted();
|
|
203
|
+
let mut previous = None;
|
|
204
|
+
let mut run = 0;
|
|
205
|
+
for (index, &at) in history.iter().enumerate() {
|
|
206
|
+
if sorted {
|
|
207
|
+
// `run` counts earlier entries of `history`: below its length.
|
|
208
|
+
run = if previous == Some(at) { run + 1 } else { 0 };
|
|
209
|
+
previous = Some(at);
|
|
160
210
|
}
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
211
|
+
if at < cutoff {
|
|
212
|
+
continue;
|
|
213
|
+
}
|
|
214
|
+
// How many times `at` came earlier in `history`: this entry is
|
|
215
|
+
// its occurrence number `rank`, wanted only past what is held.
|
|
216
|
+
let rank = if sorted {
|
|
217
|
+
run
|
|
218
|
+
} else {
|
|
219
|
+
history
|
|
220
|
+
.iter()
|
|
221
|
+
.take(index)
|
|
222
|
+
.filter(|&&earlier| earlier == at)
|
|
223
|
+
.count()
|
|
224
|
+
};
|
|
225
|
+
if rank >= occurrences(before, at) {
|
|
226
|
+
// `false` is an instant older than a full history of newer
|
|
227
|
+
// ones: dropping it is what keeping the newest means.
|
|
228
|
+
self.record(at);
|
|
169
229
|
}
|
|
170
|
-
index += 1;
|
|
171
230
|
}
|
|
172
|
-
|
|
231
|
+
added(before, self.held())
|
|
173
232
|
}
|
|
174
233
|
|
|
175
|
-
/// How long to hold off the restart just granted: `
|
|
176
|
-
///
|
|
234
|
+
/// How long to hold off the restart just granted: `base` doubled once per
|
|
235
|
+
/// restart already inside the window, capped at `ceiling`. A pure query.
|
|
177
236
|
///
|
|
178
237
|
/// Neither OTP nor [`stopped`](Self::stopped) imposes a delay, so a
|
|
179
238
|
/// supervisor that restarts the moment a child dies spends its whole
|
|
@@ -184,113 +243,170 @@ impl<const CHILDREN: usize, const HISTORY: usize> Supervisor<CHILDREN, HISTORY>
|
|
|
184
243
|
/// spreads the budget back over that window.
|
|
185
244
|
///
|
|
186
245
|
/// The count decays as the window slides, so a child that settles returns
|
|
187
|
-
/// to `
|
|
246
|
+
/// to `base` on its own. Jitter is the caller's to add.
|
|
188
247
|
///
|
|
189
248
|
/// ```
|
|
249
|
+
/// use core::time::Duration;
|
|
190
250
|
/// use supervisor_machines::{Exit, Intensity, Policy, Strategy, Supervisor};
|
|
191
251
|
///
|
|
192
|
-
///
|
|
193
|
-
///
|
|
194
|
-
/// let
|
|
195
|
-
///
|
|
196
|
-
///
|
|
197
|
-
/// );
|
|
252
|
+
/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|
253
|
+
/// let ms = Duration::from_millis;
|
|
254
|
+
/// let intensity = Intensity::new(5, ms(60_000)).ok_or("zero window")?;
|
|
255
|
+
/// let mut tree: Supervisor<1, 8> =
|
|
256
|
+
/// Supervisor::new(Strategy::OneForOne, intensity).ok_or("shape")?;
|
|
198
257
|
///
|
|
199
258
|
/// // The first restart waits the base, each further one twice as long.
|
|
200
|
-
/// tree.stopped(0, Policy::Permanent, Exit::Abnormal, 0)
|
|
201
|
-
/// assert_eq!(tree.
|
|
259
|
+
/// tree.stopped(0, Policy::Permanent, Exit::Abnormal, ms(0))?;
|
|
260
|
+
/// assert_eq!(tree.restart_delay(ms(100), ms(10_000), ms(0)), ms(100));
|
|
202
261
|
///
|
|
203
|
-
/// tree.stopped(0, Policy::Permanent, Exit::Abnormal,
|
|
204
|
-
/// assert_eq!(tree.
|
|
262
|
+
/// tree.stopped(0, Policy::Permanent, Exit::Abnormal, ms(1))?;
|
|
263
|
+
/// assert_eq!(tree.restart_delay(ms(100), ms(10_000), ms(1)), ms(200));
|
|
205
264
|
///
|
|
206
265
|
/// // Once the window has slid past them the delay is back to the base.
|
|
207
|
-
/// assert_eq!(
|
|
208
|
-
///
|
|
209
|
-
///
|
|
210
|
-
/// );
|
|
266
|
+
/// assert_eq!(tree.restart_delay(ms(100), ms(10_000), ms(120_000)), ms(100));
|
|
267
|
+
/// # Ok(())
|
|
268
|
+
/// # }
|
|
211
269
|
/// ```
|
|
212
270
|
#[must_use]
|
|
213
|
-
pub
|
|
214
|
-
let mut delay =
|
|
215
|
-
let mut doublings = self.recent_restarts(
|
|
216
|
-
while doublings > 0 && delay <
|
|
271
|
+
pub fn restart_delay(&self, base: Duration, ceiling: Duration, now: Duration) -> Duration {
|
|
272
|
+
let mut delay = base;
|
|
273
|
+
let mut doublings = self.recent_restarts(now).saturating_sub(1);
|
|
274
|
+
while doublings > 0 && delay < ceiling {
|
|
217
275
|
delay = delay.saturating_mul(2);
|
|
276
|
+
// Guarded by `doublings > 0`.
|
|
218
277
|
doublings -= 1;
|
|
219
278
|
}
|
|
220
|
-
|
|
221
|
-
ceiling_ns
|
|
222
|
-
} else {
|
|
223
|
-
delay
|
|
224
|
-
}
|
|
279
|
+
delay.min(ceiling)
|
|
225
280
|
}
|
|
226
281
|
|
|
227
|
-
/// The children a failure at `index` takes with it under this strategy
|
|
282
|
+
/// The children a failure at `index` takes with it under this strategy,
|
|
283
|
+
/// or `None` when the supervisor owns no child at `index`. A pure query.
|
|
228
284
|
#[must_use]
|
|
229
|
-
pub const fn span(&self, index: usize) -> Span {
|
|
285
|
+
pub const fn span(&self, index: usize) -> Option<Span> {
|
|
286
|
+
if index >= CHILDREN {
|
|
287
|
+
return None;
|
|
288
|
+
}
|
|
289
|
+
// `index < CHILDREN`, so `CHILDREN` is at least one.
|
|
290
|
+
let last = CHILDREN - 1;
|
|
230
291
|
match self.strategy {
|
|
231
|
-
Strategy::OneForOne => Span::
|
|
232
|
-
Strategy::OneForAll => Span
|
|
233
|
-
|
|
234
|
-
last: CHILDREN - 1,
|
|
235
|
-
},
|
|
236
|
-
Strategy::RestForOne => Span {
|
|
237
|
-
first: index,
|
|
238
|
-
last: CHILDREN - 1,
|
|
239
|
-
},
|
|
292
|
+
Strategy::OneForOne => Span::new(index, index),
|
|
293
|
+
Strategy::OneForAll => Span::new(0, last),
|
|
294
|
+
Strategy::RestForOne => Span::new(index, last),
|
|
240
295
|
}
|
|
241
296
|
}
|
|
242
297
|
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
298
|
+
/// Insert `at` in order. A full history lets its oldest instant go; an
|
|
299
|
+
/// instant older than everything a full history holds is not kept, and
|
|
300
|
+
/// the answer is `false` with nothing changed.
|
|
301
|
+
///
|
|
302
|
+
/// Both shifts carry `at` through the entries it displaces, so no index
|
|
303
|
+
/// is ever taken: `slot <= recorded`, as a partition point of what is
|
|
304
|
+
/// held.
|
|
305
|
+
fn record(&mut self, at: Duration) -> bool {
|
|
306
|
+
let slot = self.held().partition_point(|&held| held <= at);
|
|
307
|
+
let mut carry = at;
|
|
248
308
|
if self.recorded < HISTORY {
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
309
|
+
// `at` lands at `slot`; every entry from there up to `recorded`
|
|
310
|
+
// moves one right, into the first free entry. `recorded + 1`
|
|
311
|
+
// cannot overflow below `HISTORY`.
|
|
312
|
+
for held in self.history.iter_mut().take(self.recorded + 1).skip(slot) {
|
|
313
|
+
core::mem::swap(held, &mut carry);
|
|
253
314
|
}
|
|
254
|
-
self.history[slot] = at_ns;
|
|
255
315
|
self.recorded += 1;
|
|
256
|
-
|
|
257
|
-
let mut index = 1;
|
|
258
|
-
while index < slot {
|
|
259
|
-
self.history[index - 1] = self.history[index];
|
|
260
|
-
index += 1;
|
|
261
|
-
}
|
|
262
|
-
self.history[slot - 1] = at_ns;
|
|
316
|
+
return true;
|
|
263
317
|
}
|
|
318
|
+
if slot == 0 {
|
|
319
|
+
return false;
|
|
320
|
+
}
|
|
321
|
+
// Full: `at` lands at `slot - 1`, every entry below moves one left
|
|
322
|
+
// and the oldest falls off the front with the carry.
|
|
323
|
+
for held in self.history.iter_mut().take(slot).rev() {
|
|
324
|
+
core::mem::swap(held, &mut carry);
|
|
325
|
+
}
|
|
326
|
+
true
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/// The instants held, oldest first.
|
|
330
|
+
fn held(&self) -> &[Duration] {
|
|
331
|
+
let Some(held) = self.history.get(..self.recorded) else {
|
|
332
|
+
// Only `record` raises `recorded`, and only below `HISTORY`.
|
|
333
|
+
debug_assert!(false, "recorded never passes HISTORY");
|
|
334
|
+
// Past `HISTORY`, the whole history is what is held.
|
|
335
|
+
return &self.history;
|
|
336
|
+
};
|
|
337
|
+
held
|
|
264
338
|
}
|
|
265
339
|
|
|
266
340
|
/// Decide what to do about the child at `index`.
|
|
267
341
|
///
|
|
268
|
-
/// One restart is recorded per failure handled, not per child restarted
|
|
269
|
-
|
|
342
|
+
/// One restart is recorded per failure handled, not per child restarted,
|
|
343
|
+
/// and only by the answers that depend on `now`:
|
|
344
|
+
/// [`Decision::Restart`] and [`Decision::Escalate`].
|
|
345
|
+
/// [`Decision::Ignore`] and [`Decision::AlreadyEscalated`] hold whatever the
|
|
346
|
+
/// time and record nothing.
|
|
347
|
+
///
|
|
348
|
+
/// # Errors
|
|
349
|
+
///
|
|
350
|
+
/// [`Refusal::UnknownChild`] when `index` is not below `CHILDREN`, on a
|
|
351
|
+
/// terminated supervisor too. Nothing changed.
|
|
352
|
+
pub fn stopped(
|
|
270
353
|
&mut self,
|
|
271
354
|
index: usize,
|
|
272
355
|
policy: Policy,
|
|
273
356
|
exit: Exit,
|
|
274
|
-
|
|
275
|
-
) -> Decision {
|
|
357
|
+
now: Duration,
|
|
358
|
+
) -> Result<Decision, Refusal> {
|
|
359
|
+
let Some(span) = self.span(index) else {
|
|
360
|
+
return Err(Refusal::UnknownChild);
|
|
361
|
+
};
|
|
276
362
|
if self.terminated {
|
|
277
|
-
return Decision::
|
|
363
|
+
return Ok(Decision::AlreadyEscalated);
|
|
278
364
|
}
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
Policy::
|
|
282
|
-
Policy::
|
|
283
|
-
|
|
365
|
+
match (policy, exit) {
|
|
366
|
+
(Policy::Temporary, Exit::Normal | Exit::Abnormal)
|
|
367
|
+
| (Policy::Transient, Exit::Normal) => return Ok(Decision::Ignore),
|
|
368
|
+
(Policy::Permanent, Exit::Normal | Exit::Abnormal)
|
|
369
|
+
| (Policy::Transient, Exit::Abnormal) => {}
|
|
284
370
|
}
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
371
|
+
if !self.record(now) {
|
|
372
|
+
// Only a full history of newer restarts leaves `now` out, and all
|
|
373
|
+
// of those are inside the window, past what the intensity
|
|
374
|
+
// tolerates: the answer below escalates either way.
|
|
375
|
+
debug_assert!(
|
|
376
|
+
self.recent_restarts(now) > self.intensity.restarts(),
|
|
377
|
+
"a restart left out of the history is one the intensity already exceeds"
|
|
378
|
+
);
|
|
379
|
+
}
|
|
380
|
+
if self.recent_restarts(now) > self.intensity.restarts() {
|
|
288
381
|
self.terminated = true;
|
|
289
|
-
return Decision::Escalate;
|
|
382
|
+
return Ok(Decision::Escalate);
|
|
290
383
|
}
|
|
384
|
+
Ok(Decision::Restart(span))
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/// How many times `at` appears in the sorted `held`.
|
|
389
|
+
fn occurrences(held: &[Duration], at: Duration) -> usize {
|
|
390
|
+
// `held` is sorted, so the instants below `at` end no later than the
|
|
391
|
+
// instants up to `at`: the difference never saturates.
|
|
392
|
+
let after = held.partition_point(|&instant| instant <= at);
|
|
393
|
+
after.saturating_sub(held.partition_point(|&instant| instant < at))
|
|
394
|
+
}
|
|
291
395
|
|
|
292
|
-
|
|
396
|
+
/// How many entries of the sorted `after` are not matched by one in the
|
|
397
|
+
/// sorted `before`: the multiset difference, so a full history that swapped
|
|
398
|
+
/// an instant for an equal one added nothing.
|
|
399
|
+
fn added(before: &[Duration], after: &[Duration]) -> usize {
|
|
400
|
+
let mut held = before.iter().peekable();
|
|
401
|
+
let mut added = 0;
|
|
402
|
+
for at in after {
|
|
403
|
+
while held.next_if(|&instant| instant < at).is_some() {}
|
|
404
|
+
if held.next_if_eq(&at).is_none() {
|
|
405
|
+
// At most `after.len()`.
|
|
406
|
+
added += 1;
|
|
407
|
+
}
|
|
293
408
|
}
|
|
409
|
+
added
|
|
294
410
|
}
|
|
295
411
|
|
|
296
412
|
#[cfg(test)]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "supervisor_machines_native"
|
|
3
|
-
version = "0.1.
|
|
3
|
+
version = "0.1.6"
|
|
4
4
|
edition.workspace = true
|
|
5
5
|
rust-version.workspace = true
|
|
6
6
|
authors = ["Abdelkader Boudih <oss@seuros.com>"]
|
|
@@ -15,17 +15,13 @@ crate-type = ["cdylib"]
|
|
|
15
15
|
workspace = true
|
|
16
16
|
|
|
17
17
|
[dependencies]
|
|
18
|
-
supervisor-machines = { workspace = true
|
|
19
|
-
magnus = { version = "
|
|
18
|
+
supervisor-machines = { workspace = true }
|
|
19
|
+
magnus = { version = "0.9.3", default-features = false, features = ["io"] }
|
|
20
|
+
matryona-machines = { workspace = true, features = ["ruby"] }
|
|
20
21
|
# Nothing here calls rb-sys directly: this is the compiled fallback that
|
|
21
22
|
# implements the stable API for a Ruby rb-sys has no pre-generated bindings
|
|
22
23
|
# for, such as a preview. magnus asks for "stable-api" without it, and
|
|
23
24
|
# default-features = false drops the default that would have supplied it, so
|
|
24
25
|
# removing this line breaks the build on every future Ruby.
|
|
25
|
-
|
|
26
|
-
# implements the stable API for a Ruby rb-sys ships no pre-generated bindings
|
|
27
|
-
# for, such as a preview. magnus asks for "stable-api" without it, and its
|
|
28
|
-
# default-features = false drops the default that would have supplied it, so
|
|
29
|
-
# dropping this line breaks the build on every Ruby newer than rb-sys.
|
|
30
|
-
rb-sys = { version = ">= 0.9.130", default-features = false, features = ["stable-api-compiled-fallback"] }
|
|
26
|
+
rb-sys = { version = "0.9.130", default-features = false, features = ["stable-api-compiled-fallback"] }
|
|
31
27
|
|