supervisor_machines 0.1.1-aarch64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +28 -0
- data/LICENSE.txt +21 -0
- data/README.md +140 -0
- data/ext/supervisor_machines_native/extconf.rb +3 -0
- data/lib/supervisor_machines/errors.rb +9 -0
- data/lib/supervisor_machines/supervisor.rb +320 -0
- data/lib/supervisor_machines/version.rb +5 -0
- data/lib/supervisor_machines.rb +14 -0
- data/lib/supervisor_machines_native/supervisor_machines_native.so +0 -0
- data/sig/supervisor_machines.rbs +173 -0
- metadata +94 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 9cded078c2480916afef6189cdecd25dacfae8c73c8870ad0a387a9a2f202ec5
|
|
4
|
+
data.tar.gz: af96ba29d46fe254aae7da6678bb0f6bbe137f39dbba6bb1c3a1f51a46a9622d
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: fd7d57fec08de5bd2246982faa5e64de8580162a918ba8a3cc10e2d6ffb1d5e28789022ce81d9163d8a625172b5509fb5dcb2cc87516aed84e79f2fcbe751481
|
|
7
|
+
data.tar.gz: eed19f05285a49ae7f57d120937b1e8b6ad11bb782ecbbfd2ec3cb3fa78ba30602fa73105b9de031436d4074714d372dca35ebabd1a7a03e327959a0b5cc3e12
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## [0.1.1](https://github.com/seuros/supervisor_machines/compare/supervisor_machines-v0.1.0...supervisor_machines/v0.1.1) (2026-11-01)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* OTP-style supervision decisions ([013b054](https://github.com/seuros/supervisor_machines/commit/013b0543abb42eff100032c286642e9c26f5a907))
|
|
9
|
+
* pass a rubygems OTP through the environment ([d6b00e6](https://github.com/seuros/supervisor_machines/commit/d6b00e6f10606b4d8584d4f58999a184d2f749df))
|
|
10
|
+
* supervise threads over the decision core ([c423f63](https://github.com/seuros/supervisor_machines/commit/c423f63ceef340edfaef7eb2774eb353046b7556))
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
### Bug Fixes
|
|
14
|
+
|
|
15
|
+
* a source install can build, and a failed gem push fails the release ([5907461](https://github.com/seuros/supervisor_machines/commit/5907461674b3fc53bcbc6c077b39d6b61d959eaf))
|
|
16
|
+
* a started tree takes on no more children, and an unstarted one has no restarts ([49baae8](https://github.com/seuros/supervisor_machines/commit/49baae849adedfea8fc96ab18c8989e77e57edf7))
|
|
17
|
+
* a stop no longer waits out the restart backoff ([5de0f95](https://github.com/seuros/supervisor_machines/commit/5de0f95db1f15ceb2a4eb8fb8114d7ee48a5da8b))
|
|
18
|
+
* a stopped tree does not keep announcing its shutdown ([44f9e78](https://github.com/seuros/supervisor_machines/commit/44f9e78f8c5279be42fb29b10ef40ef8778b099b))
|
|
19
|
+
* an in-family floor, not a pessimistic pin ([7113e08](https://github.com/seuros/supervisor_machines/commit/7113e087b649320fe1d0b1dba1c2dabb268be175))
|
|
20
|
+
* bound the child index ([3761aab](https://github.com/seuros/supervisor_machines/commit/3761aab987111b4570aeaefc171175552d0c7948))
|
|
21
|
+
* call starters and the backoff outside the supervisor's lock ([b656111](https://github.com/seuros/supervisor_machines/commit/b656111a32fe5623a44b1dd7f665d708839b17b5))
|
|
22
|
+
* close six supervision lifecycle holes ([50d330f](https://github.com/seuros/supervisor_machines/commit/50d330f8bac893b75b8616b284bd0e1e1b006e99))
|
|
23
|
+
* collect the built gems whatever depth the download puts them at ([961776d](https://github.com/seuros/supervisor_machines/commit/961776db5edbee33b2fbe87494e6c6ac7ee0d3f7))
|
|
24
|
+
* give the tree lifecycle states so a failed start leaves nothing running ([dc7deba](https://github.com/seuros/supervisor_machines/commit/dc7debaff42f2440c80c6f43f057660c0a5c79a0))
|
|
25
|
+
* refuse a fractional restart intensity instead of tightening it quietly ([ea34650](https://github.com/seuros/supervisor_machines/commit/ea34650c0480f12e824ec488daa911daeda30ccf))
|
|
26
|
+
* ship the core's test files so the packaged crate builds ([4af208e](https://github.com/seuros/supervisor_machines/commit/4af208efd80a9e48275aedfe8e2137155acd59a0))
|
|
27
|
+
* the release tag carries the gem name, and the gate uses the aggregate output ([49982d6](https://github.com/seuros/supervisor_machines/commit/49982d60eed580789d6d52502c0f275576139df3))
|
|
28
|
+
* the tree reads the clock before taking its own lock ([6dd61ef](https://github.com/seuros/supervisor_machines/commit/6dd61efced111ae04b01ffdd7dfb4b559b0fd26e))
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Abdelkader Boudih
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# SupervisorMachines 🔁
|
|
2
|
+
|
|
3
|
+
> **OTP-style supervision trees over Ruby threads.** The command vessel that
|
|
4
|
+
> notices a station has gone quiet and decides who returns to duty.
|
|
5
|
+
|
|
6
|
+
A child died. Which children restart, and has that happened too often to keep
|
|
7
|
+
trying? Those answers come from the `no_std` [supervisor-machines][core] crate.
|
|
8
|
+
Everything the core refuses to touch, spawning, joining, killing, ordered
|
|
9
|
+
shutdown, is this gem. **AMS Armstrong** is the bridge around that decision.
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## 🚀 Launch Sequence
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
# Add to your ship's Gemfile
|
|
17
|
+
gem 'supervisor_machines'
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
Requires **Ruby 4.0+** and a Rust toolchain, or a platform gem: the core is a
|
|
21
|
+
native extension, not an option.
|
|
22
|
+
|
|
23
|
+
---
|
|
24
|
+
|
|
25
|
+
## 🎯 Mission Brief
|
|
26
|
+
|
|
27
|
+
```ruby
|
|
28
|
+
SM = SupervisorMachines
|
|
29
|
+
|
|
30
|
+
tree = SM::Supervisor.new(strategy: :one_for_all, restarts: 3, window: 60.0)
|
|
31
|
+
tree.supervise(:cache) { Thread.new { cache.run } }
|
|
32
|
+
tree.supervise(:warmup, policy: :transient) { Thread.new { warm_the_cache } }
|
|
33
|
+
tree.start
|
|
34
|
+
|
|
35
|
+
tree.children # => [:cache, :warmup]
|
|
36
|
+
tree.restarts # => 0
|
|
37
|
+
tree.terminated? # => false
|
|
38
|
+
|
|
39
|
+
tree.stop(grace: 5.0)
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
A starter block returns the `Thread` that runs the child, and is called again
|
|
43
|
+
for every restart. A child that stops is noticed, handed to the core, and
|
|
44
|
+
whatever the core decides is carried out.
|
|
45
|
+
|
|
46
|
+
---
|
|
47
|
+
|
|
48
|
+
## 🔄 Lifecycle
|
|
49
|
+
|
|
50
|
+
A tree registers children, runs them, and ends either stopped or failed. Both
|
|
51
|
+
endings are final: `start` raises `SupervisorMachines::Error` for a tree that
|
|
52
|
+
is already running, has been stopped, or gave up. There is one generation of
|
|
53
|
+
children per tree, and starting a second one is a bug, not a refresh.
|
|
54
|
+
`supervise` is refused the same way once `start` has fixed the shape of the
|
|
55
|
+
tree: a tree past registering cannot take on a child.
|
|
56
|
+
|
|
57
|
+
A `start` that cannot finish, because a starter raises, takes the children it
|
|
58
|
+
already brought up back down in reverse start order, leaves the tree failed,
|
|
59
|
+
and raises what the starter raised. Nothing is left running and nothing is left
|
|
60
|
+
unsupervised. `stop` then raises `Escalated`, like any other tree that gave up,
|
|
61
|
+
every time it is called, and a tree that was never started has nothing to take
|
|
62
|
+
down.
|
|
63
|
+
|
|
64
|
+
---
|
|
65
|
+
|
|
66
|
+
## 🎛️ Strategies
|
|
67
|
+
|
|
68
|
+
`:one_for_one` restarts only what died. `:one_for_all` restarts every child,
|
|
69
|
+
for a set that only works together. `:rest_for_one` restarts the child and
|
|
70
|
+
everything started after it, for a chain where later children depend on earlier
|
|
71
|
+
ones.
|
|
72
|
+
|
|
73
|
+
One restart is recorded per failure handled, not per child brought back, so the
|
|
74
|
+
strategy does not change how fast the restart intensity is spent. That is OTP's
|
|
75
|
+
accounting.
|
|
76
|
+
|
|
77
|
+
---
|
|
78
|
+
|
|
79
|
+
## 📋 Standing Orders
|
|
80
|
+
|
|
81
|
+
A child's policy is its standing order. `:permanent` always comes back, even
|
|
82
|
+
from a clean exit. `:transient` comes back
|
|
83
|
+
only from a bad one, because a clean exit means it finished. `:temporary` never
|
|
84
|
+
comes back, and never spends the restart budget.
|
|
85
|
+
|
|
86
|
+
```ruby
|
|
87
|
+
tree.supervise(:migration, policy: :temporary) { Thread.new { migrate } }
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
---
|
|
91
|
+
|
|
92
|
+
## 🔥 Restart Intensity
|
|
93
|
+
|
|
94
|
+
`restarts` within `window` seconds is all the thrashing a tree will do. Past
|
|
95
|
+
it, the tree stops its remaining children in reverse start order and raises
|
|
96
|
+
`SupervisorMachines::Escalated`, and the supervisor's own supervisor now owns
|
|
97
|
+
the problem. That decision is sticky: `terminated?` stays true and every later
|
|
98
|
+
`stop` raises again.
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## 🚫 Rules of Engagement
|
|
103
|
+
|
|
104
|
+
- The delay between a restart decision and the respawn comes from
|
|
105
|
+
[chrono_machines][chrono]. Pass your own `backoff:` to change the curve.
|
|
106
|
+
- The shutdown budget comes from [deadline_machines][deadline]: `stop(grace:)`
|
|
107
|
+
bounds the whole ordered teardown, and a child that outstays it is left
|
|
108
|
+
behind rather than waited on forever.
|
|
109
|
+
- Restart intensity, strategy spans and policy decisions come from the core
|
|
110
|
+
crate. None of it is reimplemented here.
|
|
111
|
+
|
|
112
|
+
---
|
|
113
|
+
|
|
114
|
+
## 📐 Shape
|
|
115
|
+
|
|
116
|
+
The core carries its child count in its type, so a tree is dispatched to one of
|
|
117
|
+
`SupervisorMachines::Decider::SUPPORTED_CHILDREN` monomorphised shapes: 1
|
|
118
|
+
through 16, and 32. Any other count raises `ArgumentError` from `#start`.
|
|
119
|
+
|
|
120
|
+
Threads are the only supervised unit.
|
|
121
|
+
|
|
122
|
+
---
|
|
123
|
+
|
|
124
|
+
## 📡 Credits
|
|
125
|
+
|
|
126
|
+
**AMS Armstrong**, NCC-3186, *Armstrong*-class command vessel.
|
|
127
|
+
|
|
128
|
+
Named for Joe Armstrong's OTP supervision trees, which settled that the way to
|
|
129
|
+
keep a system up is to let a part of it die and have something above it decide
|
|
130
|
+
what comes back.
|
|
131
|
+
|
|
132
|
+
---
|
|
133
|
+
|
|
134
|
+
## License
|
|
135
|
+
|
|
136
|
+
MIT.
|
|
137
|
+
|
|
138
|
+
[core]: ext/supervisor_machines_native/core/README.md
|
|
139
|
+
[chrono]: https://github.com/seuros/chrono_machines
|
|
140
|
+
[deadline]: https://github.com/seuros/deadline_machines
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SupervisorMachines
|
|
4
|
+
class Error < StandardError; end
|
|
5
|
+
|
|
6
|
+
# Raised once a supervisor has spent its restart intensity. Its children are
|
|
7
|
+
# down and the decision now belongs to whatever supervises the supervisor.
|
|
8
|
+
class Escalated < Error; end
|
|
9
|
+
end
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SupervisorMachines
|
|
4
|
+
# A supervision tree over threads.
|
|
5
|
+
#
|
|
6
|
+
# Which children a failure takes down, and whether restarting is still worth
|
|
7
|
+
# trying, belongs entirely to {Decider}. This class owns the lifecycle:
|
|
8
|
+
# spawning children, noticing when one stops, waiting out the backoff
|
|
9
|
+
# +chrono_machines+ hands it, and taking the tree down in reverse start order
|
|
10
|
+
# inside a +deadline_machines+ budget.
|
|
11
|
+
#
|
|
12
|
+
# A tree is +registering+ until {#start} fixes its shape, +running+ while it
|
|
13
|
+
# watches its children, and ends either +stopped+ or +failed+. Both endings
|
|
14
|
+
# are final: {#start} refuses a tree that is not registering, and a +failed+
|
|
15
|
+
# tree answers every {#stop} with {Escalated}.
|
|
16
|
+
#
|
|
17
|
+
# tree = SupervisorMachines::Supervisor.new(strategy: :one_for_all, restarts: 3, window: 60.0)
|
|
18
|
+
# tree.supervise(:cache) { Thread.new { cache.run } }
|
|
19
|
+
# tree.supervise(:warmup, policy: :transient) { Thread.new { warm } }
|
|
20
|
+
# tree.start
|
|
21
|
+
# tree.stop(grace: 5.0)
|
|
22
|
+
class Supervisor
|
|
23
|
+
# A registered child and whichever thread currently runs it. +epoch+ rises
|
|
24
|
+
# every time the child is taken down, so a thread the tree has already
|
|
25
|
+
# replaced neither reports its exit nor takes the child's place when its
|
|
26
|
+
# starter returns late. +done+ marks a child the tree has dropped, which no
|
|
27
|
+
# group restart brings back.
|
|
28
|
+
Child = Struct.new(:name, :policy, :starter, :thread, :epoch, :done)
|
|
29
|
+
|
|
30
|
+
# Wakes the monitor when the tree is being taken down.
|
|
31
|
+
STOP = Object.new
|
|
32
|
+
private_constant :STOP
|
|
33
|
+
|
|
34
|
+
attr_reader :strategy
|
|
35
|
+
|
|
36
|
+
# +restarts+ within +window+ seconds is the restart intensity; past it the
|
|
37
|
+
# tree escalates. +backoff+ is asked for the delay between a restart
|
|
38
|
+
# decision and the respawn; its attempt budget has to be one wider than the
|
|
39
|
+
# intensity, because the core escalates one restart after the last one it
|
|
40
|
+
# tolerates.
|
|
41
|
+
def initialize(strategy: :one_for_one, restarts: DEFAULT_RESTARTS,
|
|
42
|
+
window: DEFAULT_WINDOW_NS / NS_PER_SECOND.to_f, backoff: nil)
|
|
43
|
+
raise ArgumentError, "unknown strategy: #{strategy.inspect}" unless STRATEGIES.include?(strategy)
|
|
44
|
+
raise ArgumentError, "at most #{Decider::MAX_RESTARTS} restarts" if restarts > Decider::MAX_RESTARTS
|
|
45
|
+
|
|
46
|
+
@strategy = strategy
|
|
47
|
+
@restarts = restarts
|
|
48
|
+
@window = window
|
|
49
|
+
@backoff = backoff || ChronoMachines::Executor.new(max_attempts: restarts + 1)
|
|
50
|
+
@slots = []
|
|
51
|
+
@events = Thread::Queue.new
|
|
52
|
+
@mutex = Thread::Mutex.new
|
|
53
|
+
@halt = Thread::ConditionVariable.new
|
|
54
|
+
@state = :registering
|
|
55
|
+
@failed = nil
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Registers a child under +name+. The block is its starter: it returns the
|
|
59
|
+
# +Thread+ that runs the child, and is called again for every restart.
|
|
60
|
+
def supervise(name, policy: :permanent, &starter)
|
|
61
|
+
raise ArgumentError, "unknown policy: #{policy.inspect}" unless POLICIES.include?(policy)
|
|
62
|
+
|
|
63
|
+
@mutex.synchronize do
|
|
64
|
+
raise Error, "a #{@state} supervisor cannot take on a child" unless @state == :registering
|
|
65
|
+
|
|
66
|
+
@slots << Child.new(name, policy, starter, nil, 0, false)
|
|
67
|
+
end
|
|
68
|
+
self
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Fixes the shape of the tree, starts every child in registration order and
|
|
72
|
+
# begins watching them. A starter that will not run takes the children
|
|
73
|
+
# already up back down, leaves the tree failed, and raises what the starter
|
|
74
|
+
# raised.
|
|
75
|
+
#
|
|
76
|
+
# Raises {Error} for a tree that is past registering.
|
|
77
|
+
def start
|
|
78
|
+
@mutex.synchronize do
|
|
79
|
+
raise Error, "a #{@state} supervisor cannot be started" unless @state == :registering
|
|
80
|
+
|
|
81
|
+
@decider = Decider.new(children: @slots.size, strategy: @strategy,
|
|
82
|
+
restarts: @restarts, window_ns: to_ns(@window))
|
|
83
|
+
@state = :running
|
|
84
|
+
@monitor = Thread.new { monitor }
|
|
85
|
+
end
|
|
86
|
+
failure = launch
|
|
87
|
+
return self unless failure
|
|
88
|
+
|
|
89
|
+
rollback
|
|
90
|
+
raise failure
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Child names, in start order.
|
|
94
|
+
def children
|
|
95
|
+
@mutex.synchronize { @slots.map(&:name) }
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Restarts still inside the intensity window.
|
|
99
|
+
def restarts
|
|
100
|
+
now = monotonic_ns
|
|
101
|
+
@mutex.synchronize { @decider ? @decider.recent_restarts(now) : 0 }
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# True once the tree has escalated, and true from then on.
|
|
105
|
+
def terminated?
|
|
106
|
+
@mutex.synchronize { @state == :failed }
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# Takes every running child down in reverse start order, giving the whole
|
|
110
|
+
# shutdown +grace+ seconds, or all the time it needs when +grace+ is nil.
|
|
111
|
+
# A tree still registering has nothing to take down and stays registering.
|
|
112
|
+
#
|
|
113
|
+
# Raises {Escalated} for a tree that gave up, every time it is called.
|
|
114
|
+
def stop(grace: nil)
|
|
115
|
+
entered = @mutex.synchronize do
|
|
116
|
+
state = @state
|
|
117
|
+
@state = :stopped if state == :running
|
|
118
|
+
@halt.broadcast
|
|
119
|
+
state
|
|
120
|
+
end
|
|
121
|
+
return self if entered == :registering
|
|
122
|
+
|
|
123
|
+
@events.push(STOP) if entered == :running
|
|
124
|
+
failures = nil
|
|
125
|
+
DeadlineMachines.with_deadline(grace) do |deadline|
|
|
126
|
+
failures = shutdown
|
|
127
|
+
@monitor.kill unless @monitor.join(deadline.remaining)
|
|
128
|
+
end
|
|
129
|
+
raise Escalated, escalation if terminated?
|
|
130
|
+
raise failures.first unless failures.empty?
|
|
131
|
+
|
|
132
|
+
self
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Seconds a supervisor taking itself down waits for a killed child, since
|
|
136
|
+
# neither the monitor thread nor a failing {#start} has a caller budget to
|
|
137
|
+
# read.
|
|
138
|
+
ESCALATION_GRACE = 5.0
|
|
139
|
+
|
|
140
|
+
private
|
|
141
|
+
|
|
142
|
+
# Brings every child up in registration order, returning what a starter
|
|
143
|
+
# raised, or nil once the whole tree is up.
|
|
144
|
+
def launch
|
|
145
|
+
@slots.each_index { |index| spawn(index) }
|
|
146
|
+
nil
|
|
147
|
+
rescue Exception => e
|
|
148
|
+
@mutex.synchronize do
|
|
149
|
+
@state = :failed
|
|
150
|
+
@failed = unstarted(e)
|
|
151
|
+
end
|
|
152
|
+
e
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# Takes down whatever a start brought up before it failed, the children and
|
|
156
|
+
# the monitor watching them, inside one budget.
|
|
157
|
+
def rollback
|
|
158
|
+
DeadlineMachines.with_deadline(ESCALATION_GRACE) do |deadline|
|
|
159
|
+
shutdown
|
|
160
|
+
reap(@monitor, deadline.remaining)
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def running?
|
|
165
|
+
@state == :running
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def monitor
|
|
169
|
+
Thread.current.report_on_exception = false
|
|
170
|
+
loop do
|
|
171
|
+
event = @events.pop
|
|
172
|
+
break if event.equal?(STOP)
|
|
173
|
+
|
|
174
|
+
handle(*event)
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# One child stopped. The core says what that means; this carries it out.
|
|
179
|
+
def handle(index, epoch, ending)
|
|
180
|
+
now = monotonic_ns
|
|
181
|
+
decision = @mutex.synchronize do
|
|
182
|
+
child = @slots[index]
|
|
183
|
+
next unless running? && child.epoch == epoch
|
|
184
|
+
|
|
185
|
+
child.thread = nil
|
|
186
|
+
child.done = true if child.policy == :temporary
|
|
187
|
+
@decider.stopped(index: index, policy: child.policy, exit: ending, now_ns: now)
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
case decision
|
|
191
|
+
when Range then restart(decision)
|
|
192
|
+
when :escalate then escalate
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# Retires the whole span in reverse start order and waits for it to go,
|
|
197
|
+
# waits out the backoff without holding the tree, then brings the span back
|
|
198
|
+
# in index order.
|
|
199
|
+
def restart(span)
|
|
200
|
+
retiring = @mutex.synchronize { span.reverse_each.filter_map { |index| retire(@slots[index]) } }
|
|
201
|
+
DeadlineMachines.with_deadline(ESCALATION_GRACE) do |deadline|
|
|
202
|
+
retiring.each { |thread| reap(thread, deadline.remaining) }
|
|
203
|
+
end
|
|
204
|
+
return escalate(abandoned) if retiring.any?(&:alive?)
|
|
205
|
+
|
|
206
|
+
failure = respawn(span)
|
|
207
|
+
escalate(failure) if failure
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# The reason a respawn could not happen, or nil once the span is running.
|
|
211
|
+
def respawn(span)
|
|
212
|
+
delay = @backoff.next_delay(attempt: restarts)
|
|
213
|
+
@mutex.synchronize { @halt.wait(@mutex, delay) if delay && running? }
|
|
214
|
+
span.each { |index| spawn(index) }
|
|
215
|
+
nil
|
|
216
|
+
rescue Exception => e
|
|
217
|
+
unstarted(e)
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
def escalate(reason = escalation)
|
|
221
|
+
@mutex.synchronize do
|
|
222
|
+
@state = :failed
|
|
223
|
+
@failed = reason
|
|
224
|
+
end
|
|
225
|
+
shutdown(ESCALATION_GRACE)
|
|
226
|
+
raise Escalated, reason
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Reverse start order, each child killed and then waited for inside one
|
|
230
|
+
# budget shared by the whole tree. Touches no tree state, so a shutdown
|
|
231
|
+
# under way in the monitor never holds up the one in {#stop}. Returns what
|
|
232
|
+
# the children raised on their way out, having stopped all of them.
|
|
233
|
+
def shutdown(grace = DeadlineMachines.remaining)
|
|
234
|
+
failures = []
|
|
235
|
+
DeadlineMachines.with_deadline(grace) do |deadline|
|
|
236
|
+
@slots.reverse_each do |child|
|
|
237
|
+
thread = child.thread
|
|
238
|
+
next unless thread
|
|
239
|
+
|
|
240
|
+
failures << reap(thread, deadline.remaining)
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
failures.compact
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# Kills the thread and waits out the rest of the budget for it, reporting
|
|
247
|
+
# whatever it raised on the way out. A thread that outlives the budget is
|
|
248
|
+
# still alive afterwards.
|
|
249
|
+
def reap(thread, remaining)
|
|
250
|
+
thread.kill
|
|
251
|
+
thread.join(remaining)
|
|
252
|
+
nil
|
|
253
|
+
rescue Exception => e
|
|
254
|
+
e
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
# Calls the child's starter outside the lock, under the generation the tree
|
|
258
|
+
# is in when the call goes out. A child the tree has dropped, and every
|
|
259
|
+
# child of a tree that is no longer running, is left alone.
|
|
260
|
+
def spawn(index)
|
|
261
|
+
child = @slots[index]
|
|
262
|
+
generation = @mutex.synchronize { child.epoch if running? && !child.done }
|
|
263
|
+
install(index, generation, child.starter.call) if generation
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
# Hands the started thread to the child, unless the tree has moved past the
|
|
267
|
+
# generation that asked for it, in which case the thread is killed instead
|
|
268
|
+
# of left running with nothing watching it.
|
|
269
|
+
def install(index, generation, thread)
|
|
270
|
+
installed = @mutex.synchronize do
|
|
271
|
+
child = @slots[index]
|
|
272
|
+
next false unless running? && child.epoch == generation
|
|
273
|
+
|
|
274
|
+
child.thread = thread
|
|
275
|
+
watch(index, generation, thread)
|
|
276
|
+
true
|
|
277
|
+
end
|
|
278
|
+
reap(thread, ESCALATION_GRACE) unless installed
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
# Invalidates the watcher following the child and hands its thread back to
|
|
282
|
+
# be reaped outside the lock.
|
|
283
|
+
def retire(child)
|
|
284
|
+
child.epoch += 1
|
|
285
|
+
thread = child.thread
|
|
286
|
+
child.thread = nil
|
|
287
|
+
thread
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
# One thread per running child, blocked on it, reporting how it ended.
|
|
291
|
+
def watch(index, epoch, thread)
|
|
292
|
+
Thread.new do
|
|
293
|
+
thread.join
|
|
294
|
+
@events.push([index, epoch, :normal])
|
|
295
|
+
rescue Exception
|
|
296
|
+
@events.push([index, epoch, :abnormal])
|
|
297
|
+
end
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def escalation
|
|
301
|
+
@failed || "#{@strategy} supervisor escalated: more than #{@restarts} restarts in #{@window} seconds"
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
def abandoned
|
|
305
|
+
"#{@strategy} supervisor escalated: a retired child would not stop within #{ESCALATION_GRACE} seconds"
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
def unstarted(error)
|
|
309
|
+
"#{@strategy} supervisor escalated: #{error.message}"
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
def monotonic_ns
|
|
313
|
+
DeadlineMachines.clock.now_ns
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
def to_ns(seconds)
|
|
317
|
+
DeadlineMachines::Deadline.to_ns(seconds)
|
|
318
|
+
end
|
|
319
|
+
end
|
|
320
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'chrono_machines'
|
|
4
|
+
require 'deadline_machines'
|
|
5
|
+
|
|
6
|
+
require_relative 'supervisor_machines/version'
|
|
7
|
+
|
|
8
|
+
# Decider is defined by the native extension, which binds the
|
|
9
|
+
# `supervisor-machines` crate; there is no pure-Ruby implementation to fall
|
|
10
|
+
# back to.
|
|
11
|
+
require 'supervisor_machines_native/supervisor_machines_native'
|
|
12
|
+
|
|
13
|
+
require_relative 'supervisor_machines/errors'
|
|
14
|
+
require_relative 'supervisor_machines/supervisor'
|
|
Binary file
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
# OTP-style supervision over Ruby threads.
|
|
2
|
+
#
|
|
3
|
+
# `Decider` is defined by the native extension, which binds the
|
|
4
|
+
# `supervisor-machines` crate: it decides which children a failure takes down
|
|
5
|
+
# and when restarting has stopped being worth it. `Supervisor` owns everything
|
|
6
|
+
# the core refuses to touch, which is every thread.
|
|
7
|
+
module SupervisorMachines
|
|
8
|
+
VERSION: String
|
|
9
|
+
|
|
10
|
+
# Nanoseconds in a second. Seconds are what the Ruby surface takes; the core
|
|
11
|
+
# is fed nanosecond integers.
|
|
12
|
+
NS_PER_SECOND: Integer
|
|
13
|
+
|
|
14
|
+
# Restarts OTP tolerates by default before escalating.
|
|
15
|
+
DEFAULT_RESTARTS: Integer
|
|
16
|
+
|
|
17
|
+
# The window OTP measures those restarts over, five seconds.
|
|
18
|
+
DEFAULT_WINDOW_NS: Integer
|
|
19
|
+
|
|
20
|
+
# Which siblings a failure takes down with it.
|
|
21
|
+
type strategy = :one_for_one | :one_for_all | :rest_for_one
|
|
22
|
+
|
|
23
|
+
# When a child is worth restarting.
|
|
24
|
+
type policy = :permanent | :transient | :temporary
|
|
25
|
+
|
|
26
|
+
# How a child's run ended.
|
|
27
|
+
type exit_kind = :normal | :abnormal
|
|
28
|
+
|
|
29
|
+
# What the core decided: leave it stopped, restart this inclusive range of
|
|
30
|
+
# child indices, or give up.
|
|
31
|
+
type decision = :ignore | :escalate | Range[Integer]
|
|
32
|
+
|
|
33
|
+
# Every strategy, in the order the core declares them. Anything else raises
|
|
34
|
+
# `ArgumentError`.
|
|
35
|
+
STRATEGIES: Array[strategy]
|
|
36
|
+
|
|
37
|
+
# Every policy, in the order the core declares them. Anything else raises
|
|
38
|
+
# `ArgumentError`.
|
|
39
|
+
POLICIES: Array[policy]
|
|
40
|
+
|
|
41
|
+
class Error < StandardError
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Raised once a supervisor has spent its restart intensity.
|
|
45
|
+
class Escalated < Error
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# The core's decision machine. A child count and a restart history are part
|
|
49
|
+
# of its Rust type, so the count given here is dispatched to one of
|
|
50
|
+
# `SUPPORTED_CHILDREN` monomorphised shapes.
|
|
51
|
+
class Decider
|
|
52
|
+
# Restart timestamps every shape keeps.
|
|
53
|
+
HISTORY: Integer
|
|
54
|
+
|
|
55
|
+
# The child counts a decider can be built for. Anything else raises
|
|
56
|
+
# `ArgumentError`.
|
|
57
|
+
SUPPORTED_CHILDREN: Array[Integer]
|
|
58
|
+
|
|
59
|
+
def self.new: (
|
|
60
|
+
children: Integer,
|
|
61
|
+
strategy: strategy,
|
|
62
|
+
restarts: Integer,
|
|
63
|
+
window_ns: Integer
|
|
64
|
+
) -> instance
|
|
65
|
+
|
|
66
|
+
# Feeds one stopped child to the core and returns what it decided.
|
|
67
|
+
def stopped: (
|
|
68
|
+
index: Integer,
|
|
69
|
+
policy: policy,
|
|
70
|
+
exit: exit_kind,
|
|
71
|
+
now_ns: Integer
|
|
72
|
+
) -> decision
|
|
73
|
+
|
|
74
|
+
def children: () -> Integer
|
|
75
|
+
|
|
76
|
+
def strategy: () -> strategy
|
|
77
|
+
|
|
78
|
+
# True once the core has escalated, and true from then on.
|
|
79
|
+
def terminated?: () -> bool
|
|
80
|
+
|
|
81
|
+
# Restarts still inside the intensity window as of `now_ns`.
|
|
82
|
+
def recent_restarts: (Integer) -> Integer
|
|
83
|
+
|
|
84
|
+
def inspect: () -> String
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# A supervision tree over threads.
|
|
88
|
+
class Supervisor
|
|
89
|
+
# Where a tree is in its lifecycle: registering children, running them, or
|
|
90
|
+
# ended — `stopped` by its caller, `failed` by giving up.
|
|
91
|
+
type state = :registering | :running | :stopped | :failed
|
|
92
|
+
|
|
93
|
+
# Seconds a supervisor taking itself down waits for a killed child.
|
|
94
|
+
ESCALATION_GRACE: Float
|
|
95
|
+
|
|
96
|
+
# Anything that can price the wait between a restart decision and the
|
|
97
|
+
# respawn, which is `ChronoMachines::Executor` unless told otherwise.
|
|
98
|
+
interface _Backoff
|
|
99
|
+
def next_delay: (attempt: Integer) -> Numeric?
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# A registered child and whichever thread currently runs it.
|
|
103
|
+
class Child < ::Struct[untyped]
|
|
104
|
+
attr_accessor name: untyped
|
|
105
|
+
attr_accessor policy: policy
|
|
106
|
+
attr_accessor starter: ^() -> Thread
|
|
107
|
+
attr_accessor thread: Thread?
|
|
108
|
+
attr_accessor epoch: Integer
|
|
109
|
+
attr_accessor done: bool
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
attr_reader strategy: strategy
|
|
113
|
+
|
|
114
|
+
# `restarts` within `window` seconds is the restart intensity.
|
|
115
|
+
def initialize: (
|
|
116
|
+
?strategy: strategy,
|
|
117
|
+
?restarts: Integer,
|
|
118
|
+
?window: Float,
|
|
119
|
+
?backoff: _Backoff?
|
|
120
|
+
) -> void
|
|
121
|
+
|
|
122
|
+
# Registers a child under `name`. The block returns the `Thread` that runs
|
|
123
|
+
# it, and is called again for every restart.
|
|
124
|
+
def supervise: (untyped name, ?policy: policy) { () -> Thread } -> self
|
|
125
|
+
|
|
126
|
+
# Fixes the shape of the tree and starts every child in registration
|
|
127
|
+
# order. Raises `ArgumentError` for a child count the core has no shape
|
|
128
|
+
# for, `Error` for a tree that is past registering, and whatever a starter
|
|
129
|
+
# raised after taking the children already up back down.
|
|
130
|
+
def start: () -> self
|
|
131
|
+
|
|
132
|
+
# Child names, in start order.
|
|
133
|
+
def children: () -> Array[untyped]
|
|
134
|
+
|
|
135
|
+
# Restarts still inside the intensity window.
|
|
136
|
+
def restarts: () -> Integer
|
|
137
|
+
|
|
138
|
+
def terminated?: () -> bool
|
|
139
|
+
|
|
140
|
+
# Takes every running child down in reverse start order within `grace`
|
|
141
|
+
# seconds, or all the time it needs when `grace` is nil. A tree still
|
|
142
|
+
# registering has nothing to take down. Raises `Escalated` for a tree that
|
|
143
|
+
# gave up, every time it is called.
|
|
144
|
+
def stop: (?grace: Float?) -> self
|
|
145
|
+
|
|
146
|
+
private
|
|
147
|
+
|
|
148
|
+
# What a starter raised, after the children already up have been taken
|
|
149
|
+
# back down, or nil once the whole tree is up.
|
|
150
|
+
def launch: () -> Exception?
|
|
151
|
+
def rollback: () -> void
|
|
152
|
+
def running?: () -> bool
|
|
153
|
+
def monitor: () -> void
|
|
154
|
+
def handle: (Integer index, Integer epoch, exit_kind ending) -> void
|
|
155
|
+
def restart: (Range[Integer] span) -> void
|
|
156
|
+
def respawn: (Range[Integer] span) -> String?
|
|
157
|
+
def escalate: (?String reason) -> void
|
|
158
|
+
def shutdown: (?Float grace) -> Array[Exception]
|
|
159
|
+
def reap: (Thread thread, Float? remaining) -> Exception?
|
|
160
|
+
def spawn: (Integer index) -> void
|
|
161
|
+
|
|
162
|
+
# Installs a started thread under the generation that asked for it, killing
|
|
163
|
+
# it when the tree has moved past that generation.
|
|
164
|
+
def install: (Integer index, Integer generation, Thread thread) -> void
|
|
165
|
+
def retire: (Child child) -> Thread?
|
|
166
|
+
def watch: (Integer index, Integer epoch, Thread thread) -> void
|
|
167
|
+
def escalation: () -> String
|
|
168
|
+
def abandoned: () -> String
|
|
169
|
+
def unstarted: (Exception error) -> String
|
|
170
|
+
def monotonic_ns: () -> Integer
|
|
171
|
+
def to_ns: (Float seconds) -> Integer
|
|
172
|
+
end
|
|
173
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: supervisor_machines
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.1
|
|
5
|
+
platform: aarch64-linux
|
|
6
|
+
authors:
|
|
7
|
+
- Abdelkader Boudih
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2026-11-01 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: chrono_machines
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - ">="
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '0.11'
|
|
20
|
+
type: :runtime
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
- - ">="
|
|
25
|
+
- !ruby/object:Gem::Version
|
|
26
|
+
version: '0.11'
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
name: deadline_machines
|
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - ">="
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: '0.1'
|
|
34
|
+
type: :runtime
|
|
35
|
+
prerelease: false
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - ">="
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: '0.1'
|
|
41
|
+
description: 'A supervision tree that restarts what died the way OTP would: one_for_one,
|
|
42
|
+
one_for_all or rest_for_one, bounded by a restart intensity that escalates instead
|
|
43
|
+
of thrashing. The decisions come from the no_std supervisor-machines crate; Ruby
|
|
44
|
+
owns the threads, the restart backoff comes from chrono_machines and the shutdown
|
|
45
|
+
budget from deadline_machines.'
|
|
46
|
+
email:
|
|
47
|
+
- oss@seuros.com
|
|
48
|
+
executables: []
|
|
49
|
+
extensions: []
|
|
50
|
+
extra_rdoc_files: []
|
|
51
|
+
files:
|
|
52
|
+
- CHANGELOG.md
|
|
53
|
+
- LICENSE.txt
|
|
54
|
+
- README.md
|
|
55
|
+
- ext/supervisor_machines_native/extconf.rb
|
|
56
|
+
- lib/supervisor_machines.rb
|
|
57
|
+
- lib/supervisor_machines/errors.rb
|
|
58
|
+
- lib/supervisor_machines/supervisor.rb
|
|
59
|
+
- lib/supervisor_machines/version.rb
|
|
60
|
+
- lib/supervisor_machines_native/supervisor_machines_native.so
|
|
61
|
+
- sig/supervisor_machines.rbs
|
|
62
|
+
homepage: https://github.com/seuros/supervisor_machines
|
|
63
|
+
licenses:
|
|
64
|
+
- MIT
|
|
65
|
+
metadata:
|
|
66
|
+
source_code_uri: https://github.com/seuros/supervisor_machines
|
|
67
|
+
changelog_uri: https://github.com/seuros/supervisor_machines/blob/main/CHANGELOG.md
|
|
68
|
+
bug_tracker_uri: https://github.com/seuros/supervisor_machines/issues
|
|
69
|
+
rubygems_mfa_required: 'true'
|
|
70
|
+
cargo_crate_name: supervisor_machines_native
|
|
71
|
+
cargo_manifest_path: ext/supervisor_machines_native/ffi/Cargo.toml
|
|
72
|
+
post_install_message:
|
|
73
|
+
rdoc_options: []
|
|
74
|
+
require_paths:
|
|
75
|
+
- lib
|
|
76
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
77
|
+
requirements:
|
|
78
|
+
- - ">="
|
|
79
|
+
- !ruby/object:Gem::Version
|
|
80
|
+
version: '4.0'
|
|
81
|
+
- - "<"
|
|
82
|
+
- !ruby/object:Gem::Version
|
|
83
|
+
version: 4.1.dev
|
|
84
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
85
|
+
requirements:
|
|
86
|
+
- - ">="
|
|
87
|
+
- !ruby/object:Gem::Version
|
|
88
|
+
version: '0'
|
|
89
|
+
requirements: []
|
|
90
|
+
rubygems_version: 3.5.23
|
|
91
|
+
signing_key:
|
|
92
|
+
specification_version: 4
|
|
93
|
+
summary: OTP-style supervision trees over Ruby threads
|
|
94
|
+
test_files: []
|