otp-rails 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +34 -0
- data/LICENSE +21 -0
- data/README.md +149 -0
- data/exe/otp-rails +4 -0
- data/lib/otp_rails/adapter.rb +21 -0
- data/lib/otp_rails/adapters/command.rb +72 -0
- data/lib/otp_rails/adapters/puma.rb +56 -0
- data/lib/otp_rails/adapters/solid_queue.rb +40 -0
- data/lib/otp_rails/adapters/supervisor_adapter.rb +95 -0
- data/lib/otp_rails/backoff.rb +18 -0
- data/lib/otp_rails/child_spec.rb +33 -0
- data/lib/otp_rails/cli.rb +33 -0
- data/lib/otp_rails/dsl.rb +85 -0
- data/lib/otp_rails/heartbeat.rb +51 -0
- data/lib/otp_rails/orphan_guard.rb +35 -0
- data/lib/otp_rails/probe.rb +44 -0
- data/lib/otp_rails/restart_intensity.rb +22 -0
- data/lib/otp_rails/socket_server.rb +77 -0
- data/lib/otp_rails/strategy.rb +18 -0
- data/lib/otp_rails/supervisor.rb +214 -0
- data/lib/otp_rails/telemetry.rb +56 -0
- data/lib/otp_rails/version.rb +4 -0
- data/lib/otp_rails.rb +31 -0
- metadata +71 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 3fafef6955d1e366ba5cd01039a7c903a17eee4342a4179e679ecccd75fbab89
|
|
4
|
+
data.tar.gz: '09f381af732986cdb5f2bea354e3b657009d342ff340bcbf95c6e7f15c9096b6'
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: f4da29b3dd961433aefde893f2eecd8165a50ca0720ca301fa71f10cab656e862a415afc86e4f1f9201f15ebd27cf49c80af3e118c730fc522fdfff5790347a7
|
|
7
|
+
data.tar.gz: 1c74adf968934165f36f379a01bb9dd8cea47b00c0c33d1c3ee8ba2b21eb829358a4e1b6b7c66362dc0c5f52e6eeabbbcbe7ff108a0761ca76df7f62e8a4e87b
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## v0.1.0 — 2026-09-12
|
|
4
|
+
|
|
5
|
+
First release: a slim, zero-runtime-dependency, OTP-style process supervisor for the
|
|
6
|
+
processes of a Rails app. The supervisor never loads Rails (DESIGN §9).
|
|
7
|
+
|
|
8
|
+
- **Supervision core** — `ChildSpec` with `permanent`/`transient`/`temporary` restart
|
|
9
|
+
semantics; `one_for_one`, `rest_for_one`, `one_for_all` strategies; restart intensity
|
|
10
|
+
with a sliding window that escalates (exit 70) when exceeded; `none`/`constant`/
|
|
11
|
+
`exponential` backoff; generation-guarded exit handling (stale exits ignored).
|
|
12
|
+
- **Shutdown correctness** (#1) — `stop_all` drains in reverse start order, waiting per
|
|
13
|
+
child (SIGTERM, then SIGKILL of the process group after `shutdown:`). Exit codes:
|
|
14
|
+
0 clean, 70 escalation, 78 config error. Orphan prevention on Linux via
|
|
15
|
+
`prctl(PR_SET_PDEATHSIG, SIGTERM)`; macOS has no equivalent (documented limitation).
|
|
16
|
+
- **Passive probes** (#2) — `probe: { tcp: PORT }` / `probe: { http: URL }` on `:command`:
|
|
17
|
+
`:starting` until the probe answers, then `:healthy`. `start_timeout` exceeded ⇒ drain ⇒
|
|
18
|
+
counts as a crash ⇒ strategy applies.
|
|
19
|
+
- **Periodic health loop** (#3) — per-child monitor at `health_interval` (default 5s);
|
|
20
|
+
`:degraded` emits telemetry only; `degraded_restart_after: N` drains after N consecutive
|
|
21
|
+
degraded reports (flows through the crash path, so intensity applies).
|
|
22
|
+
- **`:puma` adapter, beside mode** (#4) — wraps the puma master; health = HTTP `/up` probe;
|
|
23
|
+
drain = SIGTERM (Puma graceful). `opts[:port]` or a literal `port NNNN` in the config.
|
|
24
|
+
- **Active heartbeat socket** (#5) — Unix socket (default `tmp/otp-rails.sock`, mode 0600),
|
|
25
|
+
per-boot token via `OTP_RAILS_SOCK` / `OTP_RAILS_TOKEN`; NDJSON heartbeats
|
|
26
|
+
`{"id","state","ts","token","meta"}`; bad token dropped; 3 missed intervals ⇒ `:degraded`,
|
|
27
|
+
6 ⇒ `:dead` ⇒ restart; `{"cmd":"restart","id":...}` control messages. This wire protocol
|
|
28
|
+
is the contract consumed by the Elixir sidecar (`shishi-odoshi/beam`).
|
|
29
|
+
- **`:solid_queue` adapter + heartbeat helper** (#6) — wraps `bin/jobs`; health = active
|
|
30
|
+
heartbeat, not the DB table. `require "otp_rails/heartbeat"` is Rails-free and
|
|
31
|
+
self-contained; silently a no-op when unsupervised.
|
|
32
|
+
- **Nested supervisors** (#7) — `supervisor :background do ... end` creates a subtree with
|
|
33
|
+
its own strategy/intensity/backoff; subtree escalation is an ordinary child exit in the
|
|
34
|
+
parent; restarted subtrees get a fresh intensity window.
|
data/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Tim
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
# otp-rails
|
|
2
|
+
|
|
3
|
+
OTP-style supervision trees for the processes of a Rails app. A slim supervisor — it never
|
|
4
|
+
loads Rails — that starts, links, health-checks, and restarts `web`, `jobs`, `cable`, `cron`
|
|
5
|
+
with `one_for_one` / `rest_for_one` / `one_for_all` strategies, restart intensity, and backoff.
|
|
6
|
+
|
|
7
|
+
Part of the [shishi-odoshi](https://github.com/shishi-odoshi) org. Design: `docs/DESIGN.md`.
|
|
8
|
+
The Elixir sidecar speaking the same wire protocol lives at
|
|
9
|
+
[shishi-odoshi/beam](https://github.com/shishi-odoshi/beam).
|
|
10
|
+
|
|
11
|
+
## Install
|
|
12
|
+
|
|
13
|
+
```ruby
|
|
14
|
+
# Gemfile — the supervisor is its own process; a :supervisor group keeps app boot slim
|
|
15
|
+
group :supervisor do
|
|
16
|
+
gem "otp-rails"
|
|
17
|
+
end
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
```
|
|
21
|
+
$ bundle install
|
|
22
|
+
$ otp-rails version
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Zero runtime dependencies. Ruby >= 3.2.
|
|
26
|
+
|
|
27
|
+
## Quick start
|
|
28
|
+
|
|
29
|
+
```ruby
|
|
30
|
+
# config/supervisor.rb — plain Ruby, evaluated WITHOUT Rails
|
|
31
|
+
strategy :rest_for_one
|
|
32
|
+
max_restarts 5, within: 60
|
|
33
|
+
backoff :exponential, base: 1, cap: 30
|
|
34
|
+
|
|
35
|
+
child :web, adapter: :puma, port: 3000
|
|
36
|
+
child :jobs, adapter: :solid_queue, shutdown: 60
|
|
37
|
+
child :cron, adapter: :command, cmd: "bin/rails cron", restart: :transient
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
```
|
|
41
|
+
$ otp-rails check config/supervisor.rb # validate config, print the tree
|
|
42
|
+
$ otp-rails run config/supervisor.rb # supervise (Ctrl-C drains and stops)
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Config reference
|
|
46
|
+
|
|
47
|
+
Top-level directives in `config/supervisor.rb`:
|
|
48
|
+
|
|
49
|
+
| Directive | Default | Meaning |
|
|
50
|
+
|---|---|---|
|
|
51
|
+
| `strategy KIND` | `:one_for_one` | `:one_for_one` restarts only the failed child; `:rest_for_one` also restarts children declared after it; `:one_for_all` restarts every child |
|
|
52
|
+
| `max_restarts N, within: S` | `5, within: 60` | Sliding-window restart intensity; exceeding it escalates (exit 70) |
|
|
53
|
+
| `backoff KIND, **opts` | `:exponential, base: 1, cap: 30` | `:none`, `:constant`, or `:exponential` delay between restarts |
|
|
54
|
+
| `socket PATH` | `"tmp/otp-rails.sock"` | Heartbeat/control Unix socket; `socket nil` disables it |
|
|
55
|
+
| `child ID, adapter:, **opts` | — | Declares a child; declaration order is start order |
|
|
56
|
+
| `supervisor ID do ... end` | — | Nested subtree with its own strategy/intensity/backoff; subtree escalation is an ordinary child exit in the parent |
|
|
57
|
+
|
|
58
|
+
Per-child options:
|
|
59
|
+
|
|
60
|
+
| Option | Default | Meaning |
|
|
61
|
+
|---|---|---|
|
|
62
|
+
| `restart:` | `:permanent` | `:permanent` always restarts; `:transient` restarts only on non-zero exit; `:temporary` never restarts |
|
|
63
|
+
| `shutdown:` | `30` | Seconds to wait after SIGTERM before SIGKILL of the child's process group |
|
|
64
|
+
| `start_timeout:` | `30` | Seconds to reach `:healthy`; exceeding it drains the child and counts as a crash |
|
|
65
|
+
| `health_interval:` | `5` | Seconds between health checks (and the unit for heartbeat freshness) |
|
|
66
|
+
| `degraded_restart_after:` | `nil` | N consecutive `:degraded` reports ⇒ drain and restart (counts toward intensity) |
|
|
67
|
+
|
|
68
|
+
Adapters:
|
|
69
|
+
|
|
70
|
+
- **`:command`** — any command. `cmd:` (required), `env:`, `spawn_opts:`, and optional
|
|
71
|
+
`probe: { tcp: PORT }` or `probe: { http: "http://127.0.0.1:3000/up" }` — the child is
|
|
72
|
+
`:starting` until the probe answers, `:degraded` if it stops answering later.
|
|
73
|
+
- **`:puma`** (beside mode) — wraps the puma master. `config:` (default `config/puma.rb`),
|
|
74
|
+
`port:` (or a literal `port NNNN` line in the config). Health = HTTP probe of `/up`;
|
|
75
|
+
drain = SIGTERM (Puma's graceful stop).
|
|
76
|
+
- **`:solid_queue`** — wraps `bin/jobs` (`cmd:` overrides). Health = the active heartbeat
|
|
77
|
+
below, never the `solid_queue_processes` table.
|
|
78
|
+
|
|
79
|
+
## Health & heartbeats
|
|
80
|
+
|
|
81
|
+
Passive children are probed (PID, TCP, HTTP). Active children report themselves: the
|
|
82
|
+
supervisor listens on a Unix socket (mode 0600) and exports `OTP_RAILS_SOCK` /
|
|
83
|
+
`OTP_RAILS_TOKEN` to every child. Heartbeats are newline-delimited JSON:
|
|
84
|
+
|
|
85
|
+
```json
|
|
86
|
+
{"id":"jobs","state":"healthy","ts":1757700000,"token":"…","meta":{"backlog":0}}
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
A child that has heartbeated is judged by heartbeat freshness: 3 missed `health_interval`s
|
|
90
|
+
⇒ `:degraded`, 6 ⇒ `:dead` ⇒ the strategy applies. Wrong token ⇒ the line is silently
|
|
91
|
+
dropped. The same socket accepts `{"cmd":"restart","id":"jobs","token":"…"}`.
|
|
92
|
+
|
|
93
|
+
From any child process (a Rails initializer, a Solid Queue hook — no Rails required):
|
|
94
|
+
|
|
95
|
+
```ruby
|
|
96
|
+
require "otp_rails/heartbeat" # loads nothing else
|
|
97
|
+
OtpRails::Heartbeat.start(id: "jobs") # no-op when running unsupervised
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## Telemetry reference
|
|
101
|
+
|
|
102
|
+
Event names follow `[:otp_rails, :subject, :action]`, mirroring Elixir `:telemetry` so the
|
|
103
|
+
sidecar can forward them unchanged. This list is a published contract:
|
|
104
|
+
|
|
105
|
+
```
|
|
106
|
+
[:otp_rails, :supervisor, :start] metadata: {strategy, children}
|
|
107
|
+
[:otp_rails, :supervisor, :stop]
|
|
108
|
+
[:otp_rails, :supervisor, :escalate] measurements: {restarts} metadata: {within}
|
|
109
|
+
[:otp_rails, :child, :spawn] metadata: {id, adapter, pid}
|
|
110
|
+
[:otp_rails, :child, :healthy] metadata: {id}
|
|
111
|
+
[:otp_rails, :child, :degraded] measurements: {consecutive} metadata: {id}
|
|
112
|
+
[:otp_rails, :child, :exit] measurements: {exit_code, uptime_ms} metadata: {id}
|
|
113
|
+
[:otp_rails, :child, :restart] measurements: {backoff_ms} metadata: {id, attempt, strategy}
|
|
114
|
+
[:otp_rails, :child, :drain] metadata: {id}
|
|
115
|
+
[:otp_rails, :child, :kill] metadata: {id} (drain timed out)
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
Subscribe in-process with `OtpRails::Telemetry.subscribe { |event| ... }`; a logger
|
|
119
|
+
subscriber and a JSON-lines exporter ship by default (`Telemetry::Subscribers`).
|
|
120
|
+
|
|
121
|
+
## Exit codes
|
|
122
|
+
|
|
123
|
+
| Code | Meaning |
|
|
124
|
+
|---|---|
|
|
125
|
+
| `0` | Clean stop (SIGINT/SIGTERM, all children drained) |
|
|
126
|
+
| `70` | Restart intensity exceeded — the supervisor escalated (EX_SOFTWARE). The platform (Kamal/K8s/Heroku/launchd) is the final supervisor and should restart on non-zero |
|
|
127
|
+
| `78` | Configuration error (EX_CONFIG) |
|
|
128
|
+
|
|
129
|
+
## Shutdown semantics
|
|
130
|
+
|
|
131
|
+
- `stop_all` drains children in **reverse start order**, waiting for each child to exit
|
|
132
|
+
(up to its `shutdown:` timeout, then SIGKILL of its process group) before draining the next.
|
|
133
|
+
- Orphan prevention: on Linux, children are armed with `prctl(PR_SET_PDEATHSIG, SIGTERM)`
|
|
134
|
+
between fork and exec, so they receive SIGTERM even if the supervisor is SIGKILLed.
|
|
135
|
+
**macOS/BSD limitation:** no parent-death signal exists there; a SIGKILLed supervisor
|
|
136
|
+
orphans its children to launchd/init and they keep running. Mitigation: the platform
|
|
137
|
+
restarts the supervisor; each child runs in its own session/process group, so stale
|
|
138
|
+
orphans are findable and killable by pgid.
|
|
139
|
+
- macOS also caps Unix socket paths at ~104 bytes — keep `socket PATH` short.
|
|
140
|
+
|
|
141
|
+
## Development
|
|
142
|
+
|
|
143
|
+
```
|
|
144
|
+
bundle exec rake test # full suite (real processes, no mocks)
|
|
145
|
+
ruby -Ilib -Itest test/supervisor_kill_test.rb
|
|
146
|
+
exe/otp-rails check examples/supervisor.rb
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
`docs/PLAN.md` is the backlog; `docs/DESIGN.md` is the frozen design.
|
data/exe/otp-rails
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
module OtpRails
|
|
3
|
+
# DESIGN §4. Exactly four operations (+ kill as last resort). Adapters never talk to each other.
|
|
4
|
+
class Adapter
|
|
5
|
+
REGISTRY = {}
|
|
6
|
+
|
|
7
|
+
def self.register(name, klass) = REGISTRY[name] = klass
|
|
8
|
+
def self.lookup(name) = REGISTRY.fetch(name) { raise ConfigError, "no adapter registered as #{name.inspect}" }
|
|
9
|
+
|
|
10
|
+
# @return [Object] opaque handle
|
|
11
|
+
def spawn(spec) = raise NotImplementedError
|
|
12
|
+
# Register a one-shot callback: block.call(exit_status)
|
|
13
|
+
def link(handle, &on_exit) = raise NotImplementedError
|
|
14
|
+
# @return [:starting, :healthy, :degraded, :dead]
|
|
15
|
+
def health(handle) = raise NotImplementedError
|
|
16
|
+
# Stop accepting work, finish in-flight, exit. Return true if exited within timeout.
|
|
17
|
+
def drain(handle, timeout:) = raise NotImplementedError
|
|
18
|
+
# Last resort after drain times out.
|
|
19
|
+
def kill(handle) = raise NotImplementedError
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
module OtpRails
|
|
3
|
+
module Adapters
|
|
4
|
+
# DESIGN §4.1 :command — arbitrary command. Health = PID alive (+ optional probe, TODO).
|
|
5
|
+
class Command < Adapter
|
|
6
|
+
Handle = Struct.new(:pid, :spec, :started_at, :exit_status, :waiter, :healthy_once)
|
|
7
|
+
|
|
8
|
+
def spawn(spec)
|
|
9
|
+
cmd = spec.opts.fetch(:cmd) { raise ConfigError, "#{spec.id}: :command adapter requires cmd:" }
|
|
10
|
+
env = spec.opts.fetch(:env, {}).transform_keys(&:to_s)
|
|
11
|
+
spawn_opts = spec.opts.fetch(:spawn_opts, {})
|
|
12
|
+
pid =
|
|
13
|
+
if OrphanGuard.available?
|
|
14
|
+
parent = Process.pid
|
|
15
|
+
Process.fork do
|
|
16
|
+
Process.setsid # own session ⇒ own pgroup, same as pgroup: true below
|
|
17
|
+
OrphanGuard.arm!(parent)
|
|
18
|
+
begin
|
|
19
|
+
Process.exec(env, cmd, **spawn_opts)
|
|
20
|
+
rescue SystemCallError
|
|
21
|
+
Process.exit!(127)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
else
|
|
25
|
+
Process.spawn(env, cmd, pgroup: true, **spawn_opts)
|
|
26
|
+
end
|
|
27
|
+
Handle.new(pid, spec, Process.clock_gettime(Process::CLOCK_MONOTONIC), nil, nil)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def link(handle, &on_exit)
|
|
31
|
+
handle.waiter = Thread.new do
|
|
32
|
+
_, status = Process.wait2(handle.pid)
|
|
33
|
+
handle.exit_status = status
|
|
34
|
+
on_exit.call(status)
|
|
35
|
+
rescue Errno::ECHILD
|
|
36
|
+
on_exit.call(nil)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def health(handle)
|
|
41
|
+
return :dead if handle.exit_status
|
|
42
|
+
Process.kill(0, handle.pid)
|
|
43
|
+
if Probe.answering?(handle.spec)
|
|
44
|
+
handle.healthy_once = true
|
|
45
|
+
:healthy
|
|
46
|
+
elsif handle.healthy_once
|
|
47
|
+
:degraded # was healthy, probe stopped answering, PID still alive (§5)
|
|
48
|
+
else
|
|
49
|
+
:starting
|
|
50
|
+
end
|
|
51
|
+
rescue Errno::ESRCH
|
|
52
|
+
:dead
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def drain(handle, timeout:)
|
|
56
|
+
Process.kill("TERM", handle.pid)
|
|
57
|
+
handle.waiter&.join(timeout)
|
|
58
|
+
!handle.exit_status.nil?
|
|
59
|
+
rescue Errno::ESRCH
|
|
60
|
+
true
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def kill(handle)
|
|
64
|
+
Process.kill("KILL", -handle.pid) # whole process group
|
|
65
|
+
handle.waiter&.join(2)
|
|
66
|
+
rescue Errno::ESRCH
|
|
67
|
+
nil
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
Adapter.register(:command, Adapters::Command)
|
|
72
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
module OtpRails
|
|
3
|
+
module Adapters
|
|
4
|
+
# DESIGN §4.1 / §4.2 step 1 — :puma in "beside" mode: one opaque child
|
|
5
|
+
# wrapping the puma master. Health = HTTP probe of /up (the Rails 7.1+
|
|
6
|
+
# default endpoint) on the bound port; drain = SIGTERM, which is Puma's
|
|
7
|
+
# graceful shutdown — exactly Command#drain, so it is inherited unchanged,
|
|
8
|
+
# as are link and kill.
|
|
9
|
+
#
|
|
10
|
+
# The whole adapter is Command with a derived spec: spawn builds the puma
|
|
11
|
+
# command line and injects a probe: { http: ".../up" } opt, so health()
|
|
12
|
+
# rides the PLAN 1.2 Probe path with no new code — :starting until /up
|
|
13
|
+
# answers 2xx, then :healthy; :dead when the PID goes.
|
|
14
|
+
#
|
|
15
|
+
# opts:
|
|
16
|
+
# config: puma config file path (default "config/puma.rb")
|
|
17
|
+
# port: the bound port. When absent, a literal `port NNNN` line is
|
|
18
|
+
# parsed from the config file; neither ⇒ ConfigError.
|
|
19
|
+
# env:, spawn_opts: passed through to Command verbatim.
|
|
20
|
+
# (cmd: and probe: are owned by this adapter and overwritten.)
|
|
21
|
+
class Puma < Command
|
|
22
|
+
DEFAULT_CONFIG = "config/puma.rb"
|
|
23
|
+
HEALTH_PATH = "/up"
|
|
24
|
+
|
|
25
|
+
def spawn(spec)
|
|
26
|
+
super(command_spec(spec))
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def command_spec(spec)
|
|
32
|
+
config = spec.opts.fetch(:config, DEFAULT_CONFIG)
|
|
33
|
+
port = resolve_port(spec, config)
|
|
34
|
+
ChildSpec.new(
|
|
35
|
+
id: spec.id, adapter: spec.adapter, restart: spec.restart,
|
|
36
|
+
shutdown: spec.shutdown, start_timeout: spec.start_timeout,
|
|
37
|
+
opts: spec.opts.merge(
|
|
38
|
+
cmd: "bundle exec puma -C #{config}",
|
|
39
|
+
probe: { http: "http://127.0.0.1:#{port}#{HEALTH_PATH}" }
|
|
40
|
+
)
|
|
41
|
+
)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# opts[:port] wins; else a literal `port NNNN` line in the config file.
|
|
45
|
+
# Anything fancier (ENV/ERB in the config) must pass opts[:port].
|
|
46
|
+
def resolve_port(spec, config)
|
|
47
|
+
return Integer(spec.opts[:port]) if spec.opts[:port]
|
|
48
|
+
literal = File.file?(config) && File.read(config)[/^\s*port\s+(\d+)/, 1]
|
|
49
|
+
return Integer(literal) if literal
|
|
50
|
+
raise ConfigError,
|
|
51
|
+
"#{spec.id}: :puma needs opts[:port] or a literal `port NNNN` line in #{config}"
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
Adapter.register(:puma, Adapters::Puma)
|
|
56
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
module OtpRails
|
|
3
|
+
module Adapters
|
|
4
|
+
# DESIGN §4.1 / §9 — :solid_queue wraps the Solid Queue supervisor
|
|
5
|
+
# (`bin/jobs`). Health is the ACTIVE heartbeat (§5), NOT the
|
|
6
|
+
# solid_queue_processes table: the app sends heartbeats via the tiny
|
|
7
|
+
# Rails-free hook, e.g. from an initializer:
|
|
8
|
+
#
|
|
9
|
+
# require "otp_rails/heartbeat"
|
|
10
|
+
# OtpRails::Heartbeat.start(id: "jobs")
|
|
11
|
+
#
|
|
12
|
+
# Once the first heartbeat arrives the supervisor judges the child by
|
|
13
|
+
# heartbeat freshness (3 missed intervals ⇒ :degraded, 6 ⇒ :dead);
|
|
14
|
+
# until then it falls back to this adapter's passive PID health,
|
|
15
|
+
# inherited from Command — as are link, drain (SIGTERM, which Solid
|
|
16
|
+
# Queue handles gracefully), and kill.
|
|
17
|
+
#
|
|
18
|
+
# opts:
|
|
19
|
+
# cmd: the jobs command (default "bin/jobs")
|
|
20
|
+
# env:, spawn_opts: passed through to Command verbatim.
|
|
21
|
+
class SolidQueue < Command
|
|
22
|
+
DEFAULT_CMD = "bin/jobs"
|
|
23
|
+
|
|
24
|
+
def spawn(spec)
|
|
25
|
+
super(command_spec(spec))
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def command_spec(spec)
|
|
31
|
+
ChildSpec.new(
|
|
32
|
+
id: spec.id, adapter: spec.adapter, restart: spec.restart,
|
|
33
|
+
shutdown: spec.shutdown, start_timeout: spec.start_timeout,
|
|
34
|
+
opts: spec.opts.merge(cmd: spec.opts.fetch(:cmd, DEFAULT_CMD))
|
|
35
|
+
)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
Adapter.register(:solid_queue, Adapters::SolidQueue)
|
|
40
|
+
end
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
module OtpRails
|
|
3
|
+
module Adapters
|
|
4
|
+
# DESIGN §3.1 nested supervisors. A subtree is just another child of the
|
|
5
|
+
# parent: this pseudo-adapter runs a child Supervisor on a Thread and maps
|
|
6
|
+
# the four adapter operations onto it. Escalation inside the subtree
|
|
7
|
+
# terminates the thread, which surfaces to the parent as a crashed exit —
|
|
8
|
+
# the parent then applies ITS strategy/intensity to the subtree child
|
|
9
|
+
# (restart the whole subtree, etc.), exactly per OTP semantics.
|
|
10
|
+
#
|
|
11
|
+
# opts:
|
|
12
|
+
# builder: a Proc returning a FRESH OtpRails::Supervisor each call.
|
|
13
|
+
# A restarted subtree must not inherit the old subtree's
|
|
14
|
+
# RestartIntensity window (it is stateful), so spawn re-invokes
|
|
15
|
+
# the builder on every (re)start instead of reusing an instance.
|
|
16
|
+
class SupervisorAdapter < Adapter
|
|
17
|
+
# Duck-types the two things the tree asks of an exit status:
|
|
18
|
+
# #exitstatus (telemetry) and #success? (ChildSpec#restart? semantics).
|
|
19
|
+
Status = Struct.new(:exitstatus) do
|
|
20
|
+
def success? = exitstatus == 0
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
Handle = Struct.new(:sub, :thread, :started_at, :exit_status, :waiter, :killed)
|
|
24
|
+
|
|
25
|
+
def spawn(spec)
|
|
26
|
+
builder = spec.opts.fetch(:builder) do
|
|
27
|
+
raise ConfigError, "#{spec.id}: :supervisor adapter requires builder: (a proc returning a fresh Supervisor)"
|
|
28
|
+
end
|
|
29
|
+
sub = builder.call
|
|
30
|
+
raise ConfigError, "#{spec.id}: builder must return an OtpRails::Supervisor" unless sub.is_a?(Supervisor)
|
|
31
|
+
thread = Thread.new { sub.run }
|
|
32
|
+
# Escalation out of a subtree is an expected, handled exit path — the
|
|
33
|
+
# waiter converts it into a crashed status. Don't let Ruby dump it.
|
|
34
|
+
thread.report_on_exception = false
|
|
35
|
+
Handle.new(sub, thread, Process.clock_gettime(Process::CLOCK_MONOTONIC), nil, nil, false)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def link(handle, &on_exit)
|
|
39
|
+
handle.waiter = Thread.new do
|
|
40
|
+
status =
|
|
41
|
+
begin
|
|
42
|
+
handle.thread.join # re-raises whatever terminated the subtree
|
|
43
|
+
Status.new(handle.killed ? nil : 0)
|
|
44
|
+
rescue Escalation
|
|
45
|
+
Status.new(70) # crashed: intensity exceeded inside the subtree
|
|
46
|
+
rescue StandardError
|
|
47
|
+
Status.new(1) # crashed: unexpected error in the subtree loop
|
|
48
|
+
end
|
|
49
|
+
handle.exit_status = status
|
|
50
|
+
on_exit.call(status)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# The subtree's internal health is its own supervisor's business; from
|
|
55
|
+
# the parent's seat the subtree is healthy while its loop is running.
|
|
56
|
+
# Death also arrives via link, so parents normally give subtree specs
|
|
57
|
+
# health_interval: nil (the DSL does) and skip the probe monitor.
|
|
58
|
+
def health(handle)
|
|
59
|
+
handle.thread.alive? ? :healthy : :dead
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Clean stop: the subtree's run loop breaks and its `ensure stop_all`
|
|
63
|
+
# drains the subtree's own children (reverse order) before the thread
|
|
64
|
+
# exits — so by the time this returns true, no subtree PIDs remain.
|
|
65
|
+
def drain(handle, timeout:)
|
|
66
|
+
handle.sub.stop
|
|
67
|
+
joined =
|
|
68
|
+
begin
|
|
69
|
+
handle.thread.join(timeout)
|
|
70
|
+
rescue StandardError
|
|
71
|
+
handle.thread # join re-raised => the thread has terminated
|
|
72
|
+
end
|
|
73
|
+
!joined.nil?
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Last resort. Thread#kill still runs the subtree's `ensure stop_all`;
|
|
77
|
+
# if even that wedges, best-effort drain the subtree's children directly
|
|
78
|
+
# so no grandchild PIDs are orphaned.
|
|
79
|
+
def kill(handle)
|
|
80
|
+
handle.killed = true
|
|
81
|
+
handle.thread.kill
|
|
82
|
+
joined =
|
|
83
|
+
begin
|
|
84
|
+
handle.thread.join(2)
|
|
85
|
+
rescue StandardError
|
|
86
|
+
handle.thread
|
|
87
|
+
end
|
|
88
|
+
handle.sub.send(:stop_all) if joined.nil? # loop wedged: drain grandchildren ourselves
|
|
89
|
+
rescue StandardError
|
|
90
|
+
nil
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
Adapter.register(:supervisor, Adapters::SupervisorAdapter)
|
|
95
|
+
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
module OtpRails
|
|
3
|
+
class Backoff
|
|
4
|
+
def initialize(kind: :exponential, base: 1.0, cap: 30.0)
|
|
5
|
+
@kind, @base, @cap = kind, base.to_f, cap.to_f
|
|
6
|
+
end
|
|
7
|
+
|
|
8
|
+
# attempt is 1-based
|
|
9
|
+
def delay(attempt)
|
|
10
|
+
case @kind
|
|
11
|
+
when :none then 0.0
|
|
12
|
+
when :constant then @base
|
|
13
|
+
when :exponential then [@base * (2**(attempt - 1)), @cap].min
|
|
14
|
+
else raise ConfigError, "unknown backoff #{@kind}"
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
module OtpRails
|
|
3
|
+
# DESIGN §3.2
|
|
4
|
+
class ChildSpec
|
|
5
|
+
RESTART_KINDS = %i[permanent transient temporary].freeze
|
|
6
|
+
|
|
7
|
+
attr_reader :id, :adapter, :restart, :shutdown, :start_timeout, :health_interval,
|
|
8
|
+
:degraded_restart_after, :opts
|
|
9
|
+
|
|
10
|
+
def initialize(id:, adapter:, restart: :permanent, shutdown: 30, start_timeout: 30,
|
|
11
|
+
health_interval: 5, degraded_restart_after: nil, opts: {})
|
|
12
|
+
raise ConfigError, "child id must be a Symbol" unless id.is_a?(Symbol)
|
|
13
|
+
raise ConfigError, "restart must be one of #{RESTART_KINDS}" unless RESTART_KINDS.include?(restart)
|
|
14
|
+
@id, @adapter, @restart, @shutdown, @start_timeout, @opts =
|
|
15
|
+
id, adapter, restart, shutdown, start_timeout, opts
|
|
16
|
+
@health_interval, @degraded_restart_after = health_interval, degraded_restart_after
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Should this child be restarted given how it exited? (OTP semantics)
|
|
20
|
+
def restart?(exit_status)
|
|
21
|
+
case restart
|
|
22
|
+
when :permanent then true
|
|
23
|
+
when :transient then !normal_exit?(exit_status)
|
|
24
|
+
when :temporary then false
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def normal_exit?(status)
|
|
29
|
+
return false if status.nil?
|
|
30
|
+
status.respond_to?(:success?) ? status.success? : status.to_i.zero?
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
module OtpRails
|
|
3
|
+
module CLI
|
|
4
|
+
USAGE = <<~TXT
|
|
5
|
+
usage: otp-rails run [config/supervisor.rb] # start the tree, block until shutdown
|
|
6
|
+
otp-rails check [config/supervisor.rb] # validate config, print the tree
|
|
7
|
+
otp-rails version
|
|
8
|
+
TXT
|
|
9
|
+
|
|
10
|
+
def self.run(argv)
|
|
11
|
+
cmd, path = argv[0], (argv[1] || "config/supervisor.rb")
|
|
12
|
+
case cmd
|
|
13
|
+
when "run"
|
|
14
|
+
Telemetry::Subscribers.logger
|
|
15
|
+
sup = DSL.load_file(path)
|
|
16
|
+
%w[INT TERM].each { |sig| trap(sig) { sup.stop } }
|
|
17
|
+
sup.run
|
|
18
|
+
0
|
|
19
|
+
when "check"
|
|
20
|
+
sup = DSL.load_file(path)
|
|
21
|
+
puts "strategy: #{sup.strategy}"
|
|
22
|
+
sup.children.each { |c| puts " #{c.id} (#{c.adapter}, #{c.restart}, shutdown=#{c.shutdown}s)" }
|
|
23
|
+
0
|
|
24
|
+
when "version" then puts VERSION; 0
|
|
25
|
+
else $stderr.puts USAGE; 1
|
|
26
|
+
end
|
|
27
|
+
rescue Escalation => e
|
|
28
|
+
$stderr.puts "otp-rails: #{e.message}"; 70 # EX_SOFTWARE — the platform is the final supervisor
|
|
29
|
+
rescue ConfigError, Errno::ENOENT => e
|
|
30
|
+
$stderr.puts "otp-rails: #{e.message}"; 78 # EX_CONFIG
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
module OtpRails
|
|
3
|
+
# Evaluates config/supervisor.rb WITHOUT Rails loaded (DESIGN §9).
|
|
4
|
+
#
|
|
5
|
+
# strategy :rest_for_one
|
|
6
|
+
# max_restarts 5, within: 60
|
|
7
|
+
# backoff :exponential, base: 1, cap: 30
|
|
8
|
+
# child :web, adapter: :command, cmd: "bundle exec puma -C config/puma.rb"
|
|
9
|
+
# child :jobs, adapter: :command, cmd: "bin/jobs"
|
|
10
|
+
# supervisor :background do # DESIGN §3.1: nested subtree with its
|
|
11
|
+
# strategy :one_for_all # own strategy/intensity/backoff
|
|
12
|
+
# child :cron, adapter: :command, cmd: "bin/rails cron"
|
|
13
|
+
# end
|
|
14
|
+
class DSL
|
|
15
|
+
def self.load_file(path)
|
|
16
|
+
dsl = new
|
|
17
|
+
dsl.instance_eval(File.read(path), path, 1)
|
|
18
|
+
dsl.build
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def initialize(root: true)
|
|
22
|
+
@root = root
|
|
23
|
+
@strategy = :one_for_one
|
|
24
|
+
@intensity = { max_restarts: 5, within: 60 }
|
|
25
|
+
@backoff = { kind: :exponential, base: 1, cap: 30 }
|
|
26
|
+
@children = []
|
|
27
|
+
# DESIGN §5/§9 default; socket nil disables. Only the ROOT supervisor
|
|
28
|
+
# listens — subtrees never bind their own socket.
|
|
29
|
+
@socket_path = root ? "tmp/otp-rails.sock" : nil
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def strategy(kind) = @strategy = kind
|
|
33
|
+
def max_restarts(n, within:) = @intensity = { max_restarts: n, within: within }
|
|
34
|
+
def backoff(kind, **opts) = @backoff = { kind: kind, **opts }
|
|
35
|
+
|
|
36
|
+
def socket(path)
|
|
37
|
+
raise ConfigError, "socket can only be set on the root supervisor" unless @root
|
|
38
|
+
@socket_path = path
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# DESIGN §3.1: `supervisor :background do ... end` creates a subtree — a
|
|
42
|
+
# :supervisor child whose block supports the full DSL (strategy /
|
|
43
|
+
# max_restarts / backoff / child, and further nesting). The block is kept
|
|
44
|
+
# as a builder so every (re)start constructs a FRESH child Supervisor:
|
|
45
|
+
# RestartIntensity is stateful, and a restarted subtree must start with a
|
|
46
|
+
# clean intensity window.
|
|
47
|
+
def supervisor(id, restart: :permanent, shutdown: 30, start_timeout: 30, &block)
|
|
48
|
+
raise ConfigError, "supervisor #{id.inspect} requires a block" unless block
|
|
49
|
+
builder = lambda do
|
|
50
|
+
sub = DSL.new(root: false)
|
|
51
|
+
sub.instance_eval(&block)
|
|
52
|
+
sub.build
|
|
53
|
+
end
|
|
54
|
+
builder.call # fail fast at config load, not at spawn time
|
|
55
|
+
# health_interval nil: subtree liveness arrives via link (thread death),
|
|
56
|
+
# and its internal health is the subtree supervisor's own business — no
|
|
57
|
+
# probe monitor needed in the parent.
|
|
58
|
+
@children << ChildSpec.new(id: id, adapter: :supervisor, restart: restart,
|
|
59
|
+
shutdown: shutdown, start_timeout: start_timeout,
|
|
60
|
+
health_interval: nil, opts: { builder: builder })
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def child(id, adapter:, restart: :permanent, shutdown: 30, start_timeout: 30,
|
|
64
|
+
health_interval: 5, degraded_restart_after: nil, **opts)
|
|
65
|
+
@children << ChildSpec.new(id: id, adapter: adapter, restart: restart, shutdown: shutdown,
|
|
66
|
+
start_timeout: start_timeout, health_interval: health_interval,
|
|
67
|
+
degraded_restart_after: degraded_restart_after, opts: opts)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def build
|
|
71
|
+
sup = Supervisor.new(strategy: @strategy,
|
|
72
|
+
intensity: RestartIntensity.new(**@intensity),
|
|
73
|
+
backoff: Backoff.new(**@backoff),
|
|
74
|
+
socket_path: @socket_path)
|
|
75
|
+
@children.each { |c| sup.add_child(c) }
|
|
76
|
+
sup
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def self.supervise(&block)
|
|
81
|
+
dsl = DSL.new
|
|
82
|
+
dsl.instance_eval(&block)
|
|
83
|
+
dsl.build
|
|
84
|
+
end
|
|
85
|
+
end
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require "socket"
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
# Deliberately self-contained (PLAN 1.6): any child process can
|
|
6
|
+
# require "otp_rails/heartbeat"
|
|
7
|
+
# without loading the rest of the gem — e.g. from a Rails initializer or a
|
|
8
|
+
# Solid Queue hook — and never Rails itself (hard rule 1 / DESIGN §9).
|
|
9
|
+
module OtpRails
|
|
10
|
+
# Sends DESIGN §5 NDJSON heartbeats to the supervising socket:
|
|
11
|
+
#
|
|
12
|
+
# OtpRails::Heartbeat.start(id: "jobs")
|
|
13
|
+
# OtpRails::Heartbeat.start(id: "jobs", interval: 2,
|
|
14
|
+
# state: -> { queue_backlog_ok? ? "healthy" : "degraded" },
|
|
15
|
+
# meta: -> { { backlog: backlog_size } })
|
|
16
|
+
#
|
|
17
|
+
# Silently a no-op when OTP_RAILS_SOCK / OTP_RAILS_TOKEN are absent — the
|
|
18
|
+
# child is running unsupervised (plain `bin/jobs` in development) and that
|
|
19
|
+
# must not be an error. Reconnects on socket loss; never raises into the
|
|
20
|
+
# host process.
|
|
21
|
+
class Heartbeat
|
|
22
|
+
def self.start(id:, interval: 2, state: -> { "healthy" }, meta: -> { {} })
|
|
23
|
+
new(id: id, interval: interval, state: state, meta: meta).start
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def initialize(id:, interval: 2, state: -> { "healthy" }, meta: -> { {} })
|
|
27
|
+
@id, @interval, @state, @meta = id.to_s, interval, state, meta
|
|
28
|
+
@sock_path, @token = ENV["OTP_RAILS_SOCK"], ENV["OTP_RAILS_TOKEN"]
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Returns the beating thread, or nil when unsupervised.
|
|
32
|
+
def start
|
|
33
|
+
return nil unless @sock_path && @token
|
|
34
|
+
@thread = Thread.new do
|
|
35
|
+
sock = nil
|
|
36
|
+
loop do
|
|
37
|
+
begin
|
|
38
|
+
sock ||= UNIXSocket.new(@sock_path)
|
|
39
|
+
sock.puts(JSON.generate(id: @id, state: @state.call, ts: Time.now.to_i,
|
|
40
|
+
token: @token, meta: @meta.call))
|
|
41
|
+
rescue IOError, SystemCallError
|
|
42
|
+
sock = nil # supervisor gone or restarting; try again next beat
|
|
43
|
+
end
|
|
44
|
+
sleep @interval
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def stop = @thread&.kill
|
|
50
|
+
end
|
|
51
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
module OtpRails
|
|
3
|
+
# PLAN 1.1 orphan prevention: if the supervisor is SIGKILLed, children should
|
|
4
|
+
# still receive SIGTERM. Linux delivers this via prctl(PR_SET_PDEATHSIG),
|
|
5
|
+
# armed in the forked child between fork and exec. macOS/BSD have no
|
|
6
|
+
# equivalent: children there are re-parented to launchd/init and keep running
|
|
7
|
+
# until the platform supervisor reaps them (documented limitation, README).
|
|
8
|
+
module OrphanGuard
|
|
9
|
+
PR_SET_PDEATHSIG = 1
|
|
10
|
+
|
|
11
|
+
def self.available?
|
|
12
|
+
return @available if defined?(@available)
|
|
13
|
+
@available = RUBY_PLATFORM.include?("linux") && fiddle?
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def self.fiddle?
|
|
17
|
+
require "fiddle"
|
|
18
|
+
true
|
|
19
|
+
rescue LoadError
|
|
20
|
+
false
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Runs in the forked child, pre-exec. parent_pid is the supervisor's pid at
|
|
24
|
+
# fork time: pdeathsig is not delivered if the parent died before prctl ran,
|
|
25
|
+
# so re-check the parent afterwards and exit rather than run orphaned.
|
|
26
|
+
def self.arm!(parent_pid, signal: "TERM")
|
|
27
|
+
return false unless available?
|
|
28
|
+
libc = Fiddle.dlopen(nil)
|
|
29
|
+
prctl = Fiddle::Function.new(libc["prctl"], [Fiddle::TYPE_INT] * 5, Fiddle::TYPE_INT)
|
|
30
|
+
armed = prctl.call(PR_SET_PDEATHSIG, Signal.list.fetch(signal), 0, 0, 0).zero?
|
|
31
|
+
Process.exit!(0) if Process.ppid != parent_pid
|
|
32
|
+
armed
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require "socket"
|
|
3
|
+
require "net/http"
|
|
4
|
+
require "uri"
|
|
5
|
+
|
|
6
|
+
module OtpRails
|
|
7
|
+
# Passive probes (DESIGN §4.1/§5): a child declared with probe: opts is
|
|
8
|
+
# :starting until the probe answers, then :healthy. Probes are adapter
|
|
9
|
+
# plumbing behind health(), not part of the Adapter interface (hard rule 2).
|
|
10
|
+
#
|
|
11
|
+
# probe: { tcp: 5432 }
|
|
12
|
+
# probe: { http: "http://127.0.0.1:3000/up" }
|
|
13
|
+
module Probe
|
|
14
|
+
CONNECT_TIMEOUT = 0.25
|
|
15
|
+
READ_TIMEOUT = 0.5
|
|
16
|
+
|
|
17
|
+
# true when the probe answers (or the spec declares no probe).
|
|
18
|
+
def self.answering?(spec)
|
|
19
|
+
probe = spec.opts[:probe] or return true
|
|
20
|
+
if (port = probe[:tcp])
|
|
21
|
+
tcp?(port)
|
|
22
|
+
elsif (url = probe[:http])
|
|
23
|
+
http?(url)
|
|
24
|
+
else
|
|
25
|
+
raise ConfigError, "#{spec.id}: probe must be {tcp: PORT} or {http: URL}, got #{probe.inspect}"
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def self.tcp?(port, host = "127.0.0.1")
|
|
30
|
+
Socket.tcp(host, port, connect_timeout: CONNECT_TIMEOUT) { true }
|
|
31
|
+
rescue SystemCallError, IO::TimeoutError
|
|
32
|
+
false
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def self.http?(url)
|
|
36
|
+
uri = URI(url)
|
|
37
|
+
Net::HTTP.start(uri.host, uri.port, open_timeout: CONNECT_TIMEOUT, read_timeout: READ_TIMEOUT) do |http|
|
|
38
|
+
http.get(uri.path.empty? ? "/" : uri.path).code.to_i.between?(200, 299)
|
|
39
|
+
end
|
|
40
|
+
rescue SystemCallError, IO::TimeoutError, Net::OpenTimeout, Net::ReadTimeout, EOFError
|
|
41
|
+
false
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
module OtpRails
|
|
3
|
+
# OTP restart intensity: more than `max_restarts` within `within` seconds => escalate.
|
|
4
|
+
class RestartIntensity
|
|
5
|
+
attr_reader :max_restarts, :within
|
|
6
|
+
|
|
7
|
+
def initialize(max_restarts: 5, within: 60, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
|
|
8
|
+
@max_restarts, @within, @clock = max_restarts, within, clock
|
|
9
|
+
@events = []
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
# Records a restart. Returns true if the supervisor should escalate (give up).
|
|
13
|
+
def record!
|
|
14
|
+
now = @clock.call
|
|
15
|
+
@events << now
|
|
16
|
+
@events.reject! { |t| now - t > @within }
|
|
17
|
+
@events.size > @max_restarts
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def count = @events.size
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require "socket"
|
|
3
|
+
require "json"
|
|
4
|
+
require "securerandom"
|
|
5
|
+
require "fileutils"
|
|
6
|
+
|
|
7
|
+
module OtpRails
|
|
8
|
+
# DESIGN §5 active heartbeats + §9 control transport, PLAN 1.4.
|
|
9
|
+
# Newline-delimited JSON over a Unix socket, mode 0600, per-boot token —
|
|
10
|
+
# no MessagePack, no length prefixes, no versions (hard rule 4). This wire
|
|
11
|
+
# format is the contract the Elixir `beam` repo consumes.
|
|
12
|
+
#
|
|
13
|
+
# Heartbeat: {"id":"jobs","state":"healthy","ts":1757700000,"token":"…","meta":{}}
|
|
14
|
+
# Control: {"cmd":"restart","id":"jobs","token":"…"}
|
|
15
|
+
# Any line with a missing or wrong token is dropped without a reply.
|
|
16
|
+
class SocketServer
|
|
17
|
+
attr_reader :path, :token
|
|
18
|
+
|
|
19
|
+
def initialize(path:, on_heartbeat:, on_control:)
|
|
20
|
+
@path, @on_heartbeat, @on_control = path, on_heartbeat, on_control
|
|
21
|
+
@token = SecureRandom.hex(16)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Binds, chmods, and exports OTP_RAILS_SOCK / OTP_RAILS_TOKEN so children
|
|
25
|
+
# spawned afterwards inherit them (DESIGN §9). Call before starting children.
|
|
26
|
+
def start
|
|
27
|
+
FileUtils.mkdir_p(File.dirname(@path))
|
|
28
|
+
File.unlink(@path) if File.exist?(@path) # stale socket from a dead boot
|
|
29
|
+
@server = UNIXServer.new(@path)
|
|
30
|
+
File.chmod(0o600, @path)
|
|
31
|
+
ENV["OTP_RAILS_SOCK"] = @path
|
|
32
|
+
ENV["OTP_RAILS_TOKEN"] = @token
|
|
33
|
+
@acceptor = Thread.new do
|
|
34
|
+
loop do
|
|
35
|
+
conn = @server.accept
|
|
36
|
+
Thread.new { serve(conn) }
|
|
37
|
+
rescue IOError, SystemCallError
|
|
38
|
+
break # server closed during shutdown
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def stop
|
|
44
|
+
@server&.close
|
|
45
|
+
@acceptor&.kill
|
|
46
|
+
File.unlink(@path) if File.exist?(@path)
|
|
47
|
+
rescue SystemCallError
|
|
48
|
+
nil
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
def serve(conn)
|
|
54
|
+
conn.each_line do |line|
|
|
55
|
+
msg = begin
|
|
56
|
+
JSON.parse(line)
|
|
57
|
+
rescue JSON::ParserError
|
|
58
|
+
next
|
|
59
|
+
end
|
|
60
|
+
next unless msg["token"] == @token # bad token ⇒ dropped
|
|
61
|
+
if msg["cmd"]
|
|
62
|
+
@on_control.call(msg)
|
|
63
|
+
elsif msg["id"] && msg["state"]
|
|
64
|
+
@on_heartbeat.call(msg)
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
rescue IOError, SystemCallError
|
|
68
|
+
nil
|
|
69
|
+
ensure
|
|
70
|
+
begin
|
|
71
|
+
conn.close
|
|
72
|
+
rescue IOError
|
|
73
|
+
nil
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
module OtpRails
|
|
3
|
+
# DESIGN §3.3. Given the ordered child ids and the failed id,
|
|
4
|
+
# return the ordered ids that must be stopped and restarted.
|
|
5
|
+
module Strategy
|
|
6
|
+
KINDS = %i[one_for_one rest_for_one one_for_all].freeze
|
|
7
|
+
|
|
8
|
+
def self.affected(kind, ordered_ids, failed_id)
|
|
9
|
+
raise ConfigError, "unknown strategy #{kind}" unless KINDS.include?(kind)
|
|
10
|
+
idx = ordered_ids.index(failed_id) or raise ArgumentError, "#{failed_id} not in tree"
|
|
11
|
+
case kind
|
|
12
|
+
when :one_for_one then [failed_id]
|
|
13
|
+
when :rest_for_one then ordered_ids[idx..]
|
|
14
|
+
when :one_for_all then ordered_ids.dup
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
module OtpRails
|
|
3
|
+
# DESIGN §3.1 / §3.3. One supervisor, an ordered set of children, one strategy.
|
|
4
|
+
# Nested supervisors (subtrees) are ordinary children via Adapters::SupervisorAdapter:
|
|
5
|
+
# a subtree escalating shows up here as a crashed child exit.
|
|
6
|
+
class Supervisor
|
|
7
|
+
attr_reader :children, :strategy, :intensity, :backoff
|
|
8
|
+
|
|
9
|
+
def initialize(strategy: :one_for_one, intensity: RestartIntensity.new, backoff: Backoff.new,
|
|
10
|
+
socket_path: nil)
|
|
11
|
+
raise ConfigError, "unknown strategy #{strategy}" unless Strategy::KINDS.include?(strategy)
|
|
12
|
+
@strategy, @intensity, @backoff = strategy, intensity, backoff
|
|
13
|
+
@children = [] # ordered ChildSpecs
|
|
14
|
+
@live = {} # id => { adapter:, handle:, attempts:, generation:, monitor: }
|
|
15
|
+
@queue = Queue.new
|
|
16
|
+
@stopping = false
|
|
17
|
+
@heartbeats = {} # id => { at: monotonic ts of last heartbeat, state: reported state }
|
|
18
|
+
return unless socket_path
|
|
19
|
+
@socket = SocketServer.new(
|
|
20
|
+
path: socket_path,
|
|
21
|
+
on_heartbeat: ->(msg) { @heartbeats[msg["id"].to_sym] = { at: mono_now, state: msg["state"] } },
|
|
22
|
+
on_control: ->(msg) { @queue << { type: :control, cmd: msg["cmd"], id: msg["id"].to_s.to_sym } }
|
|
23
|
+
)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# The per-boot token children must echo in every heartbeat (nil when the
|
|
27
|
+
# socket is disabled). Exported to children as OTP_RAILS_TOKEN.
|
|
28
|
+
def heartbeat_token = @socket&.token
|
|
29
|
+
|
|
30
|
+
def add_child(spec)
|
|
31
|
+
raise ConfigError, "duplicate child id #{spec.id}" if @children.any? { |c| c.id == spec.id }
|
|
32
|
+
@children << spec
|
|
33
|
+
self
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Blocks until the tree is shut down. Raises Escalation if intensity is exceeded.
|
|
37
|
+
def run
|
|
38
|
+
Telemetry.emit(:"supervisor.start", {}, { strategy: strategy, children: ids })
|
|
39
|
+
@socket&.start # before children, so they inherit OTP_RAILS_SOCK/_TOKEN
|
|
40
|
+
@children.each { |spec| start_child(spec) }
|
|
41
|
+
loop do
|
|
42
|
+
msg = @queue.pop
|
|
43
|
+
case msg[:type]
|
|
44
|
+
when :exit then handle_exit(msg[:id], msg[:generation], msg[:status])
|
|
45
|
+
when :health_dead then handle_health_dead(msg[:id], msg[:generation])
|
|
46
|
+
when :control then handle_control(msg)
|
|
47
|
+
when :stop then break
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
ensure
|
|
51
|
+
stop_all
|
|
52
|
+
@socket&.stop
|
|
53
|
+
Telemetry.emit(:"supervisor.stop")
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def stop = @queue << { type: :stop }
|
|
57
|
+
|
|
58
|
+
# Public remediation API (DESIGN §7). Over IPC in the real thing; direct call here.
|
|
59
|
+
def restart!(id)
|
|
60
|
+
spec = spec_for(id)
|
|
61
|
+
stop_child(spec)
|
|
62
|
+
start_child(spec)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def ids = @children.map(&:id)
|
|
66
|
+
|
|
67
|
+
# Debug/test hook. Not public API.
|
|
68
|
+
def live_pid(id) = @live.dig(id, :handle)&.pid
|
|
69
|
+
|
|
70
|
+
private
|
|
71
|
+
|
|
72
|
+
def spec_for(id) = @children.find { |c| c.id == id } || raise(ArgumentError, "no child #{id}")
|
|
73
|
+
|
|
74
|
+
def start_child(spec)
|
|
75
|
+
@heartbeats.delete(spec.id) # a replaced child's heartbeats must not vouch for its successor
|
|
76
|
+
adapter = Adapter.lookup(spec.adapter).new
|
|
77
|
+
handle = adapter.spawn(spec)
|
|
78
|
+
prev = @live[spec.id] || {}
|
|
79
|
+
generation = (prev[:generation] || 0) + 1
|
|
80
|
+
@live[spec.id] = { adapter: adapter, handle: handle, attempts: prev[:attempts] || 0, generation: generation }
|
|
81
|
+
adapter.link(handle) do |status|
|
|
82
|
+
@queue << { type: :exit, id: spec.id, generation: generation, status: status } unless @stopping
|
|
83
|
+
end
|
|
84
|
+
Telemetry.emit(:"child.spawn", {}, { id: spec.id, adapter: spec.adapter, pid: handle.respond_to?(:pid) ? handle.pid : nil })
|
|
85
|
+
start_monitor(spec, generation) if wait_healthy(spec, adapter, handle) == :healthy
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# PLAN 1.3: per-child polling thread. :degraded emits telemetry only,
|
|
89
|
+
# unless degraded_restart_after consecutive reports accumulate; :dead from
|
|
90
|
+
# a probe (not just SIGCHLD) goes through the exit queue like any crash.
|
|
91
|
+
def start_monitor(spec, generation)
|
|
92
|
+
return unless spec.health_interval
|
|
93
|
+
entry = @live[spec.id]
|
|
94
|
+
adapter, handle = entry[:adapter], entry[:handle]
|
|
95
|
+
degraded = 0
|
|
96
|
+
entry[:monitor] = Thread.new do
|
|
97
|
+
loop do
|
|
98
|
+
sleep spec.health_interval
|
|
99
|
+
break if @stopping || @live.dig(spec.id, :generation) != generation
|
|
100
|
+
case effective_health(spec, adapter, handle)
|
|
101
|
+
when :healthy
|
|
102
|
+
degraded = 0
|
|
103
|
+
when :degraded
|
|
104
|
+
degraded += 1
|
|
105
|
+
Telemetry.emit(:"child.degraded", { consecutive: degraded }, { id: spec.id })
|
|
106
|
+
if spec.degraded_restart_after && degraded >= spec.degraded_restart_after
|
|
107
|
+
@queue << { type: :health_dead, id: spec.id, generation: generation }
|
|
108
|
+
break
|
|
109
|
+
end
|
|
110
|
+
when :dead
|
|
111
|
+
@queue << { type: :health_dead, id: spec.id, generation: generation }
|
|
112
|
+
break
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def wait_healthy(spec, adapter, handle)
|
|
119
|
+
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + spec.start_timeout
|
|
120
|
+
loop do
|
|
121
|
+
case effective_health(spec, adapter, handle)
|
|
122
|
+
when :healthy then Telemetry.emit(:"child.healthy", {}, { id: spec.id }); return :healthy
|
|
123
|
+
when :dead then return :dead # the exit message arrives via link
|
|
124
|
+
end
|
|
125
|
+
if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
|
|
126
|
+
# PLAN 1.2: start_timeout exceeded ⇒ drain. The resulting exit flows
|
|
127
|
+
# through the normal link → handle_exit path, so it counts as a
|
|
128
|
+
# crash and the strategy + intensity apply.
|
|
129
|
+
stop_child(spec)
|
|
130
|
+
return :timeout
|
|
131
|
+
end
|
|
132
|
+
sleep 0.05
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
HEARTBEAT_STATES = { "starting" => :starting, "healthy" => :healthy,
|
|
137
|
+
"degraded" => :degraded, "dead" => :dead }.freeze
|
|
138
|
+
|
|
139
|
+
# DESIGN §5/§9: health is active-first. A child that has heartbeated is
|
|
140
|
+
# judged by heartbeat freshness and its own reported state — missing 3
|
|
141
|
+
# intervals ⇒ :degraded, 6 ⇒ :dead. Children that never heartbeat fall
|
|
142
|
+
# back to the adapter's passive probe.
|
|
143
|
+
def effective_health(spec, adapter, handle)
|
|
144
|
+
hb = @heartbeats[spec.id]
|
|
145
|
+
return adapter.health(handle) unless hb
|
|
146
|
+
missed = (mono_now - hb[:at]) / spec.health_interval
|
|
147
|
+
return :dead if missed >= 6
|
|
148
|
+
return :degraded if missed >= 3
|
|
149
|
+
HEARTBEAT_STATES.fetch(hb[:state], :healthy)
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def mono_now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
153
|
+
|
|
154
|
+
# DESIGN §7/§9: {"cmd":"restart","id":...} over the socket — the transport
|
|
155
|
+
# Rails.supervisor.restart! rides later. Unknown commands and ids are
|
|
156
|
+
# ignored; the token was already checked at the socket layer.
|
|
157
|
+
def handle_control(msg)
|
|
158
|
+
return unless msg[:cmd] == "restart"
|
|
159
|
+
return unless @children.any? { |c| c.id == msg[:id] }
|
|
160
|
+
restart!(msg[:id])
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# A monitor thread declared this child unhealthy enough to replace. Drain
|
|
164
|
+
# it; the resulting real exit flows through link → handle_exit, so the
|
|
165
|
+
# strategy and intensity apply exactly as for a crash (same path as the
|
|
166
|
+
# 1.2 start_timeout drain). If the child already exited and was replaced,
|
|
167
|
+
# the generation guard makes this a no-op.
|
|
168
|
+
def handle_health_dead(id, generation)
|
|
169
|
+
entry = @live[id]
|
|
170
|
+
return if entry.nil? || entry[:generation] != generation
|
|
171
|
+
stop_child(spec_for(id))
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def handle_exit(id, generation, status)
|
|
175
|
+
spec = spec_for(id)
|
|
176
|
+
entry = @live[id]
|
|
177
|
+
return if entry.nil? || entry[:generation] != generation # stale exit from a child we already replaced
|
|
178
|
+
|
|
179
|
+
uptime = Process.clock_gettime(Process::CLOCK_MONOTONIC) - entry[:handle].started_at
|
|
180
|
+
Telemetry.emit(:"child.exit", { exit_code: status&.exitstatus, uptime_ms: (uptime * 1000).round }, { id: id })
|
|
181
|
+
return unless spec.restart?(status)
|
|
182
|
+
|
|
183
|
+
if intensity.record!
|
|
184
|
+
Telemetry.emit(:"supervisor.escalate", { restarts: intensity.count }, { within: intensity.within })
|
|
185
|
+
raise Escalation, "restart intensity exceeded (#{intensity.count} in #{intensity.within}s)"
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
Strategy.affected(strategy, ids, id).each do |aid|
|
|
189
|
+
aspec = spec_for(aid)
|
|
190
|
+
stop_child(aspec) unless aid == id
|
|
191
|
+
attempts = (@live[aid][:attempts] += 1)
|
|
192
|
+
delay = backoff.delay(attempts)
|
|
193
|
+
Telemetry.emit(:"child.restart", { backoff_ms: (delay * 1000).round }, { id: aid, attempt: attempts, strategy: strategy })
|
|
194
|
+
sleep delay
|
|
195
|
+
start_child(aspec)
|
|
196
|
+
end
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def stop_child(spec)
|
|
200
|
+
entry = @live[spec.id] or return
|
|
201
|
+
entry[:monitor]&.kill
|
|
202
|
+
entry[:monitor] = nil
|
|
203
|
+
Telemetry.emit(:"child.drain", {}, { id: spec.id })
|
|
204
|
+
return if entry[:adapter].drain(entry[:handle], timeout: spec.shutdown)
|
|
205
|
+
Telemetry.emit(:"child.kill", {}, { id: spec.id })
|
|
206
|
+
entry[:adapter].kill(entry[:handle])
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def stop_all
|
|
210
|
+
@stopping = true
|
|
211
|
+
@children.reverse_each { |spec| stop_child(spec) }
|
|
212
|
+
end
|
|
213
|
+
end
|
|
214
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require "json"
|
|
3
|
+
|
|
4
|
+
module OtpRails
|
|
5
|
+
# Minimal event bus. Event names mirror Elixir :telemetry (DESIGN §6):
|
|
6
|
+
# event: [:otp_rails, :child, :restart], measurements: {...}, metadata: {...}
|
|
7
|
+
# A railtie will bridge these into ActiveSupport::Notifications inside children.
|
|
8
|
+
module Telemetry
|
|
9
|
+
EVENTS = %i[
|
|
10
|
+
supervisor.start supervisor.stop supervisor.escalate
|
|
11
|
+
child.spawn child.healthy child.degraded child.exit child.restart child.drain child.kill
|
|
12
|
+
].freeze
|
|
13
|
+
|
|
14
|
+
@subscribers = []
|
|
15
|
+
@mutex = Mutex.new
|
|
16
|
+
|
|
17
|
+
class << self
|
|
18
|
+
def subscribe(&block)
|
|
19
|
+
@mutex.synchronize { @subscribers << block }
|
|
20
|
+
block
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def unsubscribe(block)
|
|
24
|
+
@mutex.synchronize { @subscribers.delete(block) }
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def reset!
|
|
28
|
+
@mutex.synchronize { @subscribers.clear }
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# name: Symbol like :"child.restart" (flat in Ruby; split on "." for the Elixir side)
|
|
32
|
+
def emit(name, measurements = {}, metadata = {})
|
|
33
|
+
raise ArgumentError, "unknown event #{name}" unless EVENTS.include?(name)
|
|
34
|
+
event = { event: [:otp_rails, *name.to_s.split(".").map(&:to_sym)],
|
|
35
|
+
measurements: measurements, metadata: metadata,
|
|
36
|
+
ts: Process.clock_gettime(Process::CLOCK_REALTIME) }
|
|
37
|
+
subs = @mutex.synchronize { @subscribers.dup }
|
|
38
|
+
subs.each { |s| s.call(event) }
|
|
39
|
+
event
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Default subscribers shipped in v0.1
|
|
44
|
+
module Subscribers
|
|
45
|
+
def self.logger(io = $stderr)
|
|
46
|
+
Telemetry.subscribe do |e|
|
|
47
|
+
io.puts("[otp-rails] #{e[:event].join('.')} #{e[:metadata].inspect} #{e[:measurements].inspect}")
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def self.json_lines(io = $stdout)
|
|
52
|
+
Telemetry.subscribe { |e| io.puts(JSON.generate(e)) }
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
data/lib/otp_rails.rb
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# otp-rails: a slim, Rails-free process supervisor with OTP semantics.
|
|
4
|
+
# Nothing under lib/otp_rails may require Rails, ActiveSupport, or ActiveRecord (DESIGN §9).
|
|
5
|
+
# The Rails-side bridge (railtie) is a later deliverable and is only ever loaded inside children.
|
|
6
|
+
|
|
7
|
+
require_relative "otp_rails/version"
|
|
8
|
+
|
|
9
|
+
module OtpRails
|
|
10
|
+
class Error < StandardError; end
|
|
11
|
+
class ConfigError < Error; end
|
|
12
|
+
class Escalation < Error; end # raised when restart intensity is exceeded
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
require_relative "otp_rails/telemetry"
|
|
16
|
+
require_relative "otp_rails/child_spec"
|
|
17
|
+
require_relative "otp_rails/strategy"
|
|
18
|
+
require_relative "otp_rails/restart_intensity"
|
|
19
|
+
require_relative "otp_rails/backoff"
|
|
20
|
+
require_relative "otp_rails/adapter"
|
|
21
|
+
require_relative "otp_rails/probe"
|
|
22
|
+
require_relative "otp_rails/orphan_guard"
|
|
23
|
+
require_relative "otp_rails/adapters/command"
|
|
24
|
+
require_relative "otp_rails/adapters/puma"
|
|
25
|
+
require_relative "otp_rails/adapters/solid_queue"
|
|
26
|
+
require_relative "otp_rails/socket_server"
|
|
27
|
+
require_relative "otp_rails/heartbeat"
|
|
28
|
+
require_relative "otp_rails/supervisor"
|
|
29
|
+
require_relative "otp_rails/adapters/supervisor_adapter" # after supervisor: wraps a child Supervisor
|
|
30
|
+
require_relative "otp_rails/dsl"
|
|
31
|
+
require_relative "otp_rails/cli"
|
metadata
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: otp-rails
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Tim
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: exe
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2026-09-13 00:00:00.000000000 Z
|
|
12
|
+
dependencies: []
|
|
13
|
+
description: A slim supervisor that starts, links, health-checks, and restarts the
|
|
14
|
+
processes of a Rails app (web, jobs, cable, cron) with OTP strategies.
|
|
15
|
+
email:
|
|
16
|
+
executables:
|
|
17
|
+
- otp-rails
|
|
18
|
+
extensions: []
|
|
19
|
+
extra_rdoc_files: []
|
|
20
|
+
files:
|
|
21
|
+
- CHANGELOG.md
|
|
22
|
+
- LICENSE
|
|
23
|
+
- README.md
|
|
24
|
+
- exe/otp-rails
|
|
25
|
+
- lib/otp_rails.rb
|
|
26
|
+
- lib/otp_rails/adapter.rb
|
|
27
|
+
- lib/otp_rails/adapters/command.rb
|
|
28
|
+
- lib/otp_rails/adapters/puma.rb
|
|
29
|
+
- lib/otp_rails/adapters/solid_queue.rb
|
|
30
|
+
- lib/otp_rails/adapters/supervisor_adapter.rb
|
|
31
|
+
- lib/otp_rails/backoff.rb
|
|
32
|
+
- lib/otp_rails/child_spec.rb
|
|
33
|
+
- lib/otp_rails/cli.rb
|
|
34
|
+
- lib/otp_rails/dsl.rb
|
|
35
|
+
- lib/otp_rails/heartbeat.rb
|
|
36
|
+
- lib/otp_rails/orphan_guard.rb
|
|
37
|
+
- lib/otp_rails/probe.rb
|
|
38
|
+
- lib/otp_rails/restart_intensity.rb
|
|
39
|
+
- lib/otp_rails/socket_server.rb
|
|
40
|
+
- lib/otp_rails/strategy.rb
|
|
41
|
+
- lib/otp_rails/supervisor.rb
|
|
42
|
+
- lib/otp_rails/telemetry.rb
|
|
43
|
+
- lib/otp_rails/version.rb
|
|
44
|
+
homepage: https://github.com/shishi-odoshi/otp-rails
|
|
45
|
+
licenses:
|
|
46
|
+
- MIT
|
|
47
|
+
metadata:
|
|
48
|
+
homepage_uri: https://github.com/shishi-odoshi/otp-rails
|
|
49
|
+
source_code_uri: https://github.com/shishi-odoshi/otp-rails
|
|
50
|
+
changelog_uri: https://github.com/shishi-odoshi/otp-rails/blob/main/CHANGELOG.md
|
|
51
|
+
bug_tracker_uri: https://github.com/shishi-odoshi/otp-rails/issues
|
|
52
|
+
post_install_message:
|
|
53
|
+
rdoc_options: []
|
|
54
|
+
require_paths:
|
|
55
|
+
- lib
|
|
56
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
57
|
+
requirements:
|
|
58
|
+
- - ">="
|
|
59
|
+
- !ruby/object:Gem::Version
|
|
60
|
+
version: '3.2'
|
|
61
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
62
|
+
requirements:
|
|
63
|
+
- - ">="
|
|
64
|
+
- !ruby/object:Gem::Version
|
|
65
|
+
version: '0'
|
|
66
|
+
requirements: []
|
|
67
|
+
rubygems_version: 3.4.10
|
|
68
|
+
signing_key:
|
|
69
|
+
specification_version: 4
|
|
70
|
+
summary: OTP-style supervision trees for Rails processes
|
|
71
|
+
test_files: []
|