@cat-factory/cli 0.8.5 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -0
- package/dist/args.d.ts +21 -3
- package/dist/args.d.ts.map +1 -1
- package/dist/args.js +117 -0
- package/dist/args.js.map +1 -1
- package/dist/bin.js +5 -0
- package/dist/bin.js.map +1 -1
- package/dist/execution.js +1 -1
- package/dist/execution.js.map +1 -1
- package/dist/host-shell.d.ts +6 -0
- package/dist/host-shell.d.ts.map +1 -1
- package/dist/host-shell.js +1 -1
- package/dist/host-shell.js.map +1 -1
- package/dist/supervise-k3s.d.ts +50 -0
- package/dist/supervise-k3s.d.ts.map +1 -0
- package/dist/supervise-k3s.js +130 -0
- package/dist/supervise-k3s.js.map +1 -0
- package/dist/supervise-runtime.d.ts +169 -0
- package/dist/supervise-runtime.d.ts.map +1 -0
- package/dist/supervise-runtime.js +400 -0
- package/dist/supervise-runtime.js.map +1 -0
- package/dist/supervise.d.ts +147 -0
- package/dist/supervise.d.ts.map +1 -0
- package/dist/supervise.js +130 -0
- package/dist/supervise.js.map +1 -0
- package/dist/superviseCommand.d.ts +25 -0
- package/dist/superviseCommand.d.ts.map +1 -0
- package/dist/superviseCommand.js +136 -0
- package/dist/superviseCommand.js.map +1 -0
- package/package.json +5 -4
|
@@ -0,0 +1,400 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Effects half of `cat-factory supervise`: the health probe, the optional container dependency,
|
|
3
|
+
* the port reaper, the supervised child, and the loop that drives them from the pure decisions in
|
|
4
|
+
* `supervise.ts`.
|
|
5
|
+
*
|
|
6
|
+
* Everything the loop touches is behind a seam so `runSupervisor` can be driven by fakes — the
|
|
7
|
+
* same discipline `host-shell.ts` sets out for the k3s flow. Shell-outs go through {@link HostShell}
|
|
8
|
+
* rather than `node:child_process` directly; the one exception is the supervised child itself,
|
|
9
|
+
* which needs inherited stdio and a live handle, so it gets its own {@link ChildLauncher} seam.
|
|
10
|
+
*/
|
|
11
|
+
import { spawn } from 'node:child_process';
|
|
12
|
+
import http from 'node:http';
|
|
13
|
+
import net from 'node:net';
|
|
14
|
+
import { COMMAND_NOT_FOUND } from './host-shell.js';
|
|
15
|
+
import { initialState, stateAfterStart, step } from './supervise.js';
|
|
16
|
+
/**
|
|
17
|
+
* Thrown by a dependency that cannot be repaired without a human. The loop prints the message ONCE
|
|
18
|
+
* and stops treating the step as retryable noise — the alternative is a watchdog that repeats a
|
|
19
|
+
* hopeless action forever, which is the exact pathology (a restart loop that reads as progress)
|
|
20
|
+
* this supervisor exists to end.
|
|
21
|
+
*/
|
|
22
|
+
export class OperatorActionRequiredError extends Error {
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* NOTE the deliberately un-`unref`'d timers throughout this module. An `unref`'d timer does not keep
|
|
26
|
+
* the event loop alive, and while a spawned child DOES hold it open, that reference vanishes the
|
|
27
|
+
* moment the child dies — which is precisely when the supervisor must keep running. With `unref` the
|
|
28
|
+
* poll timer was then the only thing left, so Node exited 0 the instant its child was killed: a
|
|
29
|
+
* watchdog that died with its patient, silently and with a success code.
|
|
30
|
+
*/
|
|
31
|
+
export const systemClock = {
|
|
32
|
+
now: () => Date.now(),
|
|
33
|
+
sleep: (ms, signal) => new Promise((resolve) => {
|
|
34
|
+
if (signal?.aborted === true) {
|
|
35
|
+
resolve();
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
const timer = setTimeout(() => {
|
|
39
|
+
signal?.removeEventListener('abort', onAbort);
|
|
40
|
+
resolve();
|
|
41
|
+
}, ms);
|
|
42
|
+
// Without this a Ctrl-C would sit out the remainder of the poll interval before the loop
|
|
43
|
+
// noticed, so the shutdown that is supposed to reap the child takes up to `--poll` seconds.
|
|
44
|
+
function onAbort() {
|
|
45
|
+
clearTimeout(timer);
|
|
46
|
+
resolve();
|
|
47
|
+
}
|
|
48
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
49
|
+
}),
|
|
50
|
+
};
|
|
51
|
+
const PROBE_CONNECT_TIMEOUT_MS = 2_000;
|
|
52
|
+
const PROBE_HTTP_TIMEOUT_MS = 3_000;
|
|
53
|
+
/**
|
|
54
|
+
* The real probe. "Serving" requires BOTH signals, because the two failure modes differ: a parked
|
|
55
|
+
* `node --watch` leaves nothing bound to the port, while a server that booted but wedged (or lost
|
|
56
|
+
* its DB pool) still holds the socket and only fails the HTTP check.
|
|
57
|
+
*
|
|
58
|
+
* Both address families are tried — a Node server on `0.0.0.0` answers on 127.0.0.1, but some dev
|
|
59
|
+
* servers bind IPv6 `::1` only, and probing one family would report a false outage.
|
|
60
|
+
*/
|
|
61
|
+
export function createHealthProbe(opts) {
|
|
62
|
+
const hosts = ['127.0.0.1', '::1'];
|
|
63
|
+
const connect = (host) => new Promise((resolve) => {
|
|
64
|
+
const socket = net.connect({ host, port: opts.port, timeout: PROBE_CONNECT_TIMEOUT_MS });
|
|
65
|
+
const done = (result) => {
|
|
66
|
+
socket.destroy();
|
|
67
|
+
resolve(result);
|
|
68
|
+
};
|
|
69
|
+
socket.once('connect', () => done(true));
|
|
70
|
+
socket.once('error', () => done(false));
|
|
71
|
+
socket.once('timeout', () => done(false));
|
|
72
|
+
});
|
|
73
|
+
const healthy = (host) => new Promise((resolve) => {
|
|
74
|
+
const req = http.get({ host, port: opts.port, path: opts.healthPath, timeout: PROBE_HTTP_TIMEOUT_MS }, (res) => {
|
|
75
|
+
res.resume(); // drain, so the socket can close
|
|
76
|
+
resolve(res.statusCode === 200);
|
|
77
|
+
});
|
|
78
|
+
req.once('error', () => resolve(false));
|
|
79
|
+
req.once('timeout', () => {
|
|
80
|
+
req.destroy();
|
|
81
|
+
resolve(false);
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
return {
|
|
85
|
+
async serving() {
|
|
86
|
+
const listening = await Promise.all(hosts.map(connect));
|
|
87
|
+
if (!listening.some(Boolean))
|
|
88
|
+
return false;
|
|
89
|
+
const answers = await Promise.all(hosts.map(healthy));
|
|
90
|
+
return answers.some(Boolean);
|
|
91
|
+
},
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
const COMPOSE_READY_TIMEOUT_MS = 90_000;
|
|
95
|
+
const COMPOSE_POLL_MS = 2_000;
|
|
96
|
+
/**
|
|
97
|
+
* A `docker compose` service the supervised process needs. This is the piece that makes recovery
|
|
98
|
+
* work after the container engine itself restarted: the example compose files set no restart
|
|
99
|
+
* policy on Postgres, so anything that stops the engine leaves the DB down, and relaunching the
|
|
100
|
+
* server against a missing (or still-initialising) database just crashes it again in `migrate`.
|
|
101
|
+
*/
|
|
102
|
+
export function createComposeDependency(shell, opts) {
|
|
103
|
+
const readyTimeoutMs = opts.readyTimeoutMs ?? COMPOSE_READY_TIMEOUT_MS;
|
|
104
|
+
const readyPollMs = opts.readyPollMs ?? COMPOSE_POLL_MS;
|
|
105
|
+
const inspectReady = async (id) => {
|
|
106
|
+
const format = '{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}';
|
|
107
|
+
const result = await shell.run('docker', ['inspect', id, '--format', format], { cwd: opts.dir });
|
|
108
|
+
if (result.code !== 0)
|
|
109
|
+
return false;
|
|
110
|
+
const [status, health] = result.stdout.trim().split('|');
|
|
111
|
+
// A service with no healthcheck configured can only be judged by `running`.
|
|
112
|
+
return status === 'running' && (health === 'healthy' || health === 'none');
|
|
113
|
+
};
|
|
114
|
+
return {
|
|
115
|
+
label: `${opts.service} (docker compose)`,
|
|
116
|
+
async ensure() {
|
|
117
|
+
const up = await shell.run('docker', ['compose', 'up', '-d', opts.service], {
|
|
118
|
+
timeoutMs: 60_000,
|
|
119
|
+
cwd: opts.dir,
|
|
120
|
+
});
|
|
121
|
+
if (up.code !== 0)
|
|
122
|
+
return false;
|
|
123
|
+
const deadline = Date.now() + readyTimeoutMs;
|
|
124
|
+
while (Date.now() < deadline) {
|
|
125
|
+
const ps = await shell.run('docker', ['compose', 'ps', '-q', opts.service], {
|
|
126
|
+
cwd: opts.dir,
|
|
127
|
+
});
|
|
128
|
+
const id = ps.stdout.trim().split(/\r?\n/).filter(Boolean)[0];
|
|
129
|
+
if (id && (await inspectReady(id)))
|
|
130
|
+
return true;
|
|
131
|
+
await new Promise((resolve) => {
|
|
132
|
+
setTimeout(resolve, readyPollMs);
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
return false;
|
|
136
|
+
},
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Frees the port before a restart. Killing the child tree usually suffices, but not always: a
|
|
141
|
+
* package-manager wrapper that is killed without its subtree leaves the real `node` orphaned and
|
|
142
|
+
* still holding the socket, and the relaunch then dies with `EADDRINUSE` — turning one outage into
|
|
143
|
+
* a restart loop. Reaping by PORT is the only check that covers an orphan we never had a handle on.
|
|
144
|
+
*
|
|
145
|
+
* It is also, unavoidably, the bluntest thing this supervisor does: reaping by port means SIGKILLing
|
|
146
|
+
* a process we were never handed. If `--port` names a port some unrelated service owns, that service
|
|
147
|
+
* is what dies. There is no portable way to prove descent from our own child, so the mitigation is
|
|
148
|
+
* disclosure rather than detection — every kill NAMES the pid and, where the platform will tell us,
|
|
149
|
+
* the command behind it, and `reap()` reports what it killed so the caller can log it. Callers only
|
|
150
|
+
* ever reap AFTER their own child is confirmed dead, so a healthy stack is never a candidate.
|
|
151
|
+
*/
|
|
152
|
+
export function createPortReaper(shell, port, opts = {}) {
|
|
153
|
+
const isWindows = (opts.platform ?? process.platform) === 'win32';
|
|
154
|
+
const log = opts.log ?? (() => { });
|
|
155
|
+
/** Best-effort "what IS this pid", so a surprising kill is at least explicable after the fact. */
|
|
156
|
+
const describe = async (pid) => {
|
|
157
|
+
const result = isWindows
|
|
158
|
+
? await shell.run('tasklist', ['/FI', `PID eq ${pid}`, '/NH', '/FO', 'CSV'])
|
|
159
|
+
: await shell.run('ps', ['-p', pid, '-o', 'command=']);
|
|
160
|
+
if (result.code !== 0)
|
|
161
|
+
return `pid ${pid}`;
|
|
162
|
+
const text = result.stdout.trim().split(/\r?\n/)[0]?.trim();
|
|
163
|
+
return text ? `pid ${pid} (${text})` : `pid ${pid}`;
|
|
164
|
+
};
|
|
165
|
+
const listenerPids = async () => {
|
|
166
|
+
if (isWindows) {
|
|
167
|
+
// Plain `netstat -ano` (not `-p tcp`, which is IPv4-only) so an IPv6-only listener is seen.
|
|
168
|
+
const result = await shell.run('netstat', ['-ano']);
|
|
169
|
+
if (result.code !== 0) {
|
|
170
|
+
log(`⚠ cannot check port ${port}: netstat failed — an orphaned listener will not be reaped`);
|
|
171
|
+
return [];
|
|
172
|
+
}
|
|
173
|
+
const pids = new Set();
|
|
174
|
+
for (const line of result.stdout.split(/\r?\n/)) {
|
|
175
|
+
const match = line.match(/^\s*TCP\s+\S+:(\d+)\s+\S+\s+LISTENING\s+(\d+)/i);
|
|
176
|
+
if (match && Number(match[1]) === port)
|
|
177
|
+
pids.add(match[2]);
|
|
178
|
+
}
|
|
179
|
+
return [...pids];
|
|
180
|
+
}
|
|
181
|
+
const result = await shell.run('lsof', ['-ti', `tcp:${port}`, '-sTCP:LISTEN']);
|
|
182
|
+
// `lsof` exits 1 for "nothing matched", which is the common case and not worth a word. A MISSING
|
|
183
|
+
// lsof is different: it is not installed by default on many Linux images, so the reaper silently
|
|
184
|
+
// becomes a no-op and the EADDRINUSE restart loop it exists to prevent comes back unexplained.
|
|
185
|
+
if (result.code === COMMAND_NOT_FOUND) {
|
|
186
|
+
log(`⚠ cannot check port ${port}: lsof is not installed — an orphaned listener holding the ` +
|
|
187
|
+
'port will not be reaped, so a restart may fail with EADDRINUSE');
|
|
188
|
+
return [];
|
|
189
|
+
}
|
|
190
|
+
if (result.code !== 0)
|
|
191
|
+
return [];
|
|
192
|
+
return [...new Set(result.stdout.split(/\s+/).filter(Boolean))];
|
|
193
|
+
};
|
|
194
|
+
return {
|
|
195
|
+
async reap() {
|
|
196
|
+
const killed = [];
|
|
197
|
+
for (const pid of await listenerPids()) {
|
|
198
|
+
log(`↯ port ${port} is still held by ${await describe(pid)} — killing it`);
|
|
199
|
+
// `/T` (Windows) kills the descendants too; elsewhere the group is signalled by the caller.
|
|
200
|
+
if (isWindows)
|
|
201
|
+
await shell.run('taskkill', ['/PID', pid, '/F', '/T']);
|
|
202
|
+
else
|
|
203
|
+
await shell.run('kill', ['-9', pid]);
|
|
204
|
+
killed.push(pid);
|
|
205
|
+
}
|
|
206
|
+
return killed;
|
|
207
|
+
},
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* Launches the supervised command through a shell, with stdio inherited so its logs stay visible —
|
|
212
|
+
* the supervisor is meant to be a transparent wrapper, not a log proxy.
|
|
213
|
+
*
|
|
214
|
+
* The command is passed as ONE string with `shell: true` (rather than a command plus an args array,
|
|
215
|
+
* which trips Node's DEP0190) so a package-manager entry point resolves through its Windows `.cmd`
|
|
216
|
+
* shim. On POSIX the child gets its own process group, so killing it takes the whole tree.
|
|
217
|
+
*/
|
|
218
|
+
export function createChildLauncher(opts) {
|
|
219
|
+
const isWindows = process.platform === 'win32';
|
|
220
|
+
return {
|
|
221
|
+
start() {
|
|
222
|
+
const child = spawn(opts.command, {
|
|
223
|
+
cwd: opts.cwd,
|
|
224
|
+
stdio: 'inherit',
|
|
225
|
+
shell: true,
|
|
226
|
+
detached: !isWindows,
|
|
227
|
+
});
|
|
228
|
+
const exited = new Promise((resolve) => {
|
|
229
|
+
child.once('exit', (code, signal) => resolve({ code, signal }));
|
|
230
|
+
child.once('error', () => resolve({ code: null, signal: null }));
|
|
231
|
+
});
|
|
232
|
+
return {
|
|
233
|
+
pid: child.pid,
|
|
234
|
+
exited,
|
|
235
|
+
async kill() {
|
|
236
|
+
if (child.pid === undefined || child.exitCode !== null)
|
|
237
|
+
return;
|
|
238
|
+
try {
|
|
239
|
+
if (isWindows) {
|
|
240
|
+
// No POSIX process groups on Windows; `taskkill /T` is what reaches the subtree.
|
|
241
|
+
spawn('taskkill', ['/PID', String(child.pid), '/F', '/T'], {
|
|
242
|
+
stdio: 'ignore',
|
|
243
|
+
}).unref();
|
|
244
|
+
}
|
|
245
|
+
else {
|
|
246
|
+
process.kill(-child.pid, 'SIGKILL');
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
catch (err) {
|
|
250
|
+
// silent-catch-ok: the child had already exited between the guard above and the signal
|
|
251
|
+
// (ESRCH/EPERM). That is the outcome kill() is asking for, so there is nothing to report.
|
|
252
|
+
void err;
|
|
253
|
+
}
|
|
254
|
+
await Promise.race([
|
|
255
|
+
exited,
|
|
256
|
+
new Promise((resolve) => {
|
|
257
|
+
setTimeout(resolve, 2_000);
|
|
258
|
+
}),
|
|
259
|
+
]);
|
|
260
|
+
},
|
|
261
|
+
};
|
|
262
|
+
},
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
/**
|
|
266
|
+
* Run every dependency's `ensure`, logging each outcome. Returns the labels of any that need
|
|
267
|
+
* operator action, whose guidance is printed only the first time so a long run doesn't bury it in
|
|
268
|
+
* repeats. A blocked dependency does NOT abort the ladder: the supervised process is often still
|
|
269
|
+
* worth restarting (a dead cluster breaks environment provisioning, not the whole backend).
|
|
270
|
+
*/
|
|
271
|
+
async function ensureDependencies(dependencies, log, warned) {
|
|
272
|
+
const blocked = [];
|
|
273
|
+
for (const dependency of dependencies) {
|
|
274
|
+
try {
|
|
275
|
+
const ready = await dependency.ensure();
|
|
276
|
+
log(ready
|
|
277
|
+
? `✔ ${dependency.label} is ready`
|
|
278
|
+
: `✖ ${dependency.label} is not ready — will retry next cycle`);
|
|
279
|
+
}
|
|
280
|
+
catch (err) {
|
|
281
|
+
if (!(err instanceof OperatorActionRequiredError))
|
|
282
|
+
throw err;
|
|
283
|
+
blocked.push(dependency.label);
|
|
284
|
+
if (!warned.has(dependency.label)) {
|
|
285
|
+
warned.add(dependency.label);
|
|
286
|
+
log(`✖ ${dependency.label} NEEDS YOU: ${err.message}`);
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
return blocked;
|
|
291
|
+
}
|
|
292
|
+
const RESTART_SETTLE_MS = 1_500;
|
|
293
|
+
/**
|
|
294
|
+
* Run the supervision loop: start the child, then probe on an interval and repair when the
|
|
295
|
+
* decisions in `supervise.ts` say so. Returns when `stopSignal` aborts, when the crash-loop budget
|
|
296
|
+
* is spent, or when `maxTicks` is reached (tests) — in production, otherwise never.
|
|
297
|
+
*/
|
|
298
|
+
export async function runSupervisor(deps) {
|
|
299
|
+
const clock = deps.clock ?? systemClock;
|
|
300
|
+
const log = deps.log ?? ((message) => process.stdout.write(`${message}\n`));
|
|
301
|
+
const { config, stopSignal } = deps;
|
|
302
|
+
let repairs = 0;
|
|
303
|
+
let ticks = 0;
|
|
304
|
+
let blocked = [];
|
|
305
|
+
let gaveUp;
|
|
306
|
+
const warned = new Set();
|
|
307
|
+
// A child that has exited is a fact the probe can only infer, slowly. Tracked per generation so a
|
|
308
|
+
// dead PREDECESSOR's late `exited` can never be read as the current child having died.
|
|
309
|
+
let generation = 0;
|
|
310
|
+
let childExited = false;
|
|
311
|
+
const startChild = () => {
|
|
312
|
+
const mine = ++generation;
|
|
313
|
+
childExited = false;
|
|
314
|
+
const started = deps.launcher.start();
|
|
315
|
+
void started.exited.then(() => {
|
|
316
|
+
if (mine === generation)
|
|
317
|
+
childExited = true;
|
|
318
|
+
});
|
|
319
|
+
return started;
|
|
320
|
+
};
|
|
321
|
+
let child = startChild();
|
|
322
|
+
let state = initialState(clock.now(), config);
|
|
323
|
+
// Restarts that have not yet produced a serving stack. Reset by any successful probe, so this
|
|
324
|
+
// counts a genuine crash loop rather than a long-lived stack that has been repaired often.
|
|
325
|
+
let failedStarts = 0;
|
|
326
|
+
const restart = async () => {
|
|
327
|
+
await child.kill();
|
|
328
|
+
await clock.sleep(RESTART_SETTLE_MS);
|
|
329
|
+
// Reap AFTER killing the tree: this only catches an orphan the tree kill could not reach. Our
|
|
330
|
+
// own child is dead by now, so anything still on the port is by definition not it.
|
|
331
|
+
if (deps.reaper)
|
|
332
|
+
await deps.reaper.reap();
|
|
333
|
+
child = startChild();
|
|
334
|
+
state = stateAfterStart(clock.now(), config);
|
|
335
|
+
};
|
|
336
|
+
// Read through a function, not inline: `signal.aborted` flips underneath us, and a direct
|
|
337
|
+
// comparison in the loop condition would let the compiler narrow it to `false` for the body.
|
|
338
|
+
const stopRequested = () => stopSignal?.aborted === true;
|
|
339
|
+
while (!stopRequested() &&
|
|
340
|
+
gaveUp === undefined &&
|
|
341
|
+
(deps.maxTicks === undefined || ticks < deps.maxTicks)) {
|
|
342
|
+
await clock.sleep(config.pollMs, stopSignal);
|
|
343
|
+
if (stopRequested())
|
|
344
|
+
break;
|
|
345
|
+
ticks += 1;
|
|
346
|
+
// Sampled BEFORE the probe: the probe can take seconds against a filtered port, and folding
|
|
347
|
+
// that into the drift would read as a suspend (see `clockJumpMs`).
|
|
348
|
+
const now = clock.now();
|
|
349
|
+
const serving = await deps.probe.serving();
|
|
350
|
+
if (serving)
|
|
351
|
+
failedStarts = 0;
|
|
352
|
+
const next = step(state, { now, serving, childExited }, config);
|
|
353
|
+
state = next.state;
|
|
354
|
+
const { action } = next;
|
|
355
|
+
switch (action.kind) {
|
|
356
|
+
case 'serving':
|
|
357
|
+
case 'grace':
|
|
358
|
+
break;
|
|
359
|
+
case 'resumed':
|
|
360
|
+
log(`✔ resumed after ${Math.round(action.driftMs / 1000)}s — the stack is still serving`);
|
|
361
|
+
break;
|
|
362
|
+
case 'recovered':
|
|
363
|
+
log(`✔ serving again (after ${action.afterFailures} failed probe(s))`);
|
|
364
|
+
break;
|
|
365
|
+
case 'counting':
|
|
366
|
+
log(`• health probe failed (${action.failures}/${action.threshold})`);
|
|
367
|
+
break;
|
|
368
|
+
case 'repair': {
|
|
369
|
+
repairs += 1;
|
|
370
|
+
failedStarts += 1;
|
|
371
|
+
log(`⚠ not serving — ${action.reason}; repair #${repairs}`);
|
|
372
|
+
if (failedStarts > config.maxFailedStarts) {
|
|
373
|
+
// Reported, not retried — the same rule the wedged-cgroup path follows. A command that has
|
|
374
|
+
// never once served is not going to start serving because we killed it again.
|
|
375
|
+
gaveUp =
|
|
376
|
+
`the supervised command has failed to serve ${failedStarts} starts in a row. ` +
|
|
377
|
+
'Restarting cannot fix a command that is broken — run it directly to see why ' +
|
|
378
|
+
'(in deploy/local: `pnpm dev:raw`).';
|
|
379
|
+
log(`✖ GIVING UP: ${gaveUp}`);
|
|
380
|
+
break;
|
|
381
|
+
}
|
|
382
|
+
if (deps.dependencies?.length) {
|
|
383
|
+
blocked = await ensureDependencies(deps.dependencies, log, warned);
|
|
384
|
+
}
|
|
385
|
+
await restart();
|
|
386
|
+
log('↻ restarted the supervised command');
|
|
387
|
+
break;
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
// Shutdown is the loop's job because the loop owns the child handle.
|
|
392
|
+
await child.kill();
|
|
393
|
+
if (deps.reaper) {
|
|
394
|
+
const killed = await deps.reaper.reap();
|
|
395
|
+
if (killed.length > 0)
|
|
396
|
+
log(`↯ reaped ${killed.length} orphaned listener(s) on shutdown`);
|
|
397
|
+
}
|
|
398
|
+
return { ticks, repairs, blocked, gaveUp };
|
|
399
|
+
}
|
|
400
|
+
//# sourceMappingURL=supervise-runtime.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"supervise-runtime.js","sourceRoot":"","sources":["../src/supervise-runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AAC1C,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,GAAG,MAAM,UAAU,CAAA;AAC1B,OAAO,EAAE,iBAAiB,EAAkB,MAAM,iBAAiB,CAAA;AACnE,OAAO,EAAE,YAAY,EAAwB,eAAe,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAA;AAkB1F;;;;;GAKG;AACH,MAAM,OAAO,2BAA4B,SAAQ,KAAK;CAAG;AA6BzD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,WAAW,GAAmB;IACzC,GAAG,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE;IACrB,KAAK,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CACpB,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QACtB,IAAI,MAAM,EAAE,OAAO,KAAK,IAAI,EAAE,CAAC;YAC7B,OAAO,EAAE,CAAA;YACT,OAAM;QACR,CAAC;QACD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAC7C,OAAO,EAAE,CAAA;QACX,CAAC,EAAE,EAAE,CAAC,CAAA;QACN,yFAAyF;QACzF,4FAA4F;QAC5F,SAAS,OAAO;YACd,YAAY,CAAC,KAAK,CAAC,CAAA;YACnB,OAAO,EAAE,CAAA;QACX,CAAC;QACD,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;IAC5D,CAAC,CAAC;CACL,CAAA;AAED,MAAM,wBAAwB,GAAG,KAAK,CAAA;AACtC,MAAM,qBAAqB,GAAG,KAAK,CAAA;AAEnC;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAA0C;IAC1E,MAAM,KAAK,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,CAAA;IAElC,MAAM,OAAO,GAAG,CAAC,IAAY,EAAoB,EAAE,CACjD,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QACtB,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,wBAAwB,EAAE,CAAC,CAAA;QACxF,MAAM,IAAI,GAAG,CAAC,MAAe,EAAQ,EAAE;YACrC,MAAM,CAAC,OAAO,EAAE,CAAA;YAChB,OAAO,CAAC,MAAM,CAAC,CAAA;QACjB,CAAC,CAAA;QACD,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;QACxC,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;QACvC,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;IAC3C,CAAC,CAAC,CAAA;IAEJ,MAAM,OAAO,GAAG,CAAC,IAAY,EAAoB,EAAE,CACjD,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QACtB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAClB,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,qBAAqB,EAAE,EAChF,CAAC,GAAG,EAAE,EAAE;YACN,GAAG,CAAC,MAAM,EAAE,CAAA,CAAC,iCAAiC;YAC9C,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,GAAG,CAAC,CAAA;QACjC,CAAC,CACF,CAAA;QACD,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;QACvC,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,EAAE;YACvB,GAAG,CAAC,OAAO,EAAE,CAAA;YACb,OAAO,CAAC,KAAK,CAAC,CAAA;QAChB,CAAC,CAAC,CAAA;IACJ,CAAC,CAAC,CAAA;IAEJ,OAAO;QACL,KAAK,CAAC,OAAO;YACX,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;YACvD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC;gBAAE,OAAO,KAAK,CAAA;YAC1C,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAA;YACrD,OAAO,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC9B,CAAC;KACF,CAAA;AACH,CAAC;AAED,MAAM,wBAAwB,GAAG,MAAM,CAAA;AACvC,MAAM,eAAe,GAAG,KAAK,CAAA;AAE7B;;;;;GAKG;AACH,MAAM,UAAU,uBAAuB,CACrC,KAAgB,EAChB,IAYC;IAED,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,wBAAwB,CAAA;IACtE,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,eAAe,CAAA;IAEvD,MAAM,YAAY,GAAG,KAAK,EAAE,EAAU,EAAoB,EAAE;QAC1D,MAAM,MAAM,GACV,mFAAmF,CAAA;QACrF,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,SAAS,EAAE,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;QAChG,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,KAAK,CAAA;QACnC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QACxD,4EAA4E;QAC5E,OAAO,MAAM,KAAK,SAAS,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,MAAM,CAAC,CAAA;IAC5E,CAAC,CAAA;IAED,OAAO;QACL,KAAK,EAAE,GAAG,IAAI,CAAC,OAAO,mBAAmB;QACzC,KAAK,CAAC,MAAM;YACV,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE;gBAC1E,SAAS,EAAE,MAAM;gBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;aACd,CAAC,CAAA;YACF,IAAI,EAAE,CAAC,IAAI,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YAE/B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAA;YAC5C,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;gBAC7B,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE;oBAC1E,GAAG,EAAE,IAAI,CAAC,GAAG;iBACd,CAAC,CAAA;gBACF,MAAM,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC7D,IAAI,EAAE,IAAI,CAAC,MAAM,YAAY,CAAC,EAAE,CAAC,CAAC;oBAAE,OAAO,IAAI,CAAA;gBAC/C,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;oBAC5B,UAAU,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;gBAClC,CAAC,CAAC,CAAA;YACJ,CAAC;YACD,OAAO,KAAK,CAAA;QACd,CAAC;KACF,CAAA;AACH,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,gBAAgB,CAC9B,KAAgB,EAChB,IAAY,EACZ,IAAI,GAA2D,EAAE;IAEjE,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,OAAO,CAAA;IACjE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAS,EAAE,GAAE,CAAC,CAAC,CAAA;IAExC,kGAAkG;IAClG,MAAM,QAAQ,GAAG,KAAK,EAAE,GAAW,EAAmB,EAAE;QACtD,MAAM,MAAM,GAAG,SAAS;YACtB,CAAC,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,KAAK,EAAE,UAAU,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YAC5E,CAAC,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC,CAAA;QACxD,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,OAAO,GAAG,EAAE,CAAA;QAC1C,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAA;QAC3D,OAAO,IAAI,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,OAAO,GAAG,EAAE,CAAA;IACrD,CAAC,CAAA;IAED,MAAM,YAAY,GAAG,KAAK,IAAuB,EAAE;QACjD,IAAI,SAAS,EAAE,CAAC;YACd,4FAA4F;YAC5F,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,CAAC,CAAA;YACnD,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBACtB,GAAG,CAAC,uBAAuB,IAAI,4DAA4D,CAAC,CAAA;gBAC5F,OAAO,EAAE,CAAA;YACX,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;YAC9B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBAChD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAA;gBAC1E,IAAI,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI;oBAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAW,CAAC,CAAA;YACtE,CAAC;YACD,OAAO,CAAC,GAAG,IAAI,CAAC,CAAA;QAClB,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,OAAO,IAAI,EAAE,EAAE,cAAc,CAAC,CAAC,CAAA;QAC9E,iGAAiG;QACjG,iGAAiG;QACjG,+FAA+F;QAC/F,IAAI,MAAM,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;YACtC,GAAG,CACD,uBAAuB,IAAI,6DAA6D;gBACtF,gEAAgE,CACnE,CAAA;YACD,OAAO,EAAE,CAAA;QACX,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,EAAE,CAAA;QAChC,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IACjE,CAAC,CAAA;IAED,OAAO;QACL,KAAK,CAAC,IAAI;YACR,MAAM,MAAM,GAAa,EAAE,CAAA;YAC3B,KAAK,MAAM,GAAG,IAAI,MAAM,YAAY,EAAE,EAAE,CAAC;gBACvC,GAAG,CAAC,UAAU,IAAI,qBAAqB,MAAM,QAAQ,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;gBAC1E,4FAA4F;gBAC5F,IAAI,SAAS;oBAAE,MAAM,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAA;;oBAChE,MAAM,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAA;gBACzC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAClB,CAAC;YACD,OAAO,MAAM,CAAA;QACf,CAAC;KACF,CAAA;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAsC;IACxE,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAA;IAE9C,OAAO;QACL,KAAK;YACH,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE;gBAChC,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,KAAK,EAAE,SAAS;gBAChB,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,CAAC,SAAS;aACrB,CAAC,CAAA;YAEF,MAAM,MAAM,GAAG,IAAI,OAAO,CAAiD,CAAC,OAAO,EAAE,EAAE;gBACrF,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAA;gBAC/D,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAA;YAClE,CAAC,CAAC,CAAA;YAEF,OAAO;gBACL,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,MAAM;gBACN,KAAK,CAAC,IAAI;oBACR,IAAI,KAAK,CAAC,GAAG,KAAK,SAAS,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;wBAAE,OAAM;oBAC9D,IAAI,CAAC;wBACH,IAAI,SAAS,EAAE,CAAC;4BACd,iFAAiF;4BACjF,KAAK,CAAC,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE;gCACzD,KAAK,EAAE,QAAQ;6BAChB,CAAC,CAAC,KAAK,EAAE,CAAA;wBACZ,CAAC;6BAAM,CAAC;4BACN,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;wBACrC,CAAC;oBACH,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,uFAAuF;wBACvF,0FAA0F;wBAC1F,KAAK,GAAG,CAAA;oBACV,CAAC;oBACD,MAAM,OAAO,CAAC,IAAI,CAAC;wBACjB,MAAM;wBACN,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;4BACtB,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;wBAC5B,CAAC,CAAC;qBACH,CAAC,CAAA;gBACJ,CAAC;aACF,CAAA;QACH,CAAC;KACF,CAAA;AACH,CAAC;AAsCD;;;;;GAKG;AACH,KAAK,UAAU,kBAAkB,CAC/B,YAAiC,EACjC,GAA8B,EAC9B,MAAmB;IAEnB,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,KAAK,MAAM,UAAU,IAAI,YAAY,EAAE,CAAC;QACtC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,MAAM,EAAE,CAAA;YACvC,GAAG,CACD,KAAK;gBACH,CAAC,CAAC,KAAK,UAAU,CAAC,KAAK,WAAW;gBAClC,CAAC,CAAC,KAAK,UAAU,CAAC,KAAK,uCAAuC,CACjE,CAAA;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,CAAC,GAAG,YAAY,2BAA2B,CAAC;gBAAE,MAAM,GAAG,CAAA;YAC5D,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;YAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBAClC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAA;gBAC5B,GAAG,CAAC,KAAK,UAAU,CAAC,KAAK,eAAe,GAAG,CAAC,OAAO,EAAE,CAAC,CAAA;YACxD,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,MAAM,iBAAiB,GAAG,KAAK,CAAA;AAE/B;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,IAAoB;IACtD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,WAAW,CAAA;IACvC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,OAAe,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC,CAAA;IACnF,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,IAAI,CAAA;IAEnC,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,OAAO,GAAa,EAAE,CAAA;IAC1B,IAAI,MAA0B,CAAA;IAC9B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAA;IAEhC,kGAAkG;IAClG,uFAAuF;IACvF,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,WAAW,GAAG,KAAK,CAAA;IACvB,MAAM,UAAU,GAAG,GAAoB,EAAE;QACvC,MAAM,IAAI,GAAG,EAAE,UAAU,CAAA;QACzB,WAAW,GAAG,KAAK,CAAA;QACnB,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAA;QACrC,KAAK,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE;YAC5B,IAAI,IAAI,KAAK,UAAU;gBAAE,WAAW,GAAG,IAAI,CAAA;QAC7C,CAAC,CAAC,CAAA;QACF,OAAO,OAAO,CAAA;IAChB,CAAC,CAAA;IAED,IAAI,KAAK,GAAG,UAAU,EAAE,CAAA;IACxB,IAAI,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAA;IAE7C,8FAA8F;IAC9F,2FAA2F;IAC3F,IAAI,YAAY,GAAG,CAAC,CAAA;IAEpB,MAAM,OAAO,GAAG,KAAK,IAAmB,EAAE;QACxC,MAAM,KAAK,CAAC,IAAI,EAAE,CAAA;QAClB,MAAM,KAAK,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAA;QACpC,8FAA8F;QAC9F,mFAAmF;QACnF,IAAI,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;QACzC,KAAK,GAAG,UAAU,EAAE,CAAA;QACpB,KAAK,GAAG,eAAe,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,CAAA;IAC9C,CAAC,CAAA;IAED,0FAA0F;IAC1F,6FAA6F;IAC7F,MAAM,aAAa,GAAG,GAAY,EAAE,CAAC,UAAU,EAAE,OAAO,KAAK,IAAI,CAAA;IAEjE,OACE,CAAC,aAAa,EAAE;QAChB,MAAM,KAAK,SAAS;QACpB,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS,IAAI,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,EACtD,CAAC;QACD,MAAM,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAA;QAC5C,IAAI,aAAa,EAAE;YAAE,MAAK;QAC1B,KAAK,IAAI,CAAC,CAAA;QAEV,4FAA4F;QAC5F,mEAAmE;QACnE,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,CAAA;QACvB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAA;QAC1C,IAAI,OAAO;YAAE,YAAY,GAAG,CAAC,CAAA;QAC7B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,EAAE,MAAM,CAAC,CAAA;QAC/D,KAAK,GAAG,IAAI,CAAC,KAAK,CAAA;QAClB,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAA;QAEvB,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;YACpB,KAAK,SAAS,CAAC;YACf,KAAK,OAAO;gBACV,MAAK;YACP,KAAK,SAAS;gBACZ,GAAG,CAAC,mBAAmB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,gCAAgC,CAAC,CAAA;gBACzF,MAAK;YACP,KAAK,WAAW;gBACd,GAAG,CAAC,0BAA0B,MAAM,CAAC,aAAa,mBAAmB,CAAC,CAAA;gBACtE,MAAK;YACP,KAAK,UAAU;gBACb,GAAG,CAAC,0BAA0B,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,SAAS,GAAG,CAAC,CAAA;gBACrE,MAAK;YACP,KAAK,QAAQ,EAAE,CAAC;gBACd,OAAO,IAAI,CAAC,CAAA;gBACZ,YAAY,IAAI,CAAC,CAAA;gBACjB,GAAG,CAAC,mBAAmB,MAAM,CAAC,MAAM,aAAa,OAAO,EAAE,CAAC,CAAA;gBAC3D,IAAI,YAAY,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;oBAC1C,2FAA2F;oBAC3F,8EAA8E;oBAC9E,MAAM;wBACJ,8CAA8C,YAAY,oBAAoB;4BAC9E,8EAA8E;4BAC9E,oCAAoC,CAAA;oBACtC,GAAG,CAAC,gBAAgB,MAAM,EAAE,CAAC,CAAA;oBAC7B,MAAK;gBACP,CAAC;gBACD,IAAI,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,CAAC;oBAC9B,OAAO,GAAG,MAAM,kBAAkB,CAAC,IAAI,CAAC,YAAY,EAAE,GAAG,EAAE,MAAM,CAAC,CAAA;gBACpE,CAAC;gBACD,MAAM,OAAO,EAAE,CAAA;gBACf,GAAG,CAAC,oCAAoC,CAAC,CAAA;gBACzC,MAAK;YACP,CAAC;QACH,CAAC;IACH,CAAC;IAED,qEAAqE;IACrE,MAAM,KAAK,CAAC,IAAI,EAAE,CAAA;IAClB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAA;QACvC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,GAAG,CAAC,YAAY,MAAM,CAAC,MAAM,mCAAmC,CAAC,CAAA;IAC1F,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAA;AAC5C,CAAC"}
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Decision core for `cat-factory supervise` — the self-healing local-dev supervisor.
|
|
3
|
+
*
|
|
4
|
+
* WHY THIS EXISTS. Every local deployment runs its server under `node --watch`, and
|
|
5
|
+
* `node --watch` PARKS on crash: it restarts the entry only on a FILE CHANGE, never on a process
|
|
6
|
+
* exit. A sleeping laptop is the common trigger — on resume the Postgres/Docker connection is
|
|
7
|
+
* gone, the server dies in `migrate`, and the watcher settles at "Waiting for file changes before
|
|
8
|
+
* restarting". The result is the worst kind of failure: the wrapper PID is still alive and the
|
|
9
|
+
* ready banner has already scrolled past, so the stack LOOKS running while nothing is bound to
|
|
10
|
+
* the port, and the SPA reports only a generic "can't reach backend". It never self-heals, and it
|
|
11
|
+
* stays that way until someone notices and restarts by hand.
|
|
12
|
+
*
|
|
13
|
+
* This module is the JUDGEMENT half of the fix, kept pure — no sockets, no processes, no ambient
|
|
14
|
+
* clock — so every transition is unit-testable from a table of observations (`supervise.test.ts`).
|
|
15
|
+
* `supervise-runtime.ts` owns the effects and feeds observations in. That split is the same one
|
|
16
|
+
* `scripts/silent-catch.mjs` documents for itself: a guard whose judgement nothing tests is a
|
|
17
|
+
* guard that is trusted without evidence.
|
|
18
|
+
*/
|
|
19
|
+
/** Tuning for the supervisor loop. Resolve partial input with {@link resolveSuperviseConfig}. */
|
|
20
|
+
export interface SuperviseConfig {
|
|
21
|
+
/** How often the health probe runs. */
|
|
22
|
+
pollMs: number;
|
|
23
|
+
/**
|
|
24
|
+
* Grace window after a (re)start during which a failed probe does NOT count against the child.
|
|
25
|
+
* A cold boot builds the workspace dependency and runs migrations first, so the port legitimately
|
|
26
|
+
* stays unbound for a while.
|
|
27
|
+
*/
|
|
28
|
+
bootGraceMs: number;
|
|
29
|
+
/** Grace window after a detected resume, so a still-waking Docker/Postgres isn't blamed. */
|
|
30
|
+
resumeGraceMs: number;
|
|
31
|
+
/**
|
|
32
|
+
* A tick arriving this much later than `pollMs` means time jumped — the host slept (or stalled
|
|
33
|
+
* hard). Timers do not fire while suspended, so lateness is the signal.
|
|
34
|
+
*
|
|
35
|
+
* The measurement is deliberately taken tick-START to tick-START (see {@link step}): sampling it
|
|
36
|
+
* after the probe would fold the probe's own duration into the drift, and a probe that times out
|
|
37
|
+
* on a filtered port takes seconds — enough to read as a suspend on a short `--poll` and so to
|
|
38
|
+
* bypass `failureThreshold` entirely.
|
|
39
|
+
*/
|
|
40
|
+
clockJumpMs: number;
|
|
41
|
+
/** Consecutive failed probes required before a repair (outside any grace window). */
|
|
42
|
+
failureThreshold: number;
|
|
43
|
+
/**
|
|
44
|
+
* How many restarts in a row may fail to produce a SERVING stack before the supervisor reports
|
|
45
|
+
* and gives up. A command that is simply broken (a syntax error, a missing binary, a port already
|
|
46
|
+
* owned by something else) can never be fixed by restarting it, and looping forever on it is the
|
|
47
|
+
* exact pathology this supervisor exists to end — the motivating incident was a container that
|
|
48
|
+
* restarted 518 times, exiting 0 each time, while `docker ps` showed healthy motion.
|
|
49
|
+
*/
|
|
50
|
+
maxFailedStarts: number;
|
|
51
|
+
}
|
|
52
|
+
/** Defaults chosen for a laptop-dev loop: notice within ~30s, never fight a cold boot. */
|
|
53
|
+
export declare const SUPERVISE_DEFAULTS: {
|
|
54
|
+
readonly pollMs: 10000;
|
|
55
|
+
readonly bootGraceMs: 60000;
|
|
56
|
+
readonly resumeGraceMs: 25000;
|
|
57
|
+
readonly failureThreshold: 3;
|
|
58
|
+
readonly maxFailedStarts: 5;
|
|
59
|
+
};
|
|
60
|
+
/**
|
|
61
|
+
* Fill in defaults and derive `clockJumpMs` from the poll interval. A tick 3 intervals late is
|
|
62
|
+
* well outside normal scheduler jitter but still catches a short suspend.
|
|
63
|
+
*/
|
|
64
|
+
export declare function resolveSuperviseConfig(partial?: Partial<SuperviseConfig>): SuperviseConfig;
|
|
65
|
+
/** Loop state carried between ticks. Treated as immutable: {@link step} returns the next one. */
|
|
66
|
+
export interface SuperviseState {
|
|
67
|
+
/** Consecutive failed probes so far, reset by any success or repair. */
|
|
68
|
+
failures: number;
|
|
69
|
+
/** No failed probe counts against the child until this timestamp (boot/resume grace). */
|
|
70
|
+
quietUntil: number;
|
|
71
|
+
/** When the previous tick ran — the basis for clock-jump (sleep) detection. */
|
|
72
|
+
lastTickAt: number;
|
|
73
|
+
}
|
|
74
|
+
/** What the runtime should do about this tick. Every branch of {@link step} names one. */
|
|
75
|
+
export type SuperviseAction =
|
|
76
|
+
/** Serving, and it was serving before too — nothing to say. */
|
|
77
|
+
{
|
|
78
|
+
kind: 'serving';
|
|
79
|
+
}
|
|
80
|
+
/** Serving again after one or more failed probes, without needing a repair. */
|
|
81
|
+
| {
|
|
82
|
+
kind: 'recovered';
|
|
83
|
+
afterFailures: number;
|
|
84
|
+
}
|
|
85
|
+
/** Not serving, but inside a boot/resume grace window — wait it out. */
|
|
86
|
+
| {
|
|
87
|
+
kind: 'grace';
|
|
88
|
+
msLeft: number;
|
|
89
|
+
}
|
|
90
|
+
/** Not serving; failure counted but still below the threshold. */
|
|
91
|
+
| {
|
|
92
|
+
kind: 'counting';
|
|
93
|
+
failures: number;
|
|
94
|
+
threshold: number;
|
|
95
|
+
}
|
|
96
|
+
/** Run the recovery ladder: re-check dependencies, then restart the child. */
|
|
97
|
+
| {
|
|
98
|
+
kind: 'repair';
|
|
99
|
+
reason: string;
|
|
100
|
+
}
|
|
101
|
+
/** The host resumed from sleep and the stack is still serving. */
|
|
102
|
+
| {
|
|
103
|
+
kind: 'resumed';
|
|
104
|
+
driftMs: number;
|
|
105
|
+
};
|
|
106
|
+
/** State for a freshly started child: clean counters and a full boot grace window. */
|
|
107
|
+
export declare function initialState(now: number, config: SuperviseConfig): SuperviseState;
|
|
108
|
+
/**
|
|
109
|
+
* State to adopt right after (re)spawning a child mid-run — a fresh boot grace, counters clear.
|
|
110
|
+
*
|
|
111
|
+
* `lastTickAt` is re-based on `now` (the moment the new child started), NOT carried over from the
|
|
112
|
+
* previous tick, because a repair is not instantaneous: it runs the whole dependency ladder first,
|
|
113
|
+
* and those budgets are 90s (compose readiness) and 120s (apiserver readiness) against a default
|
|
114
|
+
* `clockJumpMs` of 30s. Carrying the old timestamp forward makes the very next tick measure the
|
|
115
|
+
* repair's own duration as drift, read a slow-but-successful recovery as a host suspend, and — since
|
|
116
|
+
* resume detection deliberately outranks the boot-grace window — immediately kill the child it just
|
|
117
|
+
* started. Re-basing means the clock-jump signal only ever measures time we were genuinely idle.
|
|
118
|
+
*/
|
|
119
|
+
export declare function stateAfterStart(now: number, config: SuperviseConfig): SuperviseState;
|
|
120
|
+
/**
|
|
121
|
+
* One tick of the supervisor: current state + what we just observed -> next state + the action to
|
|
122
|
+
* take. Pure; the caller supplies `now` and the probe result.
|
|
123
|
+
*
|
|
124
|
+
* `now` must be sampled at the START of the tick, before the probe runs — see `clockJumpMs`.
|
|
125
|
+
*
|
|
126
|
+
* Order matters:
|
|
127
|
+
* 1. The clock-jump check runs FIRST and outranks the grace windows, because a resume is precisely
|
|
128
|
+
* when the stack is most likely already dead — deferring it to the normal threshold path would
|
|
129
|
+
* idle for another `failureThreshold * pollMs` before repairing something we can already tell
|
|
130
|
+
* is broken.
|
|
131
|
+
* 2. A confirmed-serving stack short-circuits everything below it.
|
|
132
|
+
* 3. A child that has EXITED then repairs immediately, ahead of the grace window and the failure
|
|
133
|
+
* counter, because neither can tell us anything a dead process handle hasn't already: counting
|
|
134
|
+
* three more probes against a process that does not exist just adds `failureThreshold * pollMs`
|
|
135
|
+
* of downtime. This is checked only once the stack is known not to be serving, so a wrapper that
|
|
136
|
+
* exits while its grandchild keeps serving (a shell that `exec`s away, say) is left alone
|
|
137
|
+
* rather than having a healthy server restarted out from under it.
|
|
138
|
+
*/
|
|
139
|
+
export declare function step(state: SuperviseState, observation: {
|
|
140
|
+
now: number;
|
|
141
|
+
serving: boolean;
|
|
142
|
+
childExited?: boolean;
|
|
143
|
+
}, config: SuperviseConfig): {
|
|
144
|
+
state: SuperviseState;
|
|
145
|
+
action: SuperviseAction;
|
|
146
|
+
};
|
|
147
|
+
//# sourceMappingURL=supervise.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"supervise.d.ts","sourceRoot":"","sources":["../src/supervise.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,iGAAiG;AACjG,MAAM,WAAW,eAAe;IAC9B,uCAAuC;IACvC,MAAM,EAAE,MAAM,CAAA;IACd;;;;OAIG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB,4FAA4F;IAC5F,aAAa,EAAE,MAAM,CAAA;IACrB;;;;;;;;OAQG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB,qFAAqF;IACrF,gBAAgB,EAAE,MAAM,CAAA;IACxB;;;;;;OAMG;IACH,eAAe,EAAE,MAAM,CAAA;CACxB;AAED,0FAA0F;AAC1F,eAAO,MAAM,kBAAkB;aAC7B,MAAM,EAAE,KAAM;aACd,WAAW,EAAE,KAAM;aACnB,aAAa,EAAE,KAAM;aACrB,gBAAgB,EAAE,CAAC;aACnB,eAAe,EAAE,CAAC;CACV,CAAA;AAEV;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,GAAE,OAAO,CAAC,eAAe,CAAM,GAAG,eAAe,CAU9F;AAED,iGAAiG;AACjG,MAAM,WAAW,cAAc;IAC7B,wEAAwE;IACxE,QAAQ,EAAE,MAAM,CAAA;IAChB,yFAAyF;IACzF,UAAU,EAAE,MAAM,CAAA;IAClB,+EAA+E;IAC/E,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,0FAA0F;AAC1F,MAAM,MAAM,eAAe;AACzB,+DAA+D;AAC7D;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE;AACrB,+EAA+E;GAC7E;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAE;AAC9C,wEAAwE;GACtE;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE;AACnC,kEAAkE;GAChE;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE;AAC3D,8EAA8E;GAC5E;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE;AACpC,kEAAkE;GAChE;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAA;AAExC,sFAAsF;AACtF,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,GAAG,cAAc,CAEjF;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,GAAG,cAAc,CAEpF;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,IAAI,CAClB,KAAK,EAAE,cAAc,EACrB,WAAW,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAC;IAAC,WAAW,CAAC,EAAE,OAAO,CAAA;CAAE,EACrE,MAAM,EAAE,eAAe,GACtB;IAAE,KAAK,EAAE,cAAc,CAAC;IAAC,MAAM,EAAE,eAAe,CAAA;CAAE,CAwDpD"}
|