@cat-factory/cli 0.8.5 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -0
- package/dist/args.d.ts +21 -3
- package/dist/args.d.ts.map +1 -1
- package/dist/args.js +117 -0
- package/dist/args.js.map +1 -1
- package/dist/bin.js +5 -0
- package/dist/bin.js.map +1 -1
- package/dist/execution.js +1 -1
- package/dist/execution.js.map +1 -1
- package/dist/host-shell.d.ts +6 -0
- package/dist/host-shell.d.ts.map +1 -1
- package/dist/host-shell.js +1 -1
- package/dist/host-shell.js.map +1 -1
- package/dist/supervise-k3s.d.ts +50 -0
- package/dist/supervise-k3s.d.ts.map +1 -0
- package/dist/supervise-k3s.js +130 -0
- package/dist/supervise-k3s.js.map +1 -0
- package/dist/supervise-runtime.d.ts +169 -0
- package/dist/supervise-runtime.d.ts.map +1 -0
- package/dist/supervise-runtime.js +400 -0
- package/dist/supervise-runtime.js.map +1 -0
- package/dist/supervise.d.ts +147 -0
- package/dist/supervise.d.ts.map +1 -0
- package/dist/supervise.js +130 -0
- package/dist/supervise.js.map +1 -0
- package/dist/superviseCommand.d.ts +25 -0
- package/dist/superviseCommand.d.ts.map +1 -0
- package/dist/superviseCommand.js +136 -0
- package/dist/superviseCommand.js.map +1 -0
- package/package.json +5 -4
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A local Kubernetes cluster as a supervised dependency.
|
|
3
|
+
*
|
|
4
|
+
* WHY. A slept laptop takes the k3s control plane with it, and nothing brings it back. The failure
|
|
5
|
+
* is quiet in the worst way: the container engine's `unless-stopped` policy DOES retry the server
|
|
6
|
+
* node, so `docker ps` shows motion — but the load balancer in front of it boots, fails to resolve
|
|
7
|
+
* its now-missing upstream, exits 0, and is restarted forever. A zero exit code reads as healthy at
|
|
8
|
+
* a glance, so the cluster can sit dead for days while `kubectl` just times out and every
|
|
9
|
+
* environment provisioned through the Local k3s handler fails.
|
|
10
|
+
*
|
|
11
|
+
* WHAT THIS CAN AND CANNOT FIX. A cluster whose containers are merely STOPPED is repairable here:
|
|
12
|
+
* start it, wait for the apiserver. A cluster whose restart is blocked by a wedged cgroup
|
|
13
|
+
* (`runc create failed: ... cgroup.procs: device or resource busy`, the state a suspend can leave
|
|
14
|
+
* behind) is NOT: clearing it requires restarting the container engine itself, which would kill
|
|
15
|
+
* every other container — including the database this same supervisor depends on. So that case is
|
|
16
|
+
* detected and NAMED rather than retried. Looping on it would reproduce exactly the uselessness of
|
|
17
|
+
* the load-balancer restart loop this dependency exists to prevent.
|
|
18
|
+
*/
|
|
19
|
+
import { hasServerVersion, parseK3dClusters, parseKindClusters } from './k3s-probe.js';
|
|
20
|
+
import { contextName } from './k3s-provision.js';
|
|
21
|
+
import { OperatorActionRequiredError } from './supervise-runtime.js';
|
|
22
|
+
const APISERVER_READY_TIMEOUT_MS = 120_000;
|
|
23
|
+
const APISERVER_POLL_MS = 3_000;
|
|
24
|
+
const CLUSTER_START_TIMEOUT_MS = 120_000;
|
|
25
|
+
/**
|
|
26
|
+
* The cgroup-wedge signature. runc reports it when a dead container's cgroup was never torn down,
|
|
27
|
+
* so the new task cannot be created in it — the state a suspend/resume cycle can leave behind, and
|
|
28
|
+
* the one thing a start command can never talk its way out of.
|
|
29
|
+
*/
|
|
30
|
+
export function looksLikeCgroupWedge(output) {
|
|
31
|
+
const text = output.toLowerCase();
|
|
32
|
+
return (text.includes('device or resource busy') ||
|
|
33
|
+
(text.includes('cgroup') && text.includes('unable to apply cgroup configuration')));
|
|
34
|
+
}
|
|
35
|
+
/** The guidance printed when the wedge is hit — the only sequence that actually clears it. */
|
|
36
|
+
export const CGROUP_WEDGE_GUIDANCE = 'the container runtime cannot restart this cluster: a stale cgroup is blocking it ' +
|
|
37
|
+
'(runc: "device or resource busy"). This does NOT clear on its own and no retry will fix it. ' +
|
|
38
|
+
'Restart the container engine (e.g. `docker desktop restart`, or Docker Desktop > Restart), ' +
|
|
39
|
+
'then start the cluster again. Note that this also bounces every other container.';
|
|
40
|
+
/**
|
|
41
|
+
* Build the cluster dependency. `reachable` is judged from the apiserver's OWN version — the only
|
|
42
|
+
* signal that survives every intermediate lie (containers up but apiserver not listening, LB up but
|
|
43
|
+
* upstream missing, kubeconfig present but stale).
|
|
44
|
+
*/
|
|
45
|
+
export function createK3sClusterDependency(shell, opts) {
|
|
46
|
+
const context = contextName(opts.runtime, opts.cluster);
|
|
47
|
+
const readyTimeoutMs = opts.readyTimeoutMs ?? APISERVER_READY_TIMEOUT_MS;
|
|
48
|
+
const readyPollMs = opts.readyPollMs ?? APISERVER_POLL_MS;
|
|
49
|
+
const apiserverReachable = async () => {
|
|
50
|
+
const result = await shell.run('kubectl', [
|
|
51
|
+
'version',
|
|
52
|
+
'--output=json',
|
|
53
|
+
'--request-timeout=5s',
|
|
54
|
+
'--context',
|
|
55
|
+
context,
|
|
56
|
+
]);
|
|
57
|
+
// A down apiserver still yields exit 1 plus the client half of the payload, so the parse — not
|
|
58
|
+
// the exit code — is what decides. `hasServerVersion` is true only when the server answered.
|
|
59
|
+
return hasServerVersion(result.stdout);
|
|
60
|
+
};
|
|
61
|
+
/** Is this cluster known to the runtime (i.e. stopped rather than deleted)? */
|
|
62
|
+
const clusterExists = async () => {
|
|
63
|
+
if (opts.runtime === 'k3d') {
|
|
64
|
+
const result = await shell.run('k3d', ['cluster', 'list', '--output', 'json']);
|
|
65
|
+
if (result.code !== 0)
|
|
66
|
+
return false;
|
|
67
|
+
return parseK3dClusters(result.stdout).includes(opts.cluster);
|
|
68
|
+
}
|
|
69
|
+
const result = await shell.run('kind', ['get', 'clusters']);
|
|
70
|
+
if (result.code !== 0)
|
|
71
|
+
return false;
|
|
72
|
+
return parseKindClusters(result.stdout).includes(opts.cluster);
|
|
73
|
+
};
|
|
74
|
+
/**
|
|
75
|
+
* Start a stopped cluster. k3d has a first-class `cluster start`; kind has none — a kind cluster
|
|
76
|
+
* is just its node containers, so they are started directly by their conventional names.
|
|
77
|
+
*/
|
|
78
|
+
const startCluster = async () => {
|
|
79
|
+
if (opts.runtime === 'k3d') {
|
|
80
|
+
const result = await shell.run('k3d', ['cluster', 'start', opts.cluster], {
|
|
81
|
+
timeoutMs: CLUSTER_START_TIMEOUT_MS,
|
|
82
|
+
});
|
|
83
|
+
return { ok: result.code === 0, output: `${result.stdout}\n${result.stderr}` };
|
|
84
|
+
}
|
|
85
|
+
const result = await shell.run('docker', ['start', `${opts.cluster}-control-plane`], {
|
|
86
|
+
timeoutMs: CLUSTER_START_TIMEOUT_MS,
|
|
87
|
+
});
|
|
88
|
+
return { ok: result.code === 0, output: `${result.stdout}\n${result.stderr}` };
|
|
89
|
+
};
|
|
90
|
+
const waitForApiserver = async () => {
|
|
91
|
+
const deadline = Date.now() + readyTimeoutMs;
|
|
92
|
+
while (Date.now() < deadline) {
|
|
93
|
+
if (await apiserverReachable())
|
|
94
|
+
return true;
|
|
95
|
+
// Not `unref`'d — see the note on `systemClock`: an unref'd timer lets the process exit while
|
|
96
|
+
// this wait is the only thing outstanding, which is exactly mid-recovery.
|
|
97
|
+
await new Promise((resolve) => {
|
|
98
|
+
setTimeout(resolve, readyPollMs);
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
return false;
|
|
102
|
+
};
|
|
103
|
+
return {
|
|
104
|
+
label: `${opts.runtime} cluster "${opts.cluster}"`,
|
|
105
|
+
async ensure() {
|
|
106
|
+
if (await apiserverReachable())
|
|
107
|
+
return true;
|
|
108
|
+
// Absent from the runtime's list ⇒ deleted, not stopped. Creating one here would be a
|
|
109
|
+
// surprise (it owns RBAC + a service account); `cat-factory k3s` is the deliberate path.
|
|
110
|
+
if (!(await clusterExists()))
|
|
111
|
+
return false;
|
|
112
|
+
const started = await startCluster();
|
|
113
|
+
if (!started.ok) {
|
|
114
|
+
if (looksLikeCgroupWedge(started.output)) {
|
|
115
|
+
throw new K3sWedgedError(CGROUP_WEDGE_GUIDANCE);
|
|
116
|
+
}
|
|
117
|
+
return false;
|
|
118
|
+
}
|
|
119
|
+
return await waitForApiserver();
|
|
120
|
+
},
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Thrown when the cluster cannot be restarted without operator action. Distinct from a plain
|
|
125
|
+
* `false` (not ready, retry next cycle) precisely so the loop can stop repeating a hopeless step
|
|
126
|
+
* and surface the fix instead.
|
|
127
|
+
*/
|
|
128
|
+
export class K3sWedgedError extends OperatorActionRequiredError {
|
|
129
|
+
}
|
|
130
|
+
//# sourceMappingURL=supervise-k3s.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"supervise-k3s.js","sourceRoot":"","sources":["../src/supervise-k3s.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAGH,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AACtF,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAChD,OAAO,EAAE,2BAA2B,EAA0B,MAAM,wBAAwB,CAAA;AAK5F,MAAM,0BAA0B,GAAG,OAAO,CAAA;AAC1C,MAAM,iBAAiB,GAAG,KAAK,CAAA;AAC/B,MAAM,wBAAwB,GAAG,OAAO,CAAA;AAExC;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAc;IACjD,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,EAAE,CAAA;IACjC,OAAO,CACL,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC;QACxC,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,sCAAsC,CAAC,CAAC,CACnF,CAAA;AACH,CAAC;AAED,8FAA8F;AAC9F,MAAM,CAAC,MAAM,qBAAqB,GAChC,mFAAmF;IACnF,8FAA8F;IAC9F,6FAA6F;IAC7F,kFAAkF,CAAA;AAEpF;;;;GAIG;AACH,MAAM,UAAU,0BAA0B,CACxC,KAAgB,EAChB,IAMC;IAED,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;IACvD,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,0BAA0B,CAAA;IACxE,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,iBAAiB,CAAA;IAEzD,MAAM,kBAAkB,GAAG,KAAK,IAAsB,EAAE;QACtD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,SAAS,EAAE;YACxC,SAAS;YACT,eAAe;YACf,sBAAsB;YACtB,WAAW;YACX,OAAO;SACR,CAAC,CAAA;QACF,+FAA+F;QAC/F,6FAA6F;QAC7F,OAAO,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IACxC,CAAC,CAAA;IAED,+EAA+E;IAC/E,MAAM,aAAa,GAAG,KAAK,IAAsB,EAAE;QACjD,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC,CAAA;YAC9E,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;gBAAE,OAAO,KAAK,CAAA;YACnC,OAAO,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAC/D,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CAAA;QAC3D,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO,KAAK,CAAA;QACnC,OAAO,iBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAChE,CAAC,CAAA;IAED;;;OAGG;IACH,MAAM,YAAY,GAAG,KAAK,IAA8C,EAAE;QACxE,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE;gBACxE,SAAS,EAAE,wBAAwB;aACpC,CAAC,CAAA;YACF,OAAO,EAAE,EAAE,EAAE,MAAM,CAAC,IAAI,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,EAAE,CAAA;QAChF,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,gBAAgB,CAAC,EAAE;YACnF,SAAS,EAAE,wBAAwB;SACpC,CAAC,CAAA;QACF,OAAO,EAAE,EAAE,EAAE,MAAM,CAAC,IAAI,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,EAAE,CAAA;IAChF,CAAC,CAAA;IAED,MAAM,gBAAgB,GAAG,KAAK,IAAsB,EAAE;QACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAA;QAC5C,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;YAC7B,IAAI,MAAM,kBAAkB,EAAE;gBAAE,OAAO,IAAI,CAAA;YAC3C,8FAA8F;YAC9F,0EAA0E;YAC1E,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC5B,UAAU,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;YAClC,CAAC,CAAC,CAAA;QACJ,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC,CAAA;IAED,OAAO;QACL,KAAK,EAAE,GAAG,IAAI,CAAC,OAAO,aAAa,IAAI,CAAC,OAAO,GAAG;QAClD,KAAK,CAAC,MAAM;YACV,IAAI,MAAM,kBAAkB,EAAE;gBAAE,OAAO,IAAI,CAAA;YAE3C,sFAAsF;YACtF,yFAAyF;YACzF,IAAI,CAAC,CAAC,MAAM,aAAa,EAAE,CAAC;gBAAE,OAAO,KAAK,CAAA;YAE1C,MAAM,OAAO,GAAG,MAAM,YAAY,EAAE,CAAA;YACpC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;gBAChB,IAAI,oBAAoB,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;oBACzC,MAAM,IAAI,cAAc,CAAC,qBAAqB,CAAC,CAAA;gBACjD,CAAC;gBACD,OAAO,KAAK,CAAA;YACd,CAAC;YACD,OAAO,MAAM,gBAAgB,EAAE,CAAA;QACjC,CAAC;KACF,CAAA;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,OAAO,cAAe,SAAQ,2BAA2B;CAAG"}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Effects half of `cat-factory supervise`: the health probe, the optional container dependency,
|
|
3
|
+
* the port reaper, the supervised child, and the loop that drives them from the pure decisions in
|
|
4
|
+
* `supervise.ts`.
|
|
5
|
+
*
|
|
6
|
+
* Everything the loop touches is behind a seam so `runSupervisor` can be driven by fakes — the
|
|
7
|
+
* same discipline `host-shell.ts` sets out for the k3s flow. Shell-outs go through {@link HostShell}
|
|
8
|
+
* rather than `node:child_process` directly; the one exception is the supervised child itself,
|
|
9
|
+
* which needs inherited stdio and a live handle, so it gets its own {@link ChildLauncher} seam.
|
|
10
|
+
*/
|
|
11
|
+
import { type HostShell } from './host-shell.js';
|
|
12
|
+
import { type SuperviseConfig } from './supervise.js';
|
|
13
|
+
/** Probes whether the supervised service is actually SERVING (not merely booted). */
|
|
14
|
+
export interface HealthProbe {
|
|
15
|
+
serving(): Promise<boolean>;
|
|
16
|
+
}
|
|
17
|
+
/** A prerequisite the supervised service needs — a database container, a local cluster, … */
|
|
18
|
+
export interface ServiceDependency {
|
|
19
|
+
/** Human-readable name for logs. */
|
|
20
|
+
readonly label: string;
|
|
21
|
+
/**
|
|
22
|
+
* Bring it up if needed; resolve `true` once it is ready, `false` to retry on the next cycle.
|
|
23
|
+
* Throw {@link OperatorActionRequiredError} for a state no retry can clear.
|
|
24
|
+
*/
|
|
25
|
+
ensure(): Promise<boolean>;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Thrown by a dependency that cannot be repaired without a human. The loop prints the message ONCE
|
|
29
|
+
* and stops treating the step as retryable noise — the alternative is a watchdog that repeats a
|
|
30
|
+
* hopeless action forever, which is the exact pathology (a restart loop that reads as progress)
|
|
31
|
+
* this supervisor exists to end.
|
|
32
|
+
*/
|
|
33
|
+
export declare class OperatorActionRequiredError extends Error {
|
|
34
|
+
}
|
|
35
|
+
/** A running supervised child process. */
|
|
36
|
+
export interface SupervisedChild {
|
|
37
|
+
readonly pid: number | undefined;
|
|
38
|
+
/** Kill the child AND its descendants. Never throws. */
|
|
39
|
+
kill(): Promise<void>;
|
|
40
|
+
/** Resolves when the process exits, with its code/signal. */
|
|
41
|
+
readonly exited: Promise<{
|
|
42
|
+
code: number | null;
|
|
43
|
+
signal: string | null;
|
|
44
|
+
}>;
|
|
45
|
+
}
|
|
46
|
+
/** Starts the supervised command. */
|
|
47
|
+
export interface ChildLauncher {
|
|
48
|
+
start(): SupervisedChild;
|
|
49
|
+
}
|
|
50
|
+
/** Frees a port held by an orphaned listener, so a restart can bind it again. */
|
|
51
|
+
export interface PortReaper {
|
|
52
|
+
/** Resolves with the PIDs actually killed (empty when the port was already free). */
|
|
53
|
+
reap(): Promise<string[]>;
|
|
54
|
+
}
|
|
55
|
+
/** Injectable clock, so tests never wait in real time. */
|
|
56
|
+
export interface SuperviseClock {
|
|
57
|
+
now(): number;
|
|
58
|
+
/** Resolves after `ms`, or EARLY if `signal` aborts — so Ctrl-C isn't stuck behind a poll interval. */
|
|
59
|
+
sleep(ms: number, signal?: AbortSignal): Promise<void>;
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* NOTE the deliberately un-`unref`'d timers throughout this module. An `unref`'d timer does not keep
|
|
63
|
+
* the event loop alive, and while a spawned child DOES hold it open, that reference vanishes the
|
|
64
|
+
* moment the child dies — which is precisely when the supervisor must keep running. With `unref` the
|
|
65
|
+
* poll timer was then the only thing left, so Node exited 0 the instant its child was killed: a
|
|
66
|
+
* watchdog that died with its patient, silently and with a success code.
|
|
67
|
+
*/
|
|
68
|
+
export declare const systemClock: SuperviseClock;
|
|
69
|
+
/**
|
|
70
|
+
* The real probe. "Serving" requires BOTH signals, because the two failure modes differ: a parked
|
|
71
|
+
* `node --watch` leaves nothing bound to the port, while a server that booted but wedged (or lost
|
|
72
|
+
* its DB pool) still holds the socket and only fails the HTTP check.
|
|
73
|
+
*
|
|
74
|
+
* Both address families are tried — a Node server on `0.0.0.0` answers on 127.0.0.1, but some dev
|
|
75
|
+
* servers bind IPv6 `::1` only, and probing one family would report a false outage.
|
|
76
|
+
*/
|
|
77
|
+
export declare function createHealthProbe(opts: {
|
|
78
|
+
port: number;
|
|
79
|
+
healthPath: string;
|
|
80
|
+
}): HealthProbe;
|
|
81
|
+
/**
|
|
82
|
+
* A `docker compose` service the supervised process needs. This is the piece that makes recovery
|
|
83
|
+
* work after the container engine itself restarted: the example compose files set no restart
|
|
84
|
+
* policy on Postgres, so anything that stops the engine leaves the DB down, and relaunching the
|
|
85
|
+
* server against a missing (or still-initialising) database just crashes it again in `migrate`.
|
|
86
|
+
*/
|
|
87
|
+
export declare function createComposeDependency(shell: HostShell, opts: {
|
|
88
|
+
/**
|
|
89
|
+
* Directory holding the `docker-compose.yml`. Passed as the shell-out's `cwd` on EVERY compose
|
|
90
|
+
* call, never merely stored: compose resolves its project file relative to the working
|
|
91
|
+
* directory, so a supervisor started from anywhere else would address no project at all and
|
|
92
|
+
* report a permanently un-ready database instead of restoring it.
|
|
93
|
+
*/
|
|
94
|
+
dir: string;
|
|
95
|
+
service: string;
|
|
96
|
+
/** Overridable so tests don't wait out the real readiness budget. */
|
|
97
|
+
readyTimeoutMs?: number;
|
|
98
|
+
readyPollMs?: number;
|
|
99
|
+
}): ServiceDependency;
|
|
100
|
+
/**
|
|
101
|
+
* Frees the port before a restart. Killing the child tree usually suffices, but not always: a
|
|
102
|
+
* package-manager wrapper that is killed without its subtree leaves the real `node` orphaned and
|
|
103
|
+
* still holding the socket, and the relaunch then dies with `EADDRINUSE` — turning one outage into
|
|
104
|
+
* a restart loop. Reaping by PORT is the only check that covers an orphan we never had a handle on.
|
|
105
|
+
*
|
|
106
|
+
* It is also, unavoidably, the bluntest thing this supervisor does: reaping by port means SIGKILLing
|
|
107
|
+
* a process we were never handed. If `--port` names a port some unrelated service owns, that service
|
|
108
|
+
* is what dies. There is no portable way to prove descent from our own child, so the mitigation is
|
|
109
|
+
* disclosure rather than detection — every kill NAMES the pid and, where the platform will tell us,
|
|
110
|
+
* the command behind it, and `reap()` reports what it killed so the caller can log it. Callers only
|
|
111
|
+
* ever reap AFTER their own child is confirmed dead, so a healthy stack is never a candidate.
|
|
112
|
+
*/
|
|
113
|
+
export declare function createPortReaper(shell: HostShell, port: number, opts?: {
|
|
114
|
+
platform?: string;
|
|
115
|
+
log?: (message: string) => void;
|
|
116
|
+
}): PortReaper;
|
|
117
|
+
/**
|
|
118
|
+
* Launches the supervised command through a shell, with stdio inherited so its logs stay visible —
|
|
119
|
+
* the supervisor is meant to be a transparent wrapper, not a log proxy.
|
|
120
|
+
*
|
|
121
|
+
* The command is passed as ONE string with `shell: true` (rather than a command plus an args array,
|
|
122
|
+
* which trips Node's DEP0190) so a package-manager entry point resolves through its Windows `.cmd`
|
|
123
|
+
* shim. On POSIX the child gets its own process group, so killing it takes the whole tree.
|
|
124
|
+
*/
|
|
125
|
+
export declare function createChildLauncher(opts: {
|
|
126
|
+
command: string;
|
|
127
|
+
cwd: string;
|
|
128
|
+
}): ChildLauncher;
|
|
129
|
+
/** Everything `runSupervisor` needs. Every field is a seam; only `launcher` and `probe` are required. */
|
|
130
|
+
export interface SupervisorDeps {
|
|
131
|
+
config: SuperviseConfig;
|
|
132
|
+
probe: HealthProbe;
|
|
133
|
+
launcher: ChildLauncher;
|
|
134
|
+
/** Checked in order before each restart — e.g. the database, then the local cluster. */
|
|
135
|
+
dependencies?: ServiceDependency[];
|
|
136
|
+
reaper?: PortReaper;
|
|
137
|
+
clock?: SuperviseClock;
|
|
138
|
+
log?: (message: string) => void;
|
|
139
|
+
/**
|
|
140
|
+
* Aborting stops the loop and, before returning, kills the supervised child and reaps the port.
|
|
141
|
+
* The loop OWNS the child handle, so shutdown has to live here: a signal handler outside it can
|
|
142
|
+
* only reach the port, which on POSIX kills the inner listener while leaving the package-manager
|
|
143
|
+
* wrapper and its `node --watch` alive and parked — a Ctrl-C that orphans exactly the process tree
|
|
144
|
+
* this command exists to manage.
|
|
145
|
+
*/
|
|
146
|
+
stopSignal?: AbortSignal;
|
|
147
|
+
/** Stop after this many ticks — tests only; production runs until the process is signalled. */
|
|
148
|
+
maxTicks?: number;
|
|
149
|
+
}
|
|
150
|
+
/** How a finished supervisor run turned out. Useful for tests and for the command's exit code. */
|
|
151
|
+
export interface SupervisorOutcome {
|
|
152
|
+
ticks: number;
|
|
153
|
+
repairs: number;
|
|
154
|
+
/** Dependencies that reported a state only an operator can clear, by label. */
|
|
155
|
+
blocked: string[];
|
|
156
|
+
/**
|
|
157
|
+
* Set when the supervisor STOPPED trying, with the reason. Restarting cannot fix a command that
|
|
158
|
+
* is simply broken, so `maxFailedStarts` consecutive restarts that never reached a serving state
|
|
159
|
+
* end the loop and report — the caller turns this into a non-zero exit.
|
|
160
|
+
*/
|
|
161
|
+
gaveUp?: string;
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Run the supervision loop: start the child, then probe on an interval and repair when the
|
|
165
|
+
* decisions in `supervise.ts` say so. Returns when `stopSignal` aborts, when the crash-loop budget
|
|
166
|
+
* is spent, or when `maxTicks` is reached (tests) — in production, otherwise never.
|
|
167
|
+
*/
|
|
168
|
+
export declare function runSupervisor(deps: SupervisorDeps): Promise<SupervisorOutcome>;
|
|
169
|
+
//# sourceMappingURL=supervise-runtime.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"supervise-runtime.d.ts","sourceRoot":"","sources":["../src/supervise-runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAKH,OAAO,EAAqB,KAAK,SAAS,EAAE,MAAM,iBAAiB,CAAA;AACnE,OAAO,EAAgB,KAAK,eAAe,EAAyB,MAAM,gBAAgB,CAAA;AAE1F,qFAAqF;AACrF,MAAM,WAAW,WAAW;IAC1B,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAA;CAC5B;AAED,6FAA6F;AAC7F,MAAM,WAAW,iBAAiB;IAChC,oCAAoC;IACpC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB;;;OAGG;IACH,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,CAAA;CAC3B;AAED;;;;;GAKG;AACH,qBAAa,2BAA4B,SAAQ,KAAK;CAAG;AAEzD,0CAA0C;AAC1C,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;IAChC,wDAAwD;IACxD,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IACrB,6DAA6D;IAC7D,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC,CAAA;CACzE;AAED,qCAAqC;AACrC,MAAM,WAAW,aAAa;IAC5B,KAAK,IAAI,eAAe,CAAA;CACzB;AAED,iFAAiF;AACjF,MAAM,WAAW,UAAU;IACzB,qFAAqF;IACrF,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;CAC1B;AAED,0DAA0D;AAC1D,MAAM,WAAW,cAAc;IAC7B,GAAG,IAAI,MAAM,CAAA;IACb,uGAAuG;IACvG,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CACvD;AAED;;;;;;GAMG;AACH,eAAO,MAAM,WAAW,EAAE,cAoBzB,CAAA;AAKD;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CAuCzF;AAKD;;;;;GAKG;AACH,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,SAAS,EAChB,IAAI,EAAE;IACJ;;;;;OAKG;IACH,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,EAAE,MAAM,CAAA;IACf,qEAAqE;IACrE,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB,GACA,iBAAiB,CAqCnB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,SAAS,EAChB,IAAI,EAAE,MAAM,EACZ,IAAI,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;CAAO,GAChE,UAAU,CAyDZ;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,aAAa,CA8CzF;AAED,yGAAyG;AACzG,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,eAAe,CAAA;IACvB,KAAK,EAAE,WAAW,CAAA;IAClB,QAAQ,EAAE,aAAa,CAAA;IACvB,wFAAwF;IACxF,YAAY,CAAC,EAAE,iBAAiB,EAAE,CAAA;IAClC,MAAM,CAAC,EAAE,UAAU,CAAA;IACnB,KAAK,CAAC,EAAE,cAAc,CAAA;IACtB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAC/B;;;;;;OAMG;IACH,UAAU,CAAC,EAAE,WAAW,CAAA;IACxB,+FAA+F;IAC/F,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,kGAAkG;AAClG,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;IACf,+EAA+E;IAC/E,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAoCD;;;;GAIG;AACH,wBAAsB,aAAa,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,iBAAiB,CAAC,CA6GpF"}
|