@mostra/runtime 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin.js +23 -12
- package/dist/bin.js.map +1 -1
- package/dist/builder/child/main.js +20 -18
- package/dist/builder/child/main.js.map +1 -1
- package/dist/{child-xrHmytks.js → child-Chiavhdx.js} +7 -3
- package/dist/child-Chiavhdx.js.map +1 -0
- package/dist/deps/child/install.js +9 -6
- package/dist/deps/child/install.js.map +1 -1
- package/dist/directories-mJ1qlVht.js +322 -0
- package/dist/directories-mJ1qlVht.js.map +1 -0
- package/dist/files-FygSJIzK.js +388 -0
- package/dist/files-FygSJIzK.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/{instance-CzBkUyHu.js → instance-DxcGEQj_.js} +1680 -934
- package/dist/instance-DxcGEQj_.js.map +1 -0
- package/package.json +5 -5
- package/src/bin.ts +18 -16
- package/src/builder/child/dev.ts +8 -4
- package/src/builder/child/plugins.ts +12 -7
- package/src/builder/child/spa.ts +15 -9
- package/src/builder/index.ts +3 -6
- package/src/deps/child/install.ts +3 -2
- package/src/deps/index.ts +21 -16
- package/src/deps/warm.ts +3 -16
- package/src/dev/index.ts +37 -19
- package/src/faults.ts +9 -2
- package/src/gateway/capabilities.ts +2 -2
- package/src/gateway/listener.ts +36 -0
- package/src/home/acquire.ts +27 -11
- package/src/home/index.ts +1 -1
- package/src/home/paths.ts +45 -135
- package/src/home/platform.ts +17 -9
- package/src/home/record.ts +3 -3
- package/src/home/settings.ts +5 -16
- package/src/instance/index.ts +41 -9
- package/src/ledger/index.ts +5 -27
- package/src/platform/control.ts +409 -0
- package/src/platform/directories.ts +212 -0
- package/src/platform/environment.ts +64 -0
- package/src/platform/files.ts +386 -0
- package/src/platform/helper.ts +178 -0
- package/src/{processes → platform}/identity.ts +40 -8
- package/src/platform/index.ts +53 -0
- package/src/platform/launch.ts +373 -0
- package/src/processes/child.ts +11 -5
- package/src/processes/environment.ts +22 -3
- package/src/processes/index.ts +2 -1
- package/src/processes/manager.ts +76 -179
- package/src/protocol-server/client.ts +19 -18
- package/src/protocol-server/server.ts +75 -54
- package/src/publish/index.ts +2 -1
- package/src/publish/publisher.ts +17 -10
- package/src/publish/source.ts +7 -4
- package/src/registry/store.ts +7 -65
- package/src/secrets/index.ts +2 -2
- package/src/service/unit.ts +353 -33
- package/src/verify/index.ts +5 -3
- package/src/verify/typecheck.ts +3 -2
- package/src/workers/launcher.ts +3 -1
- package/src/workers/pool.ts +14 -1
- package/dist/child-xrHmytks.js.map +0 -1
- package/dist/instance-CzBkUyHu.js.map +0 -1
- package/dist/types-NjNy7gXX.js +0 -57
- package/dist/types-NjNy7gXX.js.map +0 -1
package/src/processes/manager.ts
CHANGED
|
@@ -1,42 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The process manager (RT-09, ARCHITECTURE §4, FAILURE_MODEL §2). Every child
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
2
|
+
* The process manager (RT-09, RT-10, ARCHITECTURE §4, FAILURE_MODEL §2). Every child starts
|
|
3
|
+
* behind a gate in a group of its own (a process group on POSIX, a Job Object on Windows,
|
|
4
|
+
* `platform/launch.ts`) and is recorded in the ledger's `child` table with its PID and start
|
|
5
|
+
* time before it may run. A child's group dies with it, and a supervisor that starts after a
|
|
6
|
+
* crash reaps every recorded group that is still running.
|
|
6
7
|
*/
|
|
7
|
-
import { type ChildProcess, spawn } from "node:child_process";
|
|
8
8
|
import { createInterface } from "node:readline";
|
|
9
|
-
import type { Duplex
|
|
9
|
+
import type { Duplex } from "node:stream";
|
|
10
10
|
import { type JsonRpcParams, RpcPeer } from "@mostra/protocol";
|
|
11
11
|
import * as z from "zod";
|
|
12
12
|
import { type Faults, noFaults } from "../faults.ts";
|
|
13
13
|
import type { Ledger } from "../ledger/index.ts";
|
|
14
14
|
import type { Environment, Log, Redactor } from "../log.ts";
|
|
15
|
-
import {
|
|
16
|
-
listProcesses,
|
|
17
|
-
type ProcessInfo,
|
|
18
|
-
readProcess,
|
|
19
|
-
sameBoot,
|
|
20
|
-
startedNoEarlier,
|
|
21
|
-
} from "./identity.ts";
|
|
15
|
+
import { adoptable, launch, reapRecorded } from "../platform/index.ts";
|
|
22
16
|
|
|
23
17
|
/** How long a stopping child gets between SIGTERM and SIGKILL, by default. */
|
|
24
18
|
const DEFAULT_GRACE_MS = 5000;
|
|
25
|
-
/** How long reaping waits between SIGTERM and SIGKILL. */
|
|
26
|
-
const REAP_GRACE_MS = 2000;
|
|
27
|
-
/** How long to wait for SIGKILLed processes to disappear before reporting them. */
|
|
28
|
-
const KILL_WAIT_MS = 5000;
|
|
29
19
|
/** How many stderr lines a child's exit report keeps. */
|
|
30
20
|
const STDERR_TAIL = 50;
|
|
31
21
|
|
|
32
|
-
/**
|
|
33
|
-
* The gate every child starts behind (issue #7): `/bin/sh` waits for "go" on file descriptor 3 and
|
|
34
|
-
* only then replaces itself with the real command, keeping its PID, process group and start time.
|
|
35
|
-
* The supervisor writes "go" after it has recorded the child, so a supervisor killed in between
|
|
36
|
-
* leaves a gate that reads end-of-file and exits 125 without running anything.
|
|
37
|
-
*/
|
|
38
|
-
const GATE_SCRIPT = 'IFS= read -r go <&3 && [ "$go" = go ] || exit 125; exec 3<&-; exec "$@"';
|
|
39
|
-
|
|
40
22
|
/** How a child ended. */
|
|
41
23
|
export interface ChildExit {
|
|
42
24
|
/** The exit code, or `null` when a signal ended it. */
|
|
@@ -146,6 +128,8 @@ export interface ProcessManager {
|
|
|
146
128
|
list(): readonly { pid: number; kind: string; ownerRef: string; startedAt: string }[];
|
|
147
129
|
/** Stops every child; used when the supervisor stops. */
|
|
148
130
|
stopAll(graceMs?: number): Promise<void>;
|
|
131
|
+
/** Ends every child at once without waiting: the last resort at the supervisor's exit. */
|
|
132
|
+
killAll(): void;
|
|
149
133
|
}
|
|
150
134
|
|
|
151
135
|
/** Options for {@link openProcessManager}. */
|
|
@@ -166,34 +150,10 @@ export interface ProcessManagerOptions {
|
|
|
166
150
|
export async function openProcessManager(
|
|
167
151
|
options: ProcessManagerOptions,
|
|
168
152
|
): Promise<{ manager: ProcessManager; reaped: readonly ReapedChild[] }> {
|
|
169
|
-
const reaped = await
|
|
153
|
+
const reaped = await reap(options.ledger, options.log);
|
|
170
154
|
return { manager: createManager(options), reaped };
|
|
171
155
|
}
|
|
172
156
|
|
|
173
|
-
/** Signals `pids`, ignoring ones already gone. */
|
|
174
|
-
function signalAll(pids: readonly number[], signal: NodeJS.Signals): void {
|
|
175
|
-
for (const pid of pids) {
|
|
176
|
-
try {
|
|
177
|
-
process.kill(pid, signal);
|
|
178
|
-
} catch {
|
|
179
|
-
// Already gone.
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
function signalGroup(pgid: number, signal: NodeJS.Signals): boolean {
|
|
185
|
-
try {
|
|
186
|
-
process.kill(-pgid, signal);
|
|
187
|
-
return true;
|
|
188
|
-
} catch {
|
|
189
|
-
return false;
|
|
190
|
-
}
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
const groupAlive = (pgid: number): boolean => signalGroup(pgid, 0 as unknown as NodeJS.Signals);
|
|
194
|
-
|
|
195
|
-
const delay = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
|
|
196
|
-
|
|
197
157
|
/** Waits for `promise` to settle, or `ms` at most, leaving no timer behind. */
|
|
198
158
|
async function settleWithin(promise: Promise<unknown>, ms: number): Promise<void> {
|
|
199
159
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
@@ -212,70 +172,21 @@ async function settleWithin(promise: Promise<unknown>, ms: number): Promise<void
|
|
|
212
172
|
}
|
|
213
173
|
}
|
|
214
174
|
|
|
215
|
-
|
|
216
|
-
function targetsFor(
|
|
217
|
-
row: ChildRow,
|
|
218
|
-
processes: readonly ProcessInfo[],
|
|
219
|
-
): number[] | NonNullable<ReapedChild["skipped"]> {
|
|
220
|
-
const leader = processes.find((info) => info.pid === row.pid);
|
|
221
|
-
if (leader && leader.startTime !== row.start_time) return "pid reused";
|
|
222
|
-
if (!leader) {
|
|
223
|
-
const boot = processes.find((info) => info.pid === process.pid)?.startTime;
|
|
224
|
-
if (boot === undefined || !sameBoot(boot, row.start_time)) return "other boot";
|
|
225
|
-
}
|
|
226
|
-
const members = processes
|
|
227
|
-
.filter((info) => info.pgid === row.pgid && info.pid !== process.pid)
|
|
228
|
-
.filter((info) => startedNoEarlier(info.startTime, row.start_time))
|
|
229
|
-
.map((info) => info.pid);
|
|
230
|
-
return members.length > 0 ? members : "gone";
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
async function reapRecorded(ledger: Ledger, log: Log): Promise<ReapedChild[]> {
|
|
175
|
+
async function reap(ledger: Ledger, log: Log): Promise<ReapedChild[]> {
|
|
234
176
|
const rows = ledger.read((db) => db.all<ChildRow>("SELECT * FROM child ORDER BY started_at"));
|
|
235
177
|
if (rows.length === 0) return [];
|
|
236
|
-
const
|
|
237
|
-
const plan = rows.map((row) => ({ row, targets: targetsFor(row, before) }));
|
|
238
|
-
// Only groups proven to be the recorded children's are signalled, then watched until they are
|
|
239
|
-
// empty: a member may start more processes in the group while it handles SIGTERM, so every
|
|
240
|
-
// round re-lists the groups instead of trusting the first snapshot (RT-09).
|
|
241
|
-
const owned = plan.filter((entry) => Array.isArray(entry.targets)).map((entry) => entry.row);
|
|
242
|
-
const members = (processes: readonly ProcessInfo[]): number[] => [
|
|
243
|
-
...new Set(
|
|
244
|
-
owned.flatMap((row) => {
|
|
245
|
-
const targets = targetsFor(row, processes);
|
|
246
|
-
return Array.isArray(targets) ? targets : [];
|
|
247
|
-
}),
|
|
248
|
-
),
|
|
249
|
-
];
|
|
250
|
-
// A listing is a series of reads, so a process that starts while another exits can slip
|
|
251
|
-
// between them; the groups count as gone only when the kernel has no member left either.
|
|
252
|
-
const groupsLeft = (): boolean => owned.some((row) => groupAlive(row.pgid));
|
|
253
|
-
signalAll(members(before), "SIGTERM");
|
|
254
|
-
const deadline = Date.now() + REAP_GRACE_MS;
|
|
255
|
-
let remaining = members(await listProcesses());
|
|
256
|
-
while ((remaining.length > 0 || groupsLeft()) && Date.now() < deadline) {
|
|
257
|
-
await delay(100);
|
|
258
|
-
remaining = members(await listProcesses());
|
|
259
|
-
}
|
|
260
|
-
const killDeadline = Date.now() + KILL_WAIT_MS;
|
|
261
|
-
while ((remaining.length > 0 || groupsLeft()) && Date.now() < killDeadline) {
|
|
262
|
-
signalAll(remaining, "SIGKILL");
|
|
263
|
-
await delay(50);
|
|
264
|
-
remaining = members(await listProcesses());
|
|
265
|
-
}
|
|
266
|
-
if (remaining.length > 0 || groupsLeft()) {
|
|
267
|
-
log.warn("process.reap_incomplete", { pids: remaining, pgids: owned.map((row) => row.pgid) });
|
|
268
|
-
}
|
|
178
|
+
const outcomes = await reapRecorded(rows, (event, fields) => log.warn(event, fields));
|
|
269
179
|
ledger.write((db) => {
|
|
270
|
-
for (const
|
|
180
|
+
for (const row of rows) {
|
|
271
181
|
db.run("DELETE FROM child WHERE pid = ? AND start_time = ?", row.pid, row.start_time);
|
|
272
182
|
}
|
|
273
183
|
});
|
|
274
|
-
const report =
|
|
184
|
+
const report = rows.map((row, index): ReapedChild => {
|
|
275
185
|
const base = { pid: row.pid, kind: row.kind, ownerRef: row.owner_ref };
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
186
|
+
const outcome = outcomes[index];
|
|
187
|
+
return outcome?.skipped === undefined
|
|
188
|
+
? { ...base, signalled: outcome?.ended ?? [] }
|
|
189
|
+
: { ...base, signalled: [], skipped: outcome.skipped };
|
|
279
190
|
});
|
|
280
191
|
for (const entry of report) log.info("process.reaped", { ...entry });
|
|
281
192
|
return report;
|
|
@@ -287,7 +198,7 @@ function createManager({
|
|
|
287
198
|
redactor,
|
|
288
199
|
faults = noFaults,
|
|
289
200
|
}: ProcessManagerOptions): ProcessManager {
|
|
290
|
-
const running = new Map<number, ChildHandle>();
|
|
201
|
+
const running = new Map<number, { handle: ChildHandle; kill(): void }>();
|
|
291
202
|
let stopping = false;
|
|
292
203
|
|
|
293
204
|
const record = (row: ChildRow): void => {
|
|
@@ -307,53 +218,34 @@ function createManager({
|
|
|
307
218
|
const forget = (pid: number): void => {
|
|
308
219
|
ledger.write((db) => db.run("DELETE FROM child WHERE pid = ? OR parent_pid = ?", pid, pid));
|
|
309
220
|
};
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
* Terminates groups: SIGTERM, then SIGKILL after `graceMs` to those still present, then waits
|
|
313
|
-
* (boundedly) until they are gone.
|
|
314
|
-
*/
|
|
315
|
-
const endGroups = async (pgids: readonly number[], graceMs: number): Promise<void> => {
|
|
316
|
-
for (const pgid of pgids) signalGroup(pgid, "SIGTERM");
|
|
317
|
-
const deadline = Date.now() + graceMs;
|
|
318
|
-
while (Date.now() < deadline && pgids.some(groupAlive)) await delay(25);
|
|
319
|
-
for (const pgid of pgids) if (groupAlive(pgid)) signalGroup(pgid, "SIGKILL");
|
|
320
|
-
const killDeadline = Date.now() + KILL_WAIT_MS;
|
|
321
|
-
while (Date.now() < killDeadline && pgids.some(groupAlive)) await delay(10);
|
|
322
|
-
const left = pgids.filter(groupAlive);
|
|
323
|
-
if (left.length > 0) log.warn("process.group_survived_sigkill", { pgids: left });
|
|
324
|
-
};
|
|
221
|
+
const warn = (event: string, fields: Readonly<Record<string, unknown>>): void =>
|
|
222
|
+
log.warn(event, fields);
|
|
325
223
|
|
|
326
224
|
const spawnChild = async (spec: SpawnSpec): Promise<ChildHandle> => {
|
|
327
225
|
if (stopping) throw new Error("The process manager is stopping.");
|
|
328
226
|
if (spec.ipc && spec.channel) throw new Error("A child takes ipc or a channel, not both.");
|
|
329
227
|
for (const value of spec.redact ?? []) redactor.protect(value);
|
|
330
228
|
redactor.protectEnvironment(spec.env ?? process.env);
|
|
331
|
-
// fd 3 is the gate; the IPC channel, when there is one, is fd 4
|
|
332
|
-
|
|
333
|
-
const child: ChildProcess = spawn(
|
|
334
|
-
"/bin/sh",
|
|
335
|
-
["-c", GATE_SCRIPT, `mostra-${spec.kind}`, spec.command, ...(spec.args ?? [])],
|
|
229
|
+
// fd 3 is the gate; the IPC channel or the byte channel, when there is one, is fd 4.
|
|
230
|
+
const launched = launch(
|
|
336
231
|
{
|
|
232
|
+
kind: spec.kind,
|
|
233
|
+
command: spec.command,
|
|
234
|
+
args: spec.args ?? [],
|
|
337
235
|
cwd: spec.cwd,
|
|
338
236
|
env: spec.env ?? process.env,
|
|
339
|
-
|
|
340
|
-
stdio: spec.ipc
|
|
341
|
-
? ["ignore", "pipe", "pipe", "pipe", "ipc"]
|
|
342
|
-
: spec.channel
|
|
343
|
-
? ["ignore", "pipe", "pipe", "pipe", "pipe"]
|
|
344
|
-
: ["ignore", "pipe", "pipe", "pipe"],
|
|
345
|
-
serialization: "json",
|
|
237
|
+
extra: spec.ipc ? "ipc" : spec.channel ? "channel" : undefined,
|
|
346
238
|
},
|
|
239
|
+
warn,
|
|
347
240
|
);
|
|
348
|
-
const
|
|
349
|
-
gate?.on("error", () => {
|
|
350
|
-
// The child exited before reading "go"; its exit is reported through `exited`.
|
|
351
|
-
});
|
|
241
|
+
const child = launched.process;
|
|
352
242
|
const startedAt = new Date().toISOString();
|
|
353
|
-
|
|
354
|
-
child.once("spawn", () => resolve(
|
|
243
|
+
await new Promise<void>((resolve, reject) => {
|
|
244
|
+
child.once("spawn", () => resolve());
|
|
355
245
|
child.once("error", reject);
|
|
356
246
|
});
|
|
247
|
+
// The gate's process ID until the child's own is known (the same one on POSIX).
|
|
248
|
+
let pid = child.pid ?? 0;
|
|
357
249
|
// After the spawn, errors (a failed signal or IPC send) are reported, never thrown: an
|
|
358
250
|
// unhandled 'error' event would take the supervisor down.
|
|
359
251
|
child.on("error", (error) => log.warn("process.error", { kind: spec.kind, pid, error }));
|
|
@@ -403,17 +295,19 @@ function createManager({
|
|
|
403
295
|
);
|
|
404
296
|
return {};
|
|
405
297
|
}
|
|
406
|
-
const info = await readProcess(parsed.data.pid);
|
|
407
298
|
// Only a group leader (a program in its own session, such as a PTY program) can be
|
|
408
|
-
// adopted: its whole group is then signalled, and no foreign group ever is.
|
|
409
|
-
|
|
410
|
-
|
|
299
|
+
// adopted on POSIX: its whole group is then signalled, and no foreign group ever is.
|
|
300
|
+
// On Windows every descendant already ends with the child's job.
|
|
301
|
+
const found = await adoptable(parsed.data.pid);
|
|
302
|
+
if (!found) return { adopted: false };
|
|
303
|
+
if (!found.record) return { adopted: true };
|
|
304
|
+
adopted.set(found.info.pid, found.info.pgid);
|
|
411
305
|
record({
|
|
412
|
-
pid: info.pid,
|
|
413
|
-
start_time: info.startTime,
|
|
306
|
+
pid: found.info.pid,
|
|
307
|
+
start_time: found.info.startTime,
|
|
414
308
|
kind: `${spec.kind}/adopted`,
|
|
415
309
|
owner_ref: spec.ownerRef,
|
|
416
|
-
pgid: info.pgid,
|
|
310
|
+
pgid: found.info.pgid,
|
|
417
311
|
started_at: new Date().toISOString(),
|
|
418
312
|
parent_pid: pid,
|
|
419
313
|
});
|
|
@@ -434,37 +328,36 @@ function createManager({
|
|
|
434
328
|
// Record the child before it may run its command (RT-09, issue #7), then open its gate. If it
|
|
435
329
|
// already exited, nothing is left to reap.
|
|
436
330
|
const registered = (async () => {
|
|
437
|
-
const info = await
|
|
331
|
+
const info = await launched.identify();
|
|
438
332
|
faults.at("spawn:before-record");
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
}
|
|
333
|
+
// The same point for one kind of child (`spawn:before-record:demo`), for an instance whose
|
|
334
|
+
// start already spawns children of its own (on Windows the control pipe's helper).
|
|
335
|
+
faults.at(`spawn:before-record:${spec.kind}`);
|
|
336
|
+
// A supervisor stalled here (a test's `hang`) shows the child waiting at its gate, unrecorded.
|
|
337
|
+
if (faults.hangs(`spawn:before-record:${spec.kind}`)) await new Promise(() => {});
|
|
338
|
+
if (!info) return;
|
|
339
|
+
pid = info.pid;
|
|
447
340
|
record({
|
|
448
|
-
pid,
|
|
341
|
+
pid: info.pid,
|
|
449
342
|
start_time: info.startTime,
|
|
450
343
|
kind: spec.kind,
|
|
451
344
|
owner_ref: spec.ownerRef,
|
|
452
|
-
pgid: pid,
|
|
345
|
+
pgid: info.pid,
|
|
453
346
|
started_at: startedAt,
|
|
454
347
|
parent_pid: null,
|
|
455
348
|
});
|
|
456
|
-
|
|
349
|
+
launched.release();
|
|
457
350
|
})();
|
|
458
351
|
|
|
459
352
|
// From here on, the child's group ends with it, whatever happens to its registration.
|
|
460
353
|
const exited = exit.then(async ({ code, signal }) => {
|
|
461
354
|
// The child's lifetime bounds its group and the groups it adopted.
|
|
462
|
-
await
|
|
355
|
+
await launched.sweep([...adopted.values()]);
|
|
463
356
|
// The group is gone, so nothing holds the pipes open any more; the bound covers a
|
|
464
357
|
// descendant that left the group with them.
|
|
465
358
|
await settleWithin(Promise.all(drained), 2000);
|
|
466
359
|
await registered.catch(() => {});
|
|
467
|
-
running.delete(pid);
|
|
360
|
+
running.delete(child.pid ?? 0);
|
|
468
361
|
try {
|
|
469
362
|
forget(pid);
|
|
470
363
|
} catch (error) {
|
|
@@ -478,26 +371,27 @@ function createManager({
|
|
|
478
371
|
channel?.on("error", (error) =>
|
|
479
372
|
log.warn("process.channel_error", { kind: spec.kind, pid, error }),
|
|
480
373
|
);
|
|
481
|
-
const
|
|
482
|
-
|
|
374
|
+
const stop = async (graceMs = DEFAULT_GRACE_MS): Promise<ChildExit> => {
|
|
375
|
+
if (launched.running()) await launched.end(graceMs, [...adopted.values()]);
|
|
376
|
+
return exited;
|
|
377
|
+
};
|
|
378
|
+
const handle = {
|
|
379
|
+
get pid() {
|
|
380
|
+
return pid;
|
|
381
|
+
},
|
|
483
382
|
kind: spec.kind,
|
|
484
383
|
ownerRef: spec.ownerRef,
|
|
485
384
|
startedAt,
|
|
486
385
|
...(channel ? { channel } : {}),
|
|
487
386
|
exited,
|
|
488
|
-
request(method, params, options) {
|
|
387
|
+
request(method: string, params?: JsonRpcParams, options?: { timeoutMs?: number }) {
|
|
489
388
|
if (!peer) return Promise.reject(new Error(`${spec.kind} was started without IPC.`));
|
|
490
389
|
return peer.request(method, params, options);
|
|
491
390
|
},
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
await endGroups([pid, ...adopted.values()], graceMs);
|
|
495
|
-
}
|
|
496
|
-
return exited;
|
|
497
|
-
},
|
|
498
|
-
};
|
|
391
|
+
stop,
|
|
392
|
+
} satisfies ChildHandle;
|
|
499
393
|
// Tracked from the spawn on, so stopAll reaches a child whose registration is still running.
|
|
500
|
-
running.set(pid, handle);
|
|
394
|
+
running.set(child.pid ?? 0, { handle, kill: () => launched.kill() });
|
|
501
395
|
|
|
502
396
|
try {
|
|
503
397
|
await registered;
|
|
@@ -527,17 +421,20 @@ function createManager({
|
|
|
527
421
|
return {
|
|
528
422
|
spawn: trackedSpawn,
|
|
529
423
|
list: () =>
|
|
530
|
-
[...running.values()].map((
|
|
531
|
-
pid:
|
|
532
|
-
kind:
|
|
533
|
-
ownerRef:
|
|
534
|
-
startedAt:
|
|
424
|
+
[...running.values()].map(({ handle }) => ({
|
|
425
|
+
pid: handle.pid,
|
|
426
|
+
kind: handle.kind,
|
|
427
|
+
ownerRef: handle.ownerRef,
|
|
428
|
+
startedAt: handle.startedAt,
|
|
535
429
|
})),
|
|
536
430
|
async stopAll(graceMs = DEFAULT_GRACE_MS) {
|
|
537
431
|
stopping = true;
|
|
538
432
|
// Spawns already under way finish first, so their children are tracked and stopped too.
|
|
539
433
|
await Promise.all(inflight);
|
|
540
|
-
await Promise.all([...running.values()].map((
|
|
434
|
+
await Promise.all([...running.values()].map(({ handle }) => handle.stop(graceMs)));
|
|
435
|
+
},
|
|
436
|
+
killAll() {
|
|
437
|
+
for (const { kill } of running.values()) kill();
|
|
541
438
|
},
|
|
542
439
|
};
|
|
543
440
|
}
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
* The client end of the home socket, used by a starter that finds another supervisor holding the
|
|
3
3
|
* home (D81) and by tests. SDKs and the CLI implement the same few steps in their own packages.
|
|
4
4
|
*/
|
|
5
|
-
import { createConnection, type Socket } from "node:net";
|
|
6
5
|
import {
|
|
7
6
|
type ClientInfo,
|
|
8
7
|
createDiagnostic,
|
|
@@ -12,6 +11,7 @@ import {
|
|
|
12
11
|
RpcError,
|
|
13
12
|
RpcPeer,
|
|
14
13
|
} from "@mostra/protocol";
|
|
14
|
+
import { ControlConnectError, connectControl } from "../platform/index.ts";
|
|
15
15
|
import { pump, writeLine } from "./streams.ts";
|
|
16
16
|
|
|
17
17
|
/** An open client connection to an instance's socket. */
|
|
@@ -24,21 +24,12 @@ export interface SocketClient {
|
|
|
24
24
|
|
|
25
25
|
/**
|
|
26
26
|
* Connects to a control socket. Rejects with `MOSTRA_HOST_CONNECTION_CLOSED` when nothing
|
|
27
|
-
* listens there (the file is missing or stale)
|
|
27
|
+
* listens there (the file is missing or stale), the connection doesn't open within `timeoutMs`,
|
|
28
|
+
* or, on Windows, the pipe under that name is another user's.
|
|
28
29
|
*/
|
|
29
30
|
export function connectSocket(path: string, timeoutMs = 5000): Promise<SocketClient> {
|
|
30
|
-
return
|
|
31
|
-
|
|
32
|
-
const timer = setTimeout(() => {
|
|
33
|
-
socket.destroy();
|
|
34
|
-
reject(refused(path, "timed out"));
|
|
35
|
-
}, timeoutMs);
|
|
36
|
-
socket.once("error", (error: NodeJS.ErrnoException) => {
|
|
37
|
-
clearTimeout(timer);
|
|
38
|
-
reject(refused(path, error.code ?? "failed"));
|
|
39
|
-
});
|
|
40
|
-
socket.once("connect", () => {
|
|
41
|
-
clearTimeout(timer);
|
|
31
|
+
return connectControl(path, timeoutMs).then(
|
|
32
|
+
(socket) => {
|
|
42
33
|
const peer = new RpcPeer({ send: (message) => void writeLine(socket, message) });
|
|
43
34
|
let markClosed: () => void = () => {};
|
|
44
35
|
const closed = new Promise<void>((done) => {
|
|
@@ -53,16 +44,26 @@ export function connectSocket(path: string, timeoutMs = 5000): Promise<SocketCli
|
|
|
53
44
|
markClosed();
|
|
54
45
|
},
|
|
55
46
|
);
|
|
56
|
-
|
|
47
|
+
return {
|
|
57
48
|
peer,
|
|
58
49
|
closed,
|
|
59
50
|
close: () => {
|
|
60
51
|
peer.close();
|
|
61
52
|
socket.end();
|
|
62
53
|
},
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
|
|
54
|
+
};
|
|
55
|
+
},
|
|
56
|
+
(error: unknown) => {
|
|
57
|
+
// Another user's pipe under the name is no instance of this home, like no instance at all.
|
|
58
|
+
const why =
|
|
59
|
+
error instanceof ControlConnectError && error.reason === "not-owned"
|
|
60
|
+
? "it belongs to another user"
|
|
61
|
+
: error instanceof Error
|
|
62
|
+
? error.message
|
|
63
|
+
: "failed";
|
|
64
|
+
throw refused(path, why);
|
|
65
|
+
},
|
|
66
|
+
);
|
|
66
67
|
}
|
|
67
68
|
|
|
68
69
|
function refused(path: string, why: string): RpcError {
|
|
@@ -4,9 +4,6 @@
|
|
|
4
4
|
* rules (initialize first and once, revision negotiation, the home check, port registration on
|
|
5
5
|
* the registering connection) and routes every other method to the handler a module registered.
|
|
6
6
|
*/
|
|
7
|
-
import { chmodSync, mkdirSync, type Stats, statSync, unlinkSync } from "node:fs";
|
|
8
|
-
import { createServer, type Server, type Socket } from "node:net";
|
|
9
|
-
import { dirname } from "node:path";
|
|
10
7
|
import type { Readable, Writable } from "node:stream";
|
|
11
8
|
import {
|
|
12
9
|
CLI_CLIENT_NAME,
|
|
@@ -27,6 +24,7 @@ import {
|
|
|
27
24
|
RpcPeer,
|
|
28
25
|
} from "@mostra/protocol";
|
|
29
26
|
import type { Log } from "../log.ts";
|
|
27
|
+
import { type ControlListener, type LaunchPipe, listenControl } from "../platform/index.ts";
|
|
30
28
|
import { pump, writeLine } from "./streams.ts";
|
|
31
29
|
|
|
32
30
|
/** How a connection reached the instance. */
|
|
@@ -125,8 +123,15 @@ export class ProtocolServer {
|
|
|
125
123
|
readonly #connections = new Set<Connection>();
|
|
126
124
|
#nextId = 1;
|
|
127
125
|
#refusal: Diagnostic | undefined;
|
|
128
|
-
#listener:
|
|
126
|
+
#listener: ControlListener | undefined;
|
|
127
|
+
#endpoint: string | undefined;
|
|
128
|
+
#listenOptions: { launchPipe?: LaunchPipe; onMoved?: (path: string) => void } = {};
|
|
129
129
|
#healTimer: ReturnType<typeof setInterval> | undefined;
|
|
130
|
+
/** A heal under way: one at a time, and `close` waits for it (Opus review of WP-W1). */
|
|
131
|
+
#healing: Promise<void> | undefined;
|
|
132
|
+
/** An endpoint that moved but whose `onMoved` failed: announced again by every heal until it holds. */
|
|
133
|
+
#unannounced: string | undefined;
|
|
134
|
+
#closed = false;
|
|
130
135
|
|
|
131
136
|
constructor(options: ProtocolServerOptions) {
|
|
132
137
|
this.#options = options;
|
|
@@ -200,65 +205,87 @@ export class ProtocolServer {
|
|
|
200
205
|
}
|
|
201
206
|
|
|
202
207
|
/**
|
|
203
|
-
* Listens on the home's control
|
|
204
|
-
* lock holder calls this)
|
|
205
|
-
* 10 s) that the
|
|
208
|
+
* Listens on the home's control endpoint (D81, D82): a Unix socket, replacing any leftover file
|
|
209
|
+
* (only the lock holder calls this) and owner-only; on Windows the helper's owner-only pipe
|
|
210
|
+
* (`launchPipe`). Every `healMs` (at most 10 s) it checks that the endpoint still serves and
|
|
211
|
+
* binds it again when it doesn't, calling `onMoved` when the new endpoint has another name.
|
|
206
212
|
*/
|
|
207
|
-
async listen(
|
|
208
|
-
|
|
213
|
+
async listen(
|
|
214
|
+
path: string,
|
|
215
|
+
healMs = 5000,
|
|
216
|
+
options: { launchPipe?: LaunchPipe; onMoved?: (path: string) => void } = {},
|
|
217
|
+
): Promise<void> {
|
|
218
|
+
this.#endpoint = path;
|
|
219
|
+
this.#listenOptions = options;
|
|
220
|
+
this.#listener = await this.#bind();
|
|
209
221
|
this.#healTimer = setInterval(() => void this.#heal(), Math.min(healMs, 10_000));
|
|
210
222
|
this.#healTimer.unref();
|
|
211
223
|
}
|
|
212
224
|
|
|
213
|
-
/** The
|
|
225
|
+
/** The endpoint the server listens on, as clients connect to it. */
|
|
214
226
|
get socketPath(): string | undefined {
|
|
215
227
|
return this.#listener?.path;
|
|
216
228
|
}
|
|
217
229
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
230
|
+
#bind(): Promise<ControlListener> {
|
|
231
|
+
return listenControl(
|
|
232
|
+
this.#endpoint ?? "",
|
|
233
|
+
(stream) => {
|
|
234
|
+
this.serve("socket", stream, stream);
|
|
235
|
+
},
|
|
236
|
+
{
|
|
237
|
+
...(this.#listenOptions.launchPipe ? { launchPipe: this.#listenOptions.launchPipe } : {}),
|
|
238
|
+
onError: (error) => this.#options.log.error("protocol.listener_error", { error }),
|
|
239
|
+
},
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
#heal(): Promise<void> {
|
|
244
|
+
if (this.#closed) return Promise.resolve();
|
|
245
|
+
// A bind on Windows (a gated helper and its READY) can outlast the interval: one heal at a time.
|
|
246
|
+
this.#healing ??= this.#healOnce().finally(() => {
|
|
247
|
+
this.#healing = undefined;
|
|
234
248
|
});
|
|
235
|
-
|
|
236
|
-
server.on("error", (error) => this.#options.log.error("protocol.listener_error", { error }));
|
|
237
|
-
return { server, path, stats: statSync(path) };
|
|
249
|
+
return this.#healing;
|
|
238
250
|
}
|
|
239
251
|
|
|
240
|
-
async #
|
|
252
|
+
async #healOnce(): Promise<void> {
|
|
253
|
+
if (this.#unannounced !== undefined) this.#announce(this.#unannounced);
|
|
241
254
|
const listener = this.#listener;
|
|
242
|
-
if (!listener) return;
|
|
243
|
-
let current: Stats | undefined;
|
|
244
|
-
try {
|
|
245
|
-
current = statSync(listener.path);
|
|
246
|
-
} catch {
|
|
247
|
-
current = undefined;
|
|
248
|
-
}
|
|
249
|
-
if (current && current.ino === listener.stats.ino && current.dev === listener.stats.dev) return;
|
|
255
|
+
if (!listener || listener.intact()) return;
|
|
250
256
|
// Close the old listener first: Node removes a Unix socket's file when its server closes, so
|
|
251
257
|
// closing it after binding would remove the new socket.
|
|
252
|
-
listener.
|
|
258
|
+
await listener.close();
|
|
259
|
+
if (this.#closed) return;
|
|
253
260
|
try {
|
|
254
|
-
const replacement = await this.#bind(
|
|
261
|
+
const replacement = await this.#bind();
|
|
262
|
+
if (this.#closed) {
|
|
263
|
+
// The server closed while this bind ran: the endpoint is never served or announced.
|
|
264
|
+
await replacement.close();
|
|
265
|
+
return;
|
|
266
|
+
}
|
|
255
267
|
this.#listener = replacement;
|
|
256
|
-
this.#options.log.warn("protocol.socket_recreated", { path:
|
|
268
|
+
this.#options.log.warn("protocol.socket_recreated", { path: replacement.path });
|
|
269
|
+
if (replacement.path !== listener.path) this.#announce(replacement.path);
|
|
257
270
|
} catch (error) {
|
|
258
271
|
this.#options.log.warn("protocol.socket_recreate_failed", { path: listener.path, error });
|
|
259
272
|
}
|
|
260
273
|
}
|
|
261
274
|
|
|
275
|
+
/**
|
|
276
|
+
* Tells the instance the endpoint moved (it rewrites `instance.json`); when that fails, every
|
|
277
|
+
* later heal tries again, so starters are never left with a record naming a dead pipe.
|
|
278
|
+
*/
|
|
279
|
+
#announce(path: string): void {
|
|
280
|
+
try {
|
|
281
|
+
this.#listenOptions.onMoved?.(path);
|
|
282
|
+
this.#unannounced = undefined;
|
|
283
|
+
} catch (error) {
|
|
284
|
+
this.#unannounced = path;
|
|
285
|
+
this.#options.log.warn("protocol.endpoint_announce_failed", { path, error });
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
|
|
262
289
|
/**
|
|
263
290
|
* Answers every request from now on with `diagnostic`, for an instance that can no longer serve
|
|
264
291
|
* (`MOSTRA_HOME_GONE`).
|
|
@@ -365,23 +392,17 @@ export class ProtocolServer {
|
|
|
365
392
|
|
|
366
393
|
/**
|
|
367
394
|
* Stops listening, removes the socket file if it is still this server's, and closes every
|
|
368
|
-
* connection. Responses already sent are flushed first
|
|
395
|
+
* connection. Responses already sent are flushed first: the connections end before the
|
|
396
|
+
* listener, which on Windows lets the pipe's helper hand them over before it exits.
|
|
369
397
|
*/
|
|
370
|
-
close(): void {
|
|
398
|
+
async close(): Promise<void> {
|
|
399
|
+
this.#closed = true;
|
|
371
400
|
if (this.#healTimer) clearInterval(this.#healTimer);
|
|
401
|
+
await this.#healing;
|
|
402
|
+
for (const connection of this.#connections) connection.close();
|
|
372
403
|
const listener = this.#listener;
|
|
373
404
|
this.#listener = undefined;
|
|
374
|
-
|
|
375
|
-
listener.server.close();
|
|
376
|
-
try {
|
|
377
|
-
const current = statSync(listener.path);
|
|
378
|
-
if (current.ino === listener.stats.ino && current.dev === listener.stats.dev)
|
|
379
|
-
unlinkSync(listener.path);
|
|
380
|
-
} catch {
|
|
381
|
-
// Already gone.
|
|
382
|
-
}
|
|
383
|
-
}
|
|
384
|
-
for (const connection of this.#connections) connection.close();
|
|
405
|
+
await listener?.close();
|
|
385
406
|
}
|
|
386
407
|
|
|
387
408
|
async #dispatch(session: MutableSession, method: string, params: unknown): Promise<unknown> {
|