@volter/world-runtime 2.0.24 → 2.0.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/host-cli.js +14 -3
- package/dist/src/host-ready.d.ts +4 -0
- package/dist/src/host-ready.js +48 -0
- package/dist/src/host.js +58 -17
- package/dist/src/runtime.js +50 -17
- package/dist/src/startup.d.ts +2 -0
- package/dist/src/startup.js +24 -0
- package/package.json +1 -1
- package/src/host-cli.ts +8 -3
- package/src/host-ready.ts +41 -0
- package/src/host.ts +50 -20
- package/src/runtime.ts +34 -14
- package/src/startup.ts +16 -0
package/dist/src/host-cli.js
CHANGED
|
@@ -13,6 +13,7 @@ import { keepProcessAlive } from '@volter/world-core/lifecycle';
|
|
|
13
13
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
14
14
|
import { dirname, join } from 'node:path';
|
|
15
15
|
import { startColocatedHost } from "./host.js";
|
|
16
|
+
import { publishHostReady } from "./host-ready.js";
|
|
16
17
|
const argv = process.argv.slice(2);
|
|
17
18
|
const opt = (name, fb = '') => { const i = argv.indexOf(name); return i >= 0 && argv[i + 1] ? argv[i + 1] : fb; };
|
|
18
19
|
const isolation = opt('--isolation', 'shared');
|
|
@@ -85,10 +86,20 @@ const host = await startColocatedHost(specs, {
|
|
|
85
86
|
recordGaveUp(e.twin, e.exits ?? 0, e.detail ?? '');
|
|
86
87
|
},
|
|
87
88
|
});
|
|
88
|
-
for (const [id, port] of Object.entries(host.ports))
|
|
89
|
-
process.stdout.write(`ready id=${id} port=${port}\n`);
|
|
90
|
-
process.stdout.write(`host ready (${specs.length} twins, isolation=${isolation})\n`);
|
|
91
89
|
const shutdown = async () => { await host.stop(); process.exit(0); };
|
|
92
90
|
process.on('SIGTERM', shutdown);
|
|
93
91
|
process.on('SIGINT', shutdown);
|
|
92
|
+
const readyFile = opt('--ready-file');
|
|
93
|
+
if (readyFile) {
|
|
94
|
+
try {
|
|
95
|
+
publishHostReady(readyFile, process.env.VOLTER_WORLD_BOOT_ID ?? '', host.ports);
|
|
96
|
+
}
|
|
97
|
+
catch (error) {
|
|
98
|
+
await host.stop();
|
|
99
|
+
throw error;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
for (const [id, port] of Object.entries(host.ports))
|
|
103
|
+
process.stdout.write(`ready id=${id} port=${port}\n`);
|
|
104
|
+
process.stdout.write(`host ready (${specs.length} twins, isolation=${isolation})\n`);
|
|
94
105
|
await keepProcessAlive();
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/** Only the host, after all starts have returned and ports were checked, publishes this receipt. */
|
|
2
|
+
export declare function publishHostReady(path: string, boot: string, ports: Record<string, number>): void;
|
|
3
|
+
/** Receipt admission is finite and cancellation-aware; stale or malformed state is never readiness. */
|
|
4
|
+
export declare function waitForHostReady(path: string, boot: string, ports: ReadonlyMap<string, number>, timeoutMs: number, abort: AbortSignal): Promise<void>;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { dirname } from 'node:path';
|
|
3
|
+
/** Only the host, after all starts have returned and ports were checked, publishes this receipt. */
|
|
4
|
+
export function publishHostReady(path, boot, ports) {
|
|
5
|
+
if (!boot)
|
|
6
|
+
throw new Error('A co-located host readiness receipt requires its World boot identity.');
|
|
7
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
8
|
+
const temporary = `${path}.${process.pid}.tmp`;
|
|
9
|
+
writeFileSync(temporary, `${JSON.stringify({ version: 1, boot, ports })}\n`, { mode: 0o600 });
|
|
10
|
+
renameSync(temporary, path);
|
|
11
|
+
}
|
|
12
|
+
/** Receipt admission is finite and cancellation-aware; stale or malformed state is never readiness. */
|
|
13
|
+
export async function waitForHostReady(path, boot, ports, timeoutMs, abort) {
|
|
14
|
+
const started = Date.now();
|
|
15
|
+
for (;;) {
|
|
16
|
+
abort.throwIfAborted();
|
|
17
|
+
let bytes;
|
|
18
|
+
try {
|
|
19
|
+
bytes = readFileSync(path);
|
|
20
|
+
}
|
|
21
|
+
catch (error) {
|
|
22
|
+
if (error.code !== 'ENOENT')
|
|
23
|
+
throw error;
|
|
24
|
+
}
|
|
25
|
+
if (bytes) {
|
|
26
|
+
if (bytes.byteLength > 256 * 1024)
|
|
27
|
+
throw new Error('The co-located host readiness receipt exceeds its bounds.');
|
|
28
|
+
const value = JSON.parse(bytes.toString('utf8'));
|
|
29
|
+
if (!value || value.version !== 1 || value.boot !== boot || !value.ports || typeof value.ports !== 'object' || Array.isArray(value.ports)) {
|
|
30
|
+
throw new Error('The co-located host readiness receipt does not belong to this boot.');
|
|
31
|
+
}
|
|
32
|
+
const answered = value.ports;
|
|
33
|
+
if (Object.keys(answered).length !== ports.size || [...ports].some(([id, port]) => !Object.hasOwn(answered, id) || answered[id] !== port)) {
|
|
34
|
+
throw new Error('The co-located host readiness receipt does not cover its declared ports.');
|
|
35
|
+
}
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
if (Date.now() - started >= timeoutMs)
|
|
39
|
+
throw new Error(`The co-located host did not complete every declared start within ${timeoutMs} ms.`);
|
|
40
|
+
await new Promise((resolve, reject) => {
|
|
41
|
+
const cancel = () => { clearTimeout(timer); abort.removeEventListener('abort', cancel); reject(abort.reason); };
|
|
42
|
+
const timer = setTimeout(() => { abort.removeEventListener('abort', cancel); resolve(); }, Math.min(25, Math.max(1, timeoutMs - (Date.now() - started))));
|
|
43
|
+
abort.addEventListener('abort', cancel, { once: true });
|
|
44
|
+
if (abort.aborted)
|
|
45
|
+
cancel();
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
}
|
package/dist/src/host.js
CHANGED
|
@@ -23,6 +23,7 @@ import { Worker } from 'node:worker_threads';
|
|
|
23
23
|
import { siblingScript } from "./sibling.js";
|
|
24
24
|
import { importModule } from "./import-module.js";
|
|
25
25
|
import { pathToFileURL } from 'node:url';
|
|
26
|
+
import { parallelStartup } from "./startup.js";
|
|
26
27
|
async function loadFactory(spec) {
|
|
27
28
|
const mod = await importModule(spec.module);
|
|
28
29
|
const factory = mod[spec.export];
|
|
@@ -60,26 +61,50 @@ export async function startColocatedHost(specs, options = {}) {
|
|
|
60
61
|
process.on('unhandledRejection', onRejection);
|
|
61
62
|
process.on('uncaughtException', onUncaught);
|
|
62
63
|
const servers = [];
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
64
|
+
const stop = async () => {
|
|
65
|
+
process.off('unhandledRejection', onRejection);
|
|
66
|
+
process.off('uncaughtException', onUncaught);
|
|
67
|
+
const stopped = await Promise.allSettled(servers.map(async (server) => {
|
|
68
|
+
let timeout;
|
|
69
|
+
try {
|
|
70
|
+
await Promise.race([
|
|
71
|
+
Promise.resolve().then(() => server.stop()),
|
|
72
|
+
new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error(`Twin on port ${server.port} did not stop within ${STOP_TIMEOUT_MS} ms.`)), STOP_TIMEOUT_MS); }),
|
|
73
|
+
]);
|
|
74
|
+
}
|
|
75
|
+
finally {
|
|
76
|
+
if (timeout !== undefined)
|
|
77
|
+
clearTimeout(timeout);
|
|
78
|
+
}
|
|
79
|
+
}));
|
|
80
|
+
return stopped.filter((result) => result.status === 'rejected').map((result) => result.reason);
|
|
81
|
+
};
|
|
82
|
+
try {
|
|
83
|
+
await parallelStartup(specs, async (spec) => {
|
|
84
|
+
const factory = await loadFactory(spec);
|
|
85
|
+
const server = await factory({ port: spec.port, root: spec.root, readOnly: spec.readOnly, ...(spec.scenarioPath !== undefined ? { scenarioPath: spec.scenarioPath } : {}), ...(spec.database !== undefined ? { database: spec.database } : {}) });
|
|
86
|
+
// Record the handle before validating it, so rollback also owns a bad factory result.
|
|
87
|
+
servers.push(server);
|
|
88
|
+
if (!Number.isInteger(server.port) || server.port < 1 || server.port > 65535 || (spec.port !== 0 && server.port !== spec.port)) {
|
|
89
|
+
throw new Error(`Twin "${spec.id}" bound port ${server.port}, requested ${spec.port}.`);
|
|
90
|
+
}
|
|
91
|
+
ports[spec.id] = server.port;
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
catch (error) {
|
|
95
|
+
const cleanup = await stop();
|
|
96
|
+
if (cleanup.length)
|
|
97
|
+
throw new AggregateError([error, ...cleanup], 'Co-located host startup and rollback failed.');
|
|
98
|
+
throw error;
|
|
66
99
|
}
|
|
67
100
|
return {
|
|
68
101
|
isolation,
|
|
69
102
|
ports,
|
|
70
103
|
stop: async () => {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
// The awaited-sequential first cut re-created the SIGKILL escalation it was fixing
|
|
76
|
-
// through a new door (§9 round two H2, 2026-08-31: one pending/throwing stop — fly's
|
|
77
|
-
// cleanup throws on a failed container removal — left siblings unstopped and the host
|
|
78
|
-
// ignoring SIGTERM forever). The bound stays under downWorld's 5s SIGKILL grace.
|
|
79
|
-
await Promise.allSettled(servers.map((s) => Promise.race([
|
|
80
|
-
Promise.resolve().then(() => s.stop()),
|
|
81
|
-
new Promise((resolveTimeout) => setTimeout(resolveTimeout, STOP_TIMEOUT_MS).unref?.()),
|
|
82
|
-
])));
|
|
104
|
+
// Every stop runs, even when a sibling hangs or throws. Host shutdown
|
|
105
|
+
// still resolves within its budget so the owning runtime can retire it.
|
|
106
|
+
for (const error of await stop())
|
|
107
|
+
onEvent({ twin: '(host)', kind: 'stop-failed', detail: String(error) });
|
|
83
108
|
},
|
|
84
109
|
};
|
|
85
110
|
}
|
|
@@ -96,6 +121,11 @@ export async function startColocatedHost(specs, options = {}) {
|
|
|
96
121
|
let ready = false;
|
|
97
122
|
worker.on('message', (msg) => {
|
|
98
123
|
if (msg?.type === 'ready') {
|
|
124
|
+
if (typeof msg.port !== 'number' || !Number.isInteger(msg.port) || msg.port < 1 || msg.port > 65535 || (spec.port !== 0 && msg.port !== spec.port)) {
|
|
125
|
+
rejectReady(new Error(`Twin "${spec.id}" bound port ${msg.port}, requested ${spec.port}.`));
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
ports[spec.id] = msg.port;
|
|
99
129
|
ready = true;
|
|
100
130
|
resolveReady();
|
|
101
131
|
}
|
|
@@ -107,6 +137,8 @@ export async function startColocatedHost(specs, options = {}) {
|
|
|
107
137
|
});
|
|
108
138
|
worker.on('exit', (code) => {
|
|
109
139
|
workers.delete(spec.id);
|
|
140
|
+
if (!ready)
|
|
141
|
+
rejectReady(new Error(`Twin "${spec.id}" exited before readiness (code ${code}).`));
|
|
110
142
|
if (stopping.value || code === 0)
|
|
111
143
|
return;
|
|
112
144
|
// Minimal-primitive doctrine: no keep-alive supervisor. A crashed worker is NOT
|
|
@@ -115,8 +147,17 @@ export async function startColocatedHost(specs, options = {}) {
|
|
|
115
147
|
});
|
|
116
148
|
workers.set(spec.id, worker);
|
|
117
149
|
});
|
|
118
|
-
|
|
119
|
-
await spawnWorker(spec);
|
|
150
|
+
try {
|
|
151
|
+
await parallelStartup(specs, async (spec) => spawnWorker(spec));
|
|
152
|
+
}
|
|
153
|
+
catch (error) {
|
|
154
|
+
stopping.value = true;
|
|
155
|
+
const stopped = await Promise.allSettled([...workers.values()].map((worker) => worker.terminate()));
|
|
156
|
+
const cleanup = stopped.filter((result) => result.status === 'rejected').map((result) => result.reason);
|
|
157
|
+
if (cleanup.length)
|
|
158
|
+
throw new AggregateError([error, ...cleanup], 'Co-located worker startup and rollback failed.');
|
|
159
|
+
throw error;
|
|
160
|
+
}
|
|
120
161
|
return {
|
|
121
162
|
isolation,
|
|
122
163
|
ports,
|
package/dist/src/runtime.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { claimEnvOutput, publishEnvOutput } from "./env-output.js";
|
|
2
2
|
import { offSignal } from "./signals.js";
|
|
3
|
+
import { parallelStartup } from "./startup.js";
|
|
4
|
+
import { waitForHostReady } from "./host-ready.js";
|
|
3
5
|
import { assertWorldStateRemovable, removeWorldState } from "./ancestry.js";
|
|
4
6
|
import { packCli, packEntry } from "./catalog.js";
|
|
5
7
|
import { spawn, spawnSync } from 'node:child_process';
|
|
@@ -110,16 +112,25 @@ async function untilReadyOrEnded(wait, ended, boot, what, log) {
|
|
|
110
112
|
* port alone (`early`), a twin's answer with this boot's id ends it early: a server of this boot holds the port, and
|
|
111
113
|
* every port this boot assigns is kept apart from its siblings', so short of the kernel handing the just-probed port
|
|
112
114
|
* to another of this boot's port-0 listeners, that server is the service just started. A settle standing for several
|
|
113
|
-
* ports
|
|
114
|
-
*
|
|
115
|
-
async function confirmOwnPort(port, boot, ended, twin, what, log, early = false) {
|
|
115
|
+
* ports runs in full unless an all-started receipt permits each port's own early identity confirmation.
|
|
116
|
+
* Programs that are not twins are never asked. A shared settle keeps legacy co-located ports within one guard. */
|
|
117
|
+
async function confirmOwnPort(port, boot, ended, twin, what, log, early = false, sharedSettle) {
|
|
116
118
|
// A twin is asked at once, raced with the settle: a server slow to answer costs no more than the settle did.
|
|
117
119
|
boot.signal.throwIfAborted();
|
|
118
120
|
const answer = twin ? worldBootOf(port, boot.signal) : Promise.resolve(null);
|
|
119
121
|
const own = early && twin ? answer.then((a) => (a && a.boot === boot.bootId ? 'own' : new Promise(() => undefined))) : new Promise(() => undefined);
|
|
120
122
|
// `ended` absent: the process was already settled (a co-located host settles once for all its twins).
|
|
121
123
|
if (ended) {
|
|
122
|
-
|
|
124
|
+
let timer;
|
|
125
|
+
let settled;
|
|
126
|
+
try {
|
|
127
|
+
const settle = sharedSettle ?? new Promise((resolveAlive) => { timer = setTimeout(() => resolveAlive('alive'), 300); });
|
|
128
|
+
settled = await Promise.race([own, ended.then(() => 'ended'), settle]);
|
|
129
|
+
}
|
|
130
|
+
finally {
|
|
131
|
+
if (timer !== undefined)
|
|
132
|
+
clearTimeout(timer);
|
|
133
|
+
}
|
|
123
134
|
boot.signal.throwIfAborted();
|
|
124
135
|
if (settled === 'own')
|
|
125
136
|
return;
|
|
@@ -1304,8 +1315,11 @@ function versionSatisfies(actual, pin) {
|
|
|
1304
1315
|
* all share the host child's pid — so pids/instance.json/downWorld need no special casing.
|
|
1305
1316
|
*/
|
|
1306
1317
|
async function startColocatedServices(config, colocated, isolation, worldEnv, paths, allocatedPorts, boot) {
|
|
1318
|
+
if (colocated.length === 0)
|
|
1319
|
+
return [];
|
|
1307
1320
|
const hostCli = siblingScript(import.meta.url, 'host-cli');
|
|
1308
|
-
const
|
|
1321
|
+
const readyFile = join(paths.instance, `host-ready-${boot.bootId}.json`);
|
|
1322
|
+
const args = [hostCli, '--isolation', isolation === 'colocated' ? 'shared' : 'worker', '--ready-file', readyFile];
|
|
1309
1323
|
const allocated = [];
|
|
1310
1324
|
for (const service of colocated) {
|
|
1311
1325
|
const port = service.port === undefined || service.port === 'auto' ? await allocatePort(allocatedPorts) : service.port;
|
|
@@ -1341,7 +1355,17 @@ async function startColocatedServices(config, colocated, isolation, worldEnv, pa
|
|
|
1341
1355
|
exitState.value = { code, signal };
|
|
1342
1356
|
});
|
|
1343
1357
|
const command = [process.execPath, ...args];
|
|
1344
|
-
|
|
1358
|
+
const hostTimeoutMs = allocated.reduce((timeout, { service }) => Math.max(timeout, typeof service.ready === 'object' ? service.ready.timeoutMs ?? 15_000 : 15_000), 15_000);
|
|
1359
|
+
await untilReadyOrEnded((abort) => waitForHostReady(readyFile, boot.bootId, new Map(allocated.map(({ service, port }) => [service.id, port])), hostTimeoutMs, abort), ended, boot, 'the co-located host', log).catch((error) => {
|
|
1360
|
+
if (undeadPids([child.pid]).length)
|
|
1361
|
+
recordWorldStop([log], 'world.service.stop', 'up (host startup incomplete)', 'SIGTERM');
|
|
1362
|
+
try {
|
|
1363
|
+
process.kill(-child.pid, 'SIGTERM');
|
|
1364
|
+
}
|
|
1365
|
+
catch { /* best effort cleanup */ }
|
|
1366
|
+
throw new Error(`Co-located host failed to complete startup (isolation=${isolation}): ${error instanceof Error ? error.message : String(error)}\nLog: ${log}`);
|
|
1367
|
+
});
|
|
1368
|
+
await parallelStartup(allocated, async ({ service, port }) => {
|
|
1345
1369
|
const bindTimeoutMs = typeof service.ready === 'object' && service.ready.timeoutMs !== undefined ? Math.max(service.ready.timeoutMs, 15_000) : 15_000;
|
|
1346
1370
|
await untilReadyOrEnded((abort) => waitForTcp(port, bindTimeoutMs, abort), ended, boot, 'the co-located host', log).catch((error) => {
|
|
1347
1371
|
if (undeadPids([child.pid]).length)
|
|
@@ -1352,19 +1376,28 @@ async function startColocatedServices(config, colocated, isolation, worldEnv, pa
|
|
|
1352
1376
|
catch { /* best effort cleanup */ }
|
|
1353
1377
|
throw new Error(`Co-located twin "${service.id}" failed to start (host isolation=${isolation}): ${error instanceof Error ? error.message : String(error)}\nLog: ${log}`);
|
|
1354
1378
|
});
|
|
1355
|
-
}
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1379
|
+
});
|
|
1380
|
+
let hostSettleTimer;
|
|
1381
|
+
const hostSettle = new Promise((resolveAlive) => { hostSettleTimer = setTimeout(() => resolveAlive('alive'), 300); });
|
|
1382
|
+
try {
|
|
1383
|
+
await parallelStartup(allocated, async ({ service, port }) => {
|
|
1384
|
+
// The host has completed every start. Each matching boot answer can now
|
|
1385
|
+
// end its own settle; legacy twins share one full host guard.
|
|
1386
|
+
await confirmOwnPort(port, boot, ended, true, `Co-located twin "${service.id}"`, log, true, hostSettle).catch((error) => {
|
|
1387
|
+
if (undeadPids([child.pid]).length)
|
|
1388
|
+
recordWorldStop([log], 'world.service.stop', 'up (port not its own)', 'SIGTERM');
|
|
1389
|
+
try {
|
|
1390
|
+
process.kill(-child.pid, 'SIGTERM');
|
|
1391
|
+
}
|
|
1392
|
+
catch { /* best effort cleanup */ }
|
|
1393
|
+
throw error;
|
|
1394
|
+
});
|
|
1366
1395
|
});
|
|
1367
1396
|
}
|
|
1397
|
+
finally {
|
|
1398
|
+
if (hostSettleTimer !== undefined)
|
|
1399
|
+
clearTimeout(hostSettleTimer);
|
|
1400
|
+
}
|
|
1368
1401
|
if (exitState.value !== null) {
|
|
1369
1402
|
throw new Error(`The co-located host ended during startup even though its ports answer, so other/stale processes probably hold them: the host ${endedAccount(log)}\n`
|
|
1370
1403
|
+ `Stop leftover services (e.g. \`volter-world down\`) and retry.\nLog: ${log}`);
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** Finite independent boot operations. Stop scheduling on failure and join every operation already begun. */
|
|
2
|
+
export async function parallelStartup(items, start) {
|
|
3
|
+
let next = 0;
|
|
4
|
+
let failed = false;
|
|
5
|
+
const errors = [];
|
|
6
|
+
await Promise.all(Array.from({ length: Math.min(8, items.length) }, async () => {
|
|
7
|
+
while (!failed) {
|
|
8
|
+
const index = next++;
|
|
9
|
+
if (index >= items.length)
|
|
10
|
+
return;
|
|
11
|
+
try {
|
|
12
|
+
await start(items[index], index);
|
|
13
|
+
}
|
|
14
|
+
catch (error) {
|
|
15
|
+
failed = true;
|
|
16
|
+
errors.push(error);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
}));
|
|
20
|
+
if (errors.length === 1)
|
|
21
|
+
throw errors[0];
|
|
22
|
+
if (errors.length > 1)
|
|
23
|
+
throw new AggregateError(errors, 'Independent World startup operations failed.');
|
|
24
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@volter/world-runtime",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.26",
|
|
4
4
|
"description": "World configs for twins: boot named local runtimes, allocate ports, generate world.env/instance.json, and run apps against fake-key twin worlds.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"twin",
|
package/src/host-cli.ts
CHANGED
|
@@ -13,6 +13,7 @@ import { keepProcessAlive } from '@volter/world-core/lifecycle';
|
|
|
13
13
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
14
14
|
import { dirname, join } from 'node:path';
|
|
15
15
|
import { startColocatedHost, type ColocatedTwinSpec, type HostIsolation } from './host.ts';
|
|
16
|
+
import { publishHostReady } from './host-ready.ts';
|
|
16
17
|
|
|
17
18
|
const argv = process.argv.slice(2);
|
|
18
19
|
const opt = (name: string, fb = ''): string => { const i = argv.indexOf(name); return i >= 0 && argv[i + 1] ? argv[i + 1]! : fb; };
|
|
@@ -83,10 +84,14 @@ const host = await startColocatedHost(specs, {
|
|
|
83
84
|
if (e.kind === 'gaveup') recordGaveUp(e.twin, e.exits ?? 0, e.detail ?? '');
|
|
84
85
|
},
|
|
85
86
|
});
|
|
86
|
-
for (const [id, port] of Object.entries(host.ports)) process.stdout.write(`ready id=${id} port=${port}\n`);
|
|
87
|
-
process.stdout.write(`host ready (${specs.length} twins, isolation=${isolation})\n`);
|
|
88
|
-
|
|
89
87
|
const shutdown = async () => { await host.stop(); process.exit(0); };
|
|
90
88
|
process.on('SIGTERM', shutdown);
|
|
91
89
|
process.on('SIGINT', shutdown);
|
|
90
|
+
const readyFile = opt('--ready-file');
|
|
91
|
+
if (readyFile) {
|
|
92
|
+
try { publishHostReady(readyFile, process.env.VOLTER_WORLD_BOOT_ID ?? '', host.ports); }
|
|
93
|
+
catch (error) { await host.stop(); throw error; }
|
|
94
|
+
}
|
|
95
|
+
for (const [id, port] of Object.entries(host.ports)) process.stdout.write(`ready id=${id} port=${port}\n`);
|
|
96
|
+
process.stdout.write(`host ready (${specs.length} twins, isolation=${isolation})\n`);
|
|
92
97
|
await keepProcessAlive();
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { dirname } from 'node:path';
|
|
3
|
+
|
|
4
|
+
/** Only the host, after all starts have returned and ports were checked, publishes this receipt. */
|
|
5
|
+
export function publishHostReady(path: string, boot: string, ports: Record<string, number>): void {
|
|
6
|
+
if (!boot) throw new Error('A co-located host readiness receipt requires its World boot identity.');
|
|
7
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
8
|
+
const temporary = `${path}.${process.pid}.tmp`;
|
|
9
|
+
writeFileSync(temporary, `${JSON.stringify({ version: 1, boot, ports })}\n`, { mode: 0o600 });
|
|
10
|
+
renameSync(temporary, path);
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/** Receipt admission is finite and cancellation-aware; stale or malformed state is never readiness. */
|
|
14
|
+
export async function waitForHostReady(path: string, boot: string, ports: ReadonlyMap<string, number>, timeoutMs: number, abort: AbortSignal): Promise<void> {
|
|
15
|
+
const started = Date.now();
|
|
16
|
+
for (;;) {
|
|
17
|
+
abort.throwIfAborted();
|
|
18
|
+
let bytes: Buffer | undefined;
|
|
19
|
+
try { bytes = readFileSync(path); }
|
|
20
|
+
catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
|
|
21
|
+
if (bytes) {
|
|
22
|
+
if (bytes.byteLength > 256 * 1024) throw new Error('The co-located host readiness receipt exceeds its bounds.');
|
|
23
|
+
const value = JSON.parse(bytes.toString('utf8')) as { version?: unknown; boot?: unknown; ports?: unknown } | null;
|
|
24
|
+
if (!value || value.version !== 1 || value.boot !== boot || !value.ports || typeof value.ports !== 'object' || Array.isArray(value.ports)) {
|
|
25
|
+
throw new Error('The co-located host readiness receipt does not belong to this boot.');
|
|
26
|
+
}
|
|
27
|
+
const answered = value.ports as Record<string, unknown>;
|
|
28
|
+
if (Object.keys(answered).length !== ports.size || [...ports].some(([id, port]) => !Object.hasOwn(answered, id) || answered[id] !== port)) {
|
|
29
|
+
throw new Error('The co-located host readiness receipt does not cover its declared ports.');
|
|
30
|
+
}
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
if (Date.now() - started >= timeoutMs) throw new Error(`The co-located host did not complete every declared start within ${timeoutMs} ms.`);
|
|
34
|
+
await new Promise<void>((resolve, reject) => {
|
|
35
|
+
const cancel = (): void => { clearTimeout(timer); abort.removeEventListener('abort', cancel); reject(abort.reason); };
|
|
36
|
+
const timer = setTimeout(() => { abort.removeEventListener('abort', cancel); resolve(); }, Math.min(25, Math.max(1, timeoutMs - (Date.now() - started))));
|
|
37
|
+
abort.addEventListener('abort', cancel, { once: true });
|
|
38
|
+
if (abort.aborted) cancel();
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
}
|
package/src/host.ts
CHANGED
|
@@ -23,6 +23,7 @@ import { Worker } from 'node:worker_threads';
|
|
|
23
23
|
import { siblingScript } from './sibling.ts';
|
|
24
24
|
import { importModule } from './import-module.ts';
|
|
25
25
|
import { pathToFileURL } from 'node:url';
|
|
26
|
+
import { parallelStartup } from './startup.ts';
|
|
26
27
|
|
|
27
28
|
/** One twin to mount in the host. `module`/`export` name a `({port,root,readOnly}) => {port,stop}`
|
|
28
29
|
* factory (every pack ships one, e.g. `createStripeTwinServer`). `module` is anything import()
|
|
@@ -104,28 +105,43 @@ export async function startColocatedHost(
|
|
|
104
105
|
process.on('uncaughtException', onUncaught);
|
|
105
106
|
|
|
106
107
|
const servers: TwinServer[] = [];
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
108
|
+
const stop = async (): Promise<unknown[]> => {
|
|
109
|
+
process.off('unhandledRejection', onRejection);
|
|
110
|
+
process.off('uncaughtException', onUncaught);
|
|
111
|
+
const stopped = await Promise.allSettled(servers.map(async (server) => {
|
|
112
|
+
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
113
|
+
try {
|
|
114
|
+
await Promise.race([
|
|
115
|
+
Promise.resolve().then(() => server.stop()),
|
|
116
|
+
new Promise<never>((_, reject) => { timeout = setTimeout(() => reject(new Error(`Twin on port ${server.port} did not stop within ${STOP_TIMEOUT_MS} ms.`)), STOP_TIMEOUT_MS); }),
|
|
117
|
+
]);
|
|
118
|
+
} finally { if (timeout !== undefined) clearTimeout(timeout); }
|
|
119
|
+
}));
|
|
120
|
+
return stopped.filter((result): result is PromiseRejectedResult => result.status === 'rejected').map((result) => result.reason);
|
|
121
|
+
};
|
|
122
|
+
try {
|
|
123
|
+
await parallelStartup(specs, async (spec) => {
|
|
124
|
+
const factory = await loadFactory(spec);
|
|
125
|
+
const server = await factory({ port: spec.port, root: spec.root, readOnly: spec.readOnly, ...(spec.scenarioPath !== undefined ? { scenarioPath: spec.scenarioPath } : {}), ...(spec.database !== undefined ? { database: spec.database } : {}) });
|
|
126
|
+
// Record the handle before validating it, so rollback also owns a bad factory result.
|
|
127
|
+
servers.push(server);
|
|
128
|
+
if (!Number.isInteger(server.port) || server.port < 1 || server.port > 65535 || (spec.port !== 0 && server.port !== spec.port)) {
|
|
129
|
+
throw new Error(`Twin "${spec.id}" bound port ${server.port}, requested ${spec.port}.`);
|
|
130
|
+
}
|
|
131
|
+
ports[spec.id] = server.port;
|
|
132
|
+
});
|
|
133
|
+
} catch (error) {
|
|
134
|
+
const cleanup = await stop();
|
|
135
|
+
if (cleanup.length) throw new AggregateError([error, ...cleanup], 'Co-located host startup and rollback failed.');
|
|
136
|
+
throw error;
|
|
110
137
|
}
|
|
111
138
|
return {
|
|
112
139
|
isolation,
|
|
113
140
|
ports,
|
|
114
141
|
stop: async () => {
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
// rejection, hang, or slowness — can prevent the others or wedge host.stop() itself.
|
|
119
|
-
// The awaited-sequential first cut re-created the SIGKILL escalation it was fixing
|
|
120
|
-
// through a new door (§9 round two H2, 2026-08-31: one pending/throwing stop — fly's
|
|
121
|
-
// cleanup throws on a failed container removal — left siblings unstopped and the host
|
|
122
|
-
// ignoring SIGTERM forever). The bound stays under downWorld's 5s SIGKILL grace.
|
|
123
|
-
await Promise.allSettled(servers.map((s) =>
|
|
124
|
-
Promise.race([
|
|
125
|
-
Promise.resolve().then(() => s.stop()),
|
|
126
|
-
new Promise((resolveTimeout) => setTimeout(resolveTimeout, STOP_TIMEOUT_MS).unref?.()),
|
|
127
|
-
]),
|
|
128
|
-
));
|
|
142
|
+
// Every stop runs, even when a sibling hangs or throws. Host shutdown
|
|
143
|
+
// still resolves within its budget so the owning runtime can retire it.
|
|
144
|
+
for (const error of await stop()) onEvent({ twin: '(host)', kind: 'stop-failed', detail: String(error) });
|
|
129
145
|
},
|
|
130
146
|
};
|
|
131
147
|
}
|
|
@@ -143,8 +159,14 @@ export async function startColocatedHost(
|
|
|
143
159
|
// below would emit a 2nd `spawn` for the same twin, which the isolation test asserts against.
|
|
144
160
|
onEvent({ twin: spec.id, kind: 'spawn', detail: 'worker started' });
|
|
145
161
|
let ready = false;
|
|
146
|
-
worker.on('message', (msg: { type?: string }) => {
|
|
147
|
-
if (msg?.type === 'ready') {
|
|
162
|
+
worker.on('message', (msg: { type?: string; port?: number }) => {
|
|
163
|
+
if (msg?.type === 'ready') {
|
|
164
|
+
if (typeof msg.port !== 'number' || !Number.isInteger(msg.port) || msg.port < 1 || msg.port > 65535 || (spec.port !== 0 && msg.port !== spec.port)) {
|
|
165
|
+
rejectReady(new Error(`Twin "${spec.id}" bound port ${msg.port}, requested ${spec.port}.`)); return;
|
|
166
|
+
}
|
|
167
|
+
ports[spec.id] = msg.port;
|
|
168
|
+
ready = true; resolveReady();
|
|
169
|
+
}
|
|
148
170
|
});
|
|
149
171
|
worker.on('error', (err) => {
|
|
150
172
|
onEvent({ twin: spec.id, kind: 'error', detail: String(err) });
|
|
@@ -152,6 +174,7 @@ export async function startColocatedHost(
|
|
|
152
174
|
});
|
|
153
175
|
worker.on('exit', (code) => {
|
|
154
176
|
workers.delete(spec.id);
|
|
177
|
+
if (!ready) rejectReady(new Error(`Twin "${spec.id}" exited before readiness (code ${code}).`));
|
|
155
178
|
if (stopping.value || code === 0) return;
|
|
156
179
|
// Minimal-primitive doctrine: no keep-alive supervisor. A crashed worker is NOT
|
|
157
180
|
// respawned — record the give-up (host-cli persists it → doctor red) and stay dead.
|
|
@@ -160,7 +183,14 @@ export async function startColocatedHost(
|
|
|
160
183
|
workers.set(spec.id, worker);
|
|
161
184
|
});
|
|
162
185
|
|
|
163
|
-
|
|
186
|
+
try { await parallelStartup(specs, async (spec) => spawnWorker(spec)); }
|
|
187
|
+
catch (error) {
|
|
188
|
+
stopping.value = true;
|
|
189
|
+
const stopped = await Promise.allSettled([...workers.values()].map((worker) => worker.terminate()));
|
|
190
|
+
const cleanup = stopped.filter((result): result is PromiseRejectedResult => result.status === 'rejected').map((result) => result.reason);
|
|
191
|
+
if (cleanup.length) throw new AggregateError([error, ...cleanup], 'Co-located worker startup and rollback failed.');
|
|
192
|
+
throw error;
|
|
193
|
+
}
|
|
164
194
|
|
|
165
195
|
return {
|
|
166
196
|
isolation,
|
package/src/runtime.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { claimEnvOutput, publishEnvOutput } from './env-output.ts';
|
|
2
2
|
import { offSignal } from './signals.ts';
|
|
3
|
+
import { parallelStartup } from './startup.ts';
|
|
4
|
+
import { waitForHostReady } from './host-ready.ts';
|
|
3
5
|
import { assertWorldStateRemovable, removeWorldState } from './ancestry.ts';
|
|
4
6
|
import { packCli, packEntry } from './catalog.ts';
|
|
5
7
|
import { spawn, spawnSync } from 'node:child_process';
|
|
@@ -104,16 +106,21 @@ async function untilReadyOrEnded<T>(wait: (abort: AbortSignal) => Promise<T>, en
|
|
|
104
106
|
* port alone (`early`), a twin's answer with this boot's id ends it early: a server of this boot holds the port, and
|
|
105
107
|
* every port this boot assigns is kept apart from its siblings', so short of the kernel handing the just-probed port
|
|
106
108
|
* to another of this boot's port-0 listeners, that server is the service just started. A settle standing for several
|
|
107
|
-
* ports
|
|
108
|
-
*
|
|
109
|
-
async function confirmOwnPort(port: number, boot: BootContext, ended: Promise<void> | undefined, twin: boolean, what: string, log: string, early = false): Promise<void> {
|
|
109
|
+
* ports runs in full unless an all-started receipt permits each port's own early identity confirmation.
|
|
110
|
+
* Programs that are not twins are never asked. A shared settle keeps legacy co-located ports within one guard. */
|
|
111
|
+
async function confirmOwnPort(port: number, boot: BootContext, ended: Promise<void> | undefined, twin: boolean, what: string, log: string, early = false, sharedSettle?: Promise<'alive'>): Promise<void> {
|
|
110
112
|
// A twin is asked at once, raced with the settle: a server slow to answer costs no more than the settle did.
|
|
111
113
|
boot.signal.throwIfAborted();
|
|
112
114
|
const answer = twin ? worldBootOf(port, boot.signal) : Promise.resolve(null);
|
|
113
115
|
const own = early && twin ? answer.then((a) => (a && a.boot === boot.bootId ? 'own' as const : new Promise<never>(() => undefined))) : new Promise<never>(() => undefined);
|
|
114
116
|
// `ended` absent: the process was already settled (a co-located host settles once for all its twins).
|
|
115
117
|
if (ended) {
|
|
116
|
-
|
|
118
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
119
|
+
let settled: 'own' | 'ended' | 'alive';
|
|
120
|
+
try {
|
|
121
|
+
const settle = sharedSettle ?? new Promise<'alive'>((resolveAlive) => { timer = setTimeout(() => resolveAlive('alive'), 300); });
|
|
122
|
+
settled = await Promise.race([own, ended.then(() => 'ended' as const), settle]);
|
|
123
|
+
} finally { if (timer !== undefined) clearTimeout(timer); }
|
|
117
124
|
boot.signal.throwIfAborted();
|
|
118
125
|
if (settled === 'own') return;
|
|
119
126
|
if (settled === 'ended') throw new Error(`${what} ended right after its port first answered, so another process probably holds the port: the service ${endedAccount(log)}\nLog: ${log}`);
|
|
@@ -1321,8 +1328,10 @@ async function startColocatedServices(
|
|
|
1321
1328
|
allocatedPorts: Set<number>,
|
|
1322
1329
|
boot: BootContext,
|
|
1323
1330
|
): Promise<WorldServiceInstance[]> {
|
|
1331
|
+
if (colocated.length === 0) return [];
|
|
1324
1332
|
const hostCli = siblingScript(import.meta.url, 'host-cli');
|
|
1325
|
-
const
|
|
1333
|
+
const readyFile = join(paths.instance, `host-ready-${boot.bootId}.json`);
|
|
1334
|
+
const args: string[] = [hostCli, '--isolation', isolation === 'colocated' ? 'shared' : 'worker', '--ready-file', readyFile];
|
|
1326
1335
|
const allocated: Array<{ service: WorldServiceConfig; port: number; mounted: MountedPackage }> = [];
|
|
1327
1336
|
for (const service of colocated) {
|
|
1328
1337
|
const port = service.port === undefined || service.port === 'auto' ? await allocatePort(allocatedPorts) : service.port;
|
|
@@ -1360,22 +1369,33 @@ async function startColocatedServices(
|
|
|
1360
1369
|
});
|
|
1361
1370
|
|
|
1362
1371
|
const command = [process.execPath, ...args];
|
|
1363
|
-
|
|
1372
|
+
const hostTimeoutMs = allocated.reduce((timeout, { service }) => Math.max(timeout, typeof service.ready === 'object' ? service.ready.timeoutMs ?? 15_000 : 15_000), 15_000);
|
|
1373
|
+
await untilReadyOrEnded((abort) => waitForHostReady(readyFile, boot.bootId, new Map(allocated.map(({ service, port }) => [service.id, port])), hostTimeoutMs, abort), ended, boot, 'the co-located host', log).catch((error) => {
|
|
1374
|
+
if (undeadPids([child.pid!]).length) recordWorldStop([log], 'world.service.stop', 'up (host startup incomplete)', 'SIGTERM');
|
|
1375
|
+
try { process.kill(-child.pid!, 'SIGTERM'); } catch { /* best effort cleanup */ }
|
|
1376
|
+
throw new Error(`Co-located host failed to complete startup (isolation=${isolation}): ${error instanceof Error ? error.message : String(error)}\nLog: ${log}`);
|
|
1377
|
+
});
|
|
1378
|
+
await parallelStartup(allocated, async ({ service, port }) => {
|
|
1364
1379
|
const bindTimeoutMs = typeof service.ready === 'object' && service.ready.timeoutMs !== undefined ? Math.max(service.ready.timeoutMs, 15_000) : 15_000;
|
|
1365
1380
|
await untilReadyOrEnded((abort) => waitForTcp(port, bindTimeoutMs, abort), ended, boot, 'the co-located host', log).catch((error) => {
|
|
1366
1381
|
if (undeadPids([child.pid!]).length) recordWorldStop([log], 'world.service.stop', 'up (startup failed)', 'SIGTERM');
|
|
1367
1382
|
try { process.kill(-child.pid!, 'SIGTERM'); } catch { /* best effort cleanup */ }
|
|
1368
1383
|
throw new Error(`Co-located twin "${service.id}" failed to start (host isolation=${isolation}): ${error instanceof Error ? error.message : String(error)}\nLog: ${log}`);
|
|
1369
1384
|
});
|
|
1370
|
-
}
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1385
|
+
});
|
|
1386
|
+
let hostSettleTimer: ReturnType<typeof setTimeout> | undefined;
|
|
1387
|
+
const hostSettle = new Promise<'alive'>((resolveAlive) => { hostSettleTimer = setTimeout(() => resolveAlive('alive'), 300); });
|
|
1388
|
+
try {
|
|
1389
|
+
await parallelStartup(allocated, async ({ service, port }) => {
|
|
1390
|
+
// The host has completed every start. Each matching boot answer can now
|
|
1391
|
+
// end its own settle; legacy twins share one full host guard.
|
|
1392
|
+
await confirmOwnPort(port, boot, ended, true, `Co-located twin "${service.id}"`, log, true, hostSettle).catch((error) => {
|
|
1393
|
+
if (undeadPids([child.pid!]).length) recordWorldStop([log], 'world.service.stop', 'up (port not its own)', 'SIGTERM');
|
|
1394
|
+
try { process.kill(-child.pid!, 'SIGTERM'); } catch { /* best effort cleanup */ }
|
|
1395
|
+
throw error;
|
|
1396
|
+
});
|
|
1377
1397
|
});
|
|
1378
|
-
}
|
|
1398
|
+
} finally { if (hostSettleTimer !== undefined) clearTimeout(hostSettleTimer); }
|
|
1379
1399
|
if (exitState.value !== null) {
|
|
1380
1400
|
throw new Error(
|
|
1381
1401
|
`The co-located host ended during startup even though its ports answer, so other/stale processes probably hold them: the host ${endedAccount(log)}\n`
|
package/src/startup.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/** Finite independent boot operations. Stop scheduling on failure and join every operation already begun. */
|
|
2
|
+
export async function parallelStartup<T>(items: readonly T[], start: (item: T, index: number) => Promise<void>): Promise<void> {
|
|
3
|
+
let next = 0;
|
|
4
|
+
let failed = false;
|
|
5
|
+
const errors: unknown[] = [];
|
|
6
|
+
await Promise.all(Array.from({ length: Math.min(8, items.length) }, async () => {
|
|
7
|
+
while (!failed) {
|
|
8
|
+
const index = next++;
|
|
9
|
+
if (index >= items.length) return;
|
|
10
|
+
try { await start(items[index]!, index); }
|
|
11
|
+
catch (error) { failed = true; errors.push(error); }
|
|
12
|
+
}
|
|
13
|
+
}));
|
|
14
|
+
if (errors.length === 1) throw errors[0];
|
|
15
|
+
if (errors.length > 1) throw new AggregateError(errors, 'Independent World startup operations failed.');
|
|
16
|
+
}
|