@volter/world-runtime 2.0.23 → 2.0.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/host.js +58 -17
- package/dist/src/runtime.js +62 -20
- package/dist/src/startup.d.ts +2 -0
- package/dist/src/startup.js +24 -0
- package/package.json +1 -1
- package/src/host.ts +50 -20
- package/src/runtime.ts +48 -15
- package/src/startup.ts +16 -0
package/dist/src/host.js
CHANGED
|
@@ -23,6 +23,7 @@ import { Worker } from 'node:worker_threads';
|
|
|
23
23
|
import { siblingScript } from "./sibling.js";
|
|
24
24
|
import { importModule } from "./import-module.js";
|
|
25
25
|
import { pathToFileURL } from 'node:url';
|
|
26
|
+
import { parallelStartup } from "./startup.js";
|
|
26
27
|
async function loadFactory(spec) {
|
|
27
28
|
const mod = await importModule(spec.module);
|
|
28
29
|
const factory = mod[spec.export];
|
|
@@ -60,26 +61,50 @@ export async function startColocatedHost(specs, options = {}) {
|
|
|
60
61
|
process.on('unhandledRejection', onRejection);
|
|
61
62
|
process.on('uncaughtException', onUncaught);
|
|
62
63
|
const servers = [];
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
64
|
+
const stop = async () => {
|
|
65
|
+
process.off('unhandledRejection', onRejection);
|
|
66
|
+
process.off('uncaughtException', onUncaught);
|
|
67
|
+
const stopped = await Promise.allSettled(servers.map(async (server) => {
|
|
68
|
+
let timeout;
|
|
69
|
+
try {
|
|
70
|
+
await Promise.race([
|
|
71
|
+
Promise.resolve().then(() => server.stop()),
|
|
72
|
+
new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error(`Twin on port ${server.port} did not stop within ${STOP_TIMEOUT_MS} ms.`)), STOP_TIMEOUT_MS); }),
|
|
73
|
+
]);
|
|
74
|
+
}
|
|
75
|
+
finally {
|
|
76
|
+
if (timeout !== undefined)
|
|
77
|
+
clearTimeout(timeout);
|
|
78
|
+
}
|
|
79
|
+
}));
|
|
80
|
+
return stopped.filter((result) => result.status === 'rejected').map((result) => result.reason);
|
|
81
|
+
};
|
|
82
|
+
try {
|
|
83
|
+
await parallelStartup(specs, async (spec) => {
|
|
84
|
+
const factory = await loadFactory(spec);
|
|
85
|
+
const server = await factory({ port: spec.port, root: spec.root, readOnly: spec.readOnly, ...(spec.scenarioPath !== undefined ? { scenarioPath: spec.scenarioPath } : {}), ...(spec.database !== undefined ? { database: spec.database } : {}) });
|
|
86
|
+
// Record the handle before validating it, so rollback also owns a bad factory result.
|
|
87
|
+
servers.push(server);
|
|
88
|
+
if (!Number.isInteger(server.port) || server.port < 1 || server.port > 65535 || (spec.port !== 0 && server.port !== spec.port)) {
|
|
89
|
+
throw new Error(`Twin "${spec.id}" bound port ${server.port}, requested ${spec.port}.`);
|
|
90
|
+
}
|
|
91
|
+
ports[spec.id] = server.port;
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
catch (error) {
|
|
95
|
+
const cleanup = await stop();
|
|
96
|
+
if (cleanup.length)
|
|
97
|
+
throw new AggregateError([error, ...cleanup], 'Co-located host startup and rollback failed.');
|
|
98
|
+
throw error;
|
|
66
99
|
}
|
|
67
100
|
return {
|
|
68
101
|
isolation,
|
|
69
102
|
ports,
|
|
70
103
|
stop: async () => {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
// The awaited-sequential first cut re-created the SIGKILL escalation it was fixing
|
|
76
|
-
// through a new door (§9 round two H2, 2026-08-31: one pending/throwing stop — fly's
|
|
77
|
-
// cleanup throws on a failed container removal — left siblings unstopped and the host
|
|
78
|
-
// ignoring SIGTERM forever). The bound stays under downWorld's 5s SIGKILL grace.
|
|
79
|
-
await Promise.allSettled(servers.map((s) => Promise.race([
|
|
80
|
-
Promise.resolve().then(() => s.stop()),
|
|
81
|
-
new Promise((resolveTimeout) => setTimeout(resolveTimeout, STOP_TIMEOUT_MS).unref?.()),
|
|
82
|
-
])));
|
|
104
|
+
// Every stop runs, even when a sibling hangs or throws. Host shutdown
|
|
105
|
+
// still resolves within its budget so the owning runtime can retire it.
|
|
106
|
+
for (const error of await stop())
|
|
107
|
+
onEvent({ twin: '(host)', kind: 'stop-failed', detail: String(error) });
|
|
83
108
|
},
|
|
84
109
|
};
|
|
85
110
|
}
|
|
@@ -96,6 +121,11 @@ export async function startColocatedHost(specs, options = {}) {
|
|
|
96
121
|
let ready = false;
|
|
97
122
|
worker.on('message', (msg) => {
|
|
98
123
|
if (msg?.type === 'ready') {
|
|
124
|
+
if (typeof msg.port !== 'number' || !Number.isInteger(msg.port) || msg.port < 1 || msg.port > 65535 || (spec.port !== 0 && msg.port !== spec.port)) {
|
|
125
|
+
rejectReady(new Error(`Twin "${spec.id}" bound port ${msg.port}, requested ${spec.port}.`));
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
ports[spec.id] = msg.port;
|
|
99
129
|
ready = true;
|
|
100
130
|
resolveReady();
|
|
101
131
|
}
|
|
@@ -107,6 +137,8 @@ export async function startColocatedHost(specs, options = {}) {
|
|
|
107
137
|
});
|
|
108
138
|
worker.on('exit', (code) => {
|
|
109
139
|
workers.delete(spec.id);
|
|
140
|
+
if (!ready)
|
|
141
|
+
rejectReady(new Error(`Twin "${spec.id}" exited before readiness (code ${code}).`));
|
|
110
142
|
if (stopping.value || code === 0)
|
|
111
143
|
return;
|
|
112
144
|
// Minimal-primitive doctrine: no keep-alive supervisor. A crashed worker is NOT
|
|
@@ -115,8 +147,17 @@ export async function startColocatedHost(specs, options = {}) {
|
|
|
115
147
|
});
|
|
116
148
|
workers.set(spec.id, worker);
|
|
117
149
|
});
|
|
118
|
-
|
|
119
|
-
await spawnWorker(spec);
|
|
150
|
+
try {
|
|
151
|
+
await parallelStartup(specs, async (spec) => spawnWorker(spec));
|
|
152
|
+
}
|
|
153
|
+
catch (error) {
|
|
154
|
+
stopping.value = true;
|
|
155
|
+
const stopped = await Promise.allSettled([...workers.values()].map((worker) => worker.terminate()));
|
|
156
|
+
const cleanup = stopped.filter((result) => result.status === 'rejected').map((result) => result.reason);
|
|
157
|
+
if (cleanup.length)
|
|
158
|
+
throw new AggregateError([error, ...cleanup], 'Co-located worker startup and rollback failed.');
|
|
159
|
+
throw error;
|
|
160
|
+
}
|
|
120
161
|
return {
|
|
121
162
|
isolation,
|
|
122
163
|
ports,
|
package/dist/src/runtime.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { claimEnvOutput, publishEnvOutput } from "./env-output.js";
|
|
2
2
|
import { offSignal } from "./signals.js";
|
|
3
|
+
import { parallelStartup } from "./startup.js";
|
|
3
4
|
import { assertWorldStateRemovable, removeWorldState } from "./ancestry.js";
|
|
4
5
|
import { packCli, packEntry } from "./catalog.js";
|
|
5
6
|
import { spawn, spawnSync } from 'node:child_process';
|
|
@@ -114,11 +115,13 @@ async function untilReadyOrEnded(wait, ended, boot, what, log) {
|
|
|
114
115
|
* always runs in full: one port's answer says nothing of the others'. Programs that are not twins are never asked. */
|
|
115
116
|
async function confirmOwnPort(port, boot, ended, twin, what, log, early = false) {
|
|
116
117
|
// A twin is asked at once, raced with the settle: a server slow to answer costs no more than the settle did.
|
|
117
|
-
|
|
118
|
+
boot.signal.throwIfAborted();
|
|
119
|
+
const answer = twin ? worldBootOf(port, boot.signal) : Promise.resolve(null);
|
|
118
120
|
const own = early && twin ? answer.then((a) => (a && a.boot === boot.bootId ? 'own' : new Promise(() => undefined))) : new Promise(() => undefined);
|
|
119
121
|
// `ended` absent: the process was already settled (a co-located host settles once for all its twins).
|
|
120
122
|
if (ended) {
|
|
121
123
|
const settled = await Promise.race([own, ended.then(() => 'ended'), new Promise((resolveAlive) => setTimeout(() => resolveAlive('alive'), 300))]);
|
|
124
|
+
boot.signal.throwIfAborted();
|
|
122
125
|
if (settled === 'own')
|
|
123
126
|
return;
|
|
124
127
|
if (settled === 'ended')
|
|
@@ -127,28 +130,65 @@ async function confirmOwnPort(port, boot, ended, twin, what, log, early = false)
|
|
|
127
130
|
if (!twin)
|
|
128
131
|
return;
|
|
129
132
|
const answered = await answer;
|
|
133
|
+
boot.signal.throwIfAborted();
|
|
130
134
|
if (answered && typeof answered.boot === 'string' && answered.boot !== boot.bootId) {
|
|
131
135
|
throw new Error(`${what}: its port is answered by a twin of another World boot (${answered.boot}), not this one; a leftover World holds the port. Stop it (\`volter-world list\`, then \`down\`) and retry.\nLog: ${log}`);
|
|
132
136
|
}
|
|
133
137
|
}
|
|
134
138
|
/** What the server on a loopback port says its World boot is (WORLD_BOOT_PATH), or null. A direct loopback request:
|
|
135
139
|
* never through a proxy an activated shell's environment names. */
|
|
136
|
-
function worldBootOf(port) {
|
|
140
|
+
function worldBootOf(port, abort) {
|
|
137
141
|
return new Promise((resolveBoot) => {
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
142
|
+
let request;
|
|
143
|
+
let deadline;
|
|
144
|
+
let settled = false;
|
|
145
|
+
let responseStarted = false;
|
|
146
|
+
const finish = (value) => {
|
|
147
|
+
if (settled)
|
|
148
|
+
return;
|
|
149
|
+
settled = true;
|
|
150
|
+
if (deadline !== undefined)
|
|
151
|
+
clearTimeout(deadline);
|
|
152
|
+
abort.removeEventListener('abort', cancel);
|
|
153
|
+
resolveBoot(value);
|
|
154
|
+
};
|
|
155
|
+
const cancel = () => { finish(null); request?.destroy(); };
|
|
156
|
+
if (abort.aborted) {
|
|
157
|
+
finish(null);
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
abort.addEventListener('abort', cancel, { once: true });
|
|
161
|
+
// A socket timeout alone is insufficient: destroying a request after
|
|
162
|
+
// headers can close/abort its response without emitting request 'error'.
|
|
163
|
+
// Bound the whole probe, including a peer that dribbles bytes forever.
|
|
164
|
+
deadline = setTimeout(cancel, 1000);
|
|
165
|
+
try {
|
|
166
|
+
request = httpGet({ host: '127.0.0.1', port, path: WORLD_BOOT_PATH, timeout: 1000, agent: false }, (response) => {
|
|
167
|
+
responseStarted = true;
|
|
168
|
+
let body = '';
|
|
169
|
+
response.on('error', cancel);
|
|
170
|
+
response.on('aborted', cancel);
|
|
171
|
+
response.on('close', () => finish(null));
|
|
172
|
+
response.setEncoding('utf8');
|
|
173
|
+
response.on('data', (chunk) => { body += chunk; if (body.length > 4096)
|
|
174
|
+
cancel(); });
|
|
175
|
+
response.on('end', () => {
|
|
176
|
+
try {
|
|
177
|
+
finish(response.statusCode === 200 ? JSON.parse(body) : null);
|
|
178
|
+
}
|
|
179
|
+
catch {
|
|
180
|
+
finish(null);
|
|
181
|
+
}
|
|
182
|
+
});
|
|
183
|
+
});
|
|
184
|
+
request.on('timeout', cancel);
|
|
185
|
+
request.on('error', () => finish(null));
|
|
186
|
+
request.on('close', () => { if (!responseStarted)
|
|
187
|
+
finish(null); });
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
cancel();
|
|
191
|
+
}
|
|
152
192
|
});
|
|
153
193
|
}
|
|
154
194
|
/** A stopped service's own account from its log: how it ended and the last lines it wrote. */
|
|
@@ -1265,6 +1305,8 @@ function versionSatisfies(actual, pin) {
|
|
|
1265
1305
|
* all share the host child's pid — so pids/instance.json/downWorld need no special casing.
|
|
1266
1306
|
*/
|
|
1267
1307
|
async function startColocatedServices(config, colocated, isolation, worldEnv, paths, allocatedPorts, boot) {
|
|
1308
|
+
if (colocated.length === 0)
|
|
1309
|
+
return [];
|
|
1268
1310
|
const hostCli = siblingScript(import.meta.url, 'host-cli');
|
|
1269
1311
|
const args = [hostCli, '--isolation', isolation === 'colocated' ? 'shared' : 'worker'];
|
|
1270
1312
|
const allocated = [];
|
|
@@ -1302,7 +1344,7 @@ async function startColocatedServices(config, colocated, isolation, worldEnv, pa
|
|
|
1302
1344
|
exitState.value = { code, signal };
|
|
1303
1345
|
});
|
|
1304
1346
|
const command = [process.execPath, ...args];
|
|
1305
|
-
|
|
1347
|
+
await parallelStartup(allocated, async ({ service, port }) => {
|
|
1306
1348
|
const bindTimeoutMs = typeof service.ready === 'object' && service.ready.timeoutMs !== undefined ? Math.max(service.ready.timeoutMs, 15_000) : 15_000;
|
|
1307
1349
|
await untilReadyOrEnded((abort) => waitForTcp(port, bindTimeoutMs, abort), ended, boot, 'the co-located host', log).catch((error) => {
|
|
1308
1350
|
if (undeadPids([child.pid]).length)
|
|
@@ -1313,8 +1355,8 @@ async function startColocatedServices(config, colocated, isolation, worldEnv, pa
|
|
|
1313
1355
|
catch { /* best effort cleanup */ }
|
|
1314
1356
|
throw new Error(`Co-located twin "${service.id}" failed to start (host isolation=${isolation}): ${error instanceof Error ? error.message : String(error)}\nLog: ${log}`);
|
|
1315
1357
|
});
|
|
1316
|
-
}
|
|
1317
|
-
|
|
1358
|
+
});
|
|
1359
|
+
await parallelStartup(allocated, async ({ service, port }, index) => {
|
|
1318
1360
|
// A host holding one twin settles for that one port, so its twin may be confirmed early like a lone service's.
|
|
1319
1361
|
await confirmOwnPort(port, boot, index === 0 ? ended : undefined, true, `Co-located twin "${service.id}"`, log, allocated.length === 1).catch((error) => {
|
|
1320
1362
|
if (undeadPids([child.pid]).length)
|
|
@@ -1325,7 +1367,7 @@ async function startColocatedServices(config, colocated, isolation, worldEnv, pa
|
|
|
1325
1367
|
catch { /* best effort cleanup */ }
|
|
1326
1368
|
throw error;
|
|
1327
1369
|
});
|
|
1328
|
-
}
|
|
1370
|
+
});
|
|
1329
1371
|
if (exitState.value !== null) {
|
|
1330
1372
|
throw new Error(`The co-located host ended during startup even though its ports answer, so other/stale processes probably hold them: the host ${endedAccount(log)}\n`
|
|
1331
1373
|
+ `Stop leftover services (e.g. \`volter-world down\`) and retry.\nLog: ${log}`);
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** Finite independent boot operations. Stop scheduling on failure and join every operation already begun. */
|
|
2
|
+
export async function parallelStartup(items, start) {
|
|
3
|
+
let next = 0;
|
|
4
|
+
let failed = false;
|
|
5
|
+
const errors = [];
|
|
6
|
+
await Promise.all(Array.from({ length: Math.min(8, items.length) }, async () => {
|
|
7
|
+
while (!failed) {
|
|
8
|
+
const index = next++;
|
|
9
|
+
if (index >= items.length)
|
|
10
|
+
return;
|
|
11
|
+
try {
|
|
12
|
+
await start(items[index], index);
|
|
13
|
+
}
|
|
14
|
+
catch (error) {
|
|
15
|
+
failed = true;
|
|
16
|
+
errors.push(error);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
}));
|
|
20
|
+
if (errors.length === 1)
|
|
21
|
+
throw errors[0];
|
|
22
|
+
if (errors.length > 1)
|
|
23
|
+
throw new AggregateError(errors, 'Independent World startup operations failed.');
|
|
24
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@volter/world-runtime",
|
|
3
|
-
"version": "2.0.
|
|
3
|
+
"version": "2.0.25",
|
|
4
4
|
"description": "World configs for twins: boot named local runtimes, allocate ports, generate world.env/instance.json, and run apps against fake-key twin worlds.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"twin",
|
package/src/host.ts
CHANGED
|
@@ -23,6 +23,7 @@ import { Worker } from 'node:worker_threads';
|
|
|
23
23
|
import { siblingScript } from './sibling.ts';
|
|
24
24
|
import { importModule } from './import-module.ts';
|
|
25
25
|
import { pathToFileURL } from 'node:url';
|
|
26
|
+
import { parallelStartup } from './startup.ts';
|
|
26
27
|
|
|
27
28
|
/** One twin to mount in the host. `module`/`export` name a `({port,root,readOnly}) => {port,stop}`
|
|
28
29
|
* factory (every pack ships one, e.g. `createStripeTwinServer`). `module` is anything import()
|
|
@@ -104,28 +105,43 @@ export async function startColocatedHost(
|
|
|
104
105
|
process.on('uncaughtException', onUncaught);
|
|
105
106
|
|
|
106
107
|
const servers: TwinServer[] = [];
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
108
|
+
const stop = async (): Promise<unknown[]> => {
|
|
109
|
+
process.off('unhandledRejection', onRejection);
|
|
110
|
+
process.off('uncaughtException', onUncaught);
|
|
111
|
+
const stopped = await Promise.allSettled(servers.map(async (server) => {
|
|
112
|
+
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
113
|
+
try {
|
|
114
|
+
await Promise.race([
|
|
115
|
+
Promise.resolve().then(() => server.stop()),
|
|
116
|
+
new Promise<never>((_, reject) => { timeout = setTimeout(() => reject(new Error(`Twin on port ${server.port} did not stop within ${STOP_TIMEOUT_MS} ms.`)), STOP_TIMEOUT_MS); }),
|
|
117
|
+
]);
|
|
118
|
+
} finally { if (timeout !== undefined) clearTimeout(timeout); }
|
|
119
|
+
}));
|
|
120
|
+
return stopped.filter((result): result is PromiseRejectedResult => result.status === 'rejected').map((result) => result.reason);
|
|
121
|
+
};
|
|
122
|
+
try {
|
|
123
|
+
await parallelStartup(specs, async (spec) => {
|
|
124
|
+
const factory = await loadFactory(spec);
|
|
125
|
+
const server = await factory({ port: spec.port, root: spec.root, readOnly: spec.readOnly, ...(spec.scenarioPath !== undefined ? { scenarioPath: spec.scenarioPath } : {}), ...(spec.database !== undefined ? { database: spec.database } : {}) });
|
|
126
|
+
// Record the handle before validating it, so rollback also owns a bad factory result.
|
|
127
|
+
servers.push(server);
|
|
128
|
+
if (!Number.isInteger(server.port) || server.port < 1 || server.port > 65535 || (spec.port !== 0 && server.port !== spec.port)) {
|
|
129
|
+
throw new Error(`Twin "${spec.id}" bound port ${server.port}, requested ${spec.port}.`);
|
|
130
|
+
}
|
|
131
|
+
ports[spec.id] = server.port;
|
|
132
|
+
});
|
|
133
|
+
} catch (error) {
|
|
134
|
+
const cleanup = await stop();
|
|
135
|
+
if (cleanup.length) throw new AggregateError([error, ...cleanup], 'Co-located host startup and rollback failed.');
|
|
136
|
+
throw error;
|
|
110
137
|
}
|
|
111
138
|
return {
|
|
112
139
|
isolation,
|
|
113
140
|
ports,
|
|
114
141
|
stop: async () => {
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
// rejection, hang, or slowness — can prevent the others or wedge host.stop() itself.
|
|
119
|
-
// The awaited-sequential first cut re-created the SIGKILL escalation it was fixing
|
|
120
|
-
// through a new door (§9 round two H2, 2026-08-31: one pending/throwing stop — fly's
|
|
121
|
-
// cleanup throws on a failed container removal — left siblings unstopped and the host
|
|
122
|
-
// ignoring SIGTERM forever). The bound stays under downWorld's 5s SIGKILL grace.
|
|
123
|
-
await Promise.allSettled(servers.map((s) =>
|
|
124
|
-
Promise.race([
|
|
125
|
-
Promise.resolve().then(() => s.stop()),
|
|
126
|
-
new Promise((resolveTimeout) => setTimeout(resolveTimeout, STOP_TIMEOUT_MS).unref?.()),
|
|
127
|
-
]),
|
|
128
|
-
));
|
|
142
|
+
// Every stop runs, even when a sibling hangs or throws. Host shutdown
|
|
143
|
+
// still resolves within its budget so the owning runtime can retire it.
|
|
144
|
+
for (const error of await stop()) onEvent({ twin: '(host)', kind: 'stop-failed', detail: String(error) });
|
|
129
145
|
},
|
|
130
146
|
};
|
|
131
147
|
}
|
|
@@ -143,8 +159,14 @@ export async function startColocatedHost(
|
|
|
143
159
|
// below would emit a 2nd `spawn` for the same twin, which the isolation test asserts against.
|
|
144
160
|
onEvent({ twin: spec.id, kind: 'spawn', detail: 'worker started' });
|
|
145
161
|
let ready = false;
|
|
146
|
-
worker.on('message', (msg: { type?: string }) => {
|
|
147
|
-
if (msg?.type === 'ready') {
|
|
162
|
+
worker.on('message', (msg: { type?: string; port?: number }) => {
|
|
163
|
+
if (msg?.type === 'ready') {
|
|
164
|
+
if (typeof msg.port !== 'number' || !Number.isInteger(msg.port) || msg.port < 1 || msg.port > 65535 || (spec.port !== 0 && msg.port !== spec.port)) {
|
|
165
|
+
rejectReady(new Error(`Twin "${spec.id}" bound port ${msg.port}, requested ${spec.port}.`)); return;
|
|
166
|
+
}
|
|
167
|
+
ports[spec.id] = msg.port;
|
|
168
|
+
ready = true; resolveReady();
|
|
169
|
+
}
|
|
148
170
|
});
|
|
149
171
|
worker.on('error', (err) => {
|
|
150
172
|
onEvent({ twin: spec.id, kind: 'error', detail: String(err) });
|
|
@@ -152,6 +174,7 @@ export async function startColocatedHost(
|
|
|
152
174
|
});
|
|
153
175
|
worker.on('exit', (code) => {
|
|
154
176
|
workers.delete(spec.id);
|
|
177
|
+
if (!ready) rejectReady(new Error(`Twin "${spec.id}" exited before readiness (code ${code}).`));
|
|
155
178
|
if (stopping.value || code === 0) return;
|
|
156
179
|
// Minimal-primitive doctrine: no keep-alive supervisor. A crashed worker is NOT
|
|
157
180
|
// respawned — record the give-up (host-cli persists it → doctor red) and stay dead.
|
|
@@ -160,7 +183,14 @@ export async function startColocatedHost(
|
|
|
160
183
|
workers.set(spec.id, worker);
|
|
161
184
|
});
|
|
162
185
|
|
|
163
|
-
|
|
186
|
+
try { await parallelStartup(specs, async (spec) => spawnWorker(spec)); }
|
|
187
|
+
catch (error) {
|
|
188
|
+
stopping.value = true;
|
|
189
|
+
const stopped = await Promise.allSettled([...workers.values()].map((worker) => worker.terminate()));
|
|
190
|
+
const cleanup = stopped.filter((result): result is PromiseRejectedResult => result.status === 'rejected').map((result) => result.reason);
|
|
191
|
+
if (cleanup.length) throw new AggregateError([error, ...cleanup], 'Co-located worker startup and rollback failed.');
|
|
192
|
+
throw error;
|
|
193
|
+
}
|
|
164
194
|
|
|
165
195
|
return {
|
|
166
196
|
isolation,
|
package/src/runtime.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { claimEnvOutput, publishEnvOutput } from './env-output.ts';
|
|
2
2
|
import { offSignal } from './signals.ts';
|
|
3
|
+
import { parallelStartup } from './startup.ts';
|
|
3
4
|
import { assertWorldStateRemovable, removeWorldState } from './ancestry.ts';
|
|
4
5
|
import { packCli, packEntry } from './catalog.ts';
|
|
5
6
|
import { spawn, spawnSync } from 'node:child_process';
|
|
6
|
-
import { get as httpGet } from 'node:http';
|
|
7
|
+
import { get as httpGet, type ClientRequest } from 'node:http';
|
|
7
8
|
import { siblingScript } from './sibling.ts';
|
|
8
9
|
import { commandLifetime, commandProcessGroup, CommandRetirementError } from './command-lifetime.ts';
|
|
9
10
|
import { superviseWorldRun } from './run-task.ts';
|
|
@@ -108,16 +109,19 @@ async function untilReadyOrEnded<T>(wait: (abort: AbortSignal) => Promise<T>, en
|
|
|
108
109
|
* always runs in full: one port's answer says nothing of the others'. Programs that are not twins are never asked. */
|
|
109
110
|
async function confirmOwnPort(port: number, boot: BootContext, ended: Promise<void> | undefined, twin: boolean, what: string, log: string, early = false): Promise<void> {
|
|
110
111
|
// A twin is asked at once, raced with the settle: a server slow to answer costs no more than the settle did.
|
|
111
|
-
|
|
112
|
+
boot.signal.throwIfAborted();
|
|
113
|
+
const answer = twin ? worldBootOf(port, boot.signal) : Promise.resolve(null);
|
|
112
114
|
const own = early && twin ? answer.then((a) => (a && a.boot === boot.bootId ? 'own' as const : new Promise<never>(() => undefined))) : new Promise<never>(() => undefined);
|
|
113
115
|
// `ended` absent: the process was already settled (a co-located host settles once for all its twins).
|
|
114
116
|
if (ended) {
|
|
115
117
|
const settled = await Promise.race([own, ended.then(() => 'ended' as const), new Promise<'alive'>((resolveAlive) => setTimeout(() => resolveAlive('alive'), 300))]);
|
|
118
|
+
boot.signal.throwIfAborted();
|
|
116
119
|
if (settled === 'own') return;
|
|
117
120
|
if (settled === 'ended') throw new Error(`${what} ended right after its port first answered, so another process probably holds the port: the service ${endedAccount(log)}\nLog: ${log}`);
|
|
118
121
|
}
|
|
119
122
|
if (!twin) return;
|
|
120
123
|
const answered = await answer;
|
|
124
|
+
boot.signal.throwIfAborted();
|
|
121
125
|
if (answered && typeof answered.boot === 'string' && answered.boot !== boot.bootId) {
|
|
122
126
|
throw new Error(`${what}: its port is answered by a twin of another World boot (${answered.boot}), not this one; a leftover World holds the port. Stop it (\`volter-world list\`, then \`down\`) and retry.\nLog: ${log}`);
|
|
123
127
|
}
|
|
@@ -125,16 +129,44 @@ async function confirmOwnPort(port: number, boot: BootContext, ended: Promise<vo
|
|
|
125
129
|
|
|
126
130
|
/** What the server on a loopback port says its World boot is (WORLD_BOOT_PATH), or null. A direct loopback request:
|
|
127
131
|
* never through a proxy an activated shell's environment names. */
|
|
128
|
-
function worldBootOf(port: number): Promise<{ boot?: unknown } | null> {
|
|
132
|
+
function worldBootOf(port: number, abort: AbortSignal): Promise<{ boot?: unknown } | null> {
|
|
129
133
|
return new Promise((resolveBoot) => {
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
134
|
+
let request: ClientRequest | undefined;
|
|
135
|
+
let deadline: ReturnType<typeof setTimeout> | undefined;
|
|
136
|
+
let settled = false;
|
|
137
|
+
let responseStarted = false;
|
|
138
|
+
const finish = (value: { boot?: unknown } | null): void => {
|
|
139
|
+
if (settled) return;
|
|
140
|
+
settled = true;
|
|
141
|
+
if (deadline !== undefined) clearTimeout(deadline);
|
|
142
|
+
abort.removeEventListener('abort', cancel);
|
|
143
|
+
resolveBoot(value);
|
|
144
|
+
};
|
|
145
|
+
const cancel = (): void => { finish(null); request?.destroy(); };
|
|
146
|
+
if (abort.aborted) { finish(null); return; }
|
|
147
|
+
abort.addEventListener('abort', cancel, { once: true });
|
|
148
|
+
// A socket timeout alone is insufficient: destroying a request after
|
|
149
|
+
// headers can close/abort its response without emitting request 'error'.
|
|
150
|
+
// Bound the whole probe, including a peer that dribbles bytes forever.
|
|
151
|
+
deadline = setTimeout(cancel, 1000);
|
|
152
|
+
try {
|
|
153
|
+
request = httpGet({ host: '127.0.0.1', port, path: WORLD_BOOT_PATH, timeout: 1000, agent: false }, (response) => {
|
|
154
|
+
responseStarted = true;
|
|
155
|
+
let body = '';
|
|
156
|
+
response.on('error', cancel);
|
|
157
|
+
response.on('aborted', cancel);
|
|
158
|
+
response.on('close', () => finish(null));
|
|
159
|
+
response.setEncoding('utf8');
|
|
160
|
+
response.on('data', (chunk: string) => { body += chunk; if (body.length > 4096) cancel(); });
|
|
161
|
+
response.on('end', () => {
|
|
162
|
+
try { finish(response.statusCode === 200 ? JSON.parse(body) as { boot?: unknown } : null); }
|
|
163
|
+
catch { finish(null); }
|
|
164
|
+
});
|
|
165
|
+
});
|
|
166
|
+
request.on('timeout', cancel);
|
|
167
|
+
request.on('error', () => finish(null));
|
|
168
|
+
request.on('close', () => { if (!responseStarted) finish(null); });
|
|
169
|
+
} catch { cancel(); }
|
|
138
170
|
});
|
|
139
171
|
}
|
|
140
172
|
|
|
@@ -1290,6 +1322,7 @@ async function startColocatedServices(
|
|
|
1290
1322
|
allocatedPorts: Set<number>,
|
|
1291
1323
|
boot: BootContext,
|
|
1292
1324
|
): Promise<WorldServiceInstance[]> {
|
|
1325
|
+
if (colocated.length === 0) return [];
|
|
1293
1326
|
const hostCli = siblingScript(import.meta.url, 'host-cli');
|
|
1294
1327
|
const args: string[] = [hostCli, '--isolation', isolation === 'colocated' ? 'shared' : 'worker'];
|
|
1295
1328
|
const allocated: Array<{ service: WorldServiceConfig; port: number; mounted: MountedPackage }> = [];
|
|
@@ -1329,22 +1362,22 @@ async function startColocatedServices(
|
|
|
1329
1362
|
});
|
|
1330
1363
|
|
|
1331
1364
|
const command = [process.execPath, ...args];
|
|
1332
|
-
|
|
1365
|
+
await parallelStartup(allocated, async ({ service, port }) => {
|
|
1333
1366
|
const bindTimeoutMs = typeof service.ready === 'object' && service.ready.timeoutMs !== undefined ? Math.max(service.ready.timeoutMs, 15_000) : 15_000;
|
|
1334
1367
|
await untilReadyOrEnded((abort) => waitForTcp(port, bindTimeoutMs, abort), ended, boot, 'the co-located host', log).catch((error) => {
|
|
1335
1368
|
if (undeadPids([child.pid!]).length) recordWorldStop([log], 'world.service.stop', 'up (startup failed)', 'SIGTERM');
|
|
1336
1369
|
try { process.kill(-child.pid!, 'SIGTERM'); } catch { /* best effort cleanup */ }
|
|
1337
1370
|
throw new Error(`Co-located twin "${service.id}" failed to start (host isolation=${isolation}): ${error instanceof Error ? error.message : String(error)}\nLog: ${log}`);
|
|
1338
1371
|
});
|
|
1339
|
-
}
|
|
1340
|
-
|
|
1372
|
+
});
|
|
1373
|
+
await parallelStartup(allocated, async ({ service, port }, index) => {
|
|
1341
1374
|
// A host holding one twin settles for that one port, so its twin may be confirmed early like a lone service's.
|
|
1342
1375
|
await confirmOwnPort(port, boot, index === 0 ? ended : undefined, true, `Co-located twin "${service.id}"`, log, allocated.length === 1).catch((error) => {
|
|
1343
1376
|
if (undeadPids([child.pid!]).length) recordWorldStop([log], 'world.service.stop', 'up (port not its own)', 'SIGTERM');
|
|
1344
1377
|
try { process.kill(-child.pid!, 'SIGTERM'); } catch { /* best effort cleanup */ }
|
|
1345
1378
|
throw error;
|
|
1346
1379
|
});
|
|
1347
|
-
}
|
|
1380
|
+
});
|
|
1348
1381
|
if (exitState.value !== null) {
|
|
1349
1382
|
throw new Error(
|
|
1350
1383
|
`The co-located host ended during startup even though its ports answer, so other/stale processes probably hold them: the host ${endedAccount(log)}\n`
|
package/src/startup.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/** Finite independent boot operations. Stop scheduling on failure and join every operation already begun. */
|
|
2
|
+
export async function parallelStartup<T>(items: readonly T[], start: (item: T, index: number) => Promise<void>): Promise<void> {
|
|
3
|
+
let next = 0;
|
|
4
|
+
let failed = false;
|
|
5
|
+
const errors: unknown[] = [];
|
|
6
|
+
await Promise.all(Array.from({ length: Math.min(8, items.length) }, async () => {
|
|
7
|
+
while (!failed) {
|
|
8
|
+
const index = next++;
|
|
9
|
+
if (index >= items.length) return;
|
|
10
|
+
try { await start(items[index]!, index); }
|
|
11
|
+
catch (error) { failed = true; errors.push(error); }
|
|
12
|
+
}
|
|
13
|
+
}));
|
|
14
|
+
if (errors.length === 1) throw errors[0];
|
|
15
|
+
if (errors.length > 1) throw new AggregateError(errors, 'Independent World startup operations failed.');
|
|
16
|
+
}
|