@volter/world-runtime 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/dist/known-external-services.json +1108 -0
- package/dist/src/ancestry.d.ts +2 -0
- package/dist/src/ancestry.js +42 -0
- package/dist/src/app-url.d.ts +47 -0
- package/dist/src/app-url.js +239 -0
- package/dist/src/attach.d.ts +48 -0
- package/dist/src/attach.js +87 -0
- package/dist/src/branch.d.ts +20 -0
- package/dist/src/branch.js +65 -0
- package/dist/src/browser-proxy-cli.d.ts +2 -0
- package/dist/src/browser-proxy-cli.js +41 -0
- package/dist/src/ca-trust.d.ts +5 -0
- package/dist/src/ca-trust.js +64 -0
- package/dist/src/catalog.d.ts +31 -0
- package/dist/src/catalog.js +148 -0
- package/dist/src/changeset.d.ts +142 -0
- package/dist/src/changeset.js +570 -0
- package/dist/src/cli.d.ts +2 -0
- package/dist/src/cli.js +1262 -0
- package/dist/src/command-lifetime.d.ts +15 -0
- package/dist/src/command-lifetime.js +98 -0
- package/dist/src/configs.d.ts +18 -0
- package/dist/src/configs.js +119 -0
- package/dist/src/console-apart.d.ts +38 -0
- package/dist/src/console-apart.js +107 -0
- package/dist/src/consumers.d.ts +46 -0
- package/dist/src/consumers.js +200 -0
- package/dist/src/covers.d.ts +183 -0
- package/dist/src/covers.js +800 -0
- package/dist/src/fixture-env.d.ts +42 -0
- package/dist/src/fixture-env.js +221 -0
- package/dist/src/host-cli.d.ts +2 -0
- package/dist/src/host-cli.js +92 -0
- package/dist/src/host-fault-fixture.d.ts +32 -0
- package/dist/src/host-fault-fixture.js +100 -0
- package/dist/src/host-worker.d.ts +1 -0
- package/dist/src/host-worker.js +23 -0
- package/dist/src/host.d.ts +38 -0
- package/dist/src/host.js +135 -0
- package/dist/src/index.d.ts +48 -0
- package/dist/src/index.js +35 -0
- package/dist/src/infra-cli.d.ts +2 -0
- package/dist/src/infra-cli.js +136 -0
- package/dist/src/init.d.ts +227 -0
- package/dist/src/init.js +1117 -0
- package/dist/src/inject-map.d.ts +34 -0
- package/dist/src/inject-map.js +56 -0
- package/dist/src/lifecycle-record.d.ts +47 -0
- package/dist/src/lifecycle-record.js +196 -0
- package/dist/src/origin.d.ts +31 -0
- package/dist/src/origin.js +139 -0
- package/dist/src/pack-facts.d.ts +75 -0
- package/dist/src/pack-facts.js +98 -0
- package/dist/src/pglite-backing.d.ts +21 -0
- package/dist/src/pglite-backing.js +158 -0
- package/dist/src/pglite-host.mjs +147 -0
- package/dist/src/placeholder.d.ts +20 -0
- package/dist/src/placeholder.js +100 -0
- package/dist/src/prerequisites.d.ts +21 -0
- package/dist/src/prerequisites.js +49 -0
- package/dist/src/process-groups.d.ts +4 -0
- package/dist/src/process-groups.js +49 -0
- package/dist/src/project-inspect.d.ts +109 -0
- package/dist/src/project-inspect.js +827 -0
- package/dist/src/proxy-daemon.d.ts +2 -0
- package/dist/src/proxy-daemon.js +18 -0
- package/dist/src/redirect-proxy.d.ts +105 -0
- package/dist/src/redirect-proxy.js +665 -0
- package/dist/src/reflect.d.ts +74 -0
- package/dist/src/reflect.js +392 -0
- package/dist/src/resources.d.ts +26 -0
- package/dist/src/resources.js +22 -0
- package/dist/src/root.d.ts +114 -0
- package/dist/src/root.js +312 -0
- package/dist/src/run-task-worker.d.ts +1 -0
- package/dist/src/run-task-worker.js +38 -0
- package/dist/src/run-task.d.ts +18 -0
- package/dist/src/run-task.js +48 -0
- package/dist/src/runtime-test-support.d.ts +59 -0
- package/dist/src/runtime-test-support.js +205 -0
- package/dist/src/runtime.d.ts +256 -0
- package/dist/src/runtime.js +3502 -0
- package/dist/src/schema.d.ts +449 -0
- package/dist/src/schema.js +605 -0
- package/dist/src/serve.d.ts +30 -0
- package/dist/src/serve.js +82 -0
- package/dist/src/served-world.d.ts +194 -0
- package/dist/src/served-world.js +986 -0
- package/dist/src/service-exit.d.ts +46 -0
- package/dist/src/service-exit.js +195 -0
- package/dist/src/service-recorder.d.ts +1 -0
- package/dist/src/service-recorder.js +121 -0
- package/dist/src/sibling.d.ts +1 -0
- package/dist/src/sibling.js +9 -0
- package/dist/src/signals.d.ts +1 -0
- package/dist/src/signals.js +11 -0
- package/dist/src/storage-capacity.d.ts +8 -0
- package/dist/src/storage-capacity.js +61 -0
- package/dist/src/tail.d.ts +30 -0
- package/dist/src/tail.js +160 -0
- package/dist/src/tcp-port.d.ts +2 -0
- package/dist/src/tcp-port.js +36 -0
- package/dist/src/up-task-worker.d.ts +1 -0
- package/dist/src/up-task-worker.js +61 -0
- package/dist/src/up-task.d.ts +17 -0
- package/dist/src/up-task.js +49 -0
- package/dist/src/websocket-relay.d.ts +3 -0
- package/dist/src/websocket-relay.js +40 -0
- package/known-external-services.json +1108 -0
- package/package.json +83 -0
- package/src/ancestry.ts +36 -0
- package/src/app-url.ts +253 -0
- package/src/attach.ts +117 -0
- package/src/branch.ts +63 -0
- package/src/browser-proxy-cli.ts +44 -0
- package/src/ca-trust.ts +57 -0
- package/src/catalog.ts +156 -0
- package/src/changeset.ts +627 -0
- package/src/cli.ts +1111 -0
- package/src/command-lifetime.ts +79 -0
- package/src/configs.ts +110 -0
- package/src/console-apart.ts +90 -0
- package/src/consumers.ts +185 -0
- package/src/covers.ts +934 -0
- package/src/fixture-env.ts +230 -0
- package/src/host-cli.ts +90 -0
- package/src/host-worker.ts +23 -0
- package/src/host.ts +169 -0
- package/src/index.ts +171 -0
- package/src/infra-cli.ts +133 -0
- package/src/init.ts +1316 -0
- package/src/inject-map.ts +72 -0
- package/src/lifecycle-record.ts +168 -0
- package/src/origin.ts +134 -0
- package/src/pack-facts.ts +128 -0
- package/src/pglite-backing.ts +141 -0
- package/src/pglite-host.mjs +147 -0
- package/src/placeholder.ts +89 -0
- package/src/prerequisites.ts +66 -0
- package/src/process-groups.ts +33 -0
- package/src/project-inspect.ts +770 -0
- package/src/proxy-daemon.ts +21 -0
- package/src/redirect-proxy.ts +684 -0
- package/src/reflect.ts +440 -0
- package/src/resources.ts +22 -0
- package/src/root.ts +290 -0
- package/src/run-task-worker.ts +27 -0
- package/src/run-task.ts +44 -0
- package/src/runtime-test-support.ts +208 -0
- package/src/runtime.ts +3357 -0
- package/src/schema.ts +922 -0
- package/src/serve.ts +102 -0
- package/src/served-world.ts +812 -0
- package/src/service-exit.ts +175 -0
- package/src/service-recorder.ts +89 -0
- package/src/sibling.ts +10 -0
- package/src/signals.ts +10 -0
- package/src/storage-capacity.ts +60 -0
- package/src/tail.ts +205 -0
- package/src/tcp-port.ts +35 -0
- package/src/up-task-worker.ts +40 -0
- package/src/up-task.ts +45 -0
- package/src/websocket-relay.ts +32 -0
|
@@ -0,0 +1,3502 @@
|
|
|
1
|
+
import { offSignal } from "./signals.js";
|
|
2
|
+
import { assertWorldStateRemovable, removeWorldState } from "./ancestry.js";
|
|
3
|
+
import { packCli, packEntry } from "./catalog.js";
|
|
4
|
+
import { spawn, spawnSync } from 'node:child_process';
|
|
5
|
+
import { get as httpGet } from 'node:http';
|
|
6
|
+
import { siblingScript } from "./sibling.js";
|
|
7
|
+
import { commandLifetime, commandProcessGroup, CommandRetirementError } from "./command-lifetime.js";
|
|
8
|
+
import { superviseWorldRun } from "./run-task.js";
|
|
9
|
+
import { survivingOwnedGroups } from "./process-groups.js";
|
|
10
|
+
import { randomUUID, createHash } from 'node:crypto';
|
|
11
|
+
import { createRequire } from 'node:module';
|
|
12
|
+
import { isIP } from 'node:net';
|
|
13
|
+
import { allocatePort } from "./tcp-port.js";
|
|
14
|
+
import { constants as osConstants, hostname as osHostname } from 'node:os';
|
|
15
|
+
import { appendFileSync, chmodSync, closeSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, writeFileSync, writeSync, existsSync } from 'node:fs';
|
|
16
|
+
import { basename, dirname, join, relative, resolve, sep } from 'node:path';
|
|
17
|
+
import { packFacts } from "./pack-facts.js";
|
|
18
|
+
import { describeServiceEnd, readServiceEnd, serviceOutput } from "./service-exit.js";
|
|
19
|
+
import { Socket } from 'node:net';
|
|
20
|
+
import { getActiveWorldStore, withFileLock, stateDirName, PROTOCOL_VERSION, PROTOCOL_MAJOR, WORLD_BOOT_PATH, WORLD_ENV_NAMES_ENV } from '@volter/world-core';
|
|
21
|
+
import { loadWorldConfig } from "./configs.js";
|
|
22
|
+
import { createWorldNetworkPolicy, WORLD_NETWORK_POLICY_ENV } from '@volter/world-core/network-policy';
|
|
23
|
+
import { resolveServicePackage } from "./catalog.js";
|
|
24
|
+
import { fakeEnvValue } from "./fixture-env.js";
|
|
25
|
+
import { MINT_ENV } from "./schema.js";
|
|
26
|
+
import { inertInjectEnvWarnings } from "./inject-map.js";
|
|
27
|
+
import { activeVendorMap, ensureCa, opensslAvailable, proxyEnvFor, startRedirectProxy, tearDownCa } from "./redirect-proxy.js";
|
|
28
|
+
import { CaTrustError, sessionTrustEnv } from "./ca-trust.js";
|
|
29
|
+
import { checkedOwner, CONSUMER_HEARTBEAT_MS, createConsumer, refreshConsumer, releaseConsumer, removeConsumerRecord, worldConsumers } from "./consumers.js";
|
|
30
|
+
import { assertLifecycleStartable, beginWorldLifecycle, handoffWorldLifecycle, recordWorldLifecycleEvent, releaseWorldLifecycle, retainWorldLifecycleForCleanup, hasWorldLifecycle, isWorldLifecycleRunning, lifecycleLogPath, releaseAbandonedWorldLifecycle } from "./lifecycle-record.js";
|
|
31
|
+
/** The latest `limit` bytes of a service log, reaching into its one rotation (`<log>.1`) when the current file is
|
|
32
|
+
* shorter: a readiness line printed just before a rotation is still found, and a poll never reads 64 MiB files. */
|
|
33
|
+
function readServiceText(log, limit = 4 * 1024 * 1024) {
|
|
34
|
+
const tailOfFile = (path, bytes) => {
|
|
35
|
+
if (bytes <= 0 || !existsSync(path))
|
|
36
|
+
return '';
|
|
37
|
+
const fd = openSync(path, 'r');
|
|
38
|
+
try {
|
|
39
|
+
const size = fstatSync(fd).size, start = Math.max(0, size - bytes), buffer = Buffer.alloc(size - start);
|
|
40
|
+
readSync(fd, buffer, 0, buffer.length, start);
|
|
41
|
+
return buffer.toString('utf8');
|
|
42
|
+
}
|
|
43
|
+
finally {
|
|
44
|
+
closeSync(fd);
|
|
45
|
+
}
|
|
46
|
+
};
|
|
47
|
+
const current = tailOfFile(log, limit);
|
|
48
|
+
return `${tailOfFile(`${log}.1`, limit - Buffer.byteLength(current))}${current}`;
|
|
49
|
+
}
|
|
50
|
+
/** The World's events file: every service's start and end, and the World's own stops, one JSON line each. */
|
|
51
|
+
export function worldEventsPath(logs) {
|
|
52
|
+
return join(logs, 'events.jsonl');
|
|
53
|
+
}
|
|
54
|
+
/** Start `command args` under the service recorder (./service-recorder.ts) as its own process group: the service writes
|
|
55
|
+
* its output straight to `log` (capped, time-marked), its start and end are recorded there and in the World's events
|
|
56
|
+
* file, and it is handed a directory of its own for the log files it writes itself (VOLTER_WORLD_SERVICE_LOG_DIR).
|
|
57
|
+
* `ended` settles when the recorder exits, right after the service ends and its exit record is written. */
|
|
58
|
+
function spawnRecorded(serviceId, command, args, cwd, env, log) {
|
|
59
|
+
const logDir = `${log.replace(/\.log$/, '')}.d`;
|
|
60
|
+
mkdirSync(logDir, { recursive: true, mode: 0o700 });
|
|
61
|
+
const out = openPrivateLog(log, 'a'); // the service's output and the recorder's records, appended
|
|
62
|
+
const recorderEnv = { ...env, VOLTER_WORLD_SERVICE_ID: serviceId, VOLTER_WORLD_SERVICE_LOG: log,
|
|
63
|
+
VOLTER_WORLD_SERVICE_LOG_DIR: logDir, VOLTER_WORLD_EVENTS: worldEventsPath(dirname(log)) };
|
|
64
|
+
if (recorderEnv.NODE_OPTIONS !== undefined) {
|
|
65
|
+
recorderEnv.VOLTER_WORLD_SERVICE_NODE_OPTIONS = recorderEnv.NODE_OPTIONS;
|
|
66
|
+
delete recorderEnv.NODE_OPTIONS;
|
|
67
|
+
}
|
|
68
|
+
const child = spawn(process.execPath, [siblingScript(import.meta.url, 'service-recorder'), command, ...args], {
|
|
69
|
+
cwd,
|
|
70
|
+
env: recorderEnv,
|
|
71
|
+
detached: true,
|
|
72
|
+
stdio: ['ignore', out, out],
|
|
73
|
+
});
|
|
74
|
+
closeSync(out);
|
|
75
|
+
const ended = new Promise((resolveEnded) => { child.once('exit', () => resolveEnded()); child.once('error', () => resolveEnded()); });
|
|
76
|
+
return { child, ended };
|
|
77
|
+
}
|
|
78
|
+
/** A startup wait that fails as soon as the service ends, with the service's own account of its end (exit code or
|
|
79
|
+
* signal, last output) instead of a timeout that names neither. */
|
|
80
|
+
async function untilReadyOrEnded(wait, ended, boot, what, log) {
|
|
81
|
+
const local = new AbortController();
|
|
82
|
+
const abort = AbortSignal.any([boot.signal, local.signal]);
|
|
83
|
+
return new Promise((resolveReady, rejectReady) => {
|
|
84
|
+
let settled = false;
|
|
85
|
+
wait(abort).then((value) => { if (!settled) {
|
|
86
|
+
settled = true;
|
|
87
|
+
local.abort();
|
|
88
|
+
resolveReady(value);
|
|
89
|
+
} }, (error) => { if (!settled) {
|
|
90
|
+
settled = true;
|
|
91
|
+
local.abort();
|
|
92
|
+
rejectReady(error);
|
|
93
|
+
} });
|
|
94
|
+
void ended.then(() => {
|
|
95
|
+
if (settled)
|
|
96
|
+
return;
|
|
97
|
+
settled = true;
|
|
98
|
+
local.abort();
|
|
99
|
+
rejectReady(new Error(`${what} ${endedAccount(log)}`));
|
|
100
|
+
});
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
/** That the process answering a service's port is the service. allocatePort closes its probe before the service
|
|
104
|
+
* binds, so another process can take the port and answer the first connect; the service then fails to bind and ends
|
|
105
|
+
* at once, which a short settle catches for any program. A World twin also answers WORLD_BOOT_PATH with the boot it
|
|
106
|
+
* belongs to, so a leftover World's twin on the port is named exactly; a server that does not answer the path (a
|
|
107
|
+
* twin from before it, a program that is not a twin) is accepted on the settle alone. When the settle guards this
|
|
108
|
+
* port alone (`early`), a twin's answer with this boot's id ends it early: a server of this boot holds the port, and
|
|
109
|
+
* every port this boot assigns is kept apart from its siblings', so short of the kernel handing the just-probed port
|
|
110
|
+
* to another of this boot's port-0 listeners, that server is the service just started. A settle standing for several
|
|
111
|
+
* ports (a co-located host's)
|
|
112
|
+
* always runs in full: one port's answer says nothing of the others'. Programs that are not twins are never asked. */
|
|
113
|
+
async function confirmOwnPort(port, boot, ended, twin, what, log, early = false) {
|
|
114
|
+
// A twin is asked at once, raced with the settle: a server slow to answer costs no more than the settle did.
|
|
115
|
+
const answer = twin ? worldBootOf(port) : Promise.resolve(null);
|
|
116
|
+
const own = early && twin ? answer.then((a) => (a && a.boot === boot.bootId ? 'own' : new Promise(() => undefined))) : new Promise(() => undefined);
|
|
117
|
+
// `ended` absent: the process was already settled (a co-located host settles once for all its twins).
|
|
118
|
+
if (ended) {
|
|
119
|
+
const settled = await Promise.race([own, ended.then(() => 'ended'), new Promise((resolveAlive) => setTimeout(() => resolveAlive('alive'), 300))]);
|
|
120
|
+
if (settled === 'own')
|
|
121
|
+
return;
|
|
122
|
+
if (settled === 'ended')
|
|
123
|
+
throw new Error(`${what} ended right after its port first answered, so another process probably holds the port: the service ${endedAccount(log)}\nLog: ${log}`);
|
|
124
|
+
}
|
|
125
|
+
if (!twin)
|
|
126
|
+
return;
|
|
127
|
+
const answered = await answer;
|
|
128
|
+
if (answered && typeof answered.boot === 'string' && answered.boot !== boot.bootId) {
|
|
129
|
+
throw new Error(`${what}: its port is answered by a twin of another World boot (${answered.boot}), not this one; a leftover World holds the port. Stop it (\`volter-world list\`, then \`down\`) and retry.\nLog: ${log}`);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
/** What the server on a loopback port says its World boot is (WORLD_BOOT_PATH), or null. A direct loopback request:
|
|
133
|
+
* never through a proxy an activated shell's environment names. */
|
|
134
|
+
function worldBootOf(port) {
|
|
135
|
+
return new Promise((resolveBoot) => {
|
|
136
|
+
const request = httpGet({ host: '127.0.0.1', port, path: WORLD_BOOT_PATH, timeout: 1000, agent: false }, (response) => {
|
|
137
|
+
let body = '';
|
|
138
|
+
response.setEncoding('utf8');
|
|
139
|
+
response.on('data', (chunk) => { body += chunk; if (body.length > 4096)
|
|
140
|
+
request.destroy(); });
|
|
141
|
+
response.on('end', () => { try {
|
|
142
|
+
resolveBoot(response.statusCode === 200 ? JSON.parse(body) : null);
|
|
143
|
+
}
|
|
144
|
+
catch {
|
|
145
|
+
resolveBoot(null);
|
|
146
|
+
} });
|
|
147
|
+
});
|
|
148
|
+
request.on('timeout', () => request.destroy());
|
|
149
|
+
request.on('error', () => resolveBoot(null));
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
/** A stopped service's own account from its log: how it ended and the last lines it wrote. */
|
|
153
|
+
function endedAccount(log) {
|
|
154
|
+
const end = readServiceEnd(log);
|
|
155
|
+
if (!end)
|
|
156
|
+
return 'ended and left no record of its end';
|
|
157
|
+
const said = end.lastOutput.length ? `\nLast output:\n${end.lastOutput.map((line) => ` ${line}`).join('\n')}` : '';
|
|
158
|
+
return `${describeServiceEnd(end)}${said}`;
|
|
159
|
+
}
|
|
160
|
+
/** The World's own stop of a service, written into the service's log beside the service's records, so its end reads as
|
|
161
|
+
* the World's act (and a SIGKILL after the grace as the World's, not the OOM killer's). */
|
|
162
|
+
function recordWorldStop(logs, event, by, signal) {
|
|
163
|
+
for (const log of new Set(logs)) {
|
|
164
|
+
try {
|
|
165
|
+
if (!existsSync(log))
|
|
166
|
+
continue;
|
|
167
|
+
const fd = openSync(log, 'r');
|
|
168
|
+
let fresh = '';
|
|
169
|
+
try {
|
|
170
|
+
const { size } = lstatSync(log);
|
|
171
|
+
const last = Buffer.alloc(1);
|
|
172
|
+
if (size && readSync(fd, last, 0, 1, size - 1) && last[0] !== 10)
|
|
173
|
+
fresh = '\n';
|
|
174
|
+
}
|
|
175
|
+
finally {
|
|
176
|
+
closeSync(fd);
|
|
177
|
+
}
|
|
178
|
+
const line = `${JSON.stringify({ event, by, signal, service: basename(log).replace(/\.log$/, ''), at: new Date().toISOString() })}\n`;
|
|
179
|
+
appendFileSync(log, `${fresh}${line}`);
|
|
180
|
+
try {
|
|
181
|
+
appendFileSync(worldEventsPath(dirname(log)), line, { mode: 0o600 });
|
|
182
|
+
}
|
|
183
|
+
catch { /* the service log has it */ }
|
|
184
|
+
}
|
|
185
|
+
catch { /* the stop proceeds without its record */ }
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
function worldBaseDir(root) {
|
|
189
|
+
// `stateDirName()`, not a literal: VOLTER_STATE_DIR is the documented way to point a process at
|
|
190
|
+
// a different state directory, and control-plane honours it everywhere. This package did not,
|
|
191
|
+
// so a caller that set it got worlds written under `.volter/` while the kernel read from the
|
|
192
|
+
// requested directory — two halves of the same world in two places.
|
|
193
|
+
return join(root, stateDirName(), 'worlds');
|
|
194
|
+
}
|
|
195
|
+
/** World names become a path segment (instanceDir/instanceLockFile) and must never let a
|
|
196
|
+
* caller escape `worldBaseDir` (no `/`, and the leading-alnum requirement rules out a bare
|
|
197
|
+
* `..`). Shared by upWorld's boot-time check and downWorld's `--purge` (the one path that
|
|
198
|
+
* recursively `rmSync`s a name-derived directory, so it is the one that most needs this to
|
|
199
|
+
* hold). */
|
|
200
|
+
function assertSafeWorldName(name) {
|
|
201
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name))
|
|
202
|
+
throw new Error(`Invalid world name: ${name}`);
|
|
203
|
+
}
|
|
204
|
+
/** Exported for the reflect attachment (reflect.ts), whose routes file lives in
|
|
205
|
+
* the instance dir — one layout owner, no duplicated path knowledge. */
|
|
206
|
+
/** THE WORLD CLOCK file (TWIN-PROGRAMMING-MODEL P3): a single frozen ISO instant every twin
|
|
207
|
+
* stamps from (kernel worldNow()); absent → real wall-clock. Written only by the
|
|
208
|
+
* `volter-world clock` operator door — time is physics, advanced explicitly, never drifting. */
|
|
209
|
+
export function clockFile(root, name) {
|
|
210
|
+
return join(instanceDir(root, name), 'clock');
|
|
211
|
+
}
|
|
212
|
+
export function instanceDir(root, name) {
|
|
213
|
+
return join(worldBaseDir(root), name);
|
|
214
|
+
}
|
|
215
|
+
function instanceFile(root, name) {
|
|
216
|
+
return join(instanceDir(root, name), 'instance.json');
|
|
217
|
+
}
|
|
218
|
+
function foregroundRunFile(root, name) {
|
|
219
|
+
return join(instanceDir(root, name), 'foreground-run.json');
|
|
220
|
+
}
|
|
221
|
+
/** Lockfile guarding the synchronous claim-of-the-instance-dir section of `upWorld` (TWIN-36).
|
|
222
|
+
* It must survive the claim's own `rmSync` of the instance dir, so it lives BESIDE the dirs in
|
|
223
|
+
* a dot-prefixed sibling: world names must start with an alphanumeric, so `.locks` can never
|
|
224
|
+
* collide with (or be wiped as) a world's instance dir. */
|
|
225
|
+
function instanceLockFile(root, name) {
|
|
226
|
+
return join(worldBaseDir(root), '.locks', `${name}.lock`);
|
|
227
|
+
}
|
|
228
|
+
function stoppingFile(root, name) {
|
|
229
|
+
return join(worldBaseDir(root), '.locks', `${name}.stop.json`);
|
|
230
|
+
}
|
|
231
|
+
const SHARE_CLAIM_STALE_MS = 60_000;
|
|
232
|
+
const SHARE_CLAIM_HEARTBEAT_MS = 5_000;
|
|
233
|
+
// Liveness is decided by the claim ITSELF (docs/contributing/architecture.md#world-resource-and-diagnostics-boundary,
|
|
234
|
+
// landed 2026-09-04): ownership is the token the claimant minted and holds in memory — no other
|
|
235
|
+
// process ever has it, so a reused pid can never look like the owner — and life is the claim's
|
|
236
|
+
// heartbeat, a REAL instant refreshed by the holder (a world's frozen clock cannot measure
|
|
237
|
+
// liveness; this is host physics, not world time). Nothing here reads a process table.
|
|
238
|
+
function sharingClaimFile(root, name) {
|
|
239
|
+
return join(worldBaseDir(root), '.locks', `${name}.share.json`);
|
|
240
|
+
}
|
|
241
|
+
function readSharingClaim(path) {
|
|
242
|
+
if (!existsSync(path))
|
|
243
|
+
return undefined;
|
|
244
|
+
try {
|
|
245
|
+
const claim = JSON.parse(readFileSync(path, 'utf8'));
|
|
246
|
+
if (!claim || typeof claim !== 'object' || typeof claim.token !== 'string'
|
|
247
|
+
|| !Number.isInteger(claim.pid) || claim.pid <= 0 || typeof claim.hostname !== 'string'
|
|
248
|
+
|| typeof claim.claimedAt !== 'string' || typeof claim.heartbeatAt !== 'string'
|
|
249
|
+
|| (claim.serviceId !== undefined && typeof claim.serviceId !== 'string'))
|
|
250
|
+
return undefined;
|
|
251
|
+
return claim;
|
|
252
|
+
}
|
|
253
|
+
catch {
|
|
254
|
+
return undefined;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
/** Exported for direct drive: the liveness rule, pure over a claim. */
|
|
258
|
+
export function sharingClaimAlive(claim) {
|
|
259
|
+
const heartbeatAt = Date.parse(claim.heartbeatAt);
|
|
260
|
+
const heartbeatAge = Date.now() - heartbeatAt;
|
|
261
|
+
const fresh = Number.isFinite(heartbeatAt)
|
|
262
|
+
&& heartbeatAge >= -SHARE_CLAIM_STALE_MS && heartbeatAge <= SHARE_CLAIM_STALE_MS;
|
|
263
|
+
if (claim.hostname !== osHostname())
|
|
264
|
+
return fresh;
|
|
265
|
+
// Same host: a dead pid is a dead claim at once; a live pid is alive only while its holder
|
|
266
|
+
// keeps the heartbeat fresh — pid reuse cannot wedge sharing, because a reused pid never
|
|
267
|
+
// holds the token and never refreshes the heartbeat, so the claim ages out on the bound.
|
|
268
|
+
if (livePids([claim.pid]).length === 0)
|
|
269
|
+
return false;
|
|
270
|
+
return fresh;
|
|
271
|
+
}
|
|
272
|
+
async function reserveSharingClaim(root, name, serviceId) {
|
|
273
|
+
const path = sharingClaimFile(root, name);
|
|
274
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
275
|
+
while (true) {
|
|
276
|
+
const claim = withFileLock(`${path}.lock`, () => {
|
|
277
|
+
if (liveBootingClaim(stoppingFile(root, name)))
|
|
278
|
+
throw new Error(`World "${name}" is being stopped`);
|
|
279
|
+
const existing = readSharingClaim(path);
|
|
280
|
+
if (existing && sharingClaimAlive(existing)) {
|
|
281
|
+
if (existing.serviceId === serviceId) {
|
|
282
|
+
throw new Error(`World "${name}" service "${serviceId}" is already being shared by pid ${existing.pid}`);
|
|
283
|
+
}
|
|
284
|
+
return undefined;
|
|
285
|
+
}
|
|
286
|
+
const now = new Date().toISOString();
|
|
287
|
+
const reserved = {
|
|
288
|
+
token: randomUUID(), pid: process.pid, hostname: osHostname(), serviceId,
|
|
289
|
+
claimedAt: now, heartbeatAt: now,
|
|
290
|
+
};
|
|
291
|
+
writeTextAtomic(path, `${JSON.stringify(reserved)}\n`);
|
|
292
|
+
return reserved;
|
|
293
|
+
});
|
|
294
|
+
if (claim)
|
|
295
|
+
return claim;
|
|
296
|
+
// A different service is mutating the same instance/pids projection. Wait for its bounded,
|
|
297
|
+
// heartbeated claim to settle, then reload the committed instance before starting this child.
|
|
298
|
+
await sleep(50);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
function refreshSharingClaim(root, name, token) {
|
|
302
|
+
const path = sharingClaimFile(root, name);
|
|
303
|
+
withFileLock(`${path}.lock`, () => {
|
|
304
|
+
const claim = readSharingClaim(path);
|
|
305
|
+
if (claim?.token === token)
|
|
306
|
+
writeTextAtomic(path, `${JSON.stringify({ ...claim, heartbeatAt: new Date().toISOString() })}\n`);
|
|
307
|
+
});
|
|
308
|
+
}
|
|
309
|
+
function releaseSharingClaim(root, name, token) {
|
|
310
|
+
const path = sharingClaimFile(root, name);
|
|
311
|
+
withFileLock(`${path}.lock`, () => {
|
|
312
|
+
if (readSharingClaim(path)?.token === token)
|
|
313
|
+
rmSync(path, { force: true });
|
|
314
|
+
});
|
|
315
|
+
}
|
|
316
|
+
/** Claim marker a winning `upWorld` writes into the freshly-wiped instance dir — under the
|
|
317
|
+
* instance lock, before releasing it — and removes once its boot attempt settles. It is what
|
|
318
|
+
* keeps later claimants out for the whole (async, possibly long) boot: mid-boot there are no
|
|
319
|
+
* recorded pids yet, so without it a second `upWorld` would see "nothing running" and rmSync
|
|
320
|
+
* the winner's dir out from under its freshly-spawned services. */
|
|
321
|
+
function bootingFile(root, name) {
|
|
322
|
+
return join(instanceDir(root, name), 'booting.json');
|
|
323
|
+
}
|
|
324
|
+
/** The live booting claim for a world, or null when there is none to honor. Only ever read
|
|
325
|
+
* under the instance lock, and the marker is written under that same lock — so a torn read is
|
|
326
|
+
* impossible; a malformed marker is a corrupt leftover and treated as stale. A claim whose
|
|
327
|
+
* recorded booter pid is dead is stale too (crashed mid-boot): the next claimant reclaims the
|
|
328
|
+
* dir, exactly like livePids() lets a fully-dead world be re-upped. */
|
|
329
|
+
function liveBootingClaim(path) {
|
|
330
|
+
if (!existsSync(path))
|
|
331
|
+
return null;
|
|
332
|
+
let claim;
|
|
333
|
+
try {
|
|
334
|
+
claim = JSON.parse(readFileSync(path, 'utf8'));
|
|
335
|
+
}
|
|
336
|
+
catch {
|
|
337
|
+
return null;
|
|
338
|
+
}
|
|
339
|
+
if (!Number.isInteger(claim.pid) || claim.pid <= 0)
|
|
340
|
+
return null;
|
|
341
|
+
return livePids([claim.pid]).length > 0 ? claim : null;
|
|
342
|
+
}
|
|
343
|
+
/** Explicit down can finish a crashed boot, but only ESRCH proves its writer is absent.
|
|
344
|
+
* Called under the instance lock, before claiming teardown or signalling any resources. */
|
|
345
|
+
function prepareBootForDown(root, name) {
|
|
346
|
+
const path = bootingFile(root, name);
|
|
347
|
+
if (!existsSync(path))
|
|
348
|
+
return;
|
|
349
|
+
let claim;
|
|
350
|
+
try {
|
|
351
|
+
claim = JSON.parse(readFileSync(path, 'utf8'));
|
|
352
|
+
if (!claim || !Number.isInteger(claim.pid) || claim.pid <= 0 || typeof claim.at !== 'string'
|
|
353
|
+
|| !Number.isFinite(Date.parse(claim.at)))
|
|
354
|
+
throw new Error('invalid boot claim');
|
|
355
|
+
}
|
|
356
|
+
catch {
|
|
357
|
+
throw new Error(`World "${name}" has unreadable boot ownership; instance and lifecycle retained`);
|
|
358
|
+
}
|
|
359
|
+
try {
|
|
360
|
+
process.kill(claim.pid, 0);
|
|
361
|
+
}
|
|
362
|
+
catch (error) {
|
|
363
|
+
if (error.code !== 'ESRCH') {
|
|
364
|
+
throw new Error(`World "${name}" boot owner is uninspectable; instance and lifecycle retained`);
|
|
365
|
+
}
|
|
366
|
+
rmSync(path); // the recorded booter cannot publish anything after this point
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
369
|
+
if (claim.pid !== process.pid)
|
|
370
|
+
throw new Error(`World "${name}" is being booted; retry after boot finishes`);
|
|
371
|
+
// An in-process boot rollback still owns this marker until its own finally runs.
|
|
372
|
+
}
|
|
373
|
+
/** Where the session-scoped CA + on-the-fly leaf certs live for a world (under the instance dir, so
|
|
374
|
+
* `down`'s teardown removes them — the CA never outlives the world). */
|
|
375
|
+
function tlsDir(root, name) {
|
|
376
|
+
return join(instanceDir(root, name), 'tls');
|
|
377
|
+
}
|
|
378
|
+
/** State of a detached ambient-redirect proxy daemon (so `activate` can export its env and `down`
|
|
379
|
+
* can stop it). Written by the daemon, read by activate/down. */
|
|
380
|
+
/** The redirect proxy daemon's log, beside the World's service logs. */
|
|
381
|
+
function proxyLogPath(root, name) {
|
|
382
|
+
return join(instanceDir(root, name), 'logs', 'proxy.log');
|
|
383
|
+
}
|
|
384
|
+
function proxyStateFile(root, name) {
|
|
385
|
+
return join(instanceDir(root, name), 'proxy.json');
|
|
386
|
+
}
|
|
387
|
+
function proxyStartingFile(root, name) {
|
|
388
|
+
return join(instanceDir(root, name), 'proxy-starting.json');
|
|
389
|
+
}
|
|
390
|
+
function assertProxyOwnershipResolved(root, name) {
|
|
391
|
+
if (existsSync(proxyStartingFile(root, name))) {
|
|
392
|
+
throw new Error(`World "${name}" has unresolved proxy startup ownership; lifecycle and instance retained. Inspect: ${proxyStartingFile(root, name)}`);
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
function proxyEnvFile(root, name) {
|
|
396
|
+
return join(instanceDir(root, name), 'proxy-env.json');
|
|
397
|
+
}
|
|
398
|
+
function readProxyState(root, name) {
|
|
399
|
+
const file = proxyStateFile(root, name);
|
|
400
|
+
if (!existsSync(file))
|
|
401
|
+
return null;
|
|
402
|
+
try {
|
|
403
|
+
return JSON.parse(readFileSync(file, 'utf8'));
|
|
404
|
+
}
|
|
405
|
+
catch {
|
|
406
|
+
return null;
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
function shellQuote(value) {
|
|
410
|
+
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
411
|
+
}
|
|
412
|
+
function envFileContents(env) {
|
|
413
|
+
return `${Object.entries(env)
|
|
414
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
415
|
+
.map(([key, value]) => `export ${key}=${shellQuote(value)}`)
|
|
416
|
+
.join('\n')}\n`;
|
|
417
|
+
}
|
|
418
|
+
function injectPreloadSpecifier() {
|
|
419
|
+
// resolved on use: importing the runtime has no side effects in a host without module resolution
|
|
420
|
+
return createRequire(import.meta.url).resolve('@volter/world-core/inject');
|
|
421
|
+
}
|
|
422
|
+
function writeProxyEnv(root, name, env) {
|
|
423
|
+
writeTextAtomic(proxyEnvFile(root, name), `${JSON.stringify(env, null, 2)}\n`);
|
|
424
|
+
}
|
|
425
|
+
function readProxyEnv(path) {
|
|
426
|
+
const env = JSON.parse(readFileSync(path, 'utf8'));
|
|
427
|
+
if (!env || typeof env !== 'object' || Array.isArray(env) || Object.values(env).some(value => typeof value !== 'string')) {
|
|
428
|
+
throw new Error('World proxy environment is unreadable; refusing network access');
|
|
429
|
+
}
|
|
430
|
+
return env;
|
|
431
|
+
}
|
|
432
|
+
/** Resolve a service's cliRedirect map (env var → template), substituting the literal `${url}`
|
|
433
|
+
* with the service's resolved URL. These flow into the world env so the vendor's real CLI lands
|
|
434
|
+
* in the twin. See docs/guides/route-a-cli-through-the-world.md. */
|
|
435
|
+
function resolveCliRedirect(cliRedirect, url) {
|
|
436
|
+
if (!cliRedirect)
|
|
437
|
+
return {};
|
|
438
|
+
const out = {};
|
|
439
|
+
for (const [key, template] of Object.entries(cliRedirect))
|
|
440
|
+
out[key] = template.split('${url}').join(url);
|
|
441
|
+
return out;
|
|
442
|
+
}
|
|
443
|
+
function resolveServiceEnvTemplates(templates, service) {
|
|
444
|
+
if (!templates)
|
|
445
|
+
return {};
|
|
446
|
+
const out = {};
|
|
447
|
+
for (const [key, template] of Object.entries(templates)) {
|
|
448
|
+
out[key] = template
|
|
449
|
+
.split('${url}').join(service.url)
|
|
450
|
+
.split('${httpUrl}').join(service.url)
|
|
451
|
+
.split('${host}').join(service.host)
|
|
452
|
+
.split('${port}').join(String(service.port));
|
|
453
|
+
}
|
|
454
|
+
return out;
|
|
455
|
+
}
|
|
456
|
+
const CONTROL_PLANE_EGRESS_ENV = [
|
|
457
|
+
'NODE_OPTIONS',
|
|
458
|
+
'HTTP_PROXY',
|
|
459
|
+
'HTTPS_PROXY',
|
|
460
|
+
'ALL_PROXY',
|
|
461
|
+
'http_proxy',
|
|
462
|
+
'https_proxy',
|
|
463
|
+
'all_proxy',
|
|
464
|
+
];
|
|
465
|
+
/** The process's environment as the world admits it: what `stripEnv` names stays outside. */
|
|
466
|
+
export function admittedEnv(strip, env = process.env) {
|
|
467
|
+
const out = {};
|
|
468
|
+
const rules = (strip ?? []).map((s) => (s.endsWith('*') ? { prefix: s.slice(0, -1) } : { name: s }));
|
|
469
|
+
for (const [k, v] of Object.entries(env)) {
|
|
470
|
+
if (v === undefined)
|
|
471
|
+
continue;
|
|
472
|
+
// Permission is published by the attached instance, never inherited from
|
|
473
|
+
// whichever World happened to launch this operator process.
|
|
474
|
+
if (k === WORLD_NETWORK_POLICY_ENV)
|
|
475
|
+
continue;
|
|
476
|
+
if (rules.some((r) => ('prefix' in r ? k.startsWith(r.prefix) : k === r.name)))
|
|
477
|
+
continue;
|
|
478
|
+
out[k] = v;
|
|
479
|
+
}
|
|
480
|
+
return out;
|
|
481
|
+
}
|
|
482
|
+
// The strip list of a running world, for `env`/`attach`: its config is the one the instance recorded.
|
|
483
|
+
function stripEnvOf(instanceEnv) {
|
|
484
|
+
// Freeze the admission policy for the instance; editing/removing its source cannot widen it.
|
|
485
|
+
const saved = instanceEnv.VOLTER_WORLD_STRIP_ENV;
|
|
486
|
+
if (saved !== undefined) {
|
|
487
|
+
try {
|
|
488
|
+
const rules = JSON.parse(saved);
|
|
489
|
+
return Array.isArray(rules) && rules.every(rule => typeof rule === 'string') ? rules : ['*'];
|
|
490
|
+
}
|
|
491
|
+
catch {
|
|
492
|
+
return ['*'];
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
// Compatibility for instances created before policy persistence. An unreadable source fails closed.
|
|
496
|
+
const path = instanceEnv.VOLTER_WORLD_CONFIG;
|
|
497
|
+
if (!path)
|
|
498
|
+
return undefined;
|
|
499
|
+
try {
|
|
500
|
+
return loadWorldConfig(path).config.stripEnv;
|
|
501
|
+
}
|
|
502
|
+
catch {
|
|
503
|
+
return ['*'];
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
function serviceProcessEnv(config, service, worldEnv, port) {
|
|
507
|
+
const env = {
|
|
508
|
+
...admittedEnv(config.stripEnv),
|
|
509
|
+
...(config.env ?? {}),
|
|
510
|
+
...worldEnv,
|
|
511
|
+
PORT: String(port),
|
|
512
|
+
};
|
|
513
|
+
env.NO_PROXY = env.NO_PROXY ? `127.0.0.1,localhost,${env.NO_PROXY}` : '127.0.0.1,localhost';
|
|
514
|
+
env.no_proxy = env.no_proxy ? `127.0.0.1,localhost,${env.no_proxy}` : '127.0.0.1,localhost';
|
|
515
|
+
if (service.controlPlane) {
|
|
516
|
+
for (const key of CONTROL_PLANE_EGRESS_ENV)
|
|
517
|
+
delete env[key];
|
|
518
|
+
}
|
|
519
|
+
return { ...env, ...(service.env ?? {}) };
|
|
520
|
+
}
|
|
521
|
+
async function waitForTcp(port, timeoutMs = 15_000, abort) {
|
|
522
|
+
const started = Date.now();
|
|
523
|
+
while (Date.now() - started < timeoutMs) {
|
|
524
|
+
abort?.throwIfAborted();
|
|
525
|
+
const ok = await new Promise((resolveOk) => {
|
|
526
|
+
const socket = new Socket();
|
|
527
|
+
socket.setTimeout(500);
|
|
528
|
+
socket.once('connect', () => {
|
|
529
|
+
socket.destroy();
|
|
530
|
+
resolveOk(true);
|
|
531
|
+
});
|
|
532
|
+
socket.once('timeout', () => {
|
|
533
|
+
socket.destroy();
|
|
534
|
+
resolveOk(false);
|
|
535
|
+
});
|
|
536
|
+
socket.once('error', () => resolveOk(false));
|
|
537
|
+
socket.connect(port, '127.0.0.1');
|
|
538
|
+
});
|
|
539
|
+
if (ok)
|
|
540
|
+
return;
|
|
541
|
+
await new Promise((resolveWait) => setTimeout(resolveWait, 25));
|
|
542
|
+
}
|
|
543
|
+
throw new Error(`Service on port ${port} did not become reachable`);
|
|
544
|
+
}
|
|
545
|
+
function readPids(path) {
|
|
546
|
+
const held = getActiveWorldStore().read(path);
|
|
547
|
+
if (held === null)
|
|
548
|
+
return [];
|
|
549
|
+
return held
|
|
550
|
+
.split(/\r?\n/)
|
|
551
|
+
.map((line) => Number(line.trim()))
|
|
552
|
+
.filter((pid) => Number.isInteger(pid) && pid > 0);
|
|
553
|
+
}
|
|
554
|
+
function livePids(pids) {
|
|
555
|
+
return pids.filter((pid) => {
|
|
556
|
+
try {
|
|
557
|
+
process.kill(pid, 0);
|
|
558
|
+
return true;
|
|
559
|
+
}
|
|
560
|
+
catch (error) {
|
|
561
|
+
return error.code !== 'ESRCH';
|
|
562
|
+
}
|
|
563
|
+
});
|
|
564
|
+
}
|
|
565
|
+
function tunnelPids(instance) {
|
|
566
|
+
return Object.values(instance.services)
|
|
567
|
+
.map((service) => service.tunnel?.pid)
|
|
568
|
+
.filter((pid) => Boolean(pid));
|
|
569
|
+
}
|
|
570
|
+
function writeTextAtomic(path, value) {
|
|
571
|
+
// through the World store: the filesystem locally (temp, rename, 0600), a hosted World's SQLite
|
|
572
|
+
getActiveWorldStore().writeAtomic(path, value, { secret: true });
|
|
573
|
+
}
|
|
574
|
+
function openPrivateLog(path, flags) {
|
|
575
|
+
const descriptor = openSync(path, flags, 0o600);
|
|
576
|
+
chmodSync(path, 0o600);
|
|
577
|
+
return descriptor;
|
|
578
|
+
}
|
|
579
|
+
/** PID ownership survives projection rewrites. This lock is always inside lifecycle locks,
|
|
580
|
+
* never held while acquiring one; a proxy append cannot race a share/unshare rebuild. */
|
|
581
|
+
/** Record `pids` as the World's. `fresh` are the processes this very call has spawned: only their start times are
|
|
582
|
+
* measured anew; every other pid keeps the start time recorded when it was spawned (re-measuring it would bless
|
|
583
|
+
* whatever process holds that number now). */
|
|
584
|
+
function writeOwnedPids(root, name, pids, fresh = []) {
|
|
585
|
+
const path = join(instanceDir(root, name), 'pids');
|
|
586
|
+
withFileLock(`${instanceLockFile(root, name)}.pids`, () => {
|
|
587
|
+
const retained = undeadPids(readPids(path));
|
|
588
|
+
const merged = [...new Set([...retained, ...pids].filter(pid => Number.isInteger(pid) && pid > 0))];
|
|
589
|
+
writeTextAtomic(path, `${merged.join('\n')}\n`);
|
|
590
|
+
recordIdentities(root, name, merged, fresh);
|
|
591
|
+
});
|
|
592
|
+
}
|
|
593
|
+
// ---- which process a recorded pid names -------------------------------------------------------------------------
|
|
594
|
+
// A pid outlives its process: once the World's process has exited the kernel may hand the number to anyone. Each owned
|
|
595
|
+
// pid's start time is recorded beside the pids file, and a pid whose running process started at another time is no
|
|
596
|
+
// longer the World's: it is never signalled and never keeps the World running.
|
|
597
|
+
function identityPath(root, name) {
|
|
598
|
+
return join(instanceDir(root, name), 'pids.identity');
|
|
599
|
+
}
|
|
600
|
+
/** Each running pid's start time (one call); a pid with no process is absent. Printed in one fixed zone and locale, so
|
|
601
|
+
* the same process reads the same whoever asks (`lstart` otherwise follows the caller's TZ and language). */
|
|
602
|
+
function processStarts(pids) {
|
|
603
|
+
const starts = new Map();
|
|
604
|
+
if (!pids.length || process.platform === 'win32')
|
|
605
|
+
return starts;
|
|
606
|
+
if (process.platform === 'linux' && existsSync('/proc/self/stat')) {
|
|
607
|
+
// Linux: the start tick since boot (field 22), which a wall-clock step does not move (lstart would).
|
|
608
|
+
for (const pid of pids) {
|
|
609
|
+
try {
|
|
610
|
+
const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
|
|
611
|
+
starts.set(pid, `tick ${stat.slice(stat.lastIndexOf(')') + 2).split(' ')[19]}`);
|
|
612
|
+
}
|
|
613
|
+
catch { /* no process */ }
|
|
614
|
+
}
|
|
615
|
+
return starts;
|
|
616
|
+
}
|
|
617
|
+
const result = spawnSync('ps', ['-o', 'pid=,lstart=', '-p', pids.join(',')], { encoding: 'utf8', timeout: 1000, killSignal: 'SIGKILL', env: { ...process.env, LC_ALL: 'C', LANG: 'C', TZ: 'UTC' } });
|
|
618
|
+
for (const line of (result.stdout ?? '').split('\n')) {
|
|
619
|
+
const match = /^\s*(\d+)\s+(.+?)\s*$/.exec(line);
|
|
620
|
+
if (match)
|
|
621
|
+
starts.set(Number(match[1]), match[2]);
|
|
622
|
+
}
|
|
623
|
+
return starts;
|
|
624
|
+
}
|
|
625
|
+
function readIdentities(root, name) {
|
|
626
|
+
try {
|
|
627
|
+
return JSON.parse(getActiveWorldStore().read(identityPath(root, name)) ?? '{}');
|
|
628
|
+
}
|
|
629
|
+
catch {
|
|
630
|
+
return {};
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
function recordIdentities(root, name, pids, fresh = []) {
|
|
634
|
+
try {
|
|
635
|
+
// A pid the World has just spawned is measured anew, whatever an older record says of that number.
|
|
636
|
+
const known = readIdentities(root, name), starts = processStarts(pids.filter((pid) => known[pid] === undefined || fresh.includes(pid)));
|
|
637
|
+
for (const pid of fresh)
|
|
638
|
+
if (starts.has(pid))
|
|
639
|
+
delete known[pid];
|
|
640
|
+
const kept = Object.fromEntries(pids.flatMap((pid) => { const start = known[pid] ?? starts.get(pid); return start ? [[String(pid), start]] : []; }));
|
|
641
|
+
writeTextAtomic(identityPath(root, name), JSON.stringify(kept));
|
|
642
|
+
}
|
|
643
|
+
catch { /* without a record a pid is trusted as before */ }
|
|
644
|
+
}
|
|
645
|
+
/** Recorded pids whose running process started at another time than recorded: the number now names another
|
|
646
|
+
* program. An unrecorded pid, and a pid with no process, are never in this list. */
|
|
647
|
+
function foreignPids(root, name, pids) {
|
|
648
|
+
const known = readIdentities(root, name);
|
|
649
|
+
const recorded = pids.filter((pid) => known[pid] !== undefined);
|
|
650
|
+
if (!recorded.length)
|
|
651
|
+
return [];
|
|
652
|
+
const starts = processStarts(recorded);
|
|
653
|
+
return recorded.flatMap((pid) => { const now = starts.get(pid); return now !== undefined && now !== known[pid] ? [{ pid, recorded: known[pid], now }] : []; });
|
|
654
|
+
}
|
|
655
|
+
/** The pids still naming the processes the World started (see foreignPids). */
|
|
656
|
+
function ownedPids(root, name, pids) {
|
|
657
|
+
const foreign = new Set(foreignPids(root, name, pids).map((entry) => entry.pid));
|
|
658
|
+
return foreign.size ? pids.filter((pid) => !foreign.has(pid)) : pids;
|
|
659
|
+
}
|
|
660
|
+
function writePidsFromInstance(instance, fresh = []) {
|
|
661
|
+
// Deduped: co-located services share the single host child's pid.
|
|
662
|
+
const pids = [...new Set([
|
|
663
|
+
...Object.values(instance.services).map((service) => service.pid),
|
|
664
|
+
...tunnelPids(instance),
|
|
665
|
+
...((instance.lifecycle?.holderPid ?? instance.resources?.holderPid) ? [instance.lifecycle?.holderPid ?? instance.resources.holderPid] : []),
|
|
666
|
+
].filter((pid) => Number.isInteger(pid) && pid > 0))];
|
|
667
|
+
writeOwnedPids(instance.root, instance.name, pids, fresh);
|
|
668
|
+
}
|
|
669
|
+
export function saveWorldInstance(instance) {
|
|
670
|
+
writeTextAtomic(instanceFile(instance.root, instance.name), `${JSON.stringify(instance, null, 2)}\n`);
|
|
671
|
+
}
|
|
672
|
+
/** The co-located host CHILD process cannot safely rewrite instance.json out from under the
|
|
673
|
+
* parent (which may still be mid-boot, or rewrite it later via share/unshare) — so a worker
|
|
674
|
+
* give-up (host.ts fires it on the first crash — never restarted) is recorded in a sidecar
|
|
675
|
+
* file next to instance.json instead
|
|
676
|
+
* (see host-cli.ts). Folding it in here means every reader (doctor, status, tests) sees the
|
|
677
|
+
* record on `services[<id>].workerGaveUp` exactly as if it had been written into instance.json. */
|
|
678
|
+
function mergeWorkerGaveUp(instance, instancePath) {
|
|
679
|
+
const held = getActiveWorldStore().read(join(dirname(instancePath), 'host-gaveup.json'));
|
|
680
|
+
if (held === null)
|
|
681
|
+
return;
|
|
682
|
+
try {
|
|
683
|
+
const gaveUp = JSON.parse(held);
|
|
684
|
+
for (const [id, record] of Object.entries(gaveUp)) {
|
|
685
|
+
const service = instance.services[id];
|
|
686
|
+
if (service)
|
|
687
|
+
service.workerGaveUp = record;
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
catch {
|
|
691
|
+
// Best-effort merge: a sidecar torn by a concurrent host-cli write must never break an
|
|
692
|
+
// instance.json read (the give-up record just fails to surface for this one read).
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
/** `world run` persists this sidecar before the foreground consumer starts. If the runner itself
|
|
696
|
+
* is hard-killed, it cannot record its own signal or execute `finally`; the next World read turns
|
|
697
|
+
* that orphaned running marker into an explicit, durable abrupt-termination diagnosis. */
|
|
698
|
+
function mergeForegroundRun(instance, persist = true) {
|
|
699
|
+
const path = foregroundRunFile(instance.root, instance.name);
|
|
700
|
+
const held = getActiveWorldStore().read(path);
|
|
701
|
+
if (held === null)
|
|
702
|
+
return;
|
|
703
|
+
try {
|
|
704
|
+
let record = JSON.parse(held);
|
|
705
|
+
if (record.state === 'running' && !isAlive(record.runnerPid)) {
|
|
706
|
+
record = {
|
|
707
|
+
state: 'abrupt',
|
|
708
|
+
runnerPid: record.runnerPid,
|
|
709
|
+
...(record.consumerPid ? { consumerPid: record.consumerPid } : {}),
|
|
710
|
+
startedAt: record.startedAt,
|
|
711
|
+
observedAt: new Date().toISOString(),
|
|
712
|
+
log: record.log,
|
|
713
|
+
error: 'World runner disappeared without an exit record; it was hard-killed or its host/session terminated abruptly',
|
|
714
|
+
};
|
|
715
|
+
if (persist)
|
|
716
|
+
writeTextAtomic(path, `${JSON.stringify(record, null, 2)}\n`);
|
|
717
|
+
}
|
|
718
|
+
instance.lastRun = record;
|
|
719
|
+
}
|
|
720
|
+
catch {
|
|
721
|
+
// Preserve instance readability if a process vanished between the sidecar's atomic writes.
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
function readWorldInstance(name, root, readOnly = false) {
|
|
725
|
+
const path = instanceFile(root, name);
|
|
726
|
+
const text = getActiveWorldStore().read(path);
|
|
727
|
+
if (text === null)
|
|
728
|
+
throw new Error(`World instance not found: ${name}`);
|
|
729
|
+
const instance = JSON.parse(text);
|
|
730
|
+
mergeWorkerGaveUp(instance, path);
|
|
731
|
+
// A read-only inspector never follows unvalidated metadata to a different World's sidecar.
|
|
732
|
+
if (!readOnly || (instance.name === name && typeof instance.root === 'string' && resolve(instance.root) === resolve(root))) {
|
|
733
|
+
mergeForegroundRun(instance, !readOnly);
|
|
734
|
+
}
|
|
735
|
+
return instance;
|
|
736
|
+
}
|
|
737
|
+
function assertMode(mode) {
|
|
738
|
+
if (mode !== 'local' && mode !== 'share' && mode !== 'sealed')
|
|
739
|
+
throw new Error(`Invalid world mode: ${mode}`);
|
|
740
|
+
}
|
|
741
|
+
function getByJsonPath(value, jsonPath) {
|
|
742
|
+
let current = value;
|
|
743
|
+
for (const rawSegment of jsonPath.split('.')) {
|
|
744
|
+
const segment = rawSegment.trim();
|
|
745
|
+
if (segment === '')
|
|
746
|
+
continue;
|
|
747
|
+
if (current === null || current === undefined)
|
|
748
|
+
return undefined;
|
|
749
|
+
if (Array.isArray(current)) {
|
|
750
|
+
const index = Number(segment);
|
|
751
|
+
if (!Number.isInteger(index))
|
|
752
|
+
return undefined;
|
|
753
|
+
current = current[index];
|
|
754
|
+
}
|
|
755
|
+
else if (typeof current === 'object') {
|
|
756
|
+
current = current[segment];
|
|
757
|
+
}
|
|
758
|
+
else {
|
|
759
|
+
return undefined;
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
return current;
|
|
763
|
+
}
|
|
764
|
+
async function runExternalCommand(command, cwd, serviceId, phase, env = process.env, abort, timeoutMs = 60_000) {
|
|
765
|
+
const [bin, ...args] = command;
|
|
766
|
+
if (!bin)
|
|
767
|
+
throw new Error(`External service "${serviceId}": ${phase} command is empty`);
|
|
768
|
+
if (!commandExists(bin))
|
|
769
|
+
throw new Error(`External service "${serviceId}": \`${bin}\` not found on PATH (the ${phase} command's tool is not installed)`);
|
|
770
|
+
abort?.throwIfAborted();
|
|
771
|
+
const child = spawn(bin, args, { cwd, env, detached: commandProcessGroup, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
772
|
+
const lifetime = commandLifetime(child, abort, true);
|
|
773
|
+
let stdout = '';
|
|
774
|
+
let stderr = '';
|
|
775
|
+
let failure;
|
|
776
|
+
const collect = (chunk, out) => {
|
|
777
|
+
if (out)
|
|
778
|
+
stdout += chunk.toString();
|
|
779
|
+
else
|
|
780
|
+
stderr += chunk.toString();
|
|
781
|
+
if (stdout.length + stderr.length > 64 * 1024 * 1024) {
|
|
782
|
+
failure = new Error('command output exceeds 64 MiB');
|
|
783
|
+
lifetime.cancel('SIGTERM');
|
|
784
|
+
}
|
|
785
|
+
};
|
|
786
|
+
child.stdout.on('data', chunk => collect(chunk, true));
|
|
787
|
+
child.stderr.on('data', chunk => collect(chunk, false));
|
|
788
|
+
child.on('error', error => { failure = error; });
|
|
789
|
+
const timer = setTimeout(() => { failure = new Error(`command timed out after ${timeoutMs}ms`); lifetime.cancel('SIGTERM'); }, timeoutMs);
|
|
790
|
+
try {
|
|
791
|
+
const result = await lifetime.finish();
|
|
792
|
+
if (failure)
|
|
793
|
+
throw new Error(`External service "${serviceId}": ${phase} command failed to run (${failure.message})`);
|
|
794
|
+
if (result.code !== 0 || lifetime.forwarded)
|
|
795
|
+
throw new Error(`External service "${serviceId}": ${phase} command exited ${result.code ?? 'null'}\n${stdout}${stderr}`);
|
|
796
|
+
return { stdout, stderr };
|
|
797
|
+
}
|
|
798
|
+
finally {
|
|
799
|
+
clearTimeout(timer);
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
/** Run an external service's `up` command, STREAMING its stdout+stderr to the service log as it runs
|
|
803
|
+
* (instead of buffering through spawnSync's 1MB cap, which a large DB restore would overflow). Returns
|
|
804
|
+
* the combined output so `discover` can read it. Throws with a bounded tail on nonzero exit. */
|
|
805
|
+
async function runExternalUp(command, cwd, serviceId, env, logPath, abort) {
|
|
806
|
+
const [bin, ...args] = command;
|
|
807
|
+
if (!bin)
|
|
808
|
+
throw new Error(`External service "${serviceId}": up command is empty`);
|
|
809
|
+
if (!commandExists(bin)) {
|
|
810
|
+
throw new Error(`External service "${serviceId}": \`${bin}\` not found on PATH (the up command's tool is not installed)`);
|
|
811
|
+
}
|
|
812
|
+
writeFileSync(logPath, `$ ${command.join(' ')}\n`, { mode: 0o600 });
|
|
813
|
+
chmodSync(logPath, 0o600);
|
|
814
|
+
const out = openPrivateLog(logPath, 'a');
|
|
815
|
+
const chunks = [];
|
|
816
|
+
try {
|
|
817
|
+
abort?.throwIfAborted();
|
|
818
|
+
const child = spawn(bin, args, { cwd, env, detached: commandProcessGroup, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
819
|
+
const lifetime = commandLifetime(child, abort, true);
|
|
820
|
+
let failure;
|
|
821
|
+
child.on('error', error => { failure = error; });
|
|
822
|
+
const onData = (buf) => {
|
|
823
|
+
const text = buf.toString('utf8');
|
|
824
|
+
chunks.push(text);
|
|
825
|
+
try {
|
|
826
|
+
writeSync(out, text);
|
|
827
|
+
}
|
|
828
|
+
catch (error) {
|
|
829
|
+
failure = error;
|
|
830
|
+
lifetime.cancel('SIGTERM');
|
|
831
|
+
}
|
|
832
|
+
};
|
|
833
|
+
child.stdout.on('data', onData);
|
|
834
|
+
child.stderr.on('data', onData);
|
|
835
|
+
const { code } = await lifetime.finish();
|
|
836
|
+
if (failure)
|
|
837
|
+
throw failure;
|
|
838
|
+
abort?.throwIfAborted();
|
|
839
|
+
if (code !== 0) {
|
|
840
|
+
const detail = chunks.join('').trim().split('\n').slice(-40).join('\n');
|
|
841
|
+
throw new Error(`External service "${serviceId}": up command exited ${code ?? 'null'}\n${detail}`);
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
finally {
|
|
845
|
+
closeSync(out);
|
|
846
|
+
}
|
|
847
|
+
return chunks.join('');
|
|
848
|
+
}
|
|
849
|
+
/** Resolve `${url}`/`${host}`/`${port}` in a readiness probe's httpUrl against an owned loopback
|
|
850
|
+
* service's allocated port (process/twin readiness — the port is auto-assigned, so it can't be a
|
|
851
|
+
* literal in the config). Other probe kinds (command/stdoutMatch) are returned unchanged. */
|
|
852
|
+
function resolveReadyProbe(ready, svc) {
|
|
853
|
+
if (ready.httpUrl === undefined)
|
|
854
|
+
return ready;
|
|
855
|
+
const httpUrl = ready.httpUrl
|
|
856
|
+
.split('${url}').join(svc.url)
|
|
857
|
+
.split('${host}').join(svc.host)
|
|
858
|
+
.split('${port}').join(String(svc.port));
|
|
859
|
+
return { ...ready, httpUrl };
|
|
860
|
+
}
|
|
861
|
+
async function awaitReadiness(ready, logPath, cwd, serviceId, env = process.env, abort) {
|
|
862
|
+
if (!ready)
|
|
863
|
+
return;
|
|
864
|
+
const timeoutMs = ready.timeoutMs ?? 60_000;
|
|
865
|
+
const intervalMs = ready.intervalMs ?? 500;
|
|
866
|
+
const started = Date.now();
|
|
867
|
+
let lastError = '';
|
|
868
|
+
while (Date.now() - started < timeoutMs) {
|
|
869
|
+
abort?.throwIfAborted();
|
|
870
|
+
if (ready.command !== undefined) {
|
|
871
|
+
if (!commandExists(ready.command)) {
|
|
872
|
+
throw new Error(`Service "${serviceId}": readiness command \`${ready.command}\` not found on PATH`);
|
|
873
|
+
}
|
|
874
|
+
try {
|
|
875
|
+
await runExternalCommand([ready.command, ...(ready.args ?? [])], cwd, serviceId, 'readiness', env, abort, Math.max(1, timeoutMs - (Date.now() - started)));
|
|
876
|
+
return;
|
|
877
|
+
}
|
|
878
|
+
catch (error) {
|
|
879
|
+
if (error instanceof CommandRetirementError)
|
|
880
|
+
throw error;
|
|
881
|
+
lastError = String(error);
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
else if (ready.httpUrl !== undefined) {
|
|
885
|
+
try {
|
|
886
|
+
const response = await fetch(ready.httpUrl, { signal: AbortSignal.any([AbortSignal.timeout(Math.max(1, timeoutMs - (Date.now() - started))), ...(abort ? [abort] : [])]) });
|
|
887
|
+
if (response.status >= 200 && response.status < 400)
|
|
888
|
+
return;
|
|
889
|
+
lastError = `HTTP ${response.status}`;
|
|
890
|
+
}
|
|
891
|
+
catch (error) {
|
|
892
|
+
lastError = error instanceof Error ? error.message : String(error);
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
else if (ready.stdoutMatch !== undefined) {
|
|
896
|
+
const text = serviceOutput(readServiceText(logPath));
|
|
897
|
+
if (new RegExp(ready.stdoutMatch).test(text))
|
|
898
|
+
return;
|
|
899
|
+
lastError = `pattern /${ready.stdoutMatch}/ not yet in service log`;
|
|
900
|
+
}
|
|
901
|
+
await sleep(intervalMs);
|
|
902
|
+
}
|
|
903
|
+
throw new Error(`Service "${serviceId}": readiness probe timed out after ${timeoutMs}ms (${lastError || 'no probe matched'})`);
|
|
904
|
+
}
|
|
905
|
+
async function startExternalService(service, paths, worldEnv, stripEnv, boot) {
|
|
906
|
+
const external = service.external;
|
|
907
|
+
if (!external)
|
|
908
|
+
throw new Error(`External service "${service.id}" is missing its external config`);
|
|
909
|
+
const log = join(paths.logs, `${service.id}.log`);
|
|
910
|
+
const cwd = service.cwd ? resolve(paths.root, service.cwd) : paths.root;
|
|
911
|
+
// External commands see the accumulated world env (process.env + earlier services' discovered/
|
|
912
|
+
// injected vars), so a later external can consume an earlier service's connection info.
|
|
913
|
+
const env = {
|
|
914
|
+
...admittedEnv(stripEnv),
|
|
915
|
+
...worldEnv,
|
|
916
|
+
...(service.env ?? {}),
|
|
917
|
+
VOLTER_WORLD_SERVICE_ID: service.id,
|
|
918
|
+
VOLTER_WORLD_SERVICE_LOG: log,
|
|
919
|
+
};
|
|
920
|
+
if (service.controlPlane)
|
|
921
|
+
for (const key of CONTROL_PLANE_EGRESS_ENV)
|
|
922
|
+
delete env[key];
|
|
923
|
+
// 1. up — start the self-managed stack, STREAMING its output to the service log (no 1MB cap).
|
|
924
|
+
if (!external.up[0] || !commandExists(external.up[0])) {
|
|
925
|
+
throw new Error(`External service "${service.id}": \`${external.up[0] ?? ''}\` not found on PATH (the up command's tool is not installed)`);
|
|
926
|
+
}
|
|
927
|
+
// Publish the declared teardown before an external command can create state.
|
|
928
|
+
boot?.external({ id: service.id, type: 'external', command: [...external.up], pid: 0, log, env: {},
|
|
929
|
+
external: { down: [...external.down], cwd, discoveredEnv: { ...(service.env ?? {}) } } });
|
|
930
|
+
const upOutput = await runExternalUp(external.up, cwd, service.id, env, log, boot?.signal);
|
|
931
|
+
// 2. readiness — Docker-backed externals take seconds to boot.
|
|
932
|
+
await awaitReadiness(external.readyWhen, log, cwd, service.id, env, boot?.signal);
|
|
933
|
+
// 3. status — discover connection info (optional; may reuse up output instead).
|
|
934
|
+
let statusOutput = '';
|
|
935
|
+
if (external.status) {
|
|
936
|
+
const statusResult = await runExternalCommand(external.status, cwd, service.id, 'status', env, boot?.signal);
|
|
937
|
+
statusOutput = statusResult.stdout || statusResult.stderr;
|
|
938
|
+
appendFileSync(log, `\n$ ${external.status.join(' ')}\n${statusOutput}`);
|
|
939
|
+
}
|
|
940
|
+
// 4. discover — apply declarative mappings → injected env vars.
|
|
941
|
+
const injected = {};
|
|
942
|
+
const sources = {
|
|
943
|
+
status: statusOutput,
|
|
944
|
+
up: upOutput,
|
|
945
|
+
};
|
|
946
|
+
const jsonCache = {};
|
|
947
|
+
for (const mapping of external.discover ?? []) {
|
|
948
|
+
const source = mapping.source ?? 'status';
|
|
949
|
+
const raw = sources[source];
|
|
950
|
+
let value;
|
|
951
|
+
if (mapping.jsonPath !== undefined) {
|
|
952
|
+
if (!(source in jsonCache)) {
|
|
953
|
+
try {
|
|
954
|
+
jsonCache[source] = JSON.parse(raw);
|
|
955
|
+
}
|
|
956
|
+
catch (error) {
|
|
957
|
+
throw new Error(`External service "${service.id}": discover "${mapping.as}" expected JSON from ${source} output but parse failed (${error instanceof Error ? error.message : String(error)})\nOutput: ${raw.slice(0, 500)}`);
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
const found = getByJsonPath(jsonCache[source], mapping.jsonPath);
|
|
961
|
+
value = found === undefined || found === null ? undefined : String(found);
|
|
962
|
+
}
|
|
963
|
+
else if (mapping.pattern !== undefined) {
|
|
964
|
+
const match = new RegExp(mapping.pattern).exec(raw);
|
|
965
|
+
value = match?.groups?.value ?? match?.[1] ?? match?.[0];
|
|
966
|
+
}
|
|
967
|
+
if (value === undefined || value === '') {
|
|
968
|
+
throw new Error(`External service "${service.id}": discover "${mapping.as}" found ${value === '' ? 'an empty' : 'no'} value (jsonPath=${mapping.jsonPath ?? '-'} pattern=${mapping.pattern ?? '-'} source=${source})\nOutput: ${raw.slice(0, 500)}`);
|
|
969
|
+
}
|
|
970
|
+
injected[mapping.as] = value;
|
|
971
|
+
}
|
|
972
|
+
return {
|
|
973
|
+
id: service.id,
|
|
974
|
+
type: 'external',
|
|
975
|
+
command: [...external.up],
|
|
976
|
+
pid: 0,
|
|
977
|
+
log,
|
|
978
|
+
env: injected,
|
|
979
|
+
// Keep the discovered env on the instance so `down` can reference connection info it needs.
|
|
980
|
+
external: { down: [...external.down], cwd, discoveredEnv: injected },
|
|
981
|
+
};
|
|
982
|
+
}
|
|
983
|
+
function trackBootingChild(child, boot) {
|
|
984
|
+
if (!child.pid)
|
|
985
|
+
return;
|
|
986
|
+
boot.pids.add(child.pid);
|
|
987
|
+
boot.recordPids();
|
|
988
|
+
}
|
|
989
|
+
async function startService(config, service, worldEnv, paths, allocatedPorts, boot) {
|
|
990
|
+
if (service.type === 'external')
|
|
991
|
+
return startExternalService(service, paths, worldEnv, config.stripEnv, boot);
|
|
992
|
+
if (!service.command)
|
|
993
|
+
throw new Error(`Service "${service.id}" must define command`);
|
|
994
|
+
const port = service.port === undefined || service.port === 'auto' ? await allocatePort(allocatedPorts) : service.port;
|
|
995
|
+
const serviceDataDir = join(paths.data, service.id);
|
|
996
|
+
mkdirSync(serviceDataDir, { recursive: true });
|
|
997
|
+
// R17 — the lockfile holds on the spawn path too: a twin that runs `packages/twin/<vendor>/src/cli.ts`
|
|
998
|
+
// is checked against its pin and its protocol standing before it is spawned.
|
|
999
|
+
const spawnedModule = (service.type ?? 'twin') === 'twin' ? spawnedTwinModule(service, paths.root) : undefined;
|
|
1000
|
+
const mounted = spawnedModule !== undefined ? lockfileCheck(service, spawnedModule) : {};
|
|
1001
|
+
const args = [...(service.args ?? [])];
|
|
1002
|
+
if (service.portArg !== false)
|
|
1003
|
+
args.push(service.portArg ?? '--port', String(port));
|
|
1004
|
+
if (service.rootArg !== false)
|
|
1005
|
+
args.push(service.rootArg ?? '--root', serviceDataDir);
|
|
1006
|
+
const log = join(paths.logs, `${service.id}.log`);
|
|
1007
|
+
const env = serviceProcessEnv(config, service, worldEnv, port);
|
|
1008
|
+
const cwd = service.cwd ? resolve(paths.root, service.cwd) : paths.root;
|
|
1009
|
+
// Augment NODE_OPTIONS with declared extra preloads, so a service can add an app-local preload
|
|
1010
|
+
// (e.g. a sandbox) WITHOUT replacing the world's injector. Paths resolve to absolute against the
|
|
1011
|
+
// service cwd — Node's `--require` treats a bare relative specifier as a package name otherwise.
|
|
1012
|
+
// (Setting service.env.NODE_OPTIONS directly still replaces the default — the explicit opt-out.)
|
|
1013
|
+
if (service.preload?.length) {
|
|
1014
|
+
const requires = service.preload.map((module) => `--require ${resolve(cwd, module)}`).join(' ');
|
|
1015
|
+
env.NODE_OPTIONS = env.NODE_OPTIONS ? `${env.NODE_OPTIONS} ${requires}` : requires;
|
|
1016
|
+
}
|
|
1017
|
+
// The service runs under the service recorder, which outlives it to write its end into its own log: `up`
|
|
1018
|
+
// returns once the World is ready, so nothing else is left to see a service stop.
|
|
1019
|
+
env.VOLTER_WORLD_BOOT_ID = boot.bootId;
|
|
1020
|
+
const { child, ended } = spawnRecorded(service.id, service.command, args, cwd, env, log);
|
|
1021
|
+
child.once('error', error => boot.fail(error));
|
|
1022
|
+
child.unref();
|
|
1023
|
+
trackBootingChild(child, boot);
|
|
1024
|
+
// Track an early exit. allocatePort() is TOCTOU (it closes its probe listener before the
|
|
1025
|
+
// child binds), so if another/stale process already holds `port`, the child gets EADDRINUSE
|
|
1026
|
+
// and dies — but waitForTcp would then see the OTHER process answering and the world would
|
|
1027
|
+
// silently record this service's URL pointing at the WRONG twin (the multi-twin crossing
|
|
1028
|
+
// bug). Detecting the child's own exit lets us fail LOUDLY instead.
|
|
1029
|
+
// Object holder (not a bare `let`) so TS keeps the union type across the exit closure.
|
|
1030
|
+
const exitState = { value: null };
|
|
1031
|
+
child.on('exit', (code, signal) => {
|
|
1032
|
+
exitState.value = { code, signal };
|
|
1033
|
+
});
|
|
1034
|
+
// A service that declares a readiness budget gets it for its bind too: under load (a box
|
|
1035
|
+
// swapping, a docker build beside the boot) a heavy service takes longer than the 15s default
|
|
1036
|
+
// to answer on its port, and failing it there wastes the whole boot.
|
|
1037
|
+
const bindTimeoutMs = typeof service.ready === 'object' && service.ready.timeoutMs !== undefined ? Math.max(service.ready.timeoutMs, 15_000) : 15_000;
|
|
1038
|
+
await untilReadyOrEnded((abort) => waitForTcp(port, bindTimeoutMs, abort), ended, boot, 'the service', log).catch((error) => {
|
|
1039
|
+
if (undeadPids([child.pid]).length)
|
|
1040
|
+
recordWorldStop([log], 'world.service.stop', 'up (startup failed)', 'SIGTERM');
|
|
1041
|
+
try {
|
|
1042
|
+
process.kill(-child.pid, 'SIGTERM');
|
|
1043
|
+
}
|
|
1044
|
+
catch {
|
|
1045
|
+
// best effort cleanup
|
|
1046
|
+
}
|
|
1047
|
+
throw new Error(`Service "${service.id}" failed to start: ${error instanceof Error ? error.message : String(error)}\nLog: ${log}`);
|
|
1048
|
+
});
|
|
1049
|
+
await confirmOwnPort(port, boot, ended, (service.type ?? 'twin') === 'twin', `Service "${service.id}"`, log, true).catch((error) => {
|
|
1050
|
+
if (undeadPids([child.pid]).length)
|
|
1051
|
+
recordWorldStop([log], 'world.service.stop', 'up (port not its own)', 'SIGTERM');
|
|
1052
|
+
try {
|
|
1053
|
+
process.kill(-child.pid, 'SIGTERM');
|
|
1054
|
+
}
|
|
1055
|
+
catch { /* best effort cleanup */ }
|
|
1056
|
+
throw error;
|
|
1057
|
+
});
|
|
1058
|
+
if (exitState.value !== null) {
|
|
1059
|
+
throw new Error(`Service "${service.id}" ended during startup even though port ${port} answers, so another/stale process probably holds the port: the service ${endedAccount(log)}\n`
|
|
1060
|
+
+ `Stop leftover services (e.g. \`volter-world down\`) and retry.\nLog: ${log}`);
|
|
1061
|
+
}
|
|
1062
|
+
const host = '127.0.0.1';
|
|
1063
|
+
const url = `http://${host}:${port}`;
|
|
1064
|
+
// Optional readiness probe beyond TCP (httpUrl/command/stdoutMatch): a process can bind its port
|
|
1065
|
+
// before it is actually serving, so this makes `up` wait until the service truly responds.
|
|
1066
|
+
if (service.ready && service.ready !== 'tcp') {
|
|
1067
|
+
await untilReadyOrEnded((abort) => awaitReadiness(resolveReadyProbe(service.ready, { host, port, url }), log, cwd, service.id, env, abort), ended, boot, 'the service', log).catch((error) => {
|
|
1068
|
+
if (undeadPids([child.pid]).length)
|
|
1069
|
+
recordWorldStop([log], 'world.service.stop', 'up (startup failed)', 'SIGTERM');
|
|
1070
|
+
try {
|
|
1071
|
+
process.kill(-child.pid, 'SIGTERM');
|
|
1072
|
+
}
|
|
1073
|
+
catch { /* best effort cleanup */ }
|
|
1074
|
+
throw new Error(`Service "${service.id}" failed its readiness probe: ${error instanceof Error ? error.message : String(error)}\nLog: ${log}`);
|
|
1075
|
+
});
|
|
1076
|
+
}
|
|
1077
|
+
return {
|
|
1078
|
+
id: service.id,
|
|
1079
|
+
type: service.type ?? 'twin',
|
|
1080
|
+
command: [service.command, ...args],
|
|
1081
|
+
port,
|
|
1082
|
+
url,
|
|
1083
|
+
...(mounted.version !== undefined ? { resolvedVersion: mounted.version } : {}),
|
|
1084
|
+
...(mounted.protocol !== undefined ? { protocol: mounted.protocol } : {}),
|
|
1085
|
+
pid: child.pid ?? 0,
|
|
1086
|
+
log,
|
|
1087
|
+
env: {
|
|
1088
|
+
...(service.injectEnv ? { [service.injectEnv]: url } : {}),
|
|
1089
|
+
...resolveServiceEnvTemplates(service.injectEnvTemplates, { host, port, url }),
|
|
1090
|
+
...resolveCliRedirect(service.cliRedirect, url),
|
|
1091
|
+
},
|
|
1092
|
+
};
|
|
1093
|
+
}
|
|
1094
|
+
/** What the host is about to mount, read WITHOUT importing the pack (the kernel never imports
|
|
1095
|
+
* packs): the nearest package.json above the module for the version, pack-facts for the
|
|
1096
|
+
* protocol standing the catalog recorded — keyed by the vendor dir the module sits in, else
|
|
1097
|
+
* the service id. */
|
|
1098
|
+
function mountedPackage(modulePath, serviceId) {
|
|
1099
|
+
const out = {};
|
|
1100
|
+
let dir = dirname(modulePath);
|
|
1101
|
+
for (let i = 0; i < 6; i++) {
|
|
1102
|
+
const p = join(dir, 'package.json');
|
|
1103
|
+
if (existsSync(p)) {
|
|
1104
|
+
try {
|
|
1105
|
+
const v = JSON.parse(readFileSync(p, 'utf8')).version;
|
|
1106
|
+
if (typeof v === 'string') {
|
|
1107
|
+
out.version = v;
|
|
1108
|
+
out.packagePath = p;
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
catch { /* unreadable: no version to hold a pin against */ }
|
|
1112
|
+
break;
|
|
1113
|
+
}
|
|
1114
|
+
const up = dirname(dir);
|
|
1115
|
+
if (up === dir)
|
|
1116
|
+
break;
|
|
1117
|
+
dir = up;
|
|
1118
|
+
}
|
|
1119
|
+
const m = /\/packages\/twin\/([^/]+)\//.exec(modulePath) ?? /@volter\/twin-([^/]+)\//.exec(modulePath);
|
|
1120
|
+
const vendor = m?.[1] ?? serviceId;
|
|
1121
|
+
try {
|
|
1122
|
+
const facts = packFacts()[vendor];
|
|
1123
|
+
if (facts?.protocol)
|
|
1124
|
+
out.protocol = facts.protocol;
|
|
1125
|
+
}
|
|
1126
|
+
catch { /* no artifact: the standing is unknown, not refused */ }
|
|
1127
|
+
return out;
|
|
1128
|
+
}
|
|
1129
|
+
/** R17 + R16 at boot, on BOTH paths (the colocated host and the spawned process): the mounted
|
|
1130
|
+
* package must satisfy the pin, its protocol standing must not be refused, and a deprecated one
|
|
1131
|
+
* is said aloud. Returns what to stamp on the instance record. */
|
|
1132
|
+
function lockfileCheck(service, modulePath) {
|
|
1133
|
+
const mounted = mountedPackage(modulePath, service.id);
|
|
1134
|
+
if (service.version !== undefined) {
|
|
1135
|
+
if (mounted.version === undefined)
|
|
1136
|
+
throw new Error(`Twin "${service.id}" pins version ${service.version} but no package.json was found above ${modulePath} — nothing to hold the pin against`);
|
|
1137
|
+
if (!versionSatisfies(mounted.version, service.version))
|
|
1138
|
+
throw new Error(`Twin "${service.id}" pins version ${service.version} and the mounted package is ${mounted.version} (${mounted.packagePath}) — re-pin the world config, or mount the version it names (volter-world outdated lists both)`);
|
|
1139
|
+
}
|
|
1140
|
+
if (mounted.protocol?.standing === 'refused')
|
|
1141
|
+
throw new Error(`Twin "${service.id}" targets protocol major ${mounted.protocol.major}; this platform serves ${PROTOCOL_VERSION} and the previous major only (runtime contract R16)`);
|
|
1142
|
+
if (mounted.protocol?.standing === 'deprecated')
|
|
1143
|
+
process.stderr.write(`volter-world: twin "${service.id}" targets protocol major ${mounted.protocol.major}, one behind this platform's ${PROTOCOL_VERSION} — served for this cycle under deprecation (R16)\n`);
|
|
1144
|
+
return mounted;
|
|
1145
|
+
}
|
|
1146
|
+
/** The module a spawned twin runs from: its first arg under packages/twin/<vendor>/ or an installed
|
|
1147
|
+
* @volter/twin-<vendor>/ package, resolved. */
|
|
1148
|
+
function spawnedTwinModule(service, root) {
|
|
1149
|
+
const first = (service.args ?? []).find((a) => /packages\/twin\/[^/]+\//.test(a) || /@volter\/twin-[^/]+\//.test(a));
|
|
1150
|
+
return first === undefined ? undefined : (first.startsWith('/') ? first : resolve(root, first));
|
|
1151
|
+
}
|
|
1152
|
+
/**
|
|
1153
|
+
* The config as the runtime boots it (docs/concepts/worlds.md#the-config-and-the-running-world): a service
|
|
1154
|
+
* naming a `package` gets its entry and colocate module resolved from the world root NOW, by
|
|
1155
|
+
* node's own walk, and a `$mint` env value becomes a freshly minted throwaway credential for this
|
|
1156
|
+
* boot. Everything downstream sees ordinary command/args/module/env.
|
|
1157
|
+
*/
|
|
1158
|
+
function materializeConfig(config, root) {
|
|
1159
|
+
const services = config.services.map((service) => {
|
|
1160
|
+
if (service.package === undefined)
|
|
1161
|
+
return service;
|
|
1162
|
+
const dir = resolveServicePackage(root, service.package);
|
|
1163
|
+
// the pack's entry and cli come from ITS MANIFEST (src/*.ts in the checkout, dist/src/*.js once
|
|
1164
|
+
// published) and the twin's process is the runtime's own — a world runs under Node as under Bun
|
|
1165
|
+
const colocate = service.colocate === undefined ? undefined : { ...service.colocate, module: service.colocate.module ?? packEntry(dir) };
|
|
1166
|
+
const cli = packCli(dir);
|
|
1167
|
+
if (service.command === undefined && cli === undefined)
|
|
1168
|
+
throw new Error(`${service.package} names no cli in its manifest (bin) and has no src/cli.ts`);
|
|
1169
|
+
return {
|
|
1170
|
+
...service,
|
|
1171
|
+
command: service.command ?? process.execPath,
|
|
1172
|
+
args: service.command === undefined ? [cli, 'serve', ...(service.args ?? [])] : service.args,
|
|
1173
|
+
...(colocate === undefined ? {} : { colocate }),
|
|
1174
|
+
};
|
|
1175
|
+
});
|
|
1176
|
+
// an empty value (a repository's .env.example key with none, in a World initialized before init left them out) is left
|
|
1177
|
+
// unset, as world-machine leaves it: set, even to nothing, it would win over the app's own env files
|
|
1178
|
+
const env = config.env === undefined ? undefined
|
|
1179
|
+
: Object.fromEntries(Object.entries(config.env).filter(([, value]) => value !== '').map(([name, value]) => [name, value === MINT_ENV ? fakeEnvValue(name) : value]));
|
|
1180
|
+
return { ...config, services, ...(env === undefined ? {} : { env }) };
|
|
1181
|
+
}
|
|
1182
|
+
/** `^a.b` admits the same major at minor ≥ b; anything else is an exact pin. */
|
|
1183
|
+
function versionSatisfies(actual, pin) {
|
|
1184
|
+
const parse = (v) => v.replace(/^\^/, '').split('.').map((x) => Number(x));
|
|
1185
|
+
const [am, an] = parse(actual);
|
|
1186
|
+
const [pm, pn] = parse(pin);
|
|
1187
|
+
if (pin.startsWith('^'))
|
|
1188
|
+
return am === pm && (an ?? 0) >= (pn ?? 0);
|
|
1189
|
+
return actual === pin;
|
|
1190
|
+
}
|
|
1191
|
+
/**
|
|
1192
|
+
* Boot every `colocate`-declaring service inside ONE `volter-world-host` child (see src/host.ts —
|
|
1193
|
+
* 'colocated' maps to the host's 'shared' mode, 'worker' to one Worker thread per twin). The
|
|
1194
|
+
* recorded instances are shaped exactly like the spawn path's (port/url/injected env), except they
|
|
1195
|
+
* all share the host child's pid — so pids/instance.json/downWorld need no special casing.
|
|
1196
|
+
*/
|
|
1197
|
+
async function startColocatedServices(config, colocated, isolation, worldEnv, paths, allocatedPorts, boot) {
|
|
1198
|
+
const hostCli = siblingScript(import.meta.url, 'host-cli');
|
|
1199
|
+
const args = [hostCli, '--isolation', isolation === 'colocated' ? 'shared' : 'worker'];
|
|
1200
|
+
const allocated = [];
|
|
1201
|
+
for (const service of colocated) {
|
|
1202
|
+
const port = service.port === undefined || service.port === 'auto' ? await allocatePort(allocatedPorts) : service.port;
|
|
1203
|
+
const serviceDataDir = join(paths.data, service.id);
|
|
1204
|
+
mkdirSync(serviceDataDir, { recursive: true });
|
|
1205
|
+
// Relative module paths resolve from the world root (import() inside the host would
|
|
1206
|
+
// otherwise resolve them against the runtime's own source directory).
|
|
1207
|
+
const declared = service.colocate.module;
|
|
1208
|
+
const module = declared.startsWith('.') ? resolve(paths.root, declared) : declared;
|
|
1209
|
+
// R17 — the lockfile check, BEFORE the host boots: what this module's own package.json says
|
|
1210
|
+
// it is, against the pin the world config carries; and R16 — the protocol standing the
|
|
1211
|
+
// catalog recorded for the package (pack-facts), refused two majors back, warned one back.
|
|
1212
|
+
const mounted = lockfileCheck(service, module);
|
|
1213
|
+
const scenario = service.colocate.scenarioPath;
|
|
1214
|
+
args.push('--spec', [service.id, module, service.colocate.export, String(port), serviceDataDir, ...(scenario ? [`scenario=${scenario.startsWith('/') ? scenario : resolve(paths.root, scenario)}`] : [])].join('|'));
|
|
1215
|
+
allocated.push({ service, port, mounted });
|
|
1216
|
+
}
|
|
1217
|
+
const log = join(paths.logs, 'host.log');
|
|
1218
|
+
// The host is world infrastructure, not an app process — same egress hygiene as controlPlane
|
|
1219
|
+
// services (no injector preload, no ambient proxy): its twins SERVE, they don't call vendors.
|
|
1220
|
+
const env = { ...admittedEnv(config.stripEnv), ...(config.env ?? {}), ...worldEnv };
|
|
1221
|
+
for (const key of CONTROL_PLANE_EGRESS_ENV)
|
|
1222
|
+
delete env[key];
|
|
1223
|
+
env.VOLTER_WORLD_BOOT_ID = boot.bootId;
|
|
1224
|
+
// Under the same exit recorder as a process service, so the host's end (and every twin it holds) is in host.log.
|
|
1225
|
+
const { child, ended } = spawnRecorded('host', process.execPath, args, paths.root, env, log);
|
|
1226
|
+
child.once('error', error => boot.fail(error));
|
|
1227
|
+
child.unref();
|
|
1228
|
+
trackBootingChild(child, boot);
|
|
1229
|
+
const exitState = { value: null };
|
|
1230
|
+
child.on('exit', (code, signal) => {
|
|
1231
|
+
exitState.value = { code, signal };
|
|
1232
|
+
});
|
|
1233
|
+
const command = [process.execPath, ...args];
|
|
1234
|
+
for (const { service, port } of allocated) {
|
|
1235
|
+
const bindTimeoutMs = typeof service.ready === 'object' && service.ready.timeoutMs !== undefined ? Math.max(service.ready.timeoutMs, 15_000) : 15_000;
|
|
1236
|
+
await untilReadyOrEnded((abort) => waitForTcp(port, bindTimeoutMs, abort), ended, boot, 'the co-located host', log).catch((error) => {
|
|
1237
|
+
if (undeadPids([child.pid]).length)
|
|
1238
|
+
recordWorldStop([log], 'world.service.stop', 'up (startup failed)', 'SIGTERM');
|
|
1239
|
+
try {
|
|
1240
|
+
process.kill(-child.pid, 'SIGTERM');
|
|
1241
|
+
}
|
|
1242
|
+
catch { /* best effort cleanup */ }
|
|
1243
|
+
throw new Error(`Co-located twin "${service.id}" failed to start (host isolation=${isolation}): ${error instanceof Error ? error.message : String(error)}\nLog: ${log}`);
|
|
1244
|
+
});
|
|
1245
|
+
}
|
|
1246
|
+
for (const [index, { service, port }] of allocated.entries()) {
|
|
1247
|
+
// A host holding one twin settles for that one port, so its twin may be confirmed early like a lone service's.
|
|
1248
|
+
await confirmOwnPort(port, boot, index === 0 ? ended : undefined, true, `Co-located twin "${service.id}"`, log, allocated.length === 1).catch((error) => {
|
|
1249
|
+
if (undeadPids([child.pid]).length)
|
|
1250
|
+
recordWorldStop([log], 'world.service.stop', 'up (port not its own)', 'SIGTERM');
|
|
1251
|
+
try {
|
|
1252
|
+
process.kill(-child.pid, 'SIGTERM');
|
|
1253
|
+
}
|
|
1254
|
+
catch { /* best effort cleanup */ }
|
|
1255
|
+
throw error;
|
|
1256
|
+
});
|
|
1257
|
+
}
|
|
1258
|
+
if (exitState.value !== null) {
|
|
1259
|
+
throw new Error(`The co-located host ended during startup even though its ports answer, so other/stale processes probably hold them: the host ${endedAccount(log)}\n`
|
|
1260
|
+
+ `Stop leftover services (e.g. \`volter-world down\`) and retry.\nLog: ${log}`);
|
|
1261
|
+
}
|
|
1262
|
+
const host = '127.0.0.1';
|
|
1263
|
+
return allocated.map(({ service, port, mounted }) => {
|
|
1264
|
+
const url = `http://${host}:${port}`;
|
|
1265
|
+
return {
|
|
1266
|
+
id: service.id,
|
|
1267
|
+
type: service.type ?? 'twin',
|
|
1268
|
+
command,
|
|
1269
|
+
port,
|
|
1270
|
+
url,
|
|
1271
|
+
...(mounted.version !== undefined ? { resolvedVersion: mounted.version } : {}),
|
|
1272
|
+
...(mounted.protocol !== undefined ? { protocol: mounted.protocol } : {}),
|
|
1273
|
+
pid: child.pid ?? 0,
|
|
1274
|
+
log,
|
|
1275
|
+
env: {
|
|
1276
|
+
...(service.injectEnv ? { [service.injectEnv]: url } : {}),
|
|
1277
|
+
...resolveServiceEnvTemplates(service.injectEnvTemplates, { host, port, url }),
|
|
1278
|
+
...resolveCliRedirect(service.cliRedirect, url),
|
|
1279
|
+
},
|
|
1280
|
+
};
|
|
1281
|
+
});
|
|
1282
|
+
}
|
|
1283
|
+
export async function upWorld(configId, options = {}, abort) {
|
|
1284
|
+
abort?.throwIfAborted();
|
|
1285
|
+
const root = resolve(options.root ?? process.cwd());
|
|
1286
|
+
const found = loadWorldConfig(configId, root);
|
|
1287
|
+
const loaded = { path: found.path, config: materializeConfig(found.config, root) };
|
|
1288
|
+
const name = options.name ?? loaded.config.id;
|
|
1289
|
+
const mode = options.mode ?? 'local';
|
|
1290
|
+
assertSafeWorldName(name);
|
|
1291
|
+
// R18 — the birth stamp: this platform reads its own protocol major's state and the previous
|
|
1292
|
+
// one's by construction, and nothing else. A world two majors back, or born under a newer
|
|
1293
|
+
// platform, is refused by name; its state travels through the snapshot doors, never an
|
|
1294
|
+
// in-place rewrite. An unstamped world (born before the stamp existed) is read as current.
|
|
1295
|
+
{
|
|
1296
|
+
const born = loaded.config.catalog?.protocol;
|
|
1297
|
+
if (born !== undefined) {
|
|
1298
|
+
const major = Number(born);
|
|
1299
|
+
if (major > PROTOCOL_MAJOR)
|
|
1300
|
+
throw new Error(`World "${name}" was born under platform protocol major ${major}; this platform is ${PROTOCOL_VERSION} and cannot read a newer world's state (runtime contract R18) — run it on the platform it was born on, or export it there and re-init here`);
|
|
1301
|
+
if (major < PROTOCOL_MAJOR - 1)
|
|
1302
|
+
throw new Error(`World "${name}" was born under platform protocol major ${major}; this platform (${PROTOCOL_VERSION}) reads one major back and no further (runtime contract R18) — export its state through the snapshot door, re-init, import`);
|
|
1303
|
+
if (major === PROTOCOL_MAJOR - 1)
|
|
1304
|
+
process.stderr.write(`volter-world: world "${name}" was born under protocol major ${major}, one behind this platform's ${PROTOCOL_VERSION}; read by construction this cycle — re-init to re-stamp it (R18)\n`);
|
|
1305
|
+
}
|
|
1306
|
+
}
|
|
1307
|
+
assertMode(mode);
|
|
1308
|
+
let isolation = options.isolation ?? loaded.config.isolation ?? 'process';
|
|
1309
|
+
// TWIN-67: schema.ts's WorldIsolation contract says process isolation ("one OS process per
|
|
1310
|
+
// service") is "the only choice for share/sealed/hosted worlds" — colocating twins in one host
|
|
1311
|
+
// process (or worker threads, which still share that process) under `share`/`sealed` puts a
|
|
1312
|
+
// semi-untrusted session on colocated twins sharing one host process, exactly what the
|
|
1313
|
+
// contract forbids. Enforce it here instead of leaving it a comment nothing checks. Checked
|
|
1314
|
+
// before the file-lock claim / any dir creation below, so a rejected combination leaves nothing
|
|
1315
|
+
// to clean up. A CONFIG-sourced isolation is a local-mode preference (init emits
|
|
1316
|
+
// `isolation: 'colocated'` so a local world costs one twin process — R2a), so share/sealed
|
|
1317
|
+
// quietly boot it per-process, their contractual shape; only an EXPLICIT --isolation flag
|
|
1318
|
+
// contradicting the mode is an error, because the caller asked for two things at once.
|
|
1319
|
+
if (mode !== 'local' && isolation !== 'process') {
|
|
1320
|
+
if (options.isolation === undefined) {
|
|
1321
|
+
isolation = 'process';
|
|
1322
|
+
}
|
|
1323
|
+
else {
|
|
1324
|
+
throw new Error(`world "${name}": mode "${mode}" requires isolation "process" (schema.ts: process isolation is the only choice for share/sealed/hosted worlds), got --isolation "${isolation}". Use --isolation process (or omit --isolation) with --mode ${mode}.`);
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1327
|
+
const instance = instanceDir(root, name);
|
|
1328
|
+
const logs = join(instance, 'logs');
|
|
1329
|
+
const data = join(instance, 'data');
|
|
1330
|
+
const pidsFile = join(instance, 'pids');
|
|
1331
|
+
const envFile = resolve(root, options.envFile ?? join(instance, 'world.env'));
|
|
1332
|
+
const instanceFile = join(instance, 'instance.json');
|
|
1333
|
+
if (loaded.config.resources)
|
|
1334
|
+
process.stderr.write(`World "${name}": config resources is deprecated; declarations are not reservations or enforced limits. Configure limits in the execution backend.\n`);
|
|
1335
|
+
checkedOwner(options.owner);
|
|
1336
|
+
let resourcesClaimed = false;
|
|
1337
|
+
let cleanupUncertain = false;
|
|
1338
|
+
const createdAt = new Date().toISOString();
|
|
1339
|
+
// Claim the instance dir under the kernel's cross-process file lock (TWIN-36). Without it,
|
|
1340
|
+
// two concurrent `upWorld` calls on one name can both observe "no live pids", both rmSync the
|
|
1341
|
+
// dir, and one clobbers the other's freshly-spawned services (orphaned processes + a corrupt
|
|
1342
|
+
// instance dir). The lock guards only the SYNCHRONOUS claim below — check, wipe, re-create,
|
|
1343
|
+
// write `booting.json` — so it is held for milliseconds and never spans the async service boot
|
|
1344
|
+
// (which can exceed withFileLock's 10s contender timeout); the booting marker written inside
|
|
1345
|
+
// the lock is what keeps contenders out for the boot's whole duration, and a contender that
|
|
1346
|
+
// finds a live booter fails FAST with a clear error instead of waiting on an open-ended boot.
|
|
1347
|
+
// The claim fn has no awaits, so two in-process callers cannot interleave inside it either;
|
|
1348
|
+
// the lockfile adds the cross-process exclusion, and an in-process lock holder always releases
|
|
1349
|
+
// before its first await, so withFileLock's synchronous contender wait can never deadlock the
|
|
1350
|
+
// event loop against a holder in the same process.
|
|
1351
|
+
try {
|
|
1352
|
+
withFileLock(instanceLockFile(root, name), () => {
|
|
1353
|
+
assertProxyOwnershipResolved(root, name);
|
|
1354
|
+
if (liveBootingClaim(stoppingFile(root, name)))
|
|
1355
|
+
throw new Error(`World "${name}" is being stopped; retry after teardown finishes`);
|
|
1356
|
+
const attached = worldConsumers(root, name);
|
|
1357
|
+
if (attached.warnings.length || attached.consumers.some((c) => c.state !== 'exited')) {
|
|
1358
|
+
throw new Error(`World "${name}" has active or uncertain attached consumers; cannot replace its instance`);
|
|
1359
|
+
}
|
|
1360
|
+
const live = livePids(readPids(pidsFile));
|
|
1361
|
+
if (live.length > 0) {
|
|
1362
|
+
throw new Error(`World "${name}" is already running (${live.join(', ')}). Run: volter-world down ${name}`);
|
|
1363
|
+
}
|
|
1364
|
+
const booting = liveBootingClaim(bootingFile(root, name));
|
|
1365
|
+
if (booting) {
|
|
1366
|
+
throw new Error(`World "${name}" is being booted by another process (pid ${booting.pid}, since ${booting.at}). Retry after it finishes (a dead booter's claim is reclaimed automatically).`);
|
|
1367
|
+
}
|
|
1368
|
+
if (!options.keepState)
|
|
1369
|
+
assertWorldStateRemovable(root, instance);
|
|
1370
|
+
// A legacy record can have been removed by an older reader while its external effects
|
|
1371
|
+
// remain. Require a receipt for the exact prior instance, not merely absent claims/PIDs.
|
|
1372
|
+
if (existsSync(instanceFile)) {
|
|
1373
|
+
const priorRaw = readFileSync(instanceFile, 'utf8');
|
|
1374
|
+
const prior = JSON.parse(priorRaw);
|
|
1375
|
+
const receiptPath = join(instance, 'stopped.json');
|
|
1376
|
+
const receipt = existsSync(receiptPath) ? JSON.parse(readFileSync(receiptPath, 'utf8')) : undefined;
|
|
1377
|
+
if (prior.name !== name || resolve(prior.root) !== root || !receipt
|
|
1378
|
+
|| receipt.instanceHash !== createHash('sha256').update(priorRaw).digest('hex')
|
|
1379
|
+
|| !Number.isFinite(Date.parse(receipt.stoppedAt))) {
|
|
1380
|
+
throw new Error(`World "${name}" has no verified teardown receipt for its prior instance; use its owning runtime to complete cleanup before replacement`);
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
assertLifecycleStartable(root, name);
|
|
1384
|
+
beginWorldLifecycle(root, name, createdAt, options.owner);
|
|
1385
|
+
resourcesClaimed = true;
|
|
1386
|
+
retainWorldLifecycleForCleanup(root, name);
|
|
1387
|
+
// The last run's logs survive this boot as logs/previous (one generation): the record of a crash must outlive the
|
|
1388
|
+
// restart that usually follows it. Moved aside before the wipe and back in after it.
|
|
1389
|
+
const previousLogs = join(dirname(instance), `.${name}.previous-logs`); // dot-prefixed: bookkeeping, never listed as a World
|
|
1390
|
+
if (existsSync(previousLogs) && !existsSync(logs)) {
|
|
1391
|
+
mkdirSync(instance, { recursive: true });
|
|
1392
|
+
renameSync(previousLogs, logs);
|
|
1393
|
+
} // an earlier failed claim's, back first
|
|
1394
|
+
rmSync(previousLogs, { recursive: true, force: true });
|
|
1395
|
+
if (existsSync(logs)) {
|
|
1396
|
+
rmSync(join(logs, 'previous'), { recursive: true, force: true });
|
|
1397
|
+
renameSync(logs, previousLogs);
|
|
1398
|
+
}
|
|
1399
|
+
try {
|
|
1400
|
+
if (options.keepState) {
|
|
1401
|
+
const KEEP = ['data', 'marks', 'changesets', 'episodes'];
|
|
1402
|
+
for (const entry of existsSync(instance) ? readdirSync(instance) : [])
|
|
1403
|
+
if (!KEEP.includes(entry))
|
|
1404
|
+
rmSync(join(instance, entry), { recursive: true, force: true });
|
|
1405
|
+
}
|
|
1406
|
+
else
|
|
1407
|
+
removeWorldState(root, instance, () => rmSync(instance, { recursive: true, force: true }));
|
|
1408
|
+
}
|
|
1409
|
+
catch (error) {
|
|
1410
|
+
// The claim failed; the logs go back where they were rather than wait aside for the next boot to discard.
|
|
1411
|
+
try {
|
|
1412
|
+
if (existsSync(previousLogs) && !existsSync(logs)) {
|
|
1413
|
+
mkdirSync(instance, { recursive: true });
|
|
1414
|
+
renameSync(previousLogs, logs);
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
catch { /* left aside; restored by the next claim */ }
|
|
1418
|
+
throw error;
|
|
1419
|
+
}
|
|
1420
|
+
mkdirSync(logs, { mode: 0o700, recursive: true });
|
|
1421
|
+
if (existsSync(previousLogs))
|
|
1422
|
+
renameSync(previousLogs, join(logs, 'previous'));
|
|
1423
|
+
mkdirSync(data, { mode: 0o700, recursive: true });
|
|
1424
|
+
chmodSync(instance, 0o700);
|
|
1425
|
+
chmodSync(logs, 0o700);
|
|
1426
|
+
chmodSync(data, 0o700);
|
|
1427
|
+
writeFileSync(bootingFile(root, name), `${JSON.stringify({ pid: process.pid, at: new Date().toISOString() })}\n`, { mode: 0o600 });
|
|
1428
|
+
});
|
|
1429
|
+
}
|
|
1430
|
+
catch (error) {
|
|
1431
|
+
if (resourcesClaimed)
|
|
1432
|
+
releaseWorldLifecycle(root, name);
|
|
1433
|
+
throw error;
|
|
1434
|
+
}
|
|
1435
|
+
// From here on this call owns the claim; the finally below releases it however the boot
|
|
1436
|
+
// settles. On success the recorded live pids (written before returning) take over as the
|
|
1437
|
+
// "already running" guard; on failure the world never came up, so the next upWorld must be
|
|
1438
|
+
// allowed to claim. Removing the marker is safe — only THIS call's marker can exist at this
|
|
1439
|
+
// point (a foreign live claim made the claim section above throw before this try).
|
|
1440
|
+
try {
|
|
1441
|
+
const services = {};
|
|
1442
|
+
const inheritedEnv = admittedEnv(loaded.config.stripEnv);
|
|
1443
|
+
const env = {
|
|
1444
|
+
...(loaded.config.env ?? {}),
|
|
1445
|
+
VOLTER_WORLD_STRIP_ENV: JSON.stringify(loaded.config.stripEnv ?? []),
|
|
1446
|
+
// the names this World sets itself, apart from what the caller's shell passes through (world-core's worldEnvValue)
|
|
1447
|
+
[WORLD_ENV_NAMES_ENV]: JSON.stringify(Object.keys(loaded.config.env ?? {}).sort()),
|
|
1448
|
+
VOLTER_WORLD_NAME: name,
|
|
1449
|
+
// the world ref the dormant attach entry (`import '@volter/world-core/attach'`) arms the injector
|
|
1450
|
+
// on — what a Bun process, which ignores NODE_OPTIONS, reads to reach the twins inside `run`
|
|
1451
|
+
VOLTER_WORLD: name,
|
|
1452
|
+
...(loaded.config.network ? { [WORLD_NETWORK_POLICY_ENV]: JSON.stringify(createWorldNetworkPolicy(name, mode === 'sealed' ? { egress: [] } : loaded.config.network)) } : {}),
|
|
1453
|
+
VOLTER_WORLD_MODE: mode,
|
|
1454
|
+
VOLTER_WORLD_CONFIG: loaded.path,
|
|
1455
|
+
VOLTER_WORLD_INSTANCE: instanceFile,
|
|
1456
|
+
VOLTER_WORLD_DATA: data,
|
|
1457
|
+
// the live env file this `up` writes: what a service that runs apps elsewhere (world-machine's guest) carries there
|
|
1458
|
+
VOLTER_WORLD_ENV_FILE: envFile,
|
|
1459
|
+
// THE WORLD CLOCK: every service stamps from this file via the kernel's worldNow();
|
|
1460
|
+
// absent until `volter-world clock <name> set/advance` writes it (→ real time).
|
|
1461
|
+
TWIN_WORLD_CLOCK_FILE: clockFile(root, name),
|
|
1462
|
+
// VOLTER_WORLD_SEALED is intent-only; VOLTER_TWIN_STRICT_EGRESS is what the ENFORCEMENT
|
|
1463
|
+
// points actually gate on (redirect-proxy.ts's CONNECT/plain-proxy handlers and
|
|
1464
|
+
// control-plane/inject.cjs's patched http/https/fetch) — both block any untwinned
|
|
1465
|
+
// non-loopback host once this is set. Without it "sealed" only records intent and every
|
|
1466
|
+
// untwinned host is blind-tunneled to the real internet (TWIN-60).
|
|
1467
|
+
...(mode === 'sealed' ? { VOLTER_WORLD_SEALED: '1', VOLTER_TWIN_STRICT_EGRESS: '1' } : {}),
|
|
1468
|
+
NODE_OPTIONS: `--require ${injectPreloadSpecifier()}${inheritedEnv.NODE_OPTIONS ? ` ${inheritedEnv.NODE_OPTIONS}` : ''}`,
|
|
1469
|
+
};
|
|
1470
|
+
env.NO_PROXY = `127.0.0.1,localhost${env.NO_PROXY || inheritedEnv.NO_PROXY ? `,${env.NO_PROXY ?? inheritedEnv.NO_PROXY}` : ''}`;
|
|
1471
|
+
env.no_proxy = `127.0.0.1,localhost${env.no_proxy || inheritedEnv.no_proxy ? `,${env.no_proxy ?? inheritedEnv.no_proxy}` : ''}`;
|
|
1472
|
+
writeProxyEnv(root, name, env);
|
|
1473
|
+
const worldInstance = {
|
|
1474
|
+
name, config: loaded.config.id, configPath: loaded.path, root, createdAt, mode,
|
|
1475
|
+
dirs: { instance, logs, data }, services, env, envFile, pidsFile,
|
|
1476
|
+
...(loaded.config.selection ? { selection: loaded.config.selection } : {}),
|
|
1477
|
+
...(loaded.config.actors ? { actors: loaded.config.actors } : {}),
|
|
1478
|
+
...(loaded.config.fixtures ? { fixtures: loaded.config.fixtures } : {}),
|
|
1479
|
+
lifecycle: { owner: options.owner, log: lifecycleLogPath(root, name) },
|
|
1480
|
+
};
|
|
1481
|
+
saveWorldInstance(worldInstance);
|
|
1482
|
+
const cancelled = new AbortController();
|
|
1483
|
+
const onBootSignal = (signal) => cancelled.abort(new Error(`World "${name}": boot interrupted by ${signal}`));
|
|
1484
|
+
const onOwnerEnd = () => cancelled.abort(new Error(typeof abort?.reason === 'string' && abort.reason.startsWith('SIG')
|
|
1485
|
+
? `World "${name}": boot interrupted by ${abort.reason}` : `World "${name}": caller ended during boot`));
|
|
1486
|
+
process.on('SIGTERM', onBootSignal);
|
|
1487
|
+
process.on('SIGINT', onBootSignal);
|
|
1488
|
+
abort?.addEventListener('abort', onOwnerEnd, { once: true });
|
|
1489
|
+
if (abort?.aborted)
|
|
1490
|
+
onOwnerEnd();
|
|
1491
|
+
const detachBootSignals = () => {
|
|
1492
|
+
offSignal('SIGTERM', onBootSignal);
|
|
1493
|
+
offSignal('SIGINT', onBootSignal);
|
|
1494
|
+
abort?.removeEventListener('abort', onOwnerEnd);
|
|
1495
|
+
};
|
|
1496
|
+
const boot = {
|
|
1497
|
+
bootId: randomUUID(), signal: cancelled.signal, fail: error => cancelled.abort(error), pids: new Set(),
|
|
1498
|
+
external: service => { services[service.id] = service; saveWorldInstance(worldInstance); },
|
|
1499
|
+
recordPids: () => writeOwnedPids(root, name, [...boot.pids], [...boot.pids]),
|
|
1500
|
+
};
|
|
1501
|
+
const recordPidsSoFar = boot.recordPids;
|
|
1502
|
+
try {
|
|
1503
|
+
// Ownership survives caller/process loss without an inert capacity-holder process.
|
|
1504
|
+
retainWorldLifecycleForCleanup(root, name);
|
|
1505
|
+
// Protect later explicit ports before selecting any automatic port, across both paths.
|
|
1506
|
+
const allocatedPorts = new Set(loaded.config.services.flatMap((service) => typeof service.port === 'number' ? [service.port] : []));
|
|
1507
|
+
// Non-'process' isolation: boot every colocate-declaring service inside ONE host child
|
|
1508
|
+
// first; anything without `colocate` (and externals) still goes through the spawn path.
|
|
1509
|
+
if (isolation !== 'process') {
|
|
1510
|
+
const colocated = loaded.config.services.filter((service) => service.colocate && service.type !== 'external');
|
|
1511
|
+
for (const started of await startColocatedServices(loaded.config, colocated, isolation, env, { root, instance, logs, data }, allocatedPorts, boot)) {
|
|
1512
|
+
services[started.id] = started;
|
|
1513
|
+
Object.assign(env, started.env);
|
|
1514
|
+
}
|
|
1515
|
+
recordPidsSoFar();
|
|
1516
|
+
writeProxyEnv(root, name, env);
|
|
1517
|
+
}
|
|
1518
|
+
let proxyForServices = null;
|
|
1519
|
+
// TWIN-64: attempt the ambient proxy once activeVendorMap(env) has at least one twin — tried
|
|
1520
|
+
// again after each service in case ITS start is what makes the map non-empty. A sealed world
|
|
1521
|
+
// must never come up silently unsealed: if openssl itself is unavailable (no proxy is even
|
|
1522
|
+
// possible) or the proxy daemon fails/times out, refuse rather than let unmodified CLIs bypass
|
|
1523
|
+
// the twins undetected.
|
|
1524
|
+
const attemptProxy = () => {
|
|
1525
|
+
if (proxyForServices || (Object.keys(activeVendorMap(env)).length === 0 && env[WORLD_NETWORK_POLICY_ENV] === undefined))
|
|
1526
|
+
return;
|
|
1527
|
+
if (!opensslAvailable()) {
|
|
1528
|
+
if (mode === 'sealed' || loaded.config.network !== undefined) {
|
|
1529
|
+
throw new Error(`world "${name}" requires its network policy proxy, but openssl is unavailable; refusing to start. Install openssl.`);
|
|
1530
|
+
}
|
|
1531
|
+
return;
|
|
1532
|
+
}
|
|
1533
|
+
proxyForServices = ensureWorldProxyFromEnv(name, root, env, boot);
|
|
1534
|
+
if (proxyForServices?.url) {
|
|
1535
|
+
Object.assign(env, proxyForServices.env);
|
|
1536
|
+
}
|
|
1537
|
+
else if (mode === 'sealed' || loaded.config.network !== undefined) {
|
|
1538
|
+
// The ambient redirect proxy timed out (see the loud WARN already emitted by
|
|
1539
|
+
// ensureWorldProxyFromEnv). Composes with the existing catch below: stops the proxy
|
|
1540
|
+
// daemon, kills started services, rethrows.
|
|
1541
|
+
throw new Error(`world "${name}" requires its network policy proxy, but it failed to start; refusing to start`);
|
|
1542
|
+
}
|
|
1543
|
+
};
|
|
1544
|
+
for (const service of loaded.config.services) {
|
|
1545
|
+
if (services[service.id])
|
|
1546
|
+
continue; // already up co-located
|
|
1547
|
+
attemptProxy();
|
|
1548
|
+
boot.signal.throwIfAborted();
|
|
1549
|
+
const started = await startService(loaded.config, service, env, { root, instance, logs, data }, allocatedPorts, boot);
|
|
1550
|
+
services[started.id] = started;
|
|
1551
|
+
recordPidsSoFar();
|
|
1552
|
+
Object.assign(env, started.env);
|
|
1553
|
+
writeProxyEnv(root, name, env);
|
|
1554
|
+
}
|
|
1555
|
+
detachBootSignals();
|
|
1556
|
+
// Re-evaluate once more after the LAST service: a single/last-twin world's OWN vendor twin
|
|
1557
|
+
// only lands in `env` once IT has started, so a check that only runs "before starting the
|
|
1558
|
+
// next service" never fires when there is no next service (TWIN-64) — the single-twin sealed
|
|
1559
|
+
// config test below exercises exactly this.
|
|
1560
|
+
attemptProxy();
|
|
1561
|
+
// Inert-injectEnv check (LibreChat blind-adoption finding): an injectEnv like AWS_TWIN_URL
|
|
1562
|
+
// lands in the world env, LOOKS wired, and does nothing — the injector reads no such vendor.
|
|
1563
|
+
// Warn loudly at `up`, when the operator is watching, instead of letting the disagreement
|
|
1564
|
+
// surface later as silently-real vendor traffic.
|
|
1565
|
+
for (const warning of inertInjectEnvWarnings(loaded.config.services)) {
|
|
1566
|
+
process.stderr.write(`${warning}\n`);
|
|
1567
|
+
}
|
|
1568
|
+
mkdirSync(dirname(envFile), { recursive: true });
|
|
1569
|
+
// Config env and discovered credentials: a file this writes is owner-only; a file the caller already made keeps
|
|
1570
|
+
// the mode the caller chose (a shared env file may be read under another uid).
|
|
1571
|
+
writeFileSync(envFile, envFileContents(env), { mode: 0o600 });
|
|
1572
|
+
writePidsFromInstance(worldInstance, Object.values(worldInstance.services).map((service) => service.pid).filter((pid) => pid > 0));
|
|
1573
|
+
saveWorldInstance(worldInstance);
|
|
1574
|
+
const servicePids = [...new Set(Object.values(services).map((service) => service.pid).filter((pid) => pid > 0))];
|
|
1575
|
+
handoffWorldLifecycle(root, name, servicePids);
|
|
1576
|
+
if (mode === 'share') {
|
|
1577
|
+
try {
|
|
1578
|
+
return await shareWorldServices(name, { root, ...(options.share ?? {}) });
|
|
1579
|
+
}
|
|
1580
|
+
catch (error) {
|
|
1581
|
+
await downWorld(name, root);
|
|
1582
|
+
throw error;
|
|
1583
|
+
}
|
|
1584
|
+
}
|
|
1585
|
+
return worldInstance;
|
|
1586
|
+
}
|
|
1587
|
+
catch (error) {
|
|
1588
|
+
cleanupUncertain = true;
|
|
1589
|
+
detachBootSignals();
|
|
1590
|
+
// Rollback never lets its own failure replace the boot's: each step's error is added to the original.
|
|
1591
|
+
const rollbackErrors = [];
|
|
1592
|
+
try {
|
|
1593
|
+
stopWorldProxy(root, name);
|
|
1594
|
+
}
|
|
1595
|
+
catch (proxyError) {
|
|
1596
|
+
rollbackErrors.push(`proxy stop failed: ${proxyError instanceof Error ? proxyError.message : String(proxyError)}`);
|
|
1597
|
+
}
|
|
1598
|
+
const pids = [...new Set([...Object.values(services).map((service) => service.pid).filter(Boolean), ...boot.pids])];
|
|
1599
|
+
const rollbackAlive = undeadPids(Object.values(services).map((service) => service.pid).filter(Boolean));
|
|
1600
|
+
const rollbackLogs = Object.values(services).filter((service) => service.type !== 'external' && service.log && rollbackAlive.includes(service.pid)).map((service) => service.log);
|
|
1601
|
+
recordWorldStop(rollbackLogs, 'world.service.stop', 'boot rollback', 'SIGTERM');
|
|
1602
|
+
signalPids(pids, 'SIGTERM');
|
|
1603
|
+
// Stop any self-managed externals that DID come up, so a failed `up` never leaves a
|
|
1604
|
+
// half-running external stack behind. Best-effort: the original error is what we throw —
|
|
1605
|
+
// but a `down` that ITSELF fails must not be silent (the operator would have no hint that
|
|
1606
|
+
// an external stack is still half-running), so those failures are appended to its message.
|
|
1607
|
+
for (const service of Object.values(services)) {
|
|
1608
|
+
if (service.type === 'external' && service.external) {
|
|
1609
|
+
// Same env shape as up/status (startExternalService) and downWorld's teardown:
|
|
1610
|
+
// Admitted caller variables keep allowed tools findable without reintroducing
|
|
1611
|
+
// credentials that were excluded when this World started.
|
|
1612
|
+
const downEnv = { ...admittedEnv(loaded.config.stripEnv), ...env, ...(service.external.discoveredEnv ?? {}) };
|
|
1613
|
+
try {
|
|
1614
|
+
await runExternalCommand(service.external.down, service.external.cwd, service.id, 'down', downEnv);
|
|
1615
|
+
}
|
|
1616
|
+
catch (downError) {
|
|
1617
|
+
rollbackErrors.push(downError instanceof Error ? downError.message : String(downError));
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
}
|
|
1621
|
+
// Same SIGTERM→SIGKILL contract as downWorld: a service that ignores SIGTERM must not
|
|
1622
|
+
// outlive a failed boot either. The poll exits as soon as everything is dead, so the
|
|
1623
|
+
// common case (services honor SIGTERM) adds no delay to the failure path.
|
|
1624
|
+
const rollbackKilled = await killSurvivorsAfterGrace(pids, DOWN_GRACE_MS_DEFAULT);
|
|
1625
|
+
recordWorldStop(Object.values(services).filter((service) => rollbackKilled.includes(service.pid) && service.log).map((service) => service.log), 'world.service.killed', 'boot rollback', 'SIGKILL');
|
|
1626
|
+
const surviving = undeadPids(pids);
|
|
1627
|
+
if (surviving.length)
|
|
1628
|
+
rollbackErrors.push('owned processes remain alive or uncertain');
|
|
1629
|
+
if (error instanceof CommandRetirementError)
|
|
1630
|
+
rollbackErrors.push(error.message);
|
|
1631
|
+
if (rollbackErrors.length > 0) {
|
|
1632
|
+
cleanupUncertain = true;
|
|
1633
|
+
try {
|
|
1634
|
+
retainWorldLifecycleForCleanup(root, name);
|
|
1635
|
+
saveWorldInstance(worldInstance);
|
|
1636
|
+
}
|
|
1637
|
+
catch (bookkeepingError) {
|
|
1638
|
+
rollbackErrors.push(`rollback bookkeeping failed: ${bookkeepingError instanceof Error ? bookkeepingError.message : String(bookkeepingError)}`);
|
|
1639
|
+
}
|
|
1640
|
+
const suffix = `\n(rollback incomplete — processes or an external stack may still be running: ${rollbackErrors.join('; ')})`;
|
|
1641
|
+
// Mutate rather than re-wrap: callers match on the ORIGINAL message (which must keep
|
|
1642
|
+
// leading) and the original stack/type stay intact.
|
|
1643
|
+
if (error instanceof Error)
|
|
1644
|
+
error.message += suffix;
|
|
1645
|
+
else
|
|
1646
|
+
throw new Error(`${String(error)}${suffix}`);
|
|
1647
|
+
}
|
|
1648
|
+
if (rollbackErrors.length === 0) {
|
|
1649
|
+
// Publication failure must retain ownership. Prune requires both a receipt and release.
|
|
1650
|
+
cleanupUncertain = true;
|
|
1651
|
+
try {
|
|
1652
|
+
rmSync(pidsFile, { force: true });
|
|
1653
|
+
const raw = readFileSync(instanceFile, 'utf8');
|
|
1654
|
+
writeTextAtomic(join(instance, 'stopped.json'), JSON.stringify({ instanceHash: createHash('sha256').update(raw).digest('hex'), stoppedAt: new Date().toISOString() }));
|
|
1655
|
+
releaseWorldLifecycle(root, name);
|
|
1656
|
+
cleanupUncertain = false;
|
|
1657
|
+
}
|
|
1658
|
+
catch (receiptError) {
|
|
1659
|
+
const note = `\n(rollback stopped every process but could not publish its teardown receipt: ${receiptError instanceof Error ? receiptError.message : String(receiptError)}; lifecycle retained)`;
|
|
1660
|
+
if (error instanceof Error)
|
|
1661
|
+
error.message += note;
|
|
1662
|
+
else
|
|
1663
|
+
throw new Error(`${String(error)}${note}`);
|
|
1664
|
+
}
|
|
1665
|
+
}
|
|
1666
|
+
throw error;
|
|
1667
|
+
}
|
|
1668
|
+
}
|
|
1669
|
+
catch (error) {
|
|
1670
|
+
if (resourcesClaimed) {
|
|
1671
|
+
try {
|
|
1672
|
+
recordWorldLifecycleEvent(root, name, `boot failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1673
|
+
}
|
|
1674
|
+
catch (diagnosticError) {
|
|
1675
|
+
process.stderr.write(`World "${name}": could not save boot failure: ${String(diagnosticError)}; original failure: ${String(error)}\n`);
|
|
1676
|
+
}
|
|
1677
|
+
if (!cleanupUncertain)
|
|
1678
|
+
releaseWorldLifecycle(root, name);
|
|
1679
|
+
}
|
|
1680
|
+
throw error;
|
|
1681
|
+
}
|
|
1682
|
+
finally {
|
|
1683
|
+
rmSync(bootingFile(root, name), { force: true });
|
|
1684
|
+
}
|
|
1685
|
+
}
|
|
1686
|
+
async function teardownExternalServices(instance) {
|
|
1687
|
+
const stopped = [];
|
|
1688
|
+
const errors = [];
|
|
1689
|
+
for (const service of Object.values(instance.services)) {
|
|
1690
|
+
if (service.type !== 'external')
|
|
1691
|
+
continue;
|
|
1692
|
+
if (!service.external || !Array.isArray(service.external.down) || !service.external.down.length
|
|
1693
|
+
|| service.external.down.some((arg) => typeof arg !== 'string') || typeof service.external.cwd !== 'string') {
|
|
1694
|
+
errors.push(`External service "${service.id}": missing or invalid teardown metadata; lifecycle retained`);
|
|
1695
|
+
continue;
|
|
1696
|
+
}
|
|
1697
|
+
const downEnv = { ...admittedEnv(stripEnvOf(instance.env ?? {})), ...(instance.env ?? {}), ...(service.external.discoveredEnv ?? {}) };
|
|
1698
|
+
try {
|
|
1699
|
+
await runExternalCommand(service.external.down, service.external.cwd, service.id, 'down', downEnv);
|
|
1700
|
+
stopped.push(service.id);
|
|
1701
|
+
}
|
|
1702
|
+
catch (error) {
|
|
1703
|
+
errors.push(error instanceof Error ? error.message : String(error));
|
|
1704
|
+
}
|
|
1705
|
+
}
|
|
1706
|
+
return { stopped, errors };
|
|
1707
|
+
}
|
|
1708
|
+
/** SIGTERM contract (documented in docs/concepts/worlds.md § Teardown): services get SIGTERM to their process
|
|
1709
|
+
* GROUP, then `downWorld` waits up to this grace for them to exit before SIGKILLing survivors.
|
|
1710
|
+
* Overridable per call via `options.graceMs` (the CLI exposes `--grace-ms`). */
|
|
1711
|
+
const DOWN_GRACE_MS_DEFAULT = 5_000;
|
|
1712
|
+
/** Signal each pid's process GROUP (services are spawned `detached`, so each pid is a group
|
|
1713
|
+
* leader), falling back to the single pid when the group signal fails. Returns the pids that
|
|
1714
|
+
* accepted the signal. */
|
|
1715
|
+
function signalPids(pids, signal) {
|
|
1716
|
+
const signalled = [];
|
|
1717
|
+
for (const pid of pids) {
|
|
1718
|
+
try {
|
|
1719
|
+
process.kill(-pid, signal);
|
|
1720
|
+
signalled.push(pid);
|
|
1721
|
+
}
|
|
1722
|
+
catch {
|
|
1723
|
+
try {
|
|
1724
|
+
process.kill(pid, signal);
|
|
1725
|
+
signalled.push(pid);
|
|
1726
|
+
}
|
|
1727
|
+
catch { }
|
|
1728
|
+
}
|
|
1729
|
+
}
|
|
1730
|
+
return signalled;
|
|
1731
|
+
}
|
|
1732
|
+
/** Pids from `pids` that are neither dead nor zombied — i.e. still need killing. Shared by both
|
|
1733
|
+
* the async and sync flavors of the grace-then-SIGKILL poll below. */
|
|
1734
|
+
function undeadPids(pids) {
|
|
1735
|
+
return survivingOwnedGroups(pids);
|
|
1736
|
+
}
|
|
1737
|
+
/** Bounded extra wait (ms) for a pid to actually finish dying once SIGKILL has been sent. SIGKILL
|
|
1738
|
+
* cannot be caught/blocked, but delivery + the kernel's exit transition is not instantaneous —
|
|
1739
|
+
* callers that gate on "confirmed dead" (purge, unshareWorld) need this, not just the signal sent. */
|
|
1740
|
+
const KILL_CONFIRM_MS = 2_000;
|
|
1741
|
+
/** The SIGKILL half of SIGTERM→SIGKILL escalation: bounded poll on pid liveness (returns as soon
|
|
1742
|
+
* as everything is dead — no fixed sleep), then SIGKILL whatever survived the grace, group-first
|
|
1743
|
+
* exactly like the SIGTERM was delivered, then a short bounded confirm-wait so survivors are
|
|
1744
|
+
* actually gone (not just signalled) by the time this returns. Returns the pids that needed
|
|
1745
|
+
* escalation. */
|
|
1746
|
+
async function killSurvivorsAfterGrace(pids, graceMs) {
|
|
1747
|
+
const deadline = Date.now() + graceMs;
|
|
1748
|
+
while (undeadPids(pids).length > 0 && Date.now() < deadline) {
|
|
1749
|
+
await new Promise((resolveWait) => setTimeout(resolveWait, 50));
|
|
1750
|
+
}
|
|
1751
|
+
const survivors = undeadPids(pids);
|
|
1752
|
+
signalPids(survivors, 'SIGKILL');
|
|
1753
|
+
const confirmDeadline = Date.now() + KILL_CONFIRM_MS;
|
|
1754
|
+
while (undeadPids(survivors).length > 0 && Date.now() < confirmDeadline) {
|
|
1755
|
+
await new Promise((resolveWait) => setTimeout(resolveWait, 20));
|
|
1756
|
+
}
|
|
1757
|
+
return survivors;
|
|
1758
|
+
}
|
|
1759
|
+
/** Synchronous twin of `killSurvivorsAfterGrace` for callers (`unshareWorld`) that must stay
|
|
1760
|
+
* synchronous — same "poll until dead or grace expires, then SIGKILL survivors, then confirm they
|
|
1761
|
+
* are actually gone" contract, using an `Atomics.wait` backoff (already this file's pattern for
|
|
1762
|
+
* synchronous polling — see `ensureWorldProxy`) instead of `setTimeout`, since a sync function has
|
|
1763
|
+
* no event loop to yield to. */
|
|
1764
|
+
function killSurvivorsAfterGraceSync(pids, graceMs) {
|
|
1765
|
+
const deadline = Date.now() + graceMs;
|
|
1766
|
+
while (undeadPids(pids).length > 0 && Date.now() < deadline) {
|
|
1767
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50);
|
|
1768
|
+
}
|
|
1769
|
+
const survivors = undeadPids(pids);
|
|
1770
|
+
signalPids(survivors, 'SIGKILL');
|
|
1771
|
+
const confirmDeadline = Date.now() + KILL_CONFIRM_MS;
|
|
1772
|
+
while (undeadPids(survivors).length > 0 && Date.now() < confirmDeadline) {
|
|
1773
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);
|
|
1774
|
+
}
|
|
1775
|
+
return survivors;
|
|
1776
|
+
}
|
|
1777
|
+
/** Async solely for the SIGTERM→SIGKILL grace poll — everything else (proxy stop, SIGTERM,
|
|
1778
|
+
* external teardown) still runs in the first synchronous slice, so a legacy caller that fires
|
|
1779
|
+
* without awaiting gets exactly the old behavior plus the deferred escalation. A blocking
|
|
1780
|
+
* sleep-loop would keep the sync signature but stall the caller's event loop for up to the whole
|
|
1781
|
+
* grace; a detached watchdog child would escalate too but leak orphan processes — the await is
|
|
1782
|
+
* the honest shape (upWorld/runWorld/the CLI are already async). */
|
|
1783
|
+
/**
|
|
1784
|
+
* Delete a world's instance dir entirely — the `--purge` half of `down` (TWIN-45 dev/02b): no
|
|
1785
|
+
* twin data (per-service `data/<id>`, logs, instance.json) survives past teardown. Plain `rm`,
|
|
1786
|
+
* same honesty as `world scrub` on the control-plane side (see docs/concepts/data-and-keys.md).
|
|
1787
|
+
*
|
|
1788
|
+
* Safety (never delete more than THIS world's own instance dir):
|
|
1789
|
+
* - `assertSafeWorldName` rejects any name containing `/` or starting with `.` — the only
|
|
1790
|
+
* names `instanceDir`/`instanceLockFile` can ever have produced — so `dir` cannot resolve
|
|
1791
|
+
* outside `worldBaseDir(root)` and cannot collide with a sibling instance's directory.
|
|
1792
|
+
* - `dir` is asserted to be an actual child of `worldBaseDir(root)` (defense in depth against
|
|
1793
|
+
* a future refactor of `instanceDir` weakening that guarantee) and can never equal
|
|
1794
|
+
* `worldBaseDir` itself (which would nuke every world) or its `.locks` sibling.
|
|
1795
|
+
* - The per-world claim lock (`instanceLockFile`) lives OUTSIDE the instance dir precisely so
|
|
1796
|
+
* `upWorld`'s claim survives an in-flight `rmSync` of the instance dir (see its own comment);
|
|
1797
|
+
* purge leaves that lock alone; its owner's finally releases it, so waiting claimants never
|
|
1798
|
+
* enter through a replacement lock while the original is still held.
|
|
1799
|
+
*/
|
|
1800
|
+
function purgeInstanceDir(root, name) {
|
|
1801
|
+
assertSafeWorldName(name);
|
|
1802
|
+
const base = worldBaseDir(root);
|
|
1803
|
+
const dir = instanceDir(root, name);
|
|
1804
|
+
const rel = relative(base, dir);
|
|
1805
|
+
if (!rel || rel === '.' || rel.startsWith('..') || rel.split(sep)[0] !== name) {
|
|
1806
|
+
throw new Error(`Refusing to purge: resolved instance dir "${dir}" is not a direct child of "${base}"`);
|
|
1807
|
+
}
|
|
1808
|
+
removeWorldState(root, dir, () => rmSync(dir, { recursive: true, force: true }));
|
|
1809
|
+
// The caller holds this lock. Never unlink it while another up may be waiting on it.
|
|
1810
|
+
return dir;
|
|
1811
|
+
}
|
|
1812
|
+
class WorldConsumersActiveError extends Error {
|
|
1813
|
+
}
|
|
1814
|
+
export async function downWorld(name, root = process.cwd(), options = {}) {
|
|
1815
|
+
assertSafeWorldName(name);
|
|
1816
|
+
const resolvedRoot = resolve(root);
|
|
1817
|
+
const token = randomUUID();
|
|
1818
|
+
// Like boot's claim: a short synchronous lock protects an async lifecycle transition.
|
|
1819
|
+
// Do not hold a filesystem lock while external shutdown or the grace period waits.
|
|
1820
|
+
withFileLock(instanceLockFile(resolvedRoot, name), () => withFileLock(`${sharingClaimFile(resolvedRoot, name)}.lock`, () => {
|
|
1821
|
+
assertProxyOwnershipResolved(resolvedRoot, name);
|
|
1822
|
+
const prior = existsSync(instanceFile(resolvedRoot, name)) ? readWorldInstance(name, resolvedRoot, true) : undefined;
|
|
1823
|
+
if (prior?.resources && !prior.lifecycle)
|
|
1824
|
+
throw new Error(`World "${name}" belongs to a legacy runtime; use its pinned runtime for teardown before migration`);
|
|
1825
|
+
if (options.expectedCreatedAt !== undefined && readWorldInstance(name, resolvedRoot, true).createdAt !== options.expectedCreatedAt) {
|
|
1826
|
+
throw new Error(`World "${name}" generation changed; teardown refused`);
|
|
1827
|
+
}
|
|
1828
|
+
if (options.purge)
|
|
1829
|
+
assertWorldStateRemovable(resolvedRoot, instanceDir(resolvedRoot, name));
|
|
1830
|
+
prepareBootForDown(resolvedRoot, name);
|
|
1831
|
+
if (liveBootingClaim(stoppingFile(resolvedRoot, name)))
|
|
1832
|
+
throw new Error(`World "${name}" is already being stopped`);
|
|
1833
|
+
const consumers = worldConsumers(resolvedRoot, name);
|
|
1834
|
+
if (consumers.warnings.length || consumers.consumers.some((c) => c.state !== 'exited')) {
|
|
1835
|
+
throw new WorldConsumersActiveError(`World "${name}" has active or uncertain attached consumers; finish the attached commands before down. Inspect: volter-world resources --root ${shellQuote(resolvedRoot)}`);
|
|
1836
|
+
}
|
|
1837
|
+
const sharePath = sharingClaimFile(resolvedRoot, name);
|
|
1838
|
+
if (existsSync(sharePath)) {
|
|
1839
|
+
const sharing = readSharingClaim(sharePath);
|
|
1840
|
+
if (!sharing || sharingClaimAlive(sharing))
|
|
1841
|
+
throw new Error(`World "${name}" is being shared or has an unreadable sharing claim; retry after sharing finishes`);
|
|
1842
|
+
rmSync(sharePath); // explicit down may reclaim a provably stale sharing claim
|
|
1843
|
+
}
|
|
1844
|
+
retainWorldLifecycleForCleanup(resolvedRoot, name);
|
|
1845
|
+
writeTextAtomic(stoppingFile(resolvedRoot, name), JSON.stringify({ pid: process.pid, at: new Date().toISOString(), token }));
|
|
1846
|
+
}));
|
|
1847
|
+
try {
|
|
1848
|
+
return await downWorldClaimed(name, resolvedRoot, options);
|
|
1849
|
+
}
|
|
1850
|
+
finally {
|
|
1851
|
+
withFileLock(instanceLockFile(resolvedRoot, name), () => {
|
|
1852
|
+
const path = stoppingFile(resolvedRoot, name);
|
|
1853
|
+
if (existsSync(path) && JSON.parse(readFileSync(path, 'utf8')).token === token)
|
|
1854
|
+
rmSync(path);
|
|
1855
|
+
});
|
|
1856
|
+
}
|
|
1857
|
+
}
|
|
1858
|
+
async function downWorldClaimed(name, root, options) {
|
|
1859
|
+
assertSafeWorldName(name);
|
|
1860
|
+
const resolvedRoot = resolve(root);
|
|
1861
|
+
const dir = instanceDir(resolvedRoot, name);
|
|
1862
|
+
const pidsFile = join(dir, 'pids');
|
|
1863
|
+
const before = existsSync(instanceFile(resolvedRoot, name)) ? readFileSync(instanceFile(resolvedRoot, name), 'utf8') : undefined;
|
|
1864
|
+
// A retry must not retain an earlier success receipt while teardown is in progress.
|
|
1865
|
+
rmSync(join(dir, 'stopped.json'), { force: true });
|
|
1866
|
+
// Tear down the ambient-redirect proxy + its session CA FIRST, so the trusted CA never outlives
|
|
1867
|
+
// the world (it was only ever trusted via per-shell env, never system-wide).
|
|
1868
|
+
let instance;
|
|
1869
|
+
try {
|
|
1870
|
+
instance = readWorldInstance(name, resolvedRoot);
|
|
1871
|
+
if (instance.name !== name || typeof instance.root !== 'string' || resolve(instance.root) !== resolvedRoot || !instance.services)
|
|
1872
|
+
throw new Error('invalid identity');
|
|
1873
|
+
}
|
|
1874
|
+
catch {
|
|
1875
|
+
instance = undefined;
|
|
1876
|
+
}
|
|
1877
|
+
// Nothing to tear down when the metadata is gone and every process the lifecycle recorded has exited.
|
|
1878
|
+
if (!instance && before === undefined && releaseAbandonedWorldLifecycle(resolvedRoot, name)) {
|
|
1879
|
+
return { name, stopped: [], escalated: [], externalStopped: [], externalErrors: [] };
|
|
1880
|
+
}
|
|
1881
|
+
if (!instance && (before !== undefined || hasWorldLifecycle(resolvedRoot, name))) {
|
|
1882
|
+
return { name, stopped: [], escalated: [], externalStopped: [],
|
|
1883
|
+
externalErrors: ['World instance or external teardown metadata missing or unreadable; cleanup refused; lifecycle evidence retained'] };
|
|
1884
|
+
}
|
|
1885
|
+
const proxyPid = readProxyState(resolvedRoot, name)?.pid;
|
|
1886
|
+
stopWorldProxy(resolvedRoot, name);
|
|
1887
|
+
const foregroundPid = instance?.lastRun?.state !== 'completed' ? instance?.lastRun?.consumerPid : undefined;
|
|
1888
|
+
const holderPid = instance?.lifecycle?.holderPid ?? instance?.resources?.holderPid;
|
|
1889
|
+
const recordedPids = [...new Set([...readPids(pidsFile), ...(foregroundPid ? [foregroundPid] : []), ...(proxyPid ? [proxyPid] : [])])].filter(pid => pid !== holderPid);
|
|
1890
|
+
const foreign = foreignPids(resolvedRoot, name, recordedPids);
|
|
1891
|
+
const pids = recordedPids.filter((pid) => !foreign.some((entry) => entry.pid === pid));
|
|
1892
|
+
const processServices = Object.values(instance?.services ?? {}).filter((service) => service.type !== 'external' && service.log);
|
|
1893
|
+
// Only a service still alive is stopped by this `down`; a dead one's end was not the World's.
|
|
1894
|
+
const stillAlive = undeadPids(processServices.map((service) => service.pid).filter((pid) => pids.includes(pid)));
|
|
1895
|
+
recordWorldStop(processServices.filter((service) => stillAlive.includes(service.pid)).map((service) => service.log), 'world.service.stop', 'down', 'SIGTERM');
|
|
1896
|
+
const stopped = signalPids(pids, 'SIGTERM');
|
|
1897
|
+
// Stop self-managed externals via their declared `down` command. A missing instance is
|
|
1898
|
+
// safe only without retained lifecycle ownership, checked before any signals above.
|
|
1899
|
+
let externalStopped = [];
|
|
1900
|
+
let externalErrors = [];
|
|
1901
|
+
try {
|
|
1902
|
+
if (!instance)
|
|
1903
|
+
throw new Error('World instance unavailable');
|
|
1904
|
+
const result = await teardownExternalServices(instance);
|
|
1905
|
+
externalStopped = result.stopped;
|
|
1906
|
+
externalErrors = result.errors;
|
|
1907
|
+
}
|
|
1908
|
+
catch {
|
|
1909
|
+
// no instance.json (never fully came up) — only owned processes needed stopping.
|
|
1910
|
+
if (before !== undefined)
|
|
1911
|
+
externalErrors.push('World instance or external teardown metadata unreadable; cleanup refused');
|
|
1912
|
+
}
|
|
1913
|
+
// Escalate AFTER the external teardown so externals never wait on the grace, and
|
|
1914
|
+
// poll the full pids list rather than `stopped`: a pid whose SIGTERM errored is almost always
|
|
1915
|
+
// already dead, and livePids() re-checks anyway.
|
|
1916
|
+
const escalated = await killSurvivorsAfterGrace(pids, options.graceMs ?? DOWN_GRACE_MS_DEFAULT);
|
|
1917
|
+
recordWorldStop(processServices.filter((service) => escalated.includes(service.pid)).map((service) => service.log), 'world.service.killed', 'down', 'SIGKILL');
|
|
1918
|
+
if (holderPid && !externalErrors.length && !undeadPids(pids).length) {
|
|
1919
|
+
pids.push(holderPid);
|
|
1920
|
+
stopped.push(...signalPids([holderPid], 'SIGTERM'));
|
|
1921
|
+
escalated.push(...await killSurvivorsAfterGrace([holderPid], options.graceMs ?? DOWN_GRACE_MS_DEFAULT));
|
|
1922
|
+
}
|
|
1923
|
+
// Finalization belongs to this exact generation, not just the reusable name. A new up may
|
|
1924
|
+
// win once the old PIDs exit; never erase its PID file, lifecycle or instance directory.
|
|
1925
|
+
return withFileLock(instanceLockFile(resolvedRoot, name), () => withFileLock(`${sharingClaimFile(resolvedRoot, name)}.lock`, () => {
|
|
1926
|
+
const notSignalled = foreign.map((entry) => `pid ${entry.pid} now names another process (started ${entry.now}; the World's started ${entry.recorded}); not signalled`);
|
|
1927
|
+
const result = { name, stopped, escalated, externalStopped, externalErrors, ...(notSignalled.length ? { notSignalled } : {}) };
|
|
1928
|
+
const path = instanceFile(resolvedRoot, name);
|
|
1929
|
+
const current = existsSync(path) ? readFileSync(path, 'utf8') : undefined;
|
|
1930
|
+
if (current !== before || existsSync(bootingFile(resolvedRoot, name))) {
|
|
1931
|
+
externalErrors.push('World generation changed or is booting; finalization refused');
|
|
1932
|
+
return result;
|
|
1933
|
+
}
|
|
1934
|
+
if (existsSync(proxyStateFile(resolvedRoot, name)) || existsSync(sharingClaimFile(resolvedRoot, name))) {
|
|
1935
|
+
externalErrors.push('Proxy or sharing lifecycle changed during teardown; cleanup refused');
|
|
1936
|
+
return result;
|
|
1937
|
+
}
|
|
1938
|
+
const surviving = undeadPids(pids);
|
|
1939
|
+
if (surviving.length)
|
|
1940
|
+
externalErrors.push(`World processes still alive or uninspectable after teardown: ${surviving.join(', ')}; cleanup refused`);
|
|
1941
|
+
if (externalErrors.length)
|
|
1942
|
+
return result;
|
|
1943
|
+
rmSync(pidsFile, { force: true });
|
|
1944
|
+
if (before) {
|
|
1945
|
+
writeTextAtomic(join(dir, 'stopped.json'), `${JSON.stringify({
|
|
1946
|
+
instanceHash: createHash('sha256').update(before).digest('hex'), stoppedAt: new Date().toISOString(),
|
|
1947
|
+
})}\n`);
|
|
1948
|
+
}
|
|
1949
|
+
releaseWorldLifecycle(resolvedRoot, name);
|
|
1950
|
+
return options.purge ? { ...result, purged: purgeInstanceDir(resolvedRoot, name) } : result;
|
|
1951
|
+
}));
|
|
1952
|
+
}
|
|
1953
|
+
export function statusWorld(name, root = process.cwd(), options = {}) {
|
|
1954
|
+
const worldInstance = readWorldInstance(name, resolve(root), options.readOnly);
|
|
1955
|
+
const live = ownedPids(resolve(root), name, livePids(readPids(worldInstance.pidsFile)));
|
|
1956
|
+
const externalOnly = Object.values(worldInstance.services).every(service => service.type === 'external');
|
|
1957
|
+
const recorded = externalOnly && !existsSync(stoppingFile(root, name)) && !existsSync(join(instanceDir(root, name), 'stopped.json'))
|
|
1958
|
+
&& isWorldLifecycleRunning(root, name, worldInstance.createdAt);
|
|
1959
|
+
// `running` is ownership (any owned process lives: a service, the proxy daemon, a tunnel), which the lifecycle guards
|
|
1960
|
+
// read; a service's own state is judged by its process group, so a dead service reads as dead beside a live daemon.
|
|
1961
|
+
const owned = Object.values(worldInstance.services).filter((service) => service.type !== 'external' && service.pid);
|
|
1962
|
+
const ownPids = ownedPids(resolve(root), name, owned.map((service) => service.pid));
|
|
1963
|
+
const leaders = livePids(ownPids);
|
|
1964
|
+
// `light` (a listing of every World) skips the process-table scan: a lingering group then reads as stopped.
|
|
1965
|
+
const groups = ownPids.length && !options.light ? survivingOwnedGroups(ownPids) : [];
|
|
1966
|
+
const serviceStates = {};
|
|
1967
|
+
for (const service of Object.values(worldInstance.services)) {
|
|
1968
|
+
serviceStates[service.id] = service.type === 'external' ? 'external'
|
|
1969
|
+
: service.workerGaveUp ? 'gave-up'
|
|
1970
|
+
: leaders.includes(service.pid) ? 'running' : groups.includes(service.pid) ? 'lingering' : 'stopped';
|
|
1971
|
+
}
|
|
1972
|
+
const running = live.length > 0 || recorded;
|
|
1973
|
+
const degraded = running && Object.values(serviceStates).some((state) => state !== 'running' && state !== 'external');
|
|
1974
|
+
return { ...worldInstance, running, livePids: live, serviceStates, degraded };
|
|
1975
|
+
}
|
|
1976
|
+
function logicalTreeBytes(path) {
|
|
1977
|
+
const stat = lstatSync(path);
|
|
1978
|
+
if (!stat.isDirectory())
|
|
1979
|
+
return stat.size; // never follow symlinks
|
|
1980
|
+
return readdirSync(path).reduce((sum, child) => sum + logicalTreeBytes(join(path, child)), 0);
|
|
1981
|
+
}
|
|
1982
|
+
/** Fail closed. Neither dead service PIDs nor an absent lifecycle record prove external teardown.
|
|
1983
|
+
* Only down's receipt for this exact instance makes its disposable data eligible. */
|
|
1984
|
+
function worldCleanupEntry(name, root) {
|
|
1985
|
+
assertSafeWorldName(name);
|
|
1986
|
+
const path = instanceDir(root, name);
|
|
1987
|
+
const entry = { name, path, eligible: false, reason: '' };
|
|
1988
|
+
try {
|
|
1989
|
+
if ([join(root, stateDirName()), worldBaseDir(root), join(worldBaseDir(root), '.locks'), path]
|
|
1990
|
+
.some((dir) => existsSync(dir) && lstatSync(dir).isSymbolicLink())) {
|
|
1991
|
+
return { ...entry, reason: 'symbolic-link state directory; retained' };
|
|
1992
|
+
}
|
|
1993
|
+
if (!lstatSync(path).isDirectory())
|
|
1994
|
+
return { ...entry, reason: 'not an instance directory' };
|
|
1995
|
+
if (existsSync(proxyStartingFile(root, name)))
|
|
1996
|
+
return { ...entry, reason: 'unresolved proxy startup ownership; retained' };
|
|
1997
|
+
if (existsSync(stoppingFile(root, name)))
|
|
1998
|
+
return { ...entry, reason: 'teardown marker present; retained' };
|
|
1999
|
+
if (existsSync(bootingFile(root, name)))
|
|
2000
|
+
return { ...entry, reason: 'boot marker present; retained' };
|
|
2001
|
+
if (existsSync(proxyStateFile(root, name)))
|
|
2002
|
+
return { ...entry, reason: 'proxy lifecycle record present; retained' };
|
|
2003
|
+
if (existsSync(sharingClaimFile(root, name)))
|
|
2004
|
+
return { ...entry, reason: 'sharing lifecycle claim present; retained' };
|
|
2005
|
+
const raw = readFileSync(instanceFile(root, name), 'utf8');
|
|
2006
|
+
const snapshot = JSON.parse(raw);
|
|
2007
|
+
if (snapshot.name !== name || typeof snapshot.root !== 'string' || resolve(snapshot.root) !== root) {
|
|
2008
|
+
return { ...entry, reason: 'inconsistent instance identity' };
|
|
2009
|
+
}
|
|
2010
|
+
const instance = statusWorld(name, root, { readOnly: true });
|
|
2011
|
+
if (instance.name !== name || typeof instance.createdAt !== 'string' || !instance.services
|
|
2012
|
+
|| typeof instance.configPath !== 'string' || resolve(instance.dirs.instance) !== path
|
|
2013
|
+
|| resolve(instance.pidsFile) !== join(path, 'pids'))
|
|
2014
|
+
return { ...entry, reason: 'inconsistent instance metadata' };
|
|
2015
|
+
const configRelative = relative(path, resolve(instance.configPath));
|
|
2016
|
+
if (!configRelative || (!configRelative.startsWith(`..${sep}`) && configRelative !== '..')) {
|
|
2017
|
+
return { ...entry, reason: 'source config inside instance; retained' };
|
|
2018
|
+
}
|
|
2019
|
+
const servicePids = Object.values(instance.services).map((service) => service.pid);
|
|
2020
|
+
const knownPids = [...instance.livePids, ...servicePids, instance.lifecycle?.holderPid ?? instance.resources?.holderPid ?? 0];
|
|
2021
|
+
if (undeadPids(knownPids.filter((pid) => Number.isInteger(pid) && pid > 0)).length) {
|
|
2022
|
+
return { ...entry, reason: 'running World; stopping requires explicit down' };
|
|
2023
|
+
}
|
|
2024
|
+
const attached = worldConsumers(root, name, instance.createdAt);
|
|
2025
|
+
if (attached.warnings.length || attached.consumers.some((c) => c.state !== 'exited')) {
|
|
2026
|
+
return { ...entry, reason: 'active or uncertain attached consumer; retained' };
|
|
2027
|
+
}
|
|
2028
|
+
const lastRun = instance.lastRun;
|
|
2029
|
+
if (existsSync(foregroundRunFile(root, name))) {
|
|
2030
|
+
const record = JSON.parse(readFileSync(foregroundRunFile(root, name), 'utf8'));
|
|
2031
|
+
if (!record || !['running', 'abrupt', 'completed'].includes(record.state)
|
|
2032
|
+
|| !Number.isInteger(record.runnerPid) || record.runnerPid <= 0)
|
|
2033
|
+
return { ...entry, reason: 'invalid foreground consumer metadata' };
|
|
2034
|
+
}
|
|
2035
|
+
if (lastRun && lastRun.state !== 'completed'
|
|
2036
|
+
&& livePids([lastRun.runnerPid, lastRun.consumerPid ?? 0].filter((pid) => pid > 0)).length) {
|
|
2037
|
+
return { ...entry, reason: 'foreground consumer still active' };
|
|
2038
|
+
}
|
|
2039
|
+
if (hasWorldLifecycle(root, name)) {
|
|
2040
|
+
return { ...entry, reason: 'lifecycle ownership remains; verified teardown required' };
|
|
2041
|
+
}
|
|
2042
|
+
const receiptPath = join(path, 'stopped.json');
|
|
2043
|
+
if (!existsSync(receiptPath))
|
|
2044
|
+
return { ...entry, reason: 'no successful teardown receipt (legacy/unknown); review before explicit down' };
|
|
2045
|
+
const receipt = JSON.parse(readFileSync(receiptPath, 'utf8'));
|
|
2046
|
+
if (receipt.instanceHash !== createHash('sha256').update(raw).digest('hex') || !Number.isFinite(Date.parse(receipt.stoppedAt))) {
|
|
2047
|
+
return { ...entry, reason: 'invalid or outdated teardown receipt' };
|
|
2048
|
+
}
|
|
2049
|
+
try {
|
|
2050
|
+
assertWorldStateRemovable(root, path);
|
|
2051
|
+
}
|
|
2052
|
+
catch (error) {
|
|
2053
|
+
return { ...entry, reason: error instanceof Error ? error.message : 'Branch ancestry unreadable; retained' };
|
|
2054
|
+
}
|
|
2055
|
+
return { ...entry, eligible: true, reason: 'successfully stopped; disposable instance data only', stoppedAt: receipt.stoppedAt, logicalBytes: logicalTreeBytes(path) };
|
|
2056
|
+
}
|
|
2057
|
+
catch (error) {
|
|
2058
|
+
// Do not print malformed metadata: it can contain env secrets.
|
|
2059
|
+
return { ...entry, reason: `unreadable or incomplete lifecycle metadata; retained (${error.code ?? 'invalid'})` };
|
|
2060
|
+
}
|
|
2061
|
+
}
|
|
2062
|
+
/** Preview by default. Apply holds up's lock across fresh eligibility checking and deletion.
|
|
2063
|
+
* Never call down here: that would turn a stale preview into permission to kill a new World. */
|
|
2064
|
+
export function pruneWorlds(options = {}) {
|
|
2065
|
+
const root = resolve(options.root ?? process.cwd());
|
|
2066
|
+
if (options.name !== undefined)
|
|
2067
|
+
assertSafeWorldName(options.name);
|
|
2068
|
+
const base = worldBaseDir(root);
|
|
2069
|
+
const names = options.name ? [options.name] : existsSync(base)
|
|
2070
|
+
? readdirSync(base).filter((name) => /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(name)).sort() : [];
|
|
2071
|
+
const entries = names.map((name) => {
|
|
2072
|
+
if (!options.apply)
|
|
2073
|
+
return worldCleanupEntry(name, root);
|
|
2074
|
+
// Refuse symlink/incomplete state before even creating a lock through its parent path.
|
|
2075
|
+
const preview = worldCleanupEntry(name, root);
|
|
2076
|
+
if (!preview.eligible)
|
|
2077
|
+
return preview;
|
|
2078
|
+
return withFileLock(instanceLockFile(root, name), () => withFileLock(`${sharingClaimFile(root, name)}.lock`, () => {
|
|
2079
|
+
const entry = worldCleanupEntry(name, root);
|
|
2080
|
+
if (entry.eligible) {
|
|
2081
|
+
removeWorldState(root, entry.path, () => rmSync(entry.path, { recursive: true }));
|
|
2082
|
+
// Keep the sibling lock alone: unlinking it here can break concurrent up's exclusion.
|
|
2083
|
+
return { ...entry, removed: true };
|
|
2084
|
+
}
|
|
2085
|
+
return entry;
|
|
2086
|
+
}));
|
|
2087
|
+
});
|
|
2088
|
+
return { root, apply: options.apply === true, entries };
|
|
2089
|
+
}
|
|
2090
|
+
function worldServiceControlRoot(name, service, root = process.cwd()) {
|
|
2091
|
+
const instance = readWorldInstance(name, resolve(root));
|
|
2092
|
+
if (!instance.services[service])
|
|
2093
|
+
throw new Error(`World ${name} has no service "${service}"`);
|
|
2094
|
+
return join(instance.dirs.data, service);
|
|
2095
|
+
}
|
|
2096
|
+
function worldServiceStateService(controlRoot, requestedService) {
|
|
2097
|
+
const stateRoot = join(controlRoot, stateDirName(), 'world');
|
|
2098
|
+
if (!existsSync(stateRoot))
|
|
2099
|
+
return requestedService;
|
|
2100
|
+
const actionServices = readdirSync(stateRoot, { withFileTypes: true })
|
|
2101
|
+
.filter((entry) => entry.isDirectory() && ['actions.jsonl', 'events.jsonl', 'branch.json'].some((f) => existsSync(join(stateRoot, entry.name, f))))
|
|
2102
|
+
.map((entry) => entry.name);
|
|
2103
|
+
if (actionServices.includes(requestedService))
|
|
2104
|
+
return requestedService;
|
|
2105
|
+
if (actionServices.length === 1)
|
|
2106
|
+
return actionServices[0];
|
|
2107
|
+
if (actionServices.length > 1) {
|
|
2108
|
+
throw new Error(`World service "${requestedService}" records actions under multiple state services (${actionServices.join(', ')}); use volter-twin plan with the explicit service data root`);
|
|
2109
|
+
}
|
|
2110
|
+
return requestedService;
|
|
2111
|
+
}
|
|
2112
|
+
function shareCommand(options, serviceUrl) {
|
|
2113
|
+
if (options.provider === 'cloudflare-quick' && options.command) {
|
|
2114
|
+
throw new Error('share provider "cloudflare-quick" uses the built-in cloudflared command and cannot be combined with a custom command');
|
|
2115
|
+
}
|
|
2116
|
+
if (!options.command && options.provider === 'command') {
|
|
2117
|
+
throw new Error(`share provider "command" requires a tunnel command (share.command / --command)`);
|
|
2118
|
+
}
|
|
2119
|
+
if (options.command) {
|
|
2120
|
+
return {
|
|
2121
|
+
provider: 'command',
|
|
2122
|
+
command: options.command,
|
|
2123
|
+
args: (options.args ?? []).map((arg) => arg.replaceAll('{url}', serviceUrl)),
|
|
2124
|
+
};
|
|
2125
|
+
}
|
|
2126
|
+
return {
|
|
2127
|
+
provider: 'cloudflare-quick',
|
|
2128
|
+
command: 'cloudflared',
|
|
2129
|
+
args: ['tunnel', '--url', serviceUrl],
|
|
2130
|
+
};
|
|
2131
|
+
}
|
|
2132
|
+
export function parseCloudflareQuickPublicUrl(candidate) {
|
|
2133
|
+
// Validate the provider's RAW stdout token before WHATWG normalization. Otherwise `%2e` in the
|
|
2134
|
+
// authority, an explicit default `:443`, or `/.` / `/%2e%2e` paths normalize into an apparently
|
|
2135
|
+
// valid origin and bypass the strict cloudflared-output boundary.
|
|
2136
|
+
if (!/^https:\/\/[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.trycloudflare\.com\/?$/.test(candidate))
|
|
2137
|
+
return undefined;
|
|
2138
|
+
let parsed;
|
|
2139
|
+
try {
|
|
2140
|
+
parsed = new URL(candidate);
|
|
2141
|
+
}
|
|
2142
|
+
catch {
|
|
2143
|
+
return undefined;
|
|
2144
|
+
}
|
|
2145
|
+
if (parsed.protocol !== 'https:' || parsed.username || parsed.password || parsed.port
|
|
2146
|
+
|| !/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.trycloudflare\.com$/.test(parsed.hostname)
|
|
2147
|
+
|| parsed.pathname !== '/' || parsed.search || parsed.hash)
|
|
2148
|
+
return undefined;
|
|
2149
|
+
return parsed.origin;
|
|
2150
|
+
}
|
|
2151
|
+
function canonicalDottedIpv4(host) {
|
|
2152
|
+
const parts = host.split('.');
|
|
2153
|
+
return parts.length === 4 && parts.every((part) => /^(?:0|[1-9][0-9]{0,2})$/.test(part) && Number(part) <= 255);
|
|
2154
|
+
}
|
|
2155
|
+
function parseCustomShareOrigin(handshake) {
|
|
2156
|
+
const raw = /^https?:\/\/(\[[0-9a-f:.]+\]|[a-z0-9.-]+)(?::([0-9]{1,5}))?\/?$/i.exec(handshake);
|
|
2157
|
+
if (!raw) {
|
|
2158
|
+
throw new Error(`Custom tunnel VOLTER_SHARE_URL must be a strict http(s) origin without path, query, fragment, or backslashes: ${handshake}`);
|
|
2159
|
+
}
|
|
2160
|
+
const rawHost = raw[1];
|
|
2161
|
+
const rawPort = raw[2];
|
|
2162
|
+
let parsed;
|
|
2163
|
+
try {
|
|
2164
|
+
parsed = new URL(handshake);
|
|
2165
|
+
}
|
|
2166
|
+
catch {
|
|
2167
|
+
throw new Error(`Custom tunnel emitted an invalid VOLTER_SHARE_URL handshake: ${handshake}`);
|
|
2168
|
+
}
|
|
2169
|
+
const normalizedHostname = parsed.hostname.replace(/^\[|\]$/g, '').toLowerCase();
|
|
2170
|
+
if (!rawHost.startsWith('[') && isIP(normalizedHostname) === 4
|
|
2171
|
+
&& (!canonicalDottedIpv4(rawHost) || rawHost !== normalizedHostname)) {
|
|
2172
|
+
throw new Error(`Custom tunnel VOLTER_SHARE_URL must use canonical dotted-decimal IPv4 syntax: ${handshake}`);
|
|
2173
|
+
}
|
|
2174
|
+
if (rawPort && rawPort.length > 1 && rawPort.startsWith('0')) {
|
|
2175
|
+
throw new Error(`Custom tunnel VOLTER_SHARE_URL must use canonical decimal port syntax: ${handshake}`);
|
|
2176
|
+
}
|
|
2177
|
+
if ((parsed.protocol !== 'http:' && parsed.protocol !== 'https:') || parsed.username || parsed.password
|
|
2178
|
+
|| parsed.pathname !== '/' || parsed.search || parsed.hash) {
|
|
2179
|
+
throw new Error(`Custom tunnel VOLTER_SHARE_URL must be a credential-free http(s) origin: ${handshake}`);
|
|
2180
|
+
}
|
|
2181
|
+
return parsed.origin;
|
|
2182
|
+
}
|
|
2183
|
+
async function waitForPublicUrl(child, logPath, timeoutMs, requireTryCloudflare) {
|
|
2184
|
+
const urlPattern = /https?:\/\/[^\s"'<>]+/gi;
|
|
2185
|
+
let exited = false;
|
|
2186
|
+
let exitSummary = '';
|
|
2187
|
+
child.once('exit', (code, signal) => {
|
|
2188
|
+
exited = true;
|
|
2189
|
+
exitSummary = `code=${code ?? 'null'} signal=${signal ?? 'null'}`;
|
|
2190
|
+
});
|
|
2191
|
+
child.once('error', (error) => {
|
|
2192
|
+
exited = true;
|
|
2193
|
+
exitSummary = error.message;
|
|
2194
|
+
});
|
|
2195
|
+
const started = Date.now();
|
|
2196
|
+
while (Date.now() - started < timeoutMs) {
|
|
2197
|
+
const text = existsSync(logPath) ? readFileSync(logPath, 'utf8') : '';
|
|
2198
|
+
if (requireTryCloudflare) {
|
|
2199
|
+
for (const candidate of text.match(urlPattern) ?? []) {
|
|
2200
|
+
const parsed = parseCloudflareQuickPublicUrl(candidate);
|
|
2201
|
+
if (parsed)
|
|
2202
|
+
return parsed;
|
|
2203
|
+
}
|
|
2204
|
+
}
|
|
2205
|
+
else {
|
|
2206
|
+
const handshake = /^VOLTER_SHARE_URL=(\S+)\s*$/m.exec(text)?.[1];
|
|
2207
|
+
if (handshake)
|
|
2208
|
+
return parseCustomShareOrigin(handshake);
|
|
2209
|
+
}
|
|
2210
|
+
if (exited)
|
|
2211
|
+
throw new Error(`Tunnel process exited before printing a public URL (${exitSummary}). Log: ${logPath}`);
|
|
2212
|
+
await new Promise((resolveWait) => setTimeout(resolveWait, 250));
|
|
2213
|
+
}
|
|
2214
|
+
throw new Error(requireTryCloudflare
|
|
2215
|
+
? `Cloudflare tunnel did not print a trycloudflare.com https URL within ${timeoutMs}ms. Log: ${logPath}`
|
|
2216
|
+
: `Custom tunnel did not print VOLTER_SHARE_URL=<http(s)-url> within ${timeoutMs}ms. Log: ${logPath}`);
|
|
2217
|
+
}
|
|
2218
|
+
function sleep(ms) {
|
|
2219
|
+
return new Promise((resolveWait) => setTimeout(resolveWait, ms));
|
|
2220
|
+
}
|
|
2221
|
+
function commandExists(command) {
|
|
2222
|
+
// Pass the live env so PATH reflects the current process (some runtimes snapshot env otherwise).
|
|
2223
|
+
const result = spawnSync('which', [command], { stdio: 'ignore', env: process.env });
|
|
2224
|
+
return result.status === 0;
|
|
2225
|
+
}
|
|
2226
|
+
function resolveWithPublicDns(hostname) {
|
|
2227
|
+
const result = spawnSync('dig', ['+short', '@1.1.1.1', hostname, 'A'], {
|
|
2228
|
+
encoding: 'utf8',
|
|
2229
|
+
});
|
|
2230
|
+
if (result.status !== 0)
|
|
2231
|
+
return undefined;
|
|
2232
|
+
return result.stdout
|
|
2233
|
+
.split(/\r?\n/)
|
|
2234
|
+
.map((line) => line.trim())
|
|
2235
|
+
.find((line) => /^\d{1,3}(?:\.\d{1,3}){3}$/.test(line));
|
|
2236
|
+
}
|
|
2237
|
+
async function waitForPublicDns(hostname, timeoutMs) {
|
|
2238
|
+
const started = Date.now();
|
|
2239
|
+
while (Date.now() - started < timeoutMs) {
|
|
2240
|
+
const ip = resolveWithPublicDns(hostname);
|
|
2241
|
+
if (ip)
|
|
2242
|
+
return ip;
|
|
2243
|
+
await sleep(1_000);
|
|
2244
|
+
}
|
|
2245
|
+
return undefined;
|
|
2246
|
+
}
|
|
2247
|
+
function publicHealthUrl(publicUrl, path) {
|
|
2248
|
+
const url = new URL(publicUrl);
|
|
2249
|
+
url.pathname = path.startsWith('/') ? path : `/${path}`;
|
|
2250
|
+
url.search = '';
|
|
2251
|
+
url.hash = '';
|
|
2252
|
+
return url.toString();
|
|
2253
|
+
}
|
|
2254
|
+
async function verifyPublicUrl(publicUrl, path, timeoutMs) {
|
|
2255
|
+
if (!commandExists('curl'))
|
|
2256
|
+
throw new Error('Public URL verification requires curl');
|
|
2257
|
+
const hostname = new URL(publicUrl).hostname;
|
|
2258
|
+
const normalizedHostname = hostname.replace(/^\[|\]$/g, '').toLowerCase();
|
|
2259
|
+
const local = normalizedHostname === 'localhost' || normalizedHostname === '::1'
|
|
2260
|
+
|| (isIP(normalizedHostname) === 4 && normalizedHostname.split('.')[0] === '127');
|
|
2261
|
+
if (local) {
|
|
2262
|
+
const target = publicHealthUrl(publicUrl, path);
|
|
2263
|
+
const result = spawnSync('curl', ['--noproxy', '*', '-sS', '-o', '-', '-w', '\n%{http_code}', '--max-time', String(Math.max(1, Math.ceil(timeoutMs / 1000))), target], { encoding: 'utf8' });
|
|
2264
|
+
const separator = result.stdout.lastIndexOf('\n');
|
|
2265
|
+
const body = separator >= 0 ? result.stdout.slice(0, separator).trim() : result.stdout.trim();
|
|
2266
|
+
const status = Number(separator >= 0 ? result.stdout.slice(separator + 1).trim() : '0');
|
|
2267
|
+
if (result.status !== 0 || status < 200 || status >= 400) {
|
|
2268
|
+
throw new Error(`Local shared URL did not pass health verification at ${target}: ${result.stderr.trim() || `HTTP ${status}`}`);
|
|
2269
|
+
}
|
|
2270
|
+
return { path, checkedAt: new Date().toISOString(), hostname, resolvedIp: normalizedHostname === 'localhost' ? '127.0.0.1' : normalizedHostname, status, body: body.slice(0, 500) };
|
|
2271
|
+
}
|
|
2272
|
+
if (!commandExists('dig'))
|
|
2273
|
+
throw new Error('Public URL verification requires dig');
|
|
2274
|
+
const started = Date.now();
|
|
2275
|
+
const resolvedIp = await waitForPublicDns(hostname, Math.min(timeoutMs, 60_000));
|
|
2276
|
+
if (!resolvedIp) {
|
|
2277
|
+
throw new Error(`Public URL did not resolve through @1.1.1.1 within ${Math.min(timeoutMs, 60_000)}ms: ${hostname}`);
|
|
2278
|
+
}
|
|
2279
|
+
const target = publicHealthUrl(publicUrl, path);
|
|
2280
|
+
let lastError = '';
|
|
2281
|
+
while (Date.now() - started < timeoutMs) {
|
|
2282
|
+
const remainingMs = Math.max(timeoutMs - (Date.now() - started), 1);
|
|
2283
|
+
const result = spawnSync('curl', [
|
|
2284
|
+
'-sS',
|
|
2285
|
+
'-o',
|
|
2286
|
+
'-',
|
|
2287
|
+
'-w',
|
|
2288
|
+
'\n%{http_code}',
|
|
2289
|
+
'--max-time',
|
|
2290
|
+
String(Math.min(10, Math.max(1, Math.ceil(remainingMs / 1000)))),
|
|
2291
|
+
'--resolve',
|
|
2292
|
+
`${hostname}:443:${resolvedIp}`,
|
|
2293
|
+
target,
|
|
2294
|
+
], { encoding: 'utf8' });
|
|
2295
|
+
if (result.status === 0) {
|
|
2296
|
+
const output = result.stdout;
|
|
2297
|
+
const separator = output.lastIndexOf('\n');
|
|
2298
|
+
const body = separator >= 0 ? output.slice(0, separator).trim() : output.trim();
|
|
2299
|
+
const status = Number(separator >= 0 ? output.slice(separator + 1).trim() : '0');
|
|
2300
|
+
if (status >= 200 && status < 400) {
|
|
2301
|
+
return {
|
|
2302
|
+
path,
|
|
2303
|
+
checkedAt: new Date().toISOString(),
|
|
2304
|
+
hostname,
|
|
2305
|
+
resolvedIp,
|
|
2306
|
+
status,
|
|
2307
|
+
body: body.slice(0, 500),
|
|
2308
|
+
};
|
|
2309
|
+
}
|
|
2310
|
+
lastError = `HTTP ${status}${body ? `: ${body.slice(0, 200)}` : ''}`;
|
|
2311
|
+
}
|
|
2312
|
+
else {
|
|
2313
|
+
lastError = result.stderr.trim() || `curl exited ${result.status}`;
|
|
2314
|
+
}
|
|
2315
|
+
await sleep(Math.min(2_000, Math.max(timeoutMs - (Date.now() - started), 0)));
|
|
2316
|
+
}
|
|
2317
|
+
throw new Error(`Public URL did not pass health verification at ${target}: ${lastError || 'timed out'}`);
|
|
2318
|
+
}
|
|
2319
|
+
export async function shareWorld(name, options = {}) {
|
|
2320
|
+
const root = resolve(options.root ?? process.cwd());
|
|
2321
|
+
const serviceId = options.service ?? 'app';
|
|
2322
|
+
assertSafeWorldName(name);
|
|
2323
|
+
const claim = await reserveSharingClaim(root, name, serviceId);
|
|
2324
|
+
const heartbeat = setInterval(() => {
|
|
2325
|
+
try {
|
|
2326
|
+
refreshSharingClaim(root, name, claim.token);
|
|
2327
|
+
}
|
|
2328
|
+
catch { /* retry next tick; same-host process identity still protects the live owner */ }
|
|
2329
|
+
}, SHARE_CLAIM_HEARTBEAT_MS);
|
|
2330
|
+
heartbeat.unref();
|
|
2331
|
+
try {
|
|
2332
|
+
return await shareWorldReserved(name, { ...options, root, service: serviceId });
|
|
2333
|
+
}
|
|
2334
|
+
finally {
|
|
2335
|
+
clearInterval(heartbeat);
|
|
2336
|
+
releaseSharingClaim(root, name, claim.token);
|
|
2337
|
+
}
|
|
2338
|
+
}
|
|
2339
|
+
async function shareWorldReserved(name, options) {
|
|
2340
|
+
const root = resolve(options.root ?? process.cwd());
|
|
2341
|
+
const serviceId = options.service ?? 'app';
|
|
2342
|
+
const instance = readWorldInstance(name, root);
|
|
2343
|
+
const service = instance.services[serviceId];
|
|
2344
|
+
if (!service)
|
|
2345
|
+
throw new Error(`World "${name}" has no service "${serviceId}"`);
|
|
2346
|
+
if (service.tunnel && livePids([service.tunnel.pid]).length > 0) {
|
|
2347
|
+
throw new Error(`World "${name}" service "${serviceId}" is already shared at ${service.publicUrl}`);
|
|
2348
|
+
}
|
|
2349
|
+
if (!service.url) {
|
|
2350
|
+
throw new Error(`World "${name}" service "${serviceId}" has no local URL to share (external/self-managed services are not assigned one)`);
|
|
2351
|
+
}
|
|
2352
|
+
const tunnel = shareCommand(options, service.url);
|
|
2353
|
+
const log = join(instance.dirs.logs, `${serviceId}.tunnel.log`);
|
|
2354
|
+
// Each spawned tunnel owns one log generation. Truncating before spawn prevents a repeated
|
|
2355
|
+
// share from accepting the previous process's URL handshake as the new process's endpoint.
|
|
2356
|
+
const out = openPrivateLog(log, 'w');
|
|
2357
|
+
const child = spawn(tunnel.command, tunnel.args, {
|
|
2358
|
+
cwd: root,
|
|
2359
|
+
env: process.env,
|
|
2360
|
+
detached: true,
|
|
2361
|
+
stdio: ['ignore', out, out],
|
|
2362
|
+
});
|
|
2363
|
+
closeSync(out);
|
|
2364
|
+
let publicUrl;
|
|
2365
|
+
let publicVerification;
|
|
2366
|
+
let pidsWritten = false;
|
|
2367
|
+
try {
|
|
2368
|
+
publicUrl = await waitForPublicUrl(child, log, options.timeoutMs ?? 20_000, tunnel.provider === 'cloudflare-quick');
|
|
2369
|
+
if (options.verifyPath !== false) {
|
|
2370
|
+
publicVerification = await verifyPublicUrl(publicUrl, options.verifyPath ?? '/health', options.timeoutMs ?? 90_000);
|
|
2371
|
+
}
|
|
2372
|
+
// Build the prospective instance off to the side. Until BOTH persistence files are written,
|
|
2373
|
+
// callers and the previously loaded instance retain the exact unshared object.
|
|
2374
|
+
const nextInstance = structuredClone(instance);
|
|
2375
|
+
const nextService = nextInstance.services[serviceId];
|
|
2376
|
+
nextService.publicUrl = publicUrl;
|
|
2377
|
+
nextService.publicUrlEphemeral = tunnel.provider === 'cloudflare-quick' || options.ephemeral === true;
|
|
2378
|
+
nextService.publicReady = options.verifyPath !== false;
|
|
2379
|
+
if (publicVerification)
|
|
2380
|
+
nextService.publicVerification = publicVerification;
|
|
2381
|
+
else
|
|
2382
|
+
delete nextService.publicVerification;
|
|
2383
|
+
nextService.tunnel = {
|
|
2384
|
+
provider: tunnel.provider,
|
|
2385
|
+
pid: child.pid ?? 0,
|
|
2386
|
+
log,
|
|
2387
|
+
command: [tunnel.command, ...tunnel.args],
|
|
2388
|
+
startedAt: new Date().toISOString(),
|
|
2389
|
+
};
|
|
2390
|
+
writePidsFromInstance(nextInstance, child.pid ? [child.pid] : []);
|
|
2391
|
+
pidsWritten = true;
|
|
2392
|
+
saveWorldInstance(nextInstance);
|
|
2393
|
+
child.unref();
|
|
2394
|
+
return nextInstance;
|
|
2395
|
+
}
|
|
2396
|
+
catch (error) {
|
|
2397
|
+
// TWIN-62: never-half-shared means the tunnel must actually be confirmed dead before we
|
|
2398
|
+
// rethrow, not just best-effort SIGTERM'd — the same SIGTERM→grace→SIGKILL contract `downWorld`
|
|
2399
|
+
// gives owned services (a cloudflared that ignores SIGTERM must not keep serving after a
|
|
2400
|
+
// failed/rejected `share`).
|
|
2401
|
+
if (child.pid) {
|
|
2402
|
+
signalPids([child.pid], 'SIGTERM');
|
|
2403
|
+
await killSurvivorsAfterGrace([child.pid], DOWN_GRACE_MS_DEFAULT);
|
|
2404
|
+
}
|
|
2405
|
+
const cleanupProblems = [];
|
|
2406
|
+
if (child.pid && undeadPids([child.pid]).length > 0)
|
|
2407
|
+
cleanupProblems.push(`tunnel pid ${child.pid} survived cleanup`);
|
|
2408
|
+
// If the pids write landed but instance.json did not, restore the previous pids projection only
|
|
2409
|
+
// after the child is confirmed dead. Atomic writes ensure a failed write itself leaves the old
|
|
2410
|
+
// file intact; this compensates the one successful write in a two-file persistence attempt.
|
|
2411
|
+
if (pidsWritten) {
|
|
2412
|
+
try {
|
|
2413
|
+
writePidsFromInstance(instance);
|
|
2414
|
+
}
|
|
2415
|
+
catch (restoreError) {
|
|
2416
|
+
cleanupProblems.push(`could not restore pids file: ${restoreError instanceof Error ? restoreError.message : String(restoreError)}`);
|
|
2417
|
+
}
|
|
2418
|
+
}
|
|
2419
|
+
if (cleanupProblems.length > 0) {
|
|
2420
|
+
throw new Error(`${error instanceof Error ? error.message : String(error)}; share cleanup incomplete: ${cleanupProblems.join('; ')}`, { cause: error });
|
|
2421
|
+
}
|
|
2422
|
+
throw error;
|
|
2423
|
+
}
|
|
2424
|
+
}
|
|
2425
|
+
function configuredShareTargets(instance, requestedService, verifyPath) {
|
|
2426
|
+
if (requestedService)
|
|
2427
|
+
return [{ id: requestedService, verifyPath }];
|
|
2428
|
+
const config = loadWorldConfig(instance.configPath, instance.root).config;
|
|
2429
|
+
if (config.share?.services.length) {
|
|
2430
|
+
return config.share.services.map((service) => ({
|
|
2431
|
+
id: service.id,
|
|
2432
|
+
verifyPath: verifyPath ?? service.verifyPath,
|
|
2433
|
+
}));
|
|
2434
|
+
}
|
|
2435
|
+
return [{ id: 'app', verifyPath }];
|
|
2436
|
+
}
|
|
2437
|
+
export async function shareWorldServices(name, options = {}) {
|
|
2438
|
+
const root = resolve(options.root ?? process.cwd());
|
|
2439
|
+
let instance = readWorldInstance(name, root);
|
|
2440
|
+
const config = loadWorldConfig(instance.configPath, instance.root).config;
|
|
2441
|
+
const targets = configuredShareTargets(instance, options.service, options.verifyPath);
|
|
2442
|
+
for (const target of targets) {
|
|
2443
|
+
const provider = options.provider ?? (options.command ? 'command' : config.share?.provider);
|
|
2444
|
+
if (provider === 'cloudflare-quick' && options.command) {
|
|
2445
|
+
throw new Error('share provider "cloudflare-quick" uses the built-in cloudflared command and cannot be combined with a custom command');
|
|
2446
|
+
}
|
|
2447
|
+
// An explicit provider override is authoritative. In particular, selecting the real
|
|
2448
|
+
// Cloudflare provider must clear a config-declared local rehearsal command instead of
|
|
2449
|
+
// launching that command while parsing its output as cloudflared.
|
|
2450
|
+
const command = provider === 'cloudflare-quick' ? undefined : options.command ?? config.share?.command;
|
|
2451
|
+
const args = provider === 'cloudflare-quick' ? undefined : options.args ?? config.share?.args;
|
|
2452
|
+
instance = await shareWorld(name, {
|
|
2453
|
+
...options,
|
|
2454
|
+
root,
|
|
2455
|
+
provider,
|
|
2456
|
+
command,
|
|
2457
|
+
args,
|
|
2458
|
+
ephemeral: options.ephemeral ?? config.share?.ephemeral,
|
|
2459
|
+
service: target.id,
|
|
2460
|
+
verifyPath: target.verifyPath,
|
|
2461
|
+
});
|
|
2462
|
+
}
|
|
2463
|
+
return instance;
|
|
2464
|
+
}
|
|
2465
|
+
/** TWIN-62: `unshare` must give tunnels the same contract plain `downWorld` already gives owned
|
|
2466
|
+
* services — SIGTERM, confirm dead (escalating to SIGKILL past the grace), and only THEN erase the
|
|
2467
|
+
* tunnel facts/pids. A best-effort SIGTERM with no confirmation can leave a `cloudflared` that
|
|
2468
|
+
* ignores SIGTERM serving the public URL forever while instance.json (and the pids file) say
|
|
2469
|
+
* nothing is shared — the exact "half-shared" state docs/concepts/worlds.md promises cannot exist. Stays
|
|
2470
|
+
* synchronous (via `killSurvivorsAfterGraceSync`) to match the existing signature/call sites
|
|
2471
|
+
* (the CLI does not await it). Returns the instance plus the tunnel pids that needed escalating,
|
|
2472
|
+
* for callers that want to report it. */
|
|
2473
|
+
export function unshareWorld(name, options = {}) {
|
|
2474
|
+
const root = resolve(options.root ?? process.cwd());
|
|
2475
|
+
const instance = readWorldInstance(name, root);
|
|
2476
|
+
const serviceIds = options.service ? [options.service] : Object.keys(instance.services);
|
|
2477
|
+
const escalated = [];
|
|
2478
|
+
const stillLive = [];
|
|
2479
|
+
for (const serviceId of serviceIds) {
|
|
2480
|
+
const service = instance.services[serviceId];
|
|
2481
|
+
if (!service)
|
|
2482
|
+
throw new Error(`World "${name}" has no service "${serviceId}"`);
|
|
2483
|
+
if (service.tunnel?.pid) {
|
|
2484
|
+
const pid = service.tunnel.pid;
|
|
2485
|
+
signalPids([pid], 'SIGTERM');
|
|
2486
|
+
const survivors = killSurvivorsAfterGraceSync([pid], options.graceMs ?? DOWN_GRACE_MS_DEFAULT);
|
|
2487
|
+
if (survivors.length > 0)
|
|
2488
|
+
escalated.push(...survivors);
|
|
2489
|
+
// `undeadPids` (not the raw `livePids`) is the right "confirmed dead" test here: a zombie
|
|
2490
|
+
// (exited but unreaped — likely since we just polled it synchronously via `Atomics.wait`,
|
|
2491
|
+
// which blocks the event loop that would otherwise let this process auto-reap its child) still
|
|
2492
|
+
// answers `kill(pid, 0)` yet is dead for every practical purpose — see `survivingOwnedGroups`' own
|
|
2493
|
+
// comment. Using plain `livePids` here would treat every zombie as "still shared".
|
|
2494
|
+
if (undeadPids([pid]).length > 0) {
|
|
2495
|
+
// Even SIGKILL didn't confirm it dead (e.g. no permission to signal it) — refuse to erase
|
|
2496
|
+
// the only record of this tunnel rather than orphan it with instance.json saying "unshared".
|
|
2497
|
+
stillLive.push(serviceId);
|
|
2498
|
+
continue;
|
|
2499
|
+
}
|
|
2500
|
+
}
|
|
2501
|
+
delete service.publicUrl;
|
|
2502
|
+
delete service.publicUrlEphemeral;
|
|
2503
|
+
delete service.publicReady;
|
|
2504
|
+
delete service.publicVerification;
|
|
2505
|
+
delete service.tunnel;
|
|
2506
|
+
}
|
|
2507
|
+
writePidsFromInstance(instance);
|
|
2508
|
+
saveWorldInstance(instance);
|
|
2509
|
+
if (stillLive.length > 0) {
|
|
2510
|
+
throw new Error(`World "${name}": tunnel process(es) for ${stillLive.join(', ')} could not be confirmed dead (even after SIGKILL) — refusing to erase their records`);
|
|
2511
|
+
}
|
|
2512
|
+
return Object.assign(instance, { escalated });
|
|
2513
|
+
}
|
|
2514
|
+
/** Short, hardcoded timeout for `doctorWorld`'s external re-probe (TWIN-68) — this is a health
|
|
2515
|
+
* CHECK, not a boot wait, so it must never make `doctor` hang: a dead external should fail FAST. */
|
|
2516
|
+
const DOCTOR_EXTERNAL_PROBE_TIMEOUT_MS = 3_000;
|
|
2517
|
+
/** Run an external service's declared readiness probe exactly ONCE (unlike `awaitReadiness`, which
|
|
2518
|
+
* polls until it passes or times out — appropriate at boot, wrong for a doctor health check, which
|
|
2519
|
+
* must report the CURRENT state, not wait for it to become healthy). Reuses the same probe shapes
|
|
2520
|
+
* (`command`/`httpUrl`/`stdoutMatch`) declared in config for the boot-time `readyWhen`. */
|
|
2521
|
+
async function probeExternalOnce(ready, cwd, env, logPath) {
|
|
2522
|
+
if (ready.command !== undefined) {
|
|
2523
|
+
if (!commandExists(ready.command)) {
|
|
2524
|
+
return { ok: false, message: `readiness command \`${ready.command}\` not found on PATH` };
|
|
2525
|
+
}
|
|
2526
|
+
const result = spawnSync(ready.command, ready.args ?? [], {
|
|
2527
|
+
cwd, encoding: 'utf8', env, timeout: DOCTOR_EXTERNAL_PROBE_TIMEOUT_MS, maxBuffer: 64 * 1024 * 1024,
|
|
2528
|
+
});
|
|
2529
|
+
if (result.status === 0)
|
|
2530
|
+
return { ok: true, message: 'probe ok (command)' };
|
|
2531
|
+
const timedOut = result.signal !== null && result.status === null;
|
|
2532
|
+
return { ok: false, message: timedOut ? `probe command timed out after ${DOCTOR_EXTERNAL_PROBE_TIMEOUT_MS}ms` : `probe command exited ${result.status ?? 'null'}: ${(result.stderr ?? '').trim().slice(0, 300)}` };
|
|
2533
|
+
}
|
|
2534
|
+
if (ready.httpUrl !== undefined) {
|
|
2535
|
+
try {
|
|
2536
|
+
const response = await fetch(ready.httpUrl, { signal: AbortSignal.timeout(DOCTOR_EXTERNAL_PROBE_TIMEOUT_MS) });
|
|
2537
|
+
if (response.status >= 200 && response.status < 400)
|
|
2538
|
+
return { ok: true, message: `probe ok (HTTP ${response.status})` };
|
|
2539
|
+
return { ok: false, message: `probe returned HTTP ${response.status}` };
|
|
2540
|
+
}
|
|
2541
|
+
catch (error) {
|
|
2542
|
+
return { ok: false, message: `probe failed: ${error instanceof Error ? error.message : String(error)}` };
|
|
2543
|
+
}
|
|
2544
|
+
}
|
|
2545
|
+
if (ready.stdoutMatch !== undefined) {
|
|
2546
|
+
const text = serviceOutput(readServiceText(logPath));
|
|
2547
|
+
if (new RegExp(ready.stdoutMatch).test(text))
|
|
2548
|
+
return { ok: true, message: 'probe ok (stdoutMatch)' };
|
|
2549
|
+
return { ok: false, message: `probe pattern /${ready.stdoutMatch}/ not found in log` };
|
|
2550
|
+
}
|
|
2551
|
+
return { ok: false, message: 'malformed readiness probe (no command/httpUrl/stdoutMatch)' };
|
|
2552
|
+
}
|
|
2553
|
+
/** A stopped service's own account for doctor: how it ended, its last output, and where its whole log is. */
|
|
2554
|
+
function stoppedServiceReport(pid, log) {
|
|
2555
|
+
const end = readServiceEnd(log);
|
|
2556
|
+
if (!end)
|
|
2557
|
+
return `pid ${pid} is not live\n log: ${log}`;
|
|
2558
|
+
const said = end.lastOutput.length ? `\n last output:\n${end.lastOutput.map((line) => ` ${line}`).join('\n')}` : '\n (no output in its last run)';
|
|
2559
|
+
return `pid ${pid} is not live: it ${describeServiceEnd(end)}${said}\n log: ${log}`;
|
|
2560
|
+
}
|
|
2561
|
+
export async function doctorWorld(name, options = {}) {
|
|
2562
|
+
const root = resolve(options.root ?? process.cwd());
|
|
2563
|
+
const checks = [];
|
|
2564
|
+
let instance;
|
|
2565
|
+
try {
|
|
2566
|
+
instance = readWorldInstance(name, root);
|
|
2567
|
+
checks.push({ id: 'instance', ok: true, message: `instance=${instanceFile(root, name)}` });
|
|
2568
|
+
}
|
|
2569
|
+
catch (error) {
|
|
2570
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2571
|
+
return { name, ok: false, checks: [{ id: 'instance', ok: false, message }] };
|
|
2572
|
+
}
|
|
2573
|
+
checks.push({ id: 'env-file', ok: existsSync(instance.envFile), message: instance.envFile });
|
|
2574
|
+
if (instance.lifecycle) {
|
|
2575
|
+
checks.push({ id: 'lifecycle', ok: hasWorldLifecycle(root, name) && existsSync(instance.lifecycle.log),
|
|
2576
|
+
message: `lifecycle ownership recorded; no capacity reserved; log=${instance.lifecycle.log}` });
|
|
2577
|
+
}
|
|
2578
|
+
const live = livePids(readPids(instance.pidsFile));
|
|
2579
|
+
checks.push({
|
|
2580
|
+
id: 'pids',
|
|
2581
|
+
ok: live.length > 0 || Object.values(instance.services).every(service => service.type === 'external'),
|
|
2582
|
+
message: live.length > 0 ? `live=${live.join(', ')}` : 'no process-backed services are running; external services are checked below',
|
|
2583
|
+
});
|
|
2584
|
+
// The redirect proxy every attached command's HTTPS goes through: a dead one refuses all of their egress.
|
|
2585
|
+
const proxy = readProxyState(root, name);
|
|
2586
|
+
if (proxy) {
|
|
2587
|
+
const proxyLive = livePids([proxy.pid]).length > 0;
|
|
2588
|
+
const proxyLog = proxyLogPath(root, name);
|
|
2589
|
+
checks.push({ id: 'proxy', ok: proxyLive, message: proxyLive ? `pid=${proxy.pid} log=${proxyLog}`
|
|
2590
|
+
: `proxy daemon pid ${proxy.pid} is not live: commands attached with its HTTPS_PROXY lose all HTTPS egress\n log: ${proxyLog}` });
|
|
2591
|
+
}
|
|
2592
|
+
// Loaded once, best-effort: only used to find each external service's declared readyWhen/status
|
|
2593
|
+
// probe (TWIN-68) — if the config can't be reloaded (moved/deleted since `up`), externals just
|
|
2594
|
+
// fall back to the "no probe declared" informational check rather than failing doctor entirely.
|
|
2595
|
+
// (Named distinctly from the `config` reloaded further below for the share-mode checks, which
|
|
2596
|
+
// intentionally keeps its own separate, non-best-effort load.)
|
|
2597
|
+
let externalProbeConfig;
|
|
2598
|
+
try {
|
|
2599
|
+
externalProbeConfig = loadWorldConfig(instance.configPath, instance.root).config;
|
|
2600
|
+
}
|
|
2601
|
+
catch {
|
|
2602
|
+
externalProbeConfig = undefined;
|
|
2603
|
+
}
|
|
2604
|
+
for (const service of Object.values(instance.services)) {
|
|
2605
|
+
if (service.type === 'external') {
|
|
2606
|
+
// We don't own the external tool's process or its port — report what it discovered, plus
|
|
2607
|
+
// (TWIN-68) re-run its declared probe once so a stack that died mid-session goes red instead
|
|
2608
|
+
// of reporting an unconditional ok:true forever.
|
|
2609
|
+
const discovered = Object.keys(service.env ?? {});
|
|
2610
|
+
const discoveredMessage = discovered.length > 0 ? `discovered ${discovered.join(', ')}` : 'self-managed (no discovered env)';
|
|
2611
|
+
const configuredExternal = externalProbeConfig?.services.find((candidate) => candidate.id === service.id)?.external;
|
|
2612
|
+
const probeCwd = service.external?.cwd ?? root;
|
|
2613
|
+
const probeEnv = { ...admittedEnv(stripEnvOf(instance.env ?? {})), ...(instance.env ?? {}), ...(service.external?.discoveredEnv ?? {}) };
|
|
2614
|
+
if (configuredExternal?.readyWhen) {
|
|
2615
|
+
const probe = await probeExternalOnce(configuredExternal.readyWhen, probeCwd, probeEnv, service.log);
|
|
2616
|
+
checks.push({ id: `service:${service.id}:external`, ok: probe.ok, message: `${discoveredMessage}; ${probe.message}` });
|
|
2617
|
+
}
|
|
2618
|
+
else if (configuredExternal?.status) {
|
|
2619
|
+
try {
|
|
2620
|
+
await runExternalCommand(configuredExternal.status, probeCwd, service.id, 'doctor-probe', probeEnv);
|
|
2621
|
+
checks.push({ id: `service:${service.id}:external`, ok: true, message: `${discoveredMessage}; probe ok (status)` });
|
|
2622
|
+
}
|
|
2623
|
+
catch (error) {
|
|
2624
|
+
checks.push({ id: `service:${service.id}:external`, ok: false, message: `${discoveredMessage}; probe failed: ${error instanceof Error ? error.message : String(error)}` });
|
|
2625
|
+
}
|
|
2626
|
+
}
|
|
2627
|
+
else {
|
|
2628
|
+
checks.push({ id: `service:${service.id}:external`, ok: true, message: `${discoveredMessage}; no probe declared — health unknown` });
|
|
2629
|
+
}
|
|
2630
|
+
continue;
|
|
2631
|
+
}
|
|
2632
|
+
const pidLive = livePids([service.pid]).length > 0;
|
|
2633
|
+
checks.push({
|
|
2634
|
+
id: `service:${service.id}:pid`,
|
|
2635
|
+
ok: pidLive,
|
|
2636
|
+
message: pidLive ? `pid=${service.pid}` : stoppedServiceReport(service.pid, service.log),
|
|
2637
|
+
});
|
|
2638
|
+
if (service.workerGaveUp) {
|
|
2639
|
+
// A worker-isolated twin whose Worker thread crashed: the host process (and its pid) is
|
|
2640
|
+
// still alive, but this twin's own worker was given up on (never restarted) — surface it
|
|
2641
|
+
// explicitly rather than let it hide behind an otherwise-generic tcp-refused check below.
|
|
2642
|
+
checks.push({
|
|
2643
|
+
id: `service:${service.id}:worker`,
|
|
2644
|
+
ok: false,
|
|
2645
|
+
message: `worker exited and was not restarted (${service.workerGaveUp.detail}), at ${service.workerGaveUp.at}`,
|
|
2646
|
+
});
|
|
2647
|
+
}
|
|
2648
|
+
try {
|
|
2649
|
+
await waitForTcp(service.port, 1_000);
|
|
2650
|
+
checks.push({ id: `service:${service.id}:tcp`, ok: true, message: service.url });
|
|
2651
|
+
// A port that accepts is not a service that answers (a stuck event loop still completes the handshake): the
|
|
2652
|
+
// readiness probe the service declared is run again, as it was at boot.
|
|
2653
|
+
// A log pattern matched at boot matches forever, so it cannot tell a hung service: only a probe that asks is re-run,
|
|
2654
|
+
// in the service's own cwd and with its PORT, as at boot.
|
|
2655
|
+
const configured = externalProbeConfig?.services.find((candidate) => candidate.id === service.id);
|
|
2656
|
+
const declared = configured?.ready;
|
|
2657
|
+
if (pidLive && declared && declared !== 'tcp' && declared.stdoutMatch === undefined) {
|
|
2658
|
+
const probe = await probeExternalOnce(resolveReadyProbe(declared, { host: '127.0.0.1', port: service.port, url: service.url }), configured?.cwd ? resolve(root, configured.cwd) : root, { ...admittedEnv(stripEnvOf(instance.env ?? {})), ...(instance.env ?? {}), ...(configured?.env ?? {}), PORT: String(service.port) }, service.log);
|
|
2659
|
+
checks.push({ id: `service:${service.id}:ready`, ok: probe.ok,
|
|
2660
|
+
message: probe.ok ? probe.message : `its port accepts but its declared readiness probe fails (${probe.message}): the service is up but not answering\n log: ${service.log}` });
|
|
2661
|
+
}
|
|
2662
|
+
}
|
|
2663
|
+
catch (error) {
|
|
2664
|
+
checks.push({
|
|
2665
|
+
id: `service:${service.id}:tcp`,
|
|
2666
|
+
ok: false,
|
|
2667
|
+
message: error instanceof Error ? error.message : String(error),
|
|
2668
|
+
});
|
|
2669
|
+
}
|
|
2670
|
+
if (service.tunnel || service.publicUrl) {
|
|
2671
|
+
const tunnelLive = service.tunnel ? livePids([service.tunnel.pid]).length > 0 : false;
|
|
2672
|
+
checks.push({
|
|
2673
|
+
id: `service:${service.id}:tunnel`,
|
|
2674
|
+
ok: tunnelLive,
|
|
2675
|
+
message: !service.tunnel ? 'missing tunnel process'
|
|
2676
|
+
: tunnelLive ? `pid=${service.tunnel.pid}`
|
|
2677
|
+
: `tunnel pid ${service.tunnel.pid} is not live; its public URL no longer reaches the service${service.tunnel.log ? `\n log: ${service.tunnel.log}` : ''}`,
|
|
2678
|
+
});
|
|
2679
|
+
checks.push({
|
|
2680
|
+
id: `service:${service.id}:public-ready`,
|
|
2681
|
+
ok: service.publicReady === true && (!service.tunnel || tunnelLive),
|
|
2682
|
+
message: service.publicUrl ?? 'missing public URL',
|
|
2683
|
+
});
|
|
2684
|
+
if (options.verifyPublic && service.publicUrl && service.publicVerification?.path) {
|
|
2685
|
+
try {
|
|
2686
|
+
const verification = await verifyPublicUrl(service.publicUrl, service.publicVerification.path, options.timeoutMs ?? 10_000);
|
|
2687
|
+
checks.push({
|
|
2688
|
+
id: `service:${service.id}:public-health`,
|
|
2689
|
+
ok: true,
|
|
2690
|
+
message: `${verification.status} ${service.publicUrl}${service.publicVerification.path}`,
|
|
2691
|
+
});
|
|
2692
|
+
}
|
|
2693
|
+
catch (error) {
|
|
2694
|
+
checks.push({
|
|
2695
|
+
id: `service:${service.id}:public-health`,
|
|
2696
|
+
ok: false,
|
|
2697
|
+
message: error instanceof Error ? error.message : String(error),
|
|
2698
|
+
});
|
|
2699
|
+
}
|
|
2700
|
+
}
|
|
2701
|
+
}
|
|
2702
|
+
}
|
|
2703
|
+
if (instance.mode === 'share') {
|
|
2704
|
+
const config = loadWorldConfig(instance.configPath, instance.root).config;
|
|
2705
|
+
const expected = config.share?.services.length ? config.share.services : [{ id: 'app' }];
|
|
2706
|
+
for (const target of expected) {
|
|
2707
|
+
const service = instance.services[target.id];
|
|
2708
|
+
checks.push({
|
|
2709
|
+
id: `share:${target.id}`,
|
|
2710
|
+
ok: service?.publicReady === true && Boolean(service.publicUrl),
|
|
2711
|
+
message: service?.publicUrl ?? 'not shared',
|
|
2712
|
+
});
|
|
2713
|
+
}
|
|
2714
|
+
}
|
|
2715
|
+
return { name: instance.name, ok: checks.every((check) => check.ok), checks };
|
|
2716
|
+
}
|
|
2717
|
+
export function urlsWorld(name, root = process.cwd()) {
|
|
2718
|
+
const instance = readWorldInstance(name, resolve(root));
|
|
2719
|
+
return {
|
|
2720
|
+
name: instance.name,
|
|
2721
|
+
mode: instance.mode,
|
|
2722
|
+
services: Object.fromEntries(Object.values(instance.services).map((service) => [
|
|
2723
|
+
service.id,
|
|
2724
|
+
{
|
|
2725
|
+
localUrl: service.url ?? '',
|
|
2726
|
+
...(service.publicUrl ? { publicUrl: service.publicUrl } : {}),
|
|
2727
|
+
...(service.publicReady !== undefined ? { publicReady: service.publicReady } : {}),
|
|
2728
|
+
},
|
|
2729
|
+
])),
|
|
2730
|
+
};
|
|
2731
|
+
}
|
|
2732
|
+
/**
|
|
2733
|
+
* The clean (unquoted) base URL(s) of a RUNNING world's services — the `volter-world url`
|
|
2734
|
+
* surface. Resolved from the generated instance, so callers never shell-parse `world.env`
|
|
2735
|
+
* (its values are single-quoted). One `{ id, url }` when `service` is named; every
|
|
2736
|
+
* URL-bearing service otherwise ('external' services have no assigned URL — their endpoints
|
|
2737
|
+
* are discovered env vars, so they are omitted from the list and refused by name).
|
|
2738
|
+
*/
|
|
2739
|
+
export function urlWorld(name, options = {}) {
|
|
2740
|
+
const status = statusWorld(name, options.root ?? process.cwd());
|
|
2741
|
+
if (!status.running) {
|
|
2742
|
+
throw new Error(`World ${name} is not running; start it with: volter-world up ${status.config} --env-file <path> --name ${name}`);
|
|
2743
|
+
}
|
|
2744
|
+
if (options.service === undefined) {
|
|
2745
|
+
return Object.values(status.services)
|
|
2746
|
+
.filter((service) => service.url)
|
|
2747
|
+
.map((service) => ({ id: service.id, url: service.url }));
|
|
2748
|
+
}
|
|
2749
|
+
const service = status.services[options.service];
|
|
2750
|
+
if (!service) {
|
|
2751
|
+
throw new Error(`World ${name} has no service "${options.service}"; it has: ${Object.keys(status.services).join(', ')}`);
|
|
2752
|
+
}
|
|
2753
|
+
if (!service.url) {
|
|
2754
|
+
throw new Error(`Service "${options.service}" has no assigned URL (type '${service.type}' endpoints are discovered env vars) — read them with: volter-world env ${name} -- env`);
|
|
2755
|
+
}
|
|
2756
|
+
return [{ id: service.id, url: service.url }];
|
|
2757
|
+
}
|
|
2758
|
+
export async function runWorld(configId, command, options = {}) {
|
|
2759
|
+
if (command.length === 0)
|
|
2760
|
+
throw new Error('Missing command after --');
|
|
2761
|
+
if (!options.keep)
|
|
2762
|
+
return superviseWorldRun(configId, command, options);
|
|
2763
|
+
return runWorldOwned(configId, command, options);
|
|
2764
|
+
}
|
|
2765
|
+
/** Internal task entry. The lifetime owner exists before admission/startup. Cancellation during
|
|
2766
|
+
* boot is latched and handled after startup settles, so no abandoned startup can create a late
|
|
2767
|
+
* service after teardown. Persistent --keep runs use the caller directly. */
|
|
2768
|
+
export async function runWorldOwned(configId, command, options, abort) {
|
|
2769
|
+
if (abort?.aborted)
|
|
2770
|
+
throw new Error('World run cancelled before admission');
|
|
2771
|
+
const instance = await upWorld(configId, options, abort);
|
|
2772
|
+
let outcome;
|
|
2773
|
+
let failure;
|
|
2774
|
+
let failed = false;
|
|
2775
|
+
let retired = true;
|
|
2776
|
+
try {
|
|
2777
|
+
outcome = await runWithWorldEnvLogged(instance.name, command, instance.root, { abort, verbose: options.verbose });
|
|
2778
|
+
instance.lastRun = outcome;
|
|
2779
|
+
saveWorldInstance(instance);
|
|
2780
|
+
}
|
|
2781
|
+
catch (error) {
|
|
2782
|
+
failure = error;
|
|
2783
|
+
failed = true;
|
|
2784
|
+
if (error instanceof CommandRetirementError) {
|
|
2785
|
+
retired = false;
|
|
2786
|
+
try {
|
|
2787
|
+
retainWorldLifecycleForCleanup(instance.root, instance.name);
|
|
2788
|
+
}
|
|
2789
|
+
catch { /* the live holder and unfinished foreground record remain authoritative */ }
|
|
2790
|
+
}
|
|
2791
|
+
}
|
|
2792
|
+
if (!options.keep && retired) {
|
|
2793
|
+
try {
|
|
2794
|
+
await finishTaskWorld(instance);
|
|
2795
|
+
}
|
|
2796
|
+
catch (error) {
|
|
2797
|
+
if (failed)
|
|
2798
|
+
throw new AggregateError([failure, error], `World execution and cleanup failed: ${String(failure)}; ${String(error)}`);
|
|
2799
|
+
throw error;
|
|
2800
|
+
}
|
|
2801
|
+
}
|
|
2802
|
+
if (failed)
|
|
2803
|
+
throw failure;
|
|
2804
|
+
return { instance, exitCode: outcome.exitCode, outcome: outcome };
|
|
2805
|
+
}
|
|
2806
|
+
async function finishTaskWorld(instance) {
|
|
2807
|
+
let waiting = false;
|
|
2808
|
+
let uncertainSince;
|
|
2809
|
+
for (;;) {
|
|
2810
|
+
try {
|
|
2811
|
+
const result = await downWorld(instance.name, instance.root, { expectedCreatedAt: instance.createdAt });
|
|
2812
|
+
if (result.externalErrors.length) {
|
|
2813
|
+
const message = `World "${instance.name}" cleanup incomplete: ${result.externalErrors.join('; ')}. Reservation and diagnostics retained.`;
|
|
2814
|
+
// The caller that would have read this on stderr may be gone; the lifecycle log keeps it.
|
|
2815
|
+
try {
|
|
2816
|
+
recordWorldLifecycleEvent(instance.root, instance.name, message);
|
|
2817
|
+
}
|
|
2818
|
+
catch { /* the error still propagates */ }
|
|
2819
|
+
throw new Error(message);
|
|
2820
|
+
}
|
|
2821
|
+
return;
|
|
2822
|
+
}
|
|
2823
|
+
catch (error) {
|
|
2824
|
+
if (!(error instanceof WorldConsumersActiveError))
|
|
2825
|
+
throw error;
|
|
2826
|
+
// Wait for attachments that are running; one that is only uncertain (a runner gone, another host) never
|
|
2827
|
+
// resolves by waiting, so the task ends and names what `consumers retire` must settle.
|
|
2828
|
+
const blocking = worldConsumers(instance.root, instance.name).consumers.filter((consumer) => consumer.state !== 'exited');
|
|
2829
|
+
const onlyUncertain = blocking.length > 0 && !blocking.some((consumer) => consumer.state === 'active');
|
|
2830
|
+
uncertainSince = onlyUncertain ? (uncertainSince ?? Date.now()) : undefined;
|
|
2831
|
+
// Two minutes, not one look: an attachment whose heartbeat lapsed while the machine slept reads uncertain until
|
|
2832
|
+
// its next beat.
|
|
2833
|
+
if (uncertainSince !== undefined && Date.now() - uncertainSince >= 120_000) {
|
|
2834
|
+
const message = `World "${instance.name}" cleanup waits on uncertain attachments (${blocking.map((consumer) => consumer.id).join(', ')}); settle them with \`volter-world consumers retire\`, then \`down\``;
|
|
2835
|
+
try {
|
|
2836
|
+
recordWorldLifecycleEvent(instance.root, instance.name, message);
|
|
2837
|
+
}
|
|
2838
|
+
catch { /* the error still propagates */ }
|
|
2839
|
+
throw new Error(message);
|
|
2840
|
+
}
|
|
2841
|
+
if (!waiting) {
|
|
2842
|
+
recordWorldLifecycleEvent(instance.root, instance.name, 'task ended; cleanup waiting for registered attachments');
|
|
2843
|
+
waiting = true;
|
|
2844
|
+
}
|
|
2845
|
+
await new Promise(resolve => setTimeout(resolve, CONSUMER_HEARTBEAT_MS));
|
|
2846
|
+
}
|
|
2847
|
+
}
|
|
2848
|
+
}
|
|
2849
|
+
export function listWorlds(root = process.cwd()) {
|
|
2850
|
+
const base = worldBaseDir(resolve(root));
|
|
2851
|
+
if (!existsSync(base))
|
|
2852
|
+
return [];
|
|
2853
|
+
return readdirSync(base, { withFileTypes: true })
|
|
2854
|
+
// World names must start with an alphanumeric (assertSafeWorldName), which is exactly why
|
|
2855
|
+
// the lock dir is dot-prefixed — so anything not world-NAMED is bookkeeping (`.locks`),
|
|
2856
|
+
// never a world, and must not be listed as one.
|
|
2857
|
+
.filter((entry) => entry.isDirectory() && /^[A-Za-z0-9]/.test(entry.name))
|
|
2858
|
+
.map((entry) => {
|
|
2859
|
+
try {
|
|
2860
|
+
const status = statusWorld(entry.name, root, { light: true });
|
|
2861
|
+
return { name: entry.name, running: status.running, degraded: status.degraded, config: status.config };
|
|
2862
|
+
}
|
|
2863
|
+
catch {
|
|
2864
|
+
return { name: entry.name, running: false };
|
|
2865
|
+
}
|
|
2866
|
+
})
|
|
2867
|
+
.sort((a, b) => a.name.localeCompare(b.name));
|
|
2868
|
+
}
|
|
2869
|
+
function worldAttachedCommandEnv(name, root, verbose = false) {
|
|
2870
|
+
const status = statusWorld(name, root);
|
|
2871
|
+
const sealed = status.env.VOLTER_WORLD_MODE === 'sealed' || status.env[WORLD_NETWORK_POLICY_ENV] !== undefined;
|
|
2872
|
+
const baseEnv = { ...admittedEnv(stripEnvOf(status.env)), ...status.env };
|
|
2873
|
+
let proxyEnv = {};
|
|
2874
|
+
try {
|
|
2875
|
+
const proxy = ensureWorldProxy(name, root, [admittedEnv(stripEnvOf(status.env)), status.env]);
|
|
2876
|
+
if (proxy?.url) {
|
|
2877
|
+
proxyEnv = proxy.env;
|
|
2878
|
+
}
|
|
2879
|
+
else if (sealed)
|
|
2880
|
+
throw new Error(`world "${name}" requires its network policy proxy: attachment proxy is unavailable; refusing to run the command`);
|
|
2881
|
+
}
|
|
2882
|
+
catch (error) {
|
|
2883
|
+
if (error instanceof CaTrustError)
|
|
2884
|
+
throw error;
|
|
2885
|
+
if (sealed)
|
|
2886
|
+
throw new Error(`world "${name}" requires its network policy proxy: attachment proxy failed; refusing to run the command (${error instanceof Error ? error.message : String(error)})`);
|
|
2887
|
+
// Keep env-only redirect if the ambient proxy cannot start.
|
|
2888
|
+
}
|
|
2889
|
+
return { instance: status, env: { ...baseEnv, ...proxyEnv, VOLTER_WORLD: name,
|
|
2890
|
+
VOLTER_TWIN_INJECT_QUIET: verbose ? '0' : baseEnv.VOLTER_TWIN_INJECT_QUIET ?? '1' } };
|
|
2891
|
+
}
|
|
2892
|
+
/** Runs the command in the world's env in the foreground and resolves to its exit code. The child is
|
|
2893
|
+
* awaited, not blocked on: a runtime that runs no synchronous child (the browser engine) still runs
|
|
2894
|
+
* this one, and the caller sees no difference — stdio is inherited either way. */
|
|
2895
|
+
export function runWithWorldEnv(name, command, root = process.cwd(), options = {}) {
|
|
2896
|
+
if (command.length === 0)
|
|
2897
|
+
throw new Error('Missing command after --');
|
|
2898
|
+
const attached = worldAttachedCommandEnv(name, root, options.verbose);
|
|
2899
|
+
return new Promise((resolveExit) => {
|
|
2900
|
+
const child = spawn(command[0], command.slice(1), {
|
|
2901
|
+
cwd: resolve(options.cwd ?? root),
|
|
2902
|
+
env: attached.env,
|
|
2903
|
+
stdio: 'inherit',
|
|
2904
|
+
});
|
|
2905
|
+
const forward = (signal) => { if (child.exitCode === null && child.signalCode === null)
|
|
2906
|
+
child.kill(signal); };
|
|
2907
|
+
process.on('SIGINT', forward);
|
|
2908
|
+
process.on('SIGTERM', forward);
|
|
2909
|
+
child.once('error', () => { offSignal('SIGINT', forward); offSignal('SIGTERM', forward); resolveExit(1); });
|
|
2910
|
+
child.once('exit', (code, signal) => { offSignal('SIGINT', forward); offSignal('SIGTERM', forward); resolveExit(code ?? (signal ? 1 : 0)); });
|
|
2911
|
+
});
|
|
2912
|
+
}
|
|
2913
|
+
/**
|
|
2914
|
+
* The owner's explicit resolution of uncertain attachments (never automatic, never implied by age).
|
|
2915
|
+
* A record is retired only when it belongs to this host and the running instance, its runner and
|
|
2916
|
+
* its command are both gone (ESRCH), and no live process remains in the command's process group
|
|
2917
|
+
* (`survivingOwnedGroups`, the lifecycle's own retirement evidence); anything else is kept with the
|
|
2918
|
+
* reason, surviving members named by their group. Active consumers are never touched.
|
|
2919
|
+
*/
|
|
2920
|
+
export function retireWorldConsumers(name, root = process.cwd(), options = {}) {
|
|
2921
|
+
assertSafeWorldName(name);
|
|
2922
|
+
const resolvedRoot = resolve(root);
|
|
2923
|
+
return withFileLock(instanceLockFile(resolvedRoot, name), () => {
|
|
2924
|
+
const current = statusWorld(name, resolvedRoot);
|
|
2925
|
+
const { consumers } = worldConsumers(resolvedRoot, name, current.createdAt);
|
|
2926
|
+
const chosen = consumers.filter(c => c.state === 'uncertain' && (options.consumer === undefined || c.id === options.consumer));
|
|
2927
|
+
if (options.consumer !== undefined && chosen.length === 0)
|
|
2928
|
+
throw new Error(`World "${name}" has no uncertain consumer ${options.consumer}`);
|
|
2929
|
+
const gone = (pid) => { try {
|
|
2930
|
+
process.kill(pid, 0);
|
|
2931
|
+
return false;
|
|
2932
|
+
}
|
|
2933
|
+
catch (error) {
|
|
2934
|
+
return error.code === 'ESRCH';
|
|
2935
|
+
} };
|
|
2936
|
+
return chosen.map((c) => {
|
|
2937
|
+
const keep = (reason) => ({ id: c.id, owner: c.owner, retired: false, reason });
|
|
2938
|
+
if (c.hostname !== osHostname())
|
|
2939
|
+
return keep(`recorded on host ${c.hostname}; retire it there`);
|
|
2940
|
+
if (c.instanceCreatedAt !== current.createdAt)
|
|
2941
|
+
return keep('belongs to another instance of this World');
|
|
2942
|
+
if (!gone(c.runnerPid))
|
|
2943
|
+
return keep(`runner ${c.runnerPid} is still alive`);
|
|
2944
|
+
if (c.consumerPid === undefined)
|
|
2945
|
+
return keep('its command was never recorded as started, so its processes cannot be proven gone');
|
|
2946
|
+
if (!gone(c.consumerPid))
|
|
2947
|
+
return keep(`command ${c.consumerPid} is still alive`);
|
|
2948
|
+
const surviving = survivingOwnedGroups([c.consumerPid]);
|
|
2949
|
+
if (surviving.length)
|
|
2950
|
+
return keep(`process group ${c.consumerPid} still has live members`);
|
|
2951
|
+
removeConsumerRecord(resolvedRoot, name, c.id);
|
|
2952
|
+
return { id: c.id, owner: c.owner, retired: true, reason: `runner ${c.runnerPid}, command ${c.consumerPid} and its process group are gone` };
|
|
2953
|
+
});
|
|
2954
|
+
});
|
|
2955
|
+
}
|
|
2956
|
+
export async function attachWorld(name, command, root = process.cwd(), options = {}) {
|
|
2957
|
+
if (command.length === 0)
|
|
2958
|
+
throw new Error('Missing command after --');
|
|
2959
|
+
assertSafeWorldName(name);
|
|
2960
|
+
const resolvedRoot = resolve(root);
|
|
2961
|
+
checkedOwner(options.owner);
|
|
2962
|
+
const attached = worldAttachedCommandEnv(name, resolvedRoot, options.verbose);
|
|
2963
|
+
const record = withFileLock(instanceLockFile(resolvedRoot, name), () => {
|
|
2964
|
+
if (existsSync(stoppingFile(resolvedRoot, name)) || existsSync(bootingFile(resolvedRoot, name))) {
|
|
2965
|
+
throw new Error(`World "${name}" is changing lifecycle; retry after it finishes`);
|
|
2966
|
+
}
|
|
2967
|
+
const current = statusWorld(name, resolvedRoot);
|
|
2968
|
+
if (!current.running || current.createdAt !== attached.instance.createdAt)
|
|
2969
|
+
throw new Error(`World "${name}" is stopped or changed while attaching`);
|
|
2970
|
+
// Attaching to a World whose services died is allowed (a command may not need them), but never silent.
|
|
2971
|
+
for (const [id, state] of Object.entries(current.serviceStates)) {
|
|
2972
|
+
if (state === 'running' || state === 'external')
|
|
2973
|
+
continue;
|
|
2974
|
+
const service = current.services[id];
|
|
2975
|
+
const end = service?.log ? readServiceEnd(service.log) : undefined;
|
|
2976
|
+
process.stderr.write(`World "${name}": service ${id} is not running${end ? `: it ${describeServiceEnd(end)}` : ''}${service?.log ? `\n log: ${service.log}` : ''}\n`);
|
|
2977
|
+
}
|
|
2978
|
+
return createConsumer(resolvedRoot, name, current.createdAt, 'attach', options.owner ?? current.lifecycle?.owner ?? current.resources?.owner, { completionRequired: true });
|
|
2979
|
+
});
|
|
2980
|
+
let timer;
|
|
2981
|
+
let retired = true;
|
|
2982
|
+
try {
|
|
2983
|
+
return await new Promise((done, reject) => {
|
|
2984
|
+
const child = spawn(command[0], command.slice(1), { cwd: resolve(options.cwd ?? root), env: attached.env, stdio: 'inherit', detached: commandProcessGroup });
|
|
2985
|
+
const lifetime = commandLifetime(child);
|
|
2986
|
+
let failure;
|
|
2987
|
+
record.consumerPid = child.pid;
|
|
2988
|
+
try {
|
|
2989
|
+
refreshConsumer(resolvedRoot, name, record);
|
|
2990
|
+
}
|
|
2991
|
+
catch (error) {
|
|
2992
|
+
failure = error;
|
|
2993
|
+
lifetime.cancel('SIGTERM');
|
|
2994
|
+
}
|
|
2995
|
+
timer = setInterval(() => {
|
|
2996
|
+
try {
|
|
2997
|
+
refreshConsumer(resolvedRoot, name, record);
|
|
2998
|
+
}
|
|
2999
|
+
catch { /* the stale heartbeat reports uncertain; never implies permission to clean up */ }
|
|
3000
|
+
}, CONSUMER_HEARTBEAT_MS);
|
|
3001
|
+
timer.unref();
|
|
3002
|
+
child.once('error', (error) => { failure = error; });
|
|
3003
|
+
void lifetime.finish().then(({ code, signal }) => {
|
|
3004
|
+
if (failure)
|
|
3005
|
+
reject(failure);
|
|
3006
|
+
else
|
|
3007
|
+
done(signal || lifetime.forwarded ? signalExitCode(signal ?? lifetime.forwarded) : code ?? 1);
|
|
3008
|
+
}, error => {
|
|
3009
|
+
retired = false;
|
|
3010
|
+
record.retirementUncertain = true;
|
|
3011
|
+
try {
|
|
3012
|
+
refreshConsumer(resolvedRoot, name, record);
|
|
3013
|
+
}
|
|
3014
|
+
catch { /* pre-spawn completion requirement already retains uncertainty */ }
|
|
3015
|
+
reject(error);
|
|
3016
|
+
});
|
|
3017
|
+
});
|
|
3018
|
+
}
|
|
3019
|
+
finally {
|
|
3020
|
+
if (timer)
|
|
3021
|
+
clearInterval(timer);
|
|
3022
|
+
if (retired)
|
|
3023
|
+
releaseConsumer(resolvedRoot, name, record);
|
|
3024
|
+
}
|
|
3025
|
+
}
|
|
3026
|
+
function signalExitCode(signal) {
|
|
3027
|
+
if (signal === null)
|
|
3028
|
+
return 1;
|
|
3029
|
+
return 128 + (osConstants.signals[signal] ?? 0);
|
|
3030
|
+
}
|
|
3031
|
+
/** `world run`'s foreground consumer: stream output to the terminal while retaining one current
|
|
3032
|
+
* World-visible log, then return a structured outcome even when spawning fails or a signal kills
|
|
3033
|
+
* the consumer. The record intentionally contains neither argv nor environment values. */
|
|
3034
|
+
async function runWithWorldEnvLogged(name, command, root, options = {}) {
|
|
3035
|
+
const startedAt = new Date().toISOString();
|
|
3036
|
+
const instance = statusWorld(name, root);
|
|
3037
|
+
const log = join(instance.dirs.logs, 'foreground.log');
|
|
3038
|
+
const running = { state: 'running', runnerPid: process.pid, startedAt, log };
|
|
3039
|
+
writeTextAtomic(foregroundRunFile(instance.root, instance.name), `${JSON.stringify(running, null, 2)}\n`);
|
|
3040
|
+
const logFd = openPrivateLog(log, 'w');
|
|
3041
|
+
try {
|
|
3042
|
+
writeSync(logFd, `World ${name}: foreground consumer started at ${startedAt}\n`);
|
|
3043
|
+
}
|
|
3044
|
+
catch (error) {
|
|
3045
|
+
closeSync(logFd);
|
|
3046
|
+
throw error;
|
|
3047
|
+
}
|
|
3048
|
+
const finish = (exitCode, signal, error) => {
|
|
3049
|
+
const finishedAt = new Date().toISOString();
|
|
3050
|
+
const detail = error ? `spawn error: ${error}` : signal ? `terminated by ${signal}` : `exited ${exitCode}`;
|
|
3051
|
+
try {
|
|
3052
|
+
writeSync(logFd, `\nWorld ${name}: foreground consumer ${detail} at ${finishedAt}\n`);
|
|
3053
|
+
}
|
|
3054
|
+
finally {
|
|
3055
|
+
closeSync(logFd);
|
|
3056
|
+
}
|
|
3057
|
+
const outcome = {
|
|
3058
|
+
state: 'completed',
|
|
3059
|
+
runnerPid: process.pid,
|
|
3060
|
+
startedAt,
|
|
3061
|
+
finishedAt,
|
|
3062
|
+
exitCode,
|
|
3063
|
+
log,
|
|
3064
|
+
...(signal ? { signal } : {}),
|
|
3065
|
+
...(error ? { error } : {}),
|
|
3066
|
+
};
|
|
3067
|
+
writeTextAtomic(foregroundRunFile(instance.root, instance.name), `${JSON.stringify(outcome, null, 2)}\n`);
|
|
3068
|
+
return outcome;
|
|
3069
|
+
};
|
|
3070
|
+
if (options.abort?.aborted) {
|
|
3071
|
+
const signal = options.abort.reason === 'SIGINT' ? 'SIGINT' : 'SIGTERM';
|
|
3072
|
+
return finish(signalExitCode(signal), signal);
|
|
3073
|
+
}
|
|
3074
|
+
let attached;
|
|
3075
|
+
try {
|
|
3076
|
+
attached = worldAttachedCommandEnv(name, root, options.verbose);
|
|
3077
|
+
}
|
|
3078
|
+
catch (cause) {
|
|
3079
|
+
return finish(1, undefined, cause instanceof Error ? cause.message : String(cause));
|
|
3080
|
+
}
|
|
3081
|
+
return await new Promise((resolveOutcome, rejectOutcome) => {
|
|
3082
|
+
let spawnError;
|
|
3083
|
+
let child;
|
|
3084
|
+
try {
|
|
3085
|
+
child = spawn(command[0], command.slice(1), {
|
|
3086
|
+
cwd: resolve(options.cwd ?? root),
|
|
3087
|
+
env: attached.env,
|
|
3088
|
+
stdio: ['inherit', 'pipe', 'pipe'],
|
|
3089
|
+
detached: commandProcessGroup,
|
|
3090
|
+
});
|
|
3091
|
+
}
|
|
3092
|
+
catch (cause) {
|
|
3093
|
+
resolveOutcome(finish(1, undefined, cause instanceof Error ? cause.message : String(cause)));
|
|
3094
|
+
return;
|
|
3095
|
+
}
|
|
3096
|
+
const lifetime = commandLifetime(child, options.abort);
|
|
3097
|
+
child.stdout.on('data', (chunk) => {
|
|
3098
|
+
try {
|
|
3099
|
+
process.stdout.write(chunk);
|
|
3100
|
+
writeSync(logFd, chunk);
|
|
3101
|
+
}
|
|
3102
|
+
catch (error) {
|
|
3103
|
+
spawnError = String(error);
|
|
3104
|
+
lifetime.cancel('SIGTERM');
|
|
3105
|
+
}
|
|
3106
|
+
});
|
|
3107
|
+
child.stderr.on('data', (chunk) => {
|
|
3108
|
+
try {
|
|
3109
|
+
process.stderr.write(chunk);
|
|
3110
|
+
writeSync(logFd, chunk);
|
|
3111
|
+
}
|
|
3112
|
+
catch (error) {
|
|
3113
|
+
spawnError = String(error);
|
|
3114
|
+
lifetime.cancel('SIGTERM');
|
|
3115
|
+
}
|
|
3116
|
+
});
|
|
3117
|
+
child.once('error', (cause) => {
|
|
3118
|
+
spawnError = cause.message;
|
|
3119
|
+
});
|
|
3120
|
+
const activeRecord = {
|
|
3121
|
+
state: 'running',
|
|
3122
|
+
runnerPid: process.pid,
|
|
3123
|
+
...(child.pid ? { consumerPid: child.pid } : {}),
|
|
3124
|
+
startedAt,
|
|
3125
|
+
log,
|
|
3126
|
+
};
|
|
3127
|
+
try {
|
|
3128
|
+
writeTextAtomic(foregroundRunFile(instance.root, instance.name), `${JSON.stringify(activeRecord, null, 2)}\n`);
|
|
3129
|
+
}
|
|
3130
|
+
catch (error) {
|
|
3131
|
+
spawnError = String(error);
|
|
3132
|
+
lifetime.cancel('SIGTERM');
|
|
3133
|
+
}
|
|
3134
|
+
void lifetime.finish().then(({ code, signal }) => {
|
|
3135
|
+
const effectiveSignal = signal ?? lifetime.forwarded;
|
|
3136
|
+
const exitCode = effectiveSignal ? signalExitCode(effectiveSignal) : (code ?? 1);
|
|
3137
|
+
try {
|
|
3138
|
+
resolveOutcome(finish(spawnError ? 1 : exitCode, effectiveSignal, spawnError));
|
|
3139
|
+
}
|
|
3140
|
+
catch (error) {
|
|
3141
|
+
rejectOutcome(error);
|
|
3142
|
+
}
|
|
3143
|
+
}, error => { closeSync(logFd); rejectOutcome(error); });
|
|
3144
|
+
});
|
|
3145
|
+
}
|
|
3146
|
+
/** The shell + env + cwd a `volter-world shell` subshell launches with (extracted for testability):
|
|
3147
|
+
* the world env (twin `*_URL`s + fake keys + the Node injector + cliRedirect endpoint vars) on top
|
|
3148
|
+
* of the caller's env, plus `VOLTER_WORLD` so a prompt can show which world is active. The ambient
|
|
3149
|
+
* TLS-proxy env (HTTPS_PROXY + the CA-trust vars) is layered in by `shellWorld` once the proxy is up,
|
|
3150
|
+
* since starting a server is async; this pure helper stays synchronous + side-effect-free for tests. */
|
|
3151
|
+
export function worldShellEnv(name, root = process.cwd()) {
|
|
3152
|
+
const status = statusWorld(name, root);
|
|
3153
|
+
if (!status.running)
|
|
3154
|
+
throw new Error(`World "${name}" is not running — run: volter-world up <config> --env-file <path> --name ${name}`);
|
|
3155
|
+
return {
|
|
3156
|
+
shell: status.env.SHELL || admittedEnv(stripEnvOf(status.env)).SHELL || '/bin/bash',
|
|
3157
|
+
cwd: resolve(root),
|
|
3158
|
+
env: { ...admittedEnv(stripEnvOf(status.env)), ...status.env, VOLTER_WORLD: name },
|
|
3159
|
+
};
|
|
3160
|
+
}
|
|
3161
|
+
/** Drop into an interactive subshell with the world active — the contained sibling of `activate`
|
|
3162
|
+
* (which wires your CURRENT shell + sets the `(world:<name>)` prompt marker). The subshell gets the
|
|
3163
|
+
* world env AND, when openssl is available, an in-process ambient TLS redirect proxy: unmodified
|
|
3164
|
+
* `gh`/`stripe`/`curl`/app in the subshell transparently hit the twins. Vendor calls in this subshell
|
|
3165
|
+
* land in the twins; `exit` restores normal (the proxy + its session CA are torn down). The CA is
|
|
3166
|
+
* trusted ONLY via this subshell's env — never installed system-wide. See docs/guides/route-a-cli-through-the-world.md. */
|
|
3167
|
+
export async function shellWorld(name, root = process.cwd()) {
|
|
3168
|
+
const { shell, cwd, env } = worldShellEnv(name, root);
|
|
3169
|
+
const resolvedRoot = resolve(root);
|
|
3170
|
+
let proxy = null;
|
|
3171
|
+
const sealed = env.VOLTER_WORLD_MODE === 'sealed' || env[WORLD_NETWORK_POLICY_ENV] !== undefined;
|
|
3172
|
+
if (opensslAvailable()) {
|
|
3173
|
+
try {
|
|
3174
|
+
proxy = await startRedirectProxy({ env, trustEnv: [admittedEnv(stripEnvOf(env)), env], tlsDir: tlsDir(resolvedRoot, name) });
|
|
3175
|
+
Object.assign(env, proxy.proxyEnv());
|
|
3176
|
+
process.stderr.write(`world '${name}' active in a subshell — vendor calls hit the twins (ambient proxy ${proxy.url}). type 'exit' to leave.\n`);
|
|
3177
|
+
}
|
|
3178
|
+
catch (error) {
|
|
3179
|
+
if (error instanceof CaTrustError)
|
|
3180
|
+
throw error;
|
|
3181
|
+
if (sealed)
|
|
3182
|
+
throw new Error(`world "${name}" requires its network policy proxy: attachment proxy failed; refusing to open a shell (${error instanceof Error ? error.message : String(error)})`);
|
|
3183
|
+
process.stderr.write(`world '${name}': ambient TLS proxy unavailable (${error instanceof Error ? error.message : String(error)}); falling back to env-only redirect.\n`);
|
|
3184
|
+
}
|
|
3185
|
+
}
|
|
3186
|
+
else {
|
|
3187
|
+
if (sealed)
|
|
3188
|
+
throw new Error(`world "${name}" requires its network policy proxy: openssl is unavailable; refusing to open a shell`);
|
|
3189
|
+
process.stderr.write(`world '${name}' active in a subshell — vendor calls hit the twins (env-only; install openssl for ambient https redirect). type 'exit' to leave.\n`);
|
|
3190
|
+
}
|
|
3191
|
+
try {
|
|
3192
|
+
const result = spawnSync(shell, ['-i'], { cwd, env, stdio: 'inherit' });
|
|
3193
|
+
return result.status ?? 0;
|
|
3194
|
+
}
|
|
3195
|
+
finally {
|
|
3196
|
+
if (proxy) {
|
|
3197
|
+
await proxy.close();
|
|
3198
|
+
tearDownCa(tlsDir(resolvedRoot, name));
|
|
3199
|
+
}
|
|
3200
|
+
}
|
|
3201
|
+
}
|
|
3202
|
+
/** The daemon-poll deadline (ms) used by `ensureWorldProxy`/`ensureWorldProxyFromEnv` below. Hardcoded
|
|
3203
|
+
* to 5000ms in production; `VOLTER_PROXY_DEADLINE_MS` is a TEST-ONLY seam so tests can force a
|
|
3204
|
+
* deterministic timeout without an actual multi-second wait. Never set this env var in real usage. */
|
|
3205
|
+
function proxyDaemonDeadlineMs() {
|
|
3206
|
+
const raw = process.env.VOLTER_PROXY_DEADLINE_MS;
|
|
3207
|
+
if (raw === undefined)
|
|
3208
|
+
return 5000;
|
|
3209
|
+
const parsed = Number(raw);
|
|
3210
|
+
return Number.isFinite(parsed) && parsed >= 0 ? parsed : 5000;
|
|
3211
|
+
}
|
|
3212
|
+
/** Ensure a DETACHED ambient-redirect proxy daemon is running for this world (used by `activate`,
|
|
3213
|
+
* which only emits a script and can't host an in-process server for the life of the shell). Idempotent:
|
|
3214
|
+
* reuses a live daemon. Returns its proxy env, or null if openssl is missing (env-only fallback). The
|
|
3215
|
+
* daemon is recorded in `proxy.json` and stopped by `down`. */
|
|
3216
|
+
export function ensureWorldProxy(name, root = process.cwd(), caller = process.env) {
|
|
3217
|
+
assertSafeWorldName(name);
|
|
3218
|
+
const resolvedRoot = resolve(root);
|
|
3219
|
+
return withFileLock(instanceLockFile(resolvedRoot, name), () => {
|
|
3220
|
+
assertProxyOwnershipResolved(resolvedRoot, name);
|
|
3221
|
+
if (liveBootingClaim(stoppingFile(resolvedRoot, name)))
|
|
3222
|
+
throw new Error(`World "${name}" is being stopped`);
|
|
3223
|
+
if (liveBootingClaim(bootingFile(resolvedRoot, name)))
|
|
3224
|
+
throw new Error(`World "${name}" is being booted`);
|
|
3225
|
+
if (!statusWorld(name, resolvedRoot, { readOnly: true }).running)
|
|
3226
|
+
throw new Error(`World "${name}" is not running`);
|
|
3227
|
+
return ensureWorldProxyUnlocked(name, resolvedRoot, caller);
|
|
3228
|
+
});
|
|
3229
|
+
}
|
|
3230
|
+
/** The World's live proxy daemon, or null. One from before its guests' listener (its state names none) keeps a World
|
|
3231
|
+
* Machine on its host's network, so it is stopped (its CA kept, which the next daemon reuses) and a new one started. */
|
|
3232
|
+
function currentProxyState(root, name) {
|
|
3233
|
+
const existing = readProxyState(root, name);
|
|
3234
|
+
if (!existing || !isAlive(existing.pid))
|
|
3235
|
+
return null;
|
|
3236
|
+
if (existing.guestUrl)
|
|
3237
|
+
return existing;
|
|
3238
|
+
try {
|
|
3239
|
+
process.kill(existing.pid, 'SIGTERM');
|
|
3240
|
+
}
|
|
3241
|
+
catch { /* gone already */ }
|
|
3242
|
+
for (let i = 0; i < 50 && isAlive(existing.pid); i += 1)
|
|
3243
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 100);
|
|
3244
|
+
try {
|
|
3245
|
+
rmSync(proxyStateFile(root, name), { force: true });
|
|
3246
|
+
}
|
|
3247
|
+
catch { /* ignore */ }
|
|
3248
|
+
return null;
|
|
3249
|
+
}
|
|
3250
|
+
function ensureWorldProxyUnlocked(name, root, caller) {
|
|
3251
|
+
const resolvedRoot = resolve(root);
|
|
3252
|
+
if (!opensslAvailable())
|
|
3253
|
+
return null;
|
|
3254
|
+
const existing = currentProxyState(resolvedRoot, name);
|
|
3255
|
+
if (existing && isAlive(existing.pid)) {
|
|
3256
|
+
return { url: existing.url, caCertPath: existing.caCertPath, env: proxyEnvFor(existing.url, existing.caCertPath, caller, existing.guestUrl) };
|
|
3257
|
+
}
|
|
3258
|
+
// Pre-create the CA synchronously so we can return its path immediately; the daemon reuses it.
|
|
3259
|
+
const ca = ensureCa(tlsDir(resolvedRoot, name));
|
|
3260
|
+
return startProxyDaemonOwned(name, resolvedRoot, ca.caCert, undefined, undefined, caller);
|
|
3261
|
+
}
|
|
3262
|
+
function ensureWorldProxyFromEnv(name, root, env, boot) {
|
|
3263
|
+
const resolvedRoot = resolve(root);
|
|
3264
|
+
if (!opensslAvailable())
|
|
3265
|
+
return null;
|
|
3266
|
+
writeProxyEnv(resolvedRoot, name, env);
|
|
3267
|
+
const existing = currentProxyState(resolvedRoot, name);
|
|
3268
|
+
if (existing && isAlive(existing.pid)) {
|
|
3269
|
+
return { url: existing.url, caCertPath: existing.caCertPath, env: proxyEnvFor(existing.url, existing.caCertPath, [admittedEnv(stripEnvOf(env)), env], existing.guestUrl) };
|
|
3270
|
+
}
|
|
3271
|
+
const ca = ensureCa(tlsDir(resolvedRoot, name));
|
|
3272
|
+
return startProxyDaemonOwned(name, resolvedRoot, ca.caCert, proxyEnvFile(resolvedRoot, name), boot, [admittedEnv(stripEnvOf(env)), env]);
|
|
3273
|
+
}
|
|
3274
|
+
function startProxyDaemonOwned(name, root, caCertPath, envFile, boot, caller = process.env) {
|
|
3275
|
+
sessionTrustEnv(caCertPath, caller); // Fail in the parent before an invalid trust input can become a daemon timeout.
|
|
3276
|
+
const marker = proxyStartingFile(root, name);
|
|
3277
|
+
// Write ahead of spawn: even permanently failed PID publication must leave durable evidence.
|
|
3278
|
+
// The instance lock (activate) or boot claim excludes a competing startup or teardown.
|
|
3279
|
+
const createdAt = readWorldInstance(name, root, true).createdAt;
|
|
3280
|
+
const markerFd = openSync(marker, 'wx', 0o600); // EEXIST must never remove another intent
|
|
3281
|
+
try {
|
|
3282
|
+
try {
|
|
3283
|
+
writeFileSync(markerFd, JSON.stringify({ token: randomUUID(), ownerPid: process.pid, createdAt }));
|
|
3284
|
+
}
|
|
3285
|
+
finally {
|
|
3286
|
+
closeSync(markerFd);
|
|
3287
|
+
}
|
|
3288
|
+
}
|
|
3289
|
+
catch (error) {
|
|
3290
|
+
rmSync(marker, { force: true }); // partial pre-spawn writes own no child
|
|
3291
|
+
throw error;
|
|
3292
|
+
}
|
|
3293
|
+
let child;
|
|
3294
|
+
try {
|
|
3295
|
+
retainWorldLifecycleForCleanup(root, name);
|
|
3296
|
+
// Its own log (a startup error, a crash, the refusals it answers): with no output the daemon's failures left nothing.
|
|
3297
|
+
mkdirSync(dirname(proxyLogPath(root, name)), { recursive: true, mode: 0o700 });
|
|
3298
|
+
const proxyOut = openPrivateLog(proxyLogPath(root, name), 'a');
|
|
3299
|
+
try {
|
|
3300
|
+
child = spawn(process.execPath, [proxyDaemonEntry(), name, '--root', root, ...(envFile ? [`--env-file=${envFile}`] : [])], {
|
|
3301
|
+
cwd: root, env: process.env, detached: true, stdio: ['ignore', proxyOut, proxyOut],
|
|
3302
|
+
});
|
|
3303
|
+
}
|
|
3304
|
+
finally {
|
|
3305
|
+
closeSync(proxyOut);
|
|
3306
|
+
}
|
|
3307
|
+
}
|
|
3308
|
+
catch (error) {
|
|
3309
|
+
rmSync(marker, { force: true }); // no child was returned by spawn
|
|
3310
|
+
throw error;
|
|
3311
|
+
}
|
|
3312
|
+
child.on('error', () => { }); // a failed spawn can emit after this synchronous operation throws
|
|
3313
|
+
const pid = child.pid;
|
|
3314
|
+
if (!pid || !Number.isInteger(pid) || pid <= 0) {
|
|
3315
|
+
rmSync(marker, { force: true });
|
|
3316
|
+
throw new Error(`World "${name}": ambient proxy process did not start`);
|
|
3317
|
+
}
|
|
3318
|
+
boot?.pids.add(pid); // before any fallible filesystem operation
|
|
3319
|
+
child.unref();
|
|
3320
|
+
let retirementAttempted = false;
|
|
3321
|
+
const retire = () => {
|
|
3322
|
+
retirementAttempted = true;
|
|
3323
|
+
signalPids([pid], 'SIGTERM');
|
|
3324
|
+
killSurvivorsAfterGraceSync([pid], DOWN_GRACE_MS_DEFAULT);
|
|
3325
|
+
if (undeadPids([pid]).length) {
|
|
3326
|
+
const error = new CommandRetirementError(`World "${name}": proxy process ${pid} retirement is unconfirmed; ownership retained`);
|
|
3327
|
+
try {
|
|
3328
|
+
retainWorldLifecycleForCleanup(root, name);
|
|
3329
|
+
}
|
|
3330
|
+
catch (cause) {
|
|
3331
|
+
error.message += `; claim update failed: ${String(cause)}`;
|
|
3332
|
+
}
|
|
3333
|
+
throw error;
|
|
3334
|
+
}
|
|
3335
|
+
// Publication can happen during the grace period. Erase only this retired child's state.
|
|
3336
|
+
if (readProxyState(root, name)?.pid === pid)
|
|
3337
|
+
rmSync(proxyStateFile(root, name), { force: true });
|
|
3338
|
+
rmSync(marker, { force: true });
|
|
3339
|
+
};
|
|
3340
|
+
try {
|
|
3341
|
+
if (boot)
|
|
3342
|
+
boot.recordPids();
|
|
3343
|
+
else
|
|
3344
|
+
writeOwnedPids(root, name, [pid], [pid]);
|
|
3345
|
+
rmSync(marker, { force: true }); // canonical ownership is now durable
|
|
3346
|
+
const deadlineMs = proxyDaemonDeadlineMs();
|
|
3347
|
+
const deadline = Date.now() + deadlineMs;
|
|
3348
|
+
while (Date.now() < deadline) {
|
|
3349
|
+
const state = readProxyState(root, name);
|
|
3350
|
+
if (state?.pid === pid) {
|
|
3351
|
+
if (!boot)
|
|
3352
|
+
handoffWorldLifecycle(root, name, readPids(join(instanceDir(root, name), 'pids')));
|
|
3353
|
+
return { url: state.url, caCertPath: state.caCertPath, env: proxyEnvFor(state.url, state.caCertPath, caller, state.guestUrl) };
|
|
3354
|
+
}
|
|
3355
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50);
|
|
3356
|
+
}
|
|
3357
|
+
retire();
|
|
3358
|
+
process.stderr.write(`!! WARN: ambient proxy daemon for world "${name}" did not report within ${deadlineMs}ms — world is NOT proxy-sealed; unmodified CLIs will bypass the twins (env-only redirect still active).\n`);
|
|
3359
|
+
return { url: '', caCertPath, env: {} };
|
|
3360
|
+
}
|
|
3361
|
+
catch (error) {
|
|
3362
|
+
if (!retirementAttempted)
|
|
3363
|
+
retire();
|
|
3364
|
+
throw error;
|
|
3365
|
+
}
|
|
3366
|
+
}
|
|
3367
|
+
function isAlive(pid) {
|
|
3368
|
+
if (!pid)
|
|
3369
|
+
return false;
|
|
3370
|
+
try {
|
|
3371
|
+
process.kill(pid, 0);
|
|
3372
|
+
return true;
|
|
3373
|
+
}
|
|
3374
|
+
catch {
|
|
3375
|
+
return false;
|
|
3376
|
+
}
|
|
3377
|
+
}
|
|
3378
|
+
function proxyDaemonEntry() {
|
|
3379
|
+
return siblingScript(import.meta.url, 'proxy-daemon');
|
|
3380
|
+
}
|
|
3381
|
+
/** The detached proxy daemon's body (invoked as its own process by `ensureWorldProxy`). Starts the
|
|
3382
|
+
* redirect proxy bound to an ephemeral port, records its pid/url/CA in `proxy.json`, and stays up
|
|
3383
|
+
* until killed by `down`. Exported so the daemon entry file can delegate to it. */
|
|
3384
|
+
export async function runProxyDaemon(name, root = process.cwd(), envFile) {
|
|
3385
|
+
const resolvedRoot = resolve(root);
|
|
3386
|
+
const envLoader = envFile
|
|
3387
|
+
? () => { const saved = readProxyEnv(envFile); return { ...admittedEnv(stripEnvOf(saved)), ...saved }; }
|
|
3388
|
+
: undefined;
|
|
3389
|
+
let env;
|
|
3390
|
+
if (!envLoader) {
|
|
3391
|
+
const status = statusWorld(name, resolvedRoot);
|
|
3392
|
+
if (!status.running)
|
|
3393
|
+
throw new Error(`World "${name}" is not running`);
|
|
3394
|
+
env = { ...admittedEnv(stripEnvOf(status.env)), ...status.env };
|
|
3395
|
+
}
|
|
3396
|
+
const proxy = await startRedirectProxy({ env, envLoader, tlsDir: tlsDir(resolvedRoot, name), note: (line) => process.stderr.write(`${line}\n`) });
|
|
3397
|
+
const state = { pid: process.pid, url: proxy.url, caCertPath: proxy.caCertPath, guestUrl: proxy.guestUrl };
|
|
3398
|
+
writeFileSync(proxyStateFile(resolvedRoot, name), JSON.stringify(state));
|
|
3399
|
+
// Not a recorded service, so it records its own start and end in the World's events (a crash included; a SIGKILL
|
|
3400
|
+
// leaves none, which doctor's proxy check then names).
|
|
3401
|
+
const events = worldEventsPath(join(instanceDir(resolvedRoot, name), 'logs'));
|
|
3402
|
+
const event = (entry) => { try {
|
|
3403
|
+
appendFileSync(events, `${JSON.stringify({ ...entry, service: 'proxy', pid: process.pid, at: new Date().toISOString() })}\n`, { mode: 0o600 });
|
|
3404
|
+
}
|
|
3405
|
+
catch { /* no events file to write */ } };
|
|
3406
|
+
event({ event: 'world.service.started' });
|
|
3407
|
+
const proxyStarted = Date.now();
|
|
3408
|
+
process.on('uncaughtException', (error) => { process.stderr.write(`${new Date().toISOString()} proxy daemon crashed: ${error.stack ?? error}\n`); process.exitCode = 1; process.exit(1); });
|
|
3409
|
+
process.on('exit', (code) => event({ event: 'world.service.exited', status: code, code, signal: null, uptimeMs: Date.now() - proxyStarted }));
|
|
3410
|
+
const stop = async () => {
|
|
3411
|
+
try {
|
|
3412
|
+
await proxy.close();
|
|
3413
|
+
}
|
|
3414
|
+
catch { /* ignore */ }
|
|
3415
|
+
try {
|
|
3416
|
+
rmSync(proxyStateFile(resolvedRoot, name), { force: true });
|
|
3417
|
+
}
|
|
3418
|
+
catch { /* ignore */ }
|
|
3419
|
+
process.exit(0);
|
|
3420
|
+
};
|
|
3421
|
+
process.on('SIGTERM', stop);
|
|
3422
|
+
process.on('SIGINT', stop);
|
|
3423
|
+
}
|
|
3424
|
+
/** Stop a world's ambient-redirect proxy daemon (if any) and drop its session CA. Best-effort —
|
|
3425
|
+
* called from `downWorld` so the trusted CA never outlives the world. */
|
|
3426
|
+
function stopWorldProxy(root, name) {
|
|
3427
|
+
const state = readProxyState(root, name);
|
|
3428
|
+
if (state && isAlive(state.pid)) {
|
|
3429
|
+
try {
|
|
3430
|
+
process.kill(state.pid, 'SIGTERM');
|
|
3431
|
+
}
|
|
3432
|
+
catch { /* ignore */ }
|
|
3433
|
+
}
|
|
3434
|
+
try {
|
|
3435
|
+
rmSync(proxyStateFile(root, name), { force: true });
|
|
3436
|
+
}
|
|
3437
|
+
catch { /* ignore */ }
|
|
3438
|
+
try {
|
|
3439
|
+
rmSync(proxyEnvFile(root, name), { force: true });
|
|
3440
|
+
}
|
|
3441
|
+
catch { /* ignore */ }
|
|
3442
|
+
tearDownCa(tlsDir(root, name));
|
|
3443
|
+
}
|
|
3444
|
+
/**
|
|
3445
|
+
* Emit a POSIX (bash/zsh) script to `eval` that ACTIVATES the world in the current shell — a
|
|
3446
|
+
* virtualenv for vendor APIs. It exports the world env (twin `*_URL`s + fake keys + the Node
|
|
3447
|
+
* injector + any `cliRedirect` endpoint vars), prepends `(world:<name>)` to the prompt so you can
|
|
3448
|
+
* always see you're pointed at twins (a safety signal, not cosmetic), and defines a `deactivate`
|
|
3449
|
+
* function that restores the prompt and unsets the vars. Usage: `eval "$(volter-world activate dev)"`.
|
|
3450
|
+
*
|
|
3451
|
+
* It ALSO starts (idempotently) the ambient TLS redirect proxy daemon for this world and exports
|
|
3452
|
+
* `HTTPS_PROXY`/`HTTP_PROXY` + the CA-trust vars (`NODE_EXTRA_CA_CERTS`, `CURL_CA_BUNDLE`, …) so an
|
|
3453
|
+
* UNMODIFIED `gh`/`stripe`/`curl`/app in the shell transparently hits the twins with zero per-tool
|
|
3454
|
+
* config (WORLD_ACTIVATE.md tier 1). The session CA is trusted ONLY via these per-shell exports,
|
|
3455
|
+
* never system-wide; `down` stops the daemon and drops the CA. If openssl is unavailable the script
|
|
3456
|
+
* still emits the env-only redirect (Phase-1 behavior) — it just skips the proxy exports.
|
|
3457
|
+
* See docs/guides/route-a-cli-through-the-world.md.
|
|
3458
|
+
*/
|
|
3459
|
+
export function activateScript(name, root = process.cwd()) {
|
|
3460
|
+
const status = statusWorld(name, root);
|
|
3461
|
+
if (!status.running)
|
|
3462
|
+
throw new Error(`World "${name}" is not running — run: volter-world up <config> --env-file <path> --name ${name}`);
|
|
3463
|
+
const baseEnv = status.env ?? {};
|
|
3464
|
+
// Bring up the ambient redirect proxy (best-effort) and fold its env in so the activated shell
|
|
3465
|
+
// routes https through the twins too — not just the Node injector / cliRedirect endpoint vars.
|
|
3466
|
+
let proxyNote = '';
|
|
3467
|
+
let env = { ...baseEnv };
|
|
3468
|
+
const sealed = baseEnv.VOLTER_WORLD_MODE === 'sealed' || baseEnv[WORLD_NETWORK_POLICY_ENV] !== undefined;
|
|
3469
|
+
try {
|
|
3470
|
+
const proxy = ensureWorldProxy(name, root, [admittedEnv(stripEnvOf(baseEnv)), baseEnv]);
|
|
3471
|
+
if (proxy && proxy.url) {
|
|
3472
|
+
env = { ...baseEnv, ...proxy.env };
|
|
3473
|
+
proxyNote = ` (ambient proxy ${proxy.url})`;
|
|
3474
|
+
}
|
|
3475
|
+
else if (sealed)
|
|
3476
|
+
throw new Error('attachment proxy is unavailable');
|
|
3477
|
+
}
|
|
3478
|
+
catch (error) {
|
|
3479
|
+
if (error instanceof CaTrustError)
|
|
3480
|
+
throw error;
|
|
3481
|
+
if (sealed)
|
|
3482
|
+
throw new Error(`world "${name}" requires its network policy proxy: attachment proxy failed; refusing activation (${error instanceof Error ? error.message : String(error)})`);
|
|
3483
|
+
// openssl missing or daemon failed — keep env-only redirect; never fabricate proxy success.
|
|
3484
|
+
}
|
|
3485
|
+
const keys = Object.keys(env).sort();
|
|
3486
|
+
const lines = [
|
|
3487
|
+
`# volter-world activate ${name} — run: eval "$(volter-world activate ${name})"`,
|
|
3488
|
+
`if [ -n "\${_VOLTER_WORLD:-}" ]; then deactivate 2>/dev/null || true; fi`,
|
|
3489
|
+
`_VOLTER_WORLD=${shellQuote(name)}`,
|
|
3490
|
+
`_VOLTER_WORLD_OLD_PS1="\${PS1:-}"`,
|
|
3491
|
+
`deactivate () {`,
|
|
3492
|
+
...keys.map((k) => ` unset ${k}`),
|
|
3493
|
+
` PS1="\${_VOLTER_WORLD_OLD_PS1:-}"; export PS1`,
|
|
3494
|
+
` unset _VOLTER_WORLD _VOLTER_WORLD_OLD_PS1`,
|
|
3495
|
+
` unset -f deactivate`,
|
|
3496
|
+
`}`,
|
|
3497
|
+
...keys.map((k) => `export ${k}=${shellQuote(env[k])}`),
|
|
3498
|
+
`PS1="(world:${name}) \${PS1:-}"; export PS1`,
|
|
3499
|
+
`printf '%s\\n' "world '${name}' active — vendor calls now hit the twins${proxyNote}. run: deactivate" >&2`,
|
|
3500
|
+
];
|
|
3501
|
+
return `${lines.join('\n')}\n`;
|
|
3502
|
+
}
|