@volter/world-runtime 3.0.162 → 3.0.164
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/ancestry.js +2 -2
- package/dist/src/app-url.d.ts +5 -1
- package/dist/src/app-url.js +14 -1
- package/dist/src/branch.js +28 -8
- package/dist/src/browser-files.d.ts +5 -0
- package/dist/src/browser-files.js +14 -0
- package/dist/src/browsers.d.ts +2 -3
- package/dist/src/browsers.js +4 -4
- package/dist/src/catalog.d.ts +2 -1
- package/dist/src/catalog.js +3 -2
- package/dist/src/changeset.js +1 -1
- package/dist/src/cli-commands.js +71 -18
- package/dist/src/cli-load-clock.d.ts +1 -0
- package/dist/src/cli-load-clock.js +2 -0
- package/dist/src/cli.js +5 -1
- package/dist/src/clickhouse-backing.d.ts +2 -0
- package/dist/src/clickhouse-backing.js +145 -0
- package/dist/src/command-lifetime.d.mts +1 -1
- package/dist/src/command-lifetime.mjs +5 -2
- package/dist/src/configs.js +1 -1
- package/dist/src/consumers.js +1 -1
- package/dist/src/covers.d.ts +4 -2
- package/dist/src/covers.js +8 -4
- package/dist/src/go-mysql-binding.d.ts +19 -0
- package/dist/src/go-mysql-binding.js +51 -0
- package/dist/src/guest-processes.d.mts +1 -1
- package/dist/src/guest-processes.mjs +22 -14
- package/dist/src/host-cli.js +8 -1
- package/dist/src/host-ready.d.ts +16 -2
- package/dist/src/host-ready.js +66 -39
- package/dist/src/host-worker.js +7 -1
- package/dist/src/host.d.ts +26 -0
- package/dist/src/host.js +93 -3
- package/dist/src/infra-definition.d.ts +35 -0
- package/dist/src/infra-definition.js +146 -0
- package/dist/src/infra.d.ts +3 -0
- package/dist/src/infra.js +375 -105
- package/dist/src/init.d.ts +46 -0
- package/dist/src/init.js +445 -18
- package/dist/src/lifecycle-record.d.ts +5 -1
- package/dist/src/lifecycle-record.js +27 -4
- package/dist/src/local-branches.js +1 -1
- package/dist/src/mariadb-bootstrap.d.ts +32 -0
- package/dist/src/mariadb-bootstrap.js +90 -0
- package/dist/src/mariadb-supplier.d.ts +23 -0
- package/dist/src/mariadb-supplier.js +215 -0
- package/dist/src/mongodb-backing.d.ts +4 -0
- package/dist/src/mongodb-backing.js +71 -1
- package/dist/src/native-mariadb-backing.d.ts +13 -0
- package/dist/src/native-mariadb-backing.js +512 -0
- package/dist/src/native-postgres-backing.d.ts +4 -10
- package/dist/src/native-postgres-backing.js +428 -43
- package/dist/src/native-server-inputs.d.ts +16 -0
- package/dist/src/native-server-inputs.js +204 -0
- package/dist/src/origin.js +3 -3
- package/dist/src/original-inputs.d.ts +14 -0
- package/dist/src/original-inputs.js +84 -0
- package/dist/src/pack-facts.d.ts +9 -1
- package/dist/src/pglite-backing.d.ts +41 -17
- package/dist/src/pglite-backing.js +213 -79
- package/dist/src/pglite-host.mjs +482 -10
- package/dist/src/process-groups.d.mts +6 -1
- package/dist/src/process-groups.d.ts +2 -1
- package/dist/src/process-groups.js +1 -1
- package/dist/src/process-groups.mjs +82 -15
- package/dist/src/process-signal.d.mts +2 -2
- package/dist/src/process-signal.mjs +29 -9
- package/dist/src/project-inspect.d.ts +12 -0
- package/dist/src/project-inspect.js +41 -4
- package/dist/src/proxy-env.d.ts +34 -0
- package/dist/src/proxy-env.js +114 -0
- package/dist/src/redirect-proxy.d.ts +1 -31
- package/dist/src/redirect-proxy.js +3 -112
- package/dist/src/redis-backing.d.ts +6 -4
- package/dist/src/redis-backing.js +20 -168
- package/dist/src/runtime.d.ts +1 -0
- package/dist/src/runtime.js +503 -218
- package/dist/src/schema.d.ts +1 -0
- package/dist/src/schema.js +5 -3
- package/dist/src/served-world.js +15 -10
- package/dist/src/service-exit.d.ts +3 -2
- package/dist/src/service-exit.js +7 -5
- package/dist/src/twin-backing.d.ts +47 -0
- package/dist/src/twin-backing.js +343 -0
- package/dist/src/up-cli.js +56 -3
- package/package.json +4 -4
- package/src/ancestry.ts +2 -2
- package/src/app-url.ts +9 -2
- package/src/branch.ts +26 -8
- package/src/browser-files.ts +15 -0
- package/src/browsers.ts +4 -4
- package/src/catalog.ts +4 -3
- package/src/changeset.ts +1 -1
- package/src/cli-commands.ts +85 -18
- package/src/cli-load-clock.ts +2 -0
- package/src/cli.ts +5 -1
- package/src/clickhouse-backing.ts +70 -0
- package/src/command-lifetime.d.mts +1 -1
- package/src/command-lifetime.mjs +5 -2
- package/src/configs.ts +1 -1
- package/src/consumers.ts +1 -1
- package/src/covers.ts +9 -4
- package/src/go-mysql-binding.ts +49 -0
- package/src/guest-processes.d.mts +1 -1
- package/src/guest-processes.mjs +22 -14
- package/src/host-cli.ts +7 -1
- package/src/host-ready.ts +62 -31
- package/src/host-worker.ts +6 -2
- package/src/host.ts +88 -6
- package/src/infra-definition.ts +135 -0
- package/src/infra.ts +250 -45
- package/src/init.ts +416 -17
- package/src/lifecycle-record.ts +18 -4
- package/src/local-branches.ts +1 -1
- package/src/mariadb-bootstrap.ts +107 -0
- package/src/mariadb-supplier.ts +199 -0
- package/src/mongodb-backing.ts +59 -1
- package/src/native-mariadb-backing.ts +418 -0
- package/src/native-postgres-backing.ts +353 -41
- package/src/native-server-inputs.ts +169 -0
- package/src/origin.ts +3 -3
- package/src/original-inputs.ts +72 -0
- package/src/pack-facts.ts +1 -1
- package/src/pglite-backing.ts +192 -80
- package/src/pglite-host.mjs +482 -10
- package/src/process-groups.d.mts +6 -1
- package/src/process-groups.mjs +82 -15
- package/src/process-groups.ts +2 -1
- package/src/process-signal.d.mts +2 -2
- package/src/process-signal.mjs +29 -9
- package/src/project-inspect.ts +37 -4
- package/src/proxy-env.ts +128 -0
- package/src/redirect-proxy.ts +3 -125
- package/src/redis-backing.ts +19 -136
- package/src/runtime.ts +342 -107
- package/src/schema.ts +5 -4
- package/src/served-world.ts +14 -10
- package/src/service-exit.ts +5 -3
- package/src/twin-backing.ts +300 -0
- package/src/up-cli.ts +52 -3
package/src/runtime.ts
CHANGED
|
@@ -1,18 +1,17 @@
|
|
|
1
1
|
import { errorMessage } from './error-message.ts';
|
|
2
|
-
import {
|
|
2
|
+
import { materializeWorldBrowserFiles } from './browser-files.ts';
|
|
3
3
|
import { claimEnvOutput, publishEnvOutput } from './env-output.ts';
|
|
4
4
|
import { offSignal } from './signals.ts';
|
|
5
5
|
import { parallelStartup } from './startup.ts';
|
|
6
|
-
import { waitForHostReady } from './host-ready.ts';
|
|
6
|
+
import { type HostReadyTimes, waitForHostReady } from './host-ready.ts';
|
|
7
7
|
import { assertWorldStateRemovable, removeWorldState } from './ancestry.ts';
|
|
8
8
|
import { packCli, packEntry } from './catalog.ts';
|
|
9
|
-
import { vendorOfPackModule } from '@volter/world-core/runtime';
|
|
9
|
+
import { vendorOfPackModule } from '@volter/world-core/runtime-state';
|
|
10
10
|
import { spawn, spawnSync } from 'node:child_process';
|
|
11
11
|
import { get as httpGet, type ClientRequest } from 'node:http';
|
|
12
12
|
import { siblingScript } from './sibling.ts';
|
|
13
13
|
import { commandLifetime, commandProcessGroup, CommandRetirementError } from './command-lifetime.ts';
|
|
14
|
-
import {
|
|
15
|
-
import { processStarts, survivingOwnedGroups } from './process-groups.ts';
|
|
14
|
+
import { processStarts, survivingOwnedGroups, setWorldEventsFile, worldEvent, type ProcessListing } from './process-groups.ts';
|
|
16
15
|
import { spawnIdentity, signalOwnedProcess, signalChild, signalHeldChild, processBirth, childIdentity } from './process-signal.mjs';
|
|
17
16
|
import type { SpawnIdentity } from './process-receipt.ts';
|
|
18
17
|
import { readGuestProcesses, releaseGuestProcesses } from './guest-processes.mjs';
|
|
@@ -27,10 +26,10 @@ import { basename, delimiter, dirname, join, relative, resolve, sep } from 'node
|
|
|
27
26
|
import { packFacts } from './pack-facts.ts';
|
|
28
27
|
import { assertOwnedRealPath, ownedPath } from './owned-path.ts';
|
|
29
28
|
import { isEnvironmentName } from './schema.ts';
|
|
30
|
-
import { infraConnections, parseInfraDefinition } from './
|
|
29
|
+
import { infraConnections, parseInfraDefinition } from './infra-definition.ts';
|
|
31
30
|
import { describeServiceEnd, readServiceEnd } from './service-exit.ts';
|
|
32
31
|
import { Socket } from 'node:net';
|
|
33
|
-
import { beginPlaceholderPull, endPlaceholderPull, placeholderPullActive, getActiveWorldStore, withFileLock, stateDirName, PROTOCOL_VERSION, PROTOCOL_MAJOR, protocolStanding, WORLD_BOOT_PATH, WORLD_DATA_ENV, WORLD_ENV_NAMES_ENV } from '@volter/world-core/runtime';
|
|
32
|
+
import { beginPlaceholderPull, endPlaceholderPull, placeholderPullActive, getActiveWorldStore, removeCarriedLocks, withFileLock, stateDirName, PROTOCOL_VERSION, PROTOCOL_MAJOR, protocolStanding, WORLD_BOOT_PATH, WORLD_DATA_ENV, WORLD_ENV_NAMES_ENV } from '@volter/world-core/runtime-state';
|
|
34
33
|
import { loadWorldConfig } from './configs.ts';
|
|
35
34
|
import { createWorldNetworkPolicy, parseWorldNetworkPolicy, normalizeWorldNetwork, intersectWorldNetwork, type WorldNetwork, WORLD_NETWORK_POLICY_ENV } from '@volter/world-core/network-policy';
|
|
36
35
|
import { overlayPackFactsFile, resolveServicePackage } from './catalog.ts';
|
|
@@ -38,11 +37,12 @@ import { fakeEnvValue } from './fixture-env.ts';
|
|
|
38
37
|
import { APP_URL_ENV, ISSUE_ENV_PREFIX, MINT_ENV } from './schema.ts';
|
|
39
38
|
import { readAppUrl } from './app-url.ts';
|
|
40
39
|
import { inertInjectEnvWarnings } from './inject-map.ts';
|
|
41
|
-
import { activeVendorMap, ensureCa, hostHasRawEgress, opensslAvailable, proxyEnvFor,
|
|
40
|
+
import { activeVendorMap, ensureCa, hostHasRawEgress, opensslAvailable, proxyEnvFor, tearDownCa } from './proxy-env.ts';
|
|
41
|
+
import type { startRedirectProxy } from './redirect-proxy.ts';
|
|
42
42
|
import { CaTrustError, sessionTrustEnv, type CaTrustInputs } from './ca-trust.ts';
|
|
43
43
|
import type { WorldConfig, WorldExternalReadyWhen, WorldInstance, WorldIsolation, WorldMode, WorldRunOutcome, WorldRunRecord, WorldServiceConfig, WorldServiceInstance } from './schema.ts';
|
|
44
44
|
import { checkedOwner, CONSUMER_HEARTBEAT_MS, createConsumer, refreshConsumer, releaseConsumer, removeConsumerRecord, worldConsumers, consumerProcessState } from './consumers.ts';
|
|
45
|
-
import { assertLifecycleStartable, beginWorldLifecycle, handoffWorldLifecycle, recordWorldLifecycleEvent, releaseWorldLifecycle, retainWorldLifecycleForCleanup, hasWorldLifecycle, isWorldLifecycleRunning, lifecycleLogPath, releaseAbandonedWorldLifecycle, exitedWorldLifecyclePids } from './lifecycle-record.ts';
|
|
45
|
+
import { assertLifecycleStartable, beginWorldLifecycle, handoffWorldLifecycle, recordWorldLifecycleEvent, releaseWorldLifecycle, retainWorldLifecycleForCleanup, hasWorldLifecycle, isWorldLifecycleRunning, describeWorldLifecycle, lifecycleLogPath, releaseAbandonedWorldLifecycle, exitedWorldLifecyclePids } from './lifecycle-record.ts';
|
|
46
46
|
|
|
47
47
|
// Births captured at actual spawn; retries never remeasure a retained PID.
|
|
48
48
|
const spawnedIdentities = new Map<number, SpawnIdentity>();
|
|
@@ -123,13 +123,18 @@ async function confirmOwnPort(port: number, boot: BootContext, ended: Promise<vo
|
|
|
123
123
|
boot.signal.throwIfAborted();
|
|
124
124
|
if (answered && typeof answered.boot === 'string') {
|
|
125
125
|
if (answered.boot === boot.bootId) return;
|
|
126
|
-
throw
|
|
126
|
+
throw foreignBoot(what, answered.boot, log);
|
|
127
127
|
}
|
|
128
128
|
// Whether the service has already ended, read now: one turn of the event loop, not a wait for it to fail.
|
|
129
129
|
const endedNow = ended ? await Promise.race([ended.then(() => true), new Promise<boolean>((resolveNow) => setImmediate(() => resolveNow(false)))]) : false;
|
|
130
130
|
if (endedNow) throw new Error(`${what} ended right after its port first answered, so another process probably holds the port: the service ${endedAccount(log)}\nLog: ${log}`);
|
|
131
131
|
}
|
|
132
132
|
|
|
133
|
+
/** A port answered by a twin of another World boot: a leftover World holds it. */
|
|
134
|
+
function foreignBoot(what: string, answered: string, log: string): Error {
|
|
135
|
+
return new Error(`${what}: its port is answered by a twin of another World boot (${answered}), not this one; a leftover World holds the port. Stop it (\`volter-world list\`, then \`down\`) and retry.\nLog: ${log}`);
|
|
136
|
+
}
|
|
137
|
+
|
|
133
138
|
/** What the server on a loopback port says its World boot is (WORLD_BOOT_PATH), or null. A direct loopback request:
|
|
134
139
|
* never through a proxy an activated shell's environment names. */
|
|
135
140
|
function worldBootOf(port: number, abort: AbortSignal): Promise<{ boot?: unknown } | null> {
|
|
@@ -812,18 +817,18 @@ async function waitForTcp(port: number, timeoutMs = 15_000, abort?: AbortSignal)
|
|
|
812
817
|
throw new Error(`Service on port ${port} did not become reachable`);
|
|
813
818
|
}
|
|
814
819
|
|
|
815
|
-
function readPids(path: string): number[] {
|
|
820
|
+
function readPids(path: string, receipts = readGuestProcesses(dirname(path))): number[] {
|
|
816
821
|
const held = getActiveWorldStore().read(path);
|
|
817
|
-
const guests =
|
|
822
|
+
const guests = receipts.map(receipt => receipt.record.pid);
|
|
818
823
|
return [...new Set([...(held ?? '')
|
|
819
824
|
.split(/\r?\n/)
|
|
820
825
|
.map((line) => Number(line.trim()))
|
|
821
826
|
.filter((pid) => Number.isInteger(pid) && pid > 0), ...guests])];
|
|
822
827
|
}
|
|
823
828
|
|
|
824
|
-
function livePids(pids: number[]): number[] {
|
|
829
|
+
function livePids(pids: number[], listing?: ProcessListing): number[] {
|
|
825
830
|
// A recorded leader may have exited while its detached descendants still run.
|
|
826
|
-
return survivingOwnedGroups(pids);
|
|
831
|
+
return survivingOwnedGroups(pids, listing);
|
|
827
832
|
}
|
|
828
833
|
|
|
829
834
|
function tunnelPids(instance: WorldInstance): number[] {
|
|
@@ -865,18 +870,23 @@ function writeOwnedPids(root: string, name: string, pids: number[], fresh: numbe
|
|
|
865
870
|
function identityPath(root: string, name: string): string {
|
|
866
871
|
return join(instanceDir(root, name), 'pids.identity');
|
|
867
872
|
}
|
|
868
|
-
function readIdentities(root: string, name: string): Record<string, string> {
|
|
873
|
+
function readIdentities(root: string, name: string, receipts = readGuestProcesses(instanceDir(root, name)), onConflict?: (pid: number) => void): Record<string, string> {
|
|
869
874
|
let known: Record<string, string> = {};
|
|
870
875
|
try {
|
|
871
876
|
const value = JSON.parse(getActiveWorldStore().read(identityPath(root, name)) ?? '{}');
|
|
872
877
|
if (value && typeof value === 'object' && !Array.isArray(value)) known = Object.fromEntries(Object.entries(value).filter(([pid, start]) => Number.isInteger(Number(pid)) && Number(pid) > 0 && typeof start === 'string' && start.length > 0)) as Record<string, string>;
|
|
873
878
|
} catch { /* Guest receipts below remain independently authoritative. */ }
|
|
874
879
|
const births = new Map<number, string | undefined>();
|
|
875
|
-
|
|
876
|
-
|
|
880
|
+
const conflicts = new Set<number>();
|
|
881
|
+
for (const { record } of receipts) {
|
|
882
|
+
if (births.has(record.pid) && births.get(record.pid) !== record.start) {
|
|
883
|
+
if (!onConflict) throw new Error('Conflicting guest spawn identities; ownership retained');
|
|
884
|
+
conflicts.add(record.pid);
|
|
885
|
+
onConflict(record.pid);
|
|
886
|
+
}
|
|
877
887
|
births.set(record.pid, record.start);
|
|
878
888
|
}
|
|
879
|
-
for (const [pid, start] of births) { if (start) known[pid] = start; else delete known[pid]; }
|
|
889
|
+
for (const [pid, start] of births) { if (start && !conflicts.has(pid)) known[pid] = start; else delete known[pid]; }
|
|
880
890
|
return known;
|
|
881
891
|
}
|
|
882
892
|
function recordIdentities(root: string, name: string, pids: number[], fresh: number[] = [], births = heldStarts()): void {
|
|
@@ -1072,8 +1082,9 @@ async function runExternalUp(
|
|
|
1072
1082
|
logPath: string,
|
|
1073
1083
|
abort?: AbortSignal,
|
|
1074
1084
|
progress?: (what: string) => void,
|
|
1085
|
+
verified?: { status?: string },
|
|
1075
1086
|
): Promise<string> {
|
|
1076
|
-
if (ownInfrastructureUp(command)) return runOwnInfrastructureUp(command, serviceId, env, logPath, abort, progress);
|
|
1087
|
+
if (ownInfrastructureUp(command)) return runOwnInfrastructureUp(command, serviceId, env, logPath, abort, progress, verified);
|
|
1077
1088
|
const [bin, ...args] = ownCommand(command);
|
|
1078
1089
|
if (!bin) throw new Error(`External service "${serviceId}": up command is empty`);
|
|
1079
1090
|
if (!commandExists(bin)) {
|
|
@@ -1153,20 +1164,23 @@ async function runOwnInfrastructureStatus(serviceId: string, phase: string, env:
|
|
|
1153
1164
|
} finally { clearTimeout(timer); abort?.removeEventListener('abort', onAbort); }
|
|
1154
1165
|
}
|
|
1155
1166
|
|
|
1156
|
-
async function runOwnInfrastructureUp(command: string[], serviceId: string, env: NodeJS.ProcessEnv, logPath: string, abort?: AbortSignal, progress?: (what: string) => void): Promise<string> {
|
|
1167
|
+
async function runOwnInfrastructureUp(command: string[], serviceId: string, env: NodeJS.ProcessEnv, logPath: string, abort?: AbortSignal, progress?: (what: string) => void, verified?: { status?: string }): Promise<string> {
|
|
1157
1168
|
writeFileSync(logPath, `$ ${command.join(' ')}\n`, { mode: 0o600 });
|
|
1158
1169
|
chmodSync(logPath, 0o600);
|
|
1159
1170
|
const out = openPrivateLog(logPath, 'a');
|
|
1160
1171
|
const chunks: string[] = [];
|
|
1161
1172
|
try {
|
|
1162
1173
|
abort?.throwIfAborted();
|
|
1174
|
+
const importing = Date.now();
|
|
1163
1175
|
const { runInfrastructure } = await import('./infra.ts');
|
|
1164
1176
|
// Its phase lines (backing chosen, host spawned, serving) are the boot's progress too.
|
|
1165
1177
|
const write = (text: string) => {
|
|
1166
1178
|
chunks.push(text); writeSync(out, text);
|
|
1167
1179
|
for (const line of text.split('\n')) if (line.startsWith('managed infrastructure')) progress?.(`${serviceId}: ${line}`);
|
|
1168
1180
|
};
|
|
1169
|
-
|
|
1181
|
+
// Its module's load, said with the line's own time (S2, Twenty's World up): what of the start before the backing's line was this.
|
|
1182
|
+
write(`managed infrastructure: its module loaded in ${Date.now() - importing} ms\n`);
|
|
1183
|
+
const code = await runInfrastructure(command[1]!, env, { out: write, err: write, ...(verified ? { verified: (status: string) => { verified.status = status; } } : {}) }, abort);
|
|
1170
1184
|
abort?.throwIfAborted();
|
|
1171
1185
|
if (code !== 0) {
|
|
1172
1186
|
const detail = chunks.join('').trim().split('\n').slice(-40).join('\n');
|
|
@@ -1196,6 +1210,7 @@ async function awaitReadiness(
|
|
|
1196
1210
|
serviceId: string,
|
|
1197
1211
|
env: NodeJS.ProcessEnv = process.env,
|
|
1198
1212
|
abort?: AbortSignal,
|
|
1213
|
+
passing?: { stdout?: string; stderr?: string },
|
|
1199
1214
|
): Promise<string> {
|
|
1200
1215
|
if (!ready) return 'no readiness declared';
|
|
1201
1216
|
const timeoutMs = ready.timeoutMs ?? 60_000;
|
|
@@ -1210,7 +1225,11 @@ async function awaitReadiness(
|
|
|
1210
1225
|
if (!commandExists(ready.command)) {
|
|
1211
1226
|
throw new Error(`Service "${serviceId}": readiness command \`${ready.command}\` not found on PATH`);
|
|
1212
1227
|
}
|
|
1213
|
-
try {
|
|
1228
|
+
try {
|
|
1229
|
+
const output = await runExternalCommand([ready.command, ...(ready.args ?? [])], cwd, serviceId, 'readiness', env, abort, Math.max(1, timeoutMs - (Date.now() - started)));
|
|
1230
|
+
if (passing) Object.assign(passing, output);
|
|
1231
|
+
return passed(`\`${[ready.command, ...(ready.args ?? [])].join(' ')}\` exited 0`);
|
|
1232
|
+
}
|
|
1214
1233
|
catch (error) { if (error instanceof CommandRetirementError) throw error; lastError = errorMessage(error); }
|
|
1215
1234
|
} else if (ready.httpUrl !== undefined) {
|
|
1216
1235
|
try {
|
|
@@ -1270,17 +1289,38 @@ async function startExternalService(
|
|
|
1270
1289
|
// Publish the declared teardown before an external command can create state.
|
|
1271
1290
|
boot?.external({ id: service.id, type: 'external', command: [...external.up], pid: 0, log, env: {},
|
|
1272
1291
|
external: { down: [...external.down], cwd, discoveredEnv: { ...(service.env ?? {}) } } });
|
|
1273
|
-
const
|
|
1292
|
+
const verified: { status?: string } = {};
|
|
1293
|
+
const upOutput = await runExternalUp(external.up, cwd, service.id, env, log, boot?.signal, boot?.progress, verified);
|
|
1274
1294
|
boot?.progress(`${service.id}: up returned`);
|
|
1275
1295
|
|
|
1276
1296
|
// 2. readiness — Docker-backed externals take seconds to boot.
|
|
1277
|
-
const
|
|
1297
|
+
const passing: { stdout?: string; stderr?: string } = {};
|
|
1298
|
+
// The runtime's own infrastructure, every service twin-served or its PostgreSQL native (no MongoDB native), proves in
|
|
1299
|
+
// its `up` what its `status` would, status's proofs asked again at hand-over though not its choice of backing (infra.ts
|
|
1300
|
+
// handOver: twin-backing.ts twinBackingHeld, each port
|
|
1301
|
+
// served by the child it started; native-postgres-backing.ts nativePostgresStatus, the postmaster's ownership, live
|
|
1302
|
+
// group, identity and answering port; Cal's status run took 70 ms, c670b965 run 2, on the mini, in a browser tab; S2
|
|
1303
|
+
// R46). Its up hands over the status it would print, so the probe,
|
|
1304
|
+
// when it is that same `status`, is not run again (LibreChat: the status run took 206 ms before it connected, e946aa4c
|
|
1305
|
+
// lap 060250 run 2, on the mini, in a browser tab; S2 R45). Any other probe, or a backing up does not prove, is
|
|
1306
|
+
// probed as before.
|
|
1307
|
+
const probed = external.readyWhen?.command === undefined ? undefined : [external.readyWhen.command, ...(external.readyWhen.args ?? [])];
|
|
1308
|
+
const upProved = verified.status !== undefined && probed !== undefined && ownInfrastructureStatus(probed);
|
|
1309
|
+
if (upProved) { passing.stdout = verified.status; passing.stderr = ''; }
|
|
1310
|
+
const readiness = upProved ? 'proved by its own up, as its status would' : await awaitReadiness(external.readyWhen, cwd, service.id, env, boot?.signal, passing);
|
|
1278
1311
|
boot?.progress(`${service.id}: ready (${readiness})`);
|
|
1279
1312
|
|
|
1280
1313
|
// 3. status — discover connection info (optional; may reuse up output instead).
|
|
1281
1314
|
let statusOutput = '';
|
|
1282
1315
|
if (external.status) {
|
|
1283
|
-
|
|
1316
|
+
// The runtime's own infrastructure probes its readiness with the very status command it then reads (init declares
|
|
1317
|
+
// both as `volter-world-infra status`): the probe that just passed is that read, taken a moment ago in this
|
|
1318
|
+
// process, so it is not run again (S2 R17: LibreChat's second run of it, 190 ms of up after its infrastructure
|
|
1319
|
+
// was ready). Any other status command, or a probe that is not the same command, is run as before.
|
|
1320
|
+
const probe = external.readyWhen?.command === undefined ? undefined : [external.readyWhen.command, ...(external.readyWhen.args ?? [])];
|
|
1321
|
+
const same = probe !== undefined && passing.stdout !== undefined && ownInfrastructureStatus(external.status)
|
|
1322
|
+
&& probe.length === external.status.length && probe.every((part, index) => part === external.status![index]);
|
|
1323
|
+
const statusResult = same ? { stdout: passing.stdout!, stderr: passing.stderr ?? '' } : await runExternalCommand(external.status, cwd, service.id, 'status', env, boot?.signal);
|
|
1284
1324
|
statusOutput = statusResult.stdout || statusResult.stderr;
|
|
1285
1325
|
appendFileSync(log, `\n$ ${external.status.join(' ')}\n${statusOutput}`);
|
|
1286
1326
|
boot?.progress(`${service.id}: status read`);
|
|
@@ -1605,17 +1645,25 @@ function takeIssued(env: Record<string, string>): Map<string, string> {
|
|
|
1605
1645
|
* own store, in milliseconds; this bounds a hung twin, not a slow one). */
|
|
1606
1646
|
const ISSUE_TIMEOUT_MS = 15_000;
|
|
1607
1647
|
|
|
1648
|
+
/** Where one credential ask's time went (S2, the twins' credentials): performance.now() at each step, read by the caller
|
|
1649
|
+
* against its own origin. `door` is the pack's door read, `sent` the request handed to fetch, `headers` its answer's
|
|
1650
|
+
* status, `answered` its body parsed, `filled` every name set or said unset. Door→sent and answered→filled include synchronous placeholder-pull
|
|
1651
|
+
* and custody file work; only sent→status isolates the twin door. */
|
|
1652
|
+
type CredentialAskTimes = { twin?: string; names?: number; door?: number; sent?: number; headers?: number; answered?: number; filled?: number };
|
|
1653
|
+
|
|
1608
1654
|
/** A twin's issued credential (its `issues`, copied from its pack's credentialDoor by init): once it is up, the door is
|
|
1609
1655
|
* asked for one and each pending name of that service is set to it. A twin that answers no credential, or has no
|
|
1610
1656
|
* door, leaves the names unset and says so, rather than handing the application a fixture the twin would refuse. */
|
|
1611
|
-
async function issueCredentials(service: WorldServiceConfig, started: WorldServiceInstance, env: Record<string, string>, pending: Map<string, string>, unissued: Set<string>, controlRoot: string): Promise<void> {
|
|
1657
|
+
async function issueCredentials(service: WorldServiceConfig, started: WorldServiceInstance, env: Record<string, string>, pending: Map<string, string>, unissued: Set<string>, controlRoot: string, times?: CredentialAskTimes): Promise<void> {
|
|
1612
1658
|
const direct = new Set([...pending].filter(([, id]) => id === `${service.id}@direct`).map(([name]) => name));
|
|
1613
1659
|
const wanted = [...pending].filter(([name, id]) => id === service.id || direct.has(name)).map(([name]) => name);
|
|
1660
|
+
if (times) { times.names = wanted.length; }
|
|
1614
1661
|
if (wanted.length === 0) return;
|
|
1615
1662
|
for (const name of wanted) pending.delete(name);
|
|
1616
1663
|
// the pack's door as it stands now (a door the pack fixed since the World was made reaches it), else the saved copy
|
|
1617
1664
|
let door = service.issues;
|
|
1618
1665
|
try { door = packFacts()[service.id]?.credentialDoor ?? door; } catch { /* no artifact: the saved copy */ }
|
|
1666
|
+
if (times) times.door = performance.now();
|
|
1619
1667
|
try {
|
|
1620
1668
|
if (!door) throw new Error('its service names no credential door');
|
|
1621
1669
|
if (!started.url) throw new Error('it has no URL to ask');
|
|
@@ -1624,13 +1672,22 @@ async function issueCredentials(service: WorldServiceConfig, started: WorldServi
|
|
|
1624
1672
|
if (!enclosingSeed) beginPlaceholderPull(service.id, controlRoot);
|
|
1625
1673
|
let answer: Record<string, unknown>;
|
|
1626
1674
|
try {
|
|
1675
|
+
if (times) times.sent = performance.now();
|
|
1627
1676
|
const res = await fetch(new URL(door.path, started.url), { method: 'POST', headers: { 'content-type': 'application/json', 'x-volter-world-activity': 'provisioning' }, body: JSON.stringify(door.body), signal: AbortSignal.timeout(ISSUE_TIMEOUT_MS) });
|
|
1628
|
-
if (
|
|
1677
|
+
if (times) times.headers = performance.now();
|
|
1678
|
+
// What the door answered is said with its status: a twin's own error is the only reading of why it issued
|
|
1679
|
+
// nothing, and it is otherwise in a log of the twin's host that the caller of `up` may never see.
|
|
1680
|
+
if (!res.ok) throw new Error(`answered ${res.status}: ${(await res.text().catch(() => '')).replace(/\s+/g, ' ').slice(0, 400) || 'no body'}`);
|
|
1629
1681
|
answer = (await res.json()) as Record<string, unknown>;
|
|
1682
|
+
if (times) times.answered = performance.now();
|
|
1630
1683
|
} finally {
|
|
1631
1684
|
if (!enclosingSeed) endPlaceholderPull(service.id, controlRoot);
|
|
1632
1685
|
}
|
|
1686
|
+
// EACH NAME IS FILLED ON ITS OWN. A door answers several things (a key, a sign-in's files); a part it could not
|
|
1687
|
+
// make, or a name whose part is not in the answer, leaves THAT name unset and said, and the others are given.
|
|
1688
|
+
// One throw here used to unset every name of the service, the key among them.
|
|
1633
1689
|
for (const name of wanted) {
|
|
1690
|
+
try {
|
|
1634
1691
|
const spec = door.fill[name] ?? '';
|
|
1635
1692
|
if (spec.startsWith('directory:') || spec.startsWith('file:')) {
|
|
1636
1693
|
if (direct.has(name)) throw new Error(`cannot give ${name}'s filesystem credential a direct URL`);
|
|
@@ -1722,6 +1779,10 @@ async function issueCredentials(service: WorldServiceConfig, started: WorldServi
|
|
|
1722
1779
|
issued.host = endpoint.host;
|
|
1723
1780
|
env[name] = issued.toString();
|
|
1724
1781
|
} else env[name] = value;
|
|
1782
|
+
} catch (error) {
|
|
1783
|
+
process.stderr.write(`volter-world: ${service.id} did not issue ${name} (${errorMessage(error)}${typeof answer.codex_unavailable === 'string' ? `; the twin says: ${answer.codex_unavailable.slice(0, 300)}` : ''}); left unset\n`);
|
|
1784
|
+
delete env[name]; unissued.add(name);
|
|
1785
|
+
}
|
|
1725
1786
|
}
|
|
1726
1787
|
if (started.url) {
|
|
1727
1788
|
const endpoint = new URL(started.url);
|
|
@@ -1733,6 +1794,7 @@ async function issueCredentials(service: WorldServiceConfig, started: WorldServi
|
|
|
1733
1794
|
process.stderr.write(`volter-world: ${service.id} did not issue ${wanted.join(', ')} (${errorMessage(error)}); left unset\n`);
|
|
1734
1795
|
for (const name of wanted) { delete env[name]; unissued.add(name); }
|
|
1735
1796
|
}
|
|
1797
|
+
if (times) times.filled = performance.now();
|
|
1736
1798
|
}
|
|
1737
1799
|
|
|
1738
1800
|
function materializeConfig(config: WorldConfig, root: string): WorldConfig {
|
|
@@ -1765,6 +1827,20 @@ function versionSatisfies(actual: string, pin: string): boolean {
|
|
|
1765
1827
|
return actual === pin;
|
|
1766
1828
|
}
|
|
1767
1829
|
|
|
1830
|
+
/** Where the host's start went, from its receipt, for the receipt's progress line (S2 R38): when its own modules were
|
|
1831
|
+
* loaded, when its twins' factories returned (the slowest named) and when it published, each after its spawn, and how
|
|
1832
|
+
* long after publication the receipt was read. A host that says none of it adds nothing. */
|
|
1833
|
+
function hostStartAccount(times: HostReadyTimes, spawnedAt: number, readAt: number): string {
|
|
1834
|
+
const after = (time: number): string => `+${Math.max(0, time - spawnedAt)} ms`;
|
|
1835
|
+
const parts: string[] = [];
|
|
1836
|
+
if (times.at !== undefined) parts.push(`${Math.max(0, readAt - times.at)} ms after the host published it`);
|
|
1837
|
+
if (times.began !== undefined) parts.push(`the host's own modules loaded ${after(times.began)} after its spawn`);
|
|
1838
|
+
const started = Object.entries(times.started ?? {}).sort((a, b) => a[1] - b[1]);
|
|
1839
|
+
if (started.length) parts.push(`its twins returned ${after(started[0]![1])} to ${after(started.at(-1)![1])} (slowest ${started.at(-1)![0]})`);
|
|
1840
|
+
if (times.at !== undefined && (times.began !== undefined || started.length)) parts.push(`published ${after(times.at)}`);
|
|
1841
|
+
return parts.length ? `, ${parts.join('; ')}` : '';
|
|
1842
|
+
}
|
|
1843
|
+
|
|
1768
1844
|
/**
|
|
1769
1845
|
* Boot every `colocate`-declaring service inside ONE `volter-world-host` child (see src/host.ts —
|
|
1770
1846
|
* 'colocated' maps to the host's 'shared' mode, 'worker' to one Worker thread per twin). The
|
|
@@ -1791,21 +1867,33 @@ async function startColocatedServices(
|
|
|
1791
1867
|
const readyFile = join(paths.instance, `host-ready-${boot.bootId}.json`);
|
|
1792
1868
|
const args: string[] = [hostCli, '--isolation', isolation === 'colocated' ? 'shared' : 'worker', '--ready-file', readyFile];
|
|
1793
1869
|
const allocated: Array<{ service: WorldServiceConfig; port: number; mounted: MountedPackage; sides: Awaited<ReturnType<typeof sidePortsFor>> }> = [];
|
|
1794
|
-
|
|
1795
|
-
|
|
1870
|
+
// Every twin's port and side ports are probed together, not one after another: a probe is a listen and a close
|
|
1871
|
+
// on IPv4 and IPv6, ~25 ms each in a tab, and Dub's fourteen twins and their sides held sixteen of them in turn
|
|
1872
|
+
// (S2 R16: 406 ms of its 1,871 ms up before the host was spawned). Concurrent probes hold distinct ports while
|
|
1873
|
+
// open, and `allocatedPorts` is checked and added to synchronously after each probe, so no port is given twice;
|
|
1874
|
+
// a declared port is in it before any automatic one is chosen, as before. Which twin gets which automatic port
|
|
1875
|
+
// is not part of any contract.
|
|
1876
|
+
// Relative module paths resolve from the world root (import() inside the host would
|
|
1877
|
+
// otherwise resolve them against the runtime's own source directory).
|
|
1878
|
+
const moduleOf = (service: WorldServiceConfig): string => {
|
|
1879
|
+
const declared = service.colocate!.module!;
|
|
1880
|
+
return declared.startsWith('.') ? resolve(paths.root, declared) : declared;
|
|
1881
|
+
};
|
|
1882
|
+
const probed = await Promise.all(colocated.map(async (service) => ({
|
|
1883
|
+
port: service.port === undefined || service.port === 'auto' ? await allocatePort(allocatedPorts) : service.port,
|
|
1884
|
+
sides: await sidePortsFor(vendorOfModule(moduleOf(service), service.id), [], allocatedPorts),
|
|
1885
|
+
})));
|
|
1886
|
+
for (const [index, service] of colocated.entries()) {
|
|
1887
|
+
const { port, sides } = probed[index]!;
|
|
1796
1888
|
const serviceDataDir = ownedPath(paths.data, service.id);
|
|
1797
1889
|
mkdirSync(serviceDataDir, { recursive: true });
|
|
1798
|
-
|
|
1799
|
-
// otherwise resolve them against the runtime's own source directory).
|
|
1800
|
-
const declared = service.colocate!.module!;
|
|
1801
|
-
const module = declared.startsWith('.') ? resolve(paths.root, declared) : declared;
|
|
1890
|
+
const module = moduleOf(service);
|
|
1802
1891
|
// R17 — the lockfile check, BEFORE the host boots: what this module's own package.json says
|
|
1803
1892
|
// it is, against the pin the world config carries; and R16 — the protocol standing the
|
|
1804
1893
|
// catalog recorded for the package (pack-facts), refused two majors back, warned one back.
|
|
1805
1894
|
const mounted = lockfileCheck(service, module);
|
|
1806
1895
|
const scenario = service.colocate!.scenarioPath;
|
|
1807
1896
|
const binding = managedDatabaseFor(vendorOfModule(module, service.id), worldEnv.VOLTER_WORLD_CONFIG, config.env ?? {});
|
|
1808
|
-
const sides = await sidePortsFor(vendorOfModule(module, service.id), [], allocatedPorts);
|
|
1809
1897
|
args.push('--spec', [service.id, module, service.colocate!.export, String(port), serviceDataDir, ...(scenario ? [`scenario=${scenario.startsWith('/') ? scenario : resolve(paths.root, scenario)}`] : []), ...(binding !== undefined ? [`database=${binding.url}`] : []), ...sides.map((side) => `side.${side.option}=${side.port}`)].join('|'));
|
|
1810
1898
|
allocated.push({ service, port, mounted, sides });
|
|
1811
1899
|
}
|
|
@@ -1819,6 +1907,7 @@ async function startColocatedServices(
|
|
|
1819
1907
|
for (const [key, value] of Object.entries(env)) if (value === APP_URL_ENV) delete env[key];
|
|
1820
1908
|
dropUnissued(env);
|
|
1821
1909
|
env.VOLTER_WORLD_BOOT_ID = boot.bootId;
|
|
1910
|
+
const spawnedAt = Date.now();
|
|
1822
1911
|
// Under the same exit recorder as a process service, so the host's end (and every twin it holds) is in host.log.
|
|
1823
1912
|
const { child, ended, identity } = spawnRecorded('host', process.execPath, args, paths.root, env, log, boot);
|
|
1824
1913
|
child.once('error', error => boot.fail(error));
|
|
@@ -1857,27 +1946,42 @@ async function startColocatedServices(
|
|
|
1857
1946
|
});
|
|
1858
1947
|
const ready = (async (): Promise<void> => {
|
|
1859
1948
|
const hostTimeoutMs = allocated.reduce((timeout, { service }) => Math.max(timeout, typeof service.ready === 'object' ? service.ready.timeoutMs ?? 15_000 : 15_000), 15_000);
|
|
1860
|
-
await untilReadyOrEnded((abort) => waitForHostReady(readyFile, boot.bootId, new Map(allocated.map(({ service, port }) => [service.id, port])), hostTimeoutMs, abort, () => { try { return statSync(log).size; } catch { return 0; } }), ended, boot, 'the co-located host', log).catch(async (error) => {
|
|
1949
|
+
const hostTimes = await untilReadyOrEnded((abort) => waitForHostReady(readyFile, boot.bootId, new Map(allocated.map(({ service, port }) => [service.id, port])), hostTimeoutMs, abort, () => { try { return statSync(log).size; } catch { return 0; } }), ended, boot, 'the co-located host', log).catch(async (error) => {
|
|
1861
1950
|
if (undeadPids([child.pid!]).length) recordWorldStop([log], 'world.service.stop', 'up (host startup incomplete)', 'SIGTERM');
|
|
1862
1951
|
try { await signalHeldChild(child, 'SIGTERM', true, identity); } catch { /* best effort cleanup */ }
|
|
1863
1952
|
throw new Error(`Co-located host failed to complete startup (isolation=${isolation}): ${errorMessage(error)}\nLog: ${log}`);
|
|
1864
1953
|
});
|
|
1865
|
-
boot.progress(
|
|
1954
|
+
boot.progress(`co-located host: its all-started receipt is read${hostStartAccount(hostTimes, spawnedAt, Date.now())}`);
|
|
1955
|
+
// The host's receipt (this boot, every requested port) is published only
|
|
1956
|
+
// after every factory bound its own port; a port another process held fails
|
|
1957
|
+
// its bind and leaves no receipt. The receipt is the host's ownership proof
|
|
1958
|
+
// for each port (ADR 0014). A port that answers with another boot's identity
|
|
1959
|
+
// is still refused.
|
|
1960
|
+
// So each port is asked its boot at once: this boot's answer is the port's reachability and its ownership in one
|
|
1961
|
+
// connection. Another boot's answer is refused at once, as confirmation refuses it; a port that answers no
|
|
1962
|
+
// boot is waited for and confirmed as before.
|
|
1963
|
+
// A twin the host's receipt names a byte stream (host.ts servesByteStream) is not asked: it would answer the request
|
|
1964
|
+
// in its own protocol, after its own greeting work (Rallly's SMTP relay answered 57 ms after the request, holding
|
|
1965
|
+
// host-ready: e946aa4c run 3, on the mini, World 3.0.2197; S2 R38), and then be asked again by confirmation. Its
|
|
1966
|
+
// admission is its TCP answer and its host still running, as a port with no boot identity's is: another holder of
|
|
1967
|
+
// its port fails its bind, so no receipt.
|
|
1968
|
+
const streams = new Set(hostTimes.streams ?? []);
|
|
1866
1969
|
await parallelStartup(allocated, async ({ service, port }) => {
|
|
1970
|
+
const byteStream = streams.has(service.id);
|
|
1971
|
+
const answered = byteStream ? null : await worldBootOf(port, boot.signal);
|
|
1972
|
+
if (answered && typeof answered.boot === 'string') {
|
|
1973
|
+
if (answered.boot === boot.bootId) return;
|
|
1974
|
+
if (undeadPids([child.pid!]).length) recordWorldStop([log], 'world.service.stop', 'up (port not its own)', 'SIGTERM');
|
|
1975
|
+
try { await signalHeldChild(child, 'SIGTERM', true, identity); } catch { /* best effort cleanup */ }
|
|
1976
|
+
throw foreignBoot(`Co-located twin "${service.id}"`, answered.boot, log);
|
|
1977
|
+
}
|
|
1867
1978
|
const bindTimeoutMs = typeof service.ready === 'object' && service.ready.timeoutMs !== undefined ? Math.max(service.ready.timeoutMs, 15_000) : 15_000;
|
|
1868
1979
|
await untilReadyOrEnded((abort) => waitForTcp(port, bindTimeoutMs, abort), ended, boot, 'the co-located host', log).catch(async (error) => {
|
|
1869
1980
|
if (undeadPids([child.pid!]).length) recordWorldStop([log], 'world.service.stop', 'up (startup failed)', 'SIGTERM');
|
|
1870
1981
|
try { await signalHeldChild(child, 'SIGTERM', true, identity); } catch { /* best effort cleanup */ }
|
|
1871
1982
|
throw new Error(`Co-located twin "${service.id}" failed to start (host isolation=${isolation}): ${errorMessage(error)}\nLog: ${log}`);
|
|
1872
1983
|
});
|
|
1873
|
-
|
|
1874
|
-
// The host's receipt (this boot, every requested port) is published only
|
|
1875
|
-
// after every factory bound its own port; a port another process held fails
|
|
1876
|
-
// its bind and leaves no receipt. The receipt is the host's ownership proof
|
|
1877
|
-
// for each port (ADR 0014). A port that answers with another boot's identity
|
|
1878
|
-
// is still refused.
|
|
1879
|
-
await parallelStartup(allocated, async ({ service, port }) => {
|
|
1880
|
-
await confirmOwnPort(port, boot, ended, true, `Co-located twin "${service.id}"`, log).catch(async (error) => {
|
|
1984
|
+
await confirmOwnPort(port, boot, ended, !byteStream, `Co-located twin "${service.id}"`, log).catch(async (error) => {
|
|
1881
1985
|
if (undeadPids([child.pid!]).length) recordWorldStop([log], 'world.service.stop', 'up (port not its own)', 'SIGTERM');
|
|
1882
1986
|
try { await signalHeldChild(child, 'SIGTERM', true, identity); } catch { /* best effort cleanup */ }
|
|
1883
1987
|
throw error;
|
|
@@ -1896,16 +2000,56 @@ async function startColocatedServices(
|
|
|
1896
2000
|
return { services, ready };
|
|
1897
2001
|
}
|
|
1898
2002
|
|
|
2003
|
+
/** How late this process's event loop ran while `up` worked (S2 R45): a 10 ms tick, unref'd, on the monotonic clock.
|
|
2004
|
+
* A tick 2 ms or more late adds its lateness to the total (less is timer jitter, not counted); the longest is kept; a
|
|
2005
|
+
* stall is a tick 10 ms or more late, and the three longest are kept with when, after up began, each began (to within
|
|
2006
|
+
* the 10 ms tick). LibreChat's up showed three windows of 206-263 ms with no work of the runtime's in its trace
|
|
2007
|
+
* (e946aa4c lap 060250 run 2, on the mini, in a browser tab): this says whether its loop was stalled in them or working. */
|
|
2008
|
+
function eventLoopLag(): { account: () => string; stop: () => void } {
|
|
2009
|
+
const TICK_MS = 10;
|
|
2010
|
+
const JITTER_MS = 2;
|
|
2011
|
+
const started = performance.now();
|
|
2012
|
+
let expected = started + TICK_MS;
|
|
2013
|
+
let total = 0;
|
|
2014
|
+
let longest = 0;
|
|
2015
|
+
const stalls: Array<{ at: number; ms: number }> = [];
|
|
2016
|
+
const timer = setInterval(() => {
|
|
2017
|
+
const now = performance.now();
|
|
2018
|
+
const late = now - expected;
|
|
2019
|
+
expected = now + TICK_MS;
|
|
2020
|
+
if (late < JITTER_MS) return;
|
|
2021
|
+
total += late;
|
|
2022
|
+
longest = Math.max(longest, late);
|
|
2023
|
+
if (late < TICK_MS) return;
|
|
2024
|
+
stalls.push({ at: now - late - started, ms: late });
|
|
2025
|
+
stalls.sort((a, b) => b.ms - a.ms);
|
|
2026
|
+
stalls.length = Math.min(stalls.length, 3);
|
|
2027
|
+
}, TICK_MS);
|
|
2028
|
+
timer.unref?.();
|
|
2029
|
+
return {
|
|
2030
|
+
account: () => `runtime event loop ${Math.round(total)} ms late in all (ticks 2 ms or more late), longest ${Math.round(longest)} ms${stalls.length ? `; stalls ${stalls.map((stall) => `${Math.round(stall.ms)} ms at +${(stall.at / 1000).toFixed(2)} s`).join(', ')} (starts ±10 ms)` : ''}`,
|
|
2031
|
+
stop: () => clearInterval(timer),
|
|
2032
|
+
};
|
|
2033
|
+
}
|
|
2034
|
+
|
|
1899
2035
|
export async function upWorld(configId: string, options: UpWorldOptions = {}, abort?: AbortSignal): Promise<WorldInstance> {
|
|
1900
|
-
abort?.throwIfAborted();
|
|
1901
2036
|
const upStarted = Date.now();
|
|
2037
|
+
const lag = eventLoopLag();
|
|
2038
|
+
try { return await upWorldRecorded(configId, options, abort, upStarted, lag); }
|
|
2039
|
+
finally { lag.stop(); }
|
|
2040
|
+
}
|
|
2041
|
+
|
|
2042
|
+
async function upWorldRecorded(configId: string, options: UpWorldOptions, abort: AbortSignal | undefined, upStarted: number, lag: { account: () => string }): Promise<WorldInstance> {
|
|
2043
|
+
abort?.throwIfAborted();
|
|
1902
2044
|
const root = resolve(options.root ?? process.cwd());
|
|
1903
2045
|
const found = loadWorldConfig(configId, root);
|
|
1904
2046
|
// the facts of the World's installed packs, over the kernel's, for this process and (through the instance file) its own
|
|
1905
2047
|
const packFactsPaths = packFactsFiles(found.config, root);
|
|
1906
2048
|
for (const file of packFactsPaths) overlayPackFactsFile(file);
|
|
1907
2049
|
// and this process's injector host rules, which `up`'s inert-injectEnv warning reads, know them too
|
|
1908
|
-
|
|
2050
|
+
// An empty overlay changes no facts. Leave built-in host rules at their first use
|
|
2051
|
+
// instead of loading and rebuilding a second facts/host table on every empty up.
|
|
2052
|
+
if (packFactsPaths.length) createRequire(import.meta.url)('@volter/world-core/pack-facts').extendPackFacts(packFactsPaths);
|
|
1909
2053
|
const loaded = { path: found.path, config: materializeConfig(found.config, root) };
|
|
1910
2054
|
const name = options.name ?? loaded.config.id;
|
|
1911
2055
|
const mode = options.mode ?? 'local';
|
|
@@ -1987,6 +2131,8 @@ export async function upWorld(configId: string, options: UpWorldOptions = {}, ab
|
|
|
1987
2131
|
// the lockfile adds the cross-process exclusion, and an in-process lock holder always releases
|
|
1988
2132
|
// before its first await, so withFileLock's synchronous contender wait can never deadlock the
|
|
1989
2133
|
// event loop against a holder in the same process.
|
|
2134
|
+
// The claim's start: a kept state's lock file older than it is its last life's (removeCarriedLocks, S2 K4).
|
|
2135
|
+
const claimStartedAt = Date.now();
|
|
1990
2136
|
try {
|
|
1991
2137
|
withFileLock(instanceLockFile(root, name), () => {
|
|
1992
2138
|
assertProxyOwnershipResolved(root, name);
|
|
@@ -2054,6 +2200,9 @@ export async function upWorld(configId: string, options: UpWorldOptions = {}, ab
|
|
|
2054
2200
|
if (options.keepState) {
|
|
2055
2201
|
const KEEP = ['data', 'clock', 'marks', 'changesets', 'episodes', 'browser-records', 'browser-loads'];
|
|
2056
2202
|
for (const entry of existsSync(instance) ? readdirSync(instance) : []) if (!KEEP.includes(entry)) rmSync(join(instance, entry), { recursive: true, force: true });
|
|
2203
|
+
// The kept state's lock files with no readable holder record are its last life's, and no one's: removed now,
|
|
2204
|
+
// under this boot's claim, before any process of it waits on one (S2 K4; world-store.ts removeCarriedLocks).
|
|
2205
|
+
removeCarriedLocks(KEEP.map(entry => join(instance, entry)).filter(path => existsSync(path)), { stateDir: stateDirName(), before: claimStartedAt });
|
|
2057
2206
|
} else removeWorldState(root, instance, () => rmSync(instance, { recursive: true, force: true }));
|
|
2058
2207
|
} catch (error) {
|
|
2059
2208
|
// The claim failed; the logs go back where they were rather than wait aside for the next boot to discard.
|
|
@@ -2188,10 +2337,44 @@ export async function upWorld(configId: string, options: UpWorldOptions = {}, ab
|
|
|
2188
2337
|
colocatedReady = async () => {
|
|
2189
2338
|
await host.ready;
|
|
2190
2339
|
if (colocated.length) boot.progress('co-located host ready: receipt read, every port admitted as its own');
|
|
2191
|
-
for
|
|
2340
|
+
// Every twin is asked for its credentials at once, and each ask's time is said: thirteen twins took 4.7 s
|
|
2341
|
+
// of one World's up and fourteen took 0.2 s of another's, and the one line after them said only that all had.
|
|
2342
|
+
// Asked in turn they were the tail of up (S2 R17: LibreChat's three, 426 ms, after its infrastructure was
|
|
2343
|
+
// ready). Each ask fills a copy of the environment as it stood before any ask; the copies' changes are then
|
|
2344
|
+
// applied in declaration order, so the environment, its file and up's return are what asking in turn made:
|
|
2345
|
+
// a twin's names are its own (`pending` is split by service before the first await), and no ask reads
|
|
2346
|
+
// another's answer. One difference: where one ask fails, the others still run to their end and write their
|
|
2347
|
+
// custody files, where asking in turn stopped at the first failure; up fails either way.
|
|
2348
|
+
const before = { ...env };
|
|
2349
|
+
// Each ask's own steps, from one origin (S2, the twins' credentials: Twenty's thirteen took 388-500 ms each and
|
|
2350
|
+
// Dub's fourteen 72-90 ms): queued (its turn of the map), its door read, sent, the status, the body, every name
|
|
2351
|
+
// filled. Asks sent together and answered together late say the host or the tab was busy; one long between its
|
|
2352
|
+
// own sent and status says that twin's door.
|
|
2353
|
+
const origin = performance.now();
|
|
2354
|
+
const steps: Array<{ id: string; queued: number; times: CredentialAskTimes }> = [];
|
|
2355
|
+
const asks = await Promise.all(host.services.map(async (started) => {
|
|
2192
2356
|
const declared = colocated.find((s) => s.id === started.id);
|
|
2193
|
-
if (declared)
|
|
2357
|
+
if (!declared) return undefined;
|
|
2358
|
+
const began = performance.now();
|
|
2359
|
+
const times: CredentialAskTimes = { ...(declared.package ? { twin: declared.package } : {}) };
|
|
2360
|
+
steps.push({ id: started.id, queued: began, times });
|
|
2361
|
+
const own: Record<string, string> = { ...before };
|
|
2362
|
+
await issueCredentials(declared, started, own, issuing, unissued, ownedPath(data, started.id), times);
|
|
2363
|
+
return { id: started.id, ms: Math.round(performance.now() - began), own };
|
|
2364
|
+
}));
|
|
2365
|
+
const at = (time: number | undefined): string => time === undefined ? '-' : `+${Math.round(time - origin)}`;
|
|
2366
|
+
const named = steps.filter((step) => (step.times.names ?? 0) > 0);
|
|
2367
|
+
if (named.length) boot.progress(`co-located twins: each credential ask's steps, ms from the asks' start (queued/door/sent/status/body/filled): ${named
|
|
2368
|
+
.sort((left, right) => (right.times.filled ?? 0) - (left.times.filled ?? 0))
|
|
2369
|
+
.map(({ id, queued, times }) => `${id}${times.twin ? ` (${times.twin})` : ''} ${at(queued)}/${at(times.door)}/${at(times.sent)}/${at(times.headers)}/${at(times.answered)}/${at(times.filled)}`).join('; ')}`);
|
|
2370
|
+
const asked: Array<{ id: string; ms: number }> = [];
|
|
2371
|
+
for (const ask of asks) {
|
|
2372
|
+
if (!ask) continue;
|
|
2373
|
+
for (const [key, value] of Object.entries(ask.own)) if (before[key] !== value) env[key] = value;
|
|
2374
|
+
for (const key of Object.keys(before)) if (!Object.hasOwn(ask.own, key)) delete env[key];
|
|
2375
|
+
asked.push({ id: ask.id, ms: ask.ms });
|
|
2194
2376
|
}
|
|
2377
|
+
if (asked.length) boot.progress(`co-located twins: each asked for its credentials at once, slowest first: ${[...asked].sort((left, right) => right.ms - left.ms).slice(0, 6).map((each) => `${each.id} ${each.ms} ms`).join(', ')}; ${asked.length} asked, ${Math.max(...asked.map((each) => each.ms))} ms the longest`);
|
|
2195
2378
|
writeProxyEnv(root, name, env);
|
|
2196
2379
|
if (colocated.length) boot.progress('co-located twins: credentials issued, proxy environment written');
|
|
2197
2380
|
};
|
|
@@ -2215,9 +2398,10 @@ export async function upWorld(configId: string, options: UpWorldOptions = {}, ab
|
|
|
2215
2398
|
// possible) or the proxy daemon fails/times out, refuse rather than let unmodified CLIs bypass
|
|
2216
2399
|
// the twins undetected.
|
|
2217
2400
|
const attemptProxy = (): void => {
|
|
2218
|
-
if (proxyForServices
|
|
2401
|
+
if (proxyForServices) return;
|
|
2219
2402
|
// a host with no raw sockets (a browser tab) has nothing for the proxy to intercept; the injector holds the policy
|
|
2220
2403
|
if (!hostHasRawEgress()) return;
|
|
2404
|
+
if (Object.keys(activeVendorMap(env)).length === 0 && env[WORLD_NETWORK_POLICY_ENV] === undefined) return;
|
|
2221
2405
|
if (!opensslAvailable()) {
|
|
2222
2406
|
if (mode === 'sealed' || loaded.config.network !== undefined) {
|
|
2223
2407
|
throw new Error(
|
|
@@ -2284,12 +2468,12 @@ export async function upWorld(configId: string, options: UpWorldOptions = {}, ab
|
|
|
2284
2468
|
delete worldInstance.previousEnvOutput;
|
|
2285
2469
|
saveWorldInstance(worldInstance);
|
|
2286
2470
|
boot.progress(`environment written: ${envFile}`);
|
|
2287
|
-
|
|
2471
|
+
await materializeWorldBrowserFiles(data, root, name, loaded.config.id);
|
|
2288
2472
|
writePidsFromInstance(worldInstance, Object.values(worldInstance.services).map((service) => service.pid).filter((pid) => pid > 0));
|
|
2289
2473
|
saveWorldInstance(worldInstance);
|
|
2290
2474
|
const servicePids = [...new Set(Object.values(services).map((service) => service.pid).filter((pid) => pid > 0))];
|
|
2291
2475
|
handoffWorldLifecycle(root, name, servicePids);
|
|
2292
|
-
boot.progress(
|
|
2476
|
+
boot.progress(`returning: the World is up; ${lag.account()}`);
|
|
2293
2477
|
if (mode === 'share') {
|
|
2294
2478
|
try {
|
|
2295
2479
|
return await shareWorldServices(name, { root, ...(options.share ?? {}) });
|
|
@@ -2532,6 +2716,7 @@ export async function downWorld(
|
|
|
2532
2716
|
assertSafeWorldName(name);
|
|
2533
2717
|
const resolvedRoot = resolve(root);
|
|
2534
2718
|
const token = randomUUID();
|
|
2719
|
+
const ownershipErrors: string[] = [];
|
|
2535
2720
|
// Like boot's claim: a short synchronous lock protects an async lifecycle transition.
|
|
2536
2721
|
// Do not hold a filesystem lock while external shutdown or the grace period waits.
|
|
2537
2722
|
withFileLock(instanceLockFile(resolvedRoot, name), () => withFileLock(`${sharingClaimFile(resolvedRoot, name)}.lock`, () => {
|
|
@@ -2544,8 +2729,9 @@ export async function downWorld(
|
|
|
2544
2729
|
|| recorded.instance !== options.expectedInstance) throw new Error();
|
|
2545
2730
|
} catch { throw new Error(`World ownership mismatch: ${name}`); }
|
|
2546
2731
|
}
|
|
2547
|
-
|
|
2548
|
-
|
|
2732
|
+
// Unresolved startup evidence prevents finalization, not retirement of known children.
|
|
2733
|
+
let prior: WorldInstance | undefined;
|
|
2734
|
+
try { prior = readWorldInstance(name, resolvedRoot, true); } catch { /* diagnosed after owned children retire */ }
|
|
2549
2735
|
if (prior?.resources && !prior.lifecycle) throw new Error(`World "${name}" belongs to a legacy runtime; use its pinned runtime for teardown before migration`);
|
|
2550
2736
|
if (options.purge) assertWorldStateRemovable(resolvedRoot, instanceDir(resolvedRoot, name));
|
|
2551
2737
|
prepareBootForDown(resolvedRoot, name);
|
|
@@ -2560,11 +2746,15 @@ export async function downWorld(
|
|
|
2560
2746
|
if (!sharing || sharingClaimAlive(sharing)) throw new Error(`World "${name}" is being shared or has an unreadable sharing claim; retry after sharing finishes`);
|
|
2561
2747
|
rmSync(sharePath); // explicit down may reclaim a provably stale sharing claim
|
|
2562
2748
|
}
|
|
2563
|
-
retainWorldLifecycleForCleanup(resolvedRoot, name);
|
|
2749
|
+
retainWorldLifecycleForCleanup(resolvedRoot, name, error => { ownershipErrors.push(`Lifecycle ownership unreadable; retained: ${errorMessage(error)}`); });
|
|
2564
2750
|
writeTextAtomic(stoppingFile(resolvedRoot, name), JSON.stringify({ pid: process.pid, at: new Date().toISOString(), token }));
|
|
2565
2751
|
}));
|
|
2566
2752
|
try {
|
|
2567
|
-
|
|
2753
|
+
// The down's own signals and its whole time are said in the World's log (process-groups.mjs, worldEvent).
|
|
2754
|
+
const downAsked = Date.now();
|
|
2755
|
+
setWorldEventsFile(worldEventsPath(join(instanceDir(resolvedRoot, name), 'logs')));
|
|
2756
|
+
try { return await downWorldClaimed(name, resolvedRoot, options, ownershipErrors); }
|
|
2757
|
+
finally { worldEvent({ event: 'world.down', world: name, ms: Date.now() - downAsked }); setWorldEventsFile(undefined); }
|
|
2568
2758
|
} finally {
|
|
2569
2759
|
withFileLock(instanceLockFile(resolvedRoot, name), () => {
|
|
2570
2760
|
const path = stoppingFile(resolvedRoot, name);
|
|
@@ -2577,16 +2767,22 @@ async function downWorldClaimed(
|
|
|
2577
2767
|
name: string,
|
|
2578
2768
|
root: string,
|
|
2579
2769
|
options: { graceMs?: number; purge?: boolean },
|
|
2770
|
+
externalErrors: string[] = [],
|
|
2580
2771
|
): Promise<{ name: string; stopped: number[]; escalated: number[]; externalStopped: string[]; externalErrors: string[]; externalAbandoned?: string[]; notSignalled?: string[]; purged?: string }> {
|
|
2772
|
+
const clearBrowserFiles = options.purge ? (await import('./browsers.ts')).clearBrowserFiles : undefined;
|
|
2581
2773
|
assertSafeWorldName(name);
|
|
2582
2774
|
const resolvedRoot = resolve(root);
|
|
2583
2775
|
const dir = instanceDir(resolvedRoot, name);
|
|
2584
2776
|
const pidsFile = join(dir, 'pids');
|
|
2585
|
-
|
|
2777
|
+
let before: string | undefined;
|
|
2778
|
+
try { before = getActiveWorldStore().read(instanceFile(resolvedRoot, name)) ?? undefined; }
|
|
2779
|
+
catch (error) { externalErrors.push(`World instance unreadable; retained: ${errorMessage(error)}`); }
|
|
2586
2780
|
// A retry must not retain an earlier success receipt while teardown is in progress.
|
|
2587
2781
|
rmSync(join(dir, 'stopped.json'), { force: true });
|
|
2588
|
-
|
|
2589
|
-
|
|
2782
|
+
const guestError = (error: unknown) => {
|
|
2783
|
+
const message = errorMessage(error);
|
|
2784
|
+
if (!externalErrors.includes(message)) externalErrors.push(message);
|
|
2785
|
+
};
|
|
2590
2786
|
let instance: WorldInstance | undefined;
|
|
2591
2787
|
try {
|
|
2592
2788
|
instance = readWorldInstance(name, resolvedRoot);
|
|
@@ -2594,48 +2790,58 @@ async function downWorldClaimed(
|
|
|
2594
2790
|
} catch {
|
|
2595
2791
|
instance = undefined;
|
|
2596
2792
|
}
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
}
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
}
|
|
2793
|
+
// Read independent process records even when a guest receipt or instance is unreadable.
|
|
2794
|
+
const readGuests = () => {
|
|
2795
|
+
try { return readGuestProcesses(dir, instance?.instance, guestError); }
|
|
2796
|
+
catch (error) { guestError(error); return []; }
|
|
2797
|
+
};
|
|
2798
|
+
const guests = readGuests();
|
|
2799
|
+
let proxyBefore: string | null = null;
|
|
2800
|
+
try { proxyBefore = getActiveWorldStore().read(proxyStateFile(resolvedRoot, name)); }
|
|
2801
|
+
catch (error) { externalErrors.push(`Proxy process record unreadable; retained: ${errorMessage(error)}`); }
|
|
2606
2802
|
const proxyState = readProxyState(resolvedRoot, name);
|
|
2607
|
-
const proxyPid = proxyState
|
|
2608
|
-
|
|
2803
|
+
const proxyPid = proxyState && Number.isInteger(proxyState.pid) && proxyState.pid > 0 ? proxyState.pid : undefined;
|
|
2804
|
+
if (proxyBefore !== null && (!proxyPid || (proxyState?.birth !== undefined && (typeof proxyState.birth !== 'string' || !proxyState.birth)))) {
|
|
2805
|
+
externalErrors.push('Proxy process record unreadable; ownership retained');
|
|
2806
|
+
}
|
|
2609
2807
|
const foregroundPid = instance?.lastRun?.state !== 'completed' ? instance?.lastRun?.consumerPid : undefined;
|
|
2610
2808
|
const holderPid = instance?.lifecycle?.holderPid ?? instance?.resources?.holderPid;
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2809
|
+
let filePids: number[] = [];
|
|
2810
|
+
try { filePids = readPids(pidsFile, guests); }
|
|
2811
|
+
catch (error) { externalErrors.push(`World PID record unreadable; retained: ${errorMessage(error)}`); }
|
|
2812
|
+
const recordedPids = [...new Set([...(proxyPid ? [proxyPid] : []), ...filePids, ...(foregroundPid ? [foregroundPid] : []), ...(holderPid ? [holderPid] : [])])].filter(pid => Number.isInteger(pid) && pid > 0);
|
|
2813
|
+
const conflicts = new Set<number>();
|
|
2814
|
+
const conflictingIdentity = (pid: number) => {
|
|
2815
|
+
if (conflicts.has(pid)) return;
|
|
2816
|
+
conflicts.add(pid);
|
|
2817
|
+
externalErrors.push(`Conflicting spawn identities for pid ${pid}; not signalled; ownership retained`);
|
|
2818
|
+
};
|
|
2819
|
+
const identities = readIdentities(resolvedRoot, name, guests, conflictingIdentity);
|
|
2820
|
+
if (proxyPid && typeof proxyState?.birth === 'string' && proxyState.birth) {
|
|
2821
|
+
if (identities[proxyPid] && identities[proxyPid] !== proxyState.birth) conflictingIdentity(proxyPid);
|
|
2822
|
+
else identities[proxyPid] = proxyState.birth;
|
|
2823
|
+
}
|
|
2614
2824
|
const foreign = (await Promise.all(recordedPids.map(async pid => {
|
|
2615
2825
|
const now = identities[pid] ? await processBirth(pid) : undefined;
|
|
2616
2826
|
return now && now !== identities[pid] ? { pid, recorded: identities[pid]!, now } : undefined;
|
|
2617
2827
|
}))).filter((entry): entry is { pid: number; recorded: string; now: string } => !!entry);
|
|
2618
|
-
const pids = recordedPids.filter((pid) => !foreign.some((entry) => entry.pid === pid));
|
|
2828
|
+
const pids = recordedPids.filter((pid) => pid !== holderPid && !conflicts.has(pid) && !foreign.some((entry) => entry.pid === pid));
|
|
2619
2829
|
const processServices = Object.values(instance?.services ?? {}).filter((service) => service.type !== 'external' && service.log);
|
|
2620
2830
|
// Only a service still alive is stopped by this `down`; a dead one's end was not the World's.
|
|
2621
2831
|
const stillAlive = undeadPids(processServices.map((service) => service.pid).filter((pid) => pids.includes(pid)));
|
|
2622
2832
|
recordWorldStop(processServices.filter((service) => stillAlive.includes(service.pid)).map((service) => service.log), 'world.service.stop', 'down', 'SIGTERM');
|
|
2623
2833
|
const stopped = await signalPids(pids, 'SIGTERM', identities, spawnedChildren);
|
|
2624
2834
|
|
|
2625
|
-
//
|
|
2626
|
-
// safe only without retained lifecycle ownership, checked before any signals above.
|
|
2835
|
+
// Missing external metadata remains a finalization error after known children retire.
|
|
2627
2836
|
let externalStopped: string[] = [];
|
|
2628
|
-
let externalErrors: string[] = [];
|
|
2629
2837
|
let unavailable: string[] = [];
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
// no instance.json (never fully came up) — only owned processes needed stopping.
|
|
2638
|
-
if (before !== undefined) externalErrors.push('World instance or external teardown metadata unreadable; cleanup refused');
|
|
2838
|
+
if (instance) {
|
|
2839
|
+
try {
|
|
2840
|
+
const result = await teardownExternalServices(instance);
|
|
2841
|
+
externalStopped = result.stopped;
|
|
2842
|
+
externalErrors.push(...result.errors);
|
|
2843
|
+
unavailable = result.unavailable;
|
|
2844
|
+
} catch (error) { externalErrors.push(errorMessage(error)); }
|
|
2639
2845
|
}
|
|
2640
2846
|
|
|
2641
2847
|
// Escalate AFTER the external teardown so externals never wait on the grace, and
|
|
@@ -2645,11 +2851,14 @@ async function downWorldClaimed(
|
|
|
2645
2851
|
recordWorldStop(processServices.filter((service) => escalated.includes(service.pid)).map((service) => service.log), 'world.service.killed', 'down', 'SIGKILL');
|
|
2646
2852
|
// A launch which had already entered its spawn seam may publish while TERM is delivered.
|
|
2647
2853
|
// Once services/agents retire, join every late receipt before certifying this generation stopped.
|
|
2648
|
-
const
|
|
2854
|
+
const settlingGuests = readGuests();
|
|
2855
|
+
readIdentities(resolvedRoot, name, settlingGuests, conflictingIdentity);
|
|
2856
|
+
const lateGuests = settlingGuests.filter(({ record }) => !conflicts.has(record.pid)
|
|
2857
|
+
&& (!recordedPids.includes(record.pid) || identities[record.pid] !== record.start));
|
|
2649
2858
|
try { await retireOwnedProcessGroups(lateGuests.map(receipt => receipt.record), options.graceMs ?? DOWN_GRACE_MS_DEFAULT); }
|
|
2650
2859
|
catch (error) { externalErrors.push(errorMessage(error)); }
|
|
2651
2860
|
pids.push(...lateGuests.map(receipt => receipt.record.pid));
|
|
2652
|
-
if (holderPid &&
|
|
2861
|
+
if (holderPid && recordedPids.includes(holderPid) && !conflicts.has(holderPid) && !foreign.some(entry => entry.pid === holderPid)) {
|
|
2653
2862
|
pids.push(holderPid);
|
|
2654
2863
|
stopped.push(...await signalPids([holderPid], 'SIGTERM', identities, spawnedChildren));
|
|
2655
2864
|
escalated.push(...await killSurvivorsAfterGrace([holderPid], options.graceMs ?? DOWN_GRACE_MS_DEFAULT, identities, spawnedChildren));
|
|
@@ -2661,19 +2870,31 @@ async function downWorldClaimed(
|
|
|
2661
2870
|
const notSignalled = foreign.map((entry) => `pid ${entry.pid} now names another process (started ${entry.now}; the World's started ${entry.recorded}); not signalled`);
|
|
2662
2871
|
const result = { name, stopped, escalated, externalStopped, externalErrors, ...(notSignalled.length ? { notSignalled } : {}) };
|
|
2663
2872
|
const path = instanceFile(resolvedRoot, name);
|
|
2664
|
-
|
|
2873
|
+
let current: string | undefined;
|
|
2874
|
+
try { current = getActiveWorldStore().read(path) ?? undefined; }
|
|
2875
|
+
catch (error) { externalErrors.push(`World generation unreadable; finalization refused: ${errorMessage(error)}`); return result; }
|
|
2665
2876
|
if (current !== before || existsSync(bootingFile(resolvedRoot, name))) {
|
|
2666
2877
|
externalErrors.push('World generation changed or is booting; finalization refused');
|
|
2667
2878
|
return result;
|
|
2668
2879
|
}
|
|
2669
|
-
|
|
2880
|
+
let currentProxy: string | null;
|
|
2881
|
+
try { currentProxy = getActiveWorldStore().read(proxyStateFile(resolvedRoot, name)); }
|
|
2882
|
+
catch (error) { externalErrors.push(`Proxy record unreadable; finalization refused: ${errorMessage(error)}`); return result; }
|
|
2883
|
+
if ((currentProxy !== null && currentProxy !== proxyBefore) || existsSync(sharingClaimFile(resolvedRoot, name))) {
|
|
2670
2884
|
externalErrors.push('Proxy or sharing lifecycle changed during teardown; cleanup refused');
|
|
2671
2885
|
return result;
|
|
2672
2886
|
}
|
|
2673
|
-
|
|
2887
|
+
if (existsSync(proxyStartingFile(resolvedRoot, name))) {
|
|
2888
|
+
externalErrors.push(`World "${name}" has unresolved proxy startup ownership; recorded children retired where identity verified; lifecycle and instance retained. Inspect: ${proxyStartingFile(resolvedRoot, name)}`);
|
|
2889
|
+
}
|
|
2890
|
+
const guestReceipts = readGuests();
|
|
2674
2891
|
const surviving = undeadPids([...pids, ...guestReceipts.map(receipt => receipt.record.pid)]);
|
|
2675
2892
|
if (surviving.length) externalErrors.push(`World processes still alive or uninspectable after teardown: ${surviving.join(', ')}; cleanup refused`);
|
|
2676
2893
|
if (guestReceipts.some(receipt => foreign.some(entry => entry.pid === receipt.record.pid))) externalErrors.push('Guest spawn identity changed; receipt retained and cleanup refused');
|
|
2894
|
+
if (!instance && (before !== undefined || hasWorldLifecycle(resolvedRoot, name))
|
|
2895
|
+
&& !(before === undefined && !externalErrors.length && releaseAbandonedWorldLifecycle(resolvedRoot, name))) {
|
|
2896
|
+
externalErrors.push('World instance or external teardown metadata missing or unreadable; cleanup refused; lifecycle evidence retained');
|
|
2897
|
+
}
|
|
2677
2898
|
let externalAbandoned: string[] | undefined;
|
|
2678
2899
|
if (options.purge && instance && externalErrors.length && externalErrors.length === unavailable.length && !foreign.length) {
|
|
2679
2900
|
const inventory = exitedWorldLifecyclePids(resolvedRoot, name, instance.createdAt);
|
|
@@ -2687,6 +2908,8 @@ async function downWorldClaimed(
|
|
|
2687
2908
|
}
|
|
2688
2909
|
}
|
|
2689
2910
|
if (externalErrors.length) return result;
|
|
2911
|
+
// Keep proxy state and session trust until its recorded processes have exited.
|
|
2912
|
+
clearWorldProxyFiles(resolvedRoot, name);
|
|
2690
2913
|
releaseGuestProcesses(guestReceipts);
|
|
2691
2914
|
rmSync(pidsFile, { force: true });
|
|
2692
2915
|
if (before) {
|
|
@@ -2698,7 +2921,7 @@ async function downWorldClaimed(
|
|
|
2698
2921
|
releaseWorldLifecycle(resolvedRoot, name, externalAbandoned ? 'recorded processes exited; explicit purge surrendered state; ownership released' : undefined);
|
|
2699
2922
|
if (!options.purge) return result;
|
|
2700
2923
|
const purged = purgeInstanceDir(resolvedRoot, name);
|
|
2701
|
-
if (purged && instance) clearBrowserFiles(instance.dirs.data, resolvedRoot);
|
|
2924
|
+
if (purged && instance) clearBrowserFiles!(instance.dirs.data, resolvedRoot);
|
|
2702
2925
|
return { ...result, ...(externalAbandoned ? { externalAbandoned } : {}), purged };
|
|
2703
2926
|
}));
|
|
2704
2927
|
}
|
|
@@ -2708,7 +2931,9 @@ async function downWorldClaimed(
|
|
|
2708
2931
|
export type ServiceState = 'running' | 'stopped' | 'lingering' | 'gave-up' | 'external';
|
|
2709
2932
|
export function statusWorld(name: string, root = process.cwd(), options: { readOnly?: boolean; light?: boolean } = {}): WorldInstance & { running: boolean; livePids: number[]; serviceStates: Record<string, ServiceState>; degraded: boolean } {
|
|
2710
2933
|
const worldInstance = readWorldInstance(name, resolve(root), options.readOnly);
|
|
2711
|
-
|
|
2934
|
+
// One status, one reading of the process table: its questions below are about the same moment.
|
|
2935
|
+
const listing: ProcessListing = {};
|
|
2936
|
+
const live = ownedPids(resolve(root), name, livePids(readPids(worldInstance.pidsFile), listing));
|
|
2712
2937
|
const externalOnly = Object.values(worldInstance.services).every(service => service.type === 'external');
|
|
2713
2938
|
const recorded = externalOnly && !existsSync(stoppingFile(root, name)) && !existsSync(join(instanceDir(root, name), 'stopped.json'))
|
|
2714
2939
|
&& isWorldLifecycleRunning(root, name, worldInstance.createdAt);
|
|
@@ -2716,9 +2941,9 @@ export function statusWorld(name: string, root = process.cwd(), options: { readO
|
|
|
2716
2941
|
// read; a service's own state is judged by its process group, so a dead service reads as dead beside a live daemon.
|
|
2717
2942
|
const owned = Object.values(worldInstance.services).filter((service) => service.type !== 'external' && service.pid);
|
|
2718
2943
|
const ownPids = ownedPids(resolve(root), name, owned.map((service) => service.pid));
|
|
2719
|
-
const leaders = livePids(ownPids);
|
|
2944
|
+
const leaders = livePids(ownPids, listing);
|
|
2720
2945
|
// `light` (a listing of every World) skips the process-table scan: a lingering group then reads as stopped.
|
|
2721
|
-
const groups = ownPids.length && !options.light ? survivingOwnedGroups(ownPids) : [];
|
|
2946
|
+
const groups = ownPids.length && !options.light ? survivingOwnedGroups(ownPids, listing) : [];
|
|
2722
2947
|
const serviceStates: Record<string, ServiceState> = {};
|
|
2723
2948
|
for (const service of Object.values(worldInstance.services)) {
|
|
2724
2949
|
serviceStates[service.id] = service.type === 'external' ? 'external'
|
|
@@ -3571,7 +3796,7 @@ export function urlWorld(name: string, options: { service?: string; root?: strin
|
|
|
3571
3796
|
|
|
3572
3797
|
export async function runWorld(configId: string, command: string[], options: RunWorldOptions = {}): Promise<{ instance: WorldInstance; exitCode: number; outcome: WorldRunOutcome }> {
|
|
3573
3798
|
if (command.length === 0) throw new Error('Missing command after --');
|
|
3574
|
-
if (!options.keep) return superviseWorldRun(configId, command, options);
|
|
3799
|
+
if (!options.keep) return (await import('./run-task.ts')).superviseWorldRun(configId, command, options);
|
|
3575
3800
|
return runWorldOwned(configId, command, options);
|
|
3576
3801
|
}
|
|
3577
3802
|
|
|
@@ -3663,13 +3888,13 @@ export function listWorlds(root = process.cwd()): Array<{ name: string; running:
|
|
|
3663
3888
|
.sort((a, b) => a.name.localeCompare(b.name));
|
|
3664
3889
|
}
|
|
3665
3890
|
|
|
3666
|
-
function worldAttachedCommandEnv(name: string, root: string, verbose = false): { instance:
|
|
3667
|
-
const status = statusWorld(name, root);
|
|
3891
|
+
function worldAttachedCommandEnv(name: string, root: string, verbose = false, instance?: WorldInstance): { instance: WorldInstance; env: NodeJS.ProcessEnv } {
|
|
3892
|
+
const status = instance ?? statusWorld(name, root);
|
|
3668
3893
|
const sealed = status.env.VOLTER_WORLD_MODE === 'sealed' || status.env[WORLD_NETWORK_POLICY_ENV] !== undefined;
|
|
3669
3894
|
const baseEnv = { ...admittedEnv(stripEnvOf(status.env)), ...status.env };
|
|
3670
3895
|
let proxyEnv: Record<string, string> = {};
|
|
3671
3896
|
try {
|
|
3672
|
-
const proxy =
|
|
3897
|
+
const proxy = ensureWorldProxyForCommand(name, root, [admittedEnv(stripEnvOf(status.env)), status.env], instance);
|
|
3673
3898
|
if (proxy?.url) {
|
|
3674
3899
|
proxyEnv = proxy.env;
|
|
3675
3900
|
// a host with no raw sockets (a browser tab) runs no proxy: the injector holds the policy there, as at `up`
|
|
@@ -3759,19 +3984,19 @@ export function retireWorldConsumers(name: string, root = process.cwd(), options
|
|
|
3759
3984
|
});
|
|
3760
3985
|
}
|
|
3761
3986
|
|
|
3762
|
-
export async function attachWorld(name: string, command: string[], root = process.cwd(), options: { cwd?: string; owner?: string; verbose?: boolean } = {}): Promise<number> {
|
|
3987
|
+
export async function attachWorld(name: string, command: string[], root = process.cwd(), options: { cwd?: string; owner?: string; verbose?: boolean; instance?: WorldInstance } = {}): Promise<number> {
|
|
3763
3988
|
if (command.length === 0) throw new Error('Missing command after --');
|
|
3764
3989
|
assertSafeWorldName(name);
|
|
3765
3990
|
const resolvedRoot = resolve(root);
|
|
3766
3991
|
checkedOwner(options.owner);
|
|
3767
|
-
const attached = worldAttachedCommandEnv(name, resolvedRoot, options.verbose);
|
|
3992
|
+
const attached = worldAttachedCommandEnv(name, resolvedRoot, options.verbose, options.instance);
|
|
3768
3993
|
const record = withFileLock(instanceLockFile(resolvedRoot, name), () => {
|
|
3769
3994
|
if (existsSync(stoppingFile(resolvedRoot, name)) || existsSync(bootingFile(resolvedRoot, name))) {
|
|
3770
3995
|
throw new Error(`World "${name}" is changing lifecycle; retry after it finishes`);
|
|
3771
3996
|
}
|
|
3772
3997
|
const current = statusWorld(name, resolvedRoot);
|
|
3773
3998
|
// Which of the two it was, with what was read: "stopped" and "another instance" are different repairs.
|
|
3774
|
-
if (!current.running) throw new Error(`World "${name}" is not running while attaching: none of its recorded processes is live and its own (recorded pids ${readPids(current.pidsFile).join(', ') || 'none'}; live and its own: none)`);
|
|
3999
|
+
if (!current.running) throw new Error(`World "${name}" is not running while attaching: none of its recorded processes is live and its own (recorded pids ${readPids(current.pidsFile).join(', ') || 'none'}; live and its own: none); ${describeWorldLifecycle(resolvedRoot, name, current.createdAt)})`);
|
|
3775
4000
|
if (current.createdAt !== attached.instance.createdAt) throw new Error(`World "${name}" changed while attaching: its instance was created ${attached.instance.createdAt} when the command's environment was read and ${current.createdAt} now`);
|
|
3776
4001
|
// Attaching to a World whose services died is allowed (a command may not need them), but never silent.
|
|
3777
4002
|
for (const [id, state] of Object.entries(current.serviceStates)) {
|
|
@@ -3781,7 +4006,7 @@ export async function attachWorld(name: string, command: string[], root = proces
|
|
|
3781
4006
|
process.stderr.write(`World "${name}": service ${id} is not running${end ? `: it ${describeServiceEnd(end)}` : ''}${service?.log ? `\n log: ${service.log}` : ''}\n`);
|
|
3782
4007
|
}
|
|
3783
4008
|
return createConsumer(resolvedRoot, name, current.createdAt, 'attach', options.owner ?? current.lifecycle?.owner ?? current.resources?.owner, { completionRequired: true });
|
|
3784
|
-
});
|
|
4009
|
+
}, { operation: 'instance attach' });
|
|
3785
4010
|
let timer: ReturnType<typeof setInterval> | undefined;
|
|
3786
4011
|
let retired = true;
|
|
3787
4012
|
try {
|
|
@@ -3938,6 +4163,7 @@ export async function shellWorld(name: string, root = process.cwd()): Promise<nu
|
|
|
3938
4163
|
process.stderr.write(`world '${name}' active in a subshell — vendor calls hit the twins (this host opens no raw sockets; the injector holds the network policy). type 'exit' to leave.\n`);
|
|
3939
4164
|
} else if (opensslAvailable()) {
|
|
3940
4165
|
try {
|
|
4166
|
+
const { startRedirectProxy } = await import('./redirect-proxy.ts');
|
|
3941
4167
|
proxy = await startRedirectProxy({ env, trustEnv: [admittedEnv(stripEnvOf(env)), env], tlsDir: tlsDir(resolvedRoot, name) });
|
|
3942
4168
|
Object.assign(env, proxy.proxyEnv());
|
|
3943
4169
|
process.stderr.write(`world '${name}' active in a subshell — vendor calls hit the twins (ambient proxy ${proxy.url}). type 'exit' to leave.\n`);
|
|
@@ -3976,13 +4202,17 @@ function proxyDaemonDeadlineMs(): number {
|
|
|
3976
4202
|
* reuses a live daemon. Returns its proxy env, or null if openssl is missing (env-only fallback). The
|
|
3977
4203
|
* daemon is recorded in `proxy.json` and stopped by `down`. */
|
|
3978
4204
|
export function ensureWorldProxy(name: string, root = process.cwd(), caller: CaTrustInputs = process.env): { url: string; caCertPath: string; env: Record<string, string> } | null {
|
|
4205
|
+
return ensureWorldProxyForCommand(name, root, caller);
|
|
4206
|
+
}
|
|
4207
|
+
|
|
4208
|
+
function ensureWorldProxyForCommand(name: string, root: string, caller: CaTrustInputs, instance?: WorldInstance): { url: string; caCertPath: string; env: Record<string, string> } | null {
|
|
3979
4209
|
assertSafeWorldName(name);
|
|
3980
4210
|
const resolvedRoot = resolve(root);
|
|
3981
4211
|
return withFileLock(instanceLockFile(resolvedRoot, name), () => {
|
|
3982
4212
|
assertProxyOwnershipResolved(resolvedRoot, name);
|
|
3983
4213
|
if (liveBootingClaim(stoppingFile(resolvedRoot, name))) throw new Error(`World "${name}" is being stopped`);
|
|
3984
4214
|
if (liveBootingClaim(bootingFile(resolvedRoot, name))) throw new Error(`World "${name}" is being booted`);
|
|
3985
|
-
if (!statusWorld(name, resolvedRoot, { readOnly: true }).running) throw new Error(`World "${name}" is not running`);
|
|
4215
|
+
if (!instance && !statusWorld(name, resolvedRoot, { readOnly: true }).running) throw new Error(`World "${name}" is not running`);
|
|
3986
4216
|
return ensureWorldProxyUnlocked(name, resolvedRoot, caller);
|
|
3987
4217
|
});
|
|
3988
4218
|
}
|
|
@@ -4153,6 +4383,7 @@ export async function runProxyDaemon(name: string, root = process.cwd(), envFile
|
|
|
4153
4383
|
if (!status.running) throw new Error(`World "${name}" is not running`);
|
|
4154
4384
|
env = { ...admittedEnv(stripEnvOf(status.env)), ...status.env };
|
|
4155
4385
|
}
|
|
4386
|
+
const { startRedirectProxy } = await import('./redirect-proxy.ts');
|
|
4156
4387
|
const proxy = await startRedirectProxy({ env, envLoader, tlsDir: tlsDir(resolvedRoot, name), note: (line) => process.stderr.write(`${line}\n`) });
|
|
4157
4388
|
const identity = await spawnIdentity(process.pid);
|
|
4158
4389
|
if (!identity.start) { await proxy.close(); throw new Error('Proxy birth capture failed'); }
|
|
@@ -4175,8 +4406,8 @@ export async function runProxyDaemon(name: string, root = process.cwd(), envFile
|
|
|
4175
4406
|
process.on('SIGINT', stop);
|
|
4176
4407
|
}
|
|
4177
4408
|
|
|
4178
|
-
/**
|
|
4179
|
-
*
|
|
4409
|
+
/** Failed boot's proxy retirement; normal down retires it with the other recorded children
|
|
4410
|
+
* and clears session trust only after confirming exit. */
|
|
4180
4411
|
async function stopWorldProxy(root: string, name: string, handles?: Map<number, ReturnType<typeof spawn>>): Promise<void> {
|
|
4181
4412
|
const state = readProxyState(root, name);
|
|
4182
4413
|
if (state && isAlive(state.pid)) {
|
|
@@ -4189,8 +4420,12 @@ async function stopWorldProxy(root: string, name: string, handles?: Map<number,
|
|
|
4189
4420
|
await signalOwnedProcess(identity, 'SIGTERM');
|
|
4190
4421
|
}
|
|
4191
4422
|
}
|
|
4192
|
-
|
|
4193
|
-
|
|
4423
|
+
clearWorldProxyFiles(root, name);
|
|
4424
|
+
}
|
|
4425
|
+
|
|
4426
|
+
function clearWorldProxyFiles(root: string, name: string): void {
|
|
4427
|
+
rmSync(proxyStateFile(root, name), { force: true });
|
|
4428
|
+
rmSync(proxyEnvFile(root, name), { force: true });
|
|
4194
4429
|
tearDownCa(tlsDir(root, name));
|
|
4195
4430
|
}
|
|
4196
4431
|
|