@henols/vice-mcp 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -0
- package/build.ts +268 -0
- package/container-guard.mts +226 -0
- package/containerpath.ts +296 -0
- package/hostpath.ts +318 -0
- package/incident-record.ts +442 -0
- package/install-resources.ts +532 -0
- package/package.json +71 -0
- package/refresh-manifest.ts +103 -0
- package/repo-root.ts +198 -0
- package/resources/broker-control.mjs +343 -0
- package/resources/broker-epoch.mjs +126 -0
- package/resources/broker-kill.mjs +491 -0
- package/resources/broker-launch.mjs +659 -0
- package/resources/broker-state.mjs +173 -0
- package/resources/container-guard.mjs +211 -0
- package/resources/vice-broker.mjs +855 -0
- package/resources/vice-launcher.sh +169 -0
- package/tools-manifest.json +1231 -0
- package/vice-broker-client.ts +899 -0
- package/vice-probe.ts +278 -0
- package/vice-proxy.ts +3093 -0
- package/vice-sync.ts +336 -0
- package/vice.ts +772 -0
|
@@ -0,0 +1,659 @@
|
|
|
1
|
+
// GENERATED FILE -- DO NOT EDIT.
|
|
2
|
+
// Compiled by `tsc` from broker-launch.mts. Edit the TypeScript source and rebuild;
|
|
3
|
+
// changes made directly to this file are silently overwritten by the next build, and are never
|
|
4
|
+
// deployed to the host on their own -- install-resources.mjs copies THIS file's on-disk contents
|
|
5
|
+
// verbatim to tools/, so an edit made only here reaches the host but is lost on the very next
|
|
6
|
+
// rebuild.
|
|
7
|
+
// broker-launch.mts
|
|
8
|
+
//
|
|
9
|
+
// C (complete, plan 02 of Phase 01.6.2): the single `in_flight` launch-guard
|
|
10
|
+
// owner (plan 01, unchanged -- every launch call site in the whole broker
|
|
11
|
+
// goes through tryLaunchOne(), which is what makes the single-owner
|
|
12
|
+
// guarantee mechanical rather than a convention plan 02's own concurrency
|
|
13
|
+
// race test can silently violate), PLUS the readiness probe's single
|
|
14
|
+
// in-process mechanism (collapsed from a three-way branch by Phase
|
|
15
|
+
// 01.6.2.1's own plan 02 -- D-05 as amended by P-05/P-06/P-07; see
|
|
16
|
+
// probeReady()'s own header comment below for the amendment's record),
|
|
17
|
+
// serialised warm-floor maintenance (one launch per pass, never more), and
|
|
18
|
+
// the fixed-order evaluation pass both surviving concerns run through.
|
|
19
|
+
//
|
|
20
|
+
// Plan 03, Task 2 grows this module into a real per-child supervisor
|
|
21
|
+
// (C2/D-23), absorbing resources/vice-supervisor.sh wholesale: superviseChild()
|
|
22
|
+
// launches an instance through tryLaunchOne() (the SAME single guarded
|
|
23
|
+
// primitive above) and installs an exit handler on the spawned child that
|
|
24
|
+
// respawns on crash (doubling backoff, clamped at a ceiling), gives up
|
|
25
|
+
// cleanly after too many crashes inside a window, never respawns a
|
|
26
|
+
// deliberately-killed instance, and writes the per-instance boot/crash log
|
|
27
|
+
// D-23 preserves at the exact path shape the retiring bash supervisor used.
|
|
28
|
+
import { spawn as nodeSpawn } from "node:child_process";
|
|
29
|
+
import { mkdirSync, openSync, closeSync, existsSync } from "node:fs";
|
|
30
|
+
import { join, basename } from "node:path";
|
|
31
|
+
// Module-level: this file, not the caller, owns the single boolean --
|
|
32
|
+
// synchronous check, synchronous set, released in a finally, with no
|
|
33
|
+
// `await` between the check and the set.
|
|
34
|
+
//
|
|
35
|
+
// D-07 (01.6.2.1-03-PLAN.md): launch PRIORITY is layered on this owner, and
|
|
36
|
+
// never replaces or weakens it. An in-flight boot always completes and is
|
|
37
|
+
// NEVER killed or abandoned to serve a later arrival -- preemption was
|
|
38
|
+
// considered and rejected (01.6.2-CONTEXT.md D-07) because a kill/relaunch
|
|
39
|
+
// overlap re-creates the exact concurrent-spawn window the 2026-08-01
|
|
40
|
+
// outage came from (one SEGV, one exit 1, one exit 0 at the identical spawn
|
|
41
|
+
// second). Once a boot reaches `ready`, a waiting request takes it
|
|
42
|
+
// regardless of which reason booted it (vice-broker.mts's
|
|
43
|
+
// selectWarmInstance() performs no `reason` check at all -- proven by
|
|
44
|
+
// vice-broker-acquire.test.ts). Priority governs only which REASON wins
|
|
45
|
+
// this slot once it next frees, via the fixed pass order (runBrokerPass()'s
|
|
46
|
+
// own invariant comment, below) -- it never decides who currently holds it.
|
|
47
|
+
let inFlight = false;
|
|
48
|
+
// The reason currently holding the slot, alongside `inFlight` -- metadata
|
|
49
|
+
// only, never a second guard: nothing branches on this value's presence to
|
|
50
|
+
// decide whether a launch may proceed (that is `inFlight` alone, checked
|
|
51
|
+
// and set synchronously exactly as before). Its only consumer is the
|
|
52
|
+
// launch-slot decision log line (D-07's standing constraint that a
|
|
53
|
+
// lifecycle decision must be reconstructable from the log after an
|
|
54
|
+
// incident -- both 2026-08-01 and 2026-08-02 were diagnosed from broker log
|
|
55
|
+
// lines).
|
|
56
|
+
let inFlightReason = null;
|
|
57
|
+
/** True while a launch is in progress -- exported for the race test plan 02
|
|
58
|
+
* writes against two concurrent tryLaunchOne() calls. */
|
|
59
|
+
export function isLaunchInFlight() {
|
|
60
|
+
return inFlight;
|
|
61
|
+
}
|
|
62
|
+
/** Resolves the emulator's own argument vector. Two shapes, matching
|
|
63
|
+
* resources/vice-supervisor.sh's own VICE_ARGS convention exactly: if
|
|
64
|
+
* VICE_ARGS is set in the environment (a single space-separated string,
|
|
65
|
+
* fully overridable -- vice-supervisor.sh's own header comment), it is used
|
|
66
|
+
* AS-IS; otherwise the MCP server flags are constructed the way the bash
|
|
67
|
+
* launcher builds them -- the MCP server flag, the MCP server host from
|
|
68
|
+
* VICE_BROKER_MCP_HOST (default 0.0.0.0), and the MCP server port set to
|
|
69
|
+
* the allocated port. The override exists because this broker's own tests
|
|
70
|
+
* (and an operator's manual dry runs) need to launch a stand-in binary
|
|
71
|
+
* (e.g. /bin/sleep) that does not understand -mcpserver flags. */
|
|
72
|
+
export function buildViceArgs(port, { mcpHost, viceArgsEnv } = {}) {
|
|
73
|
+
const rawViceArgs = viceArgsEnv ?? process.env.VICE_ARGS;
|
|
74
|
+
if (typeof rawViceArgs === "string" && rawViceArgs.trim() !== "") {
|
|
75
|
+
return rawViceArgs.trim().split(/\s+/);
|
|
76
|
+
}
|
|
77
|
+
const host = mcpHost ?? process.env.VICE_BROKER_MCP_HOST ?? "0.0.0.0";
|
|
78
|
+
return ["-mcpserver", "-mcpserverhost", host, "-mcpserverport", String(port)];
|
|
79
|
+
}
|
|
80
|
+
/** The unguarded spawn+record primitive -- no in_flight check here at all.
|
|
81
|
+
* Called from exactly two places: tryLaunchOne() below (which wraps it in
|
|
82
|
+
* the standalone synchronous guard) and acquirePortAndLaunch() further
|
|
83
|
+
* down (which holds that SAME guard across its own async port-allocation
|
|
84
|
+
* step first, then calls this directly so the guard is never
|
|
85
|
+
* double-checked against itself). Spawns via deps.spawn (defaulting to
|
|
86
|
+
* Node's own child_process.spawn), records the resolved binary path at
|
|
87
|
+
* spawn time into the instance record's expectedIdentity field -- the
|
|
88
|
+
* string the kill discipline (broker-kill.mts) checks identity against,
|
|
89
|
+
* and the VICE_BIN binary this broker spawns directly, never any
|
|
90
|
+
* intermediate script path. Logs the resolved command line before
|
|
91
|
+
* spawning, so a bad configuration value is visible rather than silently
|
|
92
|
+
* mis-parsed, exactly like the bash launcher's own logging discipline. */
|
|
93
|
+
function spawnAndRecordInstance(reason, port, deps) {
|
|
94
|
+
const spawnFn = deps.spawn ?? ((cmd, args) => nodeSpawn(cmd, args));
|
|
95
|
+
const now = deps.now ?? (() => Date.now());
|
|
96
|
+
const viceBin = deps.viceBin ?? process.env.VICE_BIN ?? "x64sc";
|
|
97
|
+
const viceArgs = buildViceArgs(port, { mcpHost: deps.mcpHost });
|
|
98
|
+
const log = deps.log ?? defaultLog;
|
|
99
|
+
log(`vice-broker: launching ${viceBin} ${viceArgs.join(" ")}`);
|
|
100
|
+
const child = spawnFn(viceBin, viceArgs);
|
|
101
|
+
const record = {
|
|
102
|
+
port,
|
|
103
|
+
url: `http://127.0.0.1:${port}/mcp`,
|
|
104
|
+
state: "launching",
|
|
105
|
+
reason,
|
|
106
|
+
epochFile: deps.epochFile,
|
|
107
|
+
supervisorDir: deps.supervisorDir,
|
|
108
|
+
pid: child.pid ?? null,
|
|
109
|
+
expectedIdentity: viceBin,
|
|
110
|
+
launchedAt: now(),
|
|
111
|
+
readyAt: null,
|
|
112
|
+
viceBin,
|
|
113
|
+
viceArgs,
|
|
114
|
+
dryRun: false,
|
|
115
|
+
};
|
|
116
|
+
deps.state.instances.set(port, record);
|
|
117
|
+
return record;
|
|
118
|
+
}
|
|
119
|
+
/** The single in_flight owner, for a caller that ALREADY knows its port.
|
|
120
|
+
* Fully SYNCHRONOUS by design -- no `await` anywhere between the guard
|
|
121
|
+
* check and the guard release. This is what makes the single-owner
|
|
122
|
+
* guarantee hold even under concurrent CALLERS: JS's run-to-completion
|
|
123
|
+
* semantics mean two invocations of a synchronous function can never
|
|
124
|
+
* interleave, regardless of how many async callers race to reach it. The
|
|
125
|
+
* moment this function itself grows an internal `await` between the check
|
|
126
|
+
* and the set, that guarantee is lost -- see broker-launch.test.ts's own
|
|
127
|
+
* "discriminating power" regression check for a demonstration.
|
|
128
|
+
*
|
|
129
|
+
* This is the RIGHT primitive when the port is already decided and fixed
|
|
130
|
+
* (most tests; any future caller with its own allocation scheme). It is
|
|
131
|
+
* deliberately NOT what handleAcquire or maintainWarmFloor call for a
|
|
132
|
+
* FRESH port, because nextFreePort() itself is asynchronous (a real
|
|
133
|
+
* port-in-use probe requires it) -- see acquirePortAndLaunch()'s own
|
|
134
|
+
* header comment for the race that creates and how it is closed. */
|
|
135
|
+
export function tryLaunchOne(reason, port, deps) {
|
|
136
|
+
if (inFlight)
|
|
137
|
+
return null;
|
|
138
|
+
inFlight = true;
|
|
139
|
+
try {
|
|
140
|
+
return spawnAndRecordInstance(reason, port, deps);
|
|
141
|
+
}
|
|
142
|
+
finally {
|
|
143
|
+
inFlight = false;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
/** Holds the SAME single in_flight owner across the ENTIRE
|
|
147
|
+
* allocate-a-port-then-launch sequence -- not merely the synchronous spawn
|
|
148
|
+
* instant tryLaunchOne() alone guards. This closes a genuine race window
|
|
149
|
+
* tryLaunchOne() cannot: nextFreePort()'s own port-in-use probe is
|
|
150
|
+
* asynchronous (plan 02, C4 -- a real bind-and-release check), so two
|
|
151
|
+
* overlapping callers (a cold acquire arriving over the TCP control
|
|
152
|
+
* listener at any moment, and a warm-floor pass on its own poll timer)
|
|
153
|
+
* could otherwise BOTH be told the SAME candidate port is free before
|
|
154
|
+
* either commits it to state.instances -- a double-launch on one port,
|
|
155
|
+
* silently overwriting the earlier record. The guard is checked and set
|
|
156
|
+
* SYNCHRONOUSLY before the first `await`, exactly like tryLaunchOne()'s
|
|
157
|
+
* own discipline, so a second concurrent call is refused immediately
|
|
158
|
+
* (`launch_in_flight`) rather than racing on the allocation.
|
|
159
|
+
*
|
|
160
|
+
* This is also the function that restores vice-broker.sh's own
|
|
161
|
+
* process_requests() throttle (its `in_flight` local, checked before a
|
|
162
|
+
* COLD launch, not only before a warm one): a cold acquire and a
|
|
163
|
+
* warm-floor pass can never launch simultaneously, matching the bash
|
|
164
|
+
* original's declined-to-change behaviour (RESEARCH.md §A1/§C). D-07
|
|
165
|
+
* (01.6.2.1-03-PLAN.md) layers non-preemptive PRIORITY on top of this same
|
|
166
|
+
* "one at a time" guard, never replacing it: this function still only ever
|
|
167
|
+
* refuses a second concurrent caller (`launch_in_flight`), and never kills
|
|
168
|
+
* or preempts whichever caller already holds the slot -- which reason wins
|
|
169
|
+
* this slot NEXT, once it frees, falls out of runBrokerPass()'s own fixed
|
|
170
|
+
* evaluation order (that function's own invariant comment), not from
|
|
171
|
+
* anything in this function. The refusal below logs which reason currently
|
|
172
|
+
* holds the slot and which reason is waiting, so the decision is
|
|
173
|
+
* reconstructable from the log after an incident. */
|
|
174
|
+
export async function acquirePortAndLaunch(reason, deps) {
|
|
175
|
+
const log = deps.log ?? defaultLog;
|
|
176
|
+
if (inFlight) {
|
|
177
|
+
log(`vice-broker: launch-slot decision -- ${inFlightReason ?? "unknown"} holds the slot; ${reason} waits (D-07)`);
|
|
178
|
+
return { ok: false, reason: "launch_in_flight" };
|
|
179
|
+
}
|
|
180
|
+
inFlight = true;
|
|
181
|
+
inFlightReason = reason;
|
|
182
|
+
try {
|
|
183
|
+
const portResult = await deps.allocatePort(deps.state);
|
|
184
|
+
if (!portResult.ok) {
|
|
185
|
+
return { ok: false, reason: "no_free_port" };
|
|
186
|
+
}
|
|
187
|
+
const port = portResult.port;
|
|
188
|
+
const supervisorDir = join(deps.stateDir, String(port));
|
|
189
|
+
const epochFile = join(supervisorDir, "epoch.json");
|
|
190
|
+
const spawn = deps.spawnFactory ? deps.spawnFactory(port) : deps.spawn;
|
|
191
|
+
const record = spawnAndRecordInstance(reason, port, {
|
|
192
|
+
state: deps.state,
|
|
193
|
+
supervisorDir,
|
|
194
|
+
epochFile,
|
|
195
|
+
spawn,
|
|
196
|
+
now: deps.now,
|
|
197
|
+
viceBin: deps.viceBin,
|
|
198
|
+
mcpHost: deps.mcpHost,
|
|
199
|
+
});
|
|
200
|
+
return { ok: true, record };
|
|
201
|
+
}
|
|
202
|
+
finally {
|
|
203
|
+
inFlight = false;
|
|
204
|
+
inFlightReason = null;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
const DEFAULT_PROBE_TIMEOUT_S = 1;
|
|
208
|
+
function defaultLog(line) {
|
|
209
|
+
process.stderr.write(`${line}\n`);
|
|
210
|
+
}
|
|
211
|
+
/** A single POST of a tools/call for vice_ping at the instance's own URL,
|
|
212
|
+
* bounded by the probe timeout -- matching the exact single-POST curl form
|
|
213
|
+
* vice-broker.sh's own probe_ready() used. Treated as ready ONLY when the
|
|
214
|
+
* response body carries BOTH the "version" and "machine" substrings a real
|
|
215
|
+
* vice_ping reply contains; a bare TCP accept is explicitly not sufficient
|
|
216
|
+
* (a C64 can accept a connection before it has finished booting). */
|
|
217
|
+
async function defaultHttpProbe(port, timeoutMs) {
|
|
218
|
+
const body = JSON.stringify({
|
|
219
|
+
jsonrpc: "2.0",
|
|
220
|
+
id: 1,
|
|
221
|
+
method: "tools/call",
|
|
222
|
+
params: { name: "vice_ping", arguments: {} },
|
|
223
|
+
});
|
|
224
|
+
const controller = new AbortController();
|
|
225
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
226
|
+
try {
|
|
227
|
+
const response = await fetch(`http://127.0.0.1:${port}/mcp`, {
|
|
228
|
+
method: "POST",
|
|
229
|
+
headers: { "Content-Type": "application/json", Accept: "application/json" },
|
|
230
|
+
body,
|
|
231
|
+
signal: controller.signal,
|
|
232
|
+
});
|
|
233
|
+
const text = await response.text();
|
|
234
|
+
return text.includes("version") && text.includes("machine");
|
|
235
|
+
}
|
|
236
|
+
catch {
|
|
237
|
+
return false;
|
|
238
|
+
}
|
|
239
|
+
finally {
|
|
240
|
+
clearTimeout(timer);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
/** D-05, AS AMENDED BY P-05 -- this comment is the amendment's record, kept
|
|
244
|
+
* in the exact place a three-branch description used to sit, per this
|
|
245
|
+
* plan's own instruction that a code reader must meet the amendment here,
|
|
246
|
+
* not merely in the plan text (`01.6.2.1-02-PLAN.md`) or the validation
|
|
247
|
+
* ledger (`01.6.2-VALIDATION.md`, consolidated by plan 06).
|
|
248
|
+
*
|
|
249
|
+
* D-05 (locked, `01.6.2-CONTEXT.md`) originally specified the probe as a
|
|
250
|
+
* bare in-process TCP connect to the instance's own monitor port, with a
|
|
251
|
+
* short timeout. The code that landed instead argued against that wording,
|
|
252
|
+
* in its OWN comment: "a bare TCP accept is explicitly not sufficient (a
|
|
253
|
+
* C64 can accept a connection before it has finished booting)" -- a
|
|
254
|
+
* booting emulator promoted to ready on nothing more than an accepted
|
|
255
|
+
* connection is exactly Defect 3's failure shape reappearing, here on the
|
|
256
|
+
* plan-01 acquire hot path.
|
|
257
|
+
*
|
|
258
|
+
* P-05 amends D-05: the ping-shaped request body stays -- it is what
|
|
259
|
+
* proves the emulator ANSWERS, not merely that a port is bound, which is
|
|
260
|
+
* the whole difference between a liveness check and a readiness check. The
|
|
261
|
+
* two OTHER branches the landed code carried (an external-command
|
|
262
|
+
* mechanism, and a "neither mechanism available -> report ready
|
|
263
|
+
* unconditionally" fallback) retire outright, per P-06: with no second
|
|
264
|
+
* mechanism to prefer and no "no mechanism" state left to report, there is
|
|
265
|
+
* no longer a pair of indistinguishable states (a deliberately-zero warm
|
|
266
|
+
* floor and a broken host) for an operator to confuse in the logs. D-05's
|
|
267
|
+
* own intent -- exactly one check, no external command, no ambiguity -- is
|
|
268
|
+
* fully honoured by this collapse, not reversed by it.
|
|
269
|
+
*
|
|
270
|
+
* No retry loop, deliberately: a still-booting instance simply fails THIS
|
|
271
|
+
* pass and is re-probed on the next one (maintainWarmFloor()'s own per-pass
|
|
272
|
+
* cadence, or a later grant-time re-probe) -- this is what makes the
|
|
273
|
+
* shortened ~1s default below safe rather than reckless: a slow host is
|
|
274
|
+
* re-probed, never starved, and the seconds-valued timeout knob
|
|
275
|
+
* (VICE_BROKER_PROBE_TIMEOUT_S) still lets an operator on a slow host raise
|
|
276
|
+
* it. */
|
|
277
|
+
export async function probeReady(port, deps = {}) {
|
|
278
|
+
const timeoutS = Number(deps.probeTimeoutSEnv ?? process.env.VICE_BROKER_PROBE_TIMEOUT_S) || DEFAULT_PROBE_TIMEOUT_S;
|
|
279
|
+
const timeoutMs = timeoutS * 1000;
|
|
280
|
+
const httpProbe = deps.httpProbe ?? defaultHttpProbe;
|
|
281
|
+
return httpProbe(port, timeoutMs);
|
|
282
|
+
}
|
|
283
|
+
// ---------------------------------------------------------------------------
|
|
284
|
+
// Serialised warm-floor maintenance
|
|
285
|
+
// ---------------------------------------------------------------------------
|
|
286
|
+
function resolveWarmFloor(override) {
|
|
287
|
+
if (typeof override === "number")
|
|
288
|
+
return override;
|
|
289
|
+
const raw = process.env.VICE_BROKER_WARM_FLOOR;
|
|
290
|
+
if (raw === undefined || raw === "")
|
|
291
|
+
return 1;
|
|
292
|
+
const n = Number(raw);
|
|
293
|
+
return Number.isFinite(n) ? n : 1;
|
|
294
|
+
}
|
|
295
|
+
function resolveCeiling(override) {
|
|
296
|
+
if (typeof override === "number")
|
|
297
|
+
return override;
|
|
298
|
+
const raw = process.env.VICE_BROKER_MAX;
|
|
299
|
+
if (raw === undefined || raw === "")
|
|
300
|
+
return 16;
|
|
301
|
+
const n = Number(raw);
|
|
302
|
+
return Number.isFinite(n) ? n : 16;
|
|
303
|
+
}
|
|
304
|
+
/** Promotes launching instances via probe, then -- unless a launch is
|
|
305
|
+
* already in flight -- launches AT MOST ONE instance toward the warm floor
|
|
306
|
+
* and returns. Never loops to reach the floor in one call: reaching
|
|
307
|
+
* VICE_BROKER_WARM_FLOOR this way costs one
|
|
308
|
+
* additional CALL per warm instance instead of one call total, which is the exact
|
|
309
|
+
* trade the 2026-08-01 outage made non-negotiable (three simultaneous
|
|
310
|
+
* x64sc launches: one SEGV, one exit 1, one exit 0 at the identical spawn
|
|
311
|
+
* second). `async`/`await` makes launching everything needed in one go
|
|
312
|
+
* look free and idiomatic; it is actively dangerous here. DO NOT gather
|
|
313
|
+
* several pending launches into a single concurrent await, and do not
|
|
314
|
+
* "helpfully" loop this function internally until the floor is met.
|
|
315
|
+
*
|
|
316
|
+
* D-05's probe-live floor evaluation (count_ready() trusting probe-live
|
|
317
|
+
* instances rather than a recorded `ready` state) is explicitly Phase
|
|
318
|
+
* 01.6.2.1's criterion L, NOT this plan's -- countReady() here still
|
|
319
|
+
* counts by RECORDED state, exactly like the bash original's count_ready()
|
|
320
|
+
* before Decision 5.2. A reviewer must not mistake this for an oversight:
|
|
321
|
+
* it is the declined-for-this-phase choice RESEARCH.md §A1 recommends, and
|
|
322
|
+
* grant_from_spare()'s own live re-probe at GRANT time (broker-kill.mts /
|
|
323
|
+
* plan 04's territory) is a separate, already-correct mechanism this plan
|
|
324
|
+
* does not touch. */
|
|
325
|
+
export async function maintainWarmFloor(deps) {
|
|
326
|
+
const log = deps.log ?? defaultLog;
|
|
327
|
+
const now = deps.now ?? (() => Date.now());
|
|
328
|
+
const probe = deps.probe ?? ((port) => probeReady(port));
|
|
329
|
+
// Step 1: promote every "launching" instance whose probe now succeeds.
|
|
330
|
+
// Runs regardless of whether a launch is in flight -- promotion and
|
|
331
|
+
// speculative warming are independent concerns; an already-launched
|
|
332
|
+
// instance becomes usable the moment it answers, whether or not this
|
|
333
|
+
// pass goes on to warm anything further.
|
|
334
|
+
for (const record of deps.state.instances.values()) {
|
|
335
|
+
if (record.state !== "launching")
|
|
336
|
+
continue;
|
|
337
|
+
const isReady = await probe(record.port);
|
|
338
|
+
if (isReady) {
|
|
339
|
+
const readyAt = now();
|
|
340
|
+
const elapsedMs = readyAt - record.launchedAt;
|
|
341
|
+
record.state = "ready";
|
|
342
|
+
record.readyAt = readyAt;
|
|
343
|
+
log(`vice-broker: port ${record.port} launching -> ready (${elapsedMs}ms)`);
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
// Step 2 (P-06: the warm-zero "no readiness mechanism" branch that used
|
|
347
|
+
// to sit here is GONE -- the surviving probe mechanism is in-process and
|
|
348
|
+
// always available, so there is no "no mechanism" state left to warm
|
|
349
|
+
// zero against). No new boot starts while one is already under way --
|
|
350
|
+
// THE single in-flight counter (countLaunching) both this function and a
|
|
351
|
+
// cold acquire (vice-broker.mts's handleAcquire) consult.
|
|
352
|
+
if (deps.countLaunching(deps.state) > 0) {
|
|
353
|
+
// D-07's launch-slot decision log line, this decision point's own half:
|
|
354
|
+
// name WHICH reason currently holds the slot (the launching record's
|
|
355
|
+
// own `reason`, whichever call produced it -- cold acquire or an
|
|
356
|
+
// earlier warming pass), not merely that warming is waiting.
|
|
357
|
+
const inFlightRecord = Array.from(deps.state.instances.values()).find((r) => r.state === "launching");
|
|
358
|
+
const winningReason = inFlightRecord?.reason ?? "unknown";
|
|
359
|
+
log(`vice-broker: launch-slot decision -- ${winningReason} holds the slot; spare waits (D-07)`);
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
const ready = deps.countReady(deps.state);
|
|
363
|
+
const total = deps.countTotal(deps.state);
|
|
364
|
+
const warmFloor = resolveWarmFloor(deps.warmFloor);
|
|
365
|
+
const ceiling = resolveCeiling(deps.ceiling);
|
|
366
|
+
if (!(ready < warmFloor && total < ceiling)) {
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
369
|
+
// acquirePortAndLaunch() holds the SAME single in_flight owner across
|
|
370
|
+
// its own async port allocation -- not merely tryLaunchOne()'s
|
|
371
|
+
// synchronous spawn instant. This is what actually closes the race
|
|
372
|
+
// between this warm-floor launch and a cold acquire (vice-broker.mts's
|
|
373
|
+
// handleAcquire) arriving over the TCP control listener at any moment:
|
|
374
|
+
// the countLaunching() check just above is a cheap PRE-check (bails
|
|
375
|
+
// early when a launch is already recorded), but nextFreePort() is
|
|
376
|
+
// itself asynchronous, so without the guard held across the allocation
|
|
377
|
+
// too, two overlapping callers could still both be told the same
|
|
378
|
+
// candidate port is free before either commits it.
|
|
379
|
+
const result = await acquirePortAndLaunch("spare", {
|
|
380
|
+
state: deps.state,
|
|
381
|
+
stateDir: deps.stateDir,
|
|
382
|
+
allocatePort: deps.allocatePort,
|
|
383
|
+
spawn: deps.spawn,
|
|
384
|
+
spawnFactory: deps.spawnFactory,
|
|
385
|
+
now: deps.now,
|
|
386
|
+
viceBin: deps.viceBin,
|
|
387
|
+
mcpHost: deps.mcpHost,
|
|
388
|
+
});
|
|
389
|
+
if (result.ok) {
|
|
390
|
+
log(`vice-broker: warmed 1 warm instance this pass -- ${ready + 1} of ${warmFloor} ready, remainder warmed on later passes`);
|
|
391
|
+
deps.onLaunched?.(result.record);
|
|
392
|
+
}
|
|
393
|
+
else if (result.reason === "no_free_port") {
|
|
394
|
+
log(`vice-broker: no free port available -- warming no further warm instances; ${ready} of ${warmFloor} ready`);
|
|
395
|
+
}
|
|
396
|
+
else {
|
|
397
|
+
// A launch started (cold or warm) between this function's own
|
|
398
|
+
// countLaunching() check above and this call -- a narrow window
|
|
399
|
+
// closed by the guard rather than assumed impossible.
|
|
400
|
+
log("vice-broker: a warm-floor launch was attempted but a launch was already in flight -- deferring to a later pass");
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
/** The fixed pass order (mirrors vice-broker.sh's own broker_once(), whose
|
|
404
|
+
* comment names the ordering as load-bearing: "the spare invariant is
|
|
405
|
+
* always re-evaluated against the freshest possible grant/teardown
|
|
406
|
+
* state"). The bash version's third concern, the grant sweep, does NOT
|
|
407
|
+
* appear here -- it is one of criterion F's six retiring file-lease
|
|
408
|
+
* mechanisms; the TCP connection itself is the lease (D-12). The
|
|
409
|
+
* broker-instances.json projection write does not appear either, per D-24
|
|
410
|
+
* (see broker-state.mts's own FINDING 2 comment). Takes plain callbacks
|
|
411
|
+
* rather than the full BrokerState/deps shape so a test can inject two
|
|
412
|
+
* instrumented no-op functions and assert call ORDER without needing a
|
|
413
|
+
* real broker, a real port or a real launch.
|
|
414
|
+
*
|
|
415
|
+
* D-07 (01.6.2.1-03-PLAN.md): THIS is where launch priority actually lives
|
|
416
|
+
* -- serving acquires before maintaining the warm floor is what lets a
|
|
417
|
+
* request-driven launch win a freed slot before a warming launch, within
|
|
418
|
+
* one pass, on top of the single in-flight owner (acquirePortAndLaunch()'s
|
|
419
|
+
* own invariant comment) that this order never weakens. Inverting this
|
|
420
|
+
* order lets a warming launch take the slot first and go untested against
|
|
421
|
+
* a concurrently arriving acquire, which is exactly the regression
|
|
422
|
+
* broker-launch.test.ts's own D-07 priority test is written to catch (its
|
|
423
|
+
* own discriminating-power demonstration inverts this exact order and
|
|
424
|
+
* observes the test go red). Priority decides only which reason wins the
|
|
425
|
+
* NEXT freed slot -- it is never a substitute for the lock, and it never
|
|
426
|
+
* kills or abandons whichever boot is already in flight. */
|
|
427
|
+
export async function runBrokerPass(deps) {
|
|
428
|
+
await deps.serveAcquires();
|
|
429
|
+
await deps.maintainWarmFloor();
|
|
430
|
+
}
|
|
431
|
+
// ===========================================================================
|
|
432
|
+
// Per-child supervision (Plan 03, Task 2 -- C2/D-23): absorbs
|
|
433
|
+
// resources/vice-supervisor.sh WHOLESALE. The respawn loop becomes an
|
|
434
|
+
// exit-event handler installed on the spawned child; the backoff shape
|
|
435
|
+
// (initial delay, doubling, ceiling), the crash-loop give-up (too many
|
|
436
|
+
// crashes inside a window), and the per-instance boot/crash log are ported
|
|
437
|
+
// exactly, per D-1's own configuration knobs -- VICE_RESTART_BACKOFF_S,
|
|
438
|
+
// VICE_RESTART_BACKOFF_MAX_S, VICE_MAX_RESTARTS, VICE_CRASH_WINDOW_S all
|
|
439
|
+
// keep their exact names and semantics.
|
|
440
|
+
// ===========================================================================
|
|
441
|
+
function resolveMs(envVar, defaultSeconds, override) {
|
|
442
|
+
if (typeof override === "number")
|
|
443
|
+
return override;
|
|
444
|
+
const raw = process.env[envVar];
|
|
445
|
+
const n = raw === undefined || raw === "" ? NaN : Number(raw);
|
|
446
|
+
return (Number.isFinite(n) ? n : defaultSeconds) * 1000;
|
|
447
|
+
}
|
|
448
|
+
function resolveCount(envVar, defaultValue, override) {
|
|
449
|
+
if (typeof override === "number")
|
|
450
|
+
return override;
|
|
451
|
+
const raw = process.env[envVar];
|
|
452
|
+
const n = raw === undefined || raw === "" ? NaN : Number(raw);
|
|
453
|
+
return Number.isFinite(n) ? n : defaultValue;
|
|
454
|
+
}
|
|
455
|
+
/** The exit-driven respawn step. Reads the JUST-crashed record (still in
|
|
456
|
+
* state.instances -- nothing here deletes it before this runs), decides
|
|
457
|
+
* among the four outcomes, and acts:
|
|
458
|
+
*
|
|
459
|
+
* - deliberateKill set AND respawnAfterKill set -> "recycled": a
|
|
460
|
+
* broker-ordered death that wants a replacement, relaunched on the SAME
|
|
461
|
+
* port through launchSupervised() -- but called DIRECTLY, bypassing every
|
|
462
|
+
* crash-accounting step below (no appended crash timestamp, no give-up
|
|
463
|
+
* evaluation, no backoff wait, no doubling): a deliberate recycle is not
|
|
464
|
+
* evidence of instability, and the crash-loop machinery exists for an
|
|
465
|
+
* UNEXPLAINED exit, not this one. The pre-kill crash history and backoff
|
|
466
|
+
* are carried forward UNCHANGED, and a pre-kill "granted" state is
|
|
467
|
+
* restored on the fresh record -- the relaunch primitive always creates a
|
|
468
|
+
* new record in the "launching" state, and leaving it there would let the
|
|
469
|
+
* warm floor's own ready-count numerator mistake a recycled session's own
|
|
470
|
+
* machine for an available warm instance.
|
|
471
|
+
* - deliberateKill set WITHOUT respawnAfterKill -> "deliberate_teardown":
|
|
472
|
+
* drop the instance, no respawn. This is T-01.6.2-21's whole point --
|
|
473
|
+
* without reading this flag, every deliberate teardown would respawn
|
|
474
|
+
* exactly what it just killed, silently breaking kill-never-recycle (a
|
|
475
|
+
* released instance must be killed and stay gone).
|
|
476
|
+
* - crash count (this instance's crash timestamps still inside the window,
|
|
477
|
+
* INCLUDING this one) at or above the configured maximum ->
|
|
478
|
+
* "given_up": drop the instance, log a line naming it and the count.
|
|
479
|
+
* Mirrors vice-supervisor.sh's own `>= VICE_MAX_RESTARTS` check exactly
|
|
480
|
+
* (T-01.6.2-20).
|
|
481
|
+
* - otherwise -> "respawned": wait the CURRENT backoff (from the crashed
|
|
482
|
+
* record, so the doubling carries forward across respawns), then relaunch
|
|
483
|
+
* through launchSupervised() below -- the SAME tryLaunchOne() primitive
|
|
484
|
+
* plan 02 established, with the crash history and the NEXT (doubled,
|
|
485
|
+
* clamped) backoff threaded into the new record. */
|
|
486
|
+
async function handleExit(reason, port, deps) {
|
|
487
|
+
const record = deps.state.instances.get(port);
|
|
488
|
+
if (!record) {
|
|
489
|
+
// Already gone by some other path (e.g. a release that removed the
|
|
490
|
+
// instance outright rather than merely marking it) -- nothing to do.
|
|
491
|
+
return;
|
|
492
|
+
}
|
|
493
|
+
const log = deps.log ?? defaultLog;
|
|
494
|
+
if (record.deliberateKill) {
|
|
495
|
+
if (record.respawnAfterKill) {
|
|
496
|
+
// Recycle. Capture the pre-kill state, crash history and backoff
|
|
497
|
+
// BEFORE launchSupervised() replaces the map entry at this port key
|
|
498
|
+
// with a brand new InstanceRecord -- nothing about those three facts
|
|
499
|
+
// survives once that overwrite happens.
|
|
500
|
+
const preKillState = record.state;
|
|
501
|
+
const preKillCrashTimes = record.crashTimes ?? [];
|
|
502
|
+
const preKillBackoffMs = record.backoffMs ?? resolveMs("VICE_RESTART_BACKOFF_S", 3, deps.initialBackoffMs);
|
|
503
|
+
const respawned = launchSupervised(reason, port, deps, preKillCrashTimes, preKillBackoffMs);
|
|
504
|
+
if (respawned && preKillState === "granted") {
|
|
505
|
+
respawned.state = "granted";
|
|
506
|
+
}
|
|
507
|
+
// Keep the matching grant's own recorded pid in sync with the
|
|
508
|
+
// respawned record's pid -- the ONE legitimate case where the SAME
|
|
509
|
+
// grant continues to own a DIFFERENT pid on the SAME port. Without
|
|
510
|
+
// this, vice-broker.mts's handleRelease() own grant-pid identity
|
|
511
|
+
// check (T-01.6.2.1-28) would misfire and refuse to tear down the
|
|
512
|
+
// very instance the grant now legitimately owns.
|
|
513
|
+
if (respawned) {
|
|
514
|
+
for (const grant of deps.state.grants.values()) {
|
|
515
|
+
if (grant.port === port) {
|
|
516
|
+
grant.pid = respawned.pid;
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
deps.onOutcome?.("recycled", port);
|
|
521
|
+
return;
|
|
522
|
+
}
|
|
523
|
+
deps.state.instances.delete(port);
|
|
524
|
+
deps.onOutcome?.("deliberate_teardown", port);
|
|
525
|
+
return;
|
|
526
|
+
}
|
|
527
|
+
const now = deps.now ?? (() => Date.now());
|
|
528
|
+
const nowMs = now();
|
|
529
|
+
const crashWindowMs = resolveMs("VICE_CRASH_WINDOW_S", 120, deps.crashWindowMs);
|
|
530
|
+
const crashTimes = [...(record.crashTimes ?? []), nowMs].filter((t) => nowMs - t <= crashWindowMs);
|
|
531
|
+
const maxRestarts = resolveCount("VICE_MAX_RESTARTS", 5, deps.maxRestarts);
|
|
532
|
+
if (crashTimes.length >= maxRestarts) {
|
|
533
|
+
log(`vice-broker: giving up on port ${port} after ${crashTimes.length} crashes within ${crashWindowMs}ms -- ` +
|
|
534
|
+
`this is not a transient crash; check VICE_ARGS and whether the port is already bound`);
|
|
535
|
+
deps.state.instances.delete(port);
|
|
536
|
+
deps.onOutcome?.("given_up", port);
|
|
537
|
+
return;
|
|
538
|
+
}
|
|
539
|
+
const sleepMs = deps.sleepMs ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
540
|
+
const currentBackoffMs = record.backoffMs ?? resolveMs("VICE_RESTART_BACKOFF_S", 3, deps.initialBackoffMs);
|
|
541
|
+
await sleepMs(currentBackoffMs);
|
|
542
|
+
const maxBackoffMs = resolveMs("VICE_RESTART_BACKOFF_MAX_S", 30, deps.maxBackoffMs);
|
|
543
|
+
const nextBackoffMs = Math.min(currentBackoffMs * 2, maxBackoffMs);
|
|
544
|
+
const respawned = launchSupervised(reason, port, deps, crashTimes, nextBackoffMs);
|
|
545
|
+
deps.onOutcome?.(respawned ? "respawned" : "given_up", port);
|
|
546
|
+
}
|
|
547
|
+
/** The single exit-listener installation point in the whole module tree.
|
|
548
|
+
* Wraps `baseSpawn` (a plain spawn function of the same shape
|
|
549
|
+
* `(command, args) => ChildProcess` every launch path already threads
|
|
550
|
+
* through) so the returned spawn function, when called, attaches a
|
|
551
|
+
* one-shot "exit" listener that drives handleExit() above -- the SAME
|
|
552
|
+
* respawn/give-up/deliberate-teardown resolution launchSupervised()'s own
|
|
553
|
+
* relaunch path already uses. Returns the child object baseSpawn produced,
|
|
554
|
+
* UNCHANGED -- a caller's own handle to the child (e.g. its pid) is never
|
|
555
|
+
* replaced or wrapped itself; only the spawn FUNCTION is composed.
|
|
556
|
+
*
|
|
557
|
+
* This is the extraction the phase's own gap closure exists to make: before
|
|
558
|
+
* this function existed, launchSupervised() built this exact listener
|
|
559
|
+
* inline, and it was the ONLY place in the tree that ever did -- both real
|
|
560
|
+
* launch paths in vice-broker.mts instead spawned through a bare,
|
|
561
|
+
* unwrapped spawn with no exit observation at all. Composing a real launch
|
|
562
|
+
* path's own spawn factory through THIS function, rather than reaching for
|
|
563
|
+
* a second inline listener, is what keeps the "exactly one installation
|
|
564
|
+
* point" invariant a structural gate (broker-launch.test.ts) can hold. */
|
|
565
|
+
export function withCrashSupervision(reason, port, baseSpawn, deps) {
|
|
566
|
+
return (cmd, args) => {
|
|
567
|
+
const child = baseSpawn(cmd, args);
|
|
568
|
+
child.once("exit", () => {
|
|
569
|
+
void handleExit(reason, port, deps);
|
|
570
|
+
});
|
|
571
|
+
return child;
|
|
572
|
+
};
|
|
573
|
+
}
|
|
574
|
+
/** Launches (or relaunches) a supervised instance: spawns through
|
|
575
|
+
* tryLaunchOne() (the SAME single guarded primitive plan 02 established --
|
|
576
|
+
* "spawn again through the SAME single guarded launch function", never a
|
|
577
|
+
* second, parallel spawn path), writes the per-instance boot/crash log at
|
|
578
|
+
* the path shape the retiring supervisor used (a `logs/` directory under
|
|
579
|
+
* the instance directory, named for the binary and a timestamp -- derived
|
|
580
|
+
* from broker-epoch.mts's instanceLogDirFor so this file and the epoch
|
|
581
|
+
* record's own `log` field can never disagree), bumps the epoch through
|
|
582
|
+
* the epoch writer (broker-epoch.mts's nextEpochFor + writeEpochRecord),
|
|
583
|
+
* and installs the exit handler that drives the NEXT crash's outcome.
|
|
584
|
+
*
|
|
585
|
+
* crashTimes/backoffMs are threaded through explicitly (not reset to
|
|
586
|
+
* defaults) so a respawn's crash history and doubling backoff survive the
|
|
587
|
+
* fact that spawnAndRecordInstance() creates a BRAND NEW InstanceRecord
|
|
588
|
+
* object on every launch, replacing the old one at the same port key. */
|
|
589
|
+
function launchSupervised(reason, port, deps, crashTimes, backoffMs) {
|
|
590
|
+
const supervisorDir = join(deps.stateDir, String(port));
|
|
591
|
+
const epochFile = deps.epoch.epochPathFor(deps.stateDir, port);
|
|
592
|
+
const logDir = deps.epoch.instanceLogDirFor(deps.stateDir, port);
|
|
593
|
+
mkdirSync(logDir, { recursive: true });
|
|
594
|
+
const epoch = deps.epoch.nextEpochFor(supervisorDir);
|
|
595
|
+
const viceBin = deps.viceBin ?? process.env.VICE_BIN ?? "x64sc";
|
|
596
|
+
// Timestamp PLUS the epoch number: Date.now() alone can collide across
|
|
597
|
+
// two respawns inside the same millisecond when the injected sleepMs
|
|
598
|
+
// resolves immediately (exactly what this module's own tests do to stay
|
|
599
|
+
// fast and deterministic) -- the epoch, guaranteed strictly increasing
|
|
600
|
+
// per instance, makes every respawn's log filename distinct regardless
|
|
601
|
+
// of wall-clock resolution.
|
|
602
|
+
const logFileName = `${basename(viceBin)}-${Date.now()}-e${epoch}.log`;
|
|
603
|
+
const logPath = join(logDir, logFileName);
|
|
604
|
+
const logRelPath = `logs/${logFileName}`;
|
|
605
|
+
const defaultRealSpawn = (cmd, args) => {
|
|
606
|
+
const fd = openSync(logPath, "a");
|
|
607
|
+
return nodeSpawn(cmd, args, { stdio: ["ignore", fd, fd] });
|
|
608
|
+
};
|
|
609
|
+
const baseSpawn = deps.spawnFactory ? deps.spawnFactory(port) : (deps.spawn ?? defaultRealSpawn);
|
|
610
|
+
const wrappedSpawn = withCrashSupervision(reason, port, baseSpawn, deps);
|
|
611
|
+
const record = tryLaunchOne(reason, port, {
|
|
612
|
+
state: deps.state,
|
|
613
|
+
supervisorDir,
|
|
614
|
+
epochFile,
|
|
615
|
+
spawn: wrappedSpawn,
|
|
616
|
+
now: deps.now,
|
|
617
|
+
viceBin: deps.viceBin,
|
|
618
|
+
mcpHost: deps.mcpHost,
|
|
619
|
+
log: deps.log,
|
|
620
|
+
});
|
|
621
|
+
if (!record)
|
|
622
|
+
return null;
|
|
623
|
+
// The log file's EXISTENCE and the epoch record's `log` field naming it
|
|
624
|
+
// must never disagree, regardless of which spawn implementation actually
|
|
625
|
+
// produced output -- a test-injected stub child never writes through
|
|
626
|
+
// defaultRealSpawn's own fd, so this touches the file into existence
|
|
627
|
+
// when nothing else has.
|
|
628
|
+
if (!existsSync(logPath)) {
|
|
629
|
+
closeSync(openSync(logPath, "a"));
|
|
630
|
+
}
|
|
631
|
+
record.epoch = epoch;
|
|
632
|
+
record.deliberateKill = false;
|
|
633
|
+
record.crashTimes = crashTimes;
|
|
634
|
+
record.backoffMs = backoffMs;
|
|
635
|
+
record.logPath = logPath;
|
|
636
|
+
deps.epoch.writeEpochRecord({
|
|
637
|
+
supervisorDir,
|
|
638
|
+
record: {
|
|
639
|
+
epoch,
|
|
640
|
+
spawned_at: new Date(record.launchedAt).toISOString(),
|
|
641
|
+
pid: record.pid,
|
|
642
|
+
supervisor_pid: process.pid,
|
|
643
|
+
vice_bin: record.viceBin,
|
|
644
|
+
vice_args: record.viceArgs,
|
|
645
|
+
log: logRelPath,
|
|
646
|
+
dry_run: false,
|
|
647
|
+
},
|
|
648
|
+
});
|
|
649
|
+
return record;
|
|
650
|
+
}
|
|
651
|
+
/** The public entry point: launches a NEW instance under full supervision
|
|
652
|
+
* (crash respawn with backoff, crash-loop give-up, kill-never-recycle via
|
|
653
|
+
* the deliberate-kill marker, and the per-instance boot/crash log), exactly
|
|
654
|
+
* mirroring resources/vice-supervisor.sh's own respawn loop but expressed
|
|
655
|
+
* as an event-loop exit handler instead of a `while true` poll. */
|
|
656
|
+
export function superviseChild(reason, port, deps) {
|
|
657
|
+
const initialBackoffMs = resolveMs("VICE_RESTART_BACKOFF_S", 3, deps.initialBackoffMs);
|
|
658
|
+
return launchSupervised(reason, port, deps, [], initialBackoffMs);
|
|
659
|
+
}
|