@geminixiang/mikan 1.0.0-beta.2 → 1.0.0-beta.21

package/dist/sandbox/gondolin-fleet.js

@@ -0,0 +1,330 @@
+import * as log from "../log.js";
+import { SandboxError } from "./errors.js";
+import { gondolinPlacements } from "./gondolin-placement.js";
+import { GondolinRemoteConnection, GondolinWorkerUnreachableError, } from "./gondolin-remote.js";
+import { GondolinRuntimeGoneError } from "./gondolin-recovery.js";
+const LEASE_TTL_SECONDS = 300;
+const QUEUE_WAIT_SECONDS = 60;
+const QUEUE_POLL_MS = 2000;
+function normalizeWorkers(settings) {
+    const inline = settings.workers ?? [{ url: settings.url }];
+    return inline.map((worker) => ({
+        ...worker,
+        name: worker.name ?? worker.url,
+        caFile: worker.caFile ?? settings.caFile,
+        certFile: worker.certFile ?? settings.certFile,
+        keyFile: worker.keyFile ?? settings.keyFile,
+        workspaceRoot: worker.workspaceRoot ?? settings.workspaceRoot,
+        maxRuntimes: worker.maxRuntimes ?? settings.maxRuntimes,
+    }));
+}
+/**
+ * Places each conversation's runtime on one worker of the fleet and keeps it
+ * there (sticky placement). Failover to another worker happens only when the
+ * placed worker is unreachable AND its lease fencing watermark has passed —
+ * the point after which the old worker's janitor has provably stopped the
+ * runtime, so shared storage never sees two writers.
+ */
+class GondolinFleetClient {
+    constructor() {
+        this.workers = new Map();
+        this.connectionOverrides = {};
+        this.placements = gondolinPlacements;
+        this.leaseTtlMs = LEASE_TTL_SECONDS * 1000;
+        this.queueWaitMs = QUEUE_WAIT_SECONDS * 1000;
+        this.queuePollMs = QUEUE_POLL_MS;
+        this.now = Date.now;
+        this.sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+    }
+    configure(settings, overrides) {
+        for (const worker of this.workers.values())
+            worker.connection.dispose();
+        this.workers.clear();
+        this.settings = settings;
+        this.placements = overrides?.placements ?? gondolinPlacements;
+        this.now = overrides?.now ?? Date.now;
+        this.sleep = overrides?.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+        this.queuePollMs = overrides?.queuePollMs ?? QUEUE_POLL_MS;
+        this.connectionOverrides = overrides?.connectionOverrides ?? {};
+        this.leaseTtlMs = (overrides?.connectionOverrides?.leaseTtlSeconds ?? LEASE_TTL_SECONDS) * 1000;
+        this.queueWaitMs = (settings?.queueWaitSeconds ?? QUEUE_WAIT_SECONDS) * 1000;
+        if (!settings)
+            return;
+        if (!settings.workers?.length && !settings.url) {
+            return; // not configured; validate() reports the actionable error
+        }
+        const createConnection = overrides?.createConnection ??
+            ((worker, connectionOverrides) => new GondolinRemoteConnection(worker, connectionOverrides));
+        for (const worker of normalizeWorkers(settings)) {
+            const name = worker.name;
+            const connection = createConnection(worker, {
+                ...overrides?.connectionOverrides,
+                onLeaseActivity: (instanceId, expiresAtMs) => {
+                    if (this.placements.get(instanceId)?.worker === name) {
+                        this.placements.touch(instanceId, expiresAtMs);
+                    }
+                },
+            });
+            this.workers.set(name, { name, settings: worker, connection });
+        }
+    }
+    isConfigured() {
+        return this.workers.size > 0;
+    }
+    /** Names of statically-configured workers with no workspaceRoot (startup guard). */
+    workersMissingWorkspaceRoot() {
+        return Array.from(this.workers.values())
+            .filter((worker) => !worker.settings.workspaceRoot)
+            .map((worker) => worker.name);
+    }
+    /**
+     * Attach a dial-home worker at runtime (the gateway calls this when a
+     * worker registers). The factory receives the fleet's connection overrides
+     * plus the lease-watermark callback, mirroring configure()'s wiring.
+     */
+    attachWorker(settings, createConnection) {
+        const name = settings.name ?? settings.url;
+        this.workers.get(name)?.connection.dispose();
+        const connection = createConnection({
+            ...this.connectionOverrides,
+            onLeaseActivity: (instanceId, expiresAtMs) => {
+                if (this.placements.get(instanceId)?.worker === name) {
+                    this.placements.touch(instanceId, expiresAtMs);
+                }
+            },
+        });
+        this.workers.set(name, { name, settings: { ...settings, name }, connection });
+    }
+    /**
+     * Record a worker's shared-workspace health (the gateway pushes this from
+     * register and ping frames). A degraded worker keeps its runtimes and its
+     * connection, but new placements skip it and runtime operations fail fast
+     * with the probe's diagnosis instead of hanging on a dead mount.
+     */
+    setWorkspaceHealth(name, error) {
+        const worker = this.workers.get(name);
+        if (!worker)
+            return;
+        worker.workspaceError = error || undefined;
+    }
+    /**
+     * Detach a dial-home worker (disconnect). Placements survive: their lease
+     * watermarks keep fencing failover until expiry, exactly as for an
+     * unreachable static worker.
+     */
+    detachWorker(name) {
+        const worker = this.workers.get(name);
+        if (!worker)
+            return;
+        worker.connection.dispose();
+        this.workers.delete(name);
+    }
+    imageSelector() {
+        return this.settings?.imageSelector;
+    }
+    /** At least one worker must answer for the sandbox mode to come up. */
+    async health() {
+        const report = await Promise.all(Array.from(this.workers.values()).map(async (worker) => {
+            try {
+                return { worker: worker.name, health: await worker.connection.health() };
+            }
+            catch {
+                return { worker: worker.name };
+            }
+        }));
+        if (report.length > 0 && !report.some((entry) => entry.health)) {
+            throw new SandboxError("Error: no gondolin remote worker is reachable");
+        }
+        return report;
+    }
+    async ensure(instanceId, spec) {
+        const placed = this.placements.get(instanceId);
+        if (placed) {
+            const worker = this.workers.get(placed.worker);
+            if (worker) {
+                try {
+                    return await this.ensureOn(worker, instanceId, spec);
+                }
+                catch (err) {
+                    if (!(err instanceof GondolinWorkerUnreachableError))
+                        throw err;
+                    this.assertFailoverAllowed(instanceId, placed.worker, placed.leaseExpiresAt, err);
+                }
+            }
+            else {
+                this.assertFailoverAllowed(instanceId, placed.worker, placed.leaseExpiresAt, new GondolinWorkerUnreachableError("worker is no longer configured"));
+            }
+            log.logWarning(`Gondolin worker '${placed.worker}' lost runtime for '${instanceId}'; failing over`);
+        }
+        const worker = await this.selectWorker();
+        return this.ensureOn(worker, instanceId, spec);
+    }
+    async stop(handle) {
+        const worker = this.workerFor(handle.workerName);
+        if (!worker) {
+            // the worker left the fleet config; its janitor stops the runtime when
+            // the lease expires, and the record keeps fencing until then
+            return;
+        }
+        try {
+            await worker.connection.stop(handle);
+            this.placements.delete(handle.instanceId);
+        }
+        catch (err) {
+            if (err instanceof GondolinWorkerUnreachableError) {
+                // cannot confirm the stop; keep the placement so its watermark still
+                // fences any failover, and let the daemon's expiry janitor finish it
+                log.logWarning(`Gondolin worker '${worker.name}' unreachable while stopping '${handle.instanceId}'; leaving it to lease expiry`);
+                return;
+            }
+            throw err;
+        }
+    }
+    async exec(handle, command, options = {}) {
+        const worker = this.workerFor(handle.workerName);
+        if (!worker)
+            throw new GondolinRuntimeGoneError("worker is no longer configured");
+        this.assertWorkspaceUsable(worker);
+        log.logInfo(`Gondolin execution '${handle.instanceId}' routed to worker '${worker.name}'`);
+        return worker.connection.exec(handle, command, options);
+    }
+    async isRuntimeAlive(handle) {
+        const worker = this.workerFor(handle.workerName);
+        if (!worker)
+            return false;
+        return worker.connection.isRuntimeAlive(handle);
+    }
+    /**
+     * Fleet reconciliation: a runtime whose conversation is placed on a
+     * different worker is a stray from a superseded placement — stop it. A
+     * runtime with no placement at all (crash between daemon ensure and the
+     * placement write) is adopted instead of killed.
+     */
+    async reconcile() {
+        for (const worker of this.workers.values()) {
+            let runtimes;
+            try {
+                runtimes = await worker.connection.listRuntimes();
+            }
+            catch {
+                continue; // unreachable workers reconcile on a later sweep
+            }
+            for (const runtime of runtimes) {
+                const record = this.placements.get(runtime.instanceId);
+                if (!record) {
+                    log.logInfo(`Adopting unplaced Gondolin runtime '${runtime.instanceId}' on worker '${worker.name}'`);
+                    this.placements.set(runtime.instanceId, worker.name, this.now() + this.leaseTtlMs);
+                    continue;
+                }
+                if (record.worker === worker.name)
+                    continue;
+                log.logInfo(`Stopping stray Gondolin runtime '${runtime.instanceId}' on worker '${worker.name}' (placed on '${record.worker}')`);
+                try {
+                    await worker.connection.stop({
+                        sessionId: runtime.sessionId,
+                        instanceId: runtime.instanceId,
+                        workerPid: 0,
+                    });
+                }
+                catch (err) {
+                    log.logWarning(`Failed to stop stray runtime '${runtime.instanceId}' on '${worker.name}'`, err instanceof Error ? err.message : String(err));
+                }
+            }
+        }
+    }
+    async ensureOn(worker, instanceId, spec) {
+        this.assertWorkspaceUsable(worker);
+        const handle = await worker.connection.ensure(instanceId, spec);
+        this.placements.set(instanceId, worker.name, this.now() + this.leaseTtlMs);
+        return { ...handle, workerName: worker.name };
+    }
+    /**
+     * Fail fast — with the probe's diagnosis — instead of dispatching work that
+     * would hang on a dead shared-workspace mount. Deliberately NOT a
+     * GondolinWorkerUnreachableError: a broken mount must not trigger failover,
+     * because the workspace is shared and likely broken for every worker.
+     */
+    assertWorkspaceUsable(worker) {
+        if (!worker.workspaceError)
+            return;
+        throw new SandboxError(`Error: Gondolin worker '${worker.name}' has an unusable shared workspace: ` +
+            `${worker.workspaceError} (fix the mount on the worker; health refreshes with its next heartbeat)`);
+    }
+    assertFailoverAllowed(instanceId, workerName, leaseExpiresAt, cause) {
+        const remainingMs = leaseExpiresAt - this.now();
+        if (remainingMs <= 0)
+            return;
+        // Failing over earlier could put a second writable VM on the same shared
+        // workspace: the old worker may be alive behind a partition, and its
+        // daemon only stops the runtime once the lease expires.
+        throw new SandboxError(`Error: Gondolin worker '${workerName}' is unreachable (${cause.message}); ` +
+            `retrying '${instanceId}' elsewhere in ${Math.ceil(remainingMs / 1000)}s once its lease has expired`);
+    }
+    async selectWorker() {
+        const deadline = this.now() + this.queueWaitMs;
+        for (;;) {
+            let best;
+            let reachable = 0;
+            let draining = 0;
+            let full = 0;
+            const degraded = [];
+            for (const worker of this.workers.values()) {
+                if (worker.settings.draining) {
+                    draining += 1;
+                    continue;
+                }
+                if (worker.workspaceError) {
+                    degraded.push(`'${worker.name}': ${worker.workspaceError}`);
+                    continue;
+                }
+                let active;
+                let workspaceError;
+                try {
+                    const health = await worker.connection.health();
+                    active = typeof health.activeRuntimes === "number" ? health.activeRuntimes : 0;
+                    workspaceError =
+                        typeof health.workspaceError === "string" && health.workspaceError
+                            ? health.workspaceError
+                            : undefined;
+                }
+                catch {
+                    continue;
+                }
+                reachable += 1;
+                if (workspaceError) {
+                    // static workers report probe results via /v1/health only
+                    degraded.push(`'${worker.name}': ${workspaceError}`);
+                    continue;
+                }
+                const cap = worker.settings.maxRuntimes;
+                if (cap !== undefined && active >= cap) {
+                    full += 1;
+                    continue;
+                }
+                const load = cap !== undefined ? active / cap : active / Number.MAX_SAFE_INTEGER;
+                if (!best || load < best.load)
+                    best = { worker, load };
+            }
+            if (best)
+                return best.worker;
+            if (reachable === 0 && draining === 0 && degraded.length === 0) {
+                throw new SandboxError("Error: no gondolin remote worker is reachable");
+            }
+            if (full === 0) {
+                const detail = degraded.length > 0 ? ` — degraded: ${degraded.join("; ")}` : "";
+                throw new SandboxError(`Error: every gondolin remote worker is draining, degraded, or unreachable${detail}`);
+            }
+            if (this.now() >= deadline) {
+                throw new SandboxError(`Error: all gondolin remote workers are at capacity (waited ${Math.round(this.queueWaitMs / 1000)}s)`);
+            }
+            await this.sleep(this.queuePollMs);
+        }
+    }
+    workerFor(workerName) {
+        if (!workerName)
+            return undefined;
+        return this.workers.get(workerName);
+    }
+}
+export const gondolinFleet = new GondolinFleetClient();
+//# sourceMappingURL=gondolin-fleet.js.map

package/dist/sandbox/gondolin-fleet.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gondolin-fleet.js","sourceRoot":"","sources":["../../src/sandbox/gondolin-fleet.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,GAAG,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,kBAAkB,EAA+B,MAAM,yBAAyB,CAAC;AAC1F,OAAO,EACL,wBAAwB,EACxB,8BAA8B,GAE/B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAQlE,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,aAAa,GAAG,IAAI,CAAC;AAuC3B,SAAS,gBAAgB,CAAC,QAAgC;IACxD,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAa,EAAE,CAAC,CAAC;IACrE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC7B,GAAG,MAAM;QACT,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,GAAG;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,QAAQ,CAAC,MAAM;QACxC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ;QAC9C,OAAO,EAAE,MAAM,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO;QAC3C,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,QAAQ,CAAC,aAAa;QAC7D,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,QAAQ,CAAC,WAAW;KACxD,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;;;;GAMG;AACH,MAAM,mBAAmB;IAAzB;QAEU,YAAO,GAAG,IAAI,GAAG,EAAuB,CAAC;QACzC,wBAAmB,GAA4B,EAAE,CAAC;QAClD,eAAU,GAA2B,kBAAkB,CAAC;QACxD,eAAU,GAAG,iBAAiB,GAAG,IAAI,CAAC;QACtC,gBAAW,GAAG,kBAAkB,GAAG,IAAI,CAAC;QACxC,gBAAW,GAAG,aAAa,CAAC;QAC5B,QAAG,GAAiB,IAAI,CAAC,GAAG,CAAC;QAC7B,UAAK,GAAkC,CAAC,EAAE,EAAE,EAAE,CACpD,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IA8UtD,CAAC;IA5UC,SAAS,CAAC,QAAiC,EAAE,SAAkC;QAC7E,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YAAE,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QACxE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,SAAS,EAAE,UAAU,IAAI,kBAAkB,CAAC;QAC9D,IAAI,CAAC,GAAG,GAAG,SAAS,EAAE,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;QACtC,IAAI,CAAC,KAAK,GAAG,SAAS,EAAE,KAAK,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7F,IAAI,CAAC,WAAW,GAAG,SAAS,EAAE,WAAW,IAAI,aAAa,CAAC;QAC3D,IAAI,CAAC,mBAAmB,GAAG,SAAS,EAAE,mBAAmB,IAAI,EAAE,CAAC;QAChE,IAAI,CAAC,UAAU,GAAG,CAAC,SAAS,EAAE,mBAAmB,EAAE,eAAe,IAAI,iBAAiB,CAAC,GAAG,IAAI,CAAC;QAChG,IAAI,CAAC,WAAW,GAAG,CAAC,QAAQ,EAAE,gBAAgB,IAAI,kBAAkB,CAAC,GAAG,IAAI,CAAC;QAC7E,IAAI,CAAC,QAAQ;YAAE,OAAO;QACtB,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC;YAC/C,OAAO,CAAC,0DAA0D;QACpE,CAAC;QACD,MAAM,gBAAgB,GACpB,SAAS,EAAE,gBAAgB;YAC3B,CAAC,CAAC,MAAoC,EAAE,mBAA4C,EAAE,EAAE,CACtF,IAAI,wBAAwB,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAC/D,KAAK,MAAM,MAAM,IAAI,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC;YAChD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAc,CAAC;YACnC,MAAM,UAAU,GAAG,gBAAgB,CAAC,MAAM,EAAE;gBAC1C,GAAG,SAAS,EAAE,mBAAmB;gBACjC,eAAe,EAAE,CAAC,UAAU,EAAE,WAAW,EAAE,EAAE;oBAC3C,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,MAAM,KAAK,IAAI,EAAE,CAAC;wBACrD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;oBACjD,CAAC;gBACH,CAAC;aACF,CAAC,CAAC;YACH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAED,YAAY;QACV,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC;IAC/B,CAAC;IAED,oFAAoF;IACpF,2BAA2B;QACzB,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;aACrC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC;aAClD,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED;;;;OAIG;IACH,YAAY,CACV,QAAsC,EACtC,gBAAkF;QAElF,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,GAAG,CAAC;QAC3C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC;QAC7C,MAAM,UAAU,GAAG,gBAAgB,CAAC;YAClC,GAAG,IAAI,CAAC,mBAAmB;YAC3B,eAAe,EAAE,CAAC,UAAU,EAAE,WAAW,EAAE,EAAE;gBAC3C,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,MAAM,KAAK,IAAI,EAAE,CAAC;oBACrD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;gBACjD,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;IAChF,CAAC;IAED;;;;;OAKG;IACH,kBAAkB,CAAC,IAAY,EAAE,KAAc;QAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM;YAAE,OAAO;QACpB,MAAM,CAAC,cAAc,GAAG,KAAK,IAAI,SAAS,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACH,YAAY,CAAC,IAAY;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACtC,IAAI,CAAC,MAAM;YAAE,OAAO;QACpB,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAED,aAAa;QACX,OAAO,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC;IACtC,CAAC;IAED,uEAAuE;IACvE,KAAK,CAAC,MAAM;QACV,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAC9B,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YACrD,IAAI,CAAC;gBACH,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3E,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QACF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/D,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,UAAkB,EAAE,IAAyB;QACxD,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC/C,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC/C,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,CAAC;oBACH,OAAO,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;gBACvD,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,IAAI,CAAC,CAAC,GAAG,YAAY,8BAA8B,CAAC;wBAAE,MAAM,GAAG,CAAC;oBAChE,IAAI,CAAC,qBAAqB,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;gBACpF,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,qBAAqB,CACxB,UAAU,EACV,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,cAAc,EACrB,IAAI,8BAA8B,CAAC,gCAAgC,CAAC,CACrE,CAAC;YACJ,CAAC;YACD,GAAG,CAAC,UAAU,CACZ,oBAAoB,MAAM,CAAC,MAAM,uBAAuB,UAAU,iBAAiB,CACpF,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QACzC,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,IAAI,CACR,MAA4F;QAE5F,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,uEAAuE;YACvE,6DAA6D;YAC7D,OAAO;QACT,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,8BAA8B,EAAE,CAAC;gBAClD,qEAAqE;gBACrE,qEAAqE;gBACrE,GAAG,CAAC,UAAU,CACZ,oBAAoB,MAAM,CAAC,IAAI,iCAAiC,MAAM,CAAC,UAAU,+BAA+B,CACjH,CAAC;gBACF,OAAO;YACT,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CACR,MAA6B,EAC7B,OAAe,EACf,OAAO,GAA2D,EAAE;QAEpE,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,wBAAwB,CAAC,gCAAgC,CAAC,CAAC;QAClF,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;QACnC,GAAG,CAAC,OAAO,CAAC,uBAAuB,MAAM,CAAC,UAAU,uBAAuB,MAAM,CAAC,IAAI,GAAG,CAAC,CAAC;QAC3F,OAAO,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAA6B;QAChD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QAC1B,OAAO,MAAM,CAAC,UAAU,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IAClD,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,SAAS;QACb,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,QAA0D,CAAC;YAC/D,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,YAAY,EAAE,CAAC;YACpD,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS,CAAC,iDAAiD;YAC7D,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBACvD,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,GAAG,CAAC,OAAO,CACT,uCAAuC,OAAO,CAAC,UAAU,gBAAgB,MAAM,CAAC,IAAI,GAAG,CACxF,CAAC;oBACF,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;oBACnF,SAAS;gBACX,CAAC;gBACD,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI;oBAAE,SAAS;gBAC5C,GAAG,CAAC,OAAO,CACT,oCAAoC,OAAO,CAAC,UAAU,gBAAgB,MAAM,CAAC,IAAI,iBAAiB,MAAM,CAAC,MAAM,IAAI,CACpH,CAAC;gBACF,IAAI,CAAC;oBACH,MAAM,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;wBAC3B,SAAS,EAAE,OAAO,CAAC,SAAS;wBAC5B,UAAU,EAAE,OAAO,CAAC,UAAU;wBAC9B,SAAS,EAAE,CAAC;qBACb,CAAC,CAAC;gBACL,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,GAAG,CAAC,UAAU,CACZ,iCAAiC,OAAO,CAAC,UAAU,SAAS,MAAM,CAAC,IAAI,GAAG,EAC1E,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CACjD,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,QAAQ,CACpB,MAAmB,EACnB,UAAkB,EAClB,IAAyB;QAEzB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAChE,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;QAC3E,OAAO,EAAE,GAAG,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;IAChD,CAAC;IAED;;;;;OAKG;IACK,qBAAqB,CAAC,MAAmB;QAC/C,IAAI,CAAC,MAAM,CAAC,cAAc;YAAE,OAAO;QACnC,MAAM,IAAI,YAAY,CACpB,2BAA2B,MAAM,CAAC,IAAI,sCAAsC;YAC1E,GAAG,MAAM,CAAC,cAAc,0EAA0E,CACrG,CAAC;IACJ,CAAC;IAEO,qBAAqB,CAC3B,UAAkB,EAClB,UAAkB,EAClB,cAAsB,EACtB,KAAY;QAEZ,MAAM,WAAW,GAAG,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAChD,IAAI,WAAW,IAAI,CAAC;YAAE,OAAO;QAC7B,yEAAyE;QACzE,qEAAqE;QACrE,wDAAwD;QACxD,MAAM,IAAI,YAAY,CACpB,2BAA2B,UAAU,qBAAqB,KAAK,CAAC,OAAO,KAAK;YAC1E,aAAa,UAAU,kBAAkB,IAAI,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,8BAA8B,CACvG,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC;QAC/C,SAAS,CAAC;YACR,IAAI,IAAuD,CAAC;YAC5D,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,IAAI,IAAI,GAAG,CAAC,CAAC;YACb,MAAM,QAAQ,GAAa,EAAE,CAAC;YAC9B,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC3C,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;oBAC7B,QAAQ,IAAI,CAAC,CAAC;oBACd,SAAS;gBACX,CAAC;gBACD,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;oBAC1B,QAAQ,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;oBAC5D,SAAS;gBACX,CAAC;gBACD,IAAI,MAAc,CAAC;gBACnB,IAAI,cAAkC,CAAC;gBACvC,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;oBAChD,MAAM,GAAG,OAAO,MAAM,CAAC,cAAc,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC/E,cAAc;wBACZ,OAAO,MAAM,CAAC,cAAc,KAAK,QAAQ,IAAI,MAAM,CAAC,cAAc;4BAChE,CAAC,CAAC,MAAM,CAAC,cAAc;4BACvB,CAAC,CAAC,SAAS,CAAC;gBAClB,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS;gBACX,CAAC;gBACD,SAAS,IAAI,CAAC,CAAC;gBACf,IAAI,cAAc,EAAE,CAAC;oBACnB,0DAA0D;oBAC1D,QAAQ,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,IAAI,MAAM,cAAc,EAAE,CAAC,CAAC;oBACrD,SAAS;gBACX,CAAC;gBACD,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC;gBACxC,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;oBACvC,IAAI,IAAI,CAAC,CAAC;oBACV,SAAS;gBACX,CAAC;gBACD,MAAM,IAAI,GAAG,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC;gBACjF,IAAI,CAAC,IAAI,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI;oBAAE,IAAI,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YACzD,CAAC;YACD,IAAI,IAAI;gBAAE,OAAO,IAAI,CAAC,MAAM,CAAC;YAC7B,IAAI,SAAS,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC/D,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;YAC1E,CAAC;YACD,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACf,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChF,MAAM,IAAI,YAAY,CACpB,4EAA4E,MAAM,EAAE,CACrF,CAAC;YACJ,CAAC;YACD,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,EAAE,CAAC;gBAC3B,MAAM,IAAI,YAAY,CACpB,8DAA8D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,IAAI,CACtG,CAAC;YACJ,CAAC;YACD,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAEO,SAAS,CAAC,UAA8B;QAC9C,IAAI,CAAC,UAAU;YAAE,OAAO,SAAS,CAAC;QAClC,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACtC,CAAC;CACF;AAED,MAAM,CAAC,MAAM,aAAa,GAAG,IAAI,mBAAmB,EAAE,CAAC","sourcesContent":["import * as log from \"../log.js\";\nimport type { GondolinRemoteSettings, GondolinRemoteWorkerSettings } from \"../types.js\";\nimport { SandboxError } from \"./errors.js\";\nimport { gondolinPlacements, type GondolinPlacementStore } from \"./gondolin-placement.js\";\nimport {\n  GondolinRemoteConnection,\n  GondolinWorkerUnreachableError,\n  type GondolinRemoteOverrides,\n} from \"./gondolin-remote.js\";\nimport { GondolinRuntimeGoneError } from \"./gondolin-recovery.js\";\nimport type {\n  GondolinRuntimeHandle,\n  GondolinRuntimeSpec,\n  GondolinRuntimeTransport,\n} from \"./gondolin-worker-client.js\";\nimport type { ExecResult } from \"./types.js\";\n\nconst LEASE_TTL_SECONDS = 300;\nconst QUEUE_WAIT_SECONDS = 60;\nconst QUEUE_POLL_MS = 2000;\n\n/** The connection surface the fleet needs from each worker. */\nexport interface GondolinWorkerConnection {\n  health(): Promise<Record<string, unknown>>;\n  ensure(instanceId: string, spec: GondolinRuntimeSpec): Promise<GondolinRuntimeHandle>;\n  stop(\n    handle: Pick<GondolinRuntimeHandle, \"workerPid\" | \"sessionId\" | \"instanceId\">,\n  ): Promise<void>;\n  exec(\n    handle: GondolinRuntimeHandle,\n    command: string,\n    options?: { env?: Record<string, string>; signal?: AbortSignal },\n  ): Promise<ExecResult>;\n  isRuntimeAlive(handle: GondolinRuntimeHandle): Promise<boolean>;\n  listRuntimes(): Promise<Array<{ sessionId: string; instanceId: string }>>;\n  dispose(): void;\n}\n\ninterface FleetWorker {\n  name: string;\n  settings: GondolinRemoteWorkerSettings;\n  connection: GondolinWorkerConnection;\n  /** Non-empty when the worker's shared workspace failed its usability probe. */\n  workspaceError?: string;\n}\n\ninterface GondolinFleetOverrides {\n  createConnection?: (\n    worker: GondolinRemoteWorkerSettings,\n    connectionOverrides: GondolinRemoteOverrides,\n  ) => GondolinWorkerConnection;\n  placements?: GondolinPlacementStore;\n  connectionOverrides?: GondolinRemoteOverrides;\n  now?: () => number;\n  queuePollMs?: number;\n  sleep?: (ms: number) => Promise<void>;\n}\n\nfunction normalizeWorkers(settings: GondolinRemoteSettings): GondolinRemoteWorkerSettings[] {\n  const inline = settings.workers ?? [{ url: settings.url as string }];\n  return inline.map((worker) => ({\n    ...worker,\n    name: worker.name ?? worker.url,\n    caFile: worker.caFile ?? settings.caFile,\n    certFile: worker.certFile ?? settings.certFile,\n    keyFile: worker.keyFile ?? settings.keyFile,\n    workspaceRoot: worker.workspaceRoot ?? settings.workspaceRoot,\n    maxRuntimes: worker.maxRuntimes ?? settings.maxRuntimes,\n  }));\n}\n\n/**\n * Places each conversation's runtime on one worker of the fleet and keeps it\n * there (sticky placement). Failover to another worker happens only when the\n * placed worker is unreachable AND its lease fencing watermark has passed —\n * the point after which the old worker's janitor has provably stopped the\n * runtime, so shared storage never sees two writers.\n */\nclass GondolinFleetClient implements GondolinRuntimeTransport {\n  private settings?: GondolinRemoteSettings;\n  private workers = new Map<string, FleetWorker>();\n  private connectionOverrides: GondolinRemoteOverrides = {};\n  private placements: GondolinPlacementStore = gondolinPlacements;\n  private leaseTtlMs = LEASE_TTL_SECONDS * 1000;\n  private queueWaitMs = QUEUE_WAIT_SECONDS * 1000;\n  private queuePollMs = QUEUE_POLL_MS;\n  private now: () => number = Date.now;\n  private sleep: (ms: number) => Promise<void> = (ms) =>\n    new Promise((resolve) => setTimeout(resolve, ms));\n\n  configure(settings?: GondolinRemoteSettings, overrides?: GondolinFleetOverrides): void {\n    for (const worker of this.workers.values()) worker.connection.dispose();\n    this.workers.clear();\n    this.settings = settings;\n    this.placements = overrides?.placements ?? gondolinPlacements;\n    this.now = overrides?.now ?? Date.now;\n    this.sleep = overrides?.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));\n    this.queuePollMs = overrides?.queuePollMs ?? QUEUE_POLL_MS;\n    this.connectionOverrides = overrides?.connectionOverrides ?? {};\n    this.leaseTtlMs = (overrides?.connectionOverrides?.leaseTtlSeconds ?? LEASE_TTL_SECONDS) * 1000;\n    this.queueWaitMs = (settings?.queueWaitSeconds ?? QUEUE_WAIT_SECONDS) * 1000;\n    if (!settings) return;\n    if (!settings.workers?.length && !settings.url) {\n      return; // not configured; validate() reports the actionable error\n    }\n    const createConnection =\n      overrides?.createConnection ??\n      ((worker: GondolinRemoteWorkerSettings, connectionOverrides: GondolinRemoteOverrides) =>\n        new GondolinRemoteConnection(worker, connectionOverrides));\n    for (const worker of normalizeWorkers(settings)) {\n      const name = worker.name as string;\n      const connection = createConnection(worker, {\n        ...overrides?.connectionOverrides,\n        onLeaseActivity: (instanceId, expiresAtMs) => {\n          if (this.placements.get(instanceId)?.worker === name) {\n            this.placements.touch(instanceId, expiresAtMs);\n          }\n        },\n      });\n      this.workers.set(name, { name, settings: worker, connection });\n    }\n  }\n\n  isConfigured(): boolean {\n    return this.workers.size > 0;\n  }\n\n  /** Names of statically-configured workers with no workspaceRoot (startup guard). */\n  workersMissingWorkspaceRoot(): string[] {\n    return Array.from(this.workers.values())\n      .filter((worker) => !worker.settings.workspaceRoot)\n      .map((worker) => worker.name);\n  }\n\n  /**\n   * Attach a dial-home worker at runtime (the gateway calls this when a\n   * worker registers). The factory receives the fleet's connection overrides\n   * plus the lease-watermark callback, mirroring configure()'s wiring.\n   */\n  attachWorker(\n    settings: GondolinRemoteWorkerSettings,\n    createConnection: (overrides: GondolinRemoteOverrides) => GondolinWorkerConnection,\n  ): void {\n    const name = settings.name ?? settings.url;\n    this.workers.get(name)?.connection.dispose();\n    const connection = createConnection({\n      ...this.connectionOverrides,\n      onLeaseActivity: (instanceId, expiresAtMs) => {\n        if (this.placements.get(instanceId)?.worker === name) {\n          this.placements.touch(instanceId, expiresAtMs);\n        }\n      },\n    });\n    this.workers.set(name, { name, settings: { ...settings, name }, connection });\n  }\n\n  /**\n   * Record a worker's shared-workspace health (the gateway pushes this from\n   * register and ping frames). A degraded worker keeps its runtimes and its\n   * connection, but new placements skip it and runtime operations fail fast\n   * with the probe's diagnosis instead of hanging on a dead mount.\n   */\n  setWorkspaceHealth(name: string, error?: string): void {\n    const worker = this.workers.get(name);\n    if (!worker) return;\n    worker.workspaceError = error || undefined;\n  }\n\n  /**\n   * Detach a dial-home worker (disconnect). Placements survive: their lease\n   * watermarks keep fencing failover until expiry, exactly as for an\n   * unreachable static worker.\n   */\n  detachWorker(name: string): void {\n    const worker = this.workers.get(name);\n    if (!worker) return;\n    worker.connection.dispose();\n    this.workers.delete(name);\n  }\n\n  imageSelector(): string | undefined {\n    return this.settings?.imageSelector;\n  }\n\n  /** At least one worker must answer for the sandbox mode to come up. */\n  async health(): Promise<Array<{ worker: string; health?: Record<string, unknown> }>> {\n    const report = await Promise.all(\n      Array.from(this.workers.values()).map(async (worker) => {\n        try {\n          return { worker: worker.name, health: await worker.connection.health() };\n        } catch {\n          return { worker: worker.name };\n        }\n      }),\n    );\n    if (report.length > 0 && !report.some((entry) => entry.health)) {\n      throw new SandboxError(\"Error: no gondolin remote worker is reachable\");\n    }\n    return report;\n  }\n\n  async ensure(instanceId: string, spec: GondolinRuntimeSpec): Promise<GondolinRuntimeHandle> {\n    const placed = this.placements.get(instanceId);\n    if (placed) {\n      const worker = this.workers.get(placed.worker);\n      if (worker) {\n        try {\n          return await this.ensureOn(worker, instanceId, spec);\n        } catch (err) {\n          if (!(err instanceof GondolinWorkerUnreachableError)) throw err;\n          this.assertFailoverAllowed(instanceId, placed.worker, placed.leaseExpiresAt, err);\n        }\n      } else {\n        this.assertFailoverAllowed(\n          instanceId,\n          placed.worker,\n          placed.leaseExpiresAt,\n          new GondolinWorkerUnreachableError(\"worker is no longer configured\"),\n        );\n      }\n      log.logWarning(\n        `Gondolin worker '${placed.worker}' lost runtime for '${instanceId}'; failing over`,\n      );\n    }\n    const worker = await this.selectWorker();\n    return this.ensureOn(worker, instanceId, spec);\n  }\n\n  async stop(\n    handle: Pick<GondolinRuntimeHandle, \"workerPid\" | \"sessionId\" | \"instanceId\" | \"workerName\">,\n  ): Promise<void> {\n    const worker = this.workerFor(handle.workerName);\n    if (!worker) {\n      // the worker left the fleet config; its janitor stops the runtime when\n      // the lease expires, and the record keeps fencing until then\n      return;\n    }\n    try {\n      await worker.connection.stop(handle);\n      this.placements.delete(handle.instanceId);\n    } catch (err) {\n      if (err instanceof GondolinWorkerUnreachableError) {\n        // cannot confirm the stop; keep the placement so its watermark still\n        // fences any failover, and let the daemon's expiry janitor finish it\n        log.logWarning(\n          `Gondolin worker '${worker.name}' unreachable while stopping '${handle.instanceId}'; leaving it to lease expiry`,\n        );\n        return;\n      }\n      throw err;\n    }\n  }\n\n  async exec(\n    handle: GondolinRuntimeHandle,\n    command: string,\n    options: { env?: Record<string, string>; signal?: AbortSignal } = {},\n  ): Promise<ExecResult> {\n    const worker = this.workerFor(handle.workerName);\n    if (!worker) throw new GondolinRuntimeGoneError(\"worker is no longer configured\");\n    this.assertWorkspaceUsable(worker);\n    log.logInfo(`Gondolin execution '${handle.instanceId}' routed to worker '${worker.name}'`);\n    return worker.connection.exec(handle, command, options);\n  }\n\n  async isRuntimeAlive(handle: GondolinRuntimeHandle): Promise<boolean> {\n    const worker = this.workerFor(handle.workerName);\n    if (!worker) return false;\n    return worker.connection.isRuntimeAlive(handle);\n  }\n\n  /**\n   * Fleet reconciliation: a runtime whose conversation is placed on a\n   * different worker is a stray from a superseded placement — stop it. A\n   * runtime with no placement at all (crash between daemon ensure and the\n   * placement write) is adopted instead of killed.\n   */\n  async reconcile(): Promise<void> {\n    for (const worker of this.workers.values()) {\n      let runtimes: Array<{ sessionId: string; instanceId: string }>;\n      try {\n        runtimes = await worker.connection.listRuntimes();\n      } catch {\n        continue; // unreachable workers reconcile on a later sweep\n      }\n      for (const runtime of runtimes) {\n        const record = this.placements.get(runtime.instanceId);\n        if (!record) {\n          log.logInfo(\n            `Adopting unplaced Gondolin runtime '${runtime.instanceId}' on worker '${worker.name}'`,\n          );\n          this.placements.set(runtime.instanceId, worker.name, this.now() + this.leaseTtlMs);\n          continue;\n        }\n        if (record.worker === worker.name) continue;\n        log.logInfo(\n          `Stopping stray Gondolin runtime '${runtime.instanceId}' on worker '${worker.name}' (placed on '${record.worker}')`,\n        );\n        try {\n          await worker.connection.stop({\n            sessionId: runtime.sessionId,\n            instanceId: runtime.instanceId,\n            workerPid: 0,\n          });\n        } catch (err) {\n          log.logWarning(\n            `Failed to stop stray runtime '${runtime.instanceId}' on '${worker.name}'`,\n            err instanceof Error ? err.message : String(err),\n          );\n        }\n      }\n    }\n  }\n\n  private async ensureOn(\n    worker: FleetWorker,\n    instanceId: string,\n    spec: GondolinRuntimeSpec,\n  ): Promise<GondolinRuntimeHandle> {\n    this.assertWorkspaceUsable(worker);\n    const handle = await worker.connection.ensure(instanceId, spec);\n    this.placements.set(instanceId, worker.name, this.now() + this.leaseTtlMs);\n    return { ...handle, workerName: worker.name };\n  }\n\n  /**\n   * Fail fast — with the probe's diagnosis — instead of dispatching work that\n   * would hang on a dead shared-workspace mount. Deliberately NOT a\n   * GondolinWorkerUnreachableError: a broken mount must not trigger failover,\n   * because the workspace is shared and likely broken for every worker.\n   */\n  private assertWorkspaceUsable(worker: FleetWorker): void {\n    if (!worker.workspaceError) return;\n    throw new SandboxError(\n      `Error: Gondolin worker '${worker.name}' has an unusable shared workspace: ` +\n        `${worker.workspaceError} (fix the mount on the worker; health refreshes with its next heartbeat)`,\n    );\n  }\n\n  private assertFailoverAllowed(\n    instanceId: string,\n    workerName: string,\n    leaseExpiresAt: number,\n    cause: Error,\n  ): void {\n    const remainingMs = leaseExpiresAt - this.now();\n    if (remainingMs <= 0) return;\n    // Failing over earlier could put a second writable VM on the same shared\n    // workspace: the old worker may be alive behind a partition, and its\n    // daemon only stops the runtime once the lease expires.\n    throw new SandboxError(\n      `Error: Gondolin worker '${workerName}' is unreachable (${cause.message}); ` +\n        `retrying '${instanceId}' elsewhere in ${Math.ceil(remainingMs / 1000)}s once its lease has expired`,\n    );\n  }\n\n  private async selectWorker(): Promise<FleetWorker> {\n    const deadline = this.now() + this.queueWaitMs;\n    for (;;) {\n      let best: { worker: FleetWorker; load: number } | undefined;\n      let reachable = 0;\n      let draining = 0;\n      let full = 0;\n      const degraded: string[] = [];\n      for (const worker of this.workers.values()) {\n        if (worker.settings.draining) {\n          draining += 1;\n          continue;\n        }\n        if (worker.workspaceError) {\n          degraded.push(`'${worker.name}': ${worker.workspaceError}`);\n          continue;\n        }\n        let active: number;\n        let workspaceError: string | undefined;\n        try {\n          const health = await worker.connection.health();\n          active = typeof health.activeRuntimes === \"number\" ? health.activeRuntimes : 0;\n          workspaceError =\n            typeof health.workspaceError === \"string\" && health.workspaceError\n              ? health.workspaceError\n              : undefined;\n        } catch {\n          continue;\n        }\n        reachable += 1;\n        if (workspaceError) {\n          // static workers report probe results via /v1/health only\n          degraded.push(`'${worker.name}': ${workspaceError}`);\n          continue;\n        }\n        const cap = worker.settings.maxRuntimes;\n        if (cap !== undefined && active >= cap) {\n          full += 1;\n          continue;\n        }\n        const load = cap !== undefined ? active / cap : active / Number.MAX_SAFE_INTEGER;\n        if (!best || load < best.load) best = { worker, load };\n      }\n      if (best) return best.worker;\n      if (reachable === 0 && draining === 0 && degraded.length === 0) {\n        throw new SandboxError(\"Error: no gondolin remote worker is reachable\");\n      }\n      if (full === 0) {\n        const detail = degraded.length > 0 ? ` — degraded: ${degraded.join(\"; \")}` : \"\";\n        throw new SandboxError(\n          `Error: every gondolin remote worker is draining, degraded, or unreachable${detail}`,\n        );\n      }\n      if (this.now() >= deadline) {\n        throw new SandboxError(\n          `Error: all gondolin remote workers are at capacity (waited ${Math.round(this.queueWaitMs / 1000)}s)`,\n        );\n      }\n      await this.sleep(this.queuePollMs);\n    }\n  }\n\n  private workerFor(workerName: string | undefined): FleetWorker | undefined {\n    if (!workerName) return undefined;\n    return this.workers.get(workerName);\n  }\n}\n\nexport const gondolinFleet = new GondolinFleetClient();\n"]}

package/dist/sandbox/gondolin-gateway.d.ts

@@ -0,0 +1,86 @@
+import type { Server, Socket } from "node:net";
+import type { GondolinGatewaySettings } from "../types.js";
+/** What a worker announces when it dials home (mirrors the Go register frame). */
+interface GondolinWorkerRegistration {
+    name: string;
+    os?: string;
+    arch?: string;
+    accelerator?: string;
+    cpus?: number;
+    memoryBytes?: number;
+    maxRuntimes?: number;
+    protocolVersion?: number;
+    runtimes?: Array<{
+        sessionId: string;
+        instanceId: string;
+    }>;
+    /** Non-empty when the worker's shared workspace failed its usability probe. */
+    workspaceError?: string;
+}
+interface GatewayOverrides {
+    /** Server factory (tests substitute a plain net.Server). */
+    createServer?: (onConnection: (socket: Socket) => void) => Server;
+    /** mTLS verification result for a socket (tests substitute outcomes). */
+    isAuthorized?: (socket: Socket) => boolean;
+    leaseTtlSeconds?: number;
+    rpcTimeoutMs?: number;
+    tunnelTimeoutMs?: number;
+}
+/**
+ * The host side of dial-home workers: one mTLS listener that accepts worker
+ * control channels (register/ping/RPC responses) and per-command dial-back
+ * tunnels, keeps the registry of connected workers, and attaches each one to
+ * the fleet as a regular worker connection. Placement, leases, fencing, and
+ * failover are unchanged — this only reverses who dials whom.
+ */
+declare class GondolinWorkerGateway {
+    private settings?;
+    private overrides;
+    private server?;
+    private readonly workers;
+    private readonly pendingTunnels;
+    configure(settings?: GondolinGatewaySettings, overrides?: GatewayOverrides): void;
+    isConfigured(): boolean;
+    /** Registered workers and their liveness (admin/observability surface). */
+    list(): Array<{
+        name: string;
+        connected: boolean;
+        activeRuntimes: number;
+        workspaceError?: string;
+        info: GondolinWorkerRegistration;
+    }>;
+    /**
+     * Start the listener. Without explicit certificate settings the gateway
+     * provisions its own: a private worker CA plus a server certificate signed
+     * by it (required for join's CA pinning to also authenticate the host).
+     */
+    start(): Promise<void>;
+    stop(): void;
+    /** Routes one inbound connection by its first frame: control or tunnel. */
+    private handleConnection;
+    private acceptControlChannel;
+    private handleControlFrame;
+    /**
+     * Track shared-workspace health from register/ping frames and push it into
+     * the fleet, logging only the transitions. A worker whose NFS mount died
+     * keeps its control channel alive — this is what turns that silent state
+     * into refused placements and fail-fast execs.
+     */
+    private applyWorkspaceHealth;
+    /** RPC to a worker over its control channel (the reverse of HTTPS requests). */
+    private rpc;
+    private rejectPendingRpc;
+    /** Ask the worker to dial back a data connection for one session tunnel. */
+    private openTunnel;
+    /**
+     * One-time token → CA-signed client certificate. The only operation an
+     * unauthenticated connection may perform; the connection closes after the
+     * response either way.
+     */
+    private handleJoin;
+    private acceptTunnel;
+    private workerBySocket;
+}
+export declare const gondolinGateway: GondolinWorkerGateway;
+export {};
+//# sourceMappingURL=gondolin-gateway.d.ts.map

package/dist/sandbox/gondolin-gateway.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gondolin-gateway.d.ts","sourceRoot":"","sources":["../../src/sandbox/gondolin-gateway.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAI/C,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AAU3D,kFAAkF;AAClF,UAAU,0BAA0B;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC5D,+EAA+E;IAC/E,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAyED,UAAU,gBAAgB;IACxB,4DAA4D;IAC5D,YAAY,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,KAAK,MAAM,CAAC;IAClE,yEAAyE;IACzE,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;IAC3C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAMD;;;;;;GAMG;AACH,cAAM,qBAAqB;IACzB,OAAO,CAAC,QAAQ,CAAC,CAA0B;IAC3C,OAAO,CAAC,SAAS,CAAwB;IACzC,OAAO,CAAC,MAAM,CAAC,CAAS;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuC;IAC/D,OAAO,CAAC,QAAQ,CAAC,cAAc,CAG3B;IAEJ,SAAS,CAAC,QAAQ,CAAC,EAAE,uBAAuB,EAAE,SAAS,CAAC,EAAE,gBAAgB,GAAG,IAAI,CAIhF;IAED,YAAY,IAAI,OAAO,CAEtB;IAED,2EAA2E;IAC3E,IAAI,IAAI,KAAK,CAAC;QACZ,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,OAAO,CAAC;QACnB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,IAAI,EAAE,0BAA0B,CAAC;KAClC,CAAC,CAQD;IAED;;;;OAIG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CA6C3B;IAED,IAAI,IAAI,IAAI,CAUX;IAED,2EAA2E;IAC3E,OAAO,CAAC,gBAAgB;IA8BxB,OAAO,CAAC,oBAAoB;IAgE5B,OAAO,CAAC,kBAAkB;IAsB1B;;;;;OAKG;IACH,OAAO,CAAC,oBAAoB;IAiB5B,gFAAgF;IAChF,OAAO,CAAC,GAAG;IAqCX,OAAO,CAAC,gBAAgB;IAQxB,4EAA4E;IAC5E,OAAO,CAAC,UAAU;IAyDlB;;;;OAIG;YACW,UAAU;IA0BxB,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,cAAc;CAMvB;AAED,eAAO,MAAM,eAAe,uBAA8B,CAAC"}