@bendyline/gezel-inference 0.0.0-stage → 0.1.0

package/dist/providers/native/supervisor.js

@@ -0,0 +1,1466 @@
+/**
+ * NativeEngineSupervisor — lifecycle for a native engine child process
+ * (e.g. `sd-server`, `llama-server`).
+ *
+ * Responsibilities:
+ *   - **Lazy start** the engine on the first `ensureRunning()` call, not
+ *     at service boot. No VRAM is held until something actually asks
+ *     for work.
+ *   - **Idle timeout**: after `idleTimeoutMs` with no activity the
+ *     engine is SIGTERM'd to free GPU/VRAM. Fresh activity lazy-starts
+ *     again.
+ *   - **Health watch**: every `healthIntervalMs` probe `/health`. After
+ *     three consecutive failures the process is killed and a restart
+ *     is attempted. Restart budget is 3 attempts in 60s, matching the
+ *     gezeld supervisor's policy.
+ *
+ * Intentionally engine-agnostic — callers (`StableDiffusionCppProvider`,
+ * `LlamaCppProvider`, future STT/TTS providers) ask `ensureRunning()`
+ * before each request, mark activity via `markUsed()`, and otherwise
+ * know nothing about subprocess plumbing. Tests inject a `spawn` hook
+ * so we can drive the lifecycle without a real binary.
+ */
+import { spawn as nodeSpawn } from 'node:child_process';
+import { basename } from 'node:path';
+import { awakeNow, createLogger } from '@bendyline/gezel';
+import { listProcessSnapshots } from '@bendyline/gezel-client/node';
+import { windowsHeadlessSpawnOptions } from '@bendyline/gezel/native';
+import { acquireNativeCapacity, } from './device-capacity.js';
+const log = createLogger('native');
+/**
+ * Convert Electron's virtual `app.asar` paths to the real files staged by
+ * electron-builder under `app.asar.unpacked`. Electron patches `fs`, so an
+ * existence check in the main process can succeed through the virtual path,
+ * but the OS receives native spawn paths unchanged and rejects them with
+ * ENOTDIR. Normalize again at the service's final launch boundary so embedded
+ * mode is safe even when a pre-set env var bypassed the Electron resolver.
+ */
+export function normalizeNativeEngineLaunch(launch) {
+    const redirect = (value) => value.replace(/(^|[\\/])app\.asar(?=[\\/]|$)/g, '$1app.asar.unpacked');
+    return {
+        ...launch,
+        command: redirect(launch.command),
+        args: launch.args.map(redirect),
+        ...(launch.cwd ? { cwd: redirect(launch.cwd) } : {}),
+        ...(launch.env
+            ? {
+                env: Object.fromEntries(Object.entries(launch.env).map(([key, value]) => [key, redirect(value)])),
+            }
+            : {}),
+    };
+}
+const RESTART_BUDGET = 3;
+const RESTART_WINDOW_MS = 60_000;
+/**
+ * Ceiling on `recoverStartup` retries within one start. Two covers the
+ * full MoE degradation ladder (partial split → all experts to RAM →
+ * engine-owned fit); a hook that keeps asking past that is looping.
+ */
+const MAX_STARTUP_RECOVERIES = 2;
+const HEALTH_FAIL_THRESHOLD = 3;
+/**
+ * How often to restate that the accelerator is under pressure and this engine
+ * cannot do anything about it. Long, because on a card sized to one model the
+ * condition is ordinary and steady — the point is a trail, not an alarm.
+ */
+const PRESSURE_HELD_LOG_INTERVAL_MS = 5 * 60_000;
+const RECENT_OUTPUT_MAX_BYTES = 64 * 1024;
+const EXIT_ATTRIBUTION_WAIT_MS = 250;
+/**
+ * After SIGKILLing reaped orphans, how long to wait for them to actually leave
+ * the process table before spawning the replacement. SIGKILL is asynchronous
+ * and a model-loaded engine holding tens of GB of wired GPU memory (ds4-server
+ * especially) takes a beat to die. ds4-server enforces a HARD singleton, so
+ * spawning into a still-dying orphan races straight into its "another ds4
+ * process is already running; refusing to start".
+ */
+const ORPHAN_REAP_WAIT_MS = 10_000;
+/**
+ * Process-wide set of engine child PIDs currently owned by a LIVE
+ * supervisor in this daemon. The orphan reaper consults it so a freshly
+ * constructed supervisor never reaps a *sibling* engine that another
+ * live supervisor is actively driving.
+ *
+ * This is the fix for a real outage: a model switch (or any second
+ * engine) spins up a new supervisor whose once-per-instance orphan
+ * sweep matches every process sharing the engine's script path —
+ * including the engine still mid-agentic-turn for another session. The
+ * old reaper SIGKILL'd it, dropping the in-flight HTTP stream, which
+ * surfaced to the user as a bare "terminated". A live, owned engine is
+ * never an orphan: its supervisor manages its lifecycle (idle stop,
+ * eviction, health restart) and can drain a turn cleanly. The reaper is
+ * only for engines whose owner is *gone* (force-quit, OS-reaped
+ * Electron), which by definition aren't in this set.
+ *
+ * Shared module singleton: every supervisor in the process registers
+ * its child here on spawn and removes it on exit.
+ */
+const liveEnginePids = new Set();
+/** Test-only: clear the live-engine registry between cases so a pid
+ *  registered by one test can't suppress an orphan reap in the next. */
+export function __resetLiveEnginePidsForTest() {
+    liveEnginePids.clear();
+}
+export class NativeEngineSupervisor {
+    capacity;
+    capacityLease;
+    capacityAbort;
+    startFlight;
+    activeRequests = 0;
+    stopFlight;
+    yieldFlight;
+    get coordinatesCapacity() {
+        return this.capacity !== undefined;
+    }
+    async withRequest(run, signal) {
+        await this.yieldForWaitingCapacity(signal);
+        signal?.throwIfAborted();
+        this.activeRequests++;
+        try {
+            return await run();
+        }
+        finally {
+            this.activeRequests--;
+            this.markUsed();
+        }
+    }
+    /** Stop accepting fresh GPU work while an older memory claim needs this model to drain. */
+    async yieldForWaitingCapacity(signal) {
+        if (this.yieldFlight)
+            return this.yieldFlight;
+        if (!this.capacityLease)
+            return;
+        const flight = (async () => {
+            if (!(await this.capacityLease?.shouldYield()))
+                return;
+            const started = awakeNow();
+            while (this.engineBusy() && this.state.kind === 'running') {
+                signal?.throwIfAborted();
+                if (awakeNow() - started > 5 * 60_000)
+                    throw new Error('Engine is busy serving requests while another model waits for memory.');
+                await sleep(100);
+            }
+            signal?.throwIfAborted();
+            if (this.state.kind !== 'running')
+                return;
+            this.onLog(`${this.logPrefix} releasing idle engine for a waiting memory request`);
+            await this.runFreeze();
+            await this.stop('memory-pressure');
+        })().finally(() => {
+            if (this.yieldFlight === flight)
+                this.yieldFlight = undefined;
+        });
+        this.yieldFlight = flight;
+        return flight;
+    }
+    state = { kind: 'stopped' };
+    resolveLaunch;
+    idleTimeoutMs;
+    freezeTimeoutMs;
+    onFreeze;
+    beforeStop;
+    beforeStopTimeoutMs;
+    isBusy;
+    memoryPressure;
+    pressureIdleTimeoutMs;
+    pressureCheckInFlight = false;
+    pressureDeadlineAt;
+    /** Awake ms at which the current unbroken stretch of pressure began. */
+    pressureHeldSince;
+    /** Awake ms of the last "still pressured, cannot act" log line. */
+    pressureHeldLoggedAt;
+    freezeTimer;
+    /** True once the current idle window has fired its Stage-1 freeze.
+     *  Reset on next `markUsed` so subsequent idle windows re-fire. */
+    freezeFired = false;
+    healthIntervalMs;
+    startupTimeoutMs;
+    spawn;
+    fetchImpl;
+    onLog;
+    onExit;
+    recoverStartup;
+    onRawLine;
+    logPrefix;
+    readinessPath;
+    readyOnAnyResponse;
+    psRunner;
+    killProcess;
+    platform;
+    // Ad-hoc log listeners attached at runtime (vs. the construction-time
+    // `onLog` / `onRawLine` hooks). Used by the owning provider to
+    // subscribe to engine log lines for the duration of a single request
+    // — e.g. `StableDiffusionCppProvider.generate` parses sd-server's
+    // sampling progress from each line and republishes it as chat
+    // events. Listeners receive the SAME prefixed line (`[sd-server] …`)
+    // that `onLog` would; supervisor errors thrown from a listener are
+    // swallowed so a buggy parser can't take down the supervisor.
+    logListeners = new Set();
+    recentStarts = [];
+    lastUsedAt = 0;
+    /**
+     * The same mark on the awake clock. Every idle/pressure DECISION reads this
+     * one; {@link lastUsedAt} stays wall-clock because it is rendered to the
+     * user. A host suspension advances wall clock without giving the engine any
+     * time, and charging that against idleness unloaded resident models on every
+     * dark-wake cycle — the next turn then paid a full cold load.
+     */
+    lastUsedAwakeAt = 0;
+    /** PID of the engine child this supervisor currently owns, registered
+     *  in {@link liveEnginePids} so a sibling supervisor's orphan reaper
+     *  leaves it alone. Set on spawn, cleared on exit. */
+    ownedPid;
+    // Tracks whether we've already done the once-per-process orphan
+    // sweep. Reaping runs ONLY before the first spawn — during restarts
+    // we'd risk killing our own freshly-launched child.
+    didOrphanSweep = false;
+    // When the owning provider detects a fatal error in the child's
+    // stdout (e.g. MLX's `ValueError: Received N parameters not in
+    // model`), it calls `abortStartup(err)` to short-circuit the
+    // `waitForReady` poll loop — otherwise the supervisor would keep
+    // pinging `/health` for the full startup-timeout window even
+    // though the worker thread has already died. Cleared on each
+    // fresh start.
+    startupAbort = null;
+    idleTimer;
+    lastStartFailureAt;
+    lastStartFailure;
+    healthTimer;
+    currentStartedAt = 0;
+    currentPid;
+    /** Sticky for the current child; remains true while an expected stop moves
+     * state to `stopped` before the OS exit event arrives. */
+    currentReachedReady = false;
+    currentDiagnostics;
+    currentOutput = '';
+    currentPanic;
+    expectedExitReason;
+    lastExit;
+    exitListeners = new Set();
+    constructor(opts) {
+        this.capacity = opts.capacity;
+        this.resolveLaunch = opts.resolveLaunch;
+        // 30 min by default — aligns with `OLLAMA_TURN_TIMEOUT_MS` in chat/manager.ts,
+        // the accepted ceiling for a single local-engine turn. The previous 10-min
+        // default killed llama-server mid-stream on legitimately-long Builder
+        // generations (tankcombat trial: 1.7 MB streamed at T+10:27 →
+        // VRAM-saver fired → empty assistant message). Callers that want faster
+        // VRAM reclaim (sd-server idle policy, tests) pass an explicit value.
+        this.idleTimeoutMs = opts.idleTimeoutMs ?? 30 * 60 * 1000;
+        // Default freeze = half of idle when both are set. Operators get
+        // the staged behavior automatically without further config; legacy
+        // callers that pass explicit idleTimeoutMs and don't want a freeze
+        // can pass freezeTimeoutMs:0 to disable it explicitly.
+        if (opts.freezeTimeoutMs !== undefined) {
+            this.freezeTimeoutMs = opts.freezeTimeoutMs;
+        }
+        else {
+            this.freezeTimeoutMs = 0; // disabled by default
+        }
+        if (opts.onFreeze)
+            this.onFreeze = opts.onFreeze;
+        if (opts.beforeStop)
+            this.beforeStop = opts.beforeStop;
+        this.beforeStopTimeoutMs = opts.beforeStopTimeoutMs ?? 30_000;
+        if (opts.isBusy)
+            this.isBusy = opts.isBusy;
+        if (opts.memoryPressure)
+            this.memoryPressure = opts.memoryPressure;
+        this.pressureIdleTimeoutMs = opts.pressureIdleTimeoutMs ?? 60_000;
+        this.healthIntervalMs = opts.healthIntervalMs ?? 15_000;
+        this.startupTimeoutMs = opts.startupTimeoutMs ?? 60_000;
+        this.spawn = opts.spawn ?? nodeSpawn;
+        this.fetchImpl = opts.fetchImpl ?? fetch;
+        this.onLog = opts.onLog ?? ((line) => log.info(line));
+        if (opts.onExit)
+            this.onExit = opts.onExit;
+        if (opts.recoverStartup)
+            this.recoverStartup = opts.recoverStartup;
+        if (opts.onRawLine)
+            this.onRawLine = opts.onRawLine;
+        this.logPrefix = opts.logPrefix ?? '[native]';
+        this.readinessPath = opts.readinessPath ?? '/health';
+        this.readyOnAnyResponse = opts.readyOnAnyResponse ?? false;
+        this.platform = opts.platform ?? process.platform;
+        this.psRunner = opts.psRunner ?? (() => defaultPsRunner(this.platform));
+        this.killProcess = opts.killProcess ?? ((pid, sig) => process.kill(pid, sig));
+    }
+    /** URL the provider should hit. Only meaningful while running. */
+    currentBaseUrl() {
+        if (this.state.kind === 'running' || this.state.kind === 'starting') {
+            return this.state.launch.baseUrl;
+        }
+        return undefined;
+    }
+    /**
+     * PID of the supervised child when running. Used by providers that
+     * want to poll the child's resource usage (e.g. resident-set size
+     * for an "engine memory" telemetry event) — the supervisor owns
+     * the lifetime of the pid, so consumers don't have to track
+     * restarts themselves.
+     */
+    currentChildPid() {
+        if (this.state.kind === 'running' || this.state.kind === 'starting') {
+            return this.state.child.pid ?? undefined;
+        }
+        return undefined;
+    }
+    /** Stable status shape for the engine pool and once-a-second memory UI. */
+    lifecycleSnapshot() {
+        const running = this.state.kind === 'running' || this.state.kind === 'starting';
+        const active = running && this.engineBusy();
+        const idleDeadline = running && !active && this.lastUsedAwakeAt > 0 && this.idleTimeoutMs > 0
+            ? this.wallClockFor(this.lastUsedAwakeAt + this.idleTimeoutMs)
+            : null;
+        const pressureDeadline = running && !active && this.pressureDeadlineAt !== undefined
+            ? this.wallClockFor(this.pressureDeadlineAt)
+            : null;
+        const usePressure = pressureDeadline !== null && (idleDeadline === null || pressureDeadline < idleDeadline);
+        const pid = this.currentChildPid();
+        return {
+            running,
+            active,
+            ...(pid !== undefined ? { pid } : {}),
+            lastUsedAt: this.lastUsedAt > 0 ? this.lastUsedAt : null,
+            unloadAt: usePressure ? pressureDeadline : idleDeadline,
+            idleTimeoutMs: this.idleTimeoutMs,
+            releaseReason: usePressure ? 'memory-pressure' : idleDeadline !== null ? 'idle' : null,
+            ...(this.lastStartFailureAt !== undefined
+                ? {
+                    startFailedAt: this.lastStartFailureAt,
+                    ...(this.lastStartFailure ? { startFailure: this.lastStartFailure } : {}),
+                }
+                : {}),
+        };
+    }
+    /** Most recently sealed child exit, including expected idle/shutdown exits. */
+    lastExitSnapshot() {
+        return this.lastExit
+            ? {
+                ...this.lastExit,
+                ...(this.lastExit.diagnostics ? { diagnostics: { ...this.lastExit.diagnostics } } : {}),
+            }
+            : undefined;
+    }
+    /**
+     * Launch provenance of the LIVE child: pid, start time, and the
+     * request-independent launch facts the spawner attached (granted context
+     * window, slots, KV dtype, backend — the payload behind the
+     * `launch {...}` log line). Undefined when no child is up, so callers
+     * (Settings → About, `/api/system/diagnostics`) never show a stale grant
+     * for an engine that already exited.
+     */
+    launchSnapshot() {
+        if (this.state.kind !== 'running' && this.state.kind !== 'starting')
+            return undefined;
+        const pid = this.currentChildPid();
+        return {
+            ...(pid !== undefined ? { pid } : {}),
+            startedAt: this.currentStartedAt,
+            ...(this.currentDiagnostics ? { diagnostics: { ...this.currentDiagnostics } } : {}),
+        };
+    }
+    /**
+     * Wait briefly for an unexpected exit attributable to a request that began
+     * at `sinceMs`. Fetch/stream teardown can race Node's child `exit` event by
+     * a few milliseconds; this bounded wait prevents CUDA SIGABRTs from being
+     * flattened into a generic undici "terminated" error.
+     */
+    async waitForUnexpectedExitSince(sinceMs, timeoutMs = EXIT_ATTRIBUTION_WAIT_MS) {
+        const matches = (snapshot) => !snapshot.expected && snapshot.exitedAt >= sinceMs;
+        if (this.lastExit && matches(this.lastExit))
+            return { ...this.lastExit };
+        if (timeoutMs <= 0)
+            return undefined;
+        return await new Promise((resolve) => {
+            const listener = (snapshot) => {
+                if (!matches(snapshot))
+                    return;
+                clearTimeout(timer);
+                this.exitListeners.delete(listener);
+                resolve({ ...snapshot });
+            };
+            this.exitListeners.add(listener);
+            const timer = setTimeout(() => {
+                this.exitListeners.delete(listener);
+                resolve(undefined);
+            }, timeoutMs);
+            timer.unref?.();
+        });
+    }
+    /**
+     * Ensure the engine is running and reachable. If it's stopped, spawn
+     * it; if it's starting, await the in-flight readiness probe; if it's
+     * already running, no-op. Rejects if the restart budget is exhausted.
+     */
+    async ensureRunning() {
+        if (this.stopFlight)
+            await this.stopFlight;
+        this.lastUsedAt = Date.now();
+        this.lastUsedAwakeAt = awakeNow();
+        this.pressureDeadlineAt = undefined;
+        if (this.startFlight)
+            await this.startFlight;
+        if (this.state.kind === 'running') {
+            this.resetIdleTimer();
+            return this.state.launch;
+        }
+        if (this.state.kind === 'starting') {
+            await this.state.readyPromise;
+            const current = this.state;
+            if (current.kind === 'running')
+                return current.launch;
+            throw new Error(`${this.logPrefix} failed to start`);
+        }
+        if (this.state.kind === 'restart-budget-exhausted') {
+            if (Date.now() - this.state.at < RESTART_WINDOW_MS) {
+                throw new Error(`${this.logPrefix} failed repeatedly. Wait a minute and try again, or check logs in Settings.`);
+            }
+            this.recentStarts = [];
+            this.state = { kind: 'stopped' };
+        }
+        await this.startFresh();
+        const current = this.state;
+        if (current.kind === 'running')
+            return current.launch;
+        throw new Error(`${this.logPrefix} did not reach running state`);
+    }
+    /**
+     * Attach a transient log listener that receives every prefixed
+     * stdout/stderr line (`[sd-server] …`) for the duration of the
+     * returned unsubscribe call. Used by the owning provider to parse
+     * engine progress for a single in-flight request — e.g. sd-server's
+     * `|==> | 3/20 - 18.20s/it` sampling lines — and translate them
+     * into structured progress events for the UI.
+     *
+     * Listeners are independent of `onLog` / `onRawLine` (which are
+     * construction-time hooks for log routing + classification). Adding
+     * one does not suppress either.
+     */
+    subscribeLogLines(listener) {
+        this.logListeners.add(listener);
+        return () => {
+            this.logListeners.delete(listener);
+        };
+    }
+    /** Update the last-used clock without triggering a start. */
+    markUsed() {
+        this.lastUsedAt = Date.now();
+        this.lastUsedAwakeAt = awakeNow();
+        this.pressureDeadlineAt = undefined;
+        this.resetIdleTimer();
+        if (this.capacityLease)
+            void this.pressureProbe();
+    }
+    async stop(reason = 'stop') {
+        if (this.stopFlight)
+            return this.stopFlight;
+        const flight = this.stopChild(reason).finally(() => {
+            if (this.stopFlight === flight)
+                this.stopFlight = undefined;
+        });
+        this.stopFlight = flight;
+        return flight;
+    }
+    async stopChild(reason) {
+        this.capacityAbort?.abort(new Error(`${this.logPrefix} engine start cancelled`));
+        this.startupAbort?.abort(new Error(`${this.logPrefix} engine start cancelled`));
+        await this.startFlight?.catch(() => { });
+        this.clearIdleTimer();
+        this.clearFreezeTimer();
+        this.clearHealthTimer();
+        if (this.state.kind === 'stopped' || this.state.kind === 'restart-budget-exhausted')
+            return;
+        const { child } = this.state;
+        if (this.state.kind === 'running' && reason !== 'health-restart') {
+            await this.runBeforeStop(reason, this.state.launch.baseUrl);
+        }
+        this.pressureDeadlineAt = undefined;
+        this.expectedExitReason = reason;
+        this.state = { kind: 'stopped' };
+        await killGracefully(child);
+    }
+    async runBeforeStop(reason, baseUrl) {
+        if (!this.beforeStop)
+            return;
+        const started = Date.now();
+        let timer;
+        try {
+            await Promise.race([
+                Promise.resolve(this.beforeStop({ reason, baseUrl })),
+                new Promise((resolve) => {
+                    timer = setTimeout(() => {
+                        this.onLog(`${this.logPrefix} pre-stop hook still running after ${this.beforeStopTimeoutMs}ms — stopping anyway`);
+                        resolve();
+                    }, this.beforeStopTimeoutMs);
+                    timer.unref?.();
+                }),
+            ]);
+            this.onLog(`${this.logPrefix} pre-stop hook done in ${Date.now() - started}ms (${reason})`);
+        }
+        catch (err) {
+            this.onLog(`${this.logPrefix} pre-stop hook failed (${reason}): ${err instanceof Error ? err.message : String(err)}`);
+        }
+        finally {
+            if (timer)
+                clearTimeout(timer);
+        }
+    }
+    async startFresh() {
+        if (this.startFlight)
+            return this.startFlight;
+        this.capacityAbort = new AbortController();
+        const flight = this.startFreshAttempt()
+            .then((result) => {
+            this.lastStartFailureAt = undefined;
+            this.lastStartFailure = undefined;
+            return result;
+        })
+            .catch((err) => {
+            this.lastStartFailureAt = Date.now();
+            this.lastStartFailure = err instanceof Error ? err.message : String(err);
+            throw err;
+        })
+            .finally(() => {
+            if (this.startFlight === flight)
+                this.startFlight = undefined;
+            this.capacityAbort = undefined;
+        });
+        this.startFlight = flight;
+        return flight;
+    }
+    async startFreshAttempt(isRetry = false, recoveryAttempt = 0) {
+        const now = Date.now();
+        this.recentStarts = this.recentStarts.filter((t) => now - t < RESTART_WINDOW_MS);
+        if (this.recentStarts.length >= RESTART_BUDGET) {
+            this.state = { kind: 'restart-budget-exhausted', at: now };
+            throw new Error(`${this.logPrefix} crashed too many times; stopping restart attempts for a minute`);
+        }
+        const launch = normalizeNativeEngineLaunch(await this.resolveLaunch());
+        // First spawn of this process? Sweep up any matching orphans from
+        // previous app launches that didn't get a chance to clean up
+        // (force-quit, OS reaped Electron, before-quit handler crashed,
+        // etc.). Match is anchored on the binary path so we don't touch
+        // unrelated processes; restarts skip the sweep so we don't shoot
+        // our own foot.
+        if (!this.didOrphanSweep) {
+            this.didOrphanSweep = true;
+            await this.reapOrphansMatching(launch).catch((err) => {
+                this.onLog(`${this.logPrefix} orphan sweep failed (continuing): ${err instanceof Error ? err.message : err}`);
+            });
+        }
+        this.capacityAbort?.signal.throwIfAborted();
+        if (this.capacity) {
+            this.capacityLease = await acquireNativeCapacity(this.capacity, launch, this.capacityAbort.signal, (message) => {
+                const line = `${this.logPrefix} memory-admission: ${message}`;
+                this.onLog(line);
+                try {
+                    this.onRawLine?.(line);
+                }
+                catch {
+                    /* progress cannot abort admission */
+                }
+                for (const listener of this.logListeners) {
+                    try {
+                        listener(line);
+                    }
+                    catch {
+                        /* progress cannot abort admission */
+                    }
+                }
+            });
+        }
+        this.recentStarts.push(Date.now());
+        let child;
+        try {
+            child = this.spawn(launch.command, launch.args, {
+                env: { ...process.env, ...(launch.env ?? {}) },
+                ...(launch.cwd ? { cwd: launch.cwd } : {}),
+                stdio: ['ignore', 'pipe', 'pipe'],
+                // The supervisor owns this engine for its entire lifetime. Keep the
+                // Windows console hidden without detaching the process from its
+                // owner; detachment is what surfaces a new terminal window.
+                ...windowsHeadlessSpawnOptions(),
+            });
+        }
+        catch (err) {
+            await this.releaseCapacity();
+            throw nativeSpawnError(this.logPrefix, launch, err);
+        }
+        // Register the child as a live, owned engine so no sibling
+        // supervisor's orphan reaper can SIGKILL it mid-turn. Cleared in
+        // handleExit. `pid` is undefined only if the spawn itself failed —
+        // handleExit will fire and tear the state down regardless.
+        this.ownedPid = typeof child.pid === 'number' ? child.pid : undefined;
+        if (this.ownedPid !== undefined)
+            liveEnginePids.add(this.ownedPid);
+        this.currentStartedAt = Date.now();
+        this.currentPid = typeof child.pid === 'number' ? child.pid : undefined;
+        this.currentReachedReady = false;
+        this.currentDiagnostics = launch.diagnostics ? { ...launch.diagnostics } : undefined;
+        this.currentOutput = '';
+        this.currentPanic = undefined;
+        this.expectedExitReason = undefined;
+        this.onLog(`${this.logPrefix} launch ${JSON.stringify({
+            pid: this.currentPid ?? null,
+            command: basename(launch.command),
+            ...(this.currentDiagnostics ?? {}),
+        })}`);
+        const emitLine = (line) => {
+            if (!line)
+                return;
+            const prefixed = `${this.logPrefix} ${line}`;
+            this.appendRecentOutput(prefixed);
+            this.onLog(prefixed);
+            if (this.onRawLine) {
+                try {
+                    this.onRawLine(prefixed);
+                }
+                catch (err) {
+                    // Never let a bad classifier take down the supervisor.
+                    log.warn(`${this.logPrefix} onRawLine threw: ${err instanceof Error ? err.message : err}`);
+                }
+            }
+            if (this.logListeners.size > 0) {
+                for (const listener of this.logListeners) {
+                    try {
+                        listener(prefixed);
+                    }
+                    catch (err) {
+                        log.warn(`${this.logPrefix} log listener threw: ${err instanceof Error ? err.message : err}`);
+                    }
+                }
+            }
+        };
+        const chunkReader = () => {
+            let pending = '';
+            const onChunk = (buf) => {
+                const text = pending + buf.toString();
+                // Split on CR *and* LF so the optional line classifier sees one
+                // logical record at a time. Most engines emit line-buffered
+                // output (\n), but tqdm-style progress bars — notably mlx's
+                // chunked-prefill `Prefill: NN%|…` meter — repaint a single line
+                // in place with a bare `\r` and emit no `\n` until the bar hits
+                // 100%. Splitting on `\n` alone buffers the entire prefill arc
+                // into one unterminated record that only flushes at completion,
+                // so the classifier never sees per-step progress and the mlx
+                // idle watchdog kills the turn mid-prefill (it never gets the
+                // progress events that reset its per-chunk budget). Treating each
+                // `\r` repaint as its own line lets progress flow. A partial
+                // final line is retained across chunks and flushed on process exit.
+                const lines = text.split(/\r\n?|\n/);
+                pending = lines.pop() ?? '';
+                for (const line of lines)
+                    emitLine(line);
+            };
+            return {
+                onChunk,
+                flush: () => {
+                    if (pending)
+                        emitLine(pending);
+                    pending = '';
+                },
+            };
+        };
+        const stdout = chunkReader();
+        const stderr = chunkReader();
+        this.startupAbort = new AbortController();
+        let spawnFailed = false;
+        child.stdout?.on('data', stdout.onChunk);
+        child.stderr?.on('data', stderr.onChunk);
+        child.on('exit', (code, signal) => {
+            stdout.flush();
+            stderr.flush();
+            this.handleExit(code, signal);
+        });
+        // `child_process.spawn` reports missing/non-traversable executables through
+        // the child's `error` event on most Node/platform combinations. Without a
+        // listener this can escape as an opaque process-level exception; abort the
+        // readiness wait with the same actionable detail as a synchronous throw.
+        child.once('error', (err) => {
+            spawnFailed = true;
+            this.startupAbort?.abort(nativeSpawnError(this.logPrefix, launch, err));
+        });
+        const readyPromise = this.waitForReady(launch, this.startupAbort.signal);
+        this.state = { kind: 'starting', launch, child, readyPromise };
+        try {
+            const lease = this.capacityLease;
+            await Promise.all([
+                readyPromise,
+                child.pid !== undefined ? lease?.bind(child.pid) : Promise.resolve(),
+            ]);
+            await lease?.ready();
+            this.startupAbort?.signal.throwIfAborted();
+            this.capacityAbort?.signal.throwIfAborted();
+            if (child.exitCode !== null || child.signalCode !== null)
+                throw new Error(`${this.logPrefix} child exited before becoming ready`);
+            this.currentReachedReady = true;
+            this.state = { kind: 'running', launch, child, healthFails: 0 };
+            this.resetIdleTimer();
+            this.startHealthWatch();
+        }
+        catch (err) {
+            // The panic classifier state is cleared by handleExit — read it now,
+            // before killGracefully drives the exit, so the recovery hook below
+            // sees the fatal line however the failure was surfaced (stdout abort
+            // vs. child death vs. readiness timeout). Cast because TS narrows the
+            // field to `undefined` from the pre-spawn reset above and cannot see
+            // the stdout-callback writes in between.
+            const panicAtFailure = this.currentPanic;
+            const attemptStartedAt = this.currentStartedAt;
+            // A child that failed before the OS assigned a pid cannot be signalled
+            // and may never emit `exit`; waiting for killGracefully would wedge the
+            // chat turn while reporting the original spawn failure.
+            if (!spawnFailed) {
+                this.expectedExitReason ??= 'startup-abort';
+                await killGracefully(child);
+            }
+            await this.releaseCapacity();
+            this.state = { kind: 'stopped' };
+            // Singleton-conflict recovery: a hard-singleton engine (ds4-server)
+            // refuses to start while an owner-less orphan from a prior launch still
+            // holds the lock, surfaced as "child exited ... before becoming ready".
+            // On the first such failure, sweep orphans again — the first-spawn sweep
+            // may have raced one, or this is a RESTART (which skips the sweep) and the
+            // engine's own prior child lingered. Only retry if the sweep actually
+            // cleared a blocking process; a failure with nothing to clear is a real
+            // startup error (bad model, OOM), not a lock conflict, so don't burn a
+            // retry on it. `isRetry` caps recovery at a single extra attempt.
+            if (!isRetry && err instanceof Error && err.message.includes('before becoming ready')) {
+                const cleared = await this.reapOrphansMatching(launch).catch(() => 0);
+                if (cleared > 0) {
+                    this.onLog(`${this.logPrefix} cleared ${cleared} blocking orphan(s) after a failed start — retrying once`);
+                    this.startupAbort = null;
+                    return await this.startFreshAttempt(true, recoveryAttempt);
+                }
+            }
+            // Owner-driven recovery: give the launch resolver a chance to degrade
+            // its plan (the MoE offload ladder after a GPU OOM) and try again.
+            // Spawn failures are excluded — a missing binary cannot be planned
+            // around, and the child never produced a diagnosable panic.
+            if (!spawnFailed && this.recoverStartup && recoveryAttempt < MAX_STARTUP_RECOVERIES) {
+                const exit = this.lastExit;
+                const panicKind = panicAtFailure?.kind ??
+                    (exit && exit.exitedAt >= attemptStartedAt ? exit.panicKind : undefined);
+                let retry = false;
+                try {
+                    retry = await this.recoverStartup({ panicKind, attempt: recoveryAttempt });
+                }
+                catch (hookErr) {
+                    this.onLog(`${this.logPrefix} recoverStartup hook threw (not retrying): ${hookErr instanceof Error ? hookErr.message : hookErr}`);
+                }
+                if (retry) {
+                    this.onLog(`${this.logPrefix} start failed (${panicKind ?? 'no panic classified'}) — owner degraded the launch plan, retrying (${recoveryAttempt + 1}/${MAX_STARTUP_RECOVERIES})`);
+                    this.startupAbort = null;
+                    return await this.startFreshAttempt(isRetry, recoveryAttempt + 1);
+                }
+            }
+            throw err;
+        }
+        finally {
+            this.startupAbort = null;
+        }
+    }
+    /**
+     * Abort an in-flight `startFresh` with a concrete error. Called by
+     * the owning provider when the child's stdout reveals a fatal
+     * condition (model-arch mismatch, OOM, missing dependency) that
+     * won't resolve by waiting longer — without this, the `waitForReady`
+     * poll would keep hitting `/health` for the full startup-timeout
+     * window even though the worker thread has already died.
+     */
+    abortStartup(err) {
+        const ac = this.startupAbort;
+        if (!ac)
+            return;
+        ac.abort(err);
+    }
+    /**
+     * Find and SIGKILL any owner-less engine process from a prior gezel
+     * launch that matches this engine. Best-effort: any error during the
+     * sweep is logged and swallowed so the spawn-on-startup path never
+     * aborts because of cleanup.
+     *
+     * Crucially, a process whose parent is still alive is NOT an orphan and
+     * is skipped. The process-local {@link liveEnginePids} registry protects
+     * sibling supervisors inside this daemon; the PPID check protects engines
+     * owned by a different live gezeld process/home. This matters for two
+     * concurrent isolated eval daemons: each uses the same engine binary, so a
+     * command-path-only sweep would otherwise SIGKILL the other's in-flight
+     * child and surface a bare `terminated` fetch error.
+     *
+     * Match strategy — anchor on every install-specific absolute path we
+     * can find in the launch spec, then check whether each `ps` row
+     * contains any of them:
+     *
+     *   1. `launch.command` — works directly when the binary is
+     *      install-specific (llama-server, sd-server: a path under
+     *      `~/.gezel/native/bin/`).
+     *   2. Any absolute-path argument that looks like a script or
+     *      executable (`*.py`, `*.js`, etc.). For Python engines this is
+     *      the case that *actually* fires: `launch.command` is the venv's
+     *      `bin/python`, but that's typically a symlink chain to the
+     *      system framework Python (e.g. `/Library/Frameworks/Python.../
+     *      MacOS/Python`) — and macOS `ps` reports the resolved path, so
+     *      the binary-path match never hits. The script path
+     *      (`gezel_mlx_server.py` under the dist folder) appears verbatim
+     *      in the ps command line and is unique to this install.
+     *
+     * `didOrphanSweep` gates this to one execution per supervisor instance
+     * (so we don't re-sweep on our own restarts). A real orphan is reparented
+     * to launchd/systemd (PPID 1) after its daemon exits. Any matching child
+     * with PPID > 1 still has a live owner and is conservatively left alone.
+     * Injected legacy `psRunner` rows may omit PPID and retain the historical
+     * match behavior; the production runner always includes it.
+     *
+     * macOS + Linux use `ps -axo pid=,ppid=,command=`. Windows uses the
+     * Win32_Process CIM snapshot; because Windows retains a dead creator's pid
+     * instead of reparenting to pid 1, absence of that ppid proves orphanhood.
+     */
+    async reapOrphansMatching(launch) {
+        const anchors = this.buildOrphanAnchors(launch);
+        if (anchors.command.length === 0 && anchors.args.length === 0)
+            return 0;
+        const procs = await this.psRunner();
+        const livePids = new Set(procs.map(({ pid }) => pid));
+        const normalizedCommandAnchors = anchors.command.map((anchor) => normalizeProcessMatch(anchor, this.platform));
+        const normalizedArgAnchors = anchors.args.map((anchor) => normalizeProcessMatch(anchor, this.platform));
+        const ourPid = process.pid;
+        const targets = [];
+        for (const { pid, ppid, command } of procs) {
+            if (pid === ourPid)
+                continue;
+            // Never reap an engine a live supervisor in this process owns —
+            // it may be mid-turn for another session. Only owner-less orphans
+            // (force-quit / OS-reaped prior launches) are ours to clean up.
+            if (liveEnginePids.has(pid))
+                continue;
+            // Cross-daemon ownership: command-path matching alone cannot
+            // distinguish isolated homes that share a native binary. Unix reparents
+            // a true orphan to pid 1. Windows leaves the creator pid on the child,
+            // so it is an orphan only when that pid is absent from this same atomic-
+            // enough snapshot. PID reuse can cause a conservative false negative,
+            // never a live-process kill. Missing PPID preserves legacy test seams.
+            if (ppid !== undefined && !isOwnerlessProcess(ppid, livePids, this.platform))
+                continue;
+            const normalizedCommand = normalizeProcessMatch(command, this.platform);
+            const matched = normalizedCommandAnchors.some((anchor) => normalizedCommand === anchor ||
+                normalizedCommand.startsWith(`${anchor} `) ||
+                normalizedCommand.startsWith(`"${anchor}"`)) || normalizedArgAnchors.some((anchor) => normalizedCommand.includes(anchor));
+            if (!matched)
+                continue;
+            targets.push(pid);
+        }
+        if (targets.length === 0)
+            return 0;
+        this.onLog(`${this.logPrefix} reaping ${targets.length} orphan(s) from prior app launches: ${targets.join(', ')}`);
+        for (const pid of targets) {
+            try {
+                this.killProcess(pid, 'SIGKILL');
+            }
+            catch (err) {
+                // Already dead, or not ours to kill — either way, ignore.
+                this.onLog(`${this.logPrefix} reap pid=${pid} failed (ignored): ${err instanceof Error ? err.message : err}`);
+            }
+        }
+        // SIGKILL only *requests* termination — block until the reaped pids are
+        // actually gone, so a hard-singleton engine isn't spawned while a just-
+        // killed orphan still holds the lock. Without this wait the new ds4-server
+        // saw the dying orphan and refused to start (the trial then stalled).
+        await this.waitForPidsGone(targets, ORPHAN_REAP_WAIT_MS);
+        return targets.length;
+    }
+    /**
+     * Poll the process table until none of `pids` remain, or `timeoutMs`
+     * elapses. Best-effort: a still-present pid after the timeout is logged (the
+     * subsequent spawn may then fail a singleton check, surfaced normally) rather
+     * than thrown, so cleanup never wedges startup.
+     */
+    async waitForPidsGone(pids, timeoutMs) {
+        if (pids.length === 0)
+            return;
+        const deadline = Date.now() + timeoutMs;
+        let remaining = pids;
+        while (remaining.length > 0 && Date.now() < deadline) {
+            await sleep(150);
+            const alive = new Set((await this.psRunner()).map((p) => p.pid));
+            remaining = remaining.filter((pid) => alive.has(pid));
+        }
+        if (remaining.length > 0) {
+            this.onLog(`${this.logPrefix} reaped orphan(s) still present after ${timeoutMs}ms: ${remaining.join(', ')} — spawn may hit the engine's singleton guard`);
+        }
+    }
+    /**
+     * Pull every install-specific absolute path out of the launch spec.
+     * Used by the orphan reaper to match against `ps` output. The
+     * filtering is deliberately conservative: only paths that look like a
+     * concrete file (have an extension) qualify, so flag values like
+     * `--port 12345` or directory paths passed via `--model` don't get
+     * promoted to anchors. Absolute model dirs would still be
+     * install-specific in practice, but matching on a directory path
+     * could collide with any tool that happens to have it on its command
+     * line (e.g. a `du -sh` invocation), so we leave them out.
+     */
+    buildOrphanAnchors(launch) {
+        const command = [];
+        const args = [];
+        const seen = new Set();
+        const add = (bucket, p) => {
+            if (!p || seen.has(p))
+                return;
+            seen.add(p);
+            bucket.push(p);
+        };
+        if (isAbsoluteProcessPath(launch.command))
+            add(command, launch.command);
+        for (const arg of launch.args) {
+            if (!isAbsoluteProcessPath(arg))
+                continue;
+            // File-shaped: ends with a dot-extension whose tail contains at
+            // least one letter (`gezel_mlx_server.py`, `model.bin`). Skips
+            // bare directory paths (`/Users/me/.gezel/.../models/qwen3.6`),
+            // including ones whose final segment looks like a version
+            // number — `.6`, `1.0.0`, `3.14` would all sneak through a
+            // letter-or-digit regex but none describe a real file extension.
+            if (!/\.[A-Za-z0-9]*[A-Za-z][A-Za-z0-9]{0,7}$/.test(arg))
+                continue;
+            add(args, arg);
+        }
+        return { command, args };
+    }
+    async waitForReady(launch, signal) {
+        const deadline = Date.now() + this.startupTimeoutMs;
+        let lastError;
+        while (Date.now() < deadline) {
+            if (signal.aborted) {
+                throw signal.reason instanceof Error
+                    ? signal.reason
+                    : new Error(`${this.logPrefix} startup aborted`);
+            }
+            try {
+                const res = await this.fetchImpl(`${launch.baseUrl}${this.readinessPath}`, { signal });
+                if (res.ok || this.readyOnAnyResponse) {
+                    // For engines without a dedicated health endpoint, returning
+                    // any HTTP status proves the port is open and accepting work
+                    // — that's the readiness signal we care about. Drain the body
+                    // so the connection isn't left half-open.
+                    void res.body?.cancel?.().catch(() => { });
+                    return;
+                }
+                lastError = new Error(`${this.readinessPath} returned ${res.status}`);
+            }
+            catch (err) {
+                if (signal.aborted) {
+                    throw signal.reason instanceof Error
+                        ? signal.reason
+                        : new Error(`${this.logPrefix} startup aborted`);
+                }
+                lastError = err;
+            }
+            await new Promise((resolve, reject) => {
+                const t = setTimeout(resolve, 500);
+                const onAbort = () => {
+                    clearTimeout(t);
+                    reject(signal.reason instanceof Error ? signal.reason : new Error('startup aborted'));
+                };
+                if (signal.aborted)
+                    onAbort();
+                else
+                    signal.addEventListener('abort', onAbort, { once: true });
+            });
+        }
+        throw new Error(`${this.logPrefix} did not become ready within ${this.startupTimeoutMs / 1000}s: ${lastError instanceof Error ? lastError.message : String(lastError)}`);
+    }
+    appendRecentOutput(line) {
+        this.currentOutput += `${line}\n`;
+        const bytes = Buffer.byteLength(this.currentOutput, 'utf8');
+        if (bytes > RECENT_OUTPUT_MAX_BYTES) {
+            const trimmed = Buffer.from(this.currentOutput, 'utf8')
+                .subarray(bytes - RECENT_OUTPUT_MAX_BYTES)
+                .toString('utf8');
+            const firstNewline = trimmed.indexOf('\n');
+            this.currentOutput = firstNewline >= 0 ? trimmed.slice(firstNewline + 1) : trimmed;
+        }
+        const panic = classifyNativeEnginePanic(line);
+        if (panic &&
+            (!this.currentPanic || panicPriority(panic.kind) >= panicPriority(this.currentPanic.kind))) {
+            this.currentPanic = panic;
+        }
+    }
+    handleExit(code, signal) {
+        void this.releaseCapacity();
+        const priorState = this.state;
+        const exitedAt = Date.now();
+        const expectedReason = this.expectedExitReason;
+        const expected = priorState.kind === 'stopped' || expectedReason !== undefined;
+        const pid = this.currentPid;
+        // A SIGILL is a classification in its own right. Only claim it when
+        // the output didn't already explain the death more precisely: an
+        // engine that printed `CUDA error: …` and then took SIGILL on the
+        // way down is a CUDA fault, not an unrunnable build.
+        const panic = this.currentPanic ??
+            (signal === 'SIGILL'
+                ? {
+                    kind: 'illegal-instruction',
+                    line: 'engine terminated by SIGILL (illegal instruction) with no diagnostic output',
+                }
+                : undefined);
+        const snapshot = {
+            incidentId: `native-${pid ?? 'unknown'}-${exitedAt}`,
+            ...(pid !== undefined ? { pid } : {}),
+            startedAt: this.currentStartedAt || exitedAt,
+            exitedAt,
+            uptimeMs: Math.max(0, exitedAt - (this.currentStartedAt || exitedAt)),
+            code,
+            signal,
+            expected,
+            reachedReady: this.currentReachedReady,
+            ...(expectedReason ? { expectedReason } : {}),
+            ...(panic ? { panicKind: panic.kind, panicLine: panic.line } : {}),
+            outputTail: this.currentOutput,
+            ...(this.currentDiagnostics ? { diagnostics: { ...this.currentDiagnostics } } : {}),
+        };
+        this.lastExit = snapshot;
+        this.onLog(`${this.logPrefix} exit ${JSON.stringify({
+            incidentId: snapshot.incidentId,
+            pid: snapshot.pid ?? null,
+            code,
+            signal: signal ?? null,
+            expected,
+            ...(expectedReason ? { expectedReason } : {}),
+            uptimeMs: snapshot.uptimeMs,
+            reachedReady: snapshot.reachedReady,
+            ...(snapshot.panicKind ? { panicKind: snapshot.panicKind } : {}),
+            ...(snapshot.panicLine ? { panicLine: snapshot.panicLine } : {}),
+            ...(snapshot.diagnostics ?? {}),
+        })}`);
+        try {
+            const callbackResult = this.onExit?.({ ...snapshot });
+            if (callbackResult) {
+                void Promise.resolve(callbackResult).catch((err) => {
+                    log.warn(`${this.logPrefix} onExit rejected: ${err instanceof Error ? err.message : String(err)}`);
+                });
+            }
+        }
+        catch (err) {
+            log.warn(`${this.logPrefix} onExit threw: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        for (const listener of this.exitListeners)
+            listener(snapshot);
+        // Deregister before any early-return: this owned engine is gone, so
+        // it's free for a future reaper to clean up if it somehow outlives us.
+        if (this.ownedPid !== undefined) {
+            liveEnginePids.delete(this.ownedPid);
+            this.ownedPid = undefined;
+        }
+        this.clearIdleTimer();
+        this.clearFreezeTimer();
+        this.clearHealthTimer();
+        this.currentPid = undefined;
+        this.currentStartedAt = 0;
+        this.currentReachedReady = false;
+        this.currentDiagnostics = undefined;
+        this.currentOutput = '';
+        this.currentPanic = undefined;
+        this.expectedExitReason = undefined;
+        if (priorState.kind === 'stopped')
+            return;
+        this.onLog(`${this.logPrefix} exited (code=${code} signal=${signal ?? 'none'})`);
+        // If the child died while we were still waiting for /health to come
+        // up, abort the startup wait immediately. Otherwise `waitForReady`
+        // would keep polling a dead port for the full startup-timeout
+        // budget (default 180s) — observed as a 3-minute wedge in chat
+        // when an unclassified Python exception kills the engine before
+        // ready. The provider's stdout classifier may still produce a
+        // better-typed error a moment later via `abortStartup(err)`; that
+        // call wins because it lands first (the classifier sees the leaf
+        // exception line slightly before the kernel reports the exit).
+        if (this.startupAbort && priorState.kind === 'starting') {
+            this.startupAbort.abort(new Error(`${this.logPrefix} child exited (code=${code} signal=${signal ?? 'none'}) before becoming ready`));
+        }
+        this.state = { kind: 'stopped' };
+    }
+    startHealthWatch() {
+        this.clearHealthTimer();
+        this.healthTimer = setInterval(() => {
+            void this.healthProbe();
+            void this.pressureProbe();
+        }, this.healthIntervalMs);
+        this.healthTimer.unref?.();
+    }
+    /**
+     * Accelerator pressure this engine is currently sitting inside, or null when
+     * there is none. Read by callers that want to explain a symptom the engine
+     * itself cannot see — a turn whose throughput collapsed has no way to know
+     * the card is oversubscribed, and "the engine is slow" is the wrong story.
+     */
+    memoryPressureHeld() {
+        if (this.pressureHeldSince === undefined)
+            return null;
+        return {
+            sinceAwakeMs: this.pressureHeldSince,
+            heldMs: Math.max(0, awakeNow() - this.pressureHeldSince),
+        };
+    }
+    engineBusy() {
+        try {
+            return this.activeRequests > 0 || this.isBusy?.() === true;
+        }
+        catch (error) {
+            this.onLog(`${this.logPrefix} busy predicate failed: ${error instanceof Error ? error.message : error}`);
+            return true;
+        }
+    }
+    async pressureProbe() {
+        if ((!this.memoryPressure && !this.capacityLease) ||
+            this.pressureCheckInFlight ||
+            this.state.kind !== 'running')
+            return;
+        this.pressureCheckInFlight = true;
+        try {
+            const requested = (await this.capacityLease?.shouldYield()) ?? false;
+            if (requested && this.yieldFlight)
+                return;
+            const pressure = requested
+                ? { pressured: true, detail: 'another engine is waiting for memory' }
+                : ((await this.memoryPressure?.()) ?? { pressured: false });
+            if (!pressure.pressured || this.state.kind !== 'running') {
+                this.pressureDeadlineAt = undefined;
+                this.pressureHeldSince = undefined;
+                this.pressureHeldLoggedAt = undefined;
+                return;
+            }
+            const now = awakeNow();
+            if (this.pressureHeldSince === undefined)
+                this.pressureHeldSince = now;
+            this.pressureDeadlineAt = this.lastUsedAwakeAt + this.pressureIdleTimeoutMs;
+            const busy = this.engineBusy();
+            if ((!requested && now < this.pressureDeadlineAt) || busy) {
+                // The only lever here is "stop MY engine once MY engine is idle", and
+                // a busy one is out of reach of it. Saying so beats the silence that
+                // preceded it: a four-hour turn on an oversubscribed card ran this
+                // probe about 960 times and emitted nothing at all, so the machine
+                // state that made every other tenant unusable left no trace in the
+                // log. Rate-limited because pressure is normal and steady on a card
+                // sized to one model.
+                if (this.pressureHeldLoggedAt === undefined ||
+                    now - this.pressureHeldLoggedAt >= PRESSURE_HELD_LOG_INTERVAL_MS) {
+                    this.pressureHeldLoggedAt = now;
+                    const held = Math.round((now - this.pressureHeldSince) / 1000);
+                    const why = busy ? 'engine is serving a turn' : 'still inside the warm-retention grace';
+                    const detail = pressure.detail ? ` (${pressure.detail})` : '';
+                    this.onLog(`${this.logPrefix} memory pressure${detail} held ${held}s — ${why}, not releasing`);
+                }
+                return;
+            }
+            if (requested) {
+                // Use the same gate as incoming requests, including while flushing.
+                // A new request must not enter the engine between idle detection and
+                // the awaited cache flush that precedes releasing its memory.
+                await this.yieldForWaitingCapacity();
+                return;
+            }
+            this.onLog(`${this.logPrefix} memory pressure${pressure.detail ? ` (${pressure.detail})` : ''} — flushing and stopping idle engine`);
+            await this.runFreeze();
+            if (this.engineBusy())
+                return;
+            await this.stop('memory-pressure');
+        }
+        catch (error) {
+            this.onLog(`${this.logPrefix} memory-pressure check failed: ${error instanceof Error ? error.message : error}`);
+        }
+        finally {
+            this.pressureCheckInFlight = false;
+        }
+    }
+    async releaseCapacity() {
+        const lease = this.capacityLease;
+        this.capacityLease = undefined;
+        await lease
+            ?.release()
+            .catch((error) => this.onLog(`${this.logPrefix} memory lease release failed: ${error}`));
+    }
+    async healthProbe() {
+        if (this.state.kind !== 'running')
+            return;
+        const launch = this.state.launch;
+        try {
+            const res = await this.fetchImpl(`${launch.baseUrl}${this.readinessPath}`);
+            if (res.ok || this.readyOnAnyResponse) {
+                void res.body?.cancel?.().catch(() => { });
+                if (this.state.kind === 'running')
+                    this.state.healthFails = 0;
+                return;
+            }
+        }
+        catch {
+            /* treat as failure below */
+        }
+        if (this.state.kind !== 'running')
+            return;
+        // A busy engine can be too busy to answer: a 12-image vision prefill held
+        // the MLX server's event loop, three probes went unanswered, and the
+        // restart threw away a 20-minute powerpoint-deck evaluate turn
+        // (qwen3.8-27b, 2026-09-23). A turn in flight has its own awake-time
+        // deadlines that end a genuinely wedged engine, so only an idle engine's
+        // silence counts here.
+        if (this.engineBusy())
+            return;
+        this.state.healthFails++;
+        if (this.state.healthFails >= HEALTH_FAIL_THRESHOLD) {
+            this.onLog(`${this.logPrefix} ${HEALTH_FAIL_THRESHOLD} health failures — restarting`);
+            await this.stop('health-restart');
+            try {
+                await this.ensureRunning();
+            }
+            catch (err) {
+                this.onLog(`${this.logPrefix} restart failed: ${err instanceof Error ? err.message : err}`);
+            }
+        }
+    }
+    resetIdleTimer() {
+        this.clearIdleTimer();
+        this.clearFreezeTimer();
+        this.freezeFired = false;
+        // Stage 1 (freeze): persist transient state to disk while the
+        // child is still healthy. Doesn't stop the engine. Skipped when
+        // freezeTimeoutMs is unset/0 (legacy single-stage behavior) or
+        // when no onFreeze callback was supplied.
+        if (this.freezeTimeoutMs > 0 && this.onFreeze && this.freezeTimeoutMs < this.idleTimeoutMs) {
+            this.freezeTimer = setTimeout(() => {
+                // Fire-and-forget; supervisor logs + swallows any error so a
+                // bad callback can't crash the host.
+                void this.runFreeze();
+            }, this.freezeTimeoutMs + 50);
+            this.freezeTimer.unref?.();
+        }
+        // Stage 2 (idle): full SIGTERM. Existing behavior — runs after
+        // idleTimeoutMs regardless of whether Stage 1 fired.
+        this.armIdleTimer(this.idleTimeoutMs);
+    }
+    /**
+     * Arm the VRAM-release timer to fire `delayMs` from now, then re-check
+     * against the AWAKE clock when it does.
+     *
+     * The re-check is the load-bearing part. A host suspension fires every
+     * overdue timer the instant the machine resumes, so a nap longer than the
+     * idle timeout used to read as "this engine has been idle that long" and
+     * unload it — observed unloading a resident 27B model on each ~16 min
+     * dark-wake cycle, so the next turn paid a full cold load for idleness
+     * nobody could have avoided. Re-arming for the shortfall matters just as
+     * much: an early return with no re-arm would leave the engine resident for
+     * good, trading a spurious unload for a permanent leak.
+     */
+    armIdleTimer(delayMs) {
+        this.clearIdleTimer();
+        if (this.idleTimeoutMs <= 0)
+            return;
+        this.idleTimer = setTimeout(() => {
+            const since = awakeNow() - this.lastUsedAwakeAt;
+            if (since < this.idleTimeoutMs) {
+                this.armIdleTimer(this.idleTimeoutMs - since);
+                return;
+            }
+            // Don't interrupt an actual native request. Logical turns parked on a
+            // tool or a person are intentionally not busy here; a later iteration
+            // can lazy-start the engine again from its flushed disk cache.
+            if (this.engineBusy()) {
+                this.onLog(`${this.logPrefix} idle deadline reached but an engine request is active — deferring VRAM stop`);
+                this.resetIdleTimer();
+                return;
+            }
+            this.onLog(`${this.logPrefix} idle timeout — stopping to free VRAM`);
+            void this.stop('idle');
+        }, delayMs + 50);
+        this.idleTimer.unref?.();
+    }
+    /**
+     * Project an awake-clock instant onto the wall clock for display. The UI
+     * renders a countdown, and a deadline held in awake time would otherwise
+     * show as an already-past timestamp after any suspension.
+     */
+    wallClockFor(awakeInstant) {
+        return Date.now() + (awakeInstant - awakeNow());
+    }
+    /**
+     * Run the user-supplied onFreeze hook with proper guarding. Marks
+     * `freezeFired` so subsequent ticks within this idle window don't
+     * re-fire (idempotency for callers, e.g. a re-entrant flush would
+     * double-write the same content).
+     */
+    async runFreeze() {
+        if (this.freezeFired)
+            return;
+        // Cache endpoints share the native server with inference; do not flush a
+        // slot while that engine is actively reading or writing it. The next idle
+        // deadline (or pressure probe) gets another chance.
+        if (this.engineBusy())
+            return;
+        this.freezeFired = true;
+        if (!this.onFreeze)
+            return;
+        this.onLog(`${this.logPrefix} idle freeze — flushing caches to disk (model stays resident)`);
+        try {
+            await this.onFreeze();
+        }
+        catch (err) {
+            this.onLog(`${this.logPrefix} freeze callback threw: ${err instanceof Error ? err.message : err}`);
+        }
+    }
+    clearIdleTimer() {
+        if (this.idleTimer) {
+            clearTimeout(this.idleTimer);
+            this.idleTimer = undefined;
+        }
+    }
+    clearFreezeTimer() {
+        if (this.freezeTimer) {
+            clearTimeout(this.freezeTimer);
+            this.freezeTimer = undefined;
+        }
+    }
+    clearHealthTimer() {
+        if (this.healthTimer) {
+            clearInterval(this.healthTimer);
+            this.healthTimer = undefined;
+        }
+    }
+}
+/** High-precision fatal signatures; attribution happens only if the child exits. */
+export function classifyNativeEnginePanic(line) {
+    const clipped = line.trim().slice(0, 1_000);
+    if (/CUDA error:\s*invalid (?:configuration )?argument/i.test(line)) {
+        return { kind: 'cuda-invalid-argument', line: clipped };
+    }
+    if (/CUDA error:\s*(?:out of memory|memory allocation)/i.test(line)) {
+        return { kind: 'cuda-out-of-memory', line: clipped };
+    }
+    // Vulkan's spellings of the same condition — the backend a non-CUDA
+    // discrete card (or a CUDA card without nvidia-smi) runs on.
+    if (/ErrorOutOfDeviceMemory|OutOfDeviceMemoryError|ggml_vulkan:.*(?:allocation|memory).*failed/i.test(line)) {
+        return { kind: 'vulkan-out-of-memory', line: clipped };
+    }
+    if (/CUDA error:\s*(?:an )?illegal memory access/i.test(line)) {
+        return { kind: 'cuda-illegal-memory-access', line: clipped };
+    }
+    if (/CUDA error:.*device-side assert/i.test(line)) {
+        return { kind: 'cuda-device-assert', line: clipped };
+    }
+    if (/CUDA error:/i.test(line))
+        return { kind: 'cuda-error', line: clipped };
+    if (/assert(?:ion)? failed|GGML_ASSERT/i.test(line)) {
+        return { kind: 'assertion-failed', line: clipped };
+    }
+    return undefined;
+}
+function panicPriority(kind) {
+    if (kind === 'cuda-error')
+        return 1;
+    if (kind === 'assertion-failed')
+        return 2;
+    return 3;
+}
+/**
+ * Default process snapshot. PPID is load-bearing: command paths identify
+ * which engine a row belongs to, while the parent snapshot distinguishes an
+ * owner-less prior-launch orphan from an engine actively owned by another
+ * gezeld process. Errors remain best-effort so cleanup cannot block startup.
+ */
+async function defaultPsRunner(platform) {
+    try {
+        return await listProcessSnapshots({ platform });
+    }
+    catch {
+        return [];
+    }
+}
+function isOwnerlessProcess(ppid, livePids, platform) {
+    return platform === 'win32' ? !livePids.has(ppid) : ppid <= 1;
+}
+function normalizeProcessMatch(value, platform) {
+    return platform === 'win32' ? value.replaceAll('/', '\\').toLowerCase() : value;
+}
+function isAbsoluteProcessPath(value) {
+    // Accept both syntaxes independent of the host running the test/build. The
+    // launch value itself is authoritative, and cross-platform fixtures exercise
+    // Windows paths on Unix CI and Unix paths on Windows CI.
+    return value.startsWith('/') || /^(?:[A-Za-z]:[\\/]|\\\\)/.test(value);
+}
+/** Parse the portable `ps -axo pid=,ppid=,command=` shape. */
+export function parseNativeProcessSnapshot(stdout) {
+    const out = [];
+    for (const rawLine of stdout.split('\n')) {
+        const line = rawLine.trimStart();
+        if (!line)
+            continue;
+        const m = line.match(/^(\d+)\s+(\d+)\s+(.*)$/);
+        if (!m)
+            continue;
+        const pid = Number.parseInt(m[1], 10);
+        const ppid = Number.parseInt(m[2], 10);
+        if (!Number.isFinite(pid) || !Number.isFinite(ppid))
+            continue;
+        out.push({ pid, ppid, command: m[3] });
+    }
+    return out;
+}
+/** `code` stamped on every engine-launch failure. See {@link isEngineLaunchError}. */
+export const ENGINE_LAUNCH_ERROR_CODE = 'engine-launch';
+/**
+ * True when the engine binary itself never started, however deep the error
+ * was rewrapped. Matches the code first, then falls back to the stable
+ * message so a rewrapped error still classifies right (same belt-and-braces
+ * shape as the pool's `engine-busy` matcher).
+ */
+export function isEngineLaunchError(err) {
+    if (err &&
+        typeof err === 'object' &&
+        err.code === ENGINE_LAUNCH_ERROR_CODE) {
+        return true;
+    }
+    const message = err instanceof Error ? err.message : String(err);
+    return /could not launch /.test(message);
+}
+function nativeSpawnError(logPrefix, launch, cause) {
+    const code = cause && typeof cause === 'object' && 'code' in cause && typeof cause.code === 'string'
+        ? cause.code
+        : undefined;
+    const detail = cause instanceof Error ? cause.message : String(cause);
+    const err = new Error(`${logPrefix} could not launch ${basename(launch.command)}${code ? ` (${code})` : ''}. ` +
+        `Executable: ${launch.command}${launch.cwd ? `; cwd: ${launch.cwd}` : ''}. ${detail}`, { cause });
+    // Consumers that report *why* a model failed need to tell "the engine
+    // never came up" apart from "the model answered badly". The engine starts
+    // lazily on the first turn, so without this marker a launch failure looks
+    // like a generation failure — which is how the fitness proeve came to
+    // report "engine spawned and served the probe session" for a model whose
+    // engine had never started.
+    return Object.assign(err, { code: ENGINE_LAUNCH_ERROR_CODE });
+}
+async function killGracefully(child) {
+    if (child.exitCode !== null || child.signalCode !== null)
+        return;
+    const exited = new Promise((resolve) => child.once('exit', () => resolve()));
+    try {
+        child.kill('SIGTERM');
+    }
+    catch {
+        /* already dead */
+    }
+    const timeout = setTimeout(() => {
+        try {
+            child.kill('SIGKILL');
+        }
+        catch {
+            /* */
+        }
+    }, 3_000);
+    timeout.unref?.();
+    await exited;
+    clearTimeout(timeout);
+}
+function sleep(ms) {
+    return new Promise((resolve) => {
+        const t = setTimeout(resolve, ms);
+        t.unref?.();
+    });
+}