@bendyline/gezel-inference 0.0.0-stage → 1.0.0

package/dist/providers/native/supervisor.d.ts

@@ -0,0 +1,503 @@
+/**
+ * NativeEngineSupervisor — lifecycle for a native engine child process
+ * (e.g. `sd-server`, `llama-server`).
+ *
+ * Responsibilities:
+ *   - **Lazy start** the engine on the first `ensureRunning()` call, not
+ *     at service boot. No VRAM is held until something actually asks
+ *     for work.
+ *   - **Idle timeout**: after `idleTimeoutMs` with no activity the
+ *     engine is SIGTERM'd to free GPU/VRAM. Fresh activity lazy-starts
+ *     again.
+ *   - **Health watch**: every `healthIntervalMs` probe `/health`. After
+ *     three consecutive failures the process is killed and a restart
+ *     is attempted. Restart budget is 3 attempts in 60s, matching the
+ *     gezeld supervisor's policy.
+ *
+ * Intentionally engine-agnostic — callers (`StableDiffusionCppProvider`,
+ * `LlamaCppProvider`, future STT/TTS providers) ask `ensureRunning()`
+ * before each request, mark activity via `markUsed()`, and otherwise
+ * know nothing about subprocess plumbing. Tests inject a `spawn` hook
+ * so we can drive the lifecycle without a real binary.
+ */
+import type { NativeEngineExitSnapshot, NativeEnginePanicKind } from '@bendyline/gezel/local-loop';
+export type { NativeEngineExitSnapshot, NativeEnginePanicKind } from '@bendyline/gezel/local-loop';
+import { spawn as nodeSpawn } from 'node:child_process';
+import { type NativeCapacityOptions } from './device-capacity.js';
+export interface NativeEngineLaunch {
+    /** Args passed to the binary. The binary path is opaque to this class. */
+    command: string;
+    args: string[];
+    /** Extra env injected into the child; PATH/HOME/etc. flow through by default. */
+    env?: Record<string, string>;
+    /**
+     * Working directory for the child. Defaults to the parent's cwd. Set when
+     * the binary resolves runtime assets relative to cwd — e.g. ds4-server
+     * loads its Metal shader sources from `./metal/*.metal`, so its launch
+     * points cwd at the bundle dir that holds them.
+     */
+    cwd?: string;
+    /** Base URL the provider will use to talk to the engine. */
+    baseUrl: string;
+    /**
+     * Safe, request-independent launch facts copied into structured lifecycle
+     * logs and crash snapshots. Never place prompts, tool arguments, secrets,
+     * or other user content here.
+     */
+    diagnostics?: Record<string, string | number | boolean>;
+}
+/**
+ * Convert Electron's virtual `app.asar` paths to the real files staged by
+ * electron-builder under `app.asar.unpacked`. Electron patches `fs`, so an
+ * existence check in the main process can succeed through the virtual path,
+ * but the OS receives native spawn paths unchanged and rejects them with
+ * ENOTDIR. Normalize again at the service's final launch boundary so embedded
+ * mode is safe even when a pre-set env var bypassed the Electron resolver.
+ */
+export declare function normalizeNativeEngineLaunch(launch: NativeEngineLaunch): NativeEngineLaunch;
+export interface NativeProcessSnapshot {
+    pid: number;
+    /** Parent pid from ps. Optional only for injected legacy test seams. */
+    ppid?: number;
+    command: string;
+}
+export interface NativeEngineLifecycleSnapshot {
+    running: boolean;
+    active: boolean;
+    pid?: number;
+    lastUsedAt: number | null;
+    unloadAt: number | null;
+    idleTimeoutMs: number;
+    releaseReason: 'idle' | 'memory-pressure' | null;
+    /**
+     * When the last start attempt failed, and why. Cleared by the next start
+     * that reaches ready. An engine that never started still holds its pool
+     * reservation, and no idle timer is armed for it (the timer is only ever
+     * reset by a RUNNING engine), so nothing would ever give that memory back
+     * — see `ProviderPool.sweepFailedStarts`.
+     */
+    startFailedAt?: number;
+    startFailure?: string;
+}
+export interface NativeEngineSupervisorOptions {
+    capacity?: NativeCapacityOptions;
+    /**
+     * Resolves to the launch spec on demand. Called each time the
+     * supervisor needs to start the engine, so the caller can observe
+     * config changes (active model, GPU backend, etc.) without recreating
+     * the supervisor.
+     */
+    resolveLaunch: () => Promise<NativeEngineLaunch>;
+    idleTimeoutMs?: number;
+    /**
+     * Stage-1 idle timeout in ms. When set and < idleTimeoutMs, the
+     * supervisor fires `onFreeze` after this much inactivity but does
+     * NOT stop the child — the model stays resident, ready to serve
+     * the next request without a cold reload. The `onFreeze` callback
+     * is the provider's chance to persist transient state (prompt
+     * caches, slot files) to disk so a SIGKILL between Stage 1 and
+     * Stage 2 doesn't lose anything. If activity resumes before
+     * Stage 2, both timers reset cleanly. Unset = single-stage idle
+     * (the legacy behavior).
+     */
+    freezeTimeoutMs?: number;
+    /**
+     * Called when the Stage-1 freeze timer fires. Best-effort; errors
+     * are caught + logged so a misbehaving callback can't take down
+     * the supervisor. Returns a promise the supervisor awaits before
+     * resetting state — the freeze is intentionally a one-shot per
+     * idle window, so callbacks do their work in one pass and the next
+     * `markUsed` re-arms.
+     */
+    onFreeze?: () => void | Promise<void>;
+    /**
+     * Called right before a deliberate stop SIGTERMs a child that reached
+     * ready, with the engine still serving. The engine's own shutdown hook
+     * gets only the SIGTERM-to-SIGKILL grace (three seconds) to persist
+     * state — not enough to write a long session's KV cache, which is what
+     * made a restart re-prefill 55k tokens. Bounded by
+     * `beforeStopTimeoutMs`; a failure or timeout never blocks the stop.
+     * Skipped for health restarts: an engine that failed its health probe
+     * cannot be trusted to answer.
+     */
+    beforeStop?: (ctx: {
+        reason: string;
+        baseUrl: string;
+    }) => void | Promise<void>;
+    /** Bound on {@link beforeStop}. Default 30s. */
+    beforeStopTimeoutMs?: number;
+    /**
+     * Busy predicate consulted at the Stage-2 idle deadline: when it returns
+     * true, the supervisor does NOT stop the engine — it reschedules the idle
+     * timer for another full window. Prevents idle-unloading an engine that a
+     * turn still needs (the "engine idle-stopped mid-turn → next request fails
+     * with `unreachable / fetch failed`" bug). The idle timer resets on every
+     * request via `markUsed`. This must describe the supervised engine's
+     * physical request gate, not a whole logical chat turn: tool calls and
+     * questions parked on a person do not touch the GPU and should not pin it.
+     */
+    isBusy?: () => boolean;
+    /**
+     * Optional low-free-accelerator signal. A pressured engine that has been
+     * physically idle for `pressureIdleTimeoutMs` flushes its cache and stops,
+     * even when the ordinary warm-retention window is longer.
+     */
+    memoryPressure?: () => Promise<{
+        pressured: boolean;
+        detail?: string;
+    }> | {
+        pressured: boolean;
+        detail?: string;
+    };
+    pressureIdleTimeoutMs?: number;
+    healthIntervalMs?: number;
+    /** Poll budget for the post-spawn readiness probe. Defaults to 60s. */
+    startupTimeoutMs?: number;
+    spawn?: typeof nodeSpawn;
+    fetchImpl?: typeof fetch;
+    onLog?: (line: string) => void;
+    /** Called for both expected and unexpected exits after a snapshot is sealed. */
+    onExit?: (snapshot: NativeEngineExitSnapshot) => void | Promise<void>;
+    /**
+     * Consulted after a start attempt fails (child died or never became
+     * ready — not spawn errors, and not crashes after readiness). Return
+     * true to re-resolve the launch spec and try again in the same
+     * `ensureRunning` call; the owner degrades its plan first (e.g. the
+     * MoE offload ladder after a GPU OOM). Capped at
+     * {@link MAX_STARTUP_RECOVERIES} retries per start, on top of the
+     * one-shot singleton-conflict retry.
+     */
+    recoverStartup?: (info: {
+        panicKind?: NativeEnginePanicKind | undefined;
+        attempt: number;
+    }) => boolean | Promise<boolean>;
+    /**
+     * Called on every raw stdout/stderr line BEFORE `onLog`. Optional —
+     * when set, callers can parse the engine's log output and emit
+     * structured signals (e.g. llama-server's startup-phase classifier).
+     * Return value is ignored; errors are caught + logged so a bad
+     * classifier can't take down the supervisor. Runs synchronously on
+     * every line, so keep the work bounded.
+     */
+    onRawLine?: (line: string) => void;
+    /**
+     * Prefix prepended to stdout/stderr log lines and lifecycle events
+     * (idle-stop, restart, exit). Identifies the engine in shared log
+     * streams. Defaults to `[native]`.
+     */
+    logPrefix?: string;
+    /**
+     * Path the readiness probe hits, relative to the launch baseUrl.
+     * Defaults to `/health` — llama-server convention. Engines without a
+     * dedicated health endpoint (sd-server treats the root as either a
+     * served HTML file or a 404) override to `/` and pair with
+     * {@link readyOnAnyResponse}.
+     */
+    readinessPath?: string;
+    /**
+     * When true, ANY successful HTTP response counts as ready — even a
+     * 404. Useful for engines that don't implement a health endpoint:
+     * the act of returning HTTP status bytes proves the port is open
+     * and the request loop is alive. Defaults to `false` (2xx only),
+     * preserving the strict semantics for engines like llama-server
+     * whose `/health` returns 503 during model load and 200 once ready.
+     */
+    readyOnAnyResponse?: boolean;
+    /**
+     * Test seam: produce the `(pid, ppid, command)` snapshot used by the
+     * orphan-reaper. Defaults to running `ps -axo pid=,ppid=,command=` and
+     * parsing the output. `ppid` is optional only for backwards-compatible
+     * test seams; the real runner always supplies it. Tests inject a
+     * deterministic list so we can assert reap behavior without spawning
+     * real processes.
+     */
+    psRunner?: () => Promise<NativeProcessSnapshot[]>;
+    /** Platform seam for Windows owner-liveness semantics in orphan tests. */
+    platform?: NodeJS.Platform;
+    /**
+     * Test seam paired with `psRunner` — invoked instead of
+     * `process.kill(pid, 'SIGKILL')`. Lets unit tests assert which pids
+     * were targeted without actually delivering signals. Defaults to
+     * `process.kill`.
+     */
+    killProcess?: (pid: number, signal: NodeJS.Signals) => void;
+}
+/** Test-only: clear the live-engine registry between cases so a pid
+ *  registered by one test can't suppress an orphan reap in the next. */
+export declare function __resetLiveEnginePidsForTest(): void;
+export declare class NativeEngineSupervisor {
+    private readonly capacity?;
+    private capacityLease;
+    private capacityAbort;
+    private startFlight;
+    private activeRequests;
+    private stopFlight;
+    private yieldFlight;
+    get coordinatesCapacity(): boolean;
+    withRequest<T>(run: () => Promise<T>, signal?: AbortSignal): Promise<T>;
+    /** Stop accepting fresh GPU work while an older memory claim needs this model to drain. */
+    yieldForWaitingCapacity(signal?: AbortSignal): Promise<void>;
+    private state;
+    private readonly resolveLaunch;
+    private readonly idleTimeoutMs;
+    private readonly freezeTimeoutMs;
+    private readonly onFreeze?;
+    private readonly beforeStop?;
+    private readonly beforeStopTimeoutMs;
+    private readonly isBusy?;
+    private readonly memoryPressure?;
+    private readonly pressureIdleTimeoutMs;
+    private pressureCheckInFlight;
+    private pressureDeadlineAt?;
+    /** Awake ms at which the current unbroken stretch of pressure began. */
+    private pressureHeldSince?;
+    /** Awake ms of the last "still pressured, cannot act" log line. */
+    private pressureHeldLoggedAt?;
+    private freezeTimer?;
+    /** True once the current idle window has fired its Stage-1 freeze.
+     *  Reset on next `markUsed` so subsequent idle windows re-fire. */
+    private freezeFired;
+    private readonly healthIntervalMs;
+    private readonly startupTimeoutMs;
+    private readonly spawn;
+    private readonly fetchImpl;
+    private readonly onLog;
+    private readonly onExit?;
+    private readonly recoverStartup?;
+    private readonly onRawLine?;
+    private readonly logPrefix;
+    private readonly readinessPath;
+    private readonly readyOnAnyResponse;
+    private readonly psRunner;
+    private readonly killProcess;
+    private readonly platform;
+    private readonly logListeners;
+    private recentStarts;
+    private lastUsedAt;
+    /**
+     * The same mark on the awake clock. Every idle/pressure DECISION reads this
+     * one; {@link lastUsedAt} stays wall-clock because it is rendered to the
+     * user. A host suspension advances wall clock without giving the engine any
+     * time, and charging that against idleness unloaded resident models on every
+     * dark-wake cycle — the next turn then paid a full cold load.
+     */
+    private lastUsedAwakeAt;
+    /** PID of the engine child this supervisor currently owns, registered
+     *  in {@link liveEnginePids} so a sibling supervisor's orphan reaper
+     *  leaves it alone. Set on spawn, cleared on exit. */
+    private ownedPid?;
+    private didOrphanSweep;
+    private startupAbort;
+    private idleTimer?;
+    private lastStartFailureAt?;
+    private lastStartFailure?;
+    private healthTimer?;
+    private currentStartedAt;
+    private currentPid?;
+    /** Sticky for the current child; remains true while an expected stop moves
+     * state to `stopped` before the OS exit event arrives. */
+    private currentReachedReady;
+    private currentDiagnostics?;
+    private currentOutput;
+    private currentPanic?;
+    private expectedExitReason?;
+    private lastExit?;
+    private readonly exitListeners;
+    constructor(opts: NativeEngineSupervisorOptions);
+    /** URL the provider should hit. Only meaningful while running. */
+    currentBaseUrl(): string | undefined;
+    /**
+     * PID of the supervised child when running. Used by providers that
+     * want to poll the child's resource usage (e.g. resident-set size
+     * for an "engine memory" telemetry event) — the supervisor owns
+     * the lifetime of the pid, so consumers don't have to track
+     * restarts themselves.
+     */
+    currentChildPid(): number | undefined;
+    /** Stable status shape for the engine pool and once-a-second memory UI. */
+    lifecycleSnapshot(): NativeEngineLifecycleSnapshot;
+    /** Most recently sealed child exit, including expected idle/shutdown exits. */
+    lastExitSnapshot(): NativeEngineExitSnapshot | undefined;
+    /**
+     * Launch provenance of the LIVE child: pid, start time, and the
+     * request-independent launch facts the spawner attached (granted context
+     * window, slots, KV dtype, backend — the payload behind the
+     * `launch {...}` log line). Undefined when no child is up, so callers
+     * (Settings → About, `/api/system/diagnostics`) never show a stale grant
+     * for an engine that already exited.
+     */
+    launchSnapshot(): {
+        pid?: number;
+        startedAt: number;
+        diagnostics?: Record<string, string | number | boolean>;
+    } | undefined;
+    /**
+     * Wait briefly for an unexpected exit attributable to a request that began
+     * at `sinceMs`. Fetch/stream teardown can race Node's child `exit` event by
+     * a few milliseconds; this bounded wait prevents CUDA SIGABRTs from being
+     * flattened into a generic undici "terminated" error.
+     */
+    waitForUnexpectedExitSince(sinceMs: number, timeoutMs?: number): Promise<NativeEngineExitSnapshot | undefined>;
+    /**
+     * Ensure the engine is running and reachable. If it's stopped, spawn
+     * it; if it's starting, await the in-flight readiness probe; if it's
+     * already running, no-op. Rejects if the restart budget is exhausted.
+     */
+    ensureRunning(): Promise<NativeEngineLaunch>;
+    /**
+     * Attach a transient log listener that receives every prefixed
+     * stdout/stderr line (`[sd-server] …`) for the duration of the
+     * returned unsubscribe call. Used by the owning provider to parse
+     * engine progress for a single in-flight request — e.g. sd-server's
+     * `|==> | 3/20 - 18.20s/it` sampling lines — and translate them
+     * into structured progress events for the UI.
+     *
+     * Listeners are independent of `onLog` / `onRawLine` (which are
+     * construction-time hooks for log routing + classification). Adding
+     * one does not suppress either.
+     */
+    subscribeLogLines(listener: (line: string) => void): () => void;
+    /** Update the last-used clock without triggering a start. */
+    markUsed(): void;
+    stop(reason?: 'stop' | 'idle' | 'memory-pressure' | 'health-restart'): Promise<void>;
+    private stopChild;
+    private runBeforeStop;
+    private startFresh;
+    private startFreshAttempt;
+    /**
+     * Abort an in-flight `startFresh` with a concrete error. Called by
+     * the owning provider when the child's stdout reveals a fatal
+     * condition (model-arch mismatch, OOM, missing dependency) that
+     * won't resolve by waiting longer — without this, the `waitForReady`
+     * poll would keep hitting `/health` for the full startup-timeout
+     * window even though the worker thread has already died.
+     */
+    abortStartup(err: Error): void;
+    /**
+     * Find and SIGKILL any owner-less engine process from a prior gezel
+     * launch that matches this engine. Best-effort: any error during the
+     * sweep is logged and swallowed so the spawn-on-startup path never
+     * aborts because of cleanup.
+     *
+     * Crucially, a process whose parent is still alive is NOT an orphan and
+     * is skipped. The process-local {@link liveEnginePids} registry protects
+     * sibling supervisors inside this daemon; the PPID check protects engines
+     * owned by a different live gezeld process/home. This matters for two
+     * concurrent isolated eval daemons: each uses the same engine binary, so a
+     * command-path-only sweep would otherwise SIGKILL the other's in-flight
+     * child and surface a bare `terminated` fetch error.
+     *
+     * Match strategy — anchor on every install-specific absolute path we
+     * can find in the launch spec, then check whether each `ps` row
+     * contains any of them:
+     *
+     *   1. `launch.command` — works directly when the binary is
+     *      install-specific (llama-server, sd-server: a path under
+     *      `~/.gezel/native/bin/`).
+     *   2. Any absolute-path argument that looks like a script or
+     *      executable (`*.py`, `*.js`, etc.). For Python engines this is
+     *      the case that *actually* fires: `launch.command` is the venv's
+     *      `bin/python`, but that's typically a symlink chain to the
+     *      system framework Python (e.g. `/Library/Frameworks/Python.../
+     *      MacOS/Python`) — and macOS `ps` reports the resolved path, so
+     *      the binary-path match never hits. The script path
+     *      (`gezel_mlx_server.py` under the dist folder) appears verbatim
+     *      in the ps command line and is unique to this install.
+     *
+     * `didOrphanSweep` gates this to one execution per supervisor instance
+     * (so we don't re-sweep on our own restarts). A real orphan is reparented
+     * to launchd/systemd (PPID 1) after its daemon exits. Any matching child
+     * with PPID > 1 still has a live owner and is conservatively left alone.
+     * Injected legacy `psRunner` rows may omit PPID and retain the historical
+     * match behavior; the production runner always includes it.
+     *
+     * macOS + Linux use `ps -axo pid=,ppid=,command=`. Windows uses the
+     * Win32_Process CIM snapshot; because Windows retains a dead creator's pid
+     * instead of reparenting to pid 1, absence of that ppid proves orphanhood.
+     */
+    private reapOrphansMatching;
+    /**
+     * Poll the process table until none of `pids` remain, or `timeoutMs`
+     * elapses. Best-effort: a still-present pid after the timeout is logged (the
+     * subsequent spawn may then fail a singleton check, surfaced normally) rather
+     * than thrown, so cleanup never wedges startup.
+     */
+    private waitForPidsGone;
+    /**
+     * Pull every install-specific absolute path out of the launch spec.
+     * Used by the orphan reaper to match against `ps` output. The
+     * filtering is deliberately conservative: only paths that look like a
+     * concrete file (have an extension) qualify, so flag values like
+     * `--port 12345` or directory paths passed via `--model` don't get
+     * promoted to anchors. Absolute model dirs would still be
+     * install-specific in practice, but matching on a directory path
+     * could collide with any tool that happens to have it on its command
+     * line (e.g. a `du -sh` invocation), so we leave them out.
+     */
+    private buildOrphanAnchors;
+    private waitForReady;
+    private appendRecentOutput;
+    private handleExit;
+    private startHealthWatch;
+    /**
+     * Accelerator pressure this engine is currently sitting inside, or null when
+     * there is none. Read by callers that want to explain a symptom the engine
+     * itself cannot see — a turn whose throughput collapsed has no way to know
+     * the card is oversubscribed, and "the engine is slow" is the wrong story.
+     */
+    memoryPressureHeld(): {
+        sinceAwakeMs: number;
+        heldMs: number;
+    } | null;
+    private engineBusy;
+    private pressureProbe;
+    private releaseCapacity;
+    private healthProbe;
+    private resetIdleTimer;
+    /**
+     * Arm the VRAM-release timer to fire `delayMs` from now, then re-check
+     * against the AWAKE clock when it does.
+     *
+     * The re-check is the load-bearing part. A host suspension fires every
+     * overdue timer the instant the machine resumes, so a nap longer than the
+     * idle timeout used to read as "this engine has been idle that long" and
+     * unload it — observed unloading a resident 27B model on each ~16 min
+     * dark-wake cycle, so the next turn paid a full cold load for idleness
+     * nobody could have avoided. Re-arming for the shortfall matters just as
+     * much: an early return with no re-arm would leave the engine resident for
+     * good, trading a spurious unload for a permanent leak.
+     */
+    private armIdleTimer;
+    /**
+     * Project an awake-clock instant onto the wall clock for display. The UI
+     * renders a countdown, and a deadline held in awake time would otherwise
+     * show as an already-past timestamp after any suspension.
+     */
+    private wallClockFor;
+    /**
+     * Run the user-supplied onFreeze hook with proper guarding. Marks
+     * `freezeFired` so subsequent ticks within this idle window don't
+     * re-fire (idempotency for callers, e.g. a re-entrant flush would
+     * double-write the same content).
+     */
+    private runFreeze;
+    private clearIdleTimer;
+    private clearFreezeTimer;
+    private clearHealthTimer;
+}
+/** High-precision fatal signatures; attribution happens only if the child exits. */
+export declare function classifyNativeEnginePanic(line: string): {
+    kind: NativeEnginePanicKind;
+    line: string;
+} | undefined;
+/** Parse the portable `ps -axo pid=,ppid=,command=` shape. */
+export declare function parseNativeProcessSnapshot(stdout: string): NativeProcessSnapshot[];
+/** `code` stamped on every engine-launch failure. See {@link isEngineLaunchError}. */
+export declare const ENGINE_LAUNCH_ERROR_CODE = "engine-launch";
+/**
+ * True when the engine binary itself never started, however deep the error
+ * was rewrapped. Matches the code first, then falls back to the stable
+ * message so a rewrapped error still classifies right (same belt-and-braces
+ * shape as the pool's `engine-busy` matcher).
+ */
+export declare function isEngineLaunchError(err: unknown): boolean;