mlx-bun 0.4.0 → 1.0.0

package/src/jobs/sse.ts

@@ -2,9 +2,9 @@
 // (offset-seek read, partial-line tolerant, polls while the row is
 // non-terminal — port of optiq lab jobs.py `tail`). `streamJobResponse`
 // wraps it in a Bun `Response` whose body is an EventSource-compatible SSE
-// stream, matching the writer style in src/server.ts.
+// stream, matching the writer style in apps/mlx-bun/src/server/http.ts.
 
-import type { JobEvent, JobStatus } from "./types";
+import type { JobEvent, JobStatus } from "./protocol";
 import type { JobStore } from "./db";
 
 const TERMINAL: ReadonlySet<JobStatus> = new Set<JobStatus>(["done", "failed", "zombie"]);
@@ -13,6 +13,7 @@ export interface TailOpts {
   /** Keep polling while the job is non-terminal (SSE use case). When false,
    *  yield what's currently logged and stop. */
   follow?: boolean;
+  signal?: AbortSignal;
   /** Poll interval in ms while waiting for new lines. */
   pollMs?: number;
 }
@@ -35,8 +36,18 @@ export async function* tailJob(
 
   let offset = 0;
   let buf = "";
+  let terminalSeen = false;
+  const eventFromLine = (line: string): JobEvent | undefined => {
+    const event = JSON.parse(line) as JobEvent;
+    if (event.type === "done" || event.type === "failed") {
+      if (terminalSeen) return undefined;
+      terminalSeen = true;
+    }
+    return event;
+  };
 
   for (;;) {
+    if (opts.signal?.aborted) return;
     const file = Bun.file(logPath);
     const size = await file.size;
     if (size > offset) {
@@ -50,7 +61,8 @@ export async function* tailJob(
         buf = buf.slice(nl + 1);
         if (!line) continue;
         try {
-          yield JSON.parse(line) as JobEvent;
+          const event = eventFromLine(line);
+          if (event) yield event;
         } catch {
           // partial / corrupt line: re-buffer it and wait for more bytes
           buf = line + "\n" + buf;
@@ -59,10 +71,10 @@ export async function* tailJob(
       }
     }
 
-    if (!follow) return;
+    if (!follow || opts.signal?.aborted) return;
 
-    const status = store.get(jobId)?.status;
-    if (status && TERMINAL.has(status)) {
+    const finalRow = store.get(jobId);
+    if (finalRow && TERMINAL.has(finalRow.status)) {
       // Drain any bytes written between the last read and the terminal flip.
       const f = Bun.file(logPath);
       const sz = await f.size;
@@ -73,42 +85,66 @@ export async function* tailJob(
       for (const line of buf.split("\n")) {
         const s = line.trim();
         if (!s) continue;
-        try { yield JSON.parse(s) as JobEvent; } catch { /* drop trailing partial */ }
+        try {
+          const event = eventFromLine(s);
+          if (event) yield event;
+        } catch { /* drop trailing partial */ }
+      }
+      if (opts.signal?.aborted) return;
+      // A crash/admission failure may only update SQLite. A child can also
+      // publish terminal status just before appending its terminal log event.
+      // Clients need a terminal event in either case, before the SSE end marker.
+      if (!terminalSeen) {
+        const endedAt = finalRow.ended_at ? Date.parse(finalRow.ended_at.replace(" ", "T") + "Z") : NaN;
+        const ts = Number.isFinite(endedAt) ? endedAt : Date.now();
+        yield finalRow.status === "done"
+          ? { type: "done", ts, output_dir: finalRow.output_path ?? undefined }
+          : { type: "failed", ts, error: finalRow.error ??
+              (finalRow.status === "zombie" ? "job interrupted by a previous process exit" : "job failed") };
       }
       return;
     }
 
-    await Bun.sleep(pollMs);
+    await new Promise<void>(resolve => {
+      const done = () => { clearTimeout(timer); opts.signal?.removeEventListener("abort", done); resolve(); };
+      const timer = setTimeout(done, pollMs);
+      opts.signal?.addEventListener("abort", done, { once: true });
+      if (opts.signal?.aborted) done();
+    });
   }
 }
 
 /** Build an SSE `Response` streaming a job's events. Writes `retry: 1500`
  *  first (so EventSource fires `onopen`), then one `data:` frame per event,
  *  and finally an `event: end` marker before closing on terminal. */
-export function streamJobResponse(store: JobStore, jobId: string): Response {
+export function streamJobResponse(store: JobStore, jobId: string, signal?: AbortSignal): Response {
+  const cancellation = new AbortController();
+  const abort = () => cancellation.abort();
+  signal?.addEventListener("abort", abort, { once: true });
+  if (signal?.aborted) abort();
+  let cancelled = false;
   const stream = new ReadableStream<Uint8Array>({
     async start(controller) {
       const enc = new TextEncoder();
+      const send = (text: string) => { if (!cancellation.signal.aborted) controller.enqueue(enc.encode(text)); };
       try {
-        controller.enqueue(enc.encode("retry: 1500\n\n"));
-        for await (const event of tailJob(store, jobId, { follow: true })) {
-          controller.enqueue(enc.encode(`data: ${JSON.stringify(event)}\n\n`));
+        send("retry: 1500\n\n");
+        for await (const event of tailJob(store, jobId, { signal: cancellation.signal })) {
+          send(`data: ${JSON.stringify(event)}\n\n`);
         }
-        controller.enqueue(enc.encode("event: end\ndata: {}\n\n"));
+        send("event: end\ndata: {}\n\n");
       } catch (e) {
         const error = e instanceof Error ? e.message : String(e);
-        controller.enqueue(enc.encode(`data: ${JSON.stringify({ type: "failed", error, ts: Date.now() })}\n\n`));
-        controller.enqueue(enc.encode("event: end\ndata: {}\n\n"));
+        send(`data: ${JSON.stringify({ type: "failed", error, ts: Date.now() })}\n\n`);
+        send("event: end\ndata: {}\n\n");
       } finally {
-        controller.close();
+        signal?.removeEventListener("abort", abort);
+        if (!cancelled) controller.close();
       }
     },
+    cancel() { cancelled = true; abort(); },
   });
-  return new Response(stream, {
-    headers: {
-      "content-type": "text/event-stream",
-      "cache-control": "no-cache",
-      connection: "keep-alive",
-    },
-  });
+  return new Response(stream, { headers: {
+    "content-type": "text/event-stream", "cache-control": "no-cache", connection: "keep-alive",
+  } });
 }

package/src/jobs/worker-memory.ts

@@ -0,0 +1,46 @@
+// What an isolation worker reports of the MLX memory its process holds (the parent has no native
+// module, so the worker measures): on its `/health` (`memory`), and as a `worker.memory` line in its
+// `/admin/events` stream at connect, on an interval when it changed, and after each finished request.
+// The parent's residency (cli/worker-unit.ts) counts a worker's active plus cache bytes once one arrives.
+
+/** MLX memory of one worker process, in bytes. */
+export interface WorkerMemory {
+  /** Memory held by live arrays: the weights, the KV the model carries and the working set of a step. */
+  readonly activeBytes: number;
+  /** Buffers the allocator keeps for reuse; the process still holds them. */
+  readonly cacheBytes: number;
+  /** The high-water mark of active memory since the process started. */
+  readonly peakBytes: number;
+  /** What the device recommends a process wire at most (`max_recommended_working_set_size`); 0 when unknown. */
+  readonly workingSetBytes: number;
+}
+
+/** The stream line's `type`: the parent consumes it and never republishes it as an app event. */
+export const WORKER_MEMORY_EVENT = "worker.memory";
+
+const bytes = (value: unknown): number | undefined => typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined;
+
+function build(active: unknown, cache: unknown, peak: unknown, workingSet: unknown): WorkerMemory | undefined {
+  const activeBytes = bytes(active), cacheBytes = bytes(cache), peakBytes = bytes(peak);
+  if (activeBytes === undefined || cacheBytes === undefined || peakBytes === undefined) return undefined;
+  return { activeBytes, cacheBytes, peakBytes, workingSetBytes: bytes(workingSet) ?? 0 };
+}
+
+/** The stream line for one reading. */
+export const memoryLine = (memory: WorkerMemory, at = Date.now()): string => JSON.stringify({ type: WORKER_MEMORY_EVENT, at, ...memory }) + "\n";
+/** A stream line's reading; undefined when it is not one. */
+export function parseMemoryLine(value: unknown): WorkerMemory | undefined {
+  const line = value as Record<string, unknown> | null;
+  return line && typeof line === "object" && line.type === WORKER_MEMORY_EVENT ? build(line.activeBytes, line.cacheBytes, line.peakBytes, line.workingSetBytes) : undefined;
+}
+
+/** `/health`'s `memory` member (snake case, like the rest of the report). */
+export const memoryHealth = (memory: WorkerMemory) => ({ active_bytes: memory.activeBytes, cache_bytes: memory.cacheBytes,
+  peak_bytes: memory.peakBytes, working_set_bytes: memory.workingSetBytes });
+export function parseMemoryHealth(value: unknown): WorkerMemory | undefined {
+  const body = (value as { memory?: Record<string, unknown> } | null)?.memory;
+  return body && typeof body === "object" ? build(body.active_bytes, body.cache_bytes, body.peak_bytes, body.working_set_bytes) : undefined;
+}
+
+/** What the residency manager counts a worker as holding: everything its process keeps. */
+export const heldBytes = (memory: WorkerMemory): number => memory.activeBytes + memory.cacheBytes;

package/src/jobs/worker-process.ts

@@ -0,0 +1,163 @@
+// Parent-side ownership of one isolation worker process: spawn it through the
+// executable captured at startup (or a supplied mlx-bun command), hand it its
+// launch record on stdin, wait for its ready line, forward its logs, and stop
+// it (SIGTERM, then SIGKILL after a grace). HTTP never passes through here;
+// the parent talks to the worker's socket directly (I3). There is no restart
+// policy yet: the budget field is carried so the parent's spec is stable when
+// I3 applies it.
+import { rmSync } from "node:fs";
+import pkg from "../../package.json" with { type: "json" };
+import { executablePath } from "./executable";
+import { pumpLines } from "./runner";
+
+/** A stdout line the worker reserves for the parent; every other line is log output. */
+export const WORKER_MESSAGE_PREFIX = "<mlx-bun-worker>";
+
+/** The app's package version (the one `--version` prints) is the worker
+ * protocol's: the launch record and the ready line both carry it, and a worker
+ * refuses a record with another package version ("worker protocol version
+ * mismatch"). Two builds with the same package version are not told apart. */
+export const WORKER_PROTOCOL_VERSION: string = pkg.version;
+
+export interface WorkerReady { type: "ready"; socketPath: string; modelId: string; pid: number; version: string }
+export type WorkerMessage = WorkerReady;
+
+export function formatWorkerMessage(message: WorkerMessage): string {
+  return WORKER_MESSAGE_PREFIX + JSON.stringify(message);
+}
+
+/** The launch record is JSON with non-finite numbers preserved: resolved serve
+ * options carry `Infinity` (an unbounded SSD cache), which JSON would null. */
+export function encodeLaunch(value: unknown): string {
+  return JSON.stringify(value, (_key, item: unknown) =>
+    typeof item === "number" && !Number.isFinite(item) ? { $number: String(item) } : item);
+}
+export function decodeLaunch(text: string): unknown {
+  return JSON.parse(text, (_key, item: unknown) =>
+    item !== null && typeof item === "object" && "$number" in item && Object.keys(item).length === 1
+      ? Number((item as { $number: string }).$number) : item);
+}
+
+export interface WorkerProcessOptions {
+  /** The worker entry for source runs; a `$bunfs` path selects the compiled
+   * binary's `__worker` dispatch. Required unless `command` is supplied. */
+  entry?: string;
+  /** Internal: the program and leading arguments of an mlx-bun CLI, spawned
+   * as `[...command, "__worker"]` instead of the captured executable and
+   * `entry`. Its package version must match: see `WORKER_PROTOCOL_VERSION`. */
+  command?: readonly string[];
+  /** The Unix socket path the worker must bind. The parent owns the file's
+   * directory; a stale file is removed before the spawn and after the exit. */
+  socketPath: string;
+  /** The launch record the worker reads from stdin (`cli/worker-entry.ts` defines it). */
+  launch: unknown;
+  /** Defaults to the executable captured at startup (Bun for source execution). */
+  bin?: string;
+  /** Override process creation for deterministic tests. */
+  spawn?: typeof Bun.spawn;
+  env?: Record<string, string | undefined>;
+  /** Worker stdout lines that are not protocol messages; default `console.log`. */
+  log?(line: string): void;
+  /** Worker stderr lines; default `console.error`. */
+  error?(line: string): void;
+  /** Give up on a worker that has not reported ready by then (weights load
+   * behind it; default 15 min, as main). The worker is stopped. */
+  readyTimeoutMs?: number;
+  /** SIGTERM first; SIGKILL when the worker is still alive after this (default 3 s). */
+  graceMs?: number;
+  /** Reserved for I3's automatic respawn; carried, not applied. */
+  restarts?: { max: number; windowMs: number; delayMs?: number };
+}
+
+export interface WorkerExit { code: number | null; signal: string | null }
+
+export class WorkerExitedError extends Error {
+  /** `reason`: why a worker refused its launch record (exit 2), from its last stderr line. */
+  constructor(readonly exit: WorkerExit, stage: string, reason?: string) {
+    super((exit.signal ? `worker was killed by ${exit.signal} ${stage}` : `worker exited with code ${exit.code} ${stage}`) + (reason ? `: ${reason}` : ""));
+    this.name = "WorkerExitedError";
+  }
+}
+
+export interface WorkerProcess {
+  readonly pid: number;
+  readonly socketPath: string;
+  /** From the ready line; undefined until then. */
+  readonly modelId: string | undefined;
+  /** Resolves once the worker has bound its socket; rejects with the exit when
+   * the worker dies first (exit 2, a refused launch record, carries the
+   * worker's reason), when it echoes another socket or protocol version, or on timeout. */
+  readonly ready: Promise<{ socketPath: string; modelId: string }>;
+  /** The worker's exit, however it happened, after its logs have drained. */
+  readonly exited: Promise<WorkerExit>;
+  /** Stop the worker and join it. Idempotent. */
+  close(): Promise<WorkerExit>;
+}
+
+export function spawnWorker(options: WorkerProcessOptions): WorkerProcess {
+  const bin = options.bin ?? executablePath;
+  if (!options.command && !options.entry) throw new Error("spawnWorker needs an entry or a command");
+  const command = options.command ? [...options.command, "__worker"] : options.entry!.includes("$bunfs") ? [bin, "__worker"] : [bin, options.entry!];
+  const version = (options.launch as { version?: unknown } | null)?.version;
+  const log = options.log ?? (line => console.log(`[worker] ${line}`));
+  const error = options.error ?? (line => console.error(`[worker] ${line}`));
+  rmSync(options.socketPath, { force: true });
+  const proc = (options.spawn ?? Bun.spawn)(command, {
+    stdin: "pipe", stdout: "pipe", stderr: "pipe",
+    env: { ...process.env, ...options.env },
+  });
+  // The pipe stays open after the launch record: its end tells the worker the
+  // parent is gone, so a crashed parent leaves no orphan holding the GPU.
+  proc.stdin.write(encodeLaunch(options.launch) + "\n");
+  void proc.stdin.flush();
+  const ready = Promise.withResolvers<{ socketPath: string; modelId: string }>();
+  let settled = false, modelId: string | undefined;
+  const settle = (outcome: { value: { socketPath: string; modelId: string } } | { error: Error }) => {
+    if (settled) return;
+    settled = true;
+    clearTimeout(timer);
+    if ("value" in outcome) ready.resolve(outcome.value); else ready.reject(outcome.error);
+  };
+  const stdout = pumpLines(proc.stdout, line => {
+    if (!line.startsWith(WORKER_MESSAGE_PREFIX)) return log(line);
+    let message: WorkerMessage;
+    try { message = JSON.parse(line.slice(WORKER_MESSAGE_PREFIX.length)) as WorkerMessage; }
+    catch { return settle({ error: new Error(`worker sent an unreadable message: ${line}`) }); }
+    if (message.type !== "ready") return settle({ error: new Error(`worker sent an unknown message: ${line}`) });
+    if (message.socketPath !== options.socketPath)
+      return settle({ error: new Error(`worker bound ${message.socketPath}, not ${options.socketPath}`) });
+    // A versioned launch needs the same package version on the other side: an older worker ignores the field and never echoes it.
+    if (typeof version === "string" && message.version !== version)
+      return settle({ error: new Error(`worker protocol version mismatch: the worker reported ${typeof message.version === "string" ? message.version : "no version"}, the launch record is ${version}`) });
+    modelId = message.modelId;
+    settle({ value: { socketPath: message.socketPath, modelId: message.modelId } });
+  });
+  let lastError: string | undefined;
+  const stderr = pumpLines(proc.stderr, line => { lastError = line; error(line); });
+  const exited = (async () => {
+    await proc.exited;
+    await Promise.all([stdout, stderr]);
+    try { proc.stdin.end(); } catch { /* already closed */ }
+    const exit: WorkerExit = { code: proc.exitCode, signal: proc.signalCode };
+    // Exit 2 is the worker refusing its launch record; its last stderr line says why.
+    settle({ error: new WorkerExitedError(exit, "before ready", exit.code === 2 ? lastError : undefined) });
+    // The socket file goes only once the child has exited and its output drained.
+    rmSync(options.socketPath, { force: true });
+    return exit;
+  })();
+  const alive = () => proc.exitCode === null && proc.signalCode === null;
+  let closing: Promise<WorkerExit> | undefined;
+  const close = () => closing ??= (async () => {
+    if (!alive()) return exited;
+    proc.kill("SIGTERM");
+    const force = setTimeout(() => { if (alive()) proc.kill("SIGKILL"); }, options.graceMs ?? 3_000);
+    try { return await exited; } finally { clearTimeout(force); }
+  })();
+  const timer = setTimeout(() => {
+    settle({ error: new Error(`worker did not report ready within ${options.readyTimeoutMs ?? 900_000} ms`) });
+    void close();
+  }, options.readyTimeoutMs ?? 900_000);
+  timer.unref();
+  void ready.promise.catch(() => {}); // observed through `ready` by the owner
+  return { pid: proc.pid, socketPath: options.socketPath, get modelId() { return modelId; }, ready: ready.promise, exited, close };
+}

package/src/jobs/worker-supervisor.ts

@@ -0,0 +1,258 @@
+// Parent-side restart policy over one isolation worker (worker-process.ts):
+// respawn after an unexpected exit within a rolling budget, expose the state
+// the proxy and `/engine` report, hand out connection-owned execution leases
+// for managed jobs, and drain the worker before stopping it. Request bodies
+// never pass through here; the proxy fetches the socket through `fetch`.
+import type { DisposableResource } from "@mlx-bun/inference/contracts/portable";
+import { spawnWorker, type WorkerExit, type WorkerProcess, type WorkerProcessOptions } from "./worker-process";
+
+export type WorkerSupervisorState = "starting" | "ready" | "restarting" | "exhausted" | "closed";
+
+/** At most three restarts in a rolling minute. One early death (within ten seconds of its
+ * spawn) respawns at once, as any crash does; a worker that dies early again straight after
+ * an early death is a crash loop, and its retry waits five seconds. */
+export interface WorkerRestartBudget { max: number; windowMs: number; delayMs?: number }
+export const DEFAULT_RESTART_BUDGET: Readonly<WorkerRestartBudget> = Object.freeze({ max: 3, windowMs: 60_000 });
+const CRASH_LOOP_WINDOW_MS = 10_000, CRASH_LOOP_DELAY_MS = 5_000;
+
+export interface WorkerSupervisorOptions extends Omit<WorkerProcessOptions, "restarts"> {
+  restarts?: Partial<WorkerRestartBudget>;
+  /** Supervisor notices (exits, respawns, the exhausted budget); default `console.error`. */
+  notice?(line: string): void;
+  /** How long `close()` lets the worker quiesce through `/admin/drain` (default 10 s). */
+  drainTimeoutMs?: number;
+}
+
+/** The typed refusal behind every 502: the worker cannot take this request now. */
+export class EngineUnavailableError extends Error {
+  constructor(readonly state: WorkerSupervisorState, readonly exit: WorkerExit | null, detail?: string) {
+    super(detail ?? describeUnavailable(state, exit));
+    this.name = "EngineUnavailableError";
+  }
+}
+/** The restart budget is spent: nothing respawns until the server restarts. */
+export class RestartBudgetExhaustedError extends EngineUnavailableError {
+  constructor(exit: WorkerExit | null, readonly budget: Readonly<WorkerRestartBudget>) {
+    super("exhausted", exit, `engine restart limit reached (${budget.max} in ${Math.round(budget.windowMs / 1000)} s)${exit ? ` after ${describeExit(exit)}` : ""}; restart the server`);
+    this.name = "RestartBudgetExhaustedError";
+  }
+}
+
+export function describeExit(exit: WorkerExit): string {
+  return exit.signal ? `the worker was killed by ${exit.signal}` : `the worker exited with code ${exit.code}`;
+}
+function describeUnavailable(state: WorkerSupervisorState, exit: WorkerExit | null): string {
+  switch (state) {
+    case "starting": return "the engine worker is still loading";
+    case "restarting": return `${exit ? describeExit(exit) : "the engine worker exited"}; respawning`;
+    case "closed": return "the server is shutting down";
+    case "exhausted": return "engine restart limit reached; restart the server";
+    case "ready": return "the engine worker did not answer";
+  }
+}
+
+export interface WorkerSupervisor {
+  readonly state: WorkerSupervisorState;
+  /** The live worker's pid while one exists (loading or serving); null between workers. */
+  readonly pid: number | null;
+  readonly socketPath: string;
+  readonly modelId: string | undefined;
+  /** Automatic respawns so far; a startup failure of the first worker is not one. */
+  readonly restarts: number;
+  readonly lastExit: WorkerExit | null;
+  readonly budget: Readonly<WorkerRestartBudget>;
+  /** The first worker's readiness. It rejects when that worker dies, echoes
+   * another socket, or misses the ready timeout; the first load is never retried. */
+  readonly ready: Promise<{ socketPath: string; modelId: string }>;
+  /** Resolves once a worker serves; rejects when the budget is spent, on close, or on the signal. */
+  whenReady(signal?: AbortSignal): Promise<void>;
+  /** One request over the worker socket. Rejects with EngineUnavailableError
+   * unless a worker is serving; a transport failure means it died mid-request. */
+  fetch(url: string, init?: RequestInit & { duplex?: "half" }): Promise<Response>;
+  /** A managed job's execution lease, owned by a `/admin/lease` connection
+   * inside the worker: it waits for a serving worker and for generation in
+   * flight; a respawn waits until every lease has been released. */
+  acquireExecutionLease(signal: AbortSignal): Promise<DisposableResource>;
+  /** Best effort `POST /admin/drain`; returns once the worker reports or the deadline passes. */
+  drain(timeoutMs?: number): Promise<void>;
+  /** Drain, stop, and join the worker; a respawn in progress is joined too. Idempotent. */
+  close(): Promise<void>;
+}
+
+export function superviseWorker(options: WorkerSupervisorOptions): WorkerSupervisor {
+  const budget: Readonly<WorkerRestartBudget> = Object.freeze({ ...DEFAULT_RESTART_BUDGET, ...options.restarts });
+  const notice = options.notice ?? (line => console.error(`[isolate] ${line}`));
+  const { restarts: _budget, notice: _notice, drainTimeoutMs: _drain, ...spawnOptions } = options;
+  const lifetime = new AbortController();
+  let state: WorkerSupervisorState = "starting";
+  let worker: WorkerProcess | undefined, live = false, started = false;
+  let restarts = 0, lastExit: WorkerExit | null = null, spawnedAt = 0, earlyDeaths = 0;
+  const exits: number[] = [];
+  let backoff: ReturnType<typeof setTimeout> | undefined, wakeBackoff: (() => void) | undefined;
+  const respawns = new Set<Promise<void>>();
+  const readyWaiters = new Set<{ resolve(): void; reject(error: unknown): void }>();
+  let held = 0;
+  const idleWaiters = new Set<() => void>();
+
+  const unavailable = () => state === "exhausted" ? new RestartBudgetExhaustedError(lastExit, budget) : new EngineUnavailableError(state, lastExit);
+  const setState = (next: WorkerSupervisorState) => {
+    state = next;
+    if (next === "ready") for (const waiter of [...readyWaiters]) { readyWaiters.delete(waiter); waiter.resolve(); }
+    else if (next === "exhausted" || next === "closed") for (const waiter of [...readyWaiters]) { readyWaiters.delete(waiter); waiter.reject(unavailable()); }
+  };
+  const wakeIdle = () => { if (held === 0) for (const wake of [...idleWaiters]) wake(); };
+  const untilIdle = (signal: AbortSignal) => new Promise<void>((resolve, reject) => {
+    if (signal.aborted) return reject(signal.reason);
+    if (held === 0) return resolve();
+    const wake = () => { idleWaiters.delete(wake); signal.removeEventListener("abort", stop); resolve(); };
+    const stop = () => { idleWaiters.delete(wake); reject(signal.reason); };
+    idleWaiters.add(wake);
+    signal.addEventListener("abort", stop, { once: true });
+  });
+
+  const spawn = (): WorkerProcess => {
+    spawnedAt = Date.now();
+    const proc = spawnWorker(spawnOptions);
+    worker = proc; live = true;
+    void proc.exited.then(exit => onExit(proc, exit));
+    return proc;
+  };
+  const onExit = (proc: WorkerProcess, exit: WorkerExit) => {
+    if (worker !== proc) return;
+    live = false; lastExit = exit;
+    // The first worker's failure reaches the owner through `ready`; a closing
+    // supervisor stopped it on purpose.
+    if (state === "closed" || !started) return;
+    const now = Date.now();
+    while (exits.length && now - exits[0]! >= budget.windowMs) exits.shift();
+    if (exits.length >= budget.max) {
+      setState("exhausted");
+      notice(`${describeExit(exit)}; restart limit reached (${budget.max} in ${Math.round(budget.windowMs / 1000)} s) — requests answer 502 until the server restarts`);
+      return;
+    }
+    exits.push(now); restarts++;
+    // Crash-loop backoff: only repeated early deaths (bad flags, OOM on load) wait before the retry instead of
+    // spinning; a single early crash, or one after a worker that served for a while, respawns at once.
+    earlyDeaths = now - spawnedAt < CRASH_LOOP_WINDOW_MS ? earlyDeaths + 1 : 0;
+    const delay = budget.delayMs ?? (earlyDeaths >= 2 ? CRASH_LOOP_DELAY_MS : 0);
+    setState("restarting");
+    notice(`${describeExit(exit)} — respawning (restart ${exits.length}/${budget.max}${delay ? `, after ${delay} ms` : ""})`);
+    const respawn = (async () => {
+      await new Promise<void>(resolve => { wakeBackoff = resolve; backoff = setTimeout(resolve, delay); });
+      backoff = undefined; wakeBackoff = undefined;
+      if (state !== "restarting") return;
+      // A managed job may own the GPU right now; the reload waits for its lease.
+      try { await untilIdle(lifetime.signal); } catch { return; }
+      if (state !== "restarting") return;
+      let next: WorkerProcess;
+      try { next = spawn(); }
+      catch (error) {
+        // The executable or entry is gone: nothing more can be respawned.
+        setState("exhausted");
+        notice(`respawn failed: ${error instanceof Error ? error.message : String(error)}; restart the server`);
+        return;
+      }
+      try {
+        await next.ready;
+        if (worker === next && state === "restarting") { setState("ready"); notice(`engine worker pid ${next.pid} ready`); }
+      } catch {
+        // A rejected handshake need not mean the process exited. Stop and join
+        // it so the exit handler can apply the budget and no live orphan stalls reload.
+        await next.close();
+      }
+    })();
+    respawns.add(respawn);
+    void respawn.finally(() => respawns.delete(respawn)).catch(() => {});
+  };
+
+  const first = spawn();
+  const ready = first.ready.then(value => {
+    started = true;
+    if (state === "starting") setState("ready");
+    return value;
+  });
+  void ready.catch(() => {});
+
+  const whenReady = (signal?: AbortSignal) => {
+    if (signal?.aborted) return Promise.reject(signal.reason);
+    if (state === "ready") return Promise.resolve();
+    if (state === "exhausted" || state === "closed") return Promise.reject(unavailable());
+    return new Promise<void>((resolve, reject) => {
+      const waiter = { resolve() { signal?.removeEventListener("abort", abort); resolve(); },
+        reject(error: unknown) { signal?.removeEventListener("abort", abort); reject(error); } };
+      const abort = () => { readyWaiters.delete(waiter); reject(signal!.reason); };
+      readyWaiters.add(waiter);
+      signal?.addEventListener("abort", abort, { once: true });
+    });
+  };
+
+  const fetchWorker = (url: string, init: RequestInit & { duplex?: "half" } = {}) => {
+    if (state !== "ready" || !worker) return Promise.reject(unavailable());
+    const signal = init.signal ? AbortSignal.any([init.signal, lifetime.signal]) : lifetime.signal;
+    // Bun's fetch has its own idle timer (about 300 s): a long non-streaming generation or a
+    // held lease is silent that long, and the parent then drops a response the worker finished.
+    return fetch(url, { ...init, signal, unix: worker.socketPath, timeout: false } as RequestInit);
+  };
+
+  const acquireExecutionLease = async (signal: AbortSignal): Promise<DisposableResource> => {
+    await whenReady(signal);
+    // Cancellation can arrive in the readiness promise continuation too.
+    signal.throwIfAborted();
+    // The connection owns the lease (main's semantics): a parent that dies
+    // releases it, and disposing here ends the stream the worker holds open.
+    const abort = new AbortController();
+    const cancel = () => abort.abort(signal.reason);
+    signal.addEventListener("abort", cancel, { once: true });
+    try {
+      const response = await fetchWorker("http://engine/admin/lease", { method: "POST", signal: abort.signal });
+      if (!response.ok) { await response.body?.cancel().catch(() => {}); throw new Error(`worker lease failed (${response.status})`); }
+      signal.throwIfAborted();
+      lifetime.signal.throwIfAborted();
+      held++;
+      let released = false;
+      return { dispose() {
+        if (released) return;
+        released = true; held--;
+        abort.abort();
+        void response.body?.cancel().catch(() => {});
+        wakeIdle();
+      } };
+    } catch (error) { abort.abort(); throw error; }
+    finally { signal.removeEventListener("abort", cancel); }
+  };
+
+  const drain = async (timeoutMs = options.drainTimeoutMs ?? 10_000) => {
+    const current = worker;
+    if (!current || !live) return;
+    try {
+      const response = await fetch("http://engine/admin/drain", { method: "POST", body: JSON.stringify({ timeout_ms: timeoutMs }),
+        signal: AbortSignal.timeout(timeoutMs + 1_000), unix: current.socketPath, timeout: false } as RequestInit);
+      await response.arrayBuffer();
+    } catch { /* best effort: the worker is stopped next */ }
+  };
+
+  let closing: Promise<void> | undefined;
+  const close = () => closing ??= (async () => {
+    const wasReady = state === "ready";
+    setState("closed");
+    lifetime.abort(new Error("engine host is closed"));
+    clearTimeout(backoff); wakeBackoff?.();
+    const current = worker;
+    if (current) {
+      if (wasReady) await drain();
+      await current.close();
+    }
+    await Promise.allSettled([first.ready, ...respawns]);
+    if (worker && worker !== current) await worker.close();
+  })();
+
+  return {
+    get state() { return state; },
+    get pid() { return live && worker ? worker.pid : null; },
+    socketPath: options.socketPath,
+    get modelId() { return worker?.modelId; },
+    get restarts() { return restarts; },
+    get lastExit() { return lastExit; },
+    budget, ready, whenReady, fetch: fetchWorker, acquireExecutionLease, drain, close,
+  };
+}