mlx-bun 0.4.0 → 1.0.0

package/src/jobs/runner.ts

@@ -1,167 +1,16 @@
-import { CancellationSource } from "../engine/cancellation";
-// Job runner: turns a registered `JobRunner` into a tracked, logged job.
-// Two execution modes — both create a row, both stream NDJSON events to the
-// log, both own terminal status:
-//
-//   submitInProcess   — runs the runner as a fire-and-forget async task in
-//                        THIS process. For pure-JS work (dataset builds) with
-//                        no MLX/GPU state to isolate.
-//   submitSubprocess  — spawns `bun job-entry.ts <jobId>` so MLX state is
-//                        isolated and an uncatchable crash (segfault,
-//                        process.exit) can't take the server down. Gated by a
-//                        single global GPU lease: only one MLX job runs at a
-//                        time; the rest wait 'queued' and drain in order.
-//
-// Port of optiq lab jobs.py `submit` + `_job_main`, split across this file
-// and job-entry.ts.
-
-import type { DisposableResource } from "../contracts/resources";
-import { createJobTaskClient } from "./task-client";
 import { appendFileSync } from "node:fs";
-import { fileURLToPath } from "node:url";
-import type { Emit, JobEvent, JobRunner } from "./types";
-import type { JobStore } from "./db";
-
-/** Absolute path to the child-process entry (`bun job-entry.ts <jobId>`). */
-export const JOB_ENTRY_PATH = fileURLToPath(new URL("./job-entry.ts", import.meta.url));
-
-// ---------------------------------------------------------------------------
-// Runner registry
-// ---------------------------------------------------------------------------
-
-const RUNNERS = new Map<string, JobRunner>();
-
-export function registerRunner(kind: string, run: JobRunner): void {
-  RUNNERS.set(kind, run);
-}
-
-export function getRunner(kind: string): JobRunner | undefined {
-  return RUNNERS.get(kind);
-}
-
-// Built-in test kinds. Real kinds (quantize/finetune) register their runner
-// on import of their module (see job-entry.ts KIND_MODULES).
-
-/** Emits a couple of stage events then resolves — exercises the happy path. */
-const noopRunner: JobRunner = async (emit) => {
-  emit({ type: "stage", stage: "warmup", progress: 0.25, message: "starting up" });
-  emit({ type: "stage", stage: "work", progress: 0.75, message: "doing the thing" });
-  return { outputPath: undefined };
-};
-
-/** Hard-exits the process — simulates an uncatchable crash (segfault-class).
- *  Only meaningful under submitSubprocess: the child dies, the parent lives. */
-const crashRunner: JobRunner = async (emit) => {
-  emit({ type: "stage", stage: "boom", message: "about to crash" });
-  process.exit(1);
-};
-
-registerRunner("noop", noopRunner);
-registerRunner("crash", crashRunner);
-
-// ---------------------------------------------------------------------------
-// Emit factory
-// ---------------------------------------------------------------------------
-
-/** Build the `emit` sink for a job: appends each event as one NDJSON line
- *  (line-buffered via appendFileSync for crash durability) and mirrors
- *  progress/message from stage/metric events onto the row. Never throws — a
- *  logging failure must not kill a job. The terminal `done`/`failed` status
- *  is owned by the submit wrapper / job-entry, NOT here. */
-export function makeEmit(store: JobStore, jobId: string, logPath: string): Emit {
-  return (e: JobEvent) => {
-    try {
-      appendFileSync(logPath, JSON.stringify(e) + "\n");
-    } catch {
-      // swallow: a job must survive a logging hiccup
-    }
-    if (e.type === "stage" || e.type === "metric") {
-      const progress = typeof e.progress === "number" ? e.progress : undefined;
-      const message = typeof e.message === "string" ? e.message : undefined;
-      if (progress !== undefined || message !== undefined) {
-        try {
-          if (progress !== undefined) store.setProgress(jobId, progress, message);
-          else if (message !== undefined) {
-            // message-only update preserves current progress
-            store.db.prepare("UPDATE jobs SET message = ? WHERE id = ?").run(message, jobId);
-          }
-        } catch {
-          // swallow: DB contention must not kill a job
-        }
-      }
-    }
-  };
-}
-
-// ---------------------------------------------------------------------------
-// In-process submit
-// ---------------------------------------------------------------------------
-
-export interface SubmitResult {
-  jobId: string;
-  outputPath?: string;
-}
-
-/** Run a registered runner as a fire-and-forget async task in this process.
- *  Sets 'running' + emits 'started' synchronously, then schedules the runner;
- *  on success sets 'done'/progress 1/ended_at + emits 'done', on throw sets
- *  'failed'/error/ended_at + emits 'failed'. Returns once the task is
- *  scheduled (does not await completion — poll the row / tail the log). */
-interface InProcessWork { cancellation: CancellationSource; done: Promise<void>; }
-const inProcess = new WeakMap<JobStore, Set<InProcessWork>>();
-
-export function submitInProcess(
-  store: JobStore,
-  kind: string,
-  config: Record<string, unknown>,
-  outputPath?: string,
-): SubmitResult {
-  if (closedStores.has(store)) throw new Error("job host is closed");
-  const row = store.create(kind, config, outputPath);
-  const emit = makeEmit(store, row.id, row.log_path);
-  store.setStatus(row.id, "running");
-  emit({ type: "started", ts: Date.now() });
+import type { DisposableResource } from "@mlx-bun/inference/contracts/portable";
+import type { JobEvent } from "./protocol";
+import { jobError, nowIso, type JobStore } from "./db";
+import { executablePath } from "./executable";
 
-  const runner = getRunner(kind);
-  const cancellation = new CancellationSource();
-  const running = inProcess.get(store) ?? new Set<InProcessWork>();
-  inProcess.set(store, running);
-  let finish!: () => void;
-  const work = { cancellation, done: new Promise<void>((resolve) => { finish = resolve; }) };
-  running.add(work);
-  void (async () => {
-    if (!runner) {
-      const error = `no runner registered for kind "${kind}"`;
-      store.setStatus(row.id, "failed", { error, endedAt: nowIso() });
-      emit({ type: "failed", error, ts: Date.now() });
-      return;
-    }
-    try {
-      const result = await createJobTaskClient(runner).run(JSON.parse(row.config_json), emit, cancellation);
-      const out = result?.outputPath;
-      if (out) store.setOutputPath(row.id, out);
-      store.setProgress(row.id, 1);
-      store.setStatus(row.id, "done", { endedAt: nowIso() });
-      emit({ type: "done", ts: Date.now(), output_dir: out ?? outputPath });
-    } catch (e) {
-      const error = errString(e);
-      store.setStatus(row.id, "failed", { error, endedAt: nowIso() });
-      emit({ type: "failed", error, ts: Date.now() });
-    }
-  })().finally(() => { running.delete(work); finish(); });
-
-  return { jobId: row.id, outputPath };
-}
-
-// ---------------------------------------------------------------------------
-// Subprocess submit + single global GPU lease
-// ---------------------------------------------------------------------------
+export interface SubmitResult { jobId: string; outputPath?: string; }
 
 interface QueuedSpawn {
-  acquire?: (signal: AbortSignal) => Promise<DisposableResource>;
+  acquire: (signal: AbortSignal) => Promise<DisposableResource>;
   lease?: DisposableResource;
   abort: AbortController;
-  proc?: Bun.Subprocess;
+  proc?: JobProcess;
   finished: Promise<void>;
   finish(): void;
   store: JobStore;
@@ -170,34 +19,33 @@ interface QueuedSpawn {
   bin: string;
   compiled: boolean;
   spawn: typeof Bun.spawn;
+  graceMs: number;
+  /** The submitter's temporary directory for the child (`TMPDIR`). */
+  scratchDir?: string;
+  /** Set when a caller stopped this job on purpose; recorded as the failure instead of the exit code. */
+  cancelReason?: string;
   onComplete?: (jobId: string, code: number) => void;
 }
 
+type JobProcess = Bun.Subprocess<"pipe", "pipe", "pipe">;
+
 let gpuLeaseHolder: string | null = null;
 const spawnQueue: QueuedSpawn[] = [];
 let activeSpawn: QueuedSpawn | undefined;
 const closedStores = new WeakSet<JobStore>();
 
-/** True while a GPU (subprocess) job is running — the server gates token
- *  generation on this so quantize/finetune don't fight live inference. */
-export function isGpuBusy(): boolean {
-  return gpuLeaseHolder !== null;
-}
-
-/** The job id currently holding the GPU lease, or null. */
-export function currentGpuJob(): string | null {
-  return gpuLeaseHolder;
-}
-
 export interface SubprocessOpts {
-  /** Host execution lease, held until child exit (including crashes). */
-  acquire?: (signal: AbortSignal) => Promise<DisposableResource>;
-  /** Override the entry script (tests). Defaults to JOB_ENTRY_PATH. */
-  entry?: string;
-  /** Override the runtime binary (tests). Defaults to "bun". */
+  /** Host execution lease, held until child exit and log drain (including crashes). */
+  acquire: (signal: AbortSignal) => Promise<DisposableResource>;
+  /** Child entry supplied by application composition. */
+  entry: string;
+  /** Defaults to the executable captured at startup (Bun for source execution). */
   bin?: string;
   /** Override process creation for deterministic failure-path tests. */
   spawn?: typeof Bun.spawn;
+  /** SIGTERM first; SIGKILL for whatever of the job's process group is still
+   * alive after this (default 3 s, as main). */
+  graceMs?: number;
   /** Called on the server (parent) after the child exits — used to invalidate
    *  caches (e.g. the Library) so a finished quantize surfaces immediately. */
   onComplete?: (jobId: string, code: number) => void;
@@ -210,8 +58,9 @@ export function submitSubprocess(
   store: JobStore,
   kind: string,
   config: Record<string, unknown>,
-  outputPath?: string,
-  opts: SubprocessOpts = {},
+  outputPath: string | undefined,
+  opts: SubprocessOpts,
+  scratchDir?: string,
 ): SubmitResult {
   if (closedStores.has(store)) throw new Error("job host is closed");
   const row = store.create(kind, config, outputPath);
@@ -221,20 +70,24 @@ export function submitSubprocess(
     abort: new AbortController(), finished, finish,
     store,
     jobId: row.id,
-    entry: opts.entry ?? JOB_ENTRY_PATH,
-    compiled: opts.entry === undefined && opts.bin === undefined && JOB_ENTRY_PATH.includes("$bunfs"),
-    bin: opts.bin ?? "bun",
+    entry: opts.entry,
+    compiled: opts.entry.includes("$bunfs"),
+    bin: opts.bin ?? executablePath,
     spawn: opts.spawn ?? Bun.spawn,
+    graceMs: opts.graceMs ?? 3000,
     onComplete: opts.onComplete,
     acquire: opts.acquire,
+    ...(scratchDir ? { scratchDir } : {}),
   };
   spawnQueue.push(item);
   drainQueue();
   return { jobId: row.id, outputPath };
 }
 
-/** Spawn the child, acquire the lease, stream stdout/stderr into the log,
- *  reconcile terminal status on exit, then release the lease and drain. */
+/** Spawn the child as the leader of its own process group, stream its
+ * stdout/stderr into the log, reconcile terminal status on exit, then release
+ * the lease and drain. Everything that can fail runs before the spawn, so a
+ * child that exists is always joined before its lease is released. */
 function spawnNow(item: QueuedSpawn): void {
   const { store, jobId, entry, bin, spawn } = item;
   gpuLeaseHolder = jobId;
@@ -242,26 +95,35 @@ function spawnNow(item: QueuedSpawn): void {
   // The child opens its OWN JobStore over the SAME DB/logs — a sqlite
   // connection can't cross the process boundary, so we hand the paths via
   // env. A :memory: DB can't be shared with a child; subprocess jobs require
-  // a file-backed DB.
-  let proc: Bun.Subprocess<"ignore", "pipe", "pipe">;
+  // a file-backed DB. `detached` makes the child a process-group leader, so
+  // stopping the job stops everything it started; a group leader no longer
+  // receives the terminal's signals, so the child watches the parent pipe on
+  // its stdin instead (cli/job-entry.ts) and stops when this process is gone.
+  let proc: JobProcess;
+  let logPath: string | undefined;
   try {
-    proc = spawn(item.compiled ? [process.execPath, "__job", jobId] : [bin, entry, jobId], {
+    logPath = store.get(jobId)?.log_path;
+    proc = spawn(item.compiled ? [bin, "__job", jobId] : [bin, entry, jobId], {
+      stdin: "pipe",
       stdout: "pipe",
       stderr: "pipe",
+      detached: true,
       env: {
         ...process.env,
         MLX_BUN_JOBS_DB: store.dbPath,
         MLX_BUN_JOBS_DIR: store.logsDir,
+        MLX_BUN_JOB_PARENT_PIPE: "1",
+        ...(item.scratchDir ? { TMPDIR: item.scratchDir } : {}),
       },
     });
   } catch (e) {
     try {
       store.setStatus(jobId, "failed", {
-        error: errString(e),
+        error: jobError(e),
         endedAt: nowIso(),
       });
     } catch (error) {
-      console.error(`[jobs] failed to record spawn failure: ${errString(error)}`);
+      console.error(`[jobs] failed to record spawn failure: ${jobError(error)}`);
     } finally {
       releaseLease(item);
     }
@@ -269,18 +131,24 @@ function spawnNow(item: QueuedSpawn): void {
   }
 
   item.proc = proc;
-  const logPath = store.get(jobId)?.log_path;
   const logLine = (line: string) => {
     if (!line || !logPath) return;
     const ev: JobEvent = { type: "log", line };
     try { appendFileSync(logPath, JSON.stringify(ev) + "\n"); } catch {}
   };
 
-  void pumpLines(proc.stdout, logLine);
-  void pumpLines(proc.stderr, logLine);
+  const logs = Promise.all([pumpLines(proc.stdout, logLine), pumpLines(proc.stderr, logLine)]);
 
   void (async () => {
     const code = await proc.exited;
+    // Descendants that outlived the child still hold its work and its log
+    // pipes: the job is joined once its whole process group is gone and its
+    // output has drained. Both waits are bounded: a process surviving SIGKILL
+    // (uninterruptible sleep) or one that left the group holding the pipes is
+    // reported and left behind rather than holding the lease forever.
+    const joined = await stopGroup(proc, item.graceMs) && await settles(logs, item.graceMs);
+    if (!joined) console.error(`[jobs] ${jobId}: a process of the job outlived SIGKILL or its output; releasing the lease without it`);
+    try { proc.stdin?.end(); } catch { /* already closed */ }
     // code 0 ⇒ trust the child's terminal status (it set done/failed itself).
     // non-zero ⇒ if the row never reached terminal (crash before the wrapper
     // could write), force it failed.
@@ -288,12 +156,12 @@ function spawnNow(item: QueuedSpawn): void {
       const cur = store.get(jobId);
       if (cur && (cur.status === "queued" || cur.status === "running")) {
         store.setStatus(jobId, "failed", {
-          error: `exited ${code}`,
+          error: item.cancelReason ?? `exited ${code}`,
           endedAt: nowIso(),
         });
       }
     } } catch (error) {
-      console.error(`[jobs] failed to record child exit: ${errString(error)}`);
+      console.error(`[jobs] failed to record child exit: ${jobError(error)}`);
     } finally {
       releaseLease(item);
       try { item.onComplete?.(jobId, code); } catch {}
@@ -302,9 +170,49 @@ function spawnNow(item: QueuedSpawn): void {
   })();
 }
 
+/** Whether any process of the group the child leads is still alive. False for
+ * a child that leads no group (a supplied spawn without process groups). */
+function groupAlive(proc: JobProcess): boolean {
+  if (!Number.isInteger(proc.pid) || proc.pid <= 0) return false;
+  try { process.kill(-proc.pid, 0); return true; } catch { return false; }
+}
+
+/** Signal the child's process group, or the child alone when it leads none. */
+function signalJob(proc: JobProcess, signal: NodeJS.Signals): void {
+  if (groupAlive(proc)) {
+    try { process.kill(-proc.pid, signal); return; } catch { /* the group just ended */ }
+  }
+  if (proc.exitCode === null && proc.signalCode == null) {
+    try { proc.kill(signal); } catch { /* already gone */ }
+  }
+}
+
+/** After the leader's exit: SIGTERM what is left of its group, SIGKILL it after
+ * the grace, and wait one more grace for it to go. Whether the group is gone. */
+async function stopGroup(proc: JobProcess, graceMs: number): Promise<boolean> {
+  if (!groupAlive(proc)) return true;
+  signalJob(proc, "SIGTERM");
+  const force = Date.now() + graceMs, giveUp = force + graceMs;
+  let killed = false;
+  while (groupAlive(proc)) {
+    if (Date.now() >= giveUp) return false;
+    if (!killed && Date.now() >= force) { killed = true; signalJob(proc, "SIGKILL"); }
+    await Bun.sleep(20);
+  }
+  return true;
+}
+
+/** Whether `work` settles within `ms`. */
+async function settles(work: Promise<unknown>, ms: number): Promise<boolean> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  const timeout = new Promise<false>((resolve) => { timer = setTimeout(() => resolve(false), ms); });
+  try { return await Promise.race([work.then(() => true, () => true), timeout]); }
+  finally { clearTimeout(timer); }
+}
+
 function releaseLease(item: QueuedSpawn): void {
   try { item.lease?.dispose(); }
-  catch (error) { console.error(`[jobs] execution lease cleanup failed: ${errString(error)}`); }
+  catch (error) { console.error(`[jobs] execution lease cleanup failed: ${jobError(error)}`); }
   finally {
     item.lease = undefined;
     if (gpuLeaseHolder === item.jobId) gpuLeaseHolder = null;
@@ -322,7 +230,7 @@ export function drainQueue(): void {
     // iteratively so a run of bad queue entries cannot recurse or wedge the
     // first startable job behind them.
     activeSpawn = next;
-    if (next.acquire) {
+    {
       gpuLeaseHolder = next.jobId; // reserve FIFO order while inference drains
       void Promise.resolve().then(() => next.acquire!(next.abort.signal)).then((lease) => {
         next.lease = lease;
@@ -330,47 +238,73 @@ export function drainQueue(): void {
         spawnNow(next);
       }).catch((error) => {
         try {
-          next.store.setStatus(next.jobId, "failed", { error: errString(error), endedAt: nowIso() });
+          next.store.setStatus(next.jobId, "failed", { error: jobError(error), endedAt: nowIso() });
         } catch (recordError) {
-          console.error(`[jobs] failed to record admission failure: ${errString(recordError)}`);
+          console.error(`[jobs] failed to record admission failure: ${jobError(recordError)}`);
         } finally { releaseLease(next); }
       }).finally(drainQueue);
-    } else spawnNow(next);
+    }
   }
 }
 
-/** Dataset work stays in-process, but belongs to the application host too. */
-export async function closeInProcessJobs(store: JobStore): Promise<void> {
-  closedStores.add(store);
-  const active = [...inProcess.get(store) ?? []];
-  for (const work of active) work.cancellation.cancel("engine_closed");
-  await Promise.all(active.map((work) => work.done));
-}
-
-/** Stop only this host's managed GPU jobs. Await process death before releasing
- * the execution lease; queued and admission-waiting jobs never spawn. */
+/** Stop only this host's managed GPU jobs. Await the death of the active child's
+ * whole process group before releasing the execution lease, bounded at one
+ * grace after SIGKILL; queued and admission-waiting jobs never spawn. */
 export async function closeSubprocessJobs(store: JobStore): Promise<void> {
   closedStores.add(store);
+  const errors: unknown[] = [];
   for (let i = spawnQueue.length - 1; i >= 0; i--) {
     const item = spawnQueue[i]!;
     if (item.store !== store) continue;
     spawnQueue.splice(i, 1);
-    store.setStatus(item.jobId, "failed", { error: "job host closed", endedAt: nowIso() });
-    item.finish();
+    try { store.setStatus(item.jobId, "failed", { error: "job host closed", endedAt: nowIso() }); }
+    catch (error) { errors.push(error); }
+    finally { item.finish(); }
   }
   const active = activeSpawn;
-  if (!active || active.store !== store) return;
-  active.abort.abort(new Error("job host closed"));
+  if (active?.store === store) await stopActive(active, "job host closed");
+  if (errors.length) throw new AggregateError(errors, "Failed to persist cancelled jobs");
+}
+
+/** Stop the active job: no child is spawned if it is still waiting for the lease, otherwise the child gets SIGTERM, then SIGKILL after
+ * the grace, and the lease is released once its process group is gone (bounded at one more grace). Resolves when the job has ended. */
+async function stopActive(active: QueuedSpawn, reason: string): Promise<void> {
+  active.abort.abort(new Error(reason));
   const proc = active.proc;
-  if (proc) proc.kill("SIGTERM");
-  const force = proc ? setTimeout(() => { if (proc.exitCode === null) proc.kill("SIGKILL"); }, 3000) : undefined;
-  try { await active.finished; }
-  finally { if (force) clearTimeout(force); }
+  if (proc) {
+    signalJob(proc, "SIGTERM");
+    const force = setTimeout(() => signalJob(proc, "SIGKILL"), active.graceMs);
+    try {
+      // SIGKILL at the grace, then one more grace for the child to die.
+      if (!(await settles(proc.exited, 2 * active.graceMs))) {
+        console.error(`[jobs] ${active.jobId}: pid ${proc.pid} outlived SIGKILL; releasing the lease without it`);
+        releaseLease(active);
+      }
+    } finally { clearTimeout(force); }
+  }
+  await active.finished;
+}
+
+/** Stop one managed job of this store: a queued job never spawns, the active one is stopped as `closeSubprocessJobs` stops it. A job that
+ * has finished but whose process group is still being joined is waited for; anything else is left alone. Resolves when the job's process is gone. */
+export async function cancelSubprocessJob(store: JobStore, jobId: string): Promise<void> {
+  const queued = spawnQueue.findIndex(item => item.store === store && item.jobId === jobId);
+  if (queued >= 0) {
+    const [item] = spawnQueue.splice(queued, 1);
+    store.setStatus(jobId, "failed", { error: "job cancelled", endedAt: nowIso() });
+    item!.finish();
+    return;
+  }
+  const active = activeSpawn;
+  if (active?.store !== store || active.jobId !== jobId) return;
+  const row = store.get(jobId);
+  if (row && row.status !== "done" && row.status !== "failed") { active.cancelReason = "job cancelled"; await stopActive(active, "job cancelled"); }
+  else await active.finished;
 }
 
 /** Read a child stream line-by-line, buffering partial trailing lines, and
- *  hand each complete line to `sink`. */
-async function pumpLines(
+ *  hand each complete line to `sink`. Shared with the isolation worker owner. */
+export async function pumpLines(
   stream: ReadableStream<Uint8Array> | undefined,
   sink: (line: string) => void,
 ): Promise<void> {
@@ -397,18 +331,3 @@ async function pumpLines(
     reader.releaseLock();
   }
 }
-
-// ---------------------------------------------------------------------------
-// helpers
-// ---------------------------------------------------------------------------
-
-function nowIso(): string {
-  // SQLite datetime('now') is "YYYY-MM-DD HH:MM:SS" in UTC — match it so the
-  // ended_at column is uniform whether set here or by a column default.
-  return new Date().toISOString().replace("T", " ").slice(0, 19);
-}
-
-function errString(e: unknown): string {
-  if (e instanceof Error) return `${e.name}: ${e.message}`;
-  return String(e);
-}

package/src/jobs/service.ts

@@ -0,0 +1,75 @@
+// The `jobs` core service over this app's job host and store: what a module's
+// job runners run on. A module submits by kind and the loaded module's runner
+// spec decides how it runs, recorded in the same job store (and streamed by
+// the same `/api/jobs` routes) as every other job: a `task` runs in this
+// process; a `process` runs as a child that stops with its parent, under the
+// host's execution lease (the child activates the module itself,
+// `cli/job-entry.ts`).
+import type { JobEmit, JobEvent, JobRecord, JobRunner, JobRunnerSpec, JobService } from "@mlx-bun/app-core";
+import type { DisposableResource } from "@mlx-bun/inference/contracts/portable";
+import type { JobStore } from "./db";
+import type { JobRow, JobRunner as AppJobRunner } from "./protocol";
+import { tailJob } from "./sse";
+
+/** The runners of the loaded modules, keyed by job kind (`LoadedModules.jobs`). */
+export type JobRunners = ReadonlyMap<string, { readonly spec: JobRunnerSpec; readonly runner: JobRunner }>;
+
+/** The part of the job host the service drives. */
+export interface JobTasks {
+  ensureStore(): JobStore;
+  submitTask(kind: string, config: Record<string, unknown>, runner: AppJobRunner, outputPath?: string): { jobId: string };
+  cancelTask(jobId: string): Promise<void>;
+  /** Queue a child process for the kind under the host's execution lease. */
+  submit(kind: string, config: Record<string, unknown>, outputPath?: string, scratchDir?: string): { jobId: string };
+  cancelProcess(jobId: string): Promise<void>;
+}
+
+export interface AppJobService extends JobService {
+  /** Serve the job kinds of the loaded modules; before this, every submission is rejected. */
+  serve(runners: JobRunners): void;
+}
+
+const recordOf = (row: JobRow): JobRecord => ({ id: row.id, kind: row.kind, status: row.status, progress: row.progress, message: row.message,
+  outputPath: row.output_path, error: row.error, startedAt: row.started_at, endedAt: row.ended_at });
+
+export interface JobServiceOptions {
+  /** The engine's execution lease: a runner that declared `gpu: "exclusive"` holds it for its whole run, so no model generates meanwhile. */
+  acquire?(signal: AbortSignal): Promise<DisposableResource>;
+}
+
+export function createJobService(host: JobTasks, options: JobServiceOptions = {}): AppJobService {
+  let runners: JobRunners = new Map();
+  const store = () => host.ensureStore();
+  return {
+    serve(served) { runners = served; },
+    async submit(submission) {
+      const registered = runners.get(submission.kind);
+      if (!registered) throw new Error(`no installed module runs job kind "${submission.kind}"`);
+      const { isolation, gpu } = registered.spec;
+      if (isolation === "process") {
+        // The host's children run under its execution lease, so no model generates meanwhile.
+        if (gpu !== "exclusive") throw new Error(`job kind "${submission.kind}": this host runs process jobs only with the exclusive GPU lease`);
+        const { jobId } = host.submit(submission.kind, { ...submission.config }, submission.outputPath, submission.scratchDir);
+        return recordOf(store().get(jobId)!);
+      }
+      // The module's runner speaks the contract's event type; the store records every event the same way.
+      const exclusive = registered.spec.gpu === "exclusive" && options.acquire;
+      const run: AppJobRunner = async (emit, config, signal) => {
+        const stop = signal ?? new AbortController().signal;
+        if (!exclusive) return registered.runner(emit as JobEmit, config, stop);
+        emit({ type: "stage", stage: "waiting", progress: 0, message: "waiting for in-flight generation to drain" });
+        const lease = await exclusive(stop);
+        try { return await registered.runner(emit as JobEmit, config, stop); } finally { lease.dispose(); }
+      };
+      const { jobId } = host.submitTask(submission.kind, { ...submission.config }, run, submission.outputPath);
+      return recordOf(store().get(jobId)!);
+    },
+    async get(id) { const row = store().get(id); return row ? recordOf(row) : undefined; },
+    async list(filter) {
+      // The store's newest-first window; a status filter applies within it.
+      return store().recent(1000, filter?.kind).filter(row => !filter?.status || row.status === filter.status).map(recordOf);
+    },
+    async cancel(id) { await Promise.all([host.cancelTask(id), host.cancelProcess(id)]); },
+    events(id, signal) { return tailJob(store(), id, { follow: true, ...(signal ? { signal } : {}) }) as AsyncIterable<JobEvent>; },
+  };
+}