mlx-bun 0.2.0 → 0.4.0

package/src/diffusion/diffusion-generate.ts

@@ -14,6 +14,10 @@
 // only RNG consumers then are the canvas draws.
 
 import type { DiffusionGemmaModel } from "../model/diffusion-gemma";
+import type { DenoisingGraph } from "../inference/denoising";
+import { assertMlxDenoisingGraph, bindLegacyDenoisingModel } from "../backends/mlx/diffusion";
+import { cleanupFailure, disposeResources, ownResource } from "../engine/resources";
+import type { DisposableResource, ResourceOwner } from "../contracts/resources";
 import { MlxArray } from "../mlx/array";
 import { Dtype } from "../mlx/ffi";
 import * as ops from "../mlx/ops";
@@ -184,14 +188,48 @@ function entropyTransferMask(entropy: MlxArray, bound: number): MlxArray {
   return out;
 }
 
-export function diffusionGenerate(
-  model: DiffusionGemmaModel,
-  promptIds: number[],
-  opts: DiffusionGenOptions,
-): DiffusionGenResult {
-  const t = model.config.text;
-  const vocab = t.vocabSize;
-  const modelCanvas = model.canvasLength;
+/** Compatibility entry. Both sync and asynchronous callers drain one algorithm. */
+export function diffusionGenerate(model: DiffusionGemmaModel, promptIds: number[], opts: DiffusionGenOptions): DiffusionGenResult {
+  return denoiseSync(bindLegacyDenoisingModel(model).graph, promptIds, opts);
+}
+
+export function denoiseSync<State>(graph: DenoisingGraph<MlxArray, State>, promptIds: number[], opts: DiffusionGenOptions): DiffusionGenResult {
+  const steps = denoisingSteps(graph, promptIds, opts);
+  while (true) {
+    const step = steps.next();
+    if (step.done) {
+      if (!step.value) throw new Error("denoising ended without a result");
+      return step.value;
+    }
+  }
+}
+
+/** One macrotask between bounded denoising units allows cancellation and I/O.
+ * No tentative canvas tokens are published, and no extra device eval is added. */
+export async function denoiseAsync<State>(
+  graph: DenoisingGraph<MlxArray, State>, promptIds: number[], opts: DiffusionGenOptions, signal?: AbortSignal,
+): Promise<DiffusionGenResult> {
+  signal?.throwIfAborted();
+  const steps = denoisingSteps(graph, promptIds, opts);
+  try {
+    while (true) {
+      signal?.throwIfAborted();
+      const step = steps.next();
+      if (step.done) {
+        if (!step.value) throw new Error("denoising ended without a result");
+        return step.value;
+      }
+      await new Promise<void>((resolve) => setImmediate(resolve));
+    }
+  } finally { steps.return(undefined); }
+}
+
+function* denoisingSteps<State>(
+  graph: DenoisingGraph<MlxArray, State>, promptIds: number[], opts: DiffusionGenOptions,
+): Generator<void, DiffusionGenResult | undefined> {
+  assertMlxDenoisingGraph(graph);
+  const vocab = graph.vocabSize;
+  const modelCanvas = graph.canvasLength;
   const maxSteps = opts.maxDenoisingSteps ?? 48;
   const fullCanvas = opts.fullCanvas ?? false;
   const maxCanvas = fullCanvas ? modelCanvas : Math.min(modelCanvas, opts.maxCanvasLength ?? modelCanvas);
@@ -214,10 +252,9 @@ export function diffusionGenerate(
 
   ops.randomSeed(opts.seed ?? 0n);
 
-  const cache = opts.visionPixels
-    ? model.prefillVision(promptIds, opts.visionPixels)
-    : model.prefill(promptIds);
-  const dequantWeight = model.dequantEmbedWeight();
+  let stateOwner: ResourceOwner<State> | undefined;
+  let dequantWeight: MlxArray | null = null;
+  let failure: { error: unknown } | undefined;
 
   const emitted: number[] = [];
   const blocks: number[][] = [];
@@ -228,6 +265,9 @@ export function diffusionGenerate(
   let isFirstBlock = true;
 
   try {
+    const cache = graph.prefill(promptIds, opts.visionPixels);
+    stateOwner = ownResource(cache, (state) => graph.closeState(state));
+    dequantWeight = graph.dequantEmbedWeight();
     while (generated < maxNewTokens) {
       const remaining = maxNewTokens - generated;
       const canvasLength = fullCanvas
@@ -238,161 +278,186 @@ export function diffusionGenerate(
         // Continuation: prefill the encoder over the previous accepted block.
         const prev = blocks[blocks.length - 1]!;
         const prevArr = MlxArray.fromInt32(Int32Array.from(prev), [1, prev.length]);
-        model.extendPrefill(prevArr, cache);
-        prevArr.dispose();
+        try { graph.extendPrefill(prevArr, cache); }
+        finally { prevArr.dispose(); }
       }
       isFirstBlock = false;
 
-      let currentCanvas = ops.randint(0, vocab, [1, canvasLength], I32); // RNG: init
-      let draftReveal = ops.zeros([1, canvasLength], Dtype.bool);
-      let draftCanvas = ops.reshape(currentCanvas, [1, canvasLength]); // alias copy
-      let argmaxCanvas = ops.reshape(currentCanvas, [1, canvasLength]);
+      let currentCanvas: MlxArray | null = null;
+      let draftReveal: MlxArray | null = null;
+      let draftCanvas: MlxArray | null = null;
+      let argmaxCanvas: MlxArray | null = null;
       let scEmbeddings: MlxArray | null = null;
       const history: MlxArray[] = [];
       let stepsThisCanvas = 0;
-
-      for (let curStep = maxSteps; curStep >= 1; curStep--) {
-        stepsThisCanvas++;
-        const logits = model.decoderLogits(currentCanvas, cache, scEmbeddings);
-        const schedT = linearTemp(curStep, maxSteps, tMin, tMax);
-        // Match the reference EXACTLY: divide (not reciprocal-multiply) — the
-        // f32 rounding differs and the confidence threshold is a hard cutoff
-        // that flips acceptance on a 1-ULP difference, diverging the trajectory.
-        const schedArr = ops.scalarLike(schedT, logits);
-        const processed = ops.div(logits, schedArr);
-        schedArr.dispose();
-        logits.dispose();
-
-        argmaxCanvas.dispose();
-        const argmax = ops.argmaxAxis(processed, -1);
-        argmaxCanvas = argmax.astype(I32); // [1, L]
-        argmax.dispose();
-
-        if (curStep === 1) {
-          processed.dispose();
-          break;
-        }
-
-        // temperature 0 → denoiser canvas is the argmax.
-        const denoiser = argmaxCanvas;
-
-        let acceptance: MlxArray;
-        let nextSc: MlxArray | null = null;
-        if (sampler === "entropy-bound") {
-          // cur_step > 1 always here (cur_step==1 breaks before the sampler):
-          // entropy AND the next soft embeddings come from the SAME entropy
-          // chain (reference _diffusion_entropy_and_soft_embeddings) — NOT the
-          // softmax-precise path the confidence sampler uses.
-          const { probs, entropy } = entropyProbsChain(processed);
-          acceptance = entropyTransferMask(entropy, entropyBound);
-          entropy.dispose();
-          nextSc = softEmbeddingsFromProbs(probs, dequantWeight, model.embedScale);
-          probs.dispose();
-          // accepted/current update (entropy variant)
-          const accepted = ops.where(acceptance, denoiser, currentCanvas);
-          const noise = ops.randint(0, vocab, [1, canvasLength], I32); // RNG: re-noise
-          const newCurrent = ops.where(acceptance, accepted, noise);
-          noise.dispose();
-          currentCanvas.dispose();
-          currentCanvas = newCurrent;
-          draftReveal.dispose();
-          const notAcceptance = ops.logicalNot(acceptance);
-          draftReveal = ops.logicalNot(notAcceptance); // = acceptance
-          notAcceptance.dispose();
-          draftCanvas.dispose();
-          draftCanvas = ops.reshape(argmaxCanvas, [1, canvasLength]);
-          accepted.dispose();
-        } else {
-          const unrevealed = ops.logicalNot(draftReveal);
-          const confidence = tokenProbability(processed, denoiser);
-          acceptance = confidenceTransferMask(confidence, unrevealed, threshold, false);
-          confidence.dispose();
-          const accepted = ops.where(acceptance, denoiser, draftCanvas);
-          const revealOrAccept = ops.logicalOr(draftReveal, acceptance);
-          const noise = ops.randint(0, vocab, [1, canvasLength], I32); // RNG: re-noise
-          const newCurrent = ops.where(revealOrAccept, accepted, noise);
-          noise.dispose();
-          currentCanvas.dispose();
-          currentCanvas = newCurrent;
-          const newReveal = ops.logicalOr(draftReveal, acceptance);
-          draftReveal.dispose();
-          draftReveal = newReveal;
-          const newDraft = ops.where(acceptance, accepted, draftCanvas);
-          draftCanvas.dispose();
-          draftCanvas = newDraft;
-          unrevealed.dispose();
-          revealOrAccept.dispose();
-          accepted.dispose();
-        }
-        acceptance.dispose();
-
-        // Early stop 1 (confidence sampler): all positions revealed.
-        if (sampler === "confidence-threshold") {
-          const allRevealed = ops.allReduce(draftReveal);
-          const done = ops.itemBool(allRevealed);
-          allRevealed.dispose();
-          if (done) {
+      try {
+        currentCanvas = ops.randint(0, vocab, [1, canvasLength], I32); // RNG: init
+        draftReveal = ops.zeros([1, canvasLength], Dtype.bool);
+        draftCanvas = ops.reshape(currentCanvas, [1, canvasLength]); // alias copy
+        argmaxCanvas = ops.reshape(currentCanvas, [1, canvasLength]);
+
+        for (let curStep = maxSteps; curStep >= 1; curStep--) {
+          yield; // safe point before the next denoising step
+          stepsThisCanvas++;
+          let logits: MlxArray | null = null;
+          let processed: MlxArray | null = null;
+          let acceptance: MlxArray | null = null;
+          let nextSc: MlxArray | null = null;
+          try {
+            logits = graph.decoderLogits(currentCanvas, cache, scEmbeddings);
+            const schedT = linearTemp(curStep, maxSteps, tMin, tMax);
+            // Match the reference EXACTLY: divide (not reciprocal-multiply) — the
+            // f32 rounding differs and the confidence threshold is a hard cutoff
+            // that flips acceptance on a 1-ULP difference, diverging the trajectory.
+            const schedArr = ops.scalarLike(schedT, logits);
+            processed = ops.div(logits, schedArr);
+            schedArr.dispose();
+            logits.dispose();
+
+            argmaxCanvas.dispose();
+            const argmax = ops.argmaxAxis(processed, -1);
+            argmaxCanvas = argmax.astype(I32); // [1, L]
+            argmax.dispose();
+
+            if (curStep === 1) {
+              processed.dispose();
+              break;
+            }
+
+            // temperature 0 → denoiser canvas is the argmax.
+            const denoiser = argmaxCanvas;
+
+            if (sampler === "entropy-bound") {
+              // cur_step > 1 always here (cur_step==1 breaks before the sampler):
+              // entropy AND the next soft embeddings come from the SAME entropy
+              // chain (reference _diffusion_entropy_and_soft_embeddings) — NOT the
+              // softmax-precise path the confidence sampler uses.
+              const { probs, entropy } = entropyProbsChain(processed);
+              acceptance = entropyTransferMask(entropy, entropyBound);
+              entropy.dispose();
+              nextSc = softEmbeddingsFromProbs(probs, dequantWeight, graph.embedScale);
+              probs.dispose();
+              // accepted/current update (entropy variant)
+              const accepted = ops.where(acceptance, denoiser, currentCanvas);
+              const noise = ops.randint(0, vocab, [1, canvasLength], I32); // RNG: re-noise
+              const newCurrent = ops.where(acceptance, accepted, noise);
+              noise.dispose();
+              currentCanvas.dispose();
+              currentCanvas = newCurrent;
+              draftReveal.dispose();
+              const notAcceptance = ops.logicalNot(acceptance);
+              draftReveal = ops.logicalNot(notAcceptance); // = acceptance
+              notAcceptance.dispose();
+              draftCanvas.dispose();
+              draftCanvas = ops.reshape(argmaxCanvas, [1, canvasLength]);
+              accepted.dispose();
+            } else {
+              const unrevealed = ops.logicalNot(draftReveal);
+              const confidence = tokenProbability(processed, denoiser);
+              acceptance = confidenceTransferMask(confidence, unrevealed, threshold, false);
+              confidence.dispose();
+              const accepted = ops.where(acceptance, denoiser, draftCanvas);
+              const revealOrAccept = ops.logicalOr(draftReveal, acceptance);
+              const noise = ops.randint(0, vocab, [1, canvasLength], I32); // RNG: re-noise
+              const newCurrent = ops.where(revealOrAccept, accepted, noise);
+              noise.dispose();
+              currentCanvas.dispose();
+              currentCanvas = newCurrent;
+              const newReveal = ops.logicalOr(draftReveal, acceptance);
+              draftReveal.dispose();
+              draftReveal = newReveal;
+              const newDraft = ops.where(acceptance, accepted, draftCanvas);
+              draftCanvas.dispose();
+              draftCanvas = newDraft;
+              unrevealed.dispose();
+              revealOrAccept.dispose();
+              accepted.dispose();
+            }
+            acceptance.dispose();
+
+            // Early stop 1 (confidence sampler): all positions revealed.
+            if (sampler === "confidence-threshold") {
+              const allRevealed = ops.allReduce(draftReveal);
+              const done = ops.itemBool(allRevealed);
+              allRevealed.dispose();
+              if (done) {
+                processed.dispose();
+                break;
+              }
+            }
+
+            // Early stop 2: stable + confident (only when explicitly configured;
+            // the as-loaded oracle has stopping_config None and never stops here).
+            if (
+              stableStop &&
+              stableAndConfident(argmaxCanvas, processed, history, stabilityThreshold, stopConfidence)
+            ) {
+              // Entropy sampling precomputes the next step's self-conditioning
+              // tensor before evaluating this stop. The break means ownership is
+              // never transferred to scEmbeddings, so release the pending value.
+              nextSc?.dispose();
+              processed.dispose();
+              break;
+            }
+
+            // Self-conditioning feedback for the next step.
+            if (curStep > 1) {
+              if (nextSc === null) nextSc = graph.softEmbeddings(processed, dequantWeight);
+              if (scEmbeddings) scEmbeddings.dispose();
+              scEmbeddings = nextSc;
+              nextSc = null; // ownership transferred to persistent feedback
+            }
             processed.dispose();
-            break;
+          } finally {
+            disposeResources([logits, processed, acceptance, nextSc]
+              .filter((array): array is MlxArray => array !== null));
           }
         }
 
-        // Early stop 2: stable + confident (only when explicitly configured;
-        // the as-loaded oracle has stopping_config None and never stops here).
-        if (
-          stableStop &&
-          stableAndConfident(argmaxCanvas, processed, history, stabilityThreshold, stopConfidence)
-        ) {
-          // Entropy sampling precomputes the next step's self-conditioning
-          // tensor before evaluating this stop. The break means ownership is
-          // never transferred to scEmbeddings, so release the pending value.
-          nextSc?.dispose();
-          processed.dispose();
-          break;
-        }
-
-        // Self-conditioning feedback for the next step.
-        if (curStep > 1) {
-          if (nextSc === null) nextSc = model.softEmbeddings(processed, dequantWeight);
-          if (scEmbeddings) scEmbeddings.dispose();
-          scEmbeddings = nextSc;
-        }
-        processed.dispose();
-      }
-
-      if (scEmbeddings) scEmbeddings.dispose();
-      for (const hh of history) hh.dispose();
-      currentCanvas.dispose();
-      draftReveal.dispose();
-      draftCanvas.dispose();
-
-      // Emit from the final argmax canvas.
-      const argmaxFloat = argmaxCanvas.astype(Dtype.float32);
-      const blockTokens = [...argmaxFloat.toFloat32()].map((x) => Math.round(x));
-      argmaxFloat.dispose();
-      argmaxCanvas.dispose();
-      blocks.push(blockTokens);
-      totalSteps += stepsThisCanvas;
-
-      for (const tok of blockTokens) {
-        generated++;
-        if (eosIds.has(tok)) {
-          stopped = true;
-          finishReason = "stop";
-          break;
-        }
-        emitted.push(tok);
-        if (generated >= maxNewTokens) {
-          stopped = true;
-          finishReason = "length";
-          break;
+        if (scEmbeddings) scEmbeddings.dispose();
+        for (const hh of history) hh.dispose();
+        currentCanvas.dispose();
+        draftReveal.dispose();
+        draftCanvas.dispose();
+
+        // Emit from the final argmax canvas.
+        const argmaxFloat = argmaxCanvas.astype(Dtype.float32);
+        let blockTokens: number[];
+        try { blockTokens = [...argmaxFloat.toFloat32()].map((x) => Math.round(x)); }
+        finally { argmaxFloat.dispose(); }
+        argmaxCanvas.dispose();
+        blocks.push(blockTokens);
+        totalSteps += stepsThisCanvas;
+
+        for (const tok of blockTokens) {
+          generated++;
+          if (eosIds.has(tok)) {
+            stopped = true;
+            finishReason = "stop";
+            break;
+          }
+          emitted.push(tok);
+          if (generated >= maxNewTokens) {
+            stopped = true;
+            finishReason = "length";
+            break;
+          }
         }
+        if (stopped) break;
+      } finally {
+        disposeResources([currentCanvas, draftReveal, draftCanvas, argmaxCanvas, scEmbeddings, ...history]
+          .filter((array): array is MlxArray => array !== null));
       }
-      if (stopped) break;
     }
+  } catch (error) {
+    failure = { error };
+    throw error;
   } finally {
-    dequantWeight.dispose();
-    for (const c of cache) c.dispose();
+    const cleanup = () => disposeResources([dequantWeight,
+      ...(stateOwner ? [{ dispose: () => stateOwner!.close() }] : [])]
+      .filter((resource): resource is DisposableResource => resource !== null));
+    if (failure) cleanupFailure(failure.error, cleanup);
+    else cleanup();
   }
 
   return { tokens: emitted, blocks, steps: totalSteps, finishReason };

package/src/embed.ts

@@ -2,7 +2,7 @@
 // L2-normalized vector) with the Qwen3-Embedding input convention so the CLI
 // (`mlx-bun embed`), the server (`/v1/embeddings`), and in-repo experiments all
 // produce the SAME vectors — the ones verified bit-exact vs mlx-lm in
-// tests/qwen3-embed-parity.test.ts.
+// tests/parity/qwen3-embed-parity.test.ts.
 
 import { Qwen3Model } from "./model/qwen3";
 import type { RuntimeModel } from "./model/factory";

package/src/engine/admission.ts

@@ -0,0 +1,59 @@
+import type { Cancellation } from "../contracts/generation";
+import type { DisposableResource } from "../contracts/resources";
+import { throwIfCancelled } from "./cancellation";
+
+export class AdmissionRejected extends Error {
+  constructor() { super("admission queue is full"); this.name = "AdmissionRejected"; }
+}
+
+/** FIFO reservations. A cancelled waiter releases no active owner's capacity. */
+export class AdmissionPool {
+  #active = 0;
+  #closed = false;
+  #waiting: Array<{ resolve(lease: DisposableResource): void; reject(error: unknown): void; unsubscribe(): void }> = [];
+  constructor(readonly capacity: number, readonly maxQueued = 64) {
+    if (!Number.isSafeInteger(capacity) || capacity < 1 || !Number.isSafeInteger(maxQueued) || maxQueued < 0)
+      throw new Error("invalid admission capacity");
+  }
+  get active(): number { return this.#active; }
+  get queued(): number { return this.#waiting.length; }
+
+  acquire(cancellation?: Cancellation): Promise<DisposableResource> {
+    if (this.#closed) return Promise.reject(new Error("admission is closed"));
+    try { if (cancellation) throwIfCancelled(cancellation); }
+    catch (error) { return Promise.reject(error); }
+    if (this.#active < this.capacity) return Promise.resolve(this.#lease());
+    if (this.#waiting.length >= this.maxQueued) return Promise.reject(new AdmissionRejected());
+    return new Promise((resolve, reject) => {
+      const waiter = { resolve, reject, unsubscribe() {} };
+      this.#waiting.push(waiter);
+      waiter.unsubscribe = cancellation?.subscribe((reason) => {
+        const index = this.#waiting.indexOf(waiter);
+        if (index < 0) return;
+        this.#waiting.splice(index, 1);
+        waiter.unsubscribe();
+        reject(new Error(`admission cancelled: ${reason}`));
+      }) ?? (() => {});
+      if (!this.#waiting.includes(waiter)) waiter.unsubscribe();
+    });
+  }
+
+  #lease(): DisposableResource {
+    this.#active++;
+    let released = false;
+    return { dispose: () => {
+      if (released) return;
+      released = true;
+      this.#active--;
+      const next = this.#waiting.shift();
+      if (next) { next.unsubscribe(); next.resolve(this.#lease()); }
+    } };
+  }
+
+  close(): void {
+    this.#closed = true;
+    for (const waiter of this.#waiting.splice(0)) {
+      waiter.unsubscribe(); waiter.reject(new Error("admission is closed"));
+    }
+  }
+}

package/src/engine/cancellation.ts

@@ -0,0 +1,38 @@
+import type { CancelReason, Cancellation } from "../contracts/generation";
+
+export class GenerationCancelled extends Error {
+  constructor(readonly reason: CancelReason) {
+    super(`generation cancelled: ${reason}`);
+    this.name = "GenerationCancelled";
+  }
+}
+
+export function throwIfCancelled(cancellation: Cancellation): void {
+  if (cancellation.reason !== undefined) throw new GenerationCancelled(cancellation.reason);
+}
+
+export class CancellationSource implements Cancellation {
+  #reason: CancelReason | undefined;
+  #listeners = new Set<(reason: CancelReason) => void>();
+  get reason(): CancelReason | undefined { return this.#reason; }
+
+  subscribe(listener: (reason: CancelReason) => void): () => void {
+    if (this.#reason !== undefined) {
+      listener(this.#reason);
+      return () => {};
+    }
+    this.#listeners.add(listener);
+    return () => { this.#listeners.delete(listener); };
+  }
+
+  cancel(reason: CancelReason): void {
+    if (this.#reason !== undefined) return;
+    this.#reason = reason;
+    const listeners = [...this.#listeners];
+    this.#listeners.clear();
+    // A faulty observer cannot prevent the remaining owners from cancelling.
+    for (const listener of listeners) {
+      try { listener(reason); } catch { /* Observers do not own settlement. */ }
+    }
+  }
+}

package/src/engine/engine.ts

@@ -0,0 +1,56 @@
+import type { ExecutionPlanner, GenerationSession, InferenceEngine, RunControl, Timer } from "../contracts/generation";
+import { EngineSession, type SessionResources } from "./session";
+
+export interface EngineOptions {
+  readonly timer: Timer;
+  /** Deadline for an unread stream or a producer blocked by an idle consumer. */
+  readonly idleTimeoutMs?: number;
+  readonly maxQueuedEvents?: number;
+  readonly maxQueuedTokens?: number;
+  readonly maxTopLogprobs?: number;
+  /** Aggregate output buffers for active collecting sessions, in tokens (4 bytes each).
+   * Once terminal, output storage belongs to the caller. */
+  readonly maxCollectTokens?: number;
+}
+
+export function createInferenceEngine<Request, Metrics>(
+  planner: ExecutionPlanner<Request, Metrics>, options: EngineOptions,
+): InferenceEngine<Request, Metrics> {
+  const idleTimeoutMs = options.idleTimeoutMs ?? 30_000;
+  const maxQueuedEvents = options.maxQueuedEvents ?? 32;
+  const maxQueuedTokens = options.maxQueuedTokens ?? 256;
+  const maxCollectTokens = options.maxCollectTokens ?? 65_536;
+  const maxTopLogprobs = options.maxTopLogprobs ?? 20;
+  for (const [name, value] of Object.entries({ idleTimeoutMs, maxQueuedEvents, maxQueuedTokens, maxCollectTokens, maxTopLogprobs }))
+    if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive safe integer`);
+  const sessions = new Set<GenerationSession<Metrics>>();
+  let closed = false;
+  let closing: Promise<void> | undefined;
+  let nextId = 0;
+  let reserved = 0;
+  const resources: SessionResources = {
+    timer: options.timer, idleTimeoutMs, maxQueuedEvents, maxQueuedTokens, maxTopLogprobs,
+    reserveOutput(tokens) {
+      if (tokens > maxCollectTokens - reserved) throw new Error("collection output capacity exceeded");
+      const buffer = new Uint32Array(tokens);
+      reserved += tokens;
+      let released = false;
+      return { buffer, release() { if (!released) { released = true; reserved -= tokens; } } };
+    },
+  };
+  return {
+    async open(request: Request, control: RunControl) {
+      if (closed) throw new Error("inference engine is closed");
+      let session: EngineSession<Metrics> | undefined;
+      session = new EngineSession(String(++nextId), (cancellation) => planner.plan(request, cancellation),
+        { ...control }, resources,
+        () => { if (session) sessions.delete(session); });
+      if (session.state !== "terminal") sessions.add(session);
+      return session;
+    },
+    close() {
+      closed = true;
+      return closing ??= Promise.all([...sessions].map((session) => session.cancel("engine_closed"))).then(() => {});
+    },
+  };
+}

package/src/engine/execution-coordinator.ts

@@ -0,0 +1,70 @@
+import type { Cancellation } from "../contracts/generation";
+import type { DisposableResource } from "../contracts/resources";
+import { AdmissionRejected } from "./admission";
+import { throwIfCancelled } from "./cancellation";
+
+type Mode = "shared" | "exclusive";
+interface Waiter {
+  mode: Mode;
+  resolve(lease: DisposableResource): void;
+  reject(error: unknown): void;
+  unsubscribe(): void;
+}
+
+/** Shared worker activity; exclusive managed jobs. FIFO writers cannot starve
+ * behind newly arriving inference. This coordinates execution, not memory size. */
+export class ExecutionCoordinator {
+  #readers = 0;
+  #writer = false;
+  #closed = false;
+  #queue: Waiter[] = [];
+  constructor(readonly maxQueued = 64) {
+    if (!Number.isSafeInteger(maxQueued) || maxQueued < 0) throw new Error("invalid coordination queue bound");
+  }
+  acquire(mode: Mode, cancellation?: Cancellation): Promise<DisposableResource> {
+    if (this.#closed) return Promise.reject(new Error("execution coordinator is closed"));
+    try { if (cancellation) throwIfCancelled(cancellation); }
+    catch (error) { return Promise.reject(error); }
+    if (!this.#queue.length && !this.#writer && (mode === "shared" || this.#readers === 0))
+      return Promise.resolve(this.#lease(mode));
+    if (this.#queue.length >= this.maxQueued) return Promise.reject(new AdmissionRejected());
+    return new Promise((resolve, reject) => {
+      const waiter: Waiter = { mode, resolve, reject, unsubscribe() {} };
+      this.#queue.push(waiter);
+      waiter.unsubscribe = cancellation?.subscribe((reason) => {
+        const index = this.#queue.indexOf(waiter);
+        if (index < 0) return;
+        this.#queue.splice(index, 1);
+        waiter.unsubscribe();
+        reject(new Error(`execution admission cancelled: ${reason}`));
+        this.#drain();
+      }) ?? (() => {});
+      if (!this.#queue.includes(waiter)) waiter.unsubscribe();
+    });
+  }
+  #lease(mode: Mode): DisposableResource {
+    if (mode === "shared") this.#readers++;
+    else this.#writer = true;
+    let disposed = false;
+    return { dispose: () => {
+      if (disposed) return;
+      disposed = true;
+      if (mode === "shared") this.#readers--;
+      else this.#writer = false;
+      this.#drain();
+    } };
+  }
+  #drain(): void {
+    while (!this.#writer && this.#queue.length) {
+      const next = this.#queue[0]!;
+      if (next.mode === "exclusive" && this.#readers > 0) return;
+      this.#queue.shift(); next.unsubscribe(); next.resolve(this.#lease(next.mode));
+    }
+  }
+  close(): void {
+    this.#closed = true;
+    for (const next of this.#queue.splice(0)) {
+      next.unsubscribe(); next.reject(new Error("execution coordinator is closed"));
+    }
+  }
+}