mlx-bun 0.4.0 → 0.5.0

package/src/cli.ts

@@ -74,6 +74,8 @@ Commands:
   generate   One-shot raw text generation from a local model (no server)
   benchmark  Measure decode/prefill speed of OUR stack on this machine
   embed      Text embeddings from a local embedding model (Qwen3-Embedding)
+  transcribe Speech-to-text from an audio file with a local Whisper model
+  dictate    Push-to-talk: stream your microphone into Whisper, print/copy/type the text
   evals      Show recent benchmark runs (all stacks)
   help       Show help for a command (also: mlx-bun <command> --help)
 
@@ -164,6 +166,19 @@ Model & quality:
                             request's \`adapter\` field, incl. "none", wins).
                             --adapter-path is accepted as the mlx_lm.server
                             alias. Hot-swap via POST /v1/adapters unchanged.
+  --whisper-model <p|query> Whisper checkpoint for /v1/audio/transcriptions
+                            [default: the first downloaded whisper model,
+                            resolved on the first request]. Serving a whisper
+                            model ALONE starts a transcription-only server.
+  --preload                 Transcription-only server: load the Whisper
+                            weights before listening instead of on the first
+                            request.
+  --whisper-idle-unload <s> Keep the Whisper weights for this many idle seconds
+                            after a take before disposing them. [default: 0 —
+                            released right after every take, so the chat model
+                            has the memory back before it prefills; a cold
+                            take costs ~150 ms more than a warm one]
+  --whisper-resident        Never dispose the Whisper weights (sotto-style).
   --draft-model <query>     Speculative decoding: a drafter proposes tokens
                             the main model verifies (exact results, faster
                             decode when drafts land). The artifact's kind is
@@ -268,6 +283,74 @@ Kill switches (bit-exact A/B levers — each selects a slower same-parity path):
                             bit-exact with the resident path`;
 
 const HELP: Record<string, string> = {
+  dictate: `mlx-bun dictate — push-to-talk dictation from your microphone
+
+Usage: mlx-bun dictate [query] [options]
+
+Captures the mic through the AVAudioEngine sidecar (16 kHz mono; macOS asks
+for Microphone permission on first use) and streams it into a Whisper
+transcription session while you speak — every finished 30 s window is
+transcribed during capture, so the result lands about one window after you
+stop. Silence never runs Whisper (Silero VAD).
+
+Modes:
+  (default)              Press Enter to start a take, Enter again to stop;
+                         q + Enter or Ctrl-C quits
+  --hotkey [keycode]     Hold a key instead (default 61 = Right Option, like
+                         sotto); needs Input Monitoring for your terminal
+Backend:
+  [query] / --model <p>  Whisper model for in-process transcription (default:
+                         the first downloaded whisper checkpoint)
+  --server <url>         Use a running mlx-bun server's /v1/audio/sessions
+                         instead of loading the model here
+  --idle-unload <s>      Release the weights after this many idle seconds
+                         (0 = right after each take; default 30); --resident
+                         keeps them loaded
+Decoding:
+  --language <code>      default en; "auto" detects
+  --beam-size <n>        beam search (default greedy)
+  --prompt "…"           initial prompt / --vocabulary "a,b,c" hint terms
+  --no-vad               skip the Silero gate
+Output (any combination; default prints):
+  --copy                 copy the transcript to the clipboard (pbcopy)
+  --type                 type it into the frontmost app via System Events
+                         keystrokes (needs Accessibility for your terminal);
+                         --type-delay <s> waits first so you can Cmd-Tab
+                         (default 1 in Enter mode, 0 with --hotkey)`,
+  transcribe: `mlx-bun transcribe — speech-to-text with a local Whisper model (no server)
+
+Usage: mlx-bun transcribe <audio-file> [query] [options]
+
+Loads a Whisper checkpoint (mlx-community/whisper-*) and prints the
+transcript. WAV is decoded natively; any other container CoreAudio reads
+(mp3/m4a/flac/ogg/aiff/…) is transcoded with afconvert first.
+
+Model:
+  [query] / --model <p>  Whisper model; omitted = the first downloaded
+                         \`whisper\` checkpoint (errors with an mlx-bun get
+                         suggestion when none)
+Decoding:
+  --language <code>      ISO code or name ("en", "japanese"); "auto" or omitted
+                         detects the language from the first 30 s
+  --task translate       Translate to English instead of transcribing
+  --beam-size <n>        Beam search width (default: greedy)
+  --temperature <t>      Sampling temperature; the default runs the
+                         (0, 0.2, …, 1.0) fallback ladder like mlx-whisper
+  --no-fallback          Temperature 0 only, no retry ladder
+  --prompt "…"           Initial prompt (vocabulary hints / style)
+  --no-timestamps        Decode text only (<|notimestamps|>)
+  --no-condition         Do not condition each window on the previous text
+  --vad                  Silero VAD gate: skip Whisper when no speech is
+                         detected (weights: ggml-org/whisper-vad in the HF cache
+                         or --vad-model <path>); --vad-trim crops to speech
+  --word-timestamps      Word-level timestamps (verbose_json / verbose output)
+  --faithful             Run the oracle-parity graph instead of the fast path
+  --audio-ctx <n>        Lab: encode only n of the 1500 encoder positions
+                         (whisper.cpp -ac); cheaper for short clips, degrades
+                         below ~1024 — off by default
+Output:
+  --format <f>           text (default) · json · verbose_json · srt · vtt
+  --verbose              Print each segment with its time range as it decodes`,
   pi: `mlx-bun pi — drop into a coding-agent session on a local model
 
 Usage: mlx-bun pi [options] [message...]
@@ -1023,6 +1106,17 @@ function serverRuntimeFlags(): { port: number; serverOptions: import("./server")
   // `mlx-bun pi` attach both go through localhost, so loopback-only is
   // transparent to them.
   serverOptions.hostname = opt("host", "127.0.0.1")!;
+  // Speech-to-text companion: explicit checkpoint + residency policy.
+  const whisperIdleRaw = opt("whisper-idle-unload");
+  if (whisperIdleRaw !== null) {
+    const n = Number(whisperIdleRaw);
+    if (!Number.isFinite(n) || n < 0) {
+      console.error(`--whisper-idle-unload expects seconds >= 0 (got "${whisperIdleRaw}")`);
+      process.exit(1);
+    }
+    serverOptions.whisperIdleUnloadSec = n;
+  }
+  if (flag("whisper-resident")) serverOptions.whisperResident = true;
   // Server-wide default for the chat template's enable_thinking variable
   // (CPM/MiniCPM5 and other hybrid-reasoning models). Unset ⇒ the model's
   // own default (false for MiniCPM5). onOff accepts true|false|on|off|1|0.
@@ -1637,6 +1731,40 @@ switch (cmd) {
     // straight from that path; anything else resolves like the positional
     // fuzzy query. Precedence: --model > positional > --query > auto-pick.
     const { m, picked } = await resolveModelAuto(opt("model") ?? positional(0) ?? opt("query"));
+    if (m.modelType === "whisper") {
+      // Transcription-only server: no chat model, weights paged in on demand.
+      const sNative0 = step("native runtime");
+      await ensureNative(sNative0);
+      sNative0.done("native runtime ready");
+      const { createTranscriptionServer } = await import("./serve/transcription-server");
+      const { server, service } = await createTranscriptionServer({
+        modelDir: m.path, modelId: m.repoId, port: rt.port,
+        ...(rt.serverOptions.hostname ? { hostname: rt.serverOptions.hostname } : {}),
+        ...(rt.serverOptions.unixSocket ? { unixSocket: rt.serverOptions.unixSocket } : {}),
+        idleUnloadSec: rt.serverOptions.whisperIdleUnloadSec,
+        resident: rt.serverOptions.whisperResident,
+        preload: flag("preload"),
+      });
+      const where = rt.serverOptions.unixSocket ?? `http://${rt.serverOptions.hostname === "0.0.0.0" ? "localhost" : rt.serverOptions.hostname}:${server.port}`;
+      box([
+        `${style.bold(m.repoId)} ${style.dim("· transcription-only server")}`,
+        `POST ${where}/v1/audio/transcriptions ${style.dim(`· ${service.stats.idle_unload_sec === null ? "always resident" : service.idleUnloadSec === 0 ? "released after every take" : `idle unload ${service.idleUnloadSec}s`}${service.resident ? " · loaded" : " · loads on first request"}`)}`,
+      ]);
+      const stop = () => { service.dispose(); server.stop(true); process.exit(0); };
+      process.once("SIGINT", stop);
+      process.once("SIGTERM", stop);
+      break;
+    }
+    const whisperQuery = opt("whisper-model");
+    if (whisperQuery !== null) {
+      const { m: wm } = await resolveModelAuto(whisperQuery);
+      if (wm.modelType !== "whisper") {
+        console.error(`--whisper-model ${whisperQuery} resolved to ${wm.repoId} (model_type ${wm.modelType}), not a Whisper checkpoint`);
+        process.exit(1);
+      }
+      rt.serverOptions.whisperModelDir = wm.path;
+      rt.serverOptions.whisperModelId = wm.repoId;
+    }
     const sFit = step("assessing fit");
     const config = await loadModelConfig(m.path);
     const report = fit(config, m.sizeBytes, 8192, undefined, undefined, m.expertsBytes);
@@ -1883,6 +2011,200 @@ switch (cmd) {
     break;
   }
 
+  case "transcribe": {
+    // Speech-to-text, one-shot, no server: audio file → Whisper → text.
+    const { openWhisperModel } = await import("./model/factory");
+    const { loadWhisperTokenizer } = await import("./audio/whisper-tokenizer");
+    const { WhisperTranscriber } = await import("./audio/whisper-transcribe");
+    const { formatTranscription, TRANSCRIPTION_FORMATS } = await import("./audio/whisper-format");
+    const { TranscriptionService } = await import("./serve/transcription-service");
+    const { formatTimestamp } = await import("./audio/whisper-format");
+
+    const audioPath = positional(0);
+    if (!audioPath) {
+      console.error("usage: mlx-bun transcribe <audio-file> [query] [--language en] [--format text|json|verbose_json|srt|vtt]");
+      process.exit(1);
+    }
+    const format = (opt("format") ?? "text") as import("./audio/whisper-format").TranscriptionFormat;
+    if (!TRANSCRIPTION_FORMATS.includes(format)) {
+      console.error(`--format must be one of ${TRANSCRIPTION_FORMATS.join(", ")}`);
+      process.exit(1);
+    }
+    const bytes = new Uint8Array(await Bun.file(audioPath).arrayBuffer());
+    const samples = await TranscriptionService.decodeAudio(bytes);
+    const durationSeconds = samples.length / 16_000;
+
+    const modelQuery = opt("model") ?? positional(1) ?? opt("query");
+    let modelDir: string;
+    if (modelQuery && existsSync(join(modelQuery, "config.json"))) modelDir = modelQuery;
+    else if (modelQuery) modelDir = (await resolveModelAuto(modelQuery)).m.path;
+    else {
+      const reg = new Registry();
+      if (reg.list().length === 0) await reg.scan();
+      const w = reg.list().find((r) => r.modelType === "whisper");
+      if (!w) {
+        console.error("no Whisper model downloaded — try: mlx-bun get mlx-community/whisper-large-v3-turbo");
+        process.exit(1);
+      }
+      modelDir = w.path;
+    }
+    let vadGate: { segments: { start: number; end: number }[] } | null = null;
+    if (flag("vad")) {
+      const { SileroVad } = await import("./audio/silero-vad");
+      const tv = performance.now();
+      const vad = SileroVad.load(opt("vad-model"));
+      const { segments } = vad.detect(samples, { threshold: Number(opt("vad-threshold") ?? 0.5) });
+      vadGate = { segments };
+      if (flag("verbose")) console.error(`vad: ${segments.length} speech segment(s) in ${(performance.now() - tv).toFixed(0)} ms`);
+      if (segments.length === 0) { process.stdout.write(format === "text" ? "\n" : JSON.stringify({ text: "", vad: { speech: false, segments: [] } }) + "\n"); break; }
+    }
+    const { model } = await openWhisperModel(modelDir);
+    const tok = await loadWhisperTokenizer(modelDir, model.dims.nVocab);
+    const transcriber = new WhisperTranscriber(model, tok);
+    void vadGate;
+    const langOpt = opt("language");
+    const task = opt("task") === "translate" ? "translate" as const : "transcribe" as const;
+    const beam = opt("beam-size");
+    const temperature = opt("temperature");
+    const verbose = flag("verbose");
+    const t0 = performance.now();
+    const result = await transcriber.transcribe(samples, {
+      language: !langOpt || langOpt === "auto" ? null : langOpt,
+      task,
+      beamSize: beam ? Number(beam) : null,
+      ...(temperature ? { temperature: Number(temperature) } : flag("no-fallback") ? { temperature: 0 } : {}),
+      initialPrompt: opt("prompt"),
+      withoutTimestamps: flag("no-timestamps"),
+      conditionOnPreviousText: !flag("no-condition"),
+      fast: !flag("faithful"),
+      wordTimestamps: flag("word-timestamps"),
+      audioCtx: opt("audio-ctx") ? Number(opt("audio-ctx")) : null,
+      onSegment: verbose
+        ? (s) => console.error(`[${formatTimestamp(s.start, ".")} --> ${formatTimestamp(s.end, ".")}] ${s.text.trim()}`)
+        : undefined,
+    });
+    const elapsed = (performance.now() - t0) / 1000;
+    if (verbose)
+      console.error(`${durationSeconds.toFixed(2)} s audio in ${elapsed.toFixed(3)} s (${(durationSeconds / elapsed).toFixed(1)}× realtime), language ${result.language}`);
+    process.stdout.write(formatTranscription(result, format, durationSeconds, task));
+    transcriber.dispose();
+    model.dispose();
+    break;
+  }
+
+  case "dictate": {
+    const { startMicCapture } = await import("./audio/mic-capture");
+    const { TranscriptionService, defaultWhisperRecord } = await import("./serve/transcription-service");
+    const { style } = await import("./tui");
+    const serverUrl = opt("server");
+    const hotkeyFlag = argv.includes("--hotkey");
+    const hotkey = hotkeyFlag ? Number(opt("hotkey") ?? 61) || 61 : null;
+    const language = opt("language") ?? "en";
+    const beam = opt("beam-size");
+    const vocabulary = opt("vocabulary")?.split(",").map((s) => s.trim()).filter(Boolean);
+    const params = {
+      language: language === "auto" ? null : language,
+      beamSize: beam ? Number(beam) : null,
+      temperature: 0,
+      prompt: opt("prompt"),
+      vocabulary,
+      vad: flag("no-vad") ? null : { threshold: 0.5, minSpeechMs: 120 },
+    };
+    const typeDelay = Number(opt("type-delay") ?? (hotkey ? 0 : 1));
+
+    // backend: in-process service or a running server's sessions
+    let service: InstanceType<typeof TranscriptionService> | null = null;
+    if (!serverUrl) {
+      const q = opt("model") ?? positional(0) ?? opt("query");
+      let modelDir: string; let modelId: string;
+      if (q && existsSync(join(q, "config.json"))) { modelDir = q; modelId = q; }
+      else if (q) { const { m } = await resolveModelAuto(q); modelDir = m.path; modelId = m.repoId; }
+      else {
+        const rec = await defaultWhisperRecord();
+        if (!rec) { console.error("no Whisper model downloaded — try: mlx-bun get mlx-community/whisper-large-v3-turbo"); process.exit(1); }
+        modelDir = rec.path; modelId = rec.repoId;
+      }
+      const idle = opt("idle-unload");
+      service = new TranscriptionService({ modelDir, modelId, idleUnloadSec: idle !== null ? Number(idle) : 30, resident: flag("resident"), log: () => {} });
+      process.stderr.write(style.dim("loading whisper… "));
+      await service.ensureLoaded();
+      service.vad();
+      process.stderr.write(style.dim("ready\n"));
+    }
+    const base = serverUrl?.replace(/\/$/, "");
+    interface Take { feed(pcm: Float32Array): Promise<void>; finish(): Promise<{ text: string; ms: number }>; }
+    const startTake = async (): Promise<Take> => {
+      if (service) {
+        const session = await service.createSession(params);
+        return {
+          feed: async (pcm) => { await session.append(pcm); },
+          finish: async () => { const t0 = performance.now(); const o = await session.finish(); return { text: o.result.text.trim(), ms: performance.now() - t0 }; },
+        };
+      }
+      const r = await fetch(`${base}/v1/audio/sessions`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ language: params.language, beam_size: params.beamSize, temperature: 0, prompt: params.prompt, vocabulary, vad: !!params.vad, vad_min_speech_ms: 120 }) });
+      if (!r.ok) throw new Error(`session create failed: ${r.status} ${await r.text()}`);
+      const { id } = await r.json() as { id: string };
+      return {
+        feed: async (pcm) => { await fetch(`${base}/v1/audio/sessions/${id}/audio`, { method: "POST", headers: { "content-type": "audio/pcm;rate=16000" }, body: new Uint8Array(pcm.buffer.slice(pcm.byteOffset, pcm.byteOffset + pcm.byteLength) as ArrayBuffer) }); },
+        finish: async () => { const t0 = performance.now(); const f = await fetch(`${base}/v1/audio/sessions/${id}/finish`, { method: "POST" }); const b = await f.json() as { text: string }; return { text: (b.text ?? "").trim(), ms: performance.now() - t0 }; },
+      };
+    };
+
+    const deliver = async (text: string) => {
+      if (!text) { console.log(style.dim("(no speech)")); return; }
+      console.log(text);
+      if (flag("copy")) { const p = Bun.spawn(["pbcopy"], { stdin: "pipe" }); p.stdin.write(text); p.stdin.end(); await p.exited; }
+      if (flag("type")) {
+        if (typeDelay > 0) await Bun.sleep(typeDelay * 1000);
+        const escaped = text.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
+        const r = Bun.spawnSync(["osascript", "-e", `tell application "System Events" to keystroke "${escaped}"`], { stderr: "pipe" });
+        if (r.exitCode !== 0) console.error(style.dim(`typing failed (grant Accessibility to your terminal): ${r.stderr.toString().trim()}`));
+      }
+    };
+
+    const mic = await startMicCapture({ hotkey });
+    let take: Take | null = null;
+    let pending: Float32Array[] = []; let pendingLen = 0;
+    let feeding: Promise<void> = Promise.resolve();
+    const flush = () => {
+      if (!take || pendingLen === 0) return;
+      const buf = new Float32Array(pendingLen); let o = 0; for (const p of pending) { buf.set(p, o); o += p.length; }
+      pending = []; pendingLen = 0;
+      const t = take;
+      feeding = feeding.then(() => t.feed(buf)).catch((e) => console.error(style.dim(`feed failed: ${(e as Error).message}`)));
+    };
+    const begin = async () => { if (take) return; take = await startTake(); pending = []; pendingLen = 0; process.stderr.write(style.bold("● recording") + style.dim(hotkey ? " (release to stop)\n" : " (Enter to stop)\n")); };
+    const end = async () => {
+      if (!take) return;
+      flush();
+      await feeding;
+      const t = take; take = null;
+      process.stderr.write(style.dim("transcribing… "));
+      try { const { text, ms } = await t.finish(); process.stderr.write(style.dim(`${ms.toFixed(0)} ms\n`)); await deliver(text); }
+      catch (e) { console.error(`transcription failed: ${(e as Error).message}`); }
+    };
+    if (!hotkey) {
+      console.error(style.dim("Enter: start/stop a take · q: quit"));
+      (async () => {
+        for await (const line of console) {
+          if (line.trim() === "q") { mic.stop(); process.exit(0); }
+          if (take) await end(); else await begin();
+        }
+      })();
+    }
+    process.once("SIGINT", () => { mic.stop(); process.exit(0); });
+    for await (const ev of mic.events) {
+      if (ev.kind === "ready") { if (hotkey) console.error(style.dim(`hold key ${hotkey} to talk · Ctrl-C quits · mic ${ev.info}`)); }
+      else if (ev.kind === "error") { console.error(ev.message); mic.stop(); process.exit(1); }
+      else if (ev.kind === "hotkey") { if (ev.down) await begin(); else await end(); }
+      else if (ev.kind === "pcm" && take) {
+        pending.push(ev.samples); pendingLen += ev.samples.length;
+        if (pendingLen >= 16_000 / 4) flush(); // every 250 ms of audio
+      }
+    }
+    break;
+  }
+
   case "bench":
   case "benchmark": {
     const { banner, step, box, style } = await import("./tui");

package/src/contracts/execution.ts

@@ -1,6 +1,7 @@
 /** Request facts only. Native resources remain with the preparation owner. */
 export interface ExecutionRequirements {
   readonly hasVision: boolean;
+  readonly hasPreparedPrefixIdentity?: boolean;
   readonly hasAdapters: boolean;
   readonly hasRepetitionPenalty: boolean;
   readonly userSeed: boolean;
@@ -18,6 +19,9 @@ export interface ExecutionCapabilities {
   readonly quantizedBatch: boolean;
   readonly grammarBatch: boolean;
   readonly adapterBatch?: boolean;
+  /** Prepared media can enter the ordinary shared execution group. */
+  readonly mediaBatch?: boolean;
+  readonly mediaPrefixCache?: boolean;
   readonly pagedBatch?: boolean;
   readonly checkpoints: boolean;
   /** Bound ordinary driver can restore/capture this request configuration. */
@@ -31,6 +35,12 @@ export interface ExecutionCapabilities {
   readonly sharedSpeculativeAdapters?: boolean;
   /** Methods supplied by the model's shared execution binding. */
   readonly groupedMethods?: readonly string[];
+  /** A shared method accepts request-owned grammar continuation proposals. */
+  readonly sharedGrammarProposals?: boolean;
+  /** A grouped method can consume this request's strict known continuations. */
+  readonly sharedFill?: boolean;
+  /** A mounted provider can consume verified external continuations. */
+  readonly sharedSpeculativeEcho?: boolean;
   /** A graph-owned compiled step exists; cache geometry can still decline it. */
   readonly compiledDecode?: boolean;
 }

package/src/contracts/object-cache.ts

@@ -0,0 +1,8 @@
+/** Exact reusable work, independent of token-prefix matching. The same cache
+ * owner chooses residency and persistence for objects and conversation state. */
+export interface ObjectCache<Value> {
+  /** Borrow immutable data through an owned lease. A miss permits recompute. */
+  take(key: string): Promise<{ value: Value; dispose(): void } | null>;
+  /** Successful return transfers ownership, including immediate eviction. */
+  put(key: string, value: Value): void;
+}

package/src/contracts/pi-provider.ts

@@ -13,7 +13,7 @@ export function piModelDefinition(options: PiModelOptions = {}) {
     name: options.name ?? "mlx-bun (local)",
     api: "openai-completions" as const,
     reasoning: options.reasoning ?? false,
-    compat: { supportsDeveloperRole: false,
+    compat: { supportsDeveloperRole: false, sendSessionAffinityHeaders: true,
       ...(options.reasoning ? { thinkingFormat: "qwen-chat-template" as const } : {}) },
     input: (options.vision ? ["text", "image"] : ["text"]) as ("text" | "image")[],
     cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },

package/src/contracts/pi.ts

@@ -160,7 +160,9 @@ export type ServerMessage =
   // provider declaration — clients use it instead of probing for a 400.
   // `thinking`: whether the model has a switchable reasoning channel (drives
   // the UI's thinking on/off toggle; false hides it).
-  | { type: "ready"; model: string; vision: boolean; audio: boolean; thinking: boolean; genDefaults: ReadyGenDefaults }
+  // transcription: a Whisper checkpoint is configured or resolvable for
+  // /v1/audio/sessions — drives the composer's hold-to-talk mic button.
+  | { type: "ready"; model: string; vision: boolean; audio: boolean; thinking: boolean; genDefaults: ReadyGenDefaults; transcription?: boolean }
   | { type: "turn_start" }
   | { type: "text_delta"; delta: string }
   | { type: "thinking_delta"; delta: string }

package/src/contracts/prefix-cache.ts

@@ -13,11 +13,21 @@ export interface PrefixCacheHit<State, Attachment = never> {
  * State is opaque here; its backend/method defines layout and alignment.
  * Implementations own prefix selection, retention and storage tiers. */
 export interface PrefixCache<State, Attachment = never> {
-  /** Return owned state for the longest usable prefix, or null on a miss.
+  /** Give optional retained state back to the allocator at an execution
+   * boundary. This never rejects or changes the active request. */
+  reclaim?(): void;
+  /** Drop session affinity without deleting reusable checkpoints. */
+  closeSession?(sessionId: string): void;
+  /** Prepare a reusable prefix before execution. The returned release ends
+   * request interest; cache residency and IO remain implementation-owned. */
+  prefetch?(prompt: number[], ns?: string, sessionId?: string): Promise<() => void>;
+  /** Return owned state for a usable session continuation, otherwise the
+   * longest usable prefix, or null on a miss. Session identity is an affinity
+   * hint and never changes the content or numerical namespace.
    * Namespace distinguishes incompatible artifacts, methods and adapters. */
-  take(prompt: number[], ns?: string): PrefixCacheHit<State, Attachment> | null;
+  take(prompt: number[], ns?: string, sessionId?: string): PrefixCacheHit<State, Attachment> | null;
   /** Transfer state and its optional backing release to storage. If this
    * throws, ownership stays with the caller; successful return transfers
    * ownership even when storage immediately spills or disposes the entry. */
-  put(tokens: number[], caches: State, ns?: string, retain?: () => void, attachments?: Attachment): void;
+  put(tokens: number[], caches: State, ns?: string, retain?: () => void, attachments?: Attachment, sessionId?: string): void;
 }

package/src/contracts/scheduling.ts

@@ -9,6 +9,10 @@ export interface ExecutionContext {
 /** A backend execution group exposes bounded work units and readiness. Queue
  * payloads, tensors, cache merging and sampling remain inside its adapter. */
 export interface ExecutionGroup {
+  /** Native preparation may borrow the execution lease between work units.
+   * Payloads and task failures belong to the backend's queue. */
+  readonly pendingTasks?: number;
+  advanceTask?(): Promise<void>;
   readonly active: number;
   readonly queued: number;
   readonly preparing: boolean;
@@ -30,6 +34,11 @@ export interface ExecutionGroup {
   admitNext(): boolean;
   /** Read-only budget check used to preserve admission before decode. */
   canBurst(): boolean;
+  /** Method-owned token demand for a compatible mixed iteration. Absent when
+   * the backend has no packed implementation for the current work. */
+  readonly mixedPreparation?: { readonly runningTokens: number; readonly minimumPreparationTokens: number };
+  readonly maxIterationTokens?: number;
+  advanceMixed?(tokenBudget: number): Promise<void>;
   advancePreparation(): Promise<void>;
   advance(): Promise<void>;
   failActive(error: unknown): void;

package/src/engine/execution-plan.ts

@@ -8,7 +8,7 @@ export function resolveExecution(
   const reasons: string[] = [];
   const sharedRequestExclusions = [
     [!capabilities.continuous, "continuous-unavailable"],
-    [request.hasVision, "media-requires-serial"],
+    [request.hasVision && !capabilities.mediaBatch, "media-requires-serial"],
     [request.hasAdapters && !capabilities.adapterBatch, "adapters-require-serial"],
     [request.kvQuant && !capabilities.quantizedBatch, "kv-scheme-requires-serial"],
     [request.turboQuant && !capabilities.turboQuantBatch, "turbo-kv-requires-serial"],
@@ -18,10 +18,16 @@ export function resolveExecution(
   const sharedAdapterMethod = capabilities.sharedSpeculativeAdapters === true &&
     !sharedRequestExclusions.some(([excluded]) => excluded) &&
     capabilities.groupedMethods?.includes("speculative") === true;
-  const speculative = capabilities.method === "autoregressive" && request.hasDraft &&
+  const grammarProposals = features.grammarJump === true && request.hasGrammar && !request.hasDraft && !request.hasVision &&
+    (!request.wantsLogprobs || capabilities.speculativeLogprobs === true) && !features.pagedKv && capabilities.method === "autoregressive" &&
+    capabilities.sharedGrammarProposals === true && capabilities.groupedMethods?.includes("speculative") === true &&
+    (!request.hasAdapters || sharedAdapterMethod) && (!request.kvQuant || capabilities.speculativeKvQuant === true) &&
+    (!request.turboQuant || capabilities.speculativeTurboQuant === true) &&
+    !sharedRequestExclusions.some(([excluded]) => excluded);
+  const speculative = grammarProposals || (capabilities.method === "autoregressive" && request.hasDraft &&
     !request.hasVision && (!request.hasAdapters || sharedAdapterMethod) && (!request.wantsLogprobs || capabilities.speculativeLogprobs === true) &&
     (!request.kvQuant || capabilities.speculativeKvQuant === true) &&
-    (!request.turboQuant || capabilities.speculativeTurboQuant === true) && !features.pagedKv;
+    (!request.turboQuant || capabilities.speculativeTurboQuant === true) && !features.pagedKv);
   const method = speculative ? "speculative" : capabilities.method;
   const continuousExclusions = [sharedRequestExclusions[0],
     [!(capabilities.groupedMethods ?? ["autoregressive"]).includes(method), "method-requires-serial"],
@@ -32,19 +38,24 @@ export function resolveExecution(
   if (request.hasDraft && !speculative) reasons.push("draft-incompatible-with-request");
   const pagedKv = features.pagedKv && !request.hasVision && !request.hasAdapters;
   if (features.pagedKv && !pagedKv) reasons.push("paged-kv-bypassed-for-media-or-adapters");
-  const promptCache = method !== "speculative" && !request.hasVision && !pagedKv;
-  const fill = features.fill && method === "autoregressive" && mechanism === "serial" &&
-    !request.hasDraft && !request.hasVision && !request.userSeed && !request.hasGrammar &&
-    !request.wantsLogprobs && !request.kvQuant && !request.turboQuant;
+  const promptCache = method !== "speculative" && (!request.hasVision ||
+    (mechanism === "continuous" && request.hasPreparedPrefixIdentity === true && capabilities.mediaPrefixCache === true));
+  const sharedFill = mechanism === "continuous" && capabilities.sharedFill === true && !pagedKv;
+  const speculativeEcho = method === "speculative" && mechanism === "continuous" &&
+    capabilities.sharedSpeculativeEcho === true;
+  const fill = features.fill && !request.hasVision && !request.hasGrammar && !request.wantsLogprobs &&
+    (speculativeEcho || (method === "autoregressive" && (mechanism === "serial" || sharedFill) &&
+      !request.hasDraft && (!request.userSeed || sharedFill)));
+  // Cache-format eligibility belongs to the method's append binding.
   if (features.fill && !fill) reasons.push("fill-incompatible-with-request");
-  const checkpoint = capabilities.checkpoints && method === "autoregressive" &&
-    (mechanism === "serial" || capabilities.sharedCheckpoints === true) && promptCache && !request.hasGrammar &&
+  const checkpoint = capabilities.checkpoints && method === "autoregressive" && !request.hasVision &&
+    (mechanism === "serial" || capabilities.sharedCheckpoints === true) && promptCache && !pagedKv && !request.hasGrammar &&
     !features.fill && !request.wantsLogprobs;
   const compiledDecode = features.compiledDecode === true && capabilities.compiledDecode === true &&
-    method === "autoregressive" && !request.hasAdapters && !pagedKv;
+    method === "autoregressive" && !request.hasAdapters && !pagedKv && !(sharedFill && fill);
   if (features.compiledDecode && !compiledDecode) reasons.push("compiled-decode-unavailable-for-request");
-  const grammarJump = features.grammarJump === true && request.hasGrammar &&
-    method === "autoregressive" && mechanism === "serial" && !request.wantsLogprobs;
+  const grammarJump = grammarProposals || (features.grammarJump === true && request.hasGrammar &&
+    method === "autoregressive" && mechanism === "serial" && !request.wantsLogprobs);
   if (features.grammarJump && request.hasGrammar && !grammarJump)
     reasons.push("grammar-jump-incompatible-with-request");
   return Object.freeze({ method, mechanism, pagedKv, promptCache, checkpoint, fill,

package/src/engine/execution-tasks.ts

@@ -0,0 +1,44 @@
+interface Task {
+  run(): Promise<void>;
+  reject(error: unknown): void;
+}
+
+/** Work whose owner requires an execution boundary, without owning a decode
+ * row. The driver supplies the lease; this queue owns waiting and cancellation.
+ * Once started, work settles before its caller can release native resources. */
+export class ExecutionTasks {
+  #pending: Task[] = [];
+  get pending(): number { return this.#pending.length; }
+
+  enqueue<T>(work: () => Promise<T>, cancellation?: Cancellation): Promise<T> {
+    if (cancellation?.reason !== undefined) return Promise.reject(new GenerationCancelled(cancellation.reason));
+    return new Promise<T>((resolve, reject) => {
+      let unsubscribe = () => {};
+      const cleanup = () => unsubscribe();
+      const task: Task = {
+        async run() {
+          cleanup();
+          try { resolve(await work()); } catch (error) { reject(error); }
+        },
+        reject(error) { cleanup(); reject(error); },
+      };
+      this.#pending.push(task);
+      unsubscribe = cancellation?.subscribe(reason => {
+        const index = this.#pending.indexOf(task);
+        if (index < 0) return;
+        this.#pending.splice(index, 1);
+        task.reject(new GenerationCancelled(reason));
+      }) ?? (() => {});
+      if (!this.#pending.includes(task)) unsubscribe();
+    });
+  }
+
+  async advance(): Promise<void> { await this.#pending.shift()?.run(); }
+
+  rejectAll(error: unknown): void {
+    const tasks = this.#pending; this.#pending = [];
+    for (const task of tasks) task.reject(error);
+  }
+}
+import type { Cancellation } from "../contracts/generation";
+import { GenerationCancelled } from "./cancellation";

package/src/engine/namespaced-cache.ts

@@ -0,0 +1,13 @@
+import type { PrefixCache } from "../contracts/prefix-cache";
+
+/** Compose numerical identity with a cache without exposing its storage tiers. */
+export function namespacedCache<State, Attachment>(cache: PrefixCache<State, Attachment>,
+  namespace: (base: string) => string): PrefixCache<State, Attachment> {
+  return {
+    closeSession: id => cache.closeSession?.(id),
+    reclaim: () => cache.reclaim?.(),
+    ...(cache.prefetch ? { prefetch: (tokens: number[], ns = "", sessionId?: string) => cache.prefetch!(tokens, namespace(ns), sessionId) } : {}),
+    take: (tokens, ns = "", sessionId) => cache.take(tokens, namespace(ns), sessionId),
+    put: (tokens, state, ns = "", retain, attachments, sessionId) => cache.put(tokens, state, namespace(ns), retain, attachments, sessionId),
+  };
+}

package/src/engine/scheduler.ts

@@ -18,8 +18,8 @@ export async function driveExecutionGroup(
   try {
     while (!group.closed) {
       group.pruneCancelled();
-      const held = group.admissionHeld;
-      if (!group.active && !group.preparing && (held || !group.queued)) {
+      let held = group.admissionHeld;
+      if (!group.active && !group.preparing && (held || (!group.queued && !group.pendingTasks))) {
         release();
         await group.waitForWork();
         continue;
@@ -28,6 +28,15 @@ export async function driveExecutionGroup(
       if (!execution && group.acquireExecution) execution = await group.acquireExecution();
       if (group.closed) break;
 
+      // One preparation task per iteration. An exclusive model mutation
+      // keeps its existing drain priority; preparation cannot prolong it.
+      if (!held && group.pendingTasks && group.advanceTask) {
+        await group.advanceTask();
+        if (group.closed) break;
+        group.pruneCancelled();
+        held = group.admissionHeld;
+      }
+
       if (!group.preparing && !held && group.queued && group.active < group.maxActive)
         group.admitNext();
       while (group.preparing && group.canPrepareMore && !held && group.queued &&
@@ -35,6 +44,17 @@ export async function driveExecutionGroup(
           (group.preparingTokens ?? 0) + (group.nextPreparationTokens ?? 0) <= (group.maxPreparationTokens ?? Infinity)) {
         if (!group.admitNext()) break;
       }
+      const mixed = group.preparing && group.active ? group.mixedPreparation : undefined;
+      if (mixed && group.advanceMixed) {
+        // Reserve the method's running-token demand first; prompt work fills
+        // the rest. A budget smaller than one token per row still progresses.
+        const budget = Math.max(group.maxIterationTokens ?? Infinity,
+          mixed.runningTokens + mixed.minimumPreparationTokens);
+        await group.advanceMixed(budget);
+        lastYield = clock.now();
+        await clock.yield();
+        continue;
+      }
       if (group.preparing) {
         const activeBefore = group.active;
         await group.advancePreparation();