mlx-bun 0.3.0 → 0.5.0

package/src/cli.ts

@@ -74,6 +74,8 @@ Commands:
   generate   One-shot raw text generation from a local model (no server)
   benchmark  Measure decode/prefill speed of OUR stack on this machine
   embed      Text embeddings from a local embedding model (Qwen3-Embedding)
+  transcribe Speech-to-text from an audio file with a local Whisper model
+  dictate    Push-to-talk: stream your microphone into Whisper, print/copy/type the text
   evals      Show recent benchmark runs (all stacks)
   help       Show help for a command (also: mlx-bun <command> --help)
 
@@ -120,10 +122,11 @@ const SERVER_FLAGS = `Server options:
   --ssd-cache-verify        Verify tensor hashes on every restore (reads all
                             bytes eagerly — integrity paranoia only)
   --generation-checkpoint <tokens>
-                            Snapshot an in-flight serial generation to the SSD
+                            Queue an in-flight generation snapshot to the SSD
                             cache every N emitted tokens. Repeating the same
                             request after restart replays and resumes it.
-                            Requires --ssd-cache and --batch 1.
+                            Requires --ssd-cache; supports ordinary requests
+                            with shared cache layouts.
   --isolate                 Run the inference engine as a CHILD process and
                             keep this process a pure UI/API proxy — the UI
                             stays instant under GPU load and survives engine
@@ -163,14 +166,27 @@ Model & quality:
                             request's \`adapter\` field, incl. "none", wins).
                             --adapter-path is accepted as the mlx_lm.server
                             alias. Hot-swap via POST /v1/adapters unchanged.
+  --whisper-model <p|query> Whisper checkpoint for /v1/audio/transcriptions
+                            [default: the first downloaded whisper model,
+                            resolved on the first request]. Serving a whisper
+                            model ALONE starts a transcription-only server.
+  --preload                 Transcription-only server: load the Whisper
+                            weights before listening instead of on the first
+                            request.
+  --whisper-idle-unload <s> Keep the Whisper weights for this many idle seconds
+                            after a take before disposing them. [default: 0 —
+                            released right after every take, so the chat model
+                            has the memory back before it prefills; a cold
+                            take costs ~150 ms more than a warm one]
+  --whisper-resident        Never dispose the Whisper weights (sotto-style).
   --draft-model <query>     Speculative decoding: a drafter proposes tokens
                             the main model verifies (exact results, faster
                             decode when drafts land). The artifact's kind is
                             auto-detected — a full same-tokenizer model
                             (mlx_lm.server parity), a Gemma "-assistant"
                             KV-borrowing drafter, or a DSpark checkpoint.
-                            Serial lane only — with --batch N a mounted draft
-                            routes every request serial, like mlx_lm.server.
+                            Supported draft providers use shared batched
+                            execution; ordinary requests can batch too.
   --draft-kind <kind>       Override draft-artifact detection:
                             two-model | assistant | dspark | deepspec | mtp | ngram
                             (deepspec = DeepSeek's released Gemma4DSparkModel
@@ -205,9 +221,8 @@ Model & quality:
                             only; sliding-window layers stay bf16)]
   --paged-kv                OPTIONAL vLLM-style paged KV storage (block pool +
                             gather before the stock SDPA; env MLX_BUN_PAGED_KV=1).
-                            v1: serial only (pins --batch 1 unless --batch is
-                            given), Gemma4-family, bf16 — refuses --batch N>1,
-                            --kv-quant, --draft-model; bypasses the prompt
+                            Gemma4-family, bf16; supports --batch N.
+                            Refuses --kv-quant and --draft-model; bypasses the prompt
                             cache; runs uncompiled decode. Bit-exact with the
                             plain path; expect a small decode cost at batch=1
                             (the gather copy). docs/design/kv-cache.md
@@ -268,6 +283,74 @@ Kill switches (bit-exact A/B levers — each selects a slower same-parity path):
                             bit-exact with the resident path`;
 
 const HELP: Record<string, string> = {
+  dictate: `mlx-bun dictate — push-to-talk dictation from your microphone
+
+Usage: mlx-bun dictate [query] [options]
+
+Captures the mic through the AVAudioEngine sidecar (16 kHz mono; macOS asks
+for Microphone permission on first use) and streams it into a Whisper
+transcription session while you speak — every finished 30 s window is
+transcribed during capture, so the result lands about one window after you
+stop. Silence never runs Whisper (Silero VAD).
+
+Modes:
+  (default)              Press Enter to start a take, Enter again to stop;
+                         q + Enter or Ctrl-C quits
+  --hotkey [keycode]     Hold a key instead (default 61 = Right Option, like
+                         sotto); needs Input Monitoring for your terminal
+Backend:
+  [query] / --model <p>  Whisper model for in-process transcription (default:
+                         the first downloaded whisper checkpoint)
+  --server <url>         Use a running mlx-bun server's /v1/audio/sessions
+                         instead of loading the model here
+  --idle-unload <s>      Release the weights after this many idle seconds
+                         (0 = right after each take; default 30); --resident
+                         keeps them loaded
+Decoding:
+  --language <code>      default en; "auto" detects
+  --beam-size <n>        beam search (default greedy)
+  --prompt "…"           initial prompt / --vocabulary "a,b,c" hint terms
+  --no-vad               skip the Silero gate
+Output (any combination; default prints):
+  --copy                 copy the transcript to the clipboard (pbcopy)
+  --type                 type it into the frontmost app via System Events
+                         keystrokes (needs Accessibility for your terminal);
+                         --type-delay <s> waits first so you can Cmd-Tab
+                         (default 1 in Enter mode, 0 with --hotkey)`,
+  transcribe: `mlx-bun transcribe — speech-to-text with a local Whisper model (no server)
+
+Usage: mlx-bun transcribe <audio-file> [query] [options]
+
+Loads a Whisper checkpoint (mlx-community/whisper-*) and prints the
+transcript. WAV is decoded natively; any other container CoreAudio reads
+(mp3/m4a/flac/ogg/aiff/…) is transcoded with afconvert first.
+
+Model:
+  [query] / --model <p>  Whisper model; omitted = the first downloaded
+                         \`whisper\` checkpoint (errors with an mlx-bun get
+                         suggestion when none)
+Decoding:
+  --language <code>      ISO code or name ("en", "japanese"); "auto" or omitted
+                         detects the language from the first 30 s
+  --task translate       Translate to English instead of transcribing
+  --beam-size <n>        Beam search width (default: greedy)
+  --temperature <t>      Sampling temperature; the default runs the
+                         (0, 0.2, …, 1.0) fallback ladder like mlx-whisper
+  --no-fallback          Temperature 0 only, no retry ladder
+  --prompt "…"           Initial prompt (vocabulary hints / style)
+  --no-timestamps        Decode text only (<|notimestamps|>)
+  --no-condition         Do not condition each window on the previous text
+  --vad                  Silero VAD gate: skip Whisper when no speech is
+                         detected (weights: ggml-org/whisper-vad in the HF cache
+                         or --vad-model <path>); --vad-trim crops to speech
+  --word-timestamps      Word-level timestamps (verbose_json / verbose output)
+  --faithful             Run the oracle-parity graph instead of the fast path
+  --audio-ctx <n>        Lab: encode only n of the 1500 encoder positions
+                         (whisper.cpp -ac); cheaper for short clips, degrades
+                         below ~1024 — off by default
+Output:
+  --format <f>           text (default) · json · verbose_json · srt · vtt
+  --verbose              Print each segment with its time range as it decodes`,
   pi: `mlx-bun pi — drop into a coding-agent session on a local model
 
 Usage: mlx-bun pi [options] [message...]
@@ -1012,22 +1095,28 @@ function serverRuntimeFlags(): { port: number; serverOptions: import("./server")
   if (route.kvQuant !== undefined) serverOptions.kvQuant = route.kvQuant;
   if (route.turboQuant !== undefined) serverOptions.turboQuant = route.turboQuant;
   // --paged-kv (env: MLX_BUN_PAGED_KV=1): OPTIONAL vLLM-style block-pool KV
-  // storage, default off (docs/design/kv-cache.md). Serial-only in v1:
-  // with no explicit --batch, pin --batch 1 (the default is 8); an explicit
-  // --batch N>1 is refused loudly by createServer rather than downgraded.
+  // storage, default off (docs/design/kv-cache.md). The state policy owns
+  // block pools and cache reuse; paging preserves the selected batch size.
   if (flag("paged-kv") || runtimeValue("MLX_BUN_PAGED_KV") === "1") {
     const bsRaw = opt("paged-kv-block-size");
     serverOptions.pagedKv = bsRaw !== null ? { blockSize: Number(bsRaw) } : {};
-    if (batchRaw === null) {
-      serverOptions.batch = 1;
-      console.log("[paged-kv] serial-only in v1 — pinning --batch 1");
-    }
   }
   // Bind loopback unless asked otherwise (mlx_lm.server parity); --host
   // 0.0.0.0 is the explicit opt-in for LAN exposure. The chat-UI open and
   // `mlx-bun pi` attach both go through localhost, so loopback-only is
   // transparent to them.
   serverOptions.hostname = opt("host", "127.0.0.1")!;
+  // Speech-to-text companion: explicit checkpoint + residency policy.
+  const whisperIdleRaw = opt("whisper-idle-unload");
+  if (whisperIdleRaw !== null) {
+    const n = Number(whisperIdleRaw);
+    if (!Number.isFinite(n) || n < 0) {
+      console.error(`--whisper-idle-unload expects seconds >= 0 (got "${whisperIdleRaw}")`);
+      process.exit(1);
+    }
+    serverOptions.whisperIdleUnloadSec = n;
+  }
+  if (flag("whisper-resident")) serverOptions.whisperResident = true;
   // Server-wide default for the chat template's enable_thinking variable
   // (CPM/MiniCPM5 and other hybrid-reasoning models). Unset ⇒ the model's
   // own default (false for MiniCPM5). onOff accepts true|false|on|off|1|0.
@@ -1642,29 +1731,44 @@ switch (cmd) {
     // straight from that path; anything else resolves like the positional
     // fuzzy query. Precedence: --model > positional > --query > auto-pick.
     const { m, picked } = await resolveModelAuto(opt("model") ?? positional(0) ?? opt("query"));
+    if (m.modelType === "whisper") {
+      // Transcription-only server: no chat model, weights paged in on demand.
+      const sNative0 = step("native runtime");
+      await ensureNative(sNative0);
+      sNative0.done("native runtime ready");
+      const { createTranscriptionServer } = await import("./serve/transcription-server");
+      const { server, service } = await createTranscriptionServer({
+        modelDir: m.path, modelId: m.repoId, port: rt.port,
+        ...(rt.serverOptions.hostname ? { hostname: rt.serverOptions.hostname } : {}),
+        ...(rt.serverOptions.unixSocket ? { unixSocket: rt.serverOptions.unixSocket } : {}),
+        idleUnloadSec: rt.serverOptions.whisperIdleUnloadSec,
+        resident: rt.serverOptions.whisperResident,
+        preload: flag("preload"),
+      });
+      const where = rt.serverOptions.unixSocket ?? `http://${rt.serverOptions.hostname === "0.0.0.0" ? "localhost" : rt.serverOptions.hostname}:${server.port}`;
+      box([
+        `${style.bold(m.repoId)} ${style.dim("· transcription-only server")}`,
+        `POST ${where}/v1/audio/transcriptions ${style.dim(`· ${service.stats.idle_unload_sec === null ? "always resident" : service.idleUnloadSec === 0 ? "released after every take" : `idle unload ${service.idleUnloadSec}s`}${service.resident ? " · loaded" : " · loads on first request"}`)}`,
+      ]);
+      const stop = () => { service.dispose(); server.stop(true); process.exit(0); };
+      process.once("SIGINT", stop);
+      process.once("SIGTERM", stop);
+      break;
+    }
+    const whisperQuery = opt("whisper-model");
+    if (whisperQuery !== null) {
+      const { m: wm } = await resolveModelAuto(whisperQuery);
+      if (wm.modelType !== "whisper") {
+        console.error(`--whisper-model ${whisperQuery} resolved to ${wm.repoId} (model_type ${wm.modelType}), not a Whisper checkpoint`);
+        process.exit(1);
+      }
+      rt.serverOptions.whisperModelDir = wm.path;
+      rt.serverOptions.whisperModelId = wm.repoId;
+    }
     const sFit = step("assessing fit");
     const config = await loadModelConfig(m.path);
     const report = fit(config, m.sizeBytes, 8192, undefined, undefined, m.expertsBytes);
     sFit.done(`${style.bold(m.repoId)} ${style.dim(`· ${gb(m.sizeBytes)} · ~${report.predictedDecodeTps.toFixed(0)} tok/s predicted`)}${picked ? style.dim(" · auto-picked (override: mlx-bun serve <query>)") : ""}`);
-    // Near-ceiling advisory (admission doctrine: advise, never refuse). A
-    // model near the DEFAULT Metal wired ceiling (~75% of RAM) dies with an
-    // UNCATCHABLE async-GPU-OOM on long prefills (M4 24 GB × 17 GB model,
-    // 2026-08-20) — surface the standard remedy up front instead.
-    {
-      const { maxRecommendedWorkingSetSize } = await import("./mlx/ffi");
-      const ceiling = maxRecommendedWorkingSetSize();
-      const sysctlRaw = Bun.spawnSync(["sysctl", "-n", "iogpu.wired_limit_mb"]).stdout.toString().trim();
-      const limitIsDefault = sysctlRaw === "0" || sysctlRaw === "";
-      if (limitIsDefault && ceiling > 0 && m.sizeBytes > 0.8 * ceiling) {
-        const ramGb = Number(Bun.spawnSync(["sysctl", "-n", "hw.memsize"]).stdout.toString().trim()) / 2 ** 30;
-        const suggestMb = Math.floor((ramGb - 2.5) * 1024);
-        console.error(style.dim(
-          `  ! ${gb(m.sizeBytes)} model vs ~${gb(ceiling)} default GPU ceiling — long prompts can\n` +
-          `    hit an uncatchable Metal OOM. If that happens, raise the limit first:\n` +
-          `      sudo sysctl iogpu.wired_limit_mb=${suggestMb}   (resets on reboot)`,
-        ));
-      }
-    }
     const sNative = step("native runtime");
     await ensureNative(sNative);
     sNative.done("native runtime ready");
@@ -1684,9 +1788,8 @@ switch (cmd) {
     }
     // Speculative decoding (mlx_lm.server parity): --draft-model <path|query>
     // resolves like the main model; --num-draft-tokens defaults to 3 (the
-    // server's default upstream — mlx_lm.generate's is 2). Serial-lane-only:
-    // with --batch N, a mounted draft routes every request serial (upstream
-    // is_batchable = draft is None).
+    // server's default upstream — mlx_lm.generate's is 2). Qualified providers
+    // use shared execution; ordinary requests retain their own placement.
     const draftQuery = opt("draft-model");
     let draftModelDir: string | undefined;
     if (draftQuery !== null) {
@@ -1908,6 +2011,200 @@ switch (cmd) {
     break;
   }
 
+  case "transcribe": {
+    // Speech-to-text, one-shot, no server: audio file → Whisper → text.
+    const { openWhisperModel } = await import("./model/factory");
+    const { loadWhisperTokenizer } = await import("./audio/whisper-tokenizer");
+    const { WhisperTranscriber } = await import("./audio/whisper-transcribe");
+    const { formatTranscription, TRANSCRIPTION_FORMATS } = await import("./audio/whisper-format");
+    const { TranscriptionService } = await import("./serve/transcription-service");
+    const { formatTimestamp } = await import("./audio/whisper-format");
+
+    const audioPath = positional(0);
+    if (!audioPath) {
+      console.error("usage: mlx-bun transcribe <audio-file> [query] [--language en] [--format text|json|verbose_json|srt|vtt]");
+      process.exit(1);
+    }
+    const format = (opt("format") ?? "text") as import("./audio/whisper-format").TranscriptionFormat;
+    if (!TRANSCRIPTION_FORMATS.includes(format)) {
+      console.error(`--format must be one of ${TRANSCRIPTION_FORMATS.join(", ")}`);
+      process.exit(1);
+    }
+    const bytes = new Uint8Array(await Bun.file(audioPath).arrayBuffer());
+    const samples = await TranscriptionService.decodeAudio(bytes);
+    const durationSeconds = samples.length / 16_000;
+
+    const modelQuery = opt("model") ?? positional(1) ?? opt("query");
+    let modelDir: string;
+    if (modelQuery && existsSync(join(modelQuery, "config.json"))) modelDir = modelQuery;
+    else if (modelQuery) modelDir = (await resolveModelAuto(modelQuery)).m.path;
+    else {
+      const reg = new Registry();
+      if (reg.list().length === 0) await reg.scan();
+      const w = reg.list().find((r) => r.modelType === "whisper");
+      if (!w) {
+        console.error("no Whisper model downloaded — try: mlx-bun get mlx-community/whisper-large-v3-turbo");
+        process.exit(1);
+      }
+      modelDir = w.path;
+    }
+    let vadGate: { segments: { start: number; end: number }[] } | null = null;
+    if (flag("vad")) {
+      const { SileroVad } = await import("./audio/silero-vad");
+      const tv = performance.now();
+      const vad = SileroVad.load(opt("vad-model"));
+      const { segments } = vad.detect(samples, { threshold: Number(opt("vad-threshold") ?? 0.5) });
+      vadGate = { segments };
+      if (flag("verbose")) console.error(`vad: ${segments.length} speech segment(s) in ${(performance.now() - tv).toFixed(0)} ms`);
+      if (segments.length === 0) { process.stdout.write(format === "text" ? "\n" : JSON.stringify({ text: "", vad: { speech: false, segments: [] } }) + "\n"); break; }
+    }
+    const { model } = await openWhisperModel(modelDir);
+    const tok = await loadWhisperTokenizer(modelDir, model.dims.nVocab);
+    const transcriber = new WhisperTranscriber(model, tok);
+    void vadGate;
+    const langOpt = opt("language");
+    const task = opt("task") === "translate" ? "translate" as const : "transcribe" as const;
+    const beam = opt("beam-size");
+    const temperature = opt("temperature");
+    const verbose = flag("verbose");
+    const t0 = performance.now();
+    const result = await transcriber.transcribe(samples, {
+      language: !langOpt || langOpt === "auto" ? null : langOpt,
+      task,
+      beamSize: beam ? Number(beam) : null,
+      ...(temperature ? { temperature: Number(temperature) } : flag("no-fallback") ? { temperature: 0 } : {}),
+      initialPrompt: opt("prompt"),
+      withoutTimestamps: flag("no-timestamps"),
+      conditionOnPreviousText: !flag("no-condition"),
+      fast: !flag("faithful"),
+      wordTimestamps: flag("word-timestamps"),
+      audioCtx: opt("audio-ctx") ? Number(opt("audio-ctx")) : null,
+      onSegment: verbose
+        ? (s) => console.error(`[${formatTimestamp(s.start, ".")} --> ${formatTimestamp(s.end, ".")}] ${s.text.trim()}`)
+        : undefined,
+    });
+    const elapsed = (performance.now() - t0) / 1000;
+    if (verbose)
+      console.error(`${durationSeconds.toFixed(2)} s audio in ${elapsed.toFixed(3)} s (${(durationSeconds / elapsed).toFixed(1)}× realtime), language ${result.language}`);
+    process.stdout.write(formatTranscription(result, format, durationSeconds, task));
+    transcriber.dispose();
+    model.dispose();
+    break;
+  }
+
+  case "dictate": {
+    const { startMicCapture } = await import("./audio/mic-capture");
+    const { TranscriptionService, defaultWhisperRecord } = await import("./serve/transcription-service");
+    const { style } = await import("./tui");
+    const serverUrl = opt("server");
+    const hotkeyFlag = argv.includes("--hotkey");
+    const hotkey = hotkeyFlag ? Number(opt("hotkey") ?? 61) || 61 : null;
+    const language = opt("language") ?? "en";
+    const beam = opt("beam-size");
+    const vocabulary = opt("vocabulary")?.split(",").map((s) => s.trim()).filter(Boolean);
+    const params = {
+      language: language === "auto" ? null : language,
+      beamSize: beam ? Number(beam) : null,
+      temperature: 0,
+      prompt: opt("prompt"),
+      vocabulary,
+      vad: flag("no-vad") ? null : { threshold: 0.5, minSpeechMs: 120 },
+    };
+    const typeDelay = Number(opt("type-delay") ?? (hotkey ? 0 : 1));
+
+    // backend: in-process service or a running server's sessions
+    let service: InstanceType<typeof TranscriptionService> | null = null;
+    if (!serverUrl) {
+      const q = opt("model") ?? positional(0) ?? opt("query");
+      let modelDir: string; let modelId: string;
+      if (q && existsSync(join(q, "config.json"))) { modelDir = q; modelId = q; }
+      else if (q) { const { m } = await resolveModelAuto(q); modelDir = m.path; modelId = m.repoId; }
+      else {
+        const rec = await defaultWhisperRecord();
+        if (!rec) { console.error("no Whisper model downloaded — try: mlx-bun get mlx-community/whisper-large-v3-turbo"); process.exit(1); }
+        modelDir = rec.path; modelId = rec.repoId;
+      }
+      const idle = opt("idle-unload");
+      service = new TranscriptionService({ modelDir, modelId, idleUnloadSec: idle !== null ? Number(idle) : 30, resident: flag("resident"), log: () => {} });
+      process.stderr.write(style.dim("loading whisper… "));
+      await service.ensureLoaded();
+      service.vad();
+      process.stderr.write(style.dim("ready\n"));
+    }
+    const base = serverUrl?.replace(/\/$/, "");
+    interface Take { feed(pcm: Float32Array): Promise<void>; finish(): Promise<{ text: string; ms: number }>; }
+    const startTake = async (): Promise<Take> => {
+      if (service) {
+        const session = await service.createSession(params);
+        return {
+          feed: async (pcm) => { await session.append(pcm); },
+          finish: async () => { const t0 = performance.now(); const o = await session.finish(); return { text: o.result.text.trim(), ms: performance.now() - t0 }; },
+        };
+      }
+      const r = await fetch(`${base}/v1/audio/sessions`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ language: params.language, beam_size: params.beamSize, temperature: 0, prompt: params.prompt, vocabulary, vad: !!params.vad, vad_min_speech_ms: 120 }) });
+      if (!r.ok) throw new Error(`session create failed: ${r.status} ${await r.text()}`);
+      const { id } = await r.json() as { id: string };
+      return {
+        feed: async (pcm) => { await fetch(`${base}/v1/audio/sessions/${id}/audio`, { method: "POST", headers: { "content-type": "audio/pcm;rate=16000" }, body: new Uint8Array(pcm.buffer.slice(pcm.byteOffset, pcm.byteOffset + pcm.byteLength) as ArrayBuffer) }); },
+        finish: async () => { const t0 = performance.now(); const f = await fetch(`${base}/v1/audio/sessions/${id}/finish`, { method: "POST" }); const b = await f.json() as { text: string }; return { text: (b.text ?? "").trim(), ms: performance.now() - t0 }; },
+      };
+    };
+
+    const deliver = async (text: string) => {
+      if (!text) { console.log(style.dim("(no speech)")); return; }
+      console.log(text);
+      if (flag("copy")) { const p = Bun.spawn(["pbcopy"], { stdin: "pipe" }); p.stdin.write(text); p.stdin.end(); await p.exited; }
+      if (flag("type")) {
+        if (typeDelay > 0) await Bun.sleep(typeDelay * 1000);
+        const escaped = text.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
+        const r = Bun.spawnSync(["osascript", "-e", `tell application "System Events" to keystroke "${escaped}"`], { stderr: "pipe" });
+        if (r.exitCode !== 0) console.error(style.dim(`typing failed (grant Accessibility to your terminal): ${r.stderr.toString().trim()}`));
+      }
+    };
+
+    const mic = await startMicCapture({ hotkey });
+    let take: Take | null = null;
+    let pending: Float32Array[] = []; let pendingLen = 0;
+    let feeding: Promise<void> = Promise.resolve();
+    const flush = () => {
+      if (!take || pendingLen === 0) return;
+      const buf = new Float32Array(pendingLen); let o = 0; for (const p of pending) { buf.set(p, o); o += p.length; }
+      pending = []; pendingLen = 0;
+      const t = take;
+      feeding = feeding.then(() => t.feed(buf)).catch((e) => console.error(style.dim(`feed failed: ${(e as Error).message}`)));
+    };
+    const begin = async () => { if (take) return; take = await startTake(); pending = []; pendingLen = 0; process.stderr.write(style.bold("● recording") + style.dim(hotkey ? " (release to stop)\n" : " (Enter to stop)\n")); };
+    const end = async () => {
+      if (!take) return;
+      flush();
+      await feeding;
+      const t = take; take = null;
+      process.stderr.write(style.dim("transcribing… "));
+      try { const { text, ms } = await t.finish(); process.stderr.write(style.dim(`${ms.toFixed(0)} ms\n`)); await deliver(text); }
+      catch (e) { console.error(`transcription failed: ${(e as Error).message}`); }
+    };
+    if (!hotkey) {
+      console.error(style.dim("Enter: start/stop a take · q: quit"));
+      (async () => {
+        for await (const line of console) {
+          if (line.trim() === "q") { mic.stop(); process.exit(0); }
+          if (take) await end(); else await begin();
+        }
+      })();
+    }
+    process.once("SIGINT", () => { mic.stop(); process.exit(0); });
+    for await (const ev of mic.events) {
+      if (ev.kind === "ready") { if (hotkey) console.error(style.dim(`hold key ${hotkey} to talk · Ctrl-C quits · mic ${ev.info}`)); }
+      else if (ev.kind === "error") { console.error(ev.message); mic.stop(); process.exit(1); }
+      else if (ev.kind === "hotkey") { if (ev.down) await begin(); else await end(); }
+      else if (ev.kind === "pcm" && take) {
+        pending.push(ev.samples); pendingLen += ev.samples.length;
+        if (pendingLen >= 16_000 / 4) flush(); // every 250 ms of audio
+      }
+    }
+    break;
+  }
+
   case "bench":
   case "benchmark": {
     const { banner, step, box, style } = await import("./tui");

package/src/contracts/execution.ts

@@ -1,6 +1,7 @@
 /** Request facts only. Native resources remain with the preparation owner. */
 export interface ExecutionRequirements {
   readonly hasVision: boolean;
+  readonly hasPreparedPrefixIdentity?: boolean;
   readonly hasAdapters: boolean;
   readonly hasRepetitionPenalty: boolean;
   readonly userSeed: boolean;
@@ -17,9 +18,29 @@ export interface ExecutionCapabilities {
   readonly continuous: boolean;
   readonly quantizedBatch: boolean;
   readonly grammarBatch: boolean;
+  readonly adapterBatch?: boolean;
+  /** Prepared media can enter the ordinary shared execution group. */
+  readonly mediaBatch?: boolean;
+  readonly mediaPrefixCache?: boolean;
+  readonly pagedBatch?: boolean;
   readonly checkpoints: boolean;
+  /** Bound ordinary driver can restore/capture this request configuration. */
+  readonly sharedCheckpoints?: boolean;
   /** Model-qualified speculative execution can retain this request's KV codec. */
   readonly speculativeKvQuant?: boolean;
+  readonly speculativeTurboQuant?: boolean;
+  readonly turboQuantBatch?: boolean;
+  readonly speculativeLogprobs?: boolean;
+  /** The grouped provider supports the target's mounted adapter context. */
+  readonly sharedSpeculativeAdapters?: boolean;
+  /** Methods supplied by the model's shared execution binding. */
+  readonly groupedMethods?: readonly string[];
+  /** A shared method accepts request-owned grammar continuation proposals. */
+  readonly sharedGrammarProposals?: boolean;
+  /** A grouped method can consume this request's strict known continuations. */
+  readonly sharedFill?: boolean;
+  /** A mounted provider can consume verified external continuations. */
+  readonly sharedSpeculativeEcho?: boolean;
   /** A graph-owned compiled step exists; cache geometry can still decline it. */
   readonly compiledDecode?: boolean;
 }

package/src/contracts/object-cache.ts

@@ -0,0 +1,8 @@
+/** Exact reusable work, independent of token-prefix matching. The same cache
+ * owner chooses residency and persistence for objects and conversation state. */
+export interface ObjectCache<Value> {
+  /** Borrow immutable data through an owned lease. A miss permits recompute. */
+  take(key: string): Promise<{ value: Value; dispose(): void } | null>;
+  /** Successful return transfers ownership, including immediate eviction. */
+  put(key: string, value: Value): void;
+}

package/src/contracts/pi-provider.ts

@@ -13,7 +13,7 @@ export function piModelDefinition(options: PiModelOptions = {}) {
     name: options.name ?? "mlx-bun (local)",
     api: "openai-completions" as const,
     reasoning: options.reasoning ?? false,
-    compat: { supportsDeveloperRole: false,
+    compat: { supportsDeveloperRole: false, sendSessionAffinityHeaders: true,
       ...(options.reasoning ? { thinkingFormat: "qwen-chat-template" as const } : {}) },
     input: (options.vision ? ["text", "image"] : ["text"]) as ("text" | "image")[],
     cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },

package/src/contracts/pi.ts

@@ -160,7 +160,9 @@ export type ServerMessage =
   // provider declaration — clients use it instead of probing for a 400.
   // `thinking`: whether the model has a switchable reasoning channel (drives
   // the UI's thinking on/off toggle; false hides it).
-  | { type: "ready"; model: string; vision: boolean; audio: boolean; thinking: boolean; genDefaults: ReadyGenDefaults }
+  // transcription: a Whisper checkpoint is configured or resolvable for
+  // /v1/audio/sessions — drives the composer's hold-to-talk mic button.
+  | { type: "ready"; model: string; vision: boolean; audio: boolean; thinking: boolean; genDefaults: ReadyGenDefaults; transcription?: boolean }
   | { type: "turn_start" }
   | { type: "text_delta"; delta: string }
   | { type: "thinking_delta"; delta: string }

package/src/contracts/prefix-cache.ts

@@ -0,0 +1,33 @@
+/** A reusable prefix and the backend/method state that covers its tokens.
+ * The caller owns a returned hit. After releasing or returning its state,
+ * it must release any retained backing through retain. */
+export interface PrefixCacheHit<State, Attachment = never> {
+  tokens: number[];
+  caches: State;
+  attachments?: Attachment;
+  retain?: () => void;
+}
+
+/** Storage is independent of request scheduling and the method producing
+ * state. Both prefill and decode can publish processed-token prefixes.
+ * State is opaque here; its backend/method defines layout and alignment.
+ * Implementations own prefix selection, retention and storage tiers. */
+export interface PrefixCache<State, Attachment = never> {
+  /** Give optional retained state back to the allocator at an execution
+   * boundary. This never rejects or changes the active request. */
+  reclaim?(): void;
+  /** Drop session affinity without deleting reusable checkpoints. */
+  closeSession?(sessionId: string): void;
+  /** Prepare a reusable prefix before execution. The returned release ends
+   * request interest; cache residency and IO remain implementation-owned. */
+  prefetch?(prompt: number[], ns?: string, sessionId?: string): Promise<() => void>;
+  /** Return owned state for a usable session continuation, otherwise the
+   * longest usable prefix, or null on a miss. Session identity is an affinity
+   * hint and never changes the content or numerical namespace.
+   * Namespace distinguishes incompatible artifacts, methods and adapters. */
+  take(prompt: number[], ns?: string, sessionId?: string): PrefixCacheHit<State, Attachment> | null;
+  /** Transfer state and its optional backing release to storage. If this
+   * throws, ownership stays with the caller; successful return transfers
+   * ownership even when storage immediately spills or disposes the entry. */
+  put(tokens: number[], caches: State, ns?: string, retain?: () => void, attachments?: Attachment, sessionId?: string): void;
+}

package/src/contracts/resources.ts

@@ -16,3 +16,10 @@ export interface StateView<T> {
 export interface DisposableResource {
   dispose(): void;
 }
+
+/** A prepared replacement for live state. Commit once between method rounds;
+ * it publishes already-built state without graph work or validation. Dispose
+ * releases the unpublished replacement, or the superseded state after commit. */
+export interface PreparedStateChange extends DisposableResource {
+  commit(): void;
+}

package/src/contracts/sampling.ts

@@ -0,0 +1,23 @@
+/** Sampling owns score processing, token selection, and request-local history.
+ * Score/token representations and sync/device execution belong to the backend;
+ * this contract has no scheduling, cache, or tensor-runtime dependency. */
+export interface SampledToken<Token, Metadata> {
+  token: Token;
+  extras: Metadata | null;
+}
+
+export interface SamplingSession<Scores, DeviceToken, Result> {
+  /** Optional stateless operation shared across compatible rows and positions.
+   * It borrows scores and returns owned device tokens, without advancing history. */
+  readonly independent?: { sample(scores: Scores): DeviceToken };
+  readonly isPlainGreedy: boolean;
+  readonly capturesLogprobs: boolean;
+  readonly needsHistory: boolean;
+  /** Borrows scores; transfers ownership of the result to the caller. */
+  sample(logits: Scores, step: number): Result;
+  seedHistory(tokens: readonly number[]): void;
+  /** Commit accepted tokens when configured for manual history updates. */
+  commitDevice(token: DeviceToken): void;
+  commitNumbers(tokens: readonly number[]): void;
+  dispose(): void;
+}

package/src/contracts/scheduling.ts

@@ -1,9 +1,31 @@
+/** Backend-owned context shared by compatible requests. A key identifies
+ * equivalent execution state; enter returns its scoped release. Scheduling
+ * observes group readiness without inspecting this context. */
+export interface ExecutionContext {
+  readonly key: string;
+  enter(): () => void;
+}
+
 /** A backend execution group exposes bounded work units and readiness. Queue
  * payloads, tensors, cache merging and sampling remain inside its adapter. */
 export interface ExecutionGroup {
+  /** Native preparation may borrow the execution lease between work units.
+   * Payloads and task failures belong to the backend's queue. */
+  readonly pendingTasks?: number;
+  advanceTask?(): Promise<void>;
   readonly active: number;
   readonly queued: number;
   readonly preparing: boolean;
+  readonly preparingRows?: number;
+  /** The preparation owner can accept another row before its next work unit. */
+  readonly canPrepareMore?: boolean;
+  /** Work estimates affect grouping only. A request larger than this budget
+   * still runs alone; it is never rejected or sent to another executor. */
+  readonly preparingTokens?: number;
+  readonly nextPreparationTokens?: number;
+  readonly maxPreparationTokens?: number;
+  /** The latest preparation published output that can be flushed by yielding. */
+  readonly preparationPublishedOutput?: boolean;
   readonly maxActive: number;
   readonly admissionHeld: boolean;
   readonly closed: boolean;
@@ -12,6 +34,11 @@ export interface ExecutionGroup {
   admitNext(): boolean;
   /** Read-only budget check used to preserve admission before decode. */
   canBurst(): boolean;
+  /** Method-owned token demand for a compatible mixed iteration. Absent when
+   * the backend has no packed implementation for the current work. */
+  readonly mixedPreparation?: { readonly runningTokens: number; readonly minimumPreparationTokens: number };
+  readonly maxIterationTokens?: number;
+  advanceMixed?(tokenBudget: number): Promise<void>;
   advancePreparation(): Promise<void>;
   advance(): Promise<void>;
   failActive(error: unknown): void;