mlx-bun 0.0.10 → 0.0.11

package/src/cli.ts

@@ -12,13 +12,12 @@
 import { existsSync } from "node:fs";
 import { basename, join, resolve } from "node:path";
 import { Registry } from "./registry";
-import { loadModelConfig } from "./config";
+import { loadModelConfig, parseTurboQuantScheme, type TurboQuantScheme } from "./config";
 import { fit, skuMatrix, thisMachine } from "./fit";
 import { EvalDB } from "./evaldb";
 import pkg from "../package.json" with { type: "json" };
 import { renderHelp } from "./tui";
 import { isSupportedModelRecord } from "./model/support";
-import { perfKernelEnabled } from "./model/fused-decode-kernel";
 import type { JobEvent } from "./jobs/types";
 
 const argv = process.argv.slice(2);
@@ -36,7 +35,7 @@ Usage: mlx-bun [command] [options]   (no command = serve)
 
 Commands:
   pi         Launch a pi coding-agent session on a local model (the appliance path)
-  serve      Start the OpenAI/Anthropic-compatible server + status page
+  serve      Start the OpenAI/Anthropic-compatible server + web app
   get        Download a model from Hugging Face (resumable, verified)
   ls         List downloaded models
   gc         Reclaim superseded model snapshots + dead blobs from the HF cache
@@ -63,7 +62,7 @@ Options:
 Examples:
   mlx-bun                          # download (first run) + serve + open the chat UI
   mlx-bun pi                       # start (download if needed) and chat with an agent
-  mlx-bun serve 12B                # serve the 12B; status page at http://localhost:8080/
+  mlx-bun serve 12B                # serve the 12B; web app at http://localhost:8080/
   mlx-bun train e4b --data ./prefs # ORPO LoRA fine-tune on {prompt,chosen,rejected} data
   mlx-bun get mlx-community/gemma-4-12B-it-OptiQ-4bit`;
 
@@ -78,7 +77,12 @@ const SERVER_FLAGS = `Server options:
   --memory-budget <GB>      Admission-control memory budget; requests that
                             cannot fit are rejected instead of crashing the
                             GPU  [default: machine RAM × 0.75, check-only]
-  --prompt-cache <GB>       Prompt (KV) cache byte cap  [default: 2 GB;
+  --kv-budget <GB>          Aggregate KV budget across concurrent batch rows
+                            (decimal GB): joiners whose projected KV
+                            (prompt + max_tokens) would exceed it queue until
+                            rows finish; a request over the budget alone is
+                            rejected. Off unless set.
+  --prompt-cache <GB>       Prompt (KV) cache byte cap  [default: 8 GB;
                             --prompt-cache 0 disables the cache]
   --ssd-cache <dir>         SSD cold tier for the prompt cache: prefix KV
                             spills to disk on eviction and persists across
@@ -86,18 +90,44 @@ const SERVER_FLAGS = `Server options:
                             full re-prefill to a zero-copy mmap restore).
                             Off unless set.
   --ssd-cache-max <GB>      SSD tier byte cap  [default: 32 GB]
+  --ssd-demote-idle <sec>   Demote prompt-cache entries idle this long to the
+                            SSD tier, freeing their GPU memory (prefixes stay
+                            reachable via zero-copy restore). 0 disables.
+                            [default: 300 when --ssd-cache is on]
   --ssd-cache-verify        Verify tensor hashes on every restore (reads all
                             bytes eagerly — integrity paranoia only)
-  --batch <n>               Max concurrent requests batched through the
-                            mlx-lm-parity engine  [default: 1 = serial].
-                            >1 opts the whole server into bf16 continuous
-                            batching (= mlx-lm B=N); see --kv-quant.
+  --isolate                 Run the inference engine as a CHILD process and
+                            keep this process a pure UI/API proxy — the UI
+                            stays instant under GPU load and survives engine
+                            crashes (respawned). /ws/chat not proxied yet.
+                            Requests naming another model (exact /v1/models
+                            id) spawn/route to THAT model's engine child.
+  --model-pool <n>          With --isolate: max RESIDENT model engines
+                            [default: 1]. Over the cap, the least-recently-
+                            used engine drains, demotes its caches to the
+                            SSD tier, and exits — switching back respawns
+                            it with all state restored from disk.
+  --unix <path>             (internal) listen on a unix socket instead of a
+                            TCP port — the engine half of --isolate.
+  --batch <n>               Max concurrent requests decoded together
+                            [default: 8]. A LONE request runs the exact
+                            serial engine (same bits, same speed); the cap
+                            engages only when concurrent requests arrive
+                            (agent fan-out). 1 pins strict serial.
                             --decode-concurrency is accepted as the
                             mlx_lm.server alias (semantics differ; see
                             docs/reference/server-config.md)
   --no-open                 Don't open the chat UI in your browser on start.
                             By default an interactive terminal opens
                             http://<host>:<port>/#/chat once the server is up.
+  --allow-private-media     Let image_url/audio_url content parts fetch from
+                            private/loopback/link-local hosts (NAS, another
+                            LAN box). Blocked by default: a request's URL is
+                            attacker-controlled input, and the default keeps
+                            the server from being steered at cloud metadata
+                            or LAN-internal services (SSRF). data: URLs are
+                            always fine; the 10 s timeout and 64 MB response
+                            cap apply either way.
 
 Model & quality:
   --adapter <dir>           Mount a LoRA adapter at startup and use it as the
@@ -105,12 +135,48 @@ Model & quality:
                             request's \`adapter\` field, incl. "none", wins).
                             --adapter-path is accepted as the mlx_lm.server
                             alias. Hot-swap via POST /v1/adapters unchanged.
+  --draft-model <query>     Speculative decoding: a drafter proposes tokens
+                            the main model verifies (exact results, faster
+                            decode when drafts land). The artifact's kind is
+                            auto-detected — a full same-tokenizer model
+                            (mlx_lm.server parity), a Gemma "-assistant"
+                            KV-borrowing drafter, or a DSpark checkpoint.
+                            Serial lane only — with --batch N a mounted draft
+                            routes every request serial, like mlx_lm.server.
+  --draft-kind <kind>       Override draft-artifact detection:
+                            two-model | assistant | dspark | deepspec | ngram
+                            (deepspec = DeepSeek's released Gemma4DSparkModel
+                            drafters; ngram = MODEL-FREE prompt lookup — no
+                            --draft-model, drafts copied from the request's
+                            own context, lossless)  [default: auto]
+  --num-draft-tokens <n>    Drafts per verify round  [default: 3; ngram: 10;
+                            DSpark pins to its trained block width]
+  --ngram-max <k>           Prompt-lookup longest match window  [default: 3]
+  --ngram-min <k>           Prompt-lookup shortest match window  [default: 1]
   --kv-quant <mode>         KV cache quantization: config (per-layer
                             kv_config.json when the model ships one), off
-                            (bf16), or 4 / 8 (uniform bits)
-                            [default: config; bf16 under --batch N — the
-                            batched engine is bf16-only, so an explicit
-                            --kv-quant routes those requests to the serial path]
+                            (bf16), 4 / 8 (uniform bits), or turbo[:k<bits>v
+                            <bits>] (rotation-based TurboQuant — default
+                            k8v3; kBits in {2,4,5,8}, vBits in {2,3,4,5,8};
+                            docs/design/turboquant-kv.md)
+                            [default: off — quantized KV measured 5-20% slower
+                            decode than bf16 at ≤16k (on mlx-lm too); it buys
+                            memory headroom, so opt in when context is tight.
+                            The batched engine (--batch N) is bf16-only: an
+                            explicit --kv-quant routes those requests serial.
+                            turbo is solo-only in v1 (full-attention layers
+                            only; sliding-window layers stay bf16)]
+  --paged-kv                OPTIONAL vLLM-style paged KV storage (block pool +
+                            gather before the stock SDPA; env MLX_BUN_PAGED_KV=1).
+                            v1: serial only (pins --batch 1 unless --batch is
+                            given), Gemma4-family, bf16 — refuses --batch N>1,
+                            --kv-quant, --draft-model; bypasses the prompt
+                            cache; runs uncompiled decode. Bit-exact with the
+                            plain path; expect a small decode cost at batch=1
+                            (the gather copy). docs/design/paged-kv-cache.md
+                            [default: off]
+  --paged-kv-block-size <n> Tokens per KV block  [default: 256 = the plain
+                            cache's growth step]
   --thinking <true|false>   Default for the chat template's enable_thinking
                             variable (CPM and other hybrid-reasoning models);
                             a request's chat_template_kwargs overrides it
@@ -134,23 +200,28 @@ Model & quality:
   --hlg-toe <nats>          HLG shadow rolloff scale  [default: 6]
   --hlg-pivot-offset <nats> HLG pivot: nats below the top token  [default: 6]
 
-Parity tier (sets the whole decode route; a per-fork flag below overrides):
-  --l1                      Bit-for-bit IDENTICAL to mlx-lm (bf16 KV, unfused)
+Parity tier (an ALIAS for the per-fork flags below; each expands to the fastest
+set of forks that still holds the guarantee — a per-fork flag overrides one):
+  --l1                      Bit-for-bit IDENTICAL to mlx-lm (bf16 KV, compiled
+                            activations — the faithful @mx.compile kernels).
+                            No tier given = --l1 (decided 2026-07-05: the L1
+                            faithful kernels match mlx-lm 1.00x on every model;
+                            anything output-changing must beat that baseline in
+                            a paired A/B to earn a default).
   --l2                      Bit-for-bit IDENTICAL to mlx-optiq (quantized KV +
                             fused prefill SDPA, stock unfused decode — the
-                            optiq-golden composition; perf-kernel stays OFF)
-  --l3                      Best performance, no bit-exact oracle (KL + test
-                            gated; adds the envelope-gated perf kernel).
-                            No tier given = the per-flag defaults below,
-                            which equal --l3 today.
+                            optiq-golden composition)
 
-Performance levers (A/B levers; defaults are the measured winners):
+Kill switches (bit-exact A/B levers — each selects a slower same-parity path):
   --compiled-decode on|off  Compiled decode graphs  [default: on]
-  --perf-kernel on|off      Fused quantized-KV decode-SDPA Metal kernel —
-                            NOT bit-exact, envelope-gated (an L3 lever; the
-                            perf side of the compat A/B)  [default: on]
-  --fused-decode on|off     Fused-decode experiment lever  [default: off]
-  --fused-sdpa on|off       Fused SDPA path  [default: on]
+  --compiled-activations on|off
+                            Compiled geglu/swiglu (mlx-lm's @mx.compile kernel —
+                            bit-exact AND fewer dispatches). off = the uncompiled
+                            composition (same parity, slower)  [default: on]
+  --fused-sdpa on|off       Fused SDPA path (quantized KV only)  [default:
+                            follows --kv-quant — on for config (the optiq
+                            composition), off for uniform 4/8 (the mlx-lm
+                            composition) and bf16]
   --force-wire              Wire weights into memory at load
   --expert-offload          MoE only: serve experts from a page-aligned file
                             mmap (built on first use) — keeps the model out of
@@ -199,7 +270,7 @@ ${SERVER_FLAGS}
 
 Endpoints: /v1/chat/completions, /v1/completions, /v1/messages, /v1/responses,
 /v1/models, /v1/adapters, /health, /stats, /fit, /library, /downloads —
-status page at /, browser chat at /chat`,
+web app at / (lands on chat) — /chat, /status, /quantize, /finetune, /dataset deep-link its views`,
 
   get: `mlx-bun get — download a model from Hugging Face
 
@@ -305,11 +376,11 @@ Options:
   --tokens <n>         Tokens to decode per run  [default: 256]
   --runs <n>           Runs (median reported)  [default: 3]
   --prompt-tokens <n>  Pad the prompt to ~n tokens (long-context decode)
-  --kv-quant <mode>    config | off | 4 | 8  [default: off, the
-                       historical baseline]
+  --kv-quant <mode>    config | off | 4 | 8 | turbo[:k<bits>v<bits>]
+                       [default: off, the historical baseline]
 
-Performance levers (--compiled-decode, --perf-kernel, --fused-decode,
---fused-sdpa) apply to the run — A/B by running twice.`,
+Kill switches (--compiled-decode, --compiled-activations, --fused-sdpa)
+apply to the run — A/B by running twice.`,
 
   embed: `mlx-bun embed — text embeddings from a local embedding model
 
@@ -358,9 +429,9 @@ Options:
   --top-k <n>          Top-k sampling
   --seed <n>           Sampler seed (reproducible runs)
 
-Decode-path levers mirror serve: --l1/--l2/--l3 and the per-fork flags
-(--perf-kernel/--fused-sdpa/--compiled-decode/--fused-decode/--kv-quant).
-The tier's KV scheme applies here too (--l2/--l3 = the model's kv_config,
+Decode-path levers mirror serve: --l1/--l2 and the per-fork flags
+(--fused-sdpa/--compiled-decode/--compiled-activations/--kv-quant).
+The tier's KV scheme applies here too (--l2 = the model's kv_config,
 same tokens as serve); with no tier/--kv-quant the KV cache stays bf16
 (the bit-exact mlx-lm greedy path). mlx-lm compat = --l1.
 
@@ -720,69 +791,96 @@ function openChatUi(url: string, hostPort: string): void {
 /** Server/runtime flags shared by every mode that loads a model
  *  (serve, pi). Env levers are set here so they're in place before the
  *  generate/compiled-decode modules read them. */
-/** Resolve the decode ROUTE: a tier alias (--l1/--l2/--l3) sets the whole route,
- *  and an explicit per-fork flag (--perf-kernel/--kv-quant/…) overrides the alias.
- *  The tier IS the intent; the forks are the escape hatch. Sets the decode env
- *  levers and returns the kv-quant mode. See docs/design/parity-tier-dag.md.
+/** Resolve the decode ROUTE: a tier alias (--l1/--l2) sets the whole route,
+ *  and an explicit per-fork flag (--kv-quant/--fused-sdpa/…) overrides the
+ *  alias. Sets the decode env levers and returns the kv-quant mode. See
+ *  docs/design/parity-tier-dag.md + unified-engine-frontier-plan.md.
  *  Each tier is a GUARANTEE about which reference you reproduce bit-for-bit:
  *    --l1  bit-for-bit IDENTICAL to mlx-lm    — drop-in replacement for mlx-lm
- *    --l2  bit-for-bit IDENTICAL to mlx-optiq — drop-in replacement for mlx-optiq.
- *          The optiq-golden decode composition (scripts/regen-kvq-goldens.ts) is
- *          fused N-tiled SDPA for L>1 + STOCK unfused L=1 decode, so L2 = fused-sdpa
- *          ON, perf-kernel OFF. The perf kernel is an mlx-bun ORIGINAL Metal kernel,
- *          envelope-gated (≥56/64 teacher-forced argmax vs OUR OWN frozen compat
- *          trajectory, tests/perf-kernel-oracle.test.ts) — an L3 node, never the
- *          bare-L2 default. (Commit f1bf5cc put it in L2 claiming the goldens track
- *          it; they don't — reverted 2026-07-01 with the evidence above.)
- *    --l3  best performance, NO bit-exact guarantee — correct, but gated by KL +
- *          tests, because mlx-optiq has no analogy for what L3 does.
- *  L2 ⊂ L3 (L3 = L2 + perf kernel). L1 is a separate target (bf16, unfused). */
-function applyDecodeRoute(): { kvQuant?: "off" | "config" | number } {
+ *          (bf16 KV; THE naked default since 2026-07-05)
+ *    --l2  bit-for-bit IDENTICAL to mlx-optiq — drop-in replacement for
+ *          mlx-optiq: quantized KV per kv_config.json + fused N-tiled SDPA for
+ *          L>1 + STOCK unfused L=1 decode (the optiq-golden composition,
+ *          scripts/regen-kvq-goldens.ts + regen-mixed-kv-goldens.ts).
+ *  There is no --l3: output-changing experiments live in the Lab (env flags
+ *  with a bench + expiry) until one beats the L1 baseline in a paired A/B —
+ *  docs/design/unified-engine-frontier-plan.md §6-7. */
+function applyDecodeRoute(): { kvQuant?: "off" | "config" | number; turboQuant?: TurboQuantScheme } {
   const onOff = (name: string): boolean | null => {
     const v = opt(name); if (v == null) return null;
     if (v === "on" || v === "1" || v === "true") return true;
     if (v === "off" || v === "0" || v === "false") return false;
     console.error(`--${name} expects on|off (got "${v}")`); process.exit(1);
   };
-  type Preset = { kv: "off" | "config"; perf: boolean; fusedSdpa: boolean; compiled: boolean; fusedDecode: boolean };
+  if (flag("l3")) {
+    console.error(
+      "--l3 was removed 2026-07-05: no output-changing kernel beat the L1 baseline " +
+      "in a paired A/B, so the tier had no content. Use --l2 for the optiq route, " +
+      "or the Lab (docs/design/unified-engine-frontier-plan.md) for experiments.",
+    );
+    process.exit(1);
+  }
+  type Preset = { kv: "off" | "config"; fusedSdpa: boolean; compiled: boolean; compiledAct: boolean };
   const TIERS: Record<string, Preset> = {
     // A tier is the GUARANTEE; within it we default to the FAST kernel that still
     // holds it, and expose the slow one as an opt-in. compiled-decode is BIT-EXACT
     // with uncompiled (compiled-decode.test: on==off) → free speed, so it's ON in
-    // EVERY tier (--compiled-decode off is the slow, same-tier opt-in). fused-sdpa
-    // matches OPTIQ bit-for-bit (tier-a goldens, fused-sdpa.test) and is
-    // quantized-KV only → it's the L2 bridge, a no-op on L1's bf16. perf-kernel is
-    // NOT: it's an original flash-decoding kernel, envelope-gated against our own
-    // frozen trajectory (perf-kernel-oracle.test) — bare --l2 must ship the
-    // optiq-golden decode composition (stock unfused L=1), so perf-kernel is
-    // L3/explicit-only (--perf-kernel on opts a tier into it by choice). This is
-    // the decode-axis difference between L2 and L3; L3 also owns the no-oracle
-    // FEATURES (HLG sampler, expert offload, batched mixed-precision).
-    l1: { kv: "off",    perf: false, fusedSdpa: false, compiled: true, fusedDecode: false }, // = mlx-lm bit-for-bit (bf16)
-    l2: { kv: "config", perf: false, fusedSdpa: true,  compiled: true, fusedDecode: false }, // = mlx-optiq bit-for-bit
-    l3: { kv: "config", perf: true,  fusedSdpa: true,  compiled: true, fusedDecode: false }, // best perf (envelope-gated decode)
+    // EVERY tier (--compiled-decode off is the slow, same-tier opt-in). compiled
+    // ACTIVATIONS (geglu/swiglu) go through the SAME libmlx as mlx-lm's `@mx.compile`
+    // → same kernel, bit-exact AND faster → ON in every tier (--compiled-activations
+    // off is the slower same-parity opt-in). fused-sdpa matches OPTIQ bit-for-bit
+    // (tier-a goldens, fused-sdpa.test) and is quantized-KV only → it's the L2 bridge,
+    // a no-op on L1's bf16.
+    l1: { kv: "off",    fusedSdpa: false, compiled: true, compiledAct: true }, // = mlx-lm bit-for-bit (bf16)
+    l2: { kv: "config", fusedSdpa: true,  compiled: true, compiledAct: true }, // = mlx-optiq bit-for-bit
   };
-  const tier = flag("l1") ? "l1" : flag("l2") ? "l2" : flag("l3") ? "l3" : null;
-  const p = tier ? TIERS[tier]! : null;
+  // NAKED DEFAULT = L1 (decided 2026-07-05): with no tier flag you get the
+  // mlx-lm bit-for-bit route. The 2026-07-05 benchmark pass showed the L1
+  // faithful kernel set matches mlx-lm 1.00× on every model while every
+  // output-changing lever failed to beat it in a paired A/B (the losers were
+  // deleted the same day — Phase 1 of unified-engine-frontier-plan.md).
+  // Quantized KV costs 5–20% decode at ≤16k on BOTH stacks and pays only in
+  // memory/context headroom → explicit opt-in (--kv-quant / --l2).
+  const tier = flag("l2") ? "l2" : "l1";
+  const p = TIERS[tier]!;
+  const kv = opt("kv-quant"); // explicit --kv-quant overrides the tier
   const pick = (name: string, base: boolean | undefined): boolean | null => {
     const ex = onOff(name); return ex !== null ? ex : (base ?? null); // explicit flag wins, else tier preset
   };
   const set = (env: string, val: boolean | null, invert = false) => {
     if (val !== null) process.env[env] = (invert ? !val : val) ? "1" : "0";
   };
-  set("MLX_BUN_COMPILED_DECODE", pick("compiled-decode", p?.compiled));
-  set("MLX_BUN_PERF_KERNEL", pick("perf-kernel", p?.perf));
-  set("MLX_BUN_FUSED_DECODE", pick("fused-decode", p?.fusedDecode));
-  set("MLX_BUN_NO_FUSED_SDPA", pick("fused-sdpa", p?.fusedSdpa), true); // inverted env
-  const kv = opt("kv-quant"); // explicit --kv-quant overrides the tier
+  // Explicit --kv-quant picks the composition its ORACLE uses: config (the
+  // optiq mixed scheme) runs optiq's fused SDPA (the L2 golden composition);
+  // uniform 4/8 runs UNFUSED (mlx-lm's quantized_scaled_dot_product_attention
+  // — the bit-exact L1-eligible scheme). --fused-sdpa still overrides either.
+  const fusedSdpaBase = kv === "config" ? true : kv != null ? false : p.fusedSdpa;
+  set("MLX_BUN_COMPILED_DECODE", pick("compiled-decode", p.compiled));
+  set("MLX_BUN_NO_FUSED_SDPA", pick("fused-sdpa", fusedSdpaBase), true); // inverted env
+  // Compiled activations (the faithful geglu/swiglu — mlx-lm's @mx.compile kernel).
+  // One fork drives both env vars; qwen3/qwen3.5/universal compile unconditionally,
+  // so this toggles gemma geglu + minicpm5 swiglu (the only unfused-capable sites).
+  set("MLX_BUN_COMPILED_GEGLU", pick("compiled-activations", p.compiledAct));
+  set("MLX_BUN_COMPILED_SWIGLU", pick("compiled-activations", p.compiledAct));
   if (kv === "off") return { kvQuant: "off" };
   if (kv === "config") return { kvQuant: "config" };
   if (kv) {
+    // TurboQuant (docs/design/turboquant-kv.md): a distinct axis from the
+    // affine kvQuant above — dequantize-on-fetch, stock unfused ops.sdpa,
+    // no kvQuant value at all (mutually exclusive with kvBits/kvConfig).
+    let turboQuant: TurboQuantScheme | null;
+    try {
+      turboQuant = parseTurboQuantScheme(kv);
+    } catch (e) {
+      console.error((e as Error).message);
+      process.exit(1);
+    }
+    if (turboQuant) return { turboQuant };
     const bits = Number(kv);
-    if (![4, 8].includes(bits)) { console.error(`--kv-quant expects config|off|4|8 (got "${kv}")`); process.exit(1); }
+    if (![4, 8].includes(bits)) { console.error(`--kv-quant expects config|off|4|8|turbo (got "${kv}")`); process.exit(1); }
     return { kvQuant: bits };
   }
-  return p ? { kvQuant: p.kv } : {};
+  return { kvQuant: p.kv };
 }
 
 function serverRuntimeFlags(): { port: number; serverOptions: import("./server").ServerOptions } {
@@ -794,14 +892,26 @@ function serverRuntimeFlags(): { port: number; serverOptions: import("./server")
     console.error(`--${name} expects on|off (got "${v}")`);
     process.exit(1);
   };
-  const route = applyDecodeRoute(); // --l1/--l2/--l3 tier alias, with per-fork flags overriding
+  const route = applyDecodeRoute(); // --l1/--l2 tier alias, with per-fork flags overriding
   if (flag("force-wire")) process.env.MLX_BUN_FORCE_WIRE = "1";
+  // Remote image_url/audio_url media fetches block private/loopback/link-
+  // local destinations by default (SSRF — src/media-fetch.ts); this is the
+  // LAN-hosts escape hatch.
+  if (flag("allow-private-media")) process.env.MLX_BUN_ALLOW_PRIVATE_MEDIA = "1";
 
   const serverOptions: import("./server").ServerOptions = {};
   const budgetGB = Number(opt("memory-budget", "0"));
   if (budgetGB > 0) serverOptions.memoryBudgetBytes = budgetGB * 1e9;
+  // Aggregate KV budget across batch rows (batching-perf-path P3 admission).
+  const kvBudgetGB = Number(opt("kv-budget", "0"));
+  if (kvBudgetGB > 0) serverOptions.kvBudgetBytes = kvBudgetGB * 1e9;
   const pcRaw = opt("prompt-cache"); // null if absent; an explicit value (incl. 0) wins → `--prompt-cache 0` DISABLES the cache
   if (pcRaw !== null) serverOptions.promptCacheBytes = Math.max(0, Number(pcRaw)) * 2 ** 30;
+  // Engine-child mode (runtime-isolation.md): listen on a unix socket
+  // instead of TCP — the isolated parent proxies here. Internal flag,
+  // spawned by --isolate; usable directly for socket-level integration.
+  const unixRaw = opt("unix");
+  if (unixRaw !== null) serverOptions.unixSocket = unixRaw;
   // SSD cold tier (docs/design/ssd-kv-cold-tier.md) — off unless a dir is given.
   const ssdDir = opt("ssd-cache");
   if (ssdDir !== null) {
@@ -809,13 +919,21 @@ function serverRuntimeFlags(): { port: number; serverOptions: import("./server")
     const maxRaw = opt("ssd-cache-max");
     if (maxRaw !== null) serverOptions.ssdCacheMaxBytes = Math.max(1, Number(maxRaw)) * 2 ** 30;
     if (flag("ssd-cache-verify")) serverOptions.ssdCacheVerify = true;
+    const demoteRaw = opt("ssd-demote-idle");
+    if (demoteRaw !== null) serverOptions.ssdDemoteIdleSec = Math.max(0, Number(demoteRaw));
+  } else {
+    // Refuse-loudly house rule: sub-flags of an absent tier must not vanish.
+    for (const f of ["ssd-cache-max", "ssd-demote-idle"] as const)
+      if (opt(f) !== null) console.warn(`--${f} has no effect without --ssd-cache — ignored`);
+    if (flag("ssd-cache-verify"))
+      console.warn("--ssd-cache-verify has no effect without --ssd-cache — ignored");
   }
   // --batch N: max concurrent requests batched through the mlx-lm-parity
   // engine (N=1 = today's serial path). --decode-concurrency is accepted for
   // drop-in compatibility with mlx_lm.server, but the semantics differ: there
   // it caps per-BatchGenerator decode parallelism (default 32); in mlx-bun it
-  // enables continuous batching with this cap (the default is the optimized
-  // serial path).
+  // caps continuous batching (default 8; `--batch 1` pins the serial path —
+  // the unified engine is bit-parity with it and costs ~nothing at B=1).
   const batchRaw = opt("batch") ?? opt("decode-concurrency");
   if (batchRaw !== null) {
     const n = Number(batchRaw);
@@ -826,6 +944,19 @@ function serverRuntimeFlags(): { port: number; serverOptions: import("./server")
     serverOptions.batch = n;
   }
   if (route.kvQuant !== undefined) serverOptions.kvQuant = route.kvQuant;
+  if (route.turboQuant !== undefined) serverOptions.turboQuant = route.turboQuant;
+  // --paged-kv (env: MLX_BUN_PAGED_KV=1): OPTIONAL vLLM-style block-pool KV
+  // storage, default off (docs/design/paged-kv-cache.md). Serial-only in v1:
+  // with no explicit --batch, pin --batch 1 (the default is 8); an explicit
+  // --batch N>1 is refused loudly by createServer rather than downgraded.
+  if (flag("paged-kv") || process.env.MLX_BUN_PAGED_KV === "1") {
+    const bsRaw = opt("paged-kv-block-size");
+    serverOptions.pagedKv = bsRaw !== null ? { blockSize: Number(bsRaw) } : {};
+    if (batchRaw === null) {
+      serverOptions.batch = 1;
+      console.log("[paged-kv] serial-only in v1 — pinning --batch 1");
+    }
+  }
   // Bind loopback unless asked otherwise (mlx_lm.server parity); --host
   // 0.0.0.0 is the explicit opt-in for LAN exposure. The chat-UI open and
   // `mlx-bun pi` attach both go through localhost, so loopback-only is
@@ -904,11 +1035,11 @@ async function mountStartupAdapter(
 
 /** One-line summary of the active runtime levers for the ready card. */
 function runtimeSummary(o: import("./server").ServerOptions): string {
-  const kv = o.kvQuant === "off" ? "off" : typeof o.kvQuant === "number" ? `kv${o.kvQuant}` : "config";
+  const kv = o.turboQuant ? `turbo-k${o.turboQuant.kBits}v${o.turboQuant.vBits}`
+    : o.kvQuant === "off" ? "off" : typeof o.kvQuant === "number" ? `kv${o.kvQuant}` : "config";
   const lever = (env: string, dflt: string) => process.env[env] ?? dflt;
   return `kv-quant ${kv} · compiled-decode ${lever("MLX_BUN_COMPILED_DECODE", "1") === "1" ? "on" : "off"}` +
-    ` · perf-kernel ${perfKernelEnabled() ? "on" : "off"}` +
-    (lever("MLX_BUN_FUSED_DECODE", "0") === "1" ? " · fused-decode on" : "") +
+    (lever("MLX_BUN_COMPILED_GEGLU", "1") === "0" ? " · compiled-activations off" : "") +
     (o.batch && o.batch > 1 ? ` · batch ${o.batch}` : "") +
     (o.defaultThinking !== undefined ? ` · thinking ${o.defaultThinking ? "on" : "off"}` : "") +
     (o.defaultTemperature !== undefined ? ` · temp ${o.defaultTemperature}` : "") +
@@ -958,7 +1089,7 @@ async function resolveModelAuto(query: string | null): Promise<{ m: import("./re
     await reg.scan();
     if (recRepo !== starterRepo) {
       console.log(`  ${style.dim(`downloading ${recRepo} (recommended for this Mac) in the background —`)}`);
-      console.log(`  ${style.dim("it becomes the default next run · progress on the status page (/downloads) · resumable")}`);
+      console.log(`  ${style.dim("it becomes the default next run · progress in the web app's Status view · resumable")}`);
       downloadModel(recRepo)
         .then(async () => {
           await new Registry().scan();
@@ -1248,7 +1379,7 @@ switch (cmd) {
       console.log(style.dim("  no command given — starting the server · mlx-bun --help for all commands"));
     const rt = serverRuntimeFlags();
     // Friendly collision check before loading gigabytes of weights.
-    {
+    if (!rt.serverOptions.unixSocket) {
       const { probeServer } = await import("./harness-pi");
       const running = await probeServer(`http://localhost:${rt.port}/v1`);
       if (running) {
@@ -1258,6 +1389,71 @@ switch (cmd) {
         process.exit(1);
       }
     }
+    // Runtime isolation (--isolate, runtime-isolation.md): THIS process
+    // becomes a pure reactor — it binds the TCP port and reverse-proxies to
+    // an engine CHILD (the full server on a unix socket, spawned from the
+    // same argv minus parent-only flags). Zero MLX calls in the parent: the
+    // UI stays instant under any GPU load, and an engine crash respawns
+    // without taking the session down.
+    if (!flag("isolate") && opt("model-pool") !== null)
+      console.warn("--model-pool has no effect without --isolate (child-per-model pool) — ignored");
+    if (flag("isolate")) {
+      const { startProxyServer, engineArgvForModel, defaultSocketPath } = await import("./serve/isolate");
+      const sock = defaultSocketPath();
+      // Re-spawn ourselves: compiled binary = execPath alone (argv[1] is a
+      // $bunfs virtual path); dev = bun + the script path.
+      const selfArgv = process.argv[1] && !process.argv[1].includes("$bunfs")
+        ? [process.execPath, process.argv[1]]
+        : [process.execPath];
+      const rawArgs = process.argv.slice(2);
+      // Resolve the DEFAULT model in the parent (registry lookup — no
+      // weights load) so the pool has its key and the child gets a pinned
+      // --model path instead of re-running fuzzy resolution.
+      const { m: m0 } = await resolveModelAuto(opt("model") ?? positional(0) ?? opt("query"));
+      // STRICT resolver for the pool (P2): only EXACT /v1/models ids
+      // switch models. Fuzzy matching here would hijack drop-in clients
+      // that send model:"gpt-4"-style strings (which must keep today's
+      // ignored-field semantics and ride the default engine).
+      const { Registry } = await import("./registry");
+      const poolResolve = (query: string): { repoId: string; path: string } | null => {
+        if (query === m0.repoId) return { repoId: m0.repoId, path: m0.path };
+        try {
+          const reg = new Registry();
+          try {
+            const hit = reg.listCanonical().find((r) => r.repoId === query);
+            return hit ? { repoId: hit.repoId, path: hit.path } : null;
+          } finally {
+            reg.close();
+          }
+        } catch {
+          return null;
+        }
+      };
+      const poolMaxRaw = opt("model-pool");
+      const poolMax = Math.max(1, Number(poolMaxRaw ?? 1) || 1);
+      const { engine, pool } = startProxyServer({
+        port: rt.port,
+        ...(rt.serverOptions.hostname ? { hostname: rt.serverOptions.hostname } : {}),
+        engine: {
+          argv: [...selfArgv, ...engineArgvForModel(rawArgs, sock, m0.path)],
+          socketPath: sock,
+        },
+        pool: {
+          rawArgs, selfArgv, poolMax,
+          resolve: poolResolve,
+          defaultKey: m0.repoId,
+        },
+      });
+      const shownHost0 = rt.serverOptions.hostname ?? "localhost";
+      console.log(style.dim(`  isolated mode: engine child starting (socket ${sock})`));
+      console.log(`  ${style.green("●")} ${style.bold("proxy up")} http://${shownHost0}:${rt.port} ${style.dim(`· ${m0.repoId} loading behind it · model pool ${poolMax}`)}`);
+      const onExit = () => { pool?.stopAll(); engine.stop(); process.exit(0); };
+      process.on("SIGINT", onExit);
+      process.on("SIGTERM", onExit);
+      await engine.ready;
+      console.log(`  ${style.green("●")} ${style.bold("engine ready")} ${style.dim(`pid ${engine.pid}`)}`);
+      await new Promise(() => {}); // serve until killed
+    }
     // --model <path-or-query> is the explicit override (mlx_lm.server's
     // spelling — previously accepted-but-IGNORED here, which silently
     // auto-picked the wrong model). A directory with config.json loads
@@ -1285,10 +1481,65 @@ switch (cmd) {
         sOff.done(`experts mmap'd ${style.dim(dir)} ${style.dim("· phys_footprint ≈ core")}`);
       }
     }
+    // Speculative decoding (mlx_lm.server parity): --draft-model <path|query>
+    // resolves like the main model; --num-draft-tokens defaults to 3 (the
+    // server's default upstream — mlx_lm.generate's is 2). Serial-lane-only:
+    // with --batch N, a mounted draft routes every request serial (upstream
+    // is_batchable = draft is None).
+    const draftQuery = opt("draft-model");
+    let draftModelDir: string | undefined;
+    if (draftQuery !== null) {
+      const { m: dm } = await resolveModelAuto(draftQuery);
+      draftModelDir = dm.path;
+    }
+    const numDraftRaw = opt("num-draft-tokens");
+    const numDraftTokens = numDraftRaw !== null ? Number(numDraftRaw) : undefined;
+    if (numDraftTokens !== undefined && (!Number.isInteger(numDraftTokens) || numDraftTokens < 1)) {
+      console.error(`--num-draft-tokens expects an integer >= 1 (got "${numDraftRaw}")`);
+      process.exit(1);
+    }
+    const draftKindRaw = opt("draft-kind");
+    const draftKind = (draftKindRaw ?? undefined) as
+      | "dspark" | "deepspec" | "assistant" | "two-model" | "ngram" | undefined;
+    if (draftKind !== undefined && !["dspark", "deepspec", "assistant", "two-model", "ngram"].includes(draftKind)) {
+      console.error(`--draft-kind expects two-model|assistant|dspark|deepspec|ngram (got "${draftKindRaw}")`);
+      process.exit(1);
+    }
+    // Prompt-lookup window bounds (ngram kind only; validated here so a typo
+    // fails the launch, not the first request).
+    const ngramInt = (name: string): number | undefined => {
+      const raw = opt(name);
+      if (raw === null) return undefined;
+      const v = Number(raw);
+      if (!Number.isInteger(v) || v < 1) {
+        console.error(`--${name} expects an integer >= 1 (got "${raw}")`);
+        process.exit(1);
+      }
+      return v;
+    };
+    const ngramMax = ngramInt("ngram-max");
+    const ngramMin = ngramInt("ngram-min");
+    if (ngramMax !== undefined && ngramMin !== undefined && ngramMin > ngramMax) {
+      console.error(`--ngram-min (${ngramMin}) must be <= --ngram-max (${ngramMax})`);
+      process.exit(1);
+    }
+    if ((ngramMax !== undefined || ngramMin !== undefined) && draftKind !== "ngram")
+      console.warn("--ngram-max/--ngram-min only apply with --draft-kind ngram — ignored");
     const sLoad = step("loading weights");
     const t0 = performance.now();
-    const ctx = await loadContext(m.path, m.repoId, { memoryBudgetBytes: rt.serverOptions.memoryBudgetBytes });
-    sLoad.done(`weights loaded ${style.dim(`in ${(performance.now() - t0).toFixed(0)} ms`)}`);
+    const ctx = await loadContext(m.path, m.repoId, {
+      memoryBudgetBytes: rt.serverOptions.memoryBudgetBytes,
+      ...(draftModelDir || draftKind === "ngram"
+        ? {
+            ...(draftModelDir ? { draftModelDir } : {}),
+            numDraftTokens,
+            ...(draftKind ? { draftKind } : {}),
+            ...(ngramMax !== undefined ? { ngramMax } : {}),
+            ...(ngramMin !== undefined ? { ngramMin } : {}),
+          }
+        : {}),
+    });
+    sLoad.done(`weights loaded ${style.dim(`in ${(performance.now() - t0).toFixed(0)} ms`)}${draftModelDir ? style.dim(` · draft: ${draftModelDir.split("/").filter(Boolean).at(-1)}`) : draftKind === "ngram" ? style.dim(" · draft: ngram (prompt lookup)") : ""}`);
     await mountStartupAdapter(ctx, rt.serverOptions);
     const server = createServer(ctx, rt.port, { ...rt.serverOptions, owner: "serve" });
     const shownHost = rt.serverOptions.hostname ?? "localhost";
@@ -1321,7 +1572,7 @@ switch (cmd) {
     // three entry points (raw / OpenAI API / chat UI) differ only in how params
     // are POPULATED; here every param is explicit. Decode-path levers mirror
     // serve so you can pin the route (mlx-lm compat = --l1).
-    // --l1/--l2/--l3 + per-fork overrides set the decode env levers; the
+    // --l1/--l2 + per-fork overrides set the decode env levers; the
     // returned KV scheme is applied below so `generate --l2` runs the same
     // quantized-KV route (and produces the same tokens) as `serve --l2`.
     const route = applyDecodeRoute();
@@ -1330,7 +1581,7 @@ switch (cmd) {
     if (!prompt) {
       console.error('usage: mlx-bun generate [query] --prompt "…" [--raw] [--max-tokens N]');
       console.error('       sampling:  --temperature N (alias --temp) --top-p N --top-k N --seed N');
-      console.error('       decode path: --l1 (= mlx-lm compat) / --l2 / --l3, or per-fork flags');
+      console.error('       decode path: --l1 (= mlx-lm compat, the default) / --l2, or per-fork flags');
       process.exit(1);
     }
     const { loadTaskModel, generateText } = await import("./eval/runner");
@@ -1341,7 +1592,8 @@ switch (cmd) {
     // start. Unset (no tier, no --kv-quant) keeps generate's historical
     // default: bf16 bit-exact greedy (generateText's parity path).
     const kvScheme =
-      route.kvQuant === "off" ? {}
+      route.turboQuant ? { turboQuant: route.turboQuant, quantizedKvStart: 0 }
+      : route.kvQuant === "off" ? {}
       : route.kvQuant === "config"
         ? (tm.config.kvQuant?.length ? { kvConfig: tm.config.kvQuant } : {})
       : typeof route.kvQuant === "number"
@@ -1427,7 +1679,7 @@ switch (cmd) {
   case "benchmark": {
     const { banner, step, box, style } = await import("./tui");
     banner(pkg.version);
-    serverRuntimeFlags(); // env levers (--compiled-decode, --perf-kernel, ...) apply to the run
+    serverRuntimeFlags(); // env levers (--compiled-decode, --fused-sdpa, ...) apply to the run
     const { m, picked } = await resolveModelAuto(opt("model") ?? positional(0) ?? opt("query"));
     const tokens = Number(opt("tokens", "256"));
     const runs = Number(opt("runs", "3"));
@@ -1462,7 +1714,16 @@ switch (cmd) {
     const rendered = template.render([{ role: "user", content: userMsg }]);
     const ids = tok.encode(rendered);
     const promptIds = ids[0] === ids[1] && ids[0] === tok.bosTokenId ? ids.slice(1) : ids;
-    const kvOptions = kvMode === "config"
+    let turboScheme: TurboQuantScheme | null;
+    try {
+      turboScheme = parseTurboQuantScheme(kvMode);
+    } catch (e) {
+      console.error((e as Error).message);
+      process.exit(1);
+    }
+    const kvOptions = turboScheme
+      ? { turboQuant: turboScheme, quantizedKvStart: 0 }
+      : kvMode === "config"
       ? (() => {
           if (!config.kvQuant?.length) { console.error("model has no kv_config.json (--kv-quant config)"); process.exit(1); }
           return { kvConfig: config.kvQuant };
@@ -1516,7 +1777,7 @@ switch (cmd) {
       `decode    ${style.green(style.bold(`${median(decodes).toFixed(1)} tok/s`))} ${style.dim(`median of ${runs} · [${decodes.map((d) => d.toFixed(1)).join(", ")}]`)}`,
       `prefill   ${style.bold(`${median(prefills).toFixed(0)} tok/s`)} ${style.dim(`· ${lastStats.promptTokens} prompt tokens`)}`,
       `peak mem  ${style.bold(gb(peak))} ${style.dim("(generation only)")}`,
-      `levers    ${style.dim(runtimeSummary({ kvQuant: kvMode === "off" ? "off" : kvMode === "config" ? undefined : Number(kvMode) }))}`,
+      `levers    ${style.dim(runtimeSummary(turboScheme ? { turboQuant: turboScheme } : { kvQuant: kvMode === "off" ? "off" : kvMode === "config" ? undefined : Number(kvMode) }))}`,
       "",
       style.dim("recorded to the eval DB — see mlx-bun evals"),
     ]);
@@ -1616,14 +1877,16 @@ switch (cmd) {
     // (src/pi-terminal.ts) — pi is bundled, nothing to install. Users who
     // already run their own pi connect it with `mlx-bun harness pi`.
     const OURS_VAL = new Set([
-      "--query", "-q", "--model", "--port", "--host", "--memory-budget", "--prompt-cache",
-      "--ssd-cache", "--ssd-cache-max", "--kv-quant",
+      "--query", "-q", "--model", "--port", "--host", "--memory-budget", "--kv-budget", "--prompt-cache",
+      "--ssd-cache", "--ssd-cache-max", "--ssd-demote-idle", "--kv-quant", "--unix",
       "--batch", "--decode-concurrency", "--adapter", "--adapter-path",
-      "--compiled-decode", "--perf-kernel", "--fused-decode", "--fused-sdpa", "--thinking",
+      "--draft-model", "--num-draft-tokens", "--draft-kind", "--ngram-max", "--ngram-min",
+      "--compiled-decode", "--compiled-activations", "--fused-sdpa", "--thinking",
       "--temperature", "--temp", "--top-p", "--top-k", "--max-tokens",
       "--hlg-sampling", "--hlg-width", "--hlg-shoulder", "--hlg-toe", "--hlg-pivot-offset",
+      "--paged-kv-block-size",
     ]);
-    const OURS_BOOL = new Set(["--force-wire", "--expert-offload", "--no-open", "--ssd-cache-verify", "--l1", "--l2", "--l3"]);
+    const OURS_BOOL = new Set(["--force-wire", "--expert-offload", "--no-open", "--ssd-cache-verify", "--l1", "--l2", "--l3", "--isolate", "--paged-kv"]);
     const passthrough: string[] = [];
     for (let i = 1; i < argv.length; i++) {
       const a = argv[i]!;
@@ -1813,14 +2076,7 @@ switch (cmd) {
 
     const { m, picked } = await resolveModelAuto(query);
 
-    // Detect Gemma/e4b and set its required training env flags BEFORE the
-    // trainer is imported (perfKernelEnabled / fused-gelu read them lazily at
-    // forward time, so setting them here — like the launcher — takes effect).
     const isGemma = (await Bun.file(`${m.path}/config.json`).text()).toLowerCase().includes("gemma");
-    if (isGemma) {
-      process.env.MLX_BUN_PERF_KERNEL ??= "0";
-      process.env.MLX_BUN_FUSED_GELU ??= "0";
-    }
 
     const modelTag = isGemma ? "e4b" : "cpm5";
     const adapter = opt("adapter") ?? `${process.env.HOME}/.cache/mlx-bun/mlx-bun-finetunes/${method}-${modelTag}`;
@@ -1874,7 +2130,7 @@ switch (cmd) {
     if (!ds.ok) { console.error(`dataset: ${ds.error}`); process.exit(1); }
 
     const planLines = [
-      `${style.green("●")} ${style.bold(`train ${method}`)} ${style.dim(`· ${m.repoId}${picked ? " (auto-picked)" : ""}${isGemma ? " · e4b env set" : ""}`)}`,
+      `${style.green("●")} ${style.bold(`train ${method}`)} ${style.dim(`· ${m.repoId}${picked ? " (auto-picked)" : ""}${isGemma ? " · e4b defaults" : ""}`)}`,
       "",
       `data       ${style.bold(`${ds.n_train} train`)}${ds.n_valid ? ` · ${ds.n_valid} valid` : ""} ${style.dim(`· format ${ds.format}`)}`,
       `loop       ${style.dim(`iters ${iters} · lr ${cfg.learning_rate} · rank ${cfg.rank} · scale ${cfg.scale} · seq ${seq} · batch ${cfg.batch_size}`)}`,
@@ -2258,11 +2514,6 @@ switch (cmd) {
       if (reg.list().length === 0) await reg.scan();
       modelDir = reg.resolve(modelArg).path;
     }
-    // Gemma/e4b: same forward env flags the trainer sets (full-sequence pass).
-    if ((await Bun.file(`${modelDir}/config.json`).text()).toLowerCase().includes("gemma")) {
-      process.env.MLX_BUN_PERF_KERNEL ??= "0";
-      process.env.MLX_BUN_FUSED_GELU ??= "0";
-    }
     const sLoad = step(`loading ${modelDir}`);
     const { Weights } = await import("./weights");
     const { createModel } = await import("./model/factory");