mlx-bun 0.0.13 → 0.2.0

package/docs/reference/server-api.md

@@ -67,7 +67,9 @@ Request body (OpenAI chat schema; unknown fields ignored):
                                  // wins. On templates with a reasoning-depth
                                  // variable (Qwen3.8) the level also maps
                                  // into the template: minimal/low → low,
-                                 // medium → medium, high/xhigh → xhigh
+                                 // medium → medium, high/xhigh → xhigh.
+                                 // Any OTHER string is a 400 — a typo used
+                                 // to silently force thinking ON
                                  // (Qwen3.8's default depth is xhigh).
   "hlg": {                       // HLG tone-curve sampling (per request).
     "enabled": true,             // merged over --hlg-sampling server defaults.
@@ -106,16 +108,37 @@ Message `content` is a string or an array of parts:
 (http/https URLs also accepted; PNG, JPEG, HEIC, AVIF, WebP, TIFF, GIF,
 BMP via native OS codecs; requires a model with the vision sidecar).
 
-Remote (http/https) media URLs — image and audio alike — go through a
-destination policy: **private/loopback/link-local hosts are refused by
-default** (the request URL is attacker-controlled input — SSRF), every
-redirect hop is re-validated, and the fetch has a **10 s timeout** and a
-**64 MB response cap**. Violations are clean `400`s
+Remote (http/https) media URLs — image, audio, and video alike — go
+through a destination policy: **private/loopback/link-local hosts are
+refused by default** (the request URL is attacker-controlled input —
+SSRF), every redirect hop is re-validated, and the fetch has a **10 s
+timeout** and a **64 MB response cap** (video: **256 MB** — clips are
+legitimately larger; frame sampling truncates long clips server-side).
+Violations are clean `400`s
 (`prompt build failed: image url rejected: …`). Serving media from a LAN
 host (a NAS, another machine) is the opt-in
 [`--allow-private-media`](server-config.md#start-flags) flag. `data:`
 URLs are decoded locally and never policy-checked.
 
+### Video input (Qwen3.5-family models)
+
+Video rides the same content-part array:
+
+```jsonc
+{ "type": "video_url", "video_url": { "url": "data:video/mp4;base64,…|http(s)://…" } }
+{ "type": "video",     "data": "<base64>" }                          // alias
+```
+
+The container decodes through macOS's own AVFoundation codecs (mp4/mov,
+H.264/HEVC/ProRes/… — whatever this macOS plays) via the
+`mlx-bun-frame-extract` helper that ships beside the binary and in the
+native pack; from a source tree it compiles on first use with the Xcode
+CLT. Frames are sampled at **2 fps, capped at 768 frames**, then run the
+gated Qwen3-VL video pipeline (temporal frame pairs, per-frame-group
+attention, 3D mRoPE positions). Qwen3.5-family models only; video never
+combines with audio parts, and a non-Qwen model answers video parts with
+a clean `400`.
+
 ### Audio input
 
 Audio rides the same content-part array (OpenAI-canonical shape plus two
@@ -188,7 +211,7 @@ Non-streaming response:
     "prompt_tokens_details": { "cached_tokens": 0 },  // prompt-cache reuse
     "lane": "serial",              // "serial" | "serial+spec" | "batched" —
                                     //   which execution lane served THIS
-                                    //   request (GenerationGateway.willBatch
+                                    //   request (GenerationGateway.place
                                     //   + whether a draft actually contributed
                                     //   accepted tokens). Chat completions
                                     //   only (streaming: final usage chunk).
@@ -344,12 +367,11 @@ All errors are `{ "error": { "message": …, ... } }`.
   `top_logprobs must be of type int` / `at least 0` / `at most 11`;
   `top_logprobs: -1` is the accepted "unset" sentinel).
 - `400` with `"type": "memory_admission"`, `"code":
-  "context_over_budget"` — the request cannot fit the memory budget's max
-  safe context. Generic budgeted serving rejects when `prompt + max_tokens`
-  exceeds the ceiling. GLM-5.2 instead caps a broad `max_tokens` upper bound
-  to the space remaining in its fixed planned context and returns this error
-  only when the prompt itself leaves no generation slot. The ceiling is visible
-  at `/stats`.
+  "context_over_budget"` — the PROMPT itself leaves no generation slot
+  within the max safe context. A prompt that fits is never rejected for a
+  broad `max_tokens`: the upper bound is capped to the remaining room
+  (`max_tokens` is a ceiling, not a promise) and generation proceeds. The
+  ceiling is visible at `/stats`.
 
 ## POST /v1/completions (raw text completion)
 
@@ -581,7 +603,9 @@ vision/audio, adapters, and training are not emulated by the serving port.
   "ssd_cache": { "dir": "...", "entries": 0, "bytes": 0, "max_bytes": 0,
                  "restores": 0, "spills": 0, "restore_ms_last": 0, "demotions": 0,
                  // bounded write-behind queue (MLX_BUN_SSD_SPILL_QUEUE_GB):
-                 "pending_spills": 0, "pending_spill_bytes": 0, "dropped_spills": 0 },
+                 "pending_snapshots": 0, "pending_spills": 0,
+                 "pending_spill_bytes": 0, "dropped_spills": 0,
+                 "failed_spills": 0, "longest_durable_prefix_tokens": 0 },
   "admission": {
     "max_safe_context": 0,            // tokens; requests above this 400
     "memory_budget_bytes": null,      // explicit budget, or null (machine default)
@@ -631,6 +655,37 @@ vision/audio, adapters, and training are not emulated by the serving port.
 }
 ```
 
+`pending_snapshots` counts prompt-cache entries scheduled for persistence but
+not yet confirmed by the atomic SSD store. `longest_durable_prefix_tokens` is
+the longest prefix already present in the SSD index, not merely queued.
+
+## POST /admin/cache/flush
+
+Forces every pending SSD prompt-cache snapshot through the serial write queue
+and waits for the atomic temp-file/fsync/rename boundary. The response is `200`
+only when no snapshot or spill remains pending and no write failed during this
+flush; otherwise it is `503` with the counters that explain why.
+
+```jsonc
+{
+  "durable": true,
+  "flushedSnapshots": 1,
+  "missingSnapshots": 0,
+  "pendingSnapshots": 0,
+  "pendingSpills": 0,
+  "pendingSpillBytes": 0,
+  "droppedSpills": 0,
+  "failedSpills": 0,
+  "elapsedMs": 842,
+  "entries": 2,
+  "longest_durable_prefix_tokens": 16384
+}
+```
+
+The standard benchmark calls this endpoint before its restart leg and records
+the result. Normal `mlx-bun serve` shutdown on `SIGINT` or `SIGTERM` performs
+the same flush after active requests drain, with a 120-second default timeout.
+
 ## GET /library
 
 Returns all models found in the local HuggingFace hub cache (via the

package/docs/reference/server-config.md

@@ -9,8 +9,8 @@ adapters admin) see [server-api.md](./server-api.md); this doc is about
 
 The flags are parsed in `serverRuntimeFlags()`
 ([src/cli.ts](../../src/cli.ts)); the runtime knobs live in
-`ServerOptions` ([src/server.ts](../../src/server.ts)). The two execution
-lanes (serial vs. `--batch N`) and the lane picker are in
+`ServerOptions` ([src/server.ts](../../src/server.ts)). The serial and
+continuous scheduling mechanisms and their declaration are in
 [src/serve/generation-gateway.ts](../../src/serve/generation-gateway.ts);
 the design rationale is in
 [docs/design/parallel-slots.md](../design/parallel-slots.md).
@@ -21,21 +21,21 @@ the design rationale is in
 | --- | --- | --- | --- |
 | `--host` | addr | `127.0.0.1` | Interface to bind. Loopback-only by default (mlx_lm.server parity); pass `--host 0.0.0.0` to expose the server on your network. |
 | `--port` | n | `8080` | Listen port. A pre-flight probe refuses to start if the port is already serving. |
-| `--memory-budget` | GB | machine RAM × 0.75; GLM-5.2: min(25 GiB, physical RAM) | Admission ceiling. Generic budgeted serving rejects requests whose `prompt + max_tokens` exceed the safe context with **400** (`type: memory_admission`) instead of risking an uncatchable GPU OOM. GLM-5.2 runs its exact process equation before opening resident weights or expert slabs and caps MLX to the plan's allocator reserve; because its context is fixed up front, a broad client `max_tokens` upper bound is capped to the remaining planned context, while a prompt that leaves no generation slot is still rejected. **Decimal GB (×10⁹).** |
+| `--memory-budget` | GB | machine RAM × 0.75; GLM-5.2: min(25 GiB, physical RAM) | Admission ceiling. A request whose prompt fits is always admitted: a broad client `max_tokens` is capped to the room remaining under the safe context (`max_tokens` is a ceiling, not a promise), so agent clients with family-wide defaults never 400 over a small overshoot. Only a prompt that leaves no generation slot is rejected with **400** (`type: memory_admission`) — the alternative is an uncatchable GPU OOM. The safe-context ceiling bills the active `--kv-quant` scheme at its true bytes/element (a 4-bit cache admits the ~3.5× larger window it actually enables; `turbo` is still billed bf16, conservatively). Admission never refuses to *start*: a budget too small for any context logs a warning and serves anyway (requests then get the per-request 400). GLM-5.2 additionally runs its exact process equation before opening resident weights or expert slabs and caps MLX to the plan's allocator reserve. **Decimal GB (×10⁹).** |
 | `--prompt-cache` | GB | `8` | RAM prefix-KV cache (byte-capped LRU; `0` disables). **PREFIX SHARING (2026-07-05): serves are NON-CONSUMING** — a hit hands out zero-copy clones and leaves the donor entry intact, so N agents (or a brand-new session) sharing a system prompt all reuse ONE prefill without destroying each other's conversation entries (the old consume-and-trim take cannibalized the donor). Extended entries supersede their prefix-ancestors (when trimmable), so a conversation stays one entry. Works on BOTH lanes and tiers over the SSD store (Layer 0). |
 | `--isolate` | (bool) | off | Runtime isolation ([runtime-isolation.md](../design/runtime-isolation.md)): the inference engine runs as a CHILD process on a unix socket; this process stays a pure UI/API reverse proxy — instant under any GPU load, survives engine crashes (auto-respawn, in-flight requests get a clean 502). Measured cost ≈0 (−0.4% tok/s, +2 ms TTFT, per-token SSE granularity preserved). `/ws/chat` is not proxied yet (501). `/engine` reports child pid/restarts. |
 | `--model-pool` | n | `1` | With `--isolate`: max RESIDENT model engines. Requests naming another model (**exact `/v1/models` id** — fuzzy strings keep mlx-lm's ignored-field semantics) spawn/route to that model's own engine child (SPAWN-OVERLAP: the new model loads while the old one keeps serving; nobody's stream is interrupted). Over the cap, the LRU engine drains, **demotes its prompt cache to the SSD tier**, and exits — switching back respawns it with state restored from disk. Measured (M1 Max, cpm5⇄qwen0.8b, pool 1): switch 1.5 s, switch-back 1.2 s with `cached_tokens` 103/104 — the conversation survived the eviction. The pool drives this eviction via `POST /admin/drain` — unix-socket-only (never exposed on the public TCP listener) — which quiesces the gateway and demotes the whole prompt cache to the SSD tier before eviction, responding `{ drained: true, demotions: N }`. |
-| `--ssd-cache` | dir | off | SSD cold tier under the prompt cache ([docs/design/ssd-kv-cold-tier.md](../design/ssd-kv-cold-tier.md)): prefix KV spills to disk on RAM eviction AND idle demotion, is snapshotted after requests settle (debounced 1 s; the flush is idle-gated per tensor so it never taxes an active decode — `MLX_BUN_SSD_WRITEBEHIND=0` disables), and **survives restarts** — a long-context agent re-attach restores via a bounded streamed copy instead of re-prefilling (measured 13.7k-token prefix: 12 s → ~0.25 s TTFT; 0% decode overhead; restore transient = live entry + one tensor, no mapping outlives the load). Entries are keyed by model fingerprint **+ effective kv scheme** + tokenizer hash + adapter ns; incompatible/corrupt files self-quarantine. **Layer 0 (2026-07-05): the tiering lives inside PromptCache.take(), so BOTH lanes restore from disk — batch-scheduler joiners included — and every put (batch rows too) schedules the write-behind snapshot.** Requires the RAM cache. |
+| `--ssd-cache` | dir | off | SSD cold tier under the prompt cache ([docs/design/ssd-kv-cold-tier.md](../design/ssd-kv-cold-tier.md)): prefix KV spills to disk on RAM eviction AND idle demotion, is snapshotted after requests settle (debounced 1 s; the flush is idle-gated per tensor so it never taxes an active decode — `MLX_BUN_SSD_WRITEBEHIND=0` disables), and **survives restarts** — a long-context agent re-attach restores via a bounded streamed copy instead of re-prefilling (measured 13.7k-token prefix: 12 s → ~0.25 s TTFT; 0% decode overhead; restore transient = live entry + one tensor, no mapping outlives the load). Dirty snapshots remain tracked until the atomic SSD store confirms them; `SIGINT`/`SIGTERM` drains active requests and flushes them before exit, and `POST /admin/cache/flush` exposes the same explicit boundary. Entries are keyed by model fingerprint **+ effective kv scheme** + tokenizer hash + adapter ns; incompatible/corrupt files self-quarantine. **Layer 0 (2026-07-05): the tiering lives inside PromptCache.take(), so BOTH lanes restore from disk — batch-scheduler joiners included — and every put (batch rows too) schedules the write-behind snapshot.** Requires the RAM cache. |
 | `--ssd-cache-max` | GB | `32` | SSD tier byte cap (binary GiB); oldest-mtime entries evicted at the cap. |
 | `--ssd-cache-verify` | (bool) | off | Verify every tensor hash on restore before it is copied in (roughly doubles restore reads) — integrity paranoia only; the header hash is always verified. |
 | `--ssd-demote-idle` | sec | `300` (with `--ssd-cache`) | Idle demotion (Layer 0): prompt-cache entries unused this long spill to the SSD tier and **free their GPU memory** — unified memory drains between agent bursts while every prefix stays reachable (the next hit restores via the bounded streamed copy, ~0.25 s for a 13.7k-token entry vs a 12 s re-prefill). Swept only when the engine is fully idle (never drains a running batch). `0` disables. |
 | `--batch` | n | `8` | Max concurrent requests decoded together. **Default flipped 1→8 (2026-07-05)** after GATE-B1-SPEED: a LONE request through the unified engine IS the serial engine — adopted serial-class caches, compiled decode, prompt cache + SSD restore, byte-identical output, 0.992–0.996 paired decode ratios — so the cap only changes behavior when concurrent requests actually arrive (the agentic sub-agent workload: 4–8 coding agents against one local server). `--batch 1` pins strict serial (arrival-independent numerics). See [Execution modes](#execution-modes-serial-vs---batch-n). `--decode-concurrency` is accepted for drop-in compatibility (mlx_lm.server's cap, default 32). |
-| `--kv-quant` | `config`\|`off`\|`4`\|`8`\|`turbo[:k<bits>v<bits>]` | `off` (bf16) | KV-cache quantization. **Default flipped to bf16 2026-07-05** (naked = L1): quantized KV measured 5–20% slower decode than bf16 at ≤16k on every model — on mlx-lm too — so it pays only in **memory headroom** (e.g. ~1.3 GB on the 12B @16k) and is an explicit opt-in (`--kv-quant …` or `--l2`, whose preset passes `config`). `off` = bf16 (L1). `4`/`8` = **uniform** bits (group 64, start 0) — the scheme mlx-lm exposes as `--kv-bits`/`--kv-group-size`/`--quantized-kv-start`. In the L1 kernel set (`--l1 --kv-quant 8` = fused-sdpa off) our unfused quantized SDPA is **op-for-op identical** to mlx-lm's `quantized_scaled_dot_product_attention`, so uniform-quant is **bit-exact L1** (the fused-sdpa-ON path is optiq-aligned). `config` = **per-layer mixed-precision** from `kv_config.json` — optiq-only, no mlx-lm analog → **L2**. `turbo[:k<bits>v<bits>]` = **TurboQuant** ([docs/design/turboquant-kv.md](../design/turboquant-kv.md)): rotation-based KV quantization (asymmetric-affine int8/int4/int2/int5 keys, FWHT + Lloyd-Max values) — default `k8v3` (2.56× compression at head_dim 128); `kBits` ∈ {2,4,5,8}, `vBits` ∈ {2,3,4,5,8}. A separate axis from the scheme above (mutually exclusive with `config`/`4`/`8`); v1 is dequantize-on-fetch (stock `ops.sdpa`, no fused kernel) and **full-attention layers only** — sliding-window layers stay bf16 (one-time warning, never a throw). Under `--batch N`, a per-layer `config` scheme **batches** — full-attention layers since Phase 3.1 (MiniCPM5), rotating/sliding layers since milestone 2 (gemma's whole kv_config), so every shipped `kv_config.json` now batches. The scheduler applies the mixed scheme per row, gated bit-exact per row vs the serial composition (`tests/batched-kv-quant-parity.test.ts` — compositions no other stack ships). Uniform `4`/`8` still routes those requests serial (quantizedKvStart threshold semantics). **`turbo` is solo-only, unconditionally** — `TurboQuantKVCache` is a novel `Cache` implementation (no merge/filter/temporalView), so it's excluded from the batch scheduler by construction; `GenerationGateway.willBatch` also refuses it explicitly (belt + braces — both layers exist on purpose). |
+| `--kv-quant` | `config`\|`off`\|`4`\|`8`\|`turbo[:k<bits>v<bits>]` | `off` (bf16) | KV-cache quantization. **Default flipped to bf16 2026-07-05** (naked = L1): quantized KV measured 5–20% slower decode than bf16 at ≤16k on every model — on mlx-lm too — so it pays only in **memory headroom** (e.g. ~1.3 GB on the 12B @16k) and is an explicit opt-in (`--kv-quant …` or `--l2`, whose preset passes `config`). `off` = bf16 (L1). `4`/`8` = **uniform** bits (group 64, start 0) — the scheme mlx-lm exposes as `--kv-bits`/`--kv-group-size`/`--quantized-kv-start`. In the L1 kernel set (`--l1 --kv-quant 8` = fused-sdpa off) our unfused quantized SDPA is **op-for-op identical** to mlx-lm's `quantized_scaled_dot_product_attention`, so uniform-quant is **bit-exact L1** (the fused-sdpa-ON path is optiq-aligned). `config` = **per-layer mixed-precision** from `kv_config.json` — optiq-only, no mlx-lm analog → **L2**. `turbo[:k<bits>v<bits>]` = **TurboQuant** ([docs/design/turboquant-kv.md](../design/turboquant-kv.md)): rotation-based KV quantization (asymmetric-affine int8/int4/int2/int5 keys, FWHT + Lloyd-Max values) — default `k8v3` (2.56× compression at head_dim 128); `kBits` ∈ {2,4,5,8}, `vBits` ∈ {2,3,4,5,8}. A separate axis from the scheme above (mutually exclusive with `config`/`4`/`8`); v1 is dequantize-on-fetch (stock `ops.sdpa`, no fused kernel) and **full-attention layers only** — sliding-window layers stay bf16 (one-time warning, never a throw). Under `--batch N`, a per-layer `config` scheme **batches** — full-attention layers since Phase 3.1 (MiniCPM5), rotating/sliding layers since milestone 2 (gemma's whole kv_config), so every shipped `kv_config.json` now batches. The scheduler applies the mixed scheme per row, gated bit-exact per row vs the serial composition (`tests/batched-kv-quant-parity.test.ts` — compositions no other stack ships). Uniform `4`/`8` still routes those requests serial (quantizedKvStart threshold semantics). **`turbo` is solo-only, unconditionally** — `TurboQuantKVCache` is a novel `Cache` implementation (no merge/filter/temporalView), so it's excluded from the batch scheduler by construction; `GenerationGateway.place` also refuses continuous scheduling explicitly (belt + braces — both layers exist on purpose). |
 | `--paged-kv` | (bool) | off | **OPTIONAL vLLM-style paged KV cache** ([docs/design/paged-kv-cache.md](../design/paged-kv-cache.md)): full-attention layers store K/V in fixed-size **block pools** (host-side block table, `ops.takeAxis` gather back to contiguous before the stock SDPA — no new attention math, no fused kernel in v1). Env: `MLX_BUN_PAGED_KV=1`. **Gated bit-exact vs the plain path** (`tests/paged-kv-parity.test.ts`: identical single-forward logits + identical 48-token greedy trajectory on the 12B). **v1 scope: serial batch=1, Gemma4-family, bf16** — with no explicit `--batch` the CLI pins `--batch 1`; the server **refuses** (fails startup, never silently downgrades) `--batch N>1`, any `--kv-quant`, and `--draft-model`; sliding-window layers keep the rotating scheme (mixed paged-full + rotating-sliding is the supported shape). Media (vision/audio) and LoRA-adapter **requests** are scoped OUT per request — they run the plain cache path (v1 non-goal cells), never a 400. Paged requests **bypass the prompt cache** (fresh caches per request; `--ssd-cache` sees nothing) and run **uncompiled decode** (a data-dependent block-list length can't shapeless-replay). Expect a small decode cost at batch=1 — the per-step gather copy buys nothing until the batched/CoW follow-ups; this ships the correctness-proven abstraction, honestly labeled. Pool sized from prompt+`max_tokens` at request setup; exhaustion is a typed error (accounting tripwire), never truncation. |
 | `--paged-kv-block-size` | n | `256` | Tokens per KV block (`--paged-kv` only). 256 = the plain cache's growth step, so v1's allocation granularity is a permutation of today's into reusable slots, not a new tuning axis. |
 | `--adapter` | dir | none | Mount a LoRA adapter at startup (same machinery as `POST /v1/adapters`; the adapter id is the directory's basename) and make it the **default** for requests that send no `adapter` field. A request's explicit `adapter` — including `"none"` — always wins, and hot-swap via `/v1/adapters` is unchanged. `--adapter-path` is accepted as the mlx_lm.server-named alias. A bad adapter fails startup loudly rather than silently serving the base model. This is the flag `mlx-bun train`'s completion message points at. |
 | `--draft-model` | path/query | none | **Speculative decoding**: a drafter proposes tokens the main model verifies in one forward — exact results, faster decode when drafts land. Resolves like the main model. The artifact's **kind is auto-detected**: a full same-tokenizer model (mlx_lm.server parity — L1: token-for-token vs mlx-lm's spec path; tokenizer-family mismatch fails startup, upstream silently accepts ~0%), a Gemma `-assistant` KV-borrowing drafter (L2 vs optiq spec_generate; ~1.09× γ=1 on 12B, no training), a locally-trained **DSpark** checkpoint (`dspark.json`), or one of **DeepSeek's released DSpark drafters** (DeepSpec `Gemma4DSparkModel` config stamp, e.g. `deepseek-ai/dspark_gemma4_12b_block7` for the 12B — oracle: DeepSpec's reference at temperature 0, token-for-token). All four share ONE serve loop. A confidence-calibrated DSpark checkpoint additionally self-schedules its draft length per round (threshold pruning — fewer wasted verify positions; uncalibrated checkpoints draft fixed-length). Mounting a draft routes **every** request to the serial lane (upstream `is_batchable = draft is None`). Pays on slow targets (12B+); fast small models lose to the draft overhead. Composes with structured output (the constrained verify walk). Prompt-cache reuse is bypassed on the spec path (v1). Telemetry: `usage.speculation` (`drafted`/`accepted`/`targetCalls`). |
-| `--draft-kind` | `two-model`\|`assistant`\|`dspark`\|`deepspec`\|`ngram` | auto | Override the draft-artifact kind detection (`dspark.json` → dspark; `Gemma4DSparkModel` architecture stamp → deepspec; `*_assistant` config → assistant; else two-model). **`ngram` is never auto-detected — it has no artifact**: MODEL-FREE prompt lookup (drafts copied from the request's own prompt+generation when its trailing k-gram occurred earlier in the stream; port of prompt-lookup decoding / vLLM's `ngram` proposer). Mount it ALONE — `--draft-kind ngram` with a `--draft-model` is refused, as is any other kind without one. Zero weights/memory, lossless by the same verify (gated: serve-loop output token-identical to non-spec greedy on e4b, tests/spec-ngram.test.ts); a no-match round degrades to one plain target step. Best on agentic/RAG/code-edit traffic that re-emits context spans. |
+| `--draft-kind` | `two-model`\|`assistant`\|`dspark`\|`deepspec`\|`mtp`\|`ngram` | auto | Override the draft-artifact kind detection (`dspark.json` → dspark; `Gemma4DSparkModel` architecture stamp → deepspec; `*_assistant` config → assistant; `*_mtp` model_type → mtp, the target-release-native multi-token-prediction head — it shares the target's embeddings/lm-head, defaults `--num-draft-tokens` to its trained `block_size - 1`, and partial rejects roll the target's recurrent DeltaNet caches back via bit-exact snapshot/replay; else two-model). `mtp` alone (no `--draft-model`) mounts the companion bundled at `<model>/mtp/` when present — single-repo artifacts — and refuses otherwise. **`ngram` is never auto-detected — it has no artifact**: MODEL-FREE prompt lookup (drafts copied from the request's own prompt+generation when its trailing k-gram occurred earlier in the stream; port of prompt-lookup decoding / vLLM's `ngram` proposer). Mount it ALONE — `--draft-kind ngram` with a `--draft-model` is refused, as is any other kind without one. Zero weights/memory, lossless by the same verify (gated: serve-loop output token-identical to non-spec greedy on e4b, tests/spec-ngram.test.ts); a no-match round degrades to one plain target step. Best on agentic/RAG/code-edit traffic that re-emits context spans. |
 | `--num-draft-tokens` | n | `3` (`ngram`: `10`) | Drafts per verify round (mlx_lm.server's default; `mlx_lm.generate`'s is 2). A DSpark draft pins this to its trained block width (`cfg.gamma`); ngram drafting is free so its default is the reference implementation's 10. |
 | `--ngram-max` / `--ngram-min` | k | `3` / `1` | `--draft-kind ngram` only: longest/shortest trailing k-gram searched for a match (the reference values — longest first, first occurrence wins). |
 | `--mtp` | `on`\|`off` | on for GLM-5.2 | Mount the checkpoint-native GLM MTP row as the server's drafter. It uses the bounded auxiliary expert tier and exact serial verify loop (`usage.lane: "serial+spec"`). `off` removes the draft so ordinary GLM requests can use continuous batching; per-row batched MTP is post-release. Mutually exclusive with an explicit `--draft-model`/`--draft-kind`. |
@@ -47,7 +47,7 @@ the design rationale is in
 | `--max-tokens` | n | GLM-5.2: `128`; otherwise `65536` chat / `512` raw completion | Completion cap when a request omits `max_tokens` (mlx_lm.server flag; its default there is 512 — `--max-tokens 512` reproduces mlx_lm.server exactly). For GLM this is also reserved by the pre-open resource equation and must fit inside `--context-length`. |
 | `--allow-private-media` | (bool) | off | Let `image_url`/`audio_url` content parts fetch from **private/loopback/link-local** hosts (a NAS, another LAN box). Off by default: a request's URL is attacker-controlled input, so remote media fetches (`src/media-fetch.ts`) refuse non-http(s) schemes and destinations in private/loopback/link-local/CGNAT ranges — including hosts that *resolve* there and every **redirect hop** (each hop re-passes the whole policy) — which keeps the server from being steered at cloud metadata (`169.254.169.254`) or LAN-internal services (SSRF). Independent of the flag, every remote fetch has a **10 s wall-clock timeout** and a **64 MB response cap** (streaming-enforced, not just Content-Length); violations surface as clean `400`s (`prompt build failed: …`). `data:` URLs decode locally and are never policy-checked. |
 | `--no-open` | (bool) | off | Skip the automatic browser open on start. By default an interactive terminal session opens `http://<host>:<port>/#/chat` once the server is ready; pass this flag to suppress it (e.g. headless or non-TTY environments already skip it). |
-| `--hlg-sampling` | `on`\|`off` | off | Piecewise tone-curve (HLG) sampling: rolls off the top-token region, boosts the mids, gentles the tail. The overall gain folds from `--temperature`. See [docs/design/hlg-sampling.md](../design/hlg-sampling.md). |
+| `--hlg-sampling` | `on`\|`off` | off | Piecewise tone-curve (HLG) sampling: rolls off the top-token region, boosts the mids, gentles the tail. The overall gain folds from `--temperature`. See [docs/archive/hlg-sampling.md](../design/hlg-sampling.md). |
 | `--hlg-width` | nats | `4` | HLG mid-region half-width (nats). Only meaningful with `--hlg-sampling on`. |
 | `--hlg-shoulder` | nats | `4` | HLG highlight rolloff scale (nats). Only meaningful with `--hlg-sampling on`. |
 | `--hlg-toe` | nats | `6` | HLG shadow rolloff scale (nats). Only meaningful with `--hlg-sampling on`. |
@@ -81,6 +81,22 @@ model loads, so they apply to `mlx-bun pi` too. They affect the
 3.2) — see
 [Levers that don't reach the batched lane](#--batch-n-is-compat-mode--perf-flags-dont-apply-by-design).
 
+## Near-ceiling models on small machines (24 GB)
+
+A model whose weights approach the DEFAULT macOS GPU wired ceiling (~75% of
+RAM — e.g. a 17 GB model on a 24 GB machine) loads and serves short prompts,
+but a long prefill can exceed the ceiling and die with an **uncatchable**
+Metal OOM (the failure is asynchronous; no server error is possible).
+`mlx-bun serve` detects this at startup and prints the remedy:
+
+```
+sudo sysctl iogpu.wired_limit_mb=<RAM_MB - ~2500>   # resets on reboot
+```
+
+For very long contexts on such machines, additionally prefer `--kv-quant`
+(e.g. `turbo`) and expect the engine to use smaller prefill chunks — the
+prefill transient scales with chunk size × context depth.
+
 ## Per-request overrides
 
 Most quality knobs can be set per request in the chat body and override
@@ -117,7 +133,8 @@ for paired A/B harnesses).
 | `MLX_BUN_GRAMMAR` | — | on (`!=="0"`) | `=0` disables structured-output mask compilation. Requested constraints still take the documented graceful-degrade route: chat prompt injection plus a `Warning` header; raw completions emit the header only. |
 | `MLX_BUN_GRAMMAR_JUMP` | — | **off** (`==="1"`) | Jump-forward decoding for structured output (SGLang's technique via xgrammar's `findJumpForwardString`): when the grammar forces a unique continuation (JSON punctuation/keys; longest under `any_whitespace:false` fixed formatting), the SERIAL lane emits its retokenized ids with ONE multi-token forward instead of one masked forward per token. Lossless in STRING space and always grammar-valid, but the token stream (and, conditioning on it, content after a forced span) can legally differ from an unjumped run — no oracle, hence opt-in. Excluded automatically when either `logprobs` or `top_logprobs` is requested, because jumped tokens have no sampled distribution row; SentencePiece-family tokenizers whose raw encode can't reproduce a mid-stream span simply never jump (guard degrades to normal decode). The batch lane's per-row grammar (`#stepGrammar`) does not jump yet. Tests: tests/grammar-jump.test.ts (contract + Llama-3.2-1B e2e). |
 | `MLX_BUN_SSD_WRITEBEHIND` | — | on (`!=="0"`) | `--ssd-cache`'s debounced write-behind snapshot (restart survival). `=0` disables it entirely — the paired-A/B lever + kill switch for the 2026-07-07 idle-gating fix (eviction/demotion spills still write). The flush is **idle-gated**: every per-tensor step (blocking GPU sync + `writeSync`) waits for the engine to go idle and pauses when a request arrives mid-flush, so durability work never taxes an active decode (pre-fix: a ~16k entry's flush overlapping cached ctx repeats cost e4b ~9% decode@ctx vs mlx-lm). The snapshot timer also never holds batch admission — while rows are active it re-arms instead of grabbing the exclusive. |
-| `MLX_BUN_SSD_SPILL_QUEUE_GB` | — | 2 | Byte cap on the write-behind queue's PENDING spill clones (they pin the evicted entries' GPU memory until the idle-gated flush runs). Over cap, the oldest queued spill drops — its clones free immediately and the entry becomes a future cache miss (never a wrong result). `=0` keeps only the newest + in-flight clone pinned (the soft cap never drops the item just enqueued). `/stats.ssd_cache` reports `pending_spills` / `pending_spill_bytes` / `dropped_spills`. Note: there is deliberately **no shutdown flush** — exiting under traffic loses queued write-behind entries (best-effort tier; cost = one re-prefill). |
+| `MLX_BUN_SSD_SPILL_QUEUE_GB` | — | 2 | Byte cap on the write-behind queue's PENDING spill clones (they pin the evicted entries' GPU memory until the idle-gated flush runs). Over cap, the oldest queued spill drops and its clones free immediately. The durability coordinator keeps the source snapshot dirty so an explicit or shutdown flush can retry it. `=0` keeps only the newest + in-flight clone pinned (the soft cap never drops the item just enqueued). `/stats.ssd_cache` reports pending, dropped, failed, and longest-durable-prefix counters. |
+| `MLX_BUN_SHUTDOWN_TIMEOUT_MS` | — | `120000` | Maximum time `mlx-bun serve` gives active requests plus the SSD durability flush after `SIGINT` or `SIGTERM`. On timeout the process logs the remaining snapshot/spill counters and exits. |
 | `MLX_BUN_BATCH_NO_PIPELINE` | — | off (`==="1"`) | Disables the batch lane's pipelined token readback (the A/B lever for the shipped pipelining, like its peers `MLX_BUN_BATCH_EXTEND` / `MLX_BUN_BATCH_SSM` / `MLX_BUN_BATCH_VEC_SAMPLE`). Numerically equivalent, slower steps. |
 | `MLX_BUN_DSPARK_MINCONF` | — | checkpoint-driven | Overrides the DSpark/DeepSpec draft scheduler's minimum-confidence threshold (Alg-1 draft-length pruning). Spec lane only; no CLI flag — a calibration/A/B lever (see [dspark-speculative-decoding.md](../design/dspark-speculative-decoding.md)). |
 | `MLX_BUN_EXPERT_OFFLOAD` | `--expert-offload` | off (`==="1"`) | Env mirror of the flag for direct/library runs (scripts that never parse serve flags). |
@@ -130,12 +147,12 @@ exporting them now does nothing.)
 
 ## Execution modes: serial vs. `--batch N`
 
-Batching is **concurrency-driven** (default cap 8, flipped 2026-07-05): a
-lone request runs the exact serial engine — its caches are ADOPTED
-serial-class objects, compiled decode replays, the prompt cache and SSD
-tier serve it — and only a second concurrent request causes a batch
-layout to exist. "How many requests you send" is the batching decision;
-the flag is just the cap.
+Scheduling is **concurrency-driven** (default cap 8, flipped 2026-07-05): a
+lone scheduler request uses the B=1 fast path — its caches are adopted
+serial-class objects, compiled decode replays, and the prompt cache and SSD
+tier serve it. Only a second concurrent request causes a batch layout to
+exist. The flag declares the concurrency cap; active rows select B=1 or B=N
+inside the scheduler.
 
 `--batch 1` pins the strict serialized single-queue path: one generation
 at a time, arrival-independent numerics (a request's bits never depend on
@@ -159,23 +176,29 @@ row for bf16, and per-row oracle-gated for the quantized compositions
   serial lane (quantizedKvStart threshold semantics; a startup warning
   is printed).
 
-### The lane picker (`GenerationGateway.willBatch`)
+### Scheduling declaration (`GenerationGateway.place`)
+
+`--batch` declares a concurrency cap. With a cap greater than one,
+`GenerationGateway.place()` freezes one scheduling mechanism for the exact
+resolved execution composition. `continuous` admits it to the scheduler; that
+scheduler chooses its B=1 fast path or B=N step from active rows. `serial`
+preserves the strict or dedicated executor. The support check never removes,
+downgrades, or substitutes MTP, KV schemes, TurboQuant, grammar, adapters, or
+sampling. A composition the scheduler does not implement drains it and runs
+unchanged through the serial mechanism.
 
-Under `--batch N`, each request is routed per-request. It joins the
-batch only if **all** of these hold; otherwise it drains the batch and
-runs solo (mlx-lm's `_is_batchable` behavior). The two lanes are
-mutually exclusive on the GPU (one `AsyncMutex`), so a serial-fallback
-request never runs alongside a batched step — but batched requests run
-concurrently with each other.
+The two mechanisms are mutually exclusive on the GPU (one `AsyncMutex`), so a
+serial request never runs alongside a continuous step, while requests admitted
+to the continuous scheduler can run concurrently with each other.
 
-| Request property | Batches? |
+| Request property | Continuous scheduler? |
 | --- | --- |
 | vision (image parts) | ❌ serial — needs offset-0 single-seq prefill + bidirectional image mask |
 | LoRA `adapter` (resolves to ≥1) | ❌ serial — `loraState.active` is one per-generation field; per-row adapters unsupported |
 | `logprobs` / `top_logprobs` | ❌ serial — the batched sampler doesn't capture logprob arrays yet |
 | explicit `seed` | ❌ serial — reproducibility ⇒ solo (matches mlx-lm) |
 | KV quant active (explicit `--kv-quant`) | ✅ batches for per-layer `config` schemes — full-attention layers (Phase 3.1) AND rotating layers (milestone 2: gemma's kv_config; unpadded row gated KL-0 vs serial); uniform bits → serial |
-| `--kv-quant turbo` (TurboQuant) | ❌ serial, unconditionally — solo-only in v1 (novel `Cache` class, no batched merge/filter/temporalView; belt-and-braces refusal in `willBatch` on top of the automatic capability-gate exclusion) |
+| `--kv-quant turbo` (TurboQuant) | ❌ serial, unconditionally — solo-only in v1 (novel `Cache` class, no batched merge/filter/temporalView; belt-and-braces refusal in placement on top of the automatic capability-gate exclusion) |
 | `--draft-model` mounted | ❌ serial, server-wide — speculation is a B=1 latency mode (upstream `is_batchable = draft is None`) |
 | `repetition_penalty` / `min_p` / `xtc_*` / `logit_bias` / presence+frequency penalties | ✅ batches — per-row logits processors over a per-row device-side history (since 2026-07-02; some models — Qwen3.5 — ship a *default* repetition penalty, which used to route everything serial) |
 | structured output (`response_format` / `guided_*`) | ✅ batches — per-row grammar matchers driven by the scheduler (`MLX_BUN_GRAMMAR_BATCH=0` forces serial) |
@@ -185,7 +208,7 @@ concurrently with each other.
 | `--thinking` / `enable_thinking` | ✅ batches (template-render concern, lane-independent) |
 | multi-turn / long prompt | ✅ batches **with prompt-cache reuse** (Phase 3.2): a joiner restores the longest usable cached prefix and prefills only the suffix (`cached_tokens` reported); a request that finishes without ever sharing the batch puts its caches back. **Prefix sharing (2026-07-05): serves are non-consuming clones**, so concurrent agents with a shared system prompt all reuse one prefill and nobody's entry is destroyed. A row that merges with others is not re-put (its entry ages out) — heavy concurrency reduces hit rate, never correctness |
 
-**Which models batch:** full-attention (CPM), sliding-window (Gemma),
+**Which models use continuous scheduling:** full-attention (CPM), sliding-window (Gemma),
 hybrid gated-DeltaNet (Qwen3.5 — the SSM batched path, token-exact vs the
 mlx-lm B=2 oracle, landed 2026-07-02; `MLX_BUN_BATCH_SSM=0` reverts it to
 serial routing), and **plain full-attention Tier-0 universal archs**
@@ -194,23 +217,24 @@ Still serial by the model-level capability gate: gemma2-family and
 sliding-window *universal* archs (unvalidated cells) and DiffusionGemma
 (non-autoregressive).
 
-A non-batchable request **drains** the batch: while it waits, the
+A request requiring the serial mechanism **drains** the scheduler: while it waits, the
 scheduler stops admitting new rows, finishes the running ones, and
 releases the GPU so the serial request runs (mlx-lm's `drain_batch`);
 admission then resumes. So a steady stream of batchable traffic cannot
 starve a serial-lane request.
 
-`--batch N` is a **mode switch, not a load-dependent fallback**:
-auto-batching "when >1 request arrives" was considered and rejected —
-an idle vs. loaded server would produce different numerics for the same
-request, breaking determinism and the drop-in-for-`mlx_lm.server`
-promise.
+Placement is not a load-dependent fallback. A request admitted to the
+continuous scheduler stays there whether it is the only active row or one of
+many; arrivals select B=1 versus B=N *inside* that mechanism. `--batch 1`
+remains the arrival-independent numerics pin. This matches mlx-lm's
+concurrency-driven serving contract while keeping the strict serial route for
+goldens and compositions that need it.
 
 ## Compatibility matrix
 
-How each option behaves in each mode. "serial" = `--batch 1`; "`--batch N`"
-= the batched mode (a given request may still take the serial *lane*
-inside it per the table above).
+How each option behaves under each configured mechanism. "serial" =
+`--batch 1`; `--batch N` permits continuous scheduling, although a declared
+composition may still require the serial mechanism as shown above.
 
 | Option | serial (`--batch 1`) | `--batch N` (N>1) |
 | --- | --- | --- |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mlx-bun",
-  "version": "0.0.13",
+  "version": "0.2.0",
   "description": "Native MLX inference for Bun on Apple Silicon — local LLM server (OpenAI/Anthropic-compatible) and TypeScript library. No Python.",
   "type": "module",
   "license": "MIT",
@@ -37,7 +37,7 @@
     "THIRD_PARTY_LICENSES.md"
   ],
   "engines": {
-    "bun": ">=1.3.14"
+    "bun": ">=1.4.0"
   },
   "os": [
     "darwin"

package/src/anthropic.ts

@@ -23,6 +23,10 @@
 //     semantics) instead of the oracle's json.dumps fallback.
 
 import type { ChatMessage, ToolDefinition } from "./chat-template";
+import type {
+  CompletionEvent,
+  CompletionStreamProtocol,
+} from "./serve/completion-sink";
 
 // ---------------------------------------------------------------------
 // Request translation: Anthropic → OpenAI (our ChatRequest shape)
@@ -128,7 +132,15 @@ export function anthropicToChatBody(body: AnthropicRequest): Record<string, unkn
           break; // prior-turn thinking is not re-fed (Anthropic semantics)
         case "image": {
           const part = imagePartFromBlock(block);
-          if (part) imageParts.push(part);
+          if (!part)
+            // e.g. a Files-API {type:"file"} source we cannot fetch —
+            // reject loudly, never answer about an image the model can't
+            // see (2026-08-18 review; the audio-doctrine below).
+            throw new Error(
+              `unsupported image source "${block.source?.type ?? "missing"}" — ` +
+                `send base64 or url image sources`,
+            );
+          imageParts.push(part);
           break;
         }
         case "tool_use":
@@ -141,15 +153,52 @@ export function anthropicToChatBody(body: AnthropicRequest): Record<string, unkn
             },
           });
           break;
-        case "tool_result":
+        case "tool_result": {
           // Anthropic packs tool results into the next user message;
           // OpenAI (and the gemma template) want role:"tool" messages.
+          // Nested image blocks (screenshot-returning tools — the
+          // computer-use shape) are PRESERVED as image parts: flattening
+          // them away had the model answering about screenshots it never
+          // saw (2026-08-18 review). An unrenderable pairing downstream
+          // fails loudly on the media-count check, never silently.
+          const trImages: Record<string, unknown>[] = [];
+          if (Array.isArray(block.content)) {
+            for (const b of block.content as AnthropicContentBlock[]) {
+              if (b && typeof b === "object" && b.type === "image") {
+                const part = imagePartFromBlock(b);
+                if (!part)
+                  throw new Error(
+                    `unsupported image source "${b.source?.type ?? "missing"}" ` +
+                      `in tool_result — send base64 or url image sources`,
+                  );
+                trImages.push(part);
+              }
+            }
+          }
+          const txt = flattenText(block.content);
           messages.push({
             role: "tool",
             tool_call_id: block.tool_use_id,
-            content: flattenText(block.content),
+            content: trImages.length
+              ? [
+                  ...(txt ? [{ type: "text", text: txt }] : []),
+                  ...trImages,
+                ] as never
+              : txt,
           });
           break;
+        }
+        case "video":
+        case "video_url":
+          // Not an Anthropic Messages block type — reject loudly instead of
+          // silently answering about a video the model never saw (same
+          // doctrine as audio below; the OpenAI surface serves video on
+          // Qwen3.5-family models).
+          throw new Error(
+            `"${block.type}" content blocks are not part of the Anthropic ` +
+              `Messages protocol — send video to the OpenAI endpoint ` +
+              `/v1/chat/completions as a video_url content part`,
+          );
         case "audio":
         case "input_audio":
         case "audio_url":
@@ -168,6 +217,11 @@ export function anthropicToChatBody(body: AnthropicRequest): Record<string, unkn
       }
     }
     if (role === "assistant") {
+      if (imageParts.length)
+        // Assistants don't produce images in the Anthropic protocol; a
+        // client that puts one here would previously have it silently
+        // discarded (2026-08-18 review — loud beats silent).
+        throw new Error("image content blocks are not supported in assistant messages");
       const msg: ChatMessage = { role, content: textParts.join("\n") || null };
       if (toolCalls.length) msg.tool_calls = toolCalls;
       if (msg.content || toolCalls.length) messages.push(msg);
@@ -457,6 +511,41 @@ export class AnthropicStreamTranslator {
   }
 }
 
+/** Direct semantic-event adapter used by the server. This avoids encoding
+ * OpenAI SSE only to parse those bytes back into protocol events. */
+export function createAnthropicStreamProtocol(model: string): CompletionStreamProtocol {
+  const translator = new AnthropicStreamTranslator(model);
+  const chunk = (delta: Record<string, unknown>) => ({ choices: [{ delta }] });
+  return {
+    start: () => translator.addChunk(chunk({ role: "assistant", content: "" })),
+    addEvents(events: CompletionEvent[]) {
+      return events.flatMap((event) => {
+        if (event.type === "reasoning") {
+          return translator.addChunk(chunk({ reasoning: event.text }));
+        }
+        if (event.type === "content") {
+          return translator.addChunk(chunk({ content: event.text }));
+        }
+        return translator.addChunk(chunk({
+          tool_calls: event.calls.map((call, index) => ({ index, ...call })),
+        }));
+      });
+    },
+    finish(reason, usage) {
+      return [
+        ...translator.addChunk({ choices: [{ delta: {}, finish_reason: reason }], usage }),
+        ...translator.finalize(),
+      ];
+    },
+    error(message) {
+      return [sse("error", {
+        type: "error",
+        error: { type: "api_error", message },
+      })];
+    },
+  };
+}
+
 /** Wrap our OpenAI SSE byte stream as an Anthropic SSE byte stream.
  *  Frames translate as they arrive (per-chunk flush preserved — the
  *  upstream already rate-limits socket flushes). */