mlx-bun 0.4.0 → 1.0.0

package/src/server/proxy-routes.ts

@@ -0,0 +1,187 @@
+// The isolated host's route group: what the parent owns, and the way a request
+// reaches a model worker. Each resident model runs in its own worker process;
+// the model router (model-routes.ts) picks the model a request names and leases
+// its worker, and `forwardToWorker` sends the request over the worker's Unix
+// socket with the body streaming through (hop-by-hop headers stripped, a client
+// abort aborting the proxied fetch so the worker sees the disconnect). The
+// parent answers `/engine`, `/health` and `/stats` (with the workers'
+// contribution) and `/v1/responses` through the history client. The workers' private admin paths never forward.
+import type { WorkerMemory } from "../jobs/worker-memory";
+import { describeExit, EngineUnavailableError, type WorkerSupervisor, type WorkerSupervisorState } from "../jobs/worker-supervisor";
+import type { createResponsesClient } from "./responses-client";
+
+const HOP_BY_HOP = new Set([
+  "connection", "keep-alive", "proxy-authenticate", "proxy-authorization",
+  "te", "trailer", "transfer-encoding", "upgrade",
+]);
+/** Never forwarded to a worker: its private socket surface (which only the
+ * parent itself calls; unmatched here, the listener answers 404) and the
+ * parent's own synthesis route (served by its earlier group). */
+const PARENT_ONLY = new Set(["/admin/lease", "/admin/drain", "/admin/memory/complete", "/admin/events", "/admin/adapters", "/v1/memory/synthesize"]);
+
+const encoder = new TextEncoder();
+/** A copy without the hop-by-hop headers and the ones `Connection` names
+ * (also used by the library host, `cli/library-host.ts`). */
+export const stripHopByHop = (headers: Headers) => {
+  const excluded = new Set(HOP_BY_HOP);
+  for (const name of (headers.get("connection") ?? "").split(",")) excluded.add(name.trim().toLowerCase());
+  const out = new Headers();
+  headers.forEach((value, key) => { if (!excluded.has(key.toLowerCase())) out.set(key, value); });
+  return out;
+};
+
+/** The SSE error frame each protocol's own error path emits, so a client
+ * parser ends the stream with an error instead of a silent truncation. */
+export function unavailableFrame(pathname: string, message: string): string {
+  if (pathname === "/v1/messages")
+    return `event: error\ndata: ${JSON.stringify({ type: "error", error: { type: "api_error", message } })}\n\n`;
+  if (pathname === "/v1/responses")
+    return `event: error\ndata: ${JSON.stringify({ type: "error", code: "engine_unavailable", message, param: null })}\n\n`;
+  return `data: ${JSON.stringify({ error: { message, type: "engine_unavailable" } })}\n\n`;
+}
+
+/** Pass an event stream through unchanged; when the worker dies under it,
+ * end with the protocol's error frame instead of dropping the connection. */
+function guardEventStream(body: ReadableStream<Uint8Array>, failure: () => string): ReadableStream<Uint8Array> {
+  const reader = body.getReader();
+  return new ReadableStream<Uint8Array>({
+    async pull(controller) {
+      let item: Awaited<ReturnType<typeof reader.read>>;
+      try { item = await reader.read(); }
+      catch { controller.enqueue(encoder.encode(failure())); controller.close(); return; }
+      if (item.done) controller.close(); else controller.enqueue(item.value);
+    },
+    cancel(reason) { return reader.cancel(reason).catch(() => {}); },
+  });
+}
+
+export interface WorkerEngine {
+  readonly id: string;
+  readonly role: "primary" | "companion";
+  readonly supervisor: WorkerSupervisor;
+  /** The MLX memory the worker last reported; undefined before its first report and while it is down. */
+  measured?(): WorkerMemory | undefined;
+}
+
+export interface ProxyRoutesOptions {
+  /** The resident workers, and the id of the model requests that name none are answered by. */
+  workers(): readonly WorkerEngine[];
+  current(): string;
+  /** The model router: it leases the worker a request belongs to and forwards to it. */
+  models: { handle(request: Request): Promise<Response | null> };
+  responses: ReturnType<typeof createResponsesClient>;
+  /** The model the server started with. */
+  modelId: string;
+  startedAt: number;
+}
+
+/** `memory` is the worker's own MLX reading (`active_bytes` and `cache_bytes` are what residency counts), null until it reports. */
+export interface WorkerReport { id: string; role: "primary" | "companion"; pid: number | null; state: WorkerSupervisorState; restarts: number; socket: string;
+  memory: { active_bytes: number; cache_bytes: number; peak_bytes: number } | null }
+
+/** The current model's worker fields, then every resident worker. `state` is `none` while no worker is resident. */
+export interface EngineReport {
+  isolated: true;
+  state: WorkerSupervisorState | "none";
+  pid: number | null;
+  restarts: number | null;
+  socket: string | null;
+  model: string;
+  last_exit: { code: number | null; signal: string | null } | null;
+  response_store: { entries: number; bytes: number; max_bytes: number; ttl_ms: number };
+  workers: WorkerReport[];
+}
+
+const unavailableMessage = (error: unknown) => {
+  if (error instanceof EngineUnavailableError) return `inference engine unavailable: ${error.message}${error.state === "restarting" ? " — retry shortly" : ""}`;
+  return "inference engine unavailable: the engine worker stopped mid-request; respawning — retry shortly";
+};
+/** The 502 a request gets when its worker cannot take it. */
+export function workerUnavailable(error: unknown, worker: WorkerSupervisor): Response {
+  return Response.json({ error: { message: unavailableMessage(error), type: "engine_unavailable",
+    state: error instanceof EngineUnavailableError ? error.state : worker.state } }, { status: 502 });
+}
+
+/** The request goes to the worker as it came, its body streaming through. A worker that cannot take it, or dies under
+ * it, answers with the protocol's own unavailable error; nothing is retried. */
+export async function forwardToWorker(worker: WorkerSupervisor, request: Request): Promise<Response> {
+  const { pathname } = new URL(request.url);
+  try {
+    request.signal.throwIfAborted();
+    const upstream = await worker.fetch(request.url, {
+      method: request.method, headers: stripHopByHop(request.headers), body: request.body,
+      signal: request.signal, redirect: "manual", duplex: "half",
+    });
+    const headers = stripHopByHop(upstream.headers);
+    const failure = () => {
+      const exit = worker.lastExit;
+      return `inference engine unavailable: the engine worker stopped while streaming this response${exit ? ` (${describeExit(exit)})` : ""}; it is being respawned`;
+    };
+    const stream = upstream.body && (upstream.headers.get("content-type") ?? "").startsWith("text/event-stream")
+      ? guardEventStream(upstream.body, () => unavailableFrame(pathname, failure()))
+      : upstream.body;
+    return new Response(stream, { status: upstream.status, statusText: upstream.statusText, headers });
+  } catch (error) {
+    if (request.signal.aborted) return new Response(null, { status: 499 });
+    return workerUnavailable(error, worker);
+  }
+}
+
+export function createProxyRoutes(options: ProxyRoutesOptions) {
+  const { responses } = options;
+  const currentWorker = () => options.workers().find(worker => worker.id === options.current() && worker.role === "primary")?.supervisor;
+  const report = (): EngineReport => {
+    const engine = currentWorker();
+    return {
+      isolated: true, state: engine?.state ?? "none", pid: engine?.pid ?? null, restarts: engine?.restarts ?? null,
+      socket: engine?.socketPath ?? null, model: options.current(), last_exit: engine?.lastExit ?? null, response_store: responses.stats,
+      workers: options.workers().map(({ id, role, supervisor, measured }) => {
+        const memory = measured?.();
+        return { id, role, pid: supervisor.pid, state: supervisor.state, restarts: supervisor.restarts, socket: supervisor.socketPath,
+          memory: memory ? { active_bytes: memory.activeBytes, cache_bytes: memory.cacheBytes, peak_bytes: memory.peakBytes } : null };
+      }),
+    };
+  };
+  const health = async (): Promise<Response> => {
+    const engine = currentWorker();
+    let contribution: Record<string, unknown> = {};
+    if (engine?.state === "ready") {
+      try {
+        const upstream = await engine.fetch("http://engine/health", { signal: AbortSignal.timeout(2_000) });
+        const body = await upstream.json() as Record<string, unknown>;
+        if (upstream.ok) contribution = { state: body.state, in_flight: body.in_flight, leases: body.leases };
+        else contribution = { state: "unreachable" };
+      } catch { contribution = { state: "unreachable" }; }
+    }
+    const { response_store: _store, isolated: _isolated, workers, ...current } = report();
+    return Response.json({ status: "ok", isolated: true, engine: { ...current, ...contribution }, workers });
+  };
+  const stats = async (request: Request): Promise<Response> => {
+    const own = { response_store: responses.stats, engine: report() };
+    const upstream = await options.models.handle(request).catch(() => null);
+    if (upstream?.ok && (upstream.headers.get("content-type") ?? "").includes("json"))
+      return Response.json({ ...(await upstream.json() as Record<string, unknown>), ...own });
+    if (upstream && !upstream.ok && upstream.status !== 502) return upstream;
+    // The worker's own reason (a 502 from the forward), else that no worker is resident.
+    const reason = await upstream?.json().then((body: { error?: { message?: string } }) => body.error?.message).catch(() => undefined);
+    if (request.signal.aborted) return new Response(null, { status: 499 });
+    return Response.json({ server: { owner: "serve", model: options.current(), started_at: options.startedAt }, ...own,
+      unavailable: reason ?? "inference engine unavailable: no model worker is resident" });
+  };
+
+  return {
+    report,
+    async handle(request: Request): Promise<Response | null> {
+      const { pathname } = new URL(request.url);
+      if (PARENT_ONLY.has(pathname)) return null;
+      if (pathname === "/engine") return request.method === "GET" ? Response.json(report()) : null;
+      if (request.method === "GET") {
+        if (pathname === "/health") return health();
+        if (pathname === "/stats") return stats(request);
+      }
+      if (pathname === "/v1/responses" && request.method === "POST")
+        return responses.forward(request, async inner => await options.models.handle(inner) ?? Response.json({ error: { message: "Not found" } }, { status: 404 }));
+      return options.models.handle(request);
+    },
+  };
+}

package/src/server/publishing-routes.ts

@@ -0,0 +1,41 @@
+import type { createHfCredentials } from "../publishing/credentials";
+import type { createPublisher, PublishRequest } from "../publishing/upload";
+
+/** HTTP parsing and wire errors stay above credential and publishing policy. */
+export function createPublishingRoutes(options: {
+  credentials: ReturnType<typeof createHfCredentials>;
+  publish: ReturnType<typeof createPublisher>;
+}) {
+  const error = (message: string, status = 400) => Response.json({ ok: false, error: message }, { status });
+  return { async handle(request: Request): Promise<Response | null> {
+    const path = new URL(request.url).pathname;
+    const settings = path === "/api/settings/hf-token";
+    const push = path.match(/^\/api\/(quantize|finetune|dataset)\/push$/);
+    if (!(settings && ["GET", "POST"].includes(request.method)) && !(push && request.method === "POST")) return null;
+    try {
+      request.signal.throwIfAborted();
+      if (settings && request.method === "GET") return Response.json({ ok: true, hasToken: options.credentials.get() !== null });
+      const raw: unknown = await request.json().catch(() => null);
+      request.signal.throwIfAborted();
+      if (!raw || typeof raw !== "object" || Array.isArray(raw)) return error("expected a JSON object");
+      const body = raw as Record<string, unknown>;
+      if (settings) {
+        if (typeof body.token !== "string" || !body.token.trim()) return error("token required");
+        options.credentials.save(body.token);
+        return Response.json({ ok: true });
+      }
+      if (typeof body.repo_id !== "string" || !body.repo_id.trim()) return error("repo_id required");
+      for (const key of ["job_id", "source_path"] as const) {
+        if (body[key] != null && (typeof body[key] !== "string" || !body[key].trim())) return error(`${key} must be a nonempty string`);
+      }
+      if (body.private != null && typeof body.private !== "boolean") return error("private must be a boolean");
+      const result = await options.publish({ kind: push![1] as PublishRequest["kind"], repoId: body.repo_id,
+        private: body.private as boolean | undefined, sourcePath: body.source_path as string | undefined,
+        jobId: body.job_id as string | undefined, signal: request.signal });
+      return Response.json({ ok: true, url: result.url });
+    } catch (failure) {
+      if (request.signal.aborted) return error("Request cancelled", 499);
+      return error(failure instanceof Error ? failure.message : String(failure));
+    }
+  } };
+}

package/src/{serve → server}/request-plan.ts

@@ -1,9 +1,9 @@
-import { snapshotGenerationPolicy } from "../backends/mlx/request-policy";
-import type { GenerateOptions } from "../generate";
-import type { RequestShape } from "./generation-gateway";
-import type { DisposableResource } from "../contracts/resources";
-import { disposeResources, ownResource } from "../engine/resources";
-export type { DisposableResource } from "../contracts/resources";
+import { snapshotGenerationPolicy } from "@mlx-bun/inference/execution/request-policy";
+import type { GenerateOptions } from "@mlx-bun/inference/generation";
+import type { RequestShape } from "../engine/completion";
+import type { DisposableResource } from "@mlx-bun/inference/contracts/portable";
+import { disposeResources, ownResource } from "@mlx-bun/inference/runtime/resources";
+export type { DisposableResource } from "@mlx-bun/inference/contracts/portable";
 
 export class RequestOwnership {
   #owner = ownResource(new Set<DisposableResource>(), disposeResources);
@@ -39,6 +39,7 @@ export interface RequestPlanInput {
   topLogprobs: number;
   adapterIds: string[];
   hasVision: boolean;
+  hasPreparedPrefixIdentity?: boolean;
   userSeed: boolean;
   hasGrammar: boolean;
   hasDraft: boolean;
@@ -137,6 +138,7 @@ export function planRequest(input: RequestPlanInput): PlanRequestResult {
 
   const shape: RequestShape = {
     hasVision: input.hasVision,
+    hasPreparedPrefixIdentity: input.hasPreparedPrefixIdentity,
     hasAdapters: !!options.adapters?.length,
     hasRepetitionPenalty: !!options.repetitionPenalty,
     hasLogitsExtras: !!(
@@ -145,8 +147,6 @@ export function planRequest(input: RequestPlanInput): PlanRequestResult {
     ),
     wantsLogprobs: captureLogprobs,
     userSeed: input.userSeed,
-    kvQuant: !!(options.kvConfig?.length || options.kvBits),
-    turboQuant: !!options.turboQuant,
     hasGrammar: input.hasGrammar,
     hasDraft: input.hasDraft,
   };

package/src/{serve → server}/request-prep.ts

@@ -1,23 +1,24 @@
+import { requireChatTemplate } from "../engine/model-host";
+import { textPrompt } from "./text-prompt";
 // Per-request preparation shared by every completion surface (chat,
 // raw text, /signal, /generate): request → GenerateOptions with the
 // server/model defaults folded in, chat-template rendering + prompt ids,
 // the stable prompt-cache boundary probe, grammar compilation, and the
 // tool-call stream router. Resolved ONCE per server (createRequestPrep) and
-// captured by the route handlers. Extracted from src/server.ts (repo-taming
-// Phase 4).
-import type { ToolDefinition } from "../chat-template";
+// captured by the route handlers.
+import type { ChatMessage, ToolDefinition } from "@mlx-bun/inference/input";
 import {
   fillEchoConfig,
   fillMaxSpan,
   FillSession,
   resolveFillMode,
-} from "../fill/fill-session";
-import { compileStrictFillRows, type StrictRowPlan } from "../fill/schema-rows";
-import type { GenerateOptions } from "../generate";
-import { compileGrammarRequest, grammarEnabled, type GrammarRequest } from "../grammar";
-import type { KvSchemeOptions } from "../kv-scheme";
-import { isMiniCPM5Config } from "../model/support";
-import type { HlgConfig } from "../sampler";
+} from "@mlx-bun/inference/generation/fill";
+import { compileStrictFillRows, type StrictRowPlan } from "@mlx-bun/inference/generation/fill";
+import type { GenerateOptions } from "@mlx-bun/inference/generation";
+import type { GrammarRequest } from "@mlx-bun/inference/sampling/grammar";
+import { runtimeValue } from "@mlx-bun/inference/runtime/config";
+import { GENERIC_GENERATION_DEFAULTS } from "@mlx-bun/inference/models/profile";
+import type { HlgConfig } from "@mlx-bun/inference/sampling";
 import {
   nextDefaultSeed,
   normalizeMessages,
@@ -26,7 +27,8 @@ import {
   resolveHlg,
   type ChatRequestParams,
 } from "./chat-request";
-import type { ServingContext } from "./model-host";
+import type { LoadedModelContext as ModelContext } from "../engine/model-host";
+import type { PromptTokenHistory } from "./generated-token-history";
 import { selectToolStreamMode, ToolAwareStream, type ToolStreamMode } from "./token-streams";
 
 /** The ServerOptions fields request preparation reads (server-wide
@@ -37,20 +39,18 @@ export interface RequestPrepOptions {
   defaultTopP?: number;
   defaultTopK?: number;
   hlg?: HlgConfig;
-  pagedKv?: { blockSize?: number };
 }
 
 export function createRequestPrep(input: {
-  ctx: ServingContext;
+  ctx: ModelContext;
   serverOptions: RequestPrepOptions;
-  /** The server-wide KV scheme (resolveKvScheme(...).generationOptions),
-   *  spread into every request's GenerateOptions. */
-  kvScheme: KvSchemeOptions;
-  /** `--max-tokens` / the GLM memory plan's generation cap; undefined = the
+  /** `--max-tokens` / a runtime memory plan's generation cap; undefined = the
    *  surface's own default (chat 65 536, raw completion 512). */
   defaultGeneratedTokens: number | undefined;
+  tokenHistory?: PromptTokenHistory;
 }) {
-  const { ctx, serverOptions, kvScheme, defaultGeneratedTokens } = input;
+  const { ctx, serverOptions, defaultGeneratedTokens } = input;
+  const generationDefaults = ctx.generationDefaults ?? GENERIC_GENERATION_DEFAULTS;
 
   // XTC never removes EOS or the newline token: mlx_lm.server passes
   // [tokenizer.eos_token_id, tokenizer.encode("\n")] as xtc_special_tokens.
@@ -65,33 +65,35 @@ export function createRequestPrep(input: {
 
   // Effective enable_thinking for a request, with the same precedence the chat
   // template uses (extracted so sampling and template rendering can't disagree):
-  // explicit chat_template_kwargs.enable_thinking → reasoning_effort ("none" =
-  // off) → server --thinking default → model default (MiniCPM5 → off). undefined
+  // explicit chat_template_kwargs.enable_thinking (null = template default) → reasoning_effort ("none" =
+  // off) → server --thinking default → the model's declared default. undefined
   // means "not a switchable-thinking model / leave the template default".
   const resolveEnableThinking = (req: ChatRequestParams): boolean | undefined => {
     const explicit = req.chat_template_kwargs?.enable_thinking;
+    if (explicit === null) return undefined;
     if (typeof explicit === "boolean") return explicit;
     const effort = req.reasoning_effort;
     if (effort !== undefined) return effort !== "none";
     if (serverOptions.defaultThinking !== undefined) return serverOptions.defaultThinking;
-    return isMiniCPM5Config(ctx.model.config) ? false : undefined;
+    return generationDefaults.enableThinking;
   };
 
   const toOptions = (req: ChatRequestParams): GenerateOptions & { stopSequences: string[] } => {
     // Sampling follows the thinking state — which the web UI's thinking button
     // drives via enable_thinking. Model authors publish a SINGLE
     // generation_config temperature (the think-mode value) but recommend a
-    // cooler one for direct, no-think replies (MiniCPM5 card: 0.9 think / 0.7
-    // no-think, top_p 0.95 for both). So with no explicit temperature set, a
-    // no-think turn runs at most NO_THINK_TEMPERATURE while a think turn keeps
-    // the model's hotter configured default. An explicit request/CLI
-    // temperature always wins. top_p is unchanged by mode.
-    const NO_THINK_TEMPERATURE = 0.7;
+    // cooler one for direct, no-think replies. So with no explicit temperature
+    // set, a no-think turn runs at most the model's declared
+    // noThinkTemperatureCap while a think turn keeps the model's hotter
+    // configured default. An explicit request/CLI temperature always wins.
+    // top_p is unchanged by mode.
     const genTemp = ctx.genDefaults.temperature ?? 0.7;
     const defaultTemp = resolveEnableThinking(req) === false
-      ? Math.min(genTemp, NO_THINK_TEMPERATURE)
+      ? Math.min(genTemp, generationDefaults.noThinkTemperatureCap)
       : genTemp;
     return {
+      cacheSessionId: typeof req.session_id === "string" ? req.session_id :
+        typeof req.prompt_cache_key === "string" ? req.prompt_cache_key : undefined,
       // Unset means generate until EOS/stop. Only a client cap or an explicit
       // operator limit constrains completion length; fit estimates do not.
       maxTokens: req.max_completion_tokens ?? req.max_tokens ??
@@ -120,10 +122,6 @@ export function createRequestPrep(input: {
       // decode layer below and halts the loop via onToken → false.
       stopSequences: (typeof req.stop === "string" ? [req.stop] : req.stop ?? [])
         .filter((s) => typeof s === "string" && s.length > 0),
-      ...kvScheme,
-      // Paged KV is a server-wide mode like the kv scheme above (v1:
-      // serial bf16 gemma4 — the createServer refusals hold the invariants).
-      ...(serverOptions.pagedKv ? { pagedKv: serverOptions.pagedKv } : {}),
     };
   };
 
@@ -134,13 +132,14 @@ export function createRequestPrep(input: {
    *  + Warning header (oMLX parity — never 500). Honors MLX_BUN_GRAMMAR=0. */
   const compileGrammarForRequest = async (
     req: ChatRequestParams,
-  ): Promise<{ controller: import("../grammar").GrammarController | null; degradeHint: string | null }> => {
-    if (!grammarEnabled()) {
+  ): Promise<{ controller: import("@mlx-bun/inference/sampling/grammar").GrammarController | null; degradeHint: string | null }> => {
+    if (runtimeValue("MLX_BUN_GRAMMAR") === "0") {
       return {
         controller: null,
         degradeHint: "grammar compilation disabled by MLX_BUN_GRAMMAR=0",
       };
     }
+    const { compileGrammarRequest } = await import("@mlx-bun/inference/sampling/grammar");
     const r = await compileGrammarRequest(
       req as GrammarRequest,
       ctx.tokenizer,
@@ -150,11 +149,11 @@ export function createRequestPrep(input: {
     return { controller: r.controller, degradeHint: r.degradeHint };
   };
 
-  // Map OpenAI reasoning_effort levels onto the Qwen3.8 template's supported
-  // set (xhigh|medium|low — the template raises on anything else). "none"
+  // Map OpenAI reasoning_effort levels onto the depth set a template that reads
+  // `reasoning_effort` supports (xhigh|medium|low — such a template raises on anything else). "none"
   // means thinking off (handled by resolveEnableThinking), so no depth is
   // passed. Only consumed for templates with readsReasoningEffort.
-  const qwenReasoningEffort = (
+  const templateReasoningEffort = (
     effort: ChatRequestParams["reasoning_effort"],
   ): "xhigh" | "medium" | "low" | undefined => {
     switch (effort) {
@@ -177,7 +176,7 @@ export function createRequestPrep(input: {
     return {
       tools,
       enableThinking: resolveEnableThinking(req),
-      reasoningEffort: qwenReasoningEffort(req.reasoning_effort),
+      reasoningEffort: templateReasoningEffort(req.reasoning_effort),
       preserveThinking: req.chat_template_kwargs?.preserve_thinking,
     };
   };
@@ -186,12 +185,12 @@ export function createRequestPrep(input: {
     req: ChatRequestParams,
     tools: ToolDefinition[] | null,
   ): { ids: number[]; startInThinking: boolean } => {
+    // A context without a template serves only through a supplied prompt builder.
+    requireChatTemplate(ctx);
     const opts = templateOptionsFor(req, tools);
-    const rendered = ctx.template.render(normalizeMessages(req.messages), opts);
-    const ids = ctx.tokenizer.encode(rendered);
-    // template includes <bos>; tokenizer post-processor also prepends one
-    const trimmed = ids[0] === ids[1] && ids[0] === ctx.tokenizer.bosTokenId ? ids.slice(1) : ids;
-    return { ids: trimmed, startInThinking: promptEndsInOpenThink(rendered) };
+    const { rendered, ids } = textPrompt(ctx.template, ctx.tokenizer, normalizeMessages(req.messages), opts);
+    return { ids: input.tokenHistory?.resolve(rendered, ids) ?? ids,
+      startInThinking: promptEndsInOpenThink(rendered) };
   };
 
   /** STABLE cache boundary: the prompt's tail is a generation primer (e.g.
@@ -213,24 +212,40 @@ export function createRequestPrep(input: {
     tools: ToolDefinition[] | null,
     trimmed: number[],
   ): number => {
+    // Outside the best-effort probe: a builder asking for it needs a template.
+    requireChatTemplate(ctx);
     const opts = templateOptionsFor(req, tools);
     const mode = `${opts.enableThinking}|${!!tools?.length}|${opts.reasoningEffort ?? ""}|${opts.preserveThinking ?? ""}`;
     const primer = primerLenByMode.get(mode);
     if (primer !== undefined) return Math.max(0, trimmed.length - primer);
     let stableLen = trimmed.length;
     try {
-      const probe = ctx.tokenizer.encode(
-        ctx.template.render(
-          [...normalizeMessages(req.messages), { role: "assistant", content: "x" }],
-          opts,
-        ),
-      );
-      const probeTrimmed =
-        probe[0] === probe[1] && probe[0] === ctx.tokenizer.bosTokenId ? probe.slice(1) : probe;
-      let i = 0;
-      const n = Math.min(trimmed.length, probeTrimmed.length);
-      while (i < n && trimmed[i] === probeTrimmed[i]) i++;
-      stableLen = i;
+      const canonical = ctx.tokenizer.encode(ctx.template.render(normalizeMessages(req.messages), opts));
+      const current = canonical[0] === canonical[1] && canonical[0] === ctx.tokenizer.bosTokenId
+        ? canonical.slice(1) : canonical;
+      const replies: ChatMessage[] = [{ role: "assistant", content: "x" }];
+      // Some templates retain an empty think primer for text replies but
+      // omit it before a tool call. Both reply forms must preserve the
+      // boundary retained by an untrimmable recurrent cache.
+      if (tools?.length) replies.push({ role: "assistant", content: null, tool_calls: [{
+        id: "cache-boundary-probe", type: "function",
+        function: { name: tools[0]!.function.name, arguments: {} },
+      }] });
+      let i = current.length;
+      for (const reply of replies) {
+        const probe = ctx.tokenizer.encode(ctx.template.render([
+          ...normalizeMessages(req.messages), reply, { role: "user", content: "x" },
+        ], opts));
+        const probeTrimmed = probe[0] === probe[1] && probe[0] === ctx.tokenizer.bosTokenId ? probe.slice(1) : probe;
+        let common = 0;
+        const n = Math.min(i, probeTrimmed.length);
+        while (common < n && current[common] === probeTrimmed[common]) common++;
+        i = common;
+      }
+      // Generated-history provenance may use a different BPE segmentation
+      // inside old turns. Measure the primer canonically, then subtract it
+      // from the actual request IDs that inference will receive.
+      stableLen = Math.max(0, trimmed.length - (current.length - i));
       // Memoize the mode's primer length (sanity-capped: a "primer" longer
       // than 64 tokens means the probe diverged for content reasons —
       // don't generalize that).
@@ -241,16 +256,15 @@ export function createRequestPrep(input: {
   };
 
   const toolStreamMode = (tools: ToolDefinition[] | null): ToolStreamMode =>
-    // Gemma-4 keeps the token-level sentinel path; every other family
-    // (MiniCPM5, Qwen3/3.5, Tier-0 generics) emits tool calls as DECODED TEXT
-    // — see selectToolStreamMode for why sentinel ids must stay family-gated.
-    selectToolStreamMode(ctx.model.config.modelType, !!tools?.length);
+    // A model that declares sentinel tokens keeps the token-level path; every
+    // other model emits tool calls as DECODED TEXT — see selectToolStreamMode.
+    selectToolStreamMode(ctx.sentinels, !!tools?.length);
 
   const toolRouter = (
     tools: ToolDefinition[] | null,
     onParseFailure?: () => void,
   ): ToolAwareStream =>
-    new ToolAwareStream(ctx.tokenizer, toolStreamMode(tools), tools, onParseFailure);
+    new ToolAwareStream(ctx.tokenizer, toolStreamMode(tools), tools, onParseFailure, ctx.sentinels);
 
   /** Strict fill rows are a function of (template mode, tool signature) only —
    *  they are token spans of the assistant turn, which opens with a special
@@ -266,15 +280,15 @@ export function createRequestPrep(input: {
       (t.function?.parameters?.additionalProperties as unknown) ?? null,
     ]));
 
-  /** Token fast-forwarding (docs/design/speculative-decoding.md "Token
+  /** Token fast-forwarding (`02d723a:docs/design/speculative-decoding.md` §7 "Token
    *  fast-forwarding"): compile this request's determined-span table, or null
    *  when the feature is off or the request's shape refuses it.
    *
    *  Refusals owned here (body-level); the ones only ChatStage can see —
-   *  a compiled grammar, media prompts, a mounted draft model, a quantized KV
-   *  scheme — are applied there. Continuous (batch) placement needs no
-   *  refusal: generate() is the only fill site, so a batched request simply
-   *  does not fill.
+   *  a compiled grammar, media prompts and a mounted draft model — are applied
+   *  there. The execution binding owns method support (every cache serves it). Strict
+   *  shared execution keeps the sampler's absolute token positions, including
+   *  known tokens; an explicit seed therefore remains replayable.
    *
    *  MLX_BUN_FILL=echo additionally arms the echo index (K3c, Lab tier): the
    *  strict rows keep policy "assert", and copied spans ride the same
@@ -287,9 +301,9 @@ export function createRequestPrep(input: {
     const mode = resolveFillMode();
     if (mode === "off") return null;
     if (!tools?.length) return null;
-    // A user-fixed seed means reproducibility: the sampler's step index would
-    // skip the injected positions, so an identical request could not replay.
-    if (req.seed !== undefined) return null;
+    // The determined spans come from the template; a builder-only context has none.
+    if (!ctx.template) return null;
+    // Seed support is selected by the execution binding after placement.
     // Injected tokens are never sampled — they have no logprob row.
     if (req.logprobs === true) return null;
     if (typeof req.top_logprobs === "number" && req.top_logprobs > 0) return null;

package/src/{serve → server}/responses-client.ts

@@ -1,21 +1,32 @@
-import { ResponseStore, resolveResponsesConversation, type ResponseHistory, type ResponsesRequest } from "../responses";
+// The parent's Responses conversation history in the isolated server (main's
+// responses-client.ts): `previous_response_id` resolves against the parent's
+// store, the resolved conversation goes to the worker without one, and the
+// completed record is remembered here, so history survives worker restarts.
+// The worker keeps its own bounded store for the requests it answers; the
+// parent never consults it.
+import { ResponseStore, resolveResponsesConversation, type ResponseHistory, type ResponsesRequest } from "./responses";
+
+const responsesError = (status: number, message: string) => Response.json({ error: {
+  message, type: "invalid_request_error", param: null, code: null,
+} }, { status });
 
-/** Application-owned conversation history over a data-only HTTP host. Workers
- * translate/run the request; the parent stores only completed response records. */
 export function createResponsesClient(store: ResponseHistory = new ResponseStore()) {
   return {
     get stats() { return { entries: store.size, bytes: store.totalBytes, max_bytes: store.maxBytes, ttl_ms: store.ttlMs }; },
+    /** `forward` sends the rewritten request to the worker and returns its response as is. */
     async forward(request: Request, forward: (request: Request) => Promise<Response>): Promise<Response> {
-      const error = (status: number, message: string) => Response.json({ error: {
-        message, type: "invalid_request_error", param: null, code: null,
-      } }, { status });
       let body: ResponsesRequest;
       try { body = await request.json() as ResponsesRequest; }
-      catch { return error(400, "invalid JSON body"); }
-      if (!body || typeof body !== "object" || Array.isArray(body)) return error(400, "invalid JSON body");
+      catch { return responsesError(400, "invalid JSON body"); }
+      if (!body || typeof body !== "object" || Array.isArray(body)) return responsesError(400, "invalid JSON body");
+      // The container is validated before the previous id resolves, as the
+      // worker does: only a missing or expired previous response is a 404.
+      if ((body.input != null && typeof body.input !== "string" && !Array.isArray(body.input)) ||
+          (body.previous_response_id != null && typeof body.previous_response_id !== "string"))
+        return responsesError(400, "input must be a string or array; previous_response_id must be a string");
       let conversation: ReturnType<typeof resolveResponsesConversation>;
       try { conversation = resolveResponsesConversation(body, store); }
-      catch (e) { return error(404, (e as Error).message); }
+      catch (error) { return responsesError(404, error instanceof Error ? error.message : String(error)); }
       const headers = new Headers(request.headers);
       headers.delete("content-length");
       headers.set("x-mlx-bun-response-owner", "parent");
@@ -32,15 +43,20 @@ export function createResponsesClient(store: ResponseHistory = new ResponseStore
       if (!body.stream) {
         const value = await response.json() as Record<string, unknown>;
         remember(value);
-        const headers = new Headers(response.headers); headers.delete("content-length");
-        return Response.json(restorePrevious(value), { status: response.status, headers });
+        const outputHeaders = new Headers(response.headers);
+        outputHeaders.delete("content-length");
+        return Response.json(restorePrevious(value), { status: response.status, headers: outputHeaders });
       }
       if (!response.body) return response;
       const decoder = new TextDecoder(), encoder = new TextEncoder();
       let pending = "";
+      // Frames the worker did not shape as JSON (keep-alive comments, [DONE])
+      // pass through; `response.completed` is remembered and every response
+      // object gets the previous id the caller sent.
       const rewrite = (frame: string): string => frame.replace(/^data: (.+)$/m, (line, data: string) => {
         if (data === "[DONE]") return line;
-        const event = JSON.parse(data) as Record<string, unknown>;
+        let event: Record<string, unknown>;
+        try { event = JSON.parse(data) as Record<string, unknown>; } catch { return line; }
         if (event.response && typeof event.response === "object") {
           const value = event.response as Record<string, unknown>;
           if (event.type === "response.completed") remember(value);
@@ -48,7 +64,8 @@ export function createResponsesClient(store: ResponseHistory = new ResponseStore
         }
         return `data: ${JSON.stringify(event)}`;
       });
-      const outputHeaders = new Headers(response.headers); outputHeaders.delete("content-length");
+      const outputHeaders = new Headers(response.headers);
+      outputHeaders.delete("content-length");
       return new Response(response.body.pipeThrough(new TransformStream<Uint8Array, Uint8Array>({
         transform(chunk, controller) {
           pending += decoder.decode(chunk, { stream: true });