mlx-bun 0.2.0 → 0.4.0

package/src/serve/parent-application.ts

@@ -0,0 +1,56 @@
+import { handleStaticRoute } from "./static-routes";
+import { STATIC_ROUTE_ASSETS } from "../web-assets";
+import { createResponsesClient } from "./responses-client";
+import type { DisposableResource } from "../contracts/resources";
+import type { JobStore } from "../jobs/db";
+import { handleLabRoute, matchLabRoute } from "./lab-routes";
+import { handleAdminRoute, matchAdminRoute } from "./admin-routes";
+
+/** CPU application state stays in the parent when model workers restart/evict.
+ * Native inspection/merge/export routes still execute inside a model worker. */
+export function createParentApplication(options: {
+  acquireGpu(signal: AbortSignal): Promise<DisposableResource>;
+  serverPort(): number | undefined;
+  invalidateLibrary(): void;
+  createJobStore?(): Promise<JobStore>;
+}) {
+  const responses = createResponsesClient();
+  let jobs: Promise<JobStore> | undefined;
+  let closed = false;
+  const ensureJobs = () => {
+    if (closed) return Promise.reject(new Error("application host is closed"));
+    return jobs ??= (async () => {
+      const { JobStore, registerRunner } = await import("../jobs");
+      const store = options.createJobStore ? await options.createJobStore() : new JobStore();
+      store.markZombies();
+      const { datasetRunner } = await import("../dataset/job");
+      registerRunner("dataset", datasetRunner);
+      return store;
+    })();
+  };
+  return {
+    get responseStats() { return responses.stats; },
+    async handle(request: Request, forward?: (request: Request) => Promise<Response>): Promise<Response | null> {
+      const url = new URL(request.url);
+      const staticResponse = handleStaticRoute(url, request, STATIC_ROUTE_ASSETS);
+      if (staticResponse) return staticResponse;
+      if (forward && request.method === "POST" && url.pathname === "/v1/responses")
+        return responses.forward(request, forward);
+      const admin = matchAdminRoute(request.method, url.pathname);
+      if (admin) return handleAdminRoute(url, request, { ensureJobs, invalidateLibrary: options.invalidateLibrary });
+      const route = matchLabRoute(request.method, url.pathname);
+      if (route !== "quantize-submit" && route !== "finetune-submit" &&
+          route !== "dataset-submit" && route !== "dataset-templates") return null;
+      return handleLabRoute(url, request, { ensureJobs, acquireGpu: options.acquireGpu,
+        serverPort: options.serverPort, invalidateLibrary: options.invalidateLibrary });
+    },
+    async close() {
+      closed = true;
+      if (!jobs) return;
+      const store = await jobs;
+      const { closeSubprocessJobs, closeInProcessJobs } = await import("../jobs");
+      await Promise.all([closeSubprocessJobs(store), closeInProcessJobs(store)]);
+      store.close();
+    },
+  };
+}

package/src/serve/pipeline.ts

@@ -0,0 +1,33 @@
+// Shared contracts of the request pipeline. Each stage is one configured
+// program — `run(input) → output` — and the server composes them in order,
+// like a shell pipe:
+//
+//   new ChatRequest(body) → ChatStage.run → InferenceRequest
+//                         → InferenceStage.run → InferenceResult → wire JSON/SSE
+//
+// A stage knows only the interface it reads and the one it writes.
+import type { CompletionEvent } from "./completion-sink";
+import type { CompletionUsage } from "./completion-executor";
+import type { PromptResponseTrace } from "./prompt-response-trace";
+
+/** A request a stage refuses. `status` is the HTTP status the wire adapter
+ *  answers with; `body` the protocol-neutral error object (adapters reshape
+ *  it: OpenAI `{error}`, Anthropic `{type:"error"}`, Responses `{error}`). */
+export class RequestError extends Error {
+  constructor(
+    readonly status: number,
+    message: string,
+    readonly body: Record<string, unknown> = { message },
+  ) {
+    super(message);
+  }
+}
+
+/** What flows back to the caller while a request is in flight: cancellation,
+ *  tracing, and (for streaming surfaces) events + live usage. */
+export interface RunControl {
+  signal?: AbortSignal;
+  trace?: PromptResponseTrace;
+  onEvents?(events: readonly CompletionEvent[]): void | false | Promise<void | false>;
+  onUsageProgress?(usage: Readonly<CompletionUsage>): void;
+}

package/src/serve/preparation.ts

@@ -0,0 +1,33 @@
+/** Host-side admission before grammar/media preparation can allocate native
+ * resources. The implementation shares the generation execution domain. */
+export interface PreparationExecutor {
+  reserve?(kind: "media" | "constraint", signal?: AbortSignal): Promise<import("../contracts/resources").DisposableResource>;
+  run<T>(prepare: () => Promise<T>, signal?: AbortSignal): Promise<T>;
+}
+
+import { AdmissionPool } from "../engine/admission";
+import { CancellationSource } from "../engine/cancellation";
+
+export async function acquireReservation(pool: Pick<AdmissionPool, "acquire">, signal?: AbortSignal) {
+  signal?.throwIfAborted();
+  const cancellation = new CancellationSource();
+  const abort = () => cancellation.cancel("requested");
+  signal?.addEventListener("abort", abort, { once: true });
+  try { return await pool.acquire(cancellation); }
+  finally { signal?.removeEventListener("abort", abort); }
+}
+
+/** One retained media preparation at a time; grammar rows may form a batch. */
+export function createPreparationExecutor(
+  run: PreparationExecutor["run"], grammarCapacity: number,
+): PreparationExecutor & { close(): void } {
+  const media = new AdmissionPool(1);
+  const constraints = new AdmissionPool(Math.max(1, grammarCapacity));
+  return {
+    run,
+    async reserve(kind, signal) {
+      return acquireReservation(kind === "media" ? media : constraints, signal);
+    },
+    close() { media.close(); constraints.close(); },
+  };
+}

package/src/serve/request-plan.ts

@@ -1,28 +1,31 @@
+import { snapshotGenerationPolicy } from "../backends/mlx/request-policy";
 import type { GenerateOptions } from "../generate";
 import type { RequestShape } from "./generation-gateway";
-
-export interface DisposableResource {
-  dispose(): void;
-}
+import type { DisposableResource } from "../contracts/resources";
+import { disposeResources, ownResource } from "../engine/resources";
+export type { DisposableResource } from "../contracts/resources";
 
 export class RequestOwnership {
-  #resources = new Set<DisposableResource>();
-  #transferred = false;
+  #owner = ownResource(new Set<DisposableResource>(), disposeResources);
+  #reservations = ownResource(new Set<DisposableResource>(), disposeResources);
+
+  /** Reservations live until completion, including after native ownership transfers. */
+  retain<T extends DisposableResource>(reservation: T): T {
+    this.#reservations.borrow().add(reservation);
+    return reservation;
+  }
 
   own<T extends DisposableResource | null | undefined>(resource: T): T {
-    if (resource) this.#resources.add(resource);
+    if (resource) this.#owner.borrow().add(resource);
     return resource;
   }
 
   transfer(): void {
-    this.#transferred = true;
-    this.#resources.clear();
+    this.#owner.transfer();
   }
 
   dispose(): void {
-    if (this.#transferred) return;
-    for (const resource of this.#resources) resource.dispose();
-    this.#resources.clear();
+    disposeResources([{ dispose: () => this.#owner.close() }, { dispose: () => this.#reservations.close() }]);
   }
 }
 
@@ -30,7 +33,7 @@ export interface RequestPlanInput {
   promptIds: number[];
   options: GenerateOptions & { stopSequences: string[] };
   requestedMaxTokens: number;
-  maxSafeContext: number;
+  contextLimit: number | null;
   stream: boolean;
   wantLogprobs: boolean;
   topLogprobs: number;
@@ -111,11 +114,12 @@ export type PlanRequestResult = RequestPlan | RequestRejection;
  * options, adapter selection, and lane shape are derived here once for chat
  * and raw completions. */
 export function planRequest(input: RequestPlanInput): PlanRequestResult {
-  const available = input.maxSafeContext - input.promptIds.length;
+  const available = input.contextLimit === null ? Infinity
+    : input.contextLimit - input.promptIds.length;
   if (available < 1) {
     return new RequestRejection(
       `prompt is ${input.promptIds.length} tokens but the memory budget caps ` +
-        `safe context at ${input.maxSafeContext} — no room to generate; ` +
+        `safe context at ${input.contextLimit} — no room to generate; ` +
         "shorten the prompt or raise --memory-budget",
       input.ownership,
     );
@@ -123,6 +127,7 @@ export function planRequest(input: RequestPlanInput): PlanRequestResult {
 
   const options = { ...input.options };
   options.maxTokens = Math.min(input.requestedMaxTokens, available);
+  options.seedWasExplicit = input.userSeed;
   if (input.adapterIds.length) options.adapters = [...input.adapterIds];
   const captureLogprobs = !input.stream && (input.wantLogprobs || input.topLogprobs > 0);
   if (captureLogprobs) {
@@ -147,9 +152,9 @@ export function planRequest(input: RequestPlanInput): PlanRequestResult {
   };
 
   return new RequestPlan({
-    promptIds: input.promptIds,
-    options,
-    shape,
+    promptIds: Object.freeze([...input.promptIds]) as number[],
+    options: snapshotGenerationPolicy(options),
+    shape: Object.freeze(shape),
     captureLogprobs,
     wantLogprobs: input.wantLogprobs,
     topLogprobs: input.topLogprobs,

package/src/serve/request-prep.ts

@@ -0,0 +1,344 @@
+// Per-request preparation shared by every completion surface (chat,
+// raw text, /signal, /generate): request → GenerateOptions with the
+// server/model defaults folded in, chat-template rendering + prompt ids,
+// the stable prompt-cache boundary probe, grammar compilation, and the
+// tool-call stream router. Resolved ONCE per server (createRequestPrep) and
+// captured by the route handlers. Extracted from src/server.ts (repo-taming
+// Phase 4).
+import type { ToolDefinition } from "../chat-template";
+import {
+  fillEchoConfig,
+  fillMaxSpan,
+  FillSession,
+  resolveFillMode,
+} from "../fill/fill-session";
+import { compileStrictFillRows, type StrictRowPlan } from "../fill/schema-rows";
+import type { GenerateOptions } from "../generate";
+import { compileGrammarRequest, grammarEnabled, type GrammarRequest } from "../grammar";
+import type { KvSchemeOptions } from "../kv-scheme";
+import { isMiniCPM5Config } from "../model/support";
+import type { HlgConfig } from "../sampler";
+import {
+  nextDefaultSeed,
+  normalizeMessages,
+  parseLogitBias,
+  promptEndsInOpenThink,
+  resolveHlg,
+  type ChatRequestParams,
+} from "./chat-request";
+import type { ServingContext } from "./model-host";
+import { selectToolStreamMode, ToolAwareStream, type ToolStreamMode } from "./token-streams";
+
+/** The ServerOptions fields request preparation reads (server-wide
+ *  defaults a request field overrides). */
+export interface RequestPrepOptions {
+  defaultThinking?: boolean;
+  defaultTemperature?: number;
+  defaultTopP?: number;
+  defaultTopK?: number;
+  hlg?: HlgConfig;
+  pagedKv?: { blockSize?: number };
+}
+
+export function createRequestPrep(input: {
+  ctx: ServingContext;
+  serverOptions: RequestPrepOptions;
+  /** The server-wide KV scheme (resolveKvScheme(...).generationOptions),
+   *  spread into every request's GenerateOptions. */
+  kvScheme: KvSchemeOptions;
+  /** `--max-tokens` / the GLM memory plan's generation cap; undefined = the
+   *  surface's own default (chat 65 536, raw completion 512). */
+  defaultGeneratedTokens: number | undefined;
+}) {
+  const { ctx, serverOptions, kvScheme, defaultGeneratedTokens } = input;
+
+  // XTC never removes EOS or the newline token: mlx_lm.server passes
+  // [tokenizer.eos_token_id, tokenizer.encode("\n")] as xtc_special_tokens.
+  // We pass the flat equivalent — ALL configured EOS ids plus the bare newline
+  // id(s), encoded WITHOUT special tokens (mlx-lm's encode("\n") can drag a
+  // BOS along on some tokenizers; protecting BOS from XTC is a no-op, so the
+  // flat/no-BOS form is behaviorally identical and cleaner).
+  const xtcSpecialTokens = [
+    ...ctx.model.config.eosTokenIds,
+    ...ctx.tokenizer.encode("\n", false),
+  ];
+
+  // Effective enable_thinking for a request, with the same precedence the chat
+  // template uses (extracted so sampling and template rendering can't disagree):
+  // explicit chat_template_kwargs.enable_thinking → reasoning_effort ("none" =
+  // off) → server --thinking default → model default (MiniCPM5 → off). undefined
+  // means "not a switchable-thinking model / leave the template default".
+  const resolveEnableThinking = (req: ChatRequestParams): boolean | undefined => {
+    const explicit = req.chat_template_kwargs?.enable_thinking;
+    if (typeof explicit === "boolean") return explicit;
+    const effort = req.reasoning_effort;
+    if (effort !== undefined) return effort !== "none";
+    if (serverOptions.defaultThinking !== undefined) return serverOptions.defaultThinking;
+    return isMiniCPM5Config(ctx.model.config) ? false : undefined;
+  };
+
+  const toOptions = (req: ChatRequestParams): GenerateOptions & { stopSequences: string[] } => {
+    // Sampling follows the thinking state — which the web UI's thinking button
+    // drives via enable_thinking. Model authors publish a SINGLE
+    // generation_config temperature (the think-mode value) but recommend a
+    // cooler one for direct, no-think replies (MiniCPM5 card: 0.9 think / 0.7
+    // no-think, top_p 0.95 for both). So with no explicit temperature set, a
+    // no-think turn runs at most NO_THINK_TEMPERATURE while a think turn keeps
+    // the model's hotter configured default. An explicit request/CLI
+    // temperature always wins. top_p is unchanged by mode.
+    const NO_THINK_TEMPERATURE = 0.7;
+    const genTemp = ctx.genDefaults.temperature ?? 0.7;
+    const defaultTemp = resolveEnableThinking(req) === false
+      ? Math.min(genTemp, NO_THINK_TEMPERATURE)
+      : genTemp;
+    return {
+      // Unset means generate until EOS/stop. Only a client cap or an explicit
+      // operator limit constrains completion length; fit estimates do not.
+      maxTokens: req.max_completion_tokens ?? req.max_tokens ??
+        defaultGeneratedTokens,
+      temperature: req.temperature ?? serverOptions.defaultTemperature ?? defaultTemp,
+      topP: req.top_p ?? serverOptions.defaultTopP ?? ctx.genDefaults.topP ?? 0,
+      topK: req.top_k ?? serverOptions.defaultTopK ?? ctx.genDefaults.topK ?? 0,
+      seed: req.seed ?? nextDefaultSeed(),
+      repetitionPenalty: req.repetition_penalty ?? ctx.genDefaults.repetitionPenalty,
+      // mlx_lm.server sampling extensions (defaults mirror server.py: min_p /
+      // xtc off, context windows 20). parseLogitBias throws on non-numeric
+      // input — callers surface it as a 400 (mlx-lm's coercion error).
+      minP: req.min_p ?? 0,
+      xtcProbability: req.xtc_probability ?? 0,
+      xtcThreshold: req.xtc_threshold ?? 0,
+      ...(req.xtc_probability ? { xtcSpecialTokens } : {}),
+      logitBias: parseLogitBias(req.logit_bias),
+      repetitionContextSize: req.repetition_context_size ?? 20,
+      presencePenalty: req.presence_penalty ?? 0,
+      presenceContextSize: req.presence_context_size ?? 20,
+      frequencyPenalty: req.frequency_penalty ?? 0,
+      frequencyContextSize: req.frequency_context_size ?? 20,
+      hlg: resolveHlg(req.hlg, serverOptions.hlg),
+      // generate() yields token ids; stop sequences match on decoded text
+      // (they can span token boundaries), so the StopMatcher sits at the
+      // decode layer below and halts the loop via onToken → false.
+      stopSequences: (typeof req.stop === "string" ? [req.stop] : req.stop ?? [])
+        .filter((s) => typeof s === "string" && s.length > 0),
+      ...kvScheme,
+      // Paged KV is a server-wide mode like the kv scheme above (v1:
+      // serial bf16 gemma4 — the createServer refusals hold the invariants).
+      ...(serverOptions.pagedKv ? { pagedKv: serverOptions.pagedKv } : {}),
+    };
+  };
+
+  /** Compile a grammar controller from the request's structured-output fields
+   *  (response_format / guided_* / structured_outputs). Returns null when no
+   *  constraint is requested OR when compile failed (degrade path). On degrade,
+   *  `degradeHint` carries a human description for the system-prompt injection
+   *  + Warning header (oMLX parity — never 500). Honors MLX_BUN_GRAMMAR=0. */
+  const compileGrammarForRequest = async (
+    req: ChatRequestParams,
+  ): Promise<{ controller: import("../grammar").GrammarController | null; degradeHint: string | null }> => {
+    if (!grammarEnabled()) {
+      return {
+        controller: null,
+        degradeHint: "grammar compilation disabled by MLX_BUN_GRAMMAR=0",
+      };
+    }
+    const r = await compileGrammarRequest(
+      req as GrammarRequest,
+      ctx.tokenizer,
+      ctx.model.config.text.vocabSize,
+    );
+    if (!r) return { controller: null, degradeHint: null };
+    return { controller: r.controller, degradeHint: r.degradeHint };
+  };
+
+  // Map OpenAI reasoning_effort levels onto the Qwen3.8 template's supported
+  // set (xhigh|medium|low — the template raises on anything else). "none"
+  // means thinking off (handled by resolveEnableThinking), so no depth is
+  // passed. Only consumed for templates with readsReasoningEffort.
+  const qwenReasoningEffort = (
+    effort: ChatRequestParams["reasoning_effort"],
+  ): "xhigh" | "medium" | "low" | undefined => {
+    switch (effort) {
+      case "minimal":
+      case "low":
+        return "low";
+      case "medium":
+        return "medium";
+      case "high":
+      case "xhigh":
+        return "xhigh";
+      default:
+        return undefined; // "none" / unset → template default (xhigh)
+    }
+  };
+
+  const templateOptionsFor = (req: ChatRequestParams, tools: ToolDefinition[] | null) => {
+    // enableThinking resolution (and its precedence) lives in
+    // resolveEnableThinking so template rendering and sampling stay in sync.
+    return {
+      tools,
+      enableThinking: resolveEnableThinking(req),
+      reasoningEffort: qwenReasoningEffort(req.reasoning_effort),
+      preserveThinking: req.chat_template_kwargs?.preserve_thinking,
+    };
+  };
+
+  const promptIdsFor = (
+    req: ChatRequestParams,
+    tools: ToolDefinition[] | null,
+  ): { ids: number[]; startInThinking: boolean } => {
+    const opts = templateOptionsFor(req, tools);
+    const rendered = ctx.template.render(normalizeMessages(req.messages), opts);
+    const ids = ctx.tokenizer.encode(rendered);
+    // template includes <bos>; tokenizer post-processor also prepends one
+    const trimmed = ids[0] === ids[1] && ids[0] === ctx.tokenizer.bosTokenId ? ids.slice(1) : ids;
+    return { ids: trimmed, startInThinking: promptEndsInOpenThink(rendered) };
+  };
+
+  /** STABLE cache boundary: the prompt's tail is a generation primer (e.g.
+   *  12B's `<|channel>thought` tokens) that the NEXT turn's re-render does
+   *  not contain — a cache entry ending there can never prefix-match again
+   *  once the rings wrap. Probe by rendering this conversation as if a
+   *  reply existed; the common prefix is what every future turn preserves.
+   *  The probe is EXPENSIVE (a second full render + encode — ~150 ms at
+   *  16k with our tokenizer; 30-50% of the 12B ctx-repeat TTFT,
+   *  2026-07-06b), but the PRIMER LENGTH it measures is constant per
+   *  template mode: the divergence sits after a special-token turn
+   *  delimiter, and special tokens break BPE merges, so message content
+   *  can't shift it. So the full probe runs ONCE per mode and later
+   *  requests pay a subtraction. (A wrong boundary is a QUALITY knob, not
+   *  a correctness one — any prefix ≤ len-1 is a valid snapshot point.) */
+  const primerLenByMode = new Map<string, number>();
+  const stableLenFor = (
+    req: ChatRequestParams,
+    tools: ToolDefinition[] | null,
+    trimmed: number[],
+  ): number => {
+    const opts = templateOptionsFor(req, tools);
+    const mode = `${opts.enableThinking}|${!!tools?.length}|${opts.reasoningEffort ?? ""}|${opts.preserveThinking ?? ""}`;
+    const primer = primerLenByMode.get(mode);
+    if (primer !== undefined) return Math.max(0, trimmed.length - primer);
+    let stableLen = trimmed.length;
+    try {
+      const probe = ctx.tokenizer.encode(
+        ctx.template.render(
+          [...normalizeMessages(req.messages), { role: "assistant", content: "x" }],
+          opts,
+        ),
+      );
+      const probeTrimmed =
+        probe[0] === probe[1] && probe[0] === ctx.tokenizer.bosTokenId ? probe.slice(1) : probe;
+      let i = 0;
+      const n = Math.min(trimmed.length, probeTrimmed.length);
+      while (i < n && trimmed[i] === probeTrimmed[i]) i++;
+      stableLen = i;
+      // Memoize the mode's primer length (sanity-capped: a "primer" longer
+      // than 64 tokens means the probe diverged for content reasons —
+      // don't generalize that).
+      const p = trimmed.length - stableLen;
+      if (p >= 0 && p <= 64) primerLenByMode.set(mode, p);
+    } catch { /* probe is best-effort; full boundary is the fallback */ }
+    return stableLen;
+  };
+
+  const toolStreamMode = (tools: ToolDefinition[] | null): ToolStreamMode =>
+    // Gemma-4 keeps the token-level sentinel path; every other family
+    // (MiniCPM5, Qwen3/3.5, Tier-0 generics) emits tool calls as DECODED TEXT
+    // — see selectToolStreamMode for why sentinel ids must stay family-gated.
+    selectToolStreamMode(ctx.model.config.modelType, !!tools?.length);
+
+  const toolRouter = (
+    tools: ToolDefinition[] | null,
+    onParseFailure?: () => void,
+  ): ToolAwareStream =>
+    new ToolAwareStream(ctx.tokenizer, toolStreamMode(tools), tools, onParseFailure);
+
+  /** Strict fill rows are a function of (template mode, tool signature) only —
+   *  they are token spans of the assistant turn, which opens with a special
+   *  token, so message content cannot shift them (the same argument that lets
+   *  stableLenFor memoize its primer length). Compiling costs several template
+   *  renders + encodes, so it happens ONCE per mode. */
+  const fillRowsByMode = new Map<string, StrictRowPlan>();
+  const toolsSignature = (tools: ToolDefinition[]): string =>
+    JSON.stringify(tools.map((t) => [
+      t.function?.name,
+      Object.keys((t.function?.parameters?.properties ?? {}) as Record<string, unknown>),
+      (t.function?.parameters?.required as unknown) ?? null,
+      (t.function?.parameters?.additionalProperties as unknown) ?? null,
+    ]));
+
+  /** Token fast-forwarding (docs/design/speculative-decoding.md "Token
+   *  fast-forwarding"): compile this request's determined-span table, or null
+   *  when the feature is off or the request's shape refuses it.
+   *
+   *  Refusals owned here (body-level); the ones only ChatStage can see —
+   *  a compiled grammar, media prompts, a mounted draft model, a quantized KV
+   *  scheme — are applied there. Continuous (batch) placement needs no
+   *  refusal: generate() is the only fill site, so a batched request simply
+   *  does not fill.
+   *
+   *  MLX_BUN_FILL=echo additionally arms the echo index (K3c, Lab tier): the
+   *  strict rows keep policy "assert", and copied spans ride the same
+   *  mechanism under policy "verify". */
+  const fillPlanFor = (
+    req: ChatRequestParams,
+    tools: ToolDefinition[] | null,
+    promptIds: number[],
+  ): FillSession | null => {
+    const mode = resolveFillMode();
+    if (mode === "off") return null;
+    if (!tools?.length) return null;
+    // A user-fixed seed means reproducibility: the sampler's step index would
+    // skip the injected positions, so an identical request could not replay.
+    if (req.seed !== undefined) return null;
+    // Injected tokens are never sampled — they have no logprob row.
+    if (req.logprobs === true) return null;
+    if (typeof req.top_logprobs === "number" && req.top_logprobs > 0) return null;
+    // Structured output owns forced-token production (grammar jump-forward).
+    if (req.response_format != null || req.guided_grammar || req.guided_regex ||
+        req.guided_choice?.length || req.structured_outputs != null) return null;
+
+    const opts = templateOptionsFor(req, tools);
+    const key = `${opts.enableThinking}|${opts.reasoningEffort ?? ""}|` +
+      `${opts.preserveThinking ?? ""}|${toolsSignature(tools)}`;
+    let plan = fillRowsByMode.get(key);
+    if (plan === undefined) {
+      plan = compileStrictFillRows({
+        template: ctx.template,
+        tokenizer: ctx.tokenizer,
+        messages: normalizeMessages(req.messages),
+        tools,
+        renderOptions: opts,
+      });
+      fillRowsByMode.set(key, plan);
+    }
+    // The echo tier can carry a request whose template yields no strict rows;
+    // strict-only mode cannot.
+    if (!plan.rows.length && mode !== "echo") return null;
+    return new FillSession(
+      {
+        rows: plan.rows,
+        echo: mode === "echo" ? fillEchoConfig() : null,
+        delimiters: new Set(plan.delimiters),
+        eos: ctx.model.config.eosTokenIds,
+      },
+      promptIds,
+      {
+        maxSpan: fillMaxSpan(), decode: (ids) => ctx.tokenizer.decode([...ids]),
+        strictContext: plan.createContext?.(promptIds),
+      },
+    );
+  };
+
+  return {
+    resolveEnableThinking,
+    toOptions,
+    compileGrammarForRequest,
+    templateOptionsFor,
+    promptIdsFor,
+    stableLenFor,
+    toolRouter,
+    fillPlanFor,
+  };
+}
+
+export type RequestPrep = ReturnType<typeof createRequestPrep>;

package/src/serve/response-lease.ts

@@ -0,0 +1,24 @@
+import type { DisposableResource } from "../contracts/resources";
+
+/** Keep host activity owned through response EOF/error/cancel, without buffering
+ * the response. The parent additionally drains worker cleanup before GPU jobs. */
+export function retainResponseLease(response: Response, lease: DisposableResource): Response {
+  if (!response.body) { lease.dispose(); return response; }
+  const reader = response.body.getReader();
+  let closed = false;
+  const release = () => {
+    if (closed) return;
+    closed = true;
+    reader.releaseLock(); lease.dispose();
+  };
+  return new Response(new ReadableStream<Uint8Array>({
+    async pull(controller) {
+      try {
+        const item = await reader.read();
+        if (item.done) { release(); controller.close(); }
+        else controller.enqueue(item.value);
+      } catch (error) { release(); controller.error(error); }
+    },
+    async cancel(reason) { try { await reader.cancel(reason); } finally { release(); } },
+  }), { status: response.status, statusText: response.statusText, headers: response.headers });
+}

package/src/serve/responses-client.ts

@@ -0,0 +1,69 @@
+import { ResponseStore, resolveResponsesConversation, type ResponseHistory, type ResponsesRequest } from "../responses";
+
+/** Application-owned conversation history over a data-only HTTP host. Workers
+ * translate/run the request; the parent stores only completed response records. */
+export function createResponsesClient(store: ResponseHistory = new ResponseStore()) {
+  return {
+    get stats() { return { entries: store.size, bytes: store.totalBytes, max_bytes: store.maxBytes, ttl_ms: store.ttlMs }; },
+    async forward(request: Request, forward: (request: Request) => Promise<Response>): Promise<Response> {
+      const error = (status: number, message: string) => Response.json({ error: {
+        message, type: "invalid_request_error", param: null, code: null,
+      } }, { status });
+      let body: ResponsesRequest;
+      try { body = await request.json() as ResponsesRequest; }
+      catch { return error(400, "invalid JSON body"); }
+      if (!body || typeof body !== "object" || Array.isArray(body)) return error(400, "invalid JSON body");
+      let conversation: ReturnType<typeof resolveResponsesConversation>;
+      try { conversation = resolveResponsesConversation(body, store); }
+      catch (e) { return error(404, (e as Error).message); }
+      const headers = new Headers(request.headers);
+      headers.delete("content-length");
+      headers.set("x-mlx-bun-response-owner", "parent");
+      const response = await forward(new Request(request.url, {
+        method: "POST", headers, signal: request.signal,
+        body: JSON.stringify({ ...conversation.body, previous_response_id: undefined }),
+      }));
+      if (!response.ok) return response;
+      const remember = (value: Record<string, unknown>) => {
+        if (typeof value.id === "string" && Array.isArray(value.output))
+          store.put(value.id, { input: conversation.input, output: value.output, instructions: conversation.instructions });
+      };
+      const restorePrevious = (value: Record<string, unknown>) => ({ ...value, previous_response_id: conversation.previousId });
+      if (!body.stream) {
+        const value = await response.json() as Record<string, unknown>;
+        remember(value);
+        const headers = new Headers(response.headers); headers.delete("content-length");
+        return Response.json(restorePrevious(value), { status: response.status, headers });
+      }
+      if (!response.body) return response;
+      const decoder = new TextDecoder(), encoder = new TextEncoder();
+      let pending = "";
+      const rewrite = (frame: string): string => frame.replace(/^data: (.+)$/m, (line, data: string) => {
+        if (data === "[DONE]") return line;
+        const event = JSON.parse(data) as Record<string, unknown>;
+        if (event.response && typeof event.response === "object") {
+          const value = event.response as Record<string, unknown>;
+          if (event.type === "response.completed") remember(value);
+          event.response = restorePrevious(value);
+        }
+        return `data: ${JSON.stringify(event)}`;
+      });
+      const outputHeaders = new Headers(response.headers); outputHeaders.delete("content-length");
+      return new Response(response.body.pipeThrough(new TransformStream<Uint8Array, Uint8Array>({
+        transform(chunk, controller) {
+          pending += decoder.decode(chunk, { stream: true });
+          let split: number;
+          while ((split = pending.indexOf("\n\n")) !== -1) {
+            const frame = pending.slice(0, split);
+            pending = pending.slice(split + 2);
+            controller.enqueue(encoder.encode(`${rewrite(frame)}\n\n`));
+          }
+        },
+        flush(controller) {
+          pending += decoder.decode();
+          if (pending) controller.enqueue(encoder.encode(rewrite(pending)));
+        },
+      })), { status: response.status, headers: outputHeaders });
+    },
+  };
+}