mlx-bun 0.0.13 → 0.3.0

package/src/serve/request-prep.ts

@@ -0,0 +1,346 @@
+// Per-request preparation shared by every completion surface (chat,
+// raw text, /signal, /generate): request → GenerateOptions with the
+// server/model defaults folded in, chat-template rendering + prompt ids,
+// the stable prompt-cache boundary probe, grammar compilation, and the
+// tool-call stream router. Resolved ONCE per server (createRequestPrep) and
+// captured by the route handlers. Extracted from src/server.ts (repo-taming
+// Phase 4).
+import type { ToolDefinition } from "../chat-template";
+import {
+  fillEchoConfig,
+  fillMaxSpan,
+  FillSession,
+  resolveFillMode,
+} from "../fill/fill-session";
+import { compileStrictFillRows, type StrictRowPlan } from "../fill/schema-rows";
+import type { GenerateOptions } from "../generate";
+import { compileGrammarRequest, grammarEnabled, type GrammarRequest } from "../grammar";
+import type { KvSchemeOptions } from "../kv-scheme";
+import { isMiniCPM5Config } from "../model/support";
+import type { HlgConfig } from "../sampler";
+import {
+  nextDefaultSeed,
+  normalizeMessages,
+  parseLogitBias,
+  promptEndsInOpenThink,
+  resolveHlg,
+  type ChatRequestParams,
+} from "./chat-request";
+import type { ServingContext } from "./model-host";
+import { selectToolStreamMode, ToolAwareStream, type ToolStreamMode } from "./token-streams";
+
+/** The ServerOptions fields request preparation reads (server-wide
+ *  defaults a request field overrides). */
+export interface RequestPrepOptions {
+  defaultThinking?: boolean;
+  defaultTemperature?: number;
+  defaultTopP?: number;
+  defaultTopK?: number;
+  hlg?: HlgConfig;
+  pagedKv?: { blockSize?: number };
+}
+
+export function createRequestPrep(input: {
+  ctx: ServingContext;
+  serverOptions: RequestPrepOptions;
+  /** The server-wide KV scheme (resolveKvScheme(...).generationOptions),
+   *  spread into every request's GenerateOptions. */
+  kvScheme: KvSchemeOptions;
+  /** `--max-tokens` / the GLM memory plan's generation cap; undefined = the
+   *  surface's own default (chat 65 536, raw completion 512). */
+  defaultGeneratedTokens: number | undefined;
+}) {
+  const { ctx, serverOptions, kvScheme, defaultGeneratedTokens } = input;
+
+  // XTC never removes EOS or the newline token: mlx_lm.server passes
+  // [tokenizer.eos_token_id, tokenizer.encode("\n")] as xtc_special_tokens.
+  // We pass the flat equivalent — ALL configured EOS ids plus the bare newline
+  // id(s), encoded WITHOUT special tokens (mlx-lm's encode("\n") can drag a
+  // BOS along on some tokenizers; protecting BOS from XTC is a no-op, so the
+  // flat/no-BOS form is behaviorally identical and cleaner).
+  const xtcSpecialTokens = [
+    ...ctx.model.config.eosTokenIds,
+    ...ctx.tokenizer.encode("\n", false),
+  ];
+
+  // Effective enable_thinking for a request, with the same precedence the chat
+  // template uses (extracted so sampling and template rendering can't disagree):
+  // explicit chat_template_kwargs.enable_thinking → reasoning_effort ("none" =
+  // off) → server --thinking default → model default (MiniCPM5 → off). undefined
+  // means "not a switchable-thinking model / leave the template default".
+  const resolveEnableThinking = (req: ChatRequestParams): boolean | undefined => {
+    const explicit = req.chat_template_kwargs?.enable_thinking;
+    if (typeof explicit === "boolean") return explicit;
+    const effort = req.reasoning_effort;
+    if (effort !== undefined) return effort !== "none";
+    if (serverOptions.defaultThinking !== undefined) return serverOptions.defaultThinking;
+    return isMiniCPM5Config(ctx.model.config) ? false : undefined;
+  };
+
+  const toOptions = (req: ChatRequestParams): GenerateOptions & { stopSequences: string[] } => {
+    // Sampling follows the thinking state — which the web UI's thinking button
+    // drives via enable_thinking. Model authors publish a SINGLE
+    // generation_config temperature (the think-mode value) but recommend a
+    // cooler one for direct, no-think replies (MiniCPM5 card: 0.9 think / 0.7
+    // no-think, top_p 0.95 for both). So with no explicit temperature set, a
+    // no-think turn runs at most NO_THINK_TEMPERATURE while a think turn keeps
+    // the model's hotter configured default. An explicit request/CLI
+    // temperature always wins. top_p is unchanged by mode.
+    const NO_THINK_TEMPERATURE = 0.7;
+    const genTemp = ctx.genDefaults.temperature ?? 0.7;
+    const defaultTemp = resolveEnableThinking(req) === false
+      ? Math.min(genTemp, NO_THINK_TEMPERATURE)
+      : genTemp;
+    return {
+      // No invented cap: unset means "generate until EOS or the admitted
+      // context is exhausted" — admission (planRequest) is the real limit
+      // and clamps to maxSafeContext − promptLen. Only an explicit request
+      // max_tokens or the operator's --max-tokens narrows it further.
+      maxTokens: req.max_completion_tokens ?? req.max_tokens ??
+        defaultGeneratedTokens,
+      temperature: req.temperature ?? serverOptions.defaultTemperature ?? defaultTemp,
+      topP: req.top_p ?? serverOptions.defaultTopP ?? ctx.genDefaults.topP ?? 0,
+      topK: req.top_k ?? serverOptions.defaultTopK ?? ctx.genDefaults.topK ?? 0,
+      seed: req.seed ?? nextDefaultSeed(),
+      repetitionPenalty: req.repetition_penalty ?? ctx.genDefaults.repetitionPenalty,
+      // mlx_lm.server sampling extensions (defaults mirror server.py: min_p /
+      // xtc off, context windows 20). parseLogitBias throws on non-numeric
+      // input — callers surface it as a 400 (mlx-lm's coercion error).
+      minP: req.min_p ?? 0,
+      xtcProbability: req.xtc_probability ?? 0,
+      xtcThreshold: req.xtc_threshold ?? 0,
+      ...(req.xtc_probability ? { xtcSpecialTokens } : {}),
+      logitBias: parseLogitBias(req.logit_bias),
+      repetitionContextSize: req.repetition_context_size ?? 20,
+      presencePenalty: req.presence_penalty ?? 0,
+      presenceContextSize: req.presence_context_size ?? 20,
+      frequencyPenalty: req.frequency_penalty ?? 0,
+      frequencyContextSize: req.frequency_context_size ?? 20,
+      hlg: resolveHlg(req.hlg, serverOptions.hlg),
+      // generate() yields token ids; stop sequences match on decoded text
+      // (they can span token boundaries), so the StopMatcher sits at the
+      // decode layer below and halts the loop via onToken → false.
+      stopSequences: (typeof req.stop === "string" ? [req.stop] : req.stop ?? [])
+        .filter((s) => typeof s === "string" && s.length > 0),
+      ...kvScheme,
+      // Paged KV is a server-wide mode like the kv scheme above (v1:
+      // serial bf16 gemma4 — the createServer refusals hold the invariants).
+      ...(serverOptions.pagedKv ? { pagedKv: serverOptions.pagedKv } : {}),
+    };
+  };
+
+  /** Compile a grammar controller from the request's structured-output fields
+   *  (response_format / guided_* / structured_outputs). Returns null when no
+   *  constraint is requested OR when compile failed (degrade path). On degrade,
+   *  `degradeHint` carries a human description for the system-prompt injection
+   *  + Warning header (oMLX parity — never 500). Honors MLX_BUN_GRAMMAR=0. */
+  const compileGrammarForRequest = async (
+    req: ChatRequestParams,
+  ): Promise<{ controller: import("../grammar").GrammarController | null; degradeHint: string | null }> => {
+    if (!grammarEnabled()) {
+      return {
+        controller: null,
+        degradeHint: "grammar compilation disabled by MLX_BUN_GRAMMAR=0",
+      };
+    }
+    const r = await compileGrammarRequest(
+      req as GrammarRequest,
+      ctx.tokenizer,
+      ctx.model.config.text.vocabSize,
+    );
+    if (!r) return { controller: null, degradeHint: null };
+    return { controller: r.controller, degradeHint: r.degradeHint };
+  };
+
+  // Map OpenAI reasoning_effort levels onto the Qwen3.8 template's supported
+  // set (xhigh|medium|low — the template raises on anything else). "none"
+  // means thinking off (handled by resolveEnableThinking), so no depth is
+  // passed. Only consumed for templates with readsReasoningEffort.
+  const qwenReasoningEffort = (
+    effort: ChatRequestParams["reasoning_effort"],
+  ): "xhigh" | "medium" | "low" | undefined => {
+    switch (effort) {
+      case "minimal":
+      case "low":
+        return "low";
+      case "medium":
+        return "medium";
+      case "high":
+      case "xhigh":
+        return "xhigh";
+      default:
+        return undefined; // "none" / unset → template default (xhigh)
+    }
+  };
+
+  const templateOptionsFor = (req: ChatRequestParams, tools: ToolDefinition[] | null) => {
+    // enableThinking resolution (and its precedence) lives in
+    // resolveEnableThinking so template rendering and sampling stay in sync.
+    return {
+      tools,
+      enableThinking: resolveEnableThinking(req),
+      reasoningEffort: qwenReasoningEffort(req.reasoning_effort),
+      preserveThinking: req.chat_template_kwargs?.preserve_thinking,
+    };
+  };
+
+  const promptIdsFor = (
+    req: ChatRequestParams,
+    tools: ToolDefinition[] | null,
+  ): { ids: number[]; startInThinking: boolean } => {
+    const opts = templateOptionsFor(req, tools);
+    const rendered = ctx.template.render(normalizeMessages(req.messages), opts);
+    const ids = ctx.tokenizer.encode(rendered);
+    // template includes <bos>; tokenizer post-processor also prepends one
+    const trimmed = ids[0] === ids[1] && ids[0] === ctx.tokenizer.bosTokenId ? ids.slice(1) : ids;
+    return { ids: trimmed, startInThinking: promptEndsInOpenThink(rendered) };
+  };
+
+  /** STABLE cache boundary: the prompt's tail is a generation primer (e.g.
+   *  12B's `<|channel>thought` tokens) that the NEXT turn's re-render does
+   *  not contain — a cache entry ending there can never prefix-match again
+   *  once the rings wrap. Probe by rendering this conversation as if a
+   *  reply existed; the common prefix is what every future turn preserves.
+   *  The probe is EXPENSIVE (a second full render + encode — ~150 ms at
+   *  16k with our tokenizer; 30-50% of the 12B ctx-repeat TTFT,
+   *  2026-07-06b), but the PRIMER LENGTH it measures is constant per
+   *  template mode: the divergence sits after a special-token turn
+   *  delimiter, and special tokens break BPE merges, so message content
+   *  can't shift it. So the full probe runs ONCE per mode and later
+   *  requests pay a subtraction. (A wrong boundary is a QUALITY knob, not
+   *  a correctness one — any prefix ≤ len-1 is a valid snapshot point.) */
+  const primerLenByMode = new Map<string, number>();
+  const stableLenFor = (
+    req: ChatRequestParams,
+    tools: ToolDefinition[] | null,
+    trimmed: number[],
+  ): number => {
+    const opts = templateOptionsFor(req, tools);
+    const mode = `${opts.enableThinking}|${!!tools?.length}|${opts.reasoningEffort ?? ""}|${opts.preserveThinking ?? ""}`;
+    const primer = primerLenByMode.get(mode);
+    if (primer !== undefined) return Math.max(0, trimmed.length - primer);
+    let stableLen = trimmed.length;
+    try {
+      const probe = ctx.tokenizer.encode(
+        ctx.template.render(
+          [...normalizeMessages(req.messages), { role: "assistant", content: "x" }],
+          opts,
+        ),
+      );
+      const probeTrimmed =
+        probe[0] === probe[1] && probe[0] === ctx.tokenizer.bosTokenId ? probe.slice(1) : probe;
+      let i = 0;
+      const n = Math.min(trimmed.length, probeTrimmed.length);
+      while (i < n && trimmed[i] === probeTrimmed[i]) i++;
+      stableLen = i;
+      // Memoize the mode's primer length (sanity-capped: a "primer" longer
+      // than 64 tokens means the probe diverged for content reasons —
+      // don't generalize that).
+      const p = trimmed.length - stableLen;
+      if (p >= 0 && p <= 64) primerLenByMode.set(mode, p);
+    } catch { /* probe is best-effort; full boundary is the fallback */ }
+    return stableLen;
+  };
+
+  const toolStreamMode = (tools: ToolDefinition[] | null): ToolStreamMode =>
+    // Gemma-4 keeps the token-level sentinel path; every other family
+    // (MiniCPM5, Qwen3/3.5, Tier-0 generics) emits tool calls as DECODED TEXT
+    // — see selectToolStreamMode for why sentinel ids must stay family-gated.
+    selectToolStreamMode(ctx.model.config.modelType, !!tools?.length);
+
+  const toolRouter = (
+    tools: ToolDefinition[] | null,
+    onParseFailure?: () => void,
+  ): ToolAwareStream =>
+    new ToolAwareStream(ctx.tokenizer, toolStreamMode(tools), tools, onParseFailure);
+
+  /** Strict fill rows are a function of (template mode, tool signature) only —
+   *  they are token spans of the assistant turn, which opens with a special
+   *  token, so message content cannot shift them (the same argument that lets
+   *  stableLenFor memoize its primer length). Compiling costs several template
+   *  renders + encodes, so it happens ONCE per mode. */
+  const fillRowsByMode = new Map<string, StrictRowPlan>();
+  const toolsSignature = (tools: ToolDefinition[]): string =>
+    JSON.stringify(tools.map((t) => [
+      t.function?.name,
+      Object.keys((t.function?.parameters?.properties ?? {}) as Record<string, unknown>),
+      (t.function?.parameters?.required as unknown) ?? null,
+      (t.function?.parameters?.additionalProperties as unknown) ?? null,
+    ]));
+
+  /** Token fast-forwarding (docs/design/speculative-decoding.md "Token
+   *  fast-forwarding"): compile this request's determined-span table, or null
+   *  when the feature is off or the request's shape refuses it.
+   *
+   *  Refusals owned here (body-level); the ones only ChatStage can see —
+   *  a compiled grammar, media prompts, a mounted draft model, a quantized KV
+   *  scheme — are applied there. Continuous (batch) placement needs no
+   *  refusal: generate() is the only fill site, so a batched request simply
+   *  does not fill.
+   *
+   *  MLX_BUN_FILL=echo additionally arms the echo index (K3c, Lab tier): the
+   *  strict rows keep policy "assert", and copied spans ride the same
+   *  mechanism under policy "verify". */
+  const fillPlanFor = (
+    req: ChatRequestParams,
+    tools: ToolDefinition[] | null,
+    promptIds: number[],
+  ): FillSession | null => {
+    const mode = resolveFillMode();
+    if (mode === "off") return null;
+    if (!tools?.length) return null;
+    // A user-fixed seed means reproducibility: the sampler's step index would
+    // skip the injected positions, so an identical request could not replay.
+    if (req.seed !== undefined) return null;
+    // Injected tokens are never sampled — they have no logprob row.
+    if (req.logprobs === true) return null;
+    if (typeof req.top_logprobs === "number" && req.top_logprobs > 0) return null;
+    // Structured output owns forced-token production (grammar jump-forward).
+    if (req.response_format != null || req.guided_grammar || req.guided_regex ||
+        req.guided_choice?.length || req.structured_outputs != null) return null;
+
+    const opts = templateOptionsFor(req, tools);
+    const key = `${opts.enableThinking}|${opts.reasoningEffort ?? ""}|` +
+      `${opts.preserveThinking ?? ""}|${toolsSignature(tools)}`;
+    let plan = fillRowsByMode.get(key);
+    if (plan === undefined) {
+      plan = compileStrictFillRows({
+        template: ctx.template,
+        tokenizer: ctx.tokenizer,
+        messages: normalizeMessages(req.messages),
+        tools,
+        renderOptions: opts,
+      });
+      fillRowsByMode.set(key, plan);
+    }
+    // The echo tier can carry a request whose template yields no strict rows;
+    // strict-only mode cannot.
+    if (!plan.rows.length && mode !== "echo") return null;
+    return new FillSession(
+      {
+        rows: plan.rows,
+        echo: mode === "echo" ? fillEchoConfig() : null,
+        delimiters: new Set(plan.delimiters),
+        eos: ctx.model.config.eosTokenIds,
+      },
+      promptIds,
+      {
+        maxSpan: fillMaxSpan(), decode: (ids) => ctx.tokenizer.decode([...ids]),
+        strictContext: plan.createContext?.(promptIds),
+      },
+    );
+  };
+
+  return {
+    resolveEnableThinking,
+    toOptions,
+    compileGrammarForRequest,
+    templateOptionsFor,
+    promptIdsFor,
+    stableLenFor,
+    toolRouter,
+    fillPlanFor,
+  };
+}
+
+export type RequestPrep = ReturnType<typeof createRequestPrep>;

package/src/serve/response-lease.ts

@@ -0,0 +1,24 @@
+import type { DisposableResource } from "../contracts/resources";
+
+/** Keep host activity owned through response EOF/error/cancel, without buffering
+ * the response. The parent additionally drains worker cleanup before GPU jobs. */
+export function retainResponseLease(response: Response, lease: DisposableResource): Response {
+  if (!response.body) { lease.dispose(); return response; }
+  const reader = response.body.getReader();
+  let closed = false;
+  const release = () => {
+    if (closed) return;
+    closed = true;
+    reader.releaseLock(); lease.dispose();
+  };
+  return new Response(new ReadableStream<Uint8Array>({
+    async pull(controller) {
+      try {
+        const item = await reader.read();
+        if (item.done) { release(); controller.close(); }
+        else controller.enqueue(item.value);
+      } catch (error) { release(); controller.error(error); }
+    },
+    async cancel(reason) { try { await reader.cancel(reason); } finally { release(); } },
+  }), { status: response.status, statusText: response.statusText, headers: response.headers });
+}

package/src/serve/responses-client.ts

@@ -0,0 +1,69 @@
+import { ResponseStore, resolveResponsesConversation, type ResponseHistory, type ResponsesRequest } from "../responses";
+
+/** Application-owned conversation history over a data-only HTTP host. Workers
+ * translate/run the request; the parent stores only completed response records. */
+export function createResponsesClient(store: ResponseHistory = new ResponseStore()) {
+  return {
+    get stats() { return { entries: store.size, bytes: store.totalBytes, max_bytes: store.maxBytes, ttl_ms: store.ttlMs }; },
+    async forward(request: Request, forward: (request: Request) => Promise<Response>): Promise<Response> {
+      const error = (status: number, message: string) => Response.json({ error: {
+        message, type: "invalid_request_error", param: null, code: null,
+      } }, { status });
+      let body: ResponsesRequest;
+      try { body = await request.json() as ResponsesRequest; }
+      catch { return error(400, "invalid JSON body"); }
+      if (!body || typeof body !== "object" || Array.isArray(body)) return error(400, "invalid JSON body");
+      let conversation: ReturnType<typeof resolveResponsesConversation>;
+      try { conversation = resolveResponsesConversation(body, store); }
+      catch (e) { return error(404, (e as Error).message); }
+      const headers = new Headers(request.headers);
+      headers.delete("content-length");
+      headers.set("x-mlx-bun-response-owner", "parent");
+      const response = await forward(new Request(request.url, {
+        method: "POST", headers, signal: request.signal,
+        body: JSON.stringify({ ...conversation.body, previous_response_id: undefined }),
+      }));
+      if (!response.ok) return response;
+      const remember = (value: Record<string, unknown>) => {
+        if (typeof value.id === "string" && Array.isArray(value.output))
+          store.put(value.id, { input: conversation.input, output: value.output, instructions: conversation.instructions });
+      };
+      const restorePrevious = (value: Record<string, unknown>) => ({ ...value, previous_response_id: conversation.previousId });
+      if (!body.stream) {
+        const value = await response.json() as Record<string, unknown>;
+        remember(value);
+        const headers = new Headers(response.headers); headers.delete("content-length");
+        return Response.json(restorePrevious(value), { status: response.status, headers });
+      }
+      if (!response.body) return response;
+      const decoder = new TextDecoder(), encoder = new TextEncoder();
+      let pending = "";
+      const rewrite = (frame: string): string => frame.replace(/^data: (.+)$/m, (line, data: string) => {
+        if (data === "[DONE]") return line;
+        const event = JSON.parse(data) as Record<string, unknown>;
+        if (event.response && typeof event.response === "object") {
+          const value = event.response as Record<string, unknown>;
+          if (event.type === "response.completed") remember(value);
+          event.response = restorePrevious(value);
+        }
+        return `data: ${JSON.stringify(event)}`;
+      });
+      const outputHeaders = new Headers(response.headers); outputHeaders.delete("content-length");
+      return new Response(response.body.pipeThrough(new TransformStream<Uint8Array, Uint8Array>({
+        transform(chunk, controller) {
+          pending += decoder.decode(chunk, { stream: true });
+          let split: number;
+          while ((split = pending.indexOf("\n\n")) !== -1) {
+            const frame = pending.slice(0, split);
+            pending = pending.slice(split + 2);
+            controller.enqueue(encoder.encode(`${rewrite(frame)}\n\n`));
+          }
+        },
+        flush(controller) {
+          pending += decoder.decode();
+          if (pending) controller.enqueue(encoder.encode(rewrite(pending)));
+        },
+      })), { status: response.status, headers: outputHeaders });
+    },
+  };
+}

package/src/serve/session-completion-engine.ts

@@ -0,0 +1,110 @@
+import { cleanupFailure } from "../engine/resources";
+import type { ExecutionPlan, InferenceMethod, Timer } from "../contracts/generation";
+import { CancellationSource, throwIfCancelled } from "../engine/cancellation";
+import { createInferenceEngine } from "../engine/engine";
+import type { GenerateStats } from "../generate";
+import type { CompletionEngine } from "./completion-executor";
+
+const timer: Timer = {
+  after(ms, callback) {
+    const handle = setTimeout(callback, ms);
+    return () => clearTimeout(handle);
+  },
+};
+
+type RuntimeArguments = Parameters<CompletionEngine["run"]>;
+
+export interface CompletionRunRequest {
+  prompt: RuntimeArguments[0]; options: RuntimeArguments[1]; vision?: RuntimeArguments[3];
+  shape: RuntimeArguments[4]; placement: RuntimeArguments[5];
+  signal?: RuntimeArguments[6]; trace?: RuntimeArguments[7];
+}
+
+/** Bind one selected runtime operation to the method-neutral session. */
+export function createCompletionMethod(
+  runtime: CompletionEngine, request: CompletionRunRequest,
+  onStarted?: () => void, onFailure?: (error: unknown) => void,
+): InferenceMethod<GenerateStats> {
+  const { prompt, options, vision, shape, placement, signal, trace } = request;
+  const maxTokens = options.maxTokens ?? 512;
+  return {
+    id: placement.execution?.method ?? placement.mechanism,
+    async createRun(cancel) {
+      throwIfCancelled(cancel);
+      const controller = new AbortController();
+      const unsubscribe = cancel.subscribe(() => controller.abort(
+        signal?.reason ?? new DOMException("generation cancelled", "AbortError"),
+      ));
+      return {
+        async execute(output) {
+          onStarted?.();
+          try {
+            const stats = await runtime.run(prompt, options,
+              (token, logprobs) => output.commit([token], logprobs ? [logprobs] : undefined),
+              vision, shape, placement, controller.signal, trace);
+            return {
+              finishReason: stats.finishReason ?? (stats.generatedTokens >= maxTokens ? "length" : "stop"),
+              metrics: stats,
+            };
+          } catch (error) { onFailure?.(error); throw error; }
+        },
+        async close() { unsubscribe(); },
+      };
+    },
+  };
+}
+
+/** HTTP already bounds its output in CompletionSink. Deliver through the same
+ * session lifecycle without inserting another queue between sampling and the
+ * stop/tool parser. The runtime retains scheduling and native resource ownership. */
+export function createSessionCompletionEngine(
+  runtime: CompletionEngine,
+  releaseUnstarted: (options: Parameters<CompletionEngine["run"]>[1],
+    vision: Parameters<CompletionEngine["run"]>[3]) => void,
+): CompletionEngine & { close(): Promise<void> } {
+  const engine = createInferenceEngine<ExecutionPlan<GenerateStats>, GenerateStats>({
+    async plan(plan, cancellation) { throwIfCancelled(cancellation); return plan; },
+  }, { timer });
+  return {
+    place: (shape, options) => runtime.place(shape, options),
+    async run(prompt, options, onToken, vision, shape, placement, signal, trace) {
+      const cancellation = new CancellationSource();
+      const abort = () => cancellation.cancel("requested");
+      signal?.addEventListener("abort", abort, { once: true });
+      if (signal?.aborted) abort();
+      let started = false;
+      let failure: unknown;
+      let failed = false;
+      const maxTokens = options.maxTokens ?? 512;
+      const method = createCompletionMethod(runtime,
+        { prompt, options, vision, shape, placement, signal, trace },
+        () => { started = true; },
+        (error) => { failed = true; failure = error; });
+      try {
+        const session = await engine.open({ id: method.id, method, outputTokenLimit: maxTokens }, {
+          output: "callback", cancellation,
+          async onTokens(ids, logprobs) {
+            for (let i = 0; i < ids.length; i++)
+              if (await onToken(ids[i]!, logprobs?.[i]) === false) return false;
+          },
+        });
+        const outcome = await session.result;
+        if (outcome.status === "completed") return outcome.result.metrics;
+        if (failed) throw failure; // preserve native/HTTP error identity and stack
+        if (outcome.status === "cancelled")
+          throw signal?.reason ?? new DOMException("generation cancelled", "AbortError");
+        throw new Error(outcome.error.message);
+      } catch (error) {
+        if (!started) {
+          started = true; // release once, preserving the original execution error
+          cleanupFailure(error, () => releaseUnstarted(options, vision));
+        }
+        throw error;
+      } finally {
+        signal?.removeEventListener("abort", abort);
+        if (!started) releaseUnstarted(options, vision);
+      }
+    },
+    close: () => engine.close(),
+  };
+}

package/src/serve/session-search.ts

@@ -258,7 +258,7 @@ export async function searchSessions(sessionDir: string, query: string): Promise
    REST handlers — pure (URL) => Response, no `ctx` dependency, same
    convention as src/memory/rest.ts / src/hub-rest.ts (dispatched from
    server.ts by path+method; unit-tested directly in
-   tests/session-search.test.ts without booting a model context).
+   tests/serve/session-search.test.ts without booting a model context).
    ──────────────────────────────────────────────────────────────────── */
 
 function jsonOk<T extends object>(body: T, init?: ResponseInit): Response {

package/src/serve/static-routes.ts

@@ -0,0 +1,101 @@
+import { readFileSync } from "node:fs";
+
+export interface StaticRouteAssets {
+  appPage: string;
+  appJs: string;
+  hljsJs: string;
+  hljsCss: string;
+  manifest: string;
+  iconSvg: string;
+  serviceWorker: string;
+  curvePage: string;
+}
+
+const ASSET_HEADERS = { "cache-control": "public, max-age=3600" } as const;
+const LEGACY_PATHS = new Set(["/status", "/chat", "/quantize", "/finetune", "/dataset"]);
+
+/** Serve the web shell and its static companion pages. Returns null when the
+ * request belongs to another route family. */
+export function handleStaticRoute(
+  url: URL,
+  request: Request,
+  assets: StaticRouteAssets,
+): Response | null {
+  if (request.method !== "GET") return null;
+
+  switch (url.pathname) {
+    case "/":
+      return new Response(assets.appPage, {
+        headers: { "content-type": "text/html; charset=utf-8" },
+      });
+    case "/assets/hljs.js":
+      return new Response(assets.hljsJs, {
+        headers: { "content-type": "text/javascript; charset=utf-8", ...ASSET_HEADERS },
+      });
+    case "/assets/hljs.css":
+      return new Response(assets.hljsCss, {
+        headers: { "content-type": "text/css; charset=utf-8", ...ASSET_HEADERS },
+      });
+    case "/assets/app.js":
+      return new Response(assets.appJs, {
+        headers: { "content-type": "text/javascript; charset=utf-8", ...ASSET_HEADERS },
+      });
+    case "/manifest.webmanifest":
+      return new Response(assets.manifest, {
+        headers: { "content-type": "application/manifest+json; charset=utf-8", ...ASSET_HEADERS },
+      });
+    case "/assets/icon.svg":
+      return new Response(assets.iconSvg, {
+        headers: { "content-type": "image/svg+xml; charset=utf-8", ...ASSET_HEADERS },
+      });
+    case "/sw.js":
+      return new Response(assets.serviceWorker, {
+        headers: { "content-type": "text/javascript; charset=utf-8", "cache-control": "no-store" },
+      });
+    case "/curves": {
+      let html = assets.curvePage;
+      try {
+        html = readFileSync(new URL("../lab/curve/curve-designer.html", import.meta.url), "utf8");
+      } catch {
+        // Compiled binaries use the embedded copy.
+      }
+      return new Response(html, {
+        headers: { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" },
+      });
+    }
+    case "/dag":
+      try {
+        const html = readFileSync(
+          new URL("./assets/training-inference-map.html", import.meta.url),
+          "utf8",
+        );
+        return new Response(html, {
+          headers: { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" },
+        });
+      } catch {
+        return new Response(
+          "DAG map artifact not found",
+          { status: 404 },
+        );
+      }
+    case "/curve-terrain":
+      try {
+        const html = readFileSync(
+          new URL("./assets/curve-terrain.html", import.meta.url),
+          "utf8",
+        );
+        return new Response(html, {
+          headers: { "content-type": "text/html; charset=utf-8", "cache-control": "no-store" },
+        });
+      } catch {
+        return new Response(
+          "curve terrain artifact not found",
+          { status: 404 },
+        );
+      }
+  }
+
+  return LEGACY_PATHS.has(url.pathname)
+    ? Response.redirect(`/#${url.pathname}`, 302)
+    : null;
+}