mlx-bun 0.4.0 → 1.0.0

package/src/server/decision-routes.ts

@@ -0,0 +1,134 @@
+import type { ModelBinding } from "../engine/model-binding";
+import type { LoadedModelContext } from "../engine/model-host";
+import type { PreparationExecutor } from "../engine/preparation";
+import type { ModelPromptBuilder } from "./prompt-contracts";
+import { modelPromptBuilder } from "./prompt-builder";
+import { createRequestPrep } from "./request-prep";
+import { RequestOwnership } from "./request-plan";
+import { RequestError } from "./pipeline";
+import { errorResponse } from "./http";
+import { readDecisionBody } from "./decision-body";
+import { decisionAnswer, decisionCandidates, decisionMessages, DecisionValidationError,
+  parseJevDecision, parseOpenAiDecision } from "./decision-wire";
+
+export function createDecisionRoutes(engine: {
+  context: LoadedModelContext;
+  binding: Pick<ModelBinding, "decide"> & Partial<Pick<ModelBinding, "gateway">>;
+  preparation: PreparationExecutor;
+}, options: { contextLimit: number | null; defaultAdapter?: string; buildPrompt?: ModelPromptBuilder }) {
+  const ctx = engine.context, build = options.buildPrompt ?? modelPromptBuilder(ctx);
+  const prep = createRequestPrep({ ctx, serverOptions: { defaultThinking: false }, defaultGeneratedTokens: undefined });
+  return {
+    async handle(request: Request): Promise<Response | null> {
+      const path = new URL(request.url).pathname;
+      if (request.method !== "POST") return null;
+      const jev = path === "/v1/systemone";
+      if (!jev && path !== "/v1/decisions") return null;
+      const requestOwnership = new RequestOwnership();
+      try {
+        const reservation = await engine.preparation.reserve?.("decision", request.signal);
+        if (reservation) requestOwnership.retain(reservation);
+        const body = await readDecisionBody(request);
+        const parsed = jev ? parseJevDecision(body) : parseOpenAiDecision(body);
+        const decide = engine.binding.decide?.bind(engine.binding);
+        if (!decide) throw new RequestError(501, `model "${ctx.modelId}" declares no decision scoring operation`,
+          { message: `model "${ctx.modelId}" declares no decision scoring operation`, type: "not_implemented", code: "unsupported_execution" });
+        const schedule = <T>(work: () => Promise<T>): Promise<T> => engine.preparation.run(work, request.signal);
+        request.signal.throwIfAborted();
+        const ending = decisionAnswerEnding(ctx, prep);
+        const answers: Record<string, unknown>[] = [];
+        let inputTokens = 0;
+        for (const question of parsed.questions) {
+          request.signal.throwIfAborted();
+          const candidates = decisionCandidates(ctx.tokenizer, question, ending);
+          const ownership = new RequestOwnership();
+          try {
+            const messages = decisionMessages(parsed.input, question);
+            const media = messages.some(message => Array.isArray(message.content) && message.content.some(part => part.type !== "text"));
+            if (media) {
+              const reservation = await engine.preparation.reserve?.("media", request.signal);
+              if (reservation) ownership.retain(reservation);
+            }
+            let built: Awaited<ReturnType<ModelPromptBuilder>>;
+            try {
+              built = await build({ messages, reasoning_effort: "none", chat_template_kwargs: { enable_thinking: false } },
+                null, ownership, prep, schedule);
+            } catch (error) {
+              if (error instanceof RequestError) throw error;
+              throw new RequestError(400, `prompt build failed: ${error instanceof Error ? error.message : String(error)}`);
+            }
+            ownership.own(built.vision?.embeddings);
+            ownership.own(built.vision?.imageMask);
+            ownership.own(built.vision?.multimodalMask);
+            ownership.own(built.diffusionPixels);
+            request.signal.throwIfAborted();
+            if (built.diffusionPixels) throw new RequestError(501, "pixel-input decision scoring is unavailable for this graph");
+            if (options.contextLimit !== null && candidates.some(answer => built.promptIds.length + answer.length - 1 > options.contextLimit!))
+              throw new RequestError(400, `decision prompt exceeds the ${options.contextLimit}-token context limit`,
+                { message: "decision prompt exceeds the context limit", type: "invalid_request_error", code: "max_tokens_exceeded" });
+            const adapters = options.defaultAdapter ? [options.defaultAdapter] : [];
+            const probabilities = await decide({ prompt: built.promptIds, candidates, vision: built.vision, signal: request.signal,
+              execute: work => schedule(async () => {
+                request.signal.throwIfAborted();
+                const restore = engine.binding.gateway?.bindAdapterContext?.(adapters, `decision:${JSON.stringify(adapters)}`).enter();
+                try { return await work(); } finally { restore?.(); }
+              }) });
+            request.signal.throwIfAborted();
+            if (probabilities.length !== candidates.length || probabilities.some(p => !Number.isFinite(p) || p < 0 || p > 1) ||
+                Math.abs(probabilities.reduce((sum, p) => sum + p, 0) - 1) > 1e-5)
+              throw new Error("decision binding returned an invalid probability distribution");
+            inputTokens += candidates.reduce((sum, answer) => sum + built.promptIds.length + answer.length - 1, 0);
+            answers.push(decisionAnswer(question, probabilities, jev));
+          } finally { ownership.dispose(); }
+          await new Promise<void>(resolve => setImmediate(resolve));
+        }
+        request.signal.throwIfAborted();
+        return Response.json({ model: ctx.modelId,
+          answers: jev ? Object.fromEntries(parsed.questions.map((q, i) => [q.name!, answers[i]])) : answers,
+          usage: jev ? { input_tokens: inputTokens, output_tokens: 0 } : {
+            input_tokens: inputTokens, input_tokens_details: { cached_tokens: 0, cache_write_tokens: 0 },
+            output_tokens: 0, output_tokens_details: { reasoning_tokens: 0 }, total_tokens: inputTokens,
+          } });
+      } catch (error) {
+        if (error instanceof DecisionValidationError) return jev
+          ? Response.json({ detail: [{ loc: ["body", ...error.field.split(".")], msg: error.message, type: "value_error" }] }, { status: 422 })
+          : Response.json({ error: { message: error.message, type: "invalid_request_error", param: error.field, code: null } }, { status: 400 });
+        return errorResponse(error, path, undefined, request.signal);
+      } finally { requestOwnership.dispose(); }
+    },
+  };
+}
+
+/** Score the template's completed assistant turn through its first stop marker,
+ * excluding layout after the marker. Encode answer + ending together so BPE
+ * merges at that boundary match the supplied text. */
+function decisionAnswerEnding(ctx: LoadedModelContext, prep: ReturnType<typeof createRequestPrep>): string {
+  const stops = [...new Set([...ctx.model.config.eosTokenIds, ...(ctx.tokenizer.eosTokenId === null ? [] : [ctx.tokenizer.eosTokenId])])]
+    .map(id => ctx.tokenizer.decode([id], false)).filter(Boolean);
+  if (ctx.template) {
+    const marker = "mlx_bun_decision_answer_9d041a";
+    const messages = [{ role: "user" as const, content: "x" }, { role: "assistant" as const, content: marker }];
+    const options = { ...prep.templateOptionsFor({ messages,
+      reasoning_effort: "none", chat_template_kwargs: { enable_thinking: false } }, null), addGenerationPrompt: false };
+    const afterAnswer = (turns: typeof messages) => {
+      const rendered = ctx.template!.render(turns, options), at = rendered.indexOf(marker);
+      if (at < 0 || rendered.indexOf(marker, at + marker.length) >= 0)
+        throw new RequestError(501, "decision scoring requires a chat template that preserves assistant answers");
+      return rendered.slice(at + marker.length);
+    };
+    const throughStop = (suffix: string) => {
+      const first = stops.map(text => ({ text, at: suffix.indexOf(text) })).filter(stop => stop.at >= 0)
+        .sort((a, b) => a.at - b.at)[0];
+      return first ? suffix.slice(0, first.at + first.text.length) : undefined;
+    };
+    const suffix = afterAnswer(messages);
+    // GLM-style templates close an assistant answer with the next role marker.
+    // Only keep that boundary, never the synthetic next user's content.
+    const ending = throughStop(suffix) ?? throughStop(afterAnswer([...messages, { role: "user", content: "x" }]));
+    if (ending !== undefined) return ending;
+    if (suffix.trim()) return suffix;
+  }
+  const fallback = ctx.tokenizer.eosTokenId ?? ctx.model.config.eosTokenIds[0];
+  if (fallback === undefined || fallback === null) throw new RequestError(501, "decision scoring requires an answer terminator token");
+  return ctx.tokenizer.decode([fallback], false);
+}

package/src/server/decision-wire.ts

@@ -0,0 +1,228 @@
+import type { LoadedTokenizer } from "@mlx-bun/inference/input";
+import type { ChatRequestParams } from "./chat-request";
+
+type Messages = ChatRequestParams["messages"];
+export interface DecisionOption { value: string | boolean | number; label: string; description: string; legend?: unknown }
+export interface DecisionQuestion {
+  type: "predicate" | "choice" | "score";
+  name: string | null;
+  instructions: string;
+  options: DecisionOption[];
+  images?: { reference: string; url: string }[];
+}
+export interface DecisionRequest {
+  model: string;
+  input: Messages;
+  questions: DecisionQuestion[];
+}
+
+/** Carries a field location for both OpenAI errors and Jev's 422 details. */
+export class DecisionValidationError extends Error {
+  constructor(readonly field: string, message: string) { super(`${field}: ${message}`); }
+}
+const fail = (field: string, message: string): never => { throw new DecisionValidationError(field, message); };
+function record(value: unknown, field: string): Record<string, unknown> {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return fail(field, "must be an object");
+  return value as Record<string, unknown>;
+}
+function string(value: unknown, field: string): string {
+  if (typeof value !== "string") return fail(field, "must be a string");
+  return value;
+}
+function list(value: unknown, field: string, minimum: number, maximum: number): unknown[] {
+  if (!Array.isArray(value) || value.length < minimum || value.length > maximum)
+    return fail(field, `must contain ${minimum}–${maximum} entries`);
+  return value;
+}
+function text(value: unknown, field: string, nullable = false): string {
+  if (value === null && nullable) return "";
+  if (typeof value !== "string" && (!value || typeof value !== "object")) return fail(field, "must be text, an object, or an array");
+  finiteJson(value, field);
+  return typeof value === "string" ? value : JSON.stringify(value);
+}
+function finiteJson(value: unknown, field: string, depth = 0): void {
+  if (depth > 64) return fail(field, "exceeds 64 nested levels");
+  if (typeof value === "number" && !Number.isFinite(value)) return fail(field, "numbers must be finite");
+  if (value && typeof value === "object") for (const child of Object.values(value)) finiteJson(child, field, depth + 1);
+}
+const predicateOptions = (): DecisionOption[] => [
+  { value: false, label: "false", description: "The answer is no; the condition is false." },
+  { value: true, label: "true", description: "The answer is yes; the condition is true." },
+];
+
+/** OpenAI Decisions request contract: https://developers.openai.com/api/reference/resources/decisions/methods/create */
+export function parseOpenAiDecision(value: unknown): DecisionRequest {
+  const body = record(value, "body");
+  const model = string(body.model, "model");
+  let imageCount = 0;
+  const input: Messages = typeof body.input === "string" ? [{ role: "user", content: body.input }]
+    : list(body.input, "input", 1, Number.MAX_SAFE_INTEGER).map((entry, i) => {
+      const at = `input.${i}`, message = record(entry, at);
+      if (message.role !== "user" || (message.type !== undefined && message.type !== "message"))
+        return fail(at, "must be a user message");
+      const content = typeof message.content === "string" ? message.content
+        : list(message.content, `${at}.content`, 1, Number.MAX_SAFE_INTEGER).map((entry, j) => {
+          const part = record(entry, `${at}.content.${j}`);
+          if (part.type === "input_text") return { type: "text", text: string(part.text, `${at}.content.${j}.text`) };
+          if (part.type === "input_image") {
+            imageCount++;
+            if (imageCount > 128) return fail("input", "supports at most 128 images");
+            const detail = part.detail;
+            if (detail != null && !["auto", "low", "high", "original"].includes(String(detail)))
+              return fail(`${at}.content.${j}.detail`, "invalid image detail");
+            return { type: "image_url", image_url: { url: string(part.image_url, `${at}.content.${j}.image_url`), ...(detail ? { detail } : {}) } };
+          }
+          return fail(`${at}.content.${j}.type`, "expected input_text or input_image");
+        });
+      return { role: "user", content };
+    });
+  const questions = list(body.questions, "questions", 1, 200).map((entry, i): DecisionQuestion => {
+    const at = `questions.${i}`, q = record(entry, at), instructions = string(q.instructions, `${at}.instructions`);
+    const name = q.name === undefined ? null : string(q.name, `${at}.name`);
+    if (q.type === "predicate") return { type: "predicate", name, instructions, options: predicateOptions() };
+    if (q.type === "choice") {
+      const options = list(q.choices, `${at}.choices`, 2, 255).map((entry, j) => {
+        const choice = record(entry, `${at}.choices.${j}`), value = choice.value;
+        if (typeof value !== "string" && typeof value !== "boolean") return fail(`${at}.choices.${j}.value`, "must be a string or boolean");
+        return { value, label: JSON.stringify(value), description: choice.description === undefined ? "" : string(choice.description, `${at}.choices.${j}.description`) };
+      });
+      if (new Set(options.map(option => JSON.stringify(option.value))).size !== options.length)
+        return fail(`${at}.choices`, "values must be unique (strings and booleans are distinct)");
+      return { type: "choice", name, instructions, options };
+    }
+    if (q.type === "score") {
+      const options = list(q.levels, `${at}.levels`, 2, 255).map((entry, j) => {
+        const level = record(entry, `${at}.levels.${j}`);
+        return { value: j, label: string(level.label, `${at}.levels.${j}.label`),
+          description: level.description === undefined ? "" : string(level.description, `${at}.levels.${j}.description`) };
+      });
+      return { type: "score", name, instructions, options };
+    }
+    return fail(`${at}.type`, "expected predicate, choice, or score");
+  });
+  return { model, input, questions };
+}
+
+/** Jev System One contract: https://docs.typesafe.ai/api */
+export function parseJevDecision(value: unknown): DecisionRequest {
+  const body = record(value, "body"), model = string(body.model, "model");
+  const state = text(body.state, "state");
+  const assets = new Map<string, string>(), used = new Set<string>();
+  for (const [reference, value] of Object.entries(body.assets == null ? {} : record(body.assets, "assets"))) {
+    if (!reference.trim() || Buffer.byteLength(reference) > 1024) return fail("assets", "invalid image reference");
+    const asset = record(value, `assets.${reference}`);
+    if (asset.type !== undefined && asset.type !== "image") return fail(`assets.${reference}.type`, "expected image");
+    if (!["png", "jpg", "jpeg"].includes(String(asset.format))) return fail(`assets.${reference}.format`, "expected png, jpg, or jpeg");
+    const data = string(asset.data_base64, `assets.${reference}.data_base64`);
+    if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(data) || !data.length)
+      return fail(`assets.${reference}.data_base64`, "invalid base64 image");
+    assets.set(reference, `data:image/${asset.format === "png" ? "png" : "jpeg"};base64,${data}`);
+  }
+  // Only values may refer to images. IDs and option keys stay text. Matching is
+  // left-to-right, with whole-value matches taking precedence over substrings.
+  const imagesIn = (value: unknown): { reference: string; url: string }[] => {
+    if (value && typeof value === "object") return Object.values(value).flatMap(imagesIn);
+    if (typeof value !== "string") return [];
+    if (assets.has(value)) { used.add(value); return [{ reference: value, url: assets.get(value)! }]; }
+    const found: { reference: string; url: string }[] = [];
+    for (let position = 0; position < value.length;) {
+      const matches = [...assets.keys()].map(reference => ({ reference, at: value.indexOf(reference, position) }))
+        .filter(match => match.at >= 0).sort((a, b) => a.at - b.at || b.reference.length - a.reference.length);
+      const match = matches[0];
+      if (!match) break;
+      found.push({ reference: match.reference, url: assets.get(match.reference)! }); used.add(match.reference);
+      position = match.at + match.reference.length;
+    }
+    return found;
+  };
+  const stateImages = imagesIn(body.state);
+  const entries = Object.entries(record(body.questions, "questions"));
+  if (!entries.length) return fail("questions", "must contain at least one question");
+  const questions = entries.map(([name, entry]): DecisionQuestion => {
+    const at = `questions.${name}`, q = record(entry, at), instructions = text(q.instructions ?? null, `${at}.instructions`, true);
+    if (q.type === "noul") {
+      const options = predicateOptions();
+      if (q.criteria != null) {
+        const criteria = record(q.criteria, `${at}.criteria`);
+        for (const option of options) if (Object.hasOwn(criteria, String(option.value)))
+          option.description = text(criteria[String(option.value)], `${at}.criteria.${option.value}`, true);
+      }
+      return { type: "predicate", name, instructions, options, images: [...imagesIn(q.instructions), ...imagesIn(q.criteria)] };
+    }
+    if (q.type === "choice") {
+      const choices = Object.entries(record(q.criteria, `${at}.criteria`));
+      if (choices.length < 1 || choices.length > 255) return fail(`${at}.criteria`, "requires 1–255 options");
+      return { type: "choice", name, instructions, images: [...imagesIn(q.instructions), ...imagesIn(q.criteria)], options: choices.map(([value, description]) => ({
+        value, label: JSON.stringify(value), description: text(description, `${at}.criteria.${value}`, true),
+      })) };
+    }
+    if (q.type === "score") return { type: "score", name, instructions, images: [...imagesIn(q.instructions), ...imagesIn(q.criteria)],
+      options: list(q.criteria, `${at}.criteria`, 2, 10).map((entry, index) => ({
+        value: index, label: text(entry, `${at}.criteria.${index}`, true) || String(index), description: "", legend: entry,
+      })) };
+    return fail(`${at}.type`, "expected noul, choice, or score");
+  });
+  if (used.size !== assets.size) return fail("assets", "every image asset must be referenced");
+  return { model, input: [{ role: "user", content: stateImages.length ? [
+    { type: "text", text: state }, ...imageParts(stateImages),
+  ] : state }], questions };
+}
+
+function imageParts(images: { reference: string; url: string }[]): { type: string; [key: string]: unknown }[] {
+  return images.flatMap(image => [{ type: "text", text: `Image reference: ${image.reference}` },
+    { type: "image_url", image_url: { url: image.url } }]);
+}
+
+/** Exact wire values encoded as JSON keep string "true" distinct from boolean
+ * true, preserve whitespace and empty strings, and never shorten multi-token
+ * answers. Score questions use their supplied labels as the answer text. */
+export function decisionCandidates(tokenizer: Pick<LoadedTokenizer, "encode">, question: DecisionQuestion, ending: string): number[][] {
+  return question.options.map(option => tokenizer.encode(
+    JSON.stringify(question.type === "score" ? option.label : option.value) + ending, false));
+}
+
+export function decisionMessages(input: Messages, question: DecisionQuestion): Messages {
+  const menu = question.options.map(option => `${question.type === "score" ? JSON.stringify(option.label) : JSON.stringify(option.value)}${option.description ? ` — ${option.description}` : ""}`).join("\n");
+  const questionText = `Evaluate the preceding input.\nQuestion: ${question.instructions}\nAllowed answers${question.type === "score" ? " (ordered from lowest to highest)" : ""}:\n${menu}\nAnswer with exactly one of the JSON values above. Do not explain.`;
+  // Both protocols supply user input. Keep it in one turn so templates that
+  // enforce user/assistant alternation accept it, retaining media order.
+  const content = input.flatMap((message, i) => [
+    ...(i ? [{ type: "text" as const, text: "\n\n" }] : []),
+    ...(typeof message.content === "string" ? [{ type: "text" as const, text: message.content }] : message.content ?? []),
+  ]);
+  content.push({ type: "text", text: `\n\n${questionText}` }, ...imageParts(question.images ?? []));
+  return [{ role: "user", content: content.every(part => part.type === "text")
+    ? content.map(part => (part as { text: string }).text).join("") : content }];
+}
+
+function winner(probabilities: readonly number[]): number {
+  return probabilities.reduce((best, p, i) => p > probabilities[best]! ? i : best, 0);
+}
+/** Jev's documented distribution-derived confidence, including ordered levels.
+ * OpenAI does not publish a confidence formula; the local adapter uses this
+ * same statistic. This is a statistic of the likelihoods, with no learned
+ * calibration stage. */
+export function decisionConfidence(probabilities: readonly number[], ordered: boolean): number {
+  const n = probabilities.length, best = winner(probabilities);
+  if (n === 1) return 1;
+  if (!ordered) return Math.max(0, Math.min(1, (probabilities[best]! * n - 1) / (n - 1)));
+  const spread = probabilities.reduce((sum, p, i) => sum + p * Math.abs(i - best), 0);
+  const uniform = probabilities.reduce((sum, _, i) => sum + Math.abs(i - (n - 1) / 2) / n, 0);
+  return Math.max(0, Math.min(1, 1 - spread / uniform));
+}
+export function decisionAnswer(question: DecisionQuestion, probabilities: number[], jev: boolean): Record<string, unknown> {
+  const { type, name, options } = question;
+  if (type === "predicate") return jev ? { type: "noul", noul: probabilities[1] }
+    : { type, name, probability: probabilities[1] };
+  const confidence = decisionConfidence(probabilities, type === "score");
+  if (type === "choice") {
+    const choice = options[winner(probabilities)]!.value;
+    return jev ? { type, choice, confidence, probabilities: Object.fromEntries(options.map((option, i) => [option.value, probabilities[i]])) }
+      : { type, name, choice, confidence, probabilities: options.map((option, i) => ({ value: option.value, probability: probabilities[i] })) };
+  }
+  const score = probabilities.reduce((sum, p, i) => sum + p * i, 0);
+  return jev ? { type, score, confidence,
+    legend: Object.fromEntries(options.map((option, i) => [i, Object.hasOwn(option, "legend") ? option.legend : option.label])),
+    probabilities: Object.fromEntries(probabilities.map((p, i) => [i, p])) }
+    : { type, name, score, confidence, probabilities: options.map((option, i) => ({ value: i, label: option.label, probability: probabilities[i] })) };
+}

package/src/{serve → server}/discovery-routes.ts

@@ -1,14 +1,12 @@
-import { modelServingBinding } from "../backends/mlx/model-serving";
+import type { Registry } from "@mlx-bun/hub/registry";
+import { openRegistry } from "../storage/paths";
+import type { ModelBinding } from "../engine/model-binding";
 import pkgJson from "../../package.json" with { type: "json" };
-import type { ServingContext } from "./model-host";
-import type { GenerationGateway } from "./generation-gateway";
-import { fit } from "../fit";
+import type { LoadedModelContext as ModelContext } from "../engine/model-host";
 
 const pkgVersion = (pkgJson as { version: string }).version;
 
 export type DiscoveryRoute =
-  | "library"
-  | "downloads"
   | "api-index"
   | "health"
   | "models";
@@ -16,8 +14,6 @@ export type DiscoveryRoute =
 export function matchDiscoveryRoute(method: string, pathname: string): DiscoveryRoute | null {
   if (method !== "GET") return null;
   switch (pathname) {
-    case "/library": return "library";
-    case "/downloads": return "downloads";
     case "/v1": return "api-index";
     case "/health": return "health";
     case "/v1/models": return "models";
@@ -27,74 +23,25 @@ export function matchDiscoveryRoute(method: string, pathname: string): Discovery
 
 export interface DiscoveryRoutes {
   handle(url: URL, request: Request): Promise<Response | null>;
-  invalidateLibrary(): void;
+}
+
+export interface TranscriptionInfo {
+  id: string;
+  resident: boolean;
 }
 
 export function createDiscoveryRoutes(
-  ctx: ServingContext,
-  gateway: Pick<GenerationGateway, "batchMode">,
+  ctx: ModelContext,
+  binding: Pick<ModelBinding, "discovery">,
   startedAt: number,
+  transcription: () => Promise<TranscriptionInfo | null> = async () => null,
+  createRegistry: () => Pick<Registry, "scan" | "listCanonical" | "close"> = () => openRegistry(),
+  /** The server's own sampling defaults (`--temperature`, `--top-p`, `--top-k`): they win over the model's, as they do for a request that names none. */
+  serverDefaults: { temperature?: number; topP?: number; topK?: number } = {},
 ): DiscoveryRoutes {
-  let libraryCache: { at: number; rows: unknown[] } | null = null;
-
   return {
-    invalidateLibrary() {
-      libraryCache = null;
-    },
-
     async handle(url, request) {
       switch (matchDiscoveryRoute(request.method, url.pathname)) {
-        case "library": {
-          if (url.searchParams.get("refresh") === "1" || !libraryCache || Date.now() - libraryCache.at > 30_000) {
-            const { Registry, visionCapable, audioCapable } = await import("../registry");
-            const { loadModelConfig } = await import("../config");
-            const { supportTier } = await import("../model/support");
-            const registry = new Registry();
-            await registry.scan();
-            const rows = [];
-            for (const model of registry.listCanonical()) {
-              const tier = supportTier(model.modelType, model.repoId);
-              const supported = tier !== null;
-              let assessment = null;
-              try {
-                const config = await loadModelConfig(model.path);
-                const result = fit(
-                  config,
-                  model.sizeBytes,
-                  8192,
-                  undefined,
-                  undefined,
-                  model.expertsBytes,
-                );
-                assessment = {
-                  fits: result.fits,
-                  max_safe_context: result.maxSafeContext,
-                  predicted_decode_tps: result.predictedDecodeTps,
-                };
-              } catch {}
-              rows.push({
-                repo_id: model.repoId,
-                model_type: model.modelType,
-                size_bytes: model.sizeBytes,
-                quant_bits: model.quantBits,
-                vision: visionCapable(model),
-                audio: audioCapable(model),
-                supported,
-                support_tier: tier,
-                serving: model.repoId === ctx.modelId,
-                assessment,
-              });
-            }
-            libraryCache = { at: Date.now(), rows };
-          }
-          return Response.json({ models: libraryCache.rows });
-        }
-
-        case "downloads": {
-          const { downloadsSnapshot } = await import("../download");
-          return Response.json({ downloads: downloadsSnapshot() });
-        }
-
         case "api-index":
           return Response.json({
             name: "mlx-bun",
@@ -106,8 +53,11 @@ export function createDiscoveryRoutes(
               "POST /v1/messages",
               "POST /v1/responses",
               "POST /v1/embeddings",
+              "POST /v1/decisions",
+              "POST /v1/systemone",
+              "POST /v1/audio/transcriptions",
+              "POST /v1/audio/translations",
               "GET /v1/models",
-              "GET/POST/DELETE /v1/adapters",
               "GET /health",
               "GET /stats",
               "GET /fit",
@@ -127,30 +77,31 @@ export function createDiscoveryRoutes(
             : null;
           const created = Math.floor(startedAt / 1000);
           const genDefaults = {
-            temperature: ctx.genDefaults.temperature ?? null,
-            top_p: ctx.genDefaults.topP ?? null,
-            top_k: ctx.genDefaults.topK ?? null,
+            temperature: serverDefaults.temperature ?? ctx.genDefaults.temperature ?? null,
+            top_p: serverDefaults.topP ?? ctx.genDefaults.topP ?? null,
+            top_k: serverDefaults.topK ?? ctx.genDefaults.topK ?? null,
           };
-          const capabilities = modelServingBinding(ctx).discovery;
+          const capabilities = binding.discovery;
+          const stt = await transcription();
           const data: Array<Record<string, unknown>> = [{
             id: ctx.modelId,
             object: "model",
             created,
             owned_by: "mlx-bun",
             context_window:
-              ctx.glmMemoryPlan?.contextTokens ??
+              ctx.memoryPlan?.contextTokens ??
               ctx.model.config.text.maxPositionEmbeddings,
-            reasoning: ctx.template.supportsThinking,
+            reasoning: ctx.template?.supportsThinking ?? false,
             vision: !!(ctx.vision || ctx.loadVision),
             audio: !!(ctx.audio || ctx.loadAudio),
-            batch_mode: gateway.batchMode,
+            batch_mode: "batch",
             tools: true,
             structured_output: true,
             embeddings: capabilities.embeddings,
             adapters: capabilities.adapters,
             training: capabilities.training,
             dsa: capabilities.dsa,
-            mtp: ctx.draft?.provider.id === "glm52-native-mtp",
+            mtp: ctx.draft?.native === true,
             capabilities: {
               chat_completions: true,
               text_completions: true,
@@ -161,22 +112,30 @@ export function createDiscoveryRoutes(
               structured_output: true,
               logprobs: true,
               embeddings: capabilities.embeddings,
+              decisions: binding.discovery.decisions ?? false,
               vision: !!(ctx.vision || ctx.loadVision),
               audio: !!(ctx.audio || ctx.loadAudio),
               adapters: capabilities.adapters,
               training: capabilities.training,
+              transcription: stt !== null,
             },
             gen_defaults: genDefaults,
           }];
+          if (stt)
+            data.push({
+              id: stt.id, object: "model", created, owned_by: "mlx-bun",
+              transcription: true, resident: stt.resident,
+              capabilities: { transcription: true, translation: true, chat_completions: false },
+            });
           try {
-            const { Registry, visionCapable } = await import("../registry");
-            const { supportTier } = await import("../model/support");
-            const registry = new Registry();
+            const { visionCapable } = await import("@mlx-bun/hub/registry");
+            const { listedSupportTier } = await import("@mlx-bun/inference/models/support");
+            const registry = createRegistry();
             try {
-              if (registry.list().length === 0) await registry.scan();
+              if (registry.listCanonical().length === 0) await registry.scan();
               for (const model of registry.listCanonical()) {
-                if (model.repoId === ctx.modelId) continue;
-                const tier = supportTier(model.modelType, model.repoId);
+                if (model.repoId === ctx.modelId || model.repoId === stt?.id) continue;
+                const tier = listedSupportTier(model);
                 if (tier === null) continue;
                 data.push({
                   id: model.repoId,

package/src/server/generated-token-history.ts

@@ -0,0 +1,57 @@
+/** Token provenance for rendered multi-turn prompts. A generated BPE token
+ * sequence need not equal encode(decode(ids)). Preserve the sequence that
+ * produced the saved inference state when its rendered text is unchanged.
+ * This stores only text and IDs; KV ownership and persistence live elsewhere. */
+export interface PromptTokenHistory {
+  remember(tokens: readonly number[]): void;
+  resolve(rendered: string, canonical: number[]): number[];
+}
+
+interface TokenCodec {
+  encode(text: string, addSpecialTokens?: boolean): number[];
+  decode(ids: number[], skipSpecialTokens?: boolean): string;
+}
+
+export class GeneratedTokenHistory implements PromptTokenHistory {
+  readonly #entries = new Map<string, number[]>();
+  #chars = 0;
+  constructor(readonly codec: TokenCodec, readonly maxChars = 2 ** 21, readonly maxEntries = 16) {}
+
+  remember(tokens: readonly number[]): void {
+    if (!tokens.length) return;
+    // Repeated requests can publish the same generated checkpoint. Its text
+    // and IDs are already owned here; refresh recency without decoding the
+    // entire conversation again on the request's completion path.
+    for (const [text, ids] of this.#entries) {
+      if (ids.length !== tokens.length || !ids.every((id, i) => id === tokens[i])) continue;
+      this.#entries.delete(text);
+      this.#entries.set(text, ids);
+      return;
+    }
+    const ids = [...tokens], text = this.codec.decode(ids, false);
+    // An unfinished UTF-8 token is not a text boundary. A later checkpoint
+    // can supply a complete prefix instead.
+    if (!text.length || text.endsWith("\uFFFD") || text.length > this.maxChars) return;
+    if (this.#entries.delete(text)) this.#chars -= text.length;
+    this.#entries.set(text, ids); this.#chars += text.length;
+    while (this.#entries.size > this.maxEntries || this.#chars > this.maxChars) {
+      const oldest = this.#entries.keys().next().value!;
+      this.#entries.delete(oldest); this.#chars -= oldest.length;
+    }
+  }
+
+  resolve(rendered: string, canonical: number[]): number[] {
+    let best: { text: string; ids: number[] } | undefined;
+    for (const [text, ids] of this.#entries) {
+      if (text.length >= rendered.length || (best && text.length <= best.text.length)) continue;
+      if (rendered.startsWith(text)) best = { text, ids };
+    }
+    if (!best) return canonical;
+    // Normal canonical output keeps the existing tokenizer fast path.
+    if (best.ids.every((id, i) => canonical[i] === id)) return canonical;
+    const ids = [...best.ids, ...this.codec.encode(rendered.slice(best.text.length), false)];
+    // Decoder cleanup/normalization (e.g. SentencePiece's leading-space
+    // rules) can make a standalone suffix unsafe to append.
+    return this.codec.decode(ids, false) === rendered ? ids : canonical;
+  }
+}