mlx-bun 0.5.0 → 1.0.0

package/src/server/publishing-routes.ts

@@ -0,0 +1,41 @@
+import type { createHfCredentials } from "../publishing/credentials";
+import type { createPublisher, PublishRequest } from "../publishing/upload";
+
+/** HTTP parsing and wire errors stay above credential and publishing policy. */
+export function createPublishingRoutes(options: {
+  credentials: ReturnType<typeof createHfCredentials>;
+  publish: ReturnType<typeof createPublisher>;
+}) {
+  const error = (message: string, status = 400) => Response.json({ ok: false, error: message }, { status });
+  return { async handle(request: Request): Promise<Response | null> {
+    const path = new URL(request.url).pathname;
+    const settings = path === "/api/settings/hf-token";
+    const push = path.match(/^\/api\/(quantize|finetune|dataset)\/push$/);
+    if (!(settings && ["GET", "POST"].includes(request.method)) && !(push && request.method === "POST")) return null;
+    try {
+      request.signal.throwIfAborted();
+      if (settings && request.method === "GET") return Response.json({ ok: true, hasToken: options.credentials.get() !== null });
+      const raw: unknown = await request.json().catch(() => null);
+      request.signal.throwIfAborted();
+      if (!raw || typeof raw !== "object" || Array.isArray(raw)) return error("expected a JSON object");
+      const body = raw as Record<string, unknown>;
+      if (settings) {
+        if (typeof body.token !== "string" || !body.token.trim()) return error("token required");
+        options.credentials.save(body.token);
+        return Response.json({ ok: true });
+      }
+      if (typeof body.repo_id !== "string" || !body.repo_id.trim()) return error("repo_id required");
+      for (const key of ["job_id", "source_path"] as const) {
+        if (body[key] != null && (typeof body[key] !== "string" || !body[key].trim())) return error(`${key} must be a nonempty string`);
+      }
+      if (body.private != null && typeof body.private !== "boolean") return error("private must be a boolean");
+      const result = await options.publish({ kind: push![1] as PublishRequest["kind"], repoId: body.repo_id,
+        private: body.private as boolean | undefined, sourcePath: body.source_path as string | undefined,
+        jobId: body.job_id as string | undefined, signal: request.signal });
+      return Response.json({ ok: true, url: result.url });
+    } catch (failure) {
+      if (request.signal.aborted) return error("Request cancelled", 499);
+      return error(failure instanceof Error ? failure.message : String(failure));
+    }
+  } };
+}

package/src/{serve → server}/request-plan.ts

@@ -1,9 +1,9 @@
-import { snapshotGenerationPolicy } from "../backends/mlx/request-policy";
-import type { GenerateOptions } from "../generate";
-import type { RequestShape } from "./generation-gateway";
-import type { DisposableResource } from "../contracts/resources";
-import { disposeResources, ownResource } from "../engine/resources";
-export type { DisposableResource } from "../contracts/resources";
+import { snapshotGenerationPolicy } from "@mlx-bun/inference/execution/request-policy";
+import type { GenerateOptions } from "@mlx-bun/inference/generation";
+import type { RequestShape } from "../engine/completion";
+import type { DisposableResource } from "@mlx-bun/inference/contracts/portable";
+import { disposeResources, ownResource } from "@mlx-bun/inference/runtime/resources";
+export type { DisposableResource } from "@mlx-bun/inference/contracts/portable";
 
 export class RequestOwnership {
   #owner = ownResource(new Set<DisposableResource>(), disposeResources);
@@ -147,8 +147,6 @@ export function planRequest(input: RequestPlanInput): PlanRequestResult {
     ),
     wantsLogprobs: captureLogprobs,
     userSeed: input.userSeed,
-    kvQuant: !!(options.kvConfig?.length || options.kvBits),
-    turboQuant: !!options.turboQuant,
     hasGrammar: input.hasGrammar,
     hasDraft: input.hasDraft,
   };

package/src/{serve → server}/request-prep.ts

@@ -1,23 +1,24 @@
+import { requireChatTemplate } from "../engine/model-host";
+import { textPrompt } from "./text-prompt";
 // Per-request preparation shared by every completion surface (chat,
 // raw text, /signal, /generate): request → GenerateOptions with the
 // server/model defaults folded in, chat-template rendering + prompt ids,
 // the stable prompt-cache boundary probe, grammar compilation, and the
 // tool-call stream router. Resolved ONCE per server (createRequestPrep) and
-// captured by the route handlers. Extracted from src/server.ts (repo-taming
-// Phase 4).
-import type { ChatMessage, ToolDefinition } from "../chat-template";
+// captured by the route handlers.
+import type { ChatMessage, ToolDefinition } from "@mlx-bun/inference/input";
 import {
   fillEchoConfig,
   fillMaxSpan,
   FillSession,
   resolveFillMode,
-} from "../fill/fill-session";
-import { compileStrictFillRows, type StrictRowPlan } from "../fill/schema-rows";
-import type { GenerateOptions } from "../generate";
-import { compileGrammarRequest, grammarEnabled, type GrammarRequest } from "../grammar";
-import type { KvSchemeOptions } from "../kv-scheme";
-import { isMiniCPM5Config } from "../model/support";
-import type { HlgConfig } from "../sampler";
+} from "@mlx-bun/inference/generation/fill";
+import { compileStrictFillRows, type StrictRowPlan } from "@mlx-bun/inference/generation/fill";
+import type { GenerateOptions } from "@mlx-bun/inference/generation";
+import type { GrammarRequest } from "@mlx-bun/inference/sampling/grammar";
+import { runtimeValue } from "@mlx-bun/inference/runtime/config";
+import { GENERIC_GENERATION_DEFAULTS } from "@mlx-bun/inference/models/profile";
+import type { HlgConfig } from "@mlx-bun/inference/sampling";
 import {
   nextDefaultSeed,
   normalizeMessages,
@@ -26,7 +27,7 @@ import {
   resolveHlg,
   type ChatRequestParams,
 } from "./chat-request";
-import type { ServingContext } from "./model-host";
+import type { LoadedModelContext as ModelContext } from "../engine/model-host";
 import type { PromptTokenHistory } from "./generated-token-history";
 import { selectToolStreamMode, ToolAwareStream, type ToolStreamMode } from "./token-streams";
 
@@ -38,21 +39,18 @@ export interface RequestPrepOptions {
   defaultTopP?: number;
   defaultTopK?: number;
   hlg?: HlgConfig;
-  pagedKv?: { blockSize?: number };
 }
 
 export function createRequestPrep(input: {
-  ctx: ServingContext;
+  ctx: ModelContext;
   serverOptions: RequestPrepOptions;
-  /** The server-wide KV scheme (resolveKvScheme(...).generationOptions),
-   *  spread into every request's GenerateOptions. */
-  kvScheme: KvSchemeOptions;
-  /** `--max-tokens` / the GLM memory plan's generation cap; undefined = the
+  /** `--max-tokens` / a runtime memory plan's generation cap; undefined = the
    *  surface's own default (chat 65 536, raw completion 512). */
   defaultGeneratedTokens: number | undefined;
   tokenHistory?: PromptTokenHistory;
 }) {
-  const { ctx, serverOptions, kvScheme, defaultGeneratedTokens } = input;
+  const { ctx, serverOptions, defaultGeneratedTokens } = input;
+  const generationDefaults = ctx.generationDefaults ?? GENERIC_GENERATION_DEFAULTS;
 
   // XTC never removes EOS or the newline token: mlx_lm.server passes
   // [tokenizer.eos_token_id, tokenizer.encode("\n")] as xtc_special_tokens.
@@ -67,31 +65,31 @@ export function createRequestPrep(input: {
 
   // Effective enable_thinking for a request, with the same precedence the chat
   // template uses (extracted so sampling and template rendering can't disagree):
-  // explicit chat_template_kwargs.enable_thinking → reasoning_effort ("none" =
-  // off) → server --thinking default → model default (MiniCPM5 → off). undefined
+  // explicit chat_template_kwargs.enable_thinking (null = template default) → reasoning_effort ("none" =
+  // off) → server --thinking default → the model's declared default. undefined
   // means "not a switchable-thinking model / leave the template default".
   const resolveEnableThinking = (req: ChatRequestParams): boolean | undefined => {
     const explicit = req.chat_template_kwargs?.enable_thinking;
+    if (explicit === null) return undefined;
     if (typeof explicit === "boolean") return explicit;
     const effort = req.reasoning_effort;
     if (effort !== undefined) return effort !== "none";
     if (serverOptions.defaultThinking !== undefined) return serverOptions.defaultThinking;
-    return isMiniCPM5Config(ctx.model.config) ? false : undefined;
+    return generationDefaults.enableThinking;
   };
 
   const toOptions = (req: ChatRequestParams): GenerateOptions & { stopSequences: string[] } => {
     // Sampling follows the thinking state — which the web UI's thinking button
     // drives via enable_thinking. Model authors publish a SINGLE
     // generation_config temperature (the think-mode value) but recommend a
-    // cooler one for direct, no-think replies (MiniCPM5 card: 0.9 think / 0.7
-    // no-think, top_p 0.95 for both). So with no explicit temperature set, a
-    // no-think turn runs at most NO_THINK_TEMPERATURE while a think turn keeps
-    // the model's hotter configured default. An explicit request/CLI
-    // temperature always wins. top_p is unchanged by mode.
-    const NO_THINK_TEMPERATURE = 0.7;
+    // cooler one for direct, no-think replies. So with no explicit temperature
+    // set, a no-think turn runs at most the model's declared
+    // noThinkTemperatureCap while a think turn keeps the model's hotter
+    // configured default. An explicit request/CLI temperature always wins.
+    // top_p is unchanged by mode.
     const genTemp = ctx.genDefaults.temperature ?? 0.7;
     const defaultTemp = resolveEnableThinking(req) === false
-      ? Math.min(genTemp, NO_THINK_TEMPERATURE)
+      ? Math.min(genTemp, generationDefaults.noThinkTemperatureCap)
       : genTemp;
     return {
       cacheSessionId: typeof req.session_id === "string" ? req.session_id :
@@ -124,10 +122,6 @@ export function createRequestPrep(input: {
       // decode layer below and halts the loop via onToken → false.
       stopSequences: (typeof req.stop === "string" ? [req.stop] : req.stop ?? [])
         .filter((s) => typeof s === "string" && s.length > 0),
-      ...kvScheme,
-      // Paged KV is a server-wide mode like the kv scheme above (v1:
-      // serial bf16 gemma4 — the createServer refusals hold the invariants).
-      ...(serverOptions.pagedKv ? { pagedKv: serverOptions.pagedKv } : {}),
     };
   };
 
@@ -138,13 +132,14 @@ export function createRequestPrep(input: {
    *  + Warning header (oMLX parity — never 500). Honors MLX_BUN_GRAMMAR=0. */
   const compileGrammarForRequest = async (
     req: ChatRequestParams,
-  ): Promise<{ controller: import("../grammar").GrammarController | null; degradeHint: string | null }> => {
-    if (!grammarEnabled()) {
+  ): Promise<{ controller: import("@mlx-bun/inference/sampling/grammar").GrammarController | null; degradeHint: string | null }> => {
+    if (runtimeValue("MLX_BUN_GRAMMAR") === "0") {
       return {
         controller: null,
         degradeHint: "grammar compilation disabled by MLX_BUN_GRAMMAR=0",
       };
     }
+    const { compileGrammarRequest } = await import("@mlx-bun/inference/sampling/grammar");
     const r = await compileGrammarRequest(
       req as GrammarRequest,
       ctx.tokenizer,
@@ -154,11 +149,11 @@ export function createRequestPrep(input: {
     return { controller: r.controller, degradeHint: r.degradeHint };
   };
 
-  // Map OpenAI reasoning_effort levels onto the Qwen3.8 template's supported
-  // set (xhigh|medium|low — the template raises on anything else). "none"
+  // Map OpenAI reasoning_effort levels onto the depth set a template that reads
+  // `reasoning_effort` supports (xhigh|medium|low — such a template raises on anything else). "none"
   // means thinking off (handled by resolveEnableThinking), so no depth is
   // passed. Only consumed for templates with readsReasoningEffort.
-  const qwenReasoningEffort = (
+  const templateReasoningEffort = (
     effort: ChatRequestParams["reasoning_effort"],
   ): "xhigh" | "medium" | "low" | undefined => {
     switch (effort) {
@@ -181,7 +176,7 @@ export function createRequestPrep(input: {
     return {
       tools,
       enableThinking: resolveEnableThinking(req),
-      reasoningEffort: qwenReasoningEffort(req.reasoning_effort),
+      reasoningEffort: templateReasoningEffort(req.reasoning_effort),
       preserveThinking: req.chat_template_kwargs?.preserve_thinking,
     };
   };
@@ -190,12 +185,11 @@ export function createRequestPrep(input: {
     req: ChatRequestParams,
     tools: ToolDefinition[] | null,
   ): { ids: number[]; startInThinking: boolean } => {
+    // A context without a template serves only through a supplied prompt builder.
+    requireChatTemplate(ctx);
     const opts = templateOptionsFor(req, tools);
-    const rendered = ctx.template.render(normalizeMessages(req.messages), opts);
-    const ids = ctx.tokenizer.encode(rendered);
-    // template includes <bos>; tokenizer post-processor also prepends one
-    const trimmed = ids[0] === ids[1] && ids[0] === ctx.tokenizer.bosTokenId ? ids.slice(1) : ids;
-    return { ids: input.tokenHistory?.resolve(rendered, trimmed) ?? trimmed,
+    const { rendered, ids } = textPrompt(ctx.template, ctx.tokenizer, normalizeMessages(req.messages), opts);
+    return { ids: input.tokenHistory?.resolve(rendered, ids) ?? ids,
       startInThinking: promptEndsInOpenThink(rendered) };
   };
 
@@ -218,6 +212,8 @@ export function createRequestPrep(input: {
     tools: ToolDefinition[] | null,
     trimmed: number[],
   ): number => {
+    // Outside the best-effort probe: a builder asking for it needs a template.
+    requireChatTemplate(ctx);
     const opts = templateOptionsFor(req, tools);
     const mode = `${opts.enableThinking}|${!!tools?.length}|${opts.reasoningEffort ?? ""}|${opts.preserveThinking ?? ""}`;
     const primer = primerLenByMode.get(mode);
@@ -260,16 +256,15 @@ export function createRequestPrep(input: {
   };
 
   const toolStreamMode = (tools: ToolDefinition[] | null): ToolStreamMode =>
-    // Gemma-4 keeps the token-level sentinel path; every other family
-    // (MiniCPM5, Qwen3/3.5, Tier-0 generics) emits tool calls as DECODED TEXT
-    // — see selectToolStreamMode for why sentinel ids must stay family-gated.
-    selectToolStreamMode(ctx.model.config.modelType, !!tools?.length);
+    // A model that declares sentinel tokens keeps the token-level path; every
+    // other model emits tool calls as DECODED TEXT — see selectToolStreamMode.
+    selectToolStreamMode(ctx.sentinels, !!tools?.length);
 
   const toolRouter = (
     tools: ToolDefinition[] | null,
     onParseFailure?: () => void,
   ): ToolAwareStream =>
-    new ToolAwareStream(ctx.tokenizer, toolStreamMode(tools), tools, onParseFailure);
+    new ToolAwareStream(ctx.tokenizer, toolStreamMode(tools), tools, onParseFailure, ctx.sentinels);
 
   /** Strict fill rows are a function of (template mode, tool signature) only —
    *  they are token spans of the assistant turn, which opens with a special
@@ -285,13 +280,13 @@ export function createRequestPrep(input: {
       (t.function?.parameters?.additionalProperties as unknown) ?? null,
     ]));
 
-  /** Token fast-forwarding (docs/design/speculative-decoding.md "Token
+  /** Token fast-forwarding (`02d723a:docs/design/speculative-decoding.md` §7 "Token
    *  fast-forwarding"): compile this request's determined-span table, or null
    *  when the feature is off or the request's shape refuses it.
    *
    *  Refusals owned here (body-level); the ones only ChatStage can see —
    *  a compiled grammar, media prompts and a mounted draft model — are applied
-   *  there. The execution binding owns cache-format and method support. Strict
+   *  there. The execution binding owns method support (every cache serves it). Strict
    *  shared execution keeps the sampler's absolute token positions, including
    *  known tokens; an explicit seed therefore remains replayable.
    *
@@ -306,6 +301,8 @@ export function createRequestPrep(input: {
     const mode = resolveFillMode();
     if (mode === "off") return null;
     if (!tools?.length) return null;
+    // The determined spans come from the template; a builder-only context has none.
+    if (!ctx.template) return null;
     // Seed support is selected by the execution binding after placement.
     // Injected tokens are never sampled — they have no logprob row.
     if (req.logprobs === true) return null;

package/src/{serve → server}/responses-client.ts

@@ -1,21 +1,32 @@
-import { ResponseStore, resolveResponsesConversation, type ResponseHistory, type ResponsesRequest } from "../responses";
+// The parent's Responses conversation history in the isolated server (main's
+// responses-client.ts): `previous_response_id` resolves against the parent's
+// store, the resolved conversation goes to the worker without one, and the
+// completed record is remembered here, so history survives worker restarts.
+// The worker keeps its own bounded store for the requests it answers; the
+// parent never consults it.
+import { ResponseStore, resolveResponsesConversation, type ResponseHistory, type ResponsesRequest } from "./responses";
+
+const responsesError = (status: number, message: string) => Response.json({ error: {
+  message, type: "invalid_request_error", param: null, code: null,
+} }, { status });
 
-/** Application-owned conversation history over a data-only HTTP host. Workers
- * translate/run the request; the parent stores only completed response records. */
 export function createResponsesClient(store: ResponseHistory = new ResponseStore()) {
   return {
     get stats() { return { entries: store.size, bytes: store.totalBytes, max_bytes: store.maxBytes, ttl_ms: store.ttlMs }; },
+    /** `forward` sends the rewritten request to the worker and returns its response as is. */
     async forward(request: Request, forward: (request: Request) => Promise<Response>): Promise<Response> {
-      const error = (status: number, message: string) => Response.json({ error: {
-        message, type: "invalid_request_error", param: null, code: null,
-      } }, { status });
       let body: ResponsesRequest;
       try { body = await request.json() as ResponsesRequest; }
-      catch { return error(400, "invalid JSON body"); }
-      if (!body || typeof body !== "object" || Array.isArray(body)) return error(400, "invalid JSON body");
+      catch { return responsesError(400, "invalid JSON body"); }
+      if (!body || typeof body !== "object" || Array.isArray(body)) return responsesError(400, "invalid JSON body");
+      // The container is validated before the previous id resolves, as the
+      // worker does: only a missing or expired previous response is a 404.
+      if ((body.input != null && typeof body.input !== "string" && !Array.isArray(body.input)) ||
+          (body.previous_response_id != null && typeof body.previous_response_id !== "string"))
+        return responsesError(400, "input must be a string or array; previous_response_id must be a string");
       let conversation: ReturnType<typeof resolveResponsesConversation>;
       try { conversation = resolveResponsesConversation(body, store); }
-      catch (e) { return error(404, (e as Error).message); }
+      catch (error) { return responsesError(404, error instanceof Error ? error.message : String(error)); }
       const headers = new Headers(request.headers);
       headers.delete("content-length");
       headers.set("x-mlx-bun-response-owner", "parent");
@@ -32,15 +43,20 @@ export function createResponsesClient(store: ResponseHistory = new ResponseStore
       if (!body.stream) {
         const value = await response.json() as Record<string, unknown>;
         remember(value);
-        const headers = new Headers(response.headers); headers.delete("content-length");
-        return Response.json(restorePrevious(value), { status: response.status, headers });
+        const outputHeaders = new Headers(response.headers);
+        outputHeaders.delete("content-length");
+        return Response.json(restorePrevious(value), { status: response.status, headers: outputHeaders });
       }
       if (!response.body) return response;
       const decoder = new TextDecoder(), encoder = new TextEncoder();
       let pending = "";
+      // Frames the worker did not shape as JSON (keep-alive comments, [DONE])
+      // pass through; `response.completed` is remembered and every response
+      // object gets the previous id the caller sent.
       const rewrite = (frame: string): string => frame.replace(/^data: (.+)$/m, (line, data: string) => {
         if (data === "[DONE]") return line;
-        const event = JSON.parse(data) as Record<string, unknown>;
+        let event: Record<string, unknown>;
+        try { event = JSON.parse(data) as Record<string, unknown>; } catch { return line; }
         if (event.response && typeof event.response === "object") {
           const value = event.response as Record<string, unknown>;
           if (event.type === "response.completed") remember(value);
@@ -48,7 +64,8 @@ export function createResponsesClient(store: ResponseHistory = new ResponseStore
         }
         return `data: ${JSON.stringify(event)}`;
       });
-      const outputHeaders = new Headers(response.headers); outputHeaders.delete("content-length");
+      const outputHeaders = new Headers(response.headers);
+      outputHeaders.delete("content-length");
       return new Response(response.body.pipeThrough(new TransformStream<Uint8Array, Uint8Array>({
         transform(chunk, controller) {
           pending += decoder.decode(chunk, { stream: true });

package/src/{responses.ts → server/responses.ts}

@@ -1,5 +1,5 @@
 // OpenAI Responses API translation layer + response store for the
-// mlx-bun server (Phase 11).
+// mlx-bun server.
 //
 // Ported from: mlx-optiq optiq/responses_shim.py (MIT) —
 // responses_to_openai_body / openai_to_responses_response /
@@ -20,7 +20,7 @@
 import type {
   CompletionEvent,
   CompletionStreamProtocol,
-} from "./serve/completion-sink";
+} from "./completion-sink";
 
 // ---------------------------------------------------------------------
 // Response store (previous_response_id resumption)
@@ -815,6 +815,7 @@ export function createResponsesStreamProtocol(
 ): CompletionStreamProtocol {
   const translator = new ResponsesStreamTranslator(model, previousResponseId);
   const chunk = (delta: Record<string, unknown>) => ({ choices: [{ delta }] });
+  let failure: string | undefined;
   return {
     start: () => translator.addChunk(chunk({ role: "assistant", content: "" })),
     addEvents(events: CompletionEvent[]) {
@@ -831,14 +832,19 @@ export function createResponsesStreamProtocol(
       });
     },
     finish(reason, usage) {
-      const frames = [
-        ...translator.addChunk({ choices: [{ delta: {}, finish_reason: reason }], usage }),
-        ...translator.finalize(),
-      ];
+      const frames = translator.addChunk({ choices: [{ delta: {}, finish_reason: reason }], usage });
+      // The shared sink finishes after reporting an error. Give clients a
+      // terminal response without saving failed output as resumable history.
+      if (failure !== undefined) return [...frames, sse("response.failed", {
+        type: "response.failed", response: { ...translator.finalResponse(), status: "failed",
+          error: { code: "server_error", message: failure } },
+      })];
+      frames.push(...translator.finalize());
       onComplete(translator.finalResponse());
       return frames;
     },
     error(message) {
+      failure = message;
       return [sse("error", {
         type: "error",
         code: "server_error",
@@ -848,90 +854,3 @@ export function createResponsesStreamProtocol(
     },
   };
 }
-
-/** Wrap our OpenAI SSE byte stream as a Responses SSE byte stream.
- *  onComplete fires after response.completed with the final payload so
- *  the route can store it for previous_response_id resumption. */
-export function translateOpenAiSseToResponses(
-  upstream: ReadableStream<Uint8Array>,
-  model: string,
-  previousResponseId: string | null,
-  onComplete: (final: Record<string, unknown>) => void,
-): ReadableStream<Uint8Array> {
-  const translator = new ResponsesStreamTranslator(model, previousResponseId);
-  const enc = new TextEncoder();
-  const dec = new TextDecoder();
-  let buf = "";
-  let finalized = false;
-  let reader: ReadableStreamDefaultReader<Uint8Array> | null = null;
-  let cancelled = false;
-  return new ReadableStream<Uint8Array>({
-    start(controller) {
-      const emit = (frames: string[]) => {
-        for (const f of frames) controller.enqueue(enc.encode(f));
-      };
-      const finish = () => {
-        if (finalized) return;
-        finalized = true;
-        emit(translator.finalize());
-        onComplete(translator.finalResponse());
-      };
-      reader = upstream.getReader();
-      void (async () => {
-        try {
-          while (!cancelled) {
-            const { done, value } = await reader!.read();
-            if (done) break;
-            buf += dec.decode(value, { stream: true });
-            let sep: number;
-            while ((sep = buf.indexOf("\n\n")) !== -1) {
-              const frame = buf.slice(0, sep);
-              buf = buf.slice(sep + 2);
-              const data = frame
-                .split("\n")
-                .find((l) => l.startsWith("data: "))
-                ?.slice(6);
-              if (data == null) continue;
-              if (data === "[DONE]" || data === '"[DONE]"') {
-                finish();
-                continue;
-              }
-              const parsed = JSON.parse(data);
-              if (parsed?.error) {
-                emit([
-                  sse("error", {
-                    type: "error",
-                    code: "server_error",
-                    message: parsed.error.message ?? "generation failed",
-                    param: null,
-                  }),
-                ]);
-                continue;
-              }
-              emit(translator.addChunk(parsed));
-            }
-          }
-          if (!cancelled) finish();
-        } catch (e) {
-          if (!cancelled)
-            emit([
-              sse("error", {
-                type: "error",
-                code: "server_error",
-                message: (e as Error).message,
-                param: null,
-              }),
-            ]);
-        } finally {
-          reader?.releaseLock();
-          reader = null;
-          if (!cancelled) controller.close();
-        }
-      })();
-    },
-    async cancel(reason) {
-      cancelled = true;
-      await reader?.cancel(reason);
-    },
-  });
-}