@jxsuite/ai 0.37.2 → 0.38.0

package/src/gateway/index.ts

@@ -0,0 +1,26 @@
+/**
+ * `@jxsuite/ai/gateway`: the server half of the `ai/chat` and `ai/models` routes, written once.
+ *
+ * Every Jx backend that proxies the assistant to a provider does the same wire work: read the
+ * client's body, forward it as an OpenAI-compatible request, normalize the provider's stream into
+ * the route's frames, and frame those as SSE. Before this module each backend carried its own copy,
+ * and the copies drifted. Now `@jxsuite/server` runs this one, and any other backend can.
+ *
+ * What stays with the host is policy: whose key a request may use, which base URLs are safe to
+ * reach, how requests are admitted, and what the model catalogue says. The host supplies those as
+ * `ChatGatewayOptions`, and the gateway never second-guesses them.
+ *
+ * Worker-safe: it imports `@jxsuite/protocol`'s problem modules and nothing host-bound
+ * (`tests/worker-safety.test.ts`, `tsconfig.worker.json`).
+ *
+ * @module @jxsuite/ai/gateway
+ * @docs extending/embedding/backend-protocol
+ */
+
+export { createChatHandler } from "./chat.ts";
+export { modelsResponse } from "./models.ts";
+export { normalizeOpenAIStream } from "./normalize.ts";
+export { problemResponse } from "./problem.ts";
+export { encodeSse } from "./sse.ts";
+export { extractUpstreamErrorMessage } from "./upstream-error.ts";
+export type { ChatAdmission, ChatGatewayOptions, GatewayRefusal, Upstream } from "./types.ts";

package/src/gateway/models.ts

@@ -0,0 +1,22 @@
+/**
+ * The models route's answer.
+ *
+ * @module @jxsuite/ai/gateway
+ */
+
+import type { AiModelsResponse } from "@jxsuite/protocol/types";
+
+/**
+ * Answer the models route with `body`, always at `200`.
+ *
+ * The models route is a capability probe (ai.md §2.1), so every credential state is a 200: "no
+ * credentials", "credentials lapsed" and "the upstream could not list" are all answers the client
+ * reads, and a non-2xx makes the probe throw. Which catalogue to answer with is the host's business
+ * (its defaults, the upstream's listing, a brokered one); this only fixes the envelope.
+ *
+ * @param {AiModelsResponse} body - The catalogue and the credential state
+ * @returns {Response}
+ */
+export function modelsResponse(body: AiModelsResponse): Response {
+  return Response.json(body, { headers: { "Content-Type": "application/json" }, status: 200 });
+}

package/src/gateway/normalize.ts

@@ -0,0 +1,302 @@
+/**
+ * The OpenAI-compatible stream normalizer: an upstream chat-completions response in, the chat
+ * route's frames out.
+ *
+ * This is `@jxsuite/server`'s normalizer, moved here verbatim (specs/ai.md §2.4), so that every
+ * backend serving the chat route can run the same one. Its frames are frozen by the server's
+ * upstream fixtures (`packages/server/tests/fixtures/ai-upstream/*.server.json`); a change to what
+ * this file yields is a behaviour change and belongs in a change whose spec fragment names it.
+ *
+ * @module @jxsuite/ai/gateway
+ */
+
+import { problemDetails } from "@jxsuite/protocol/problems";
+import type { StreamEvent, StreamUsageEvent } from "../streaming-client.ts";
+import { problemTypeForStatus } from "./problem.ts";
+import { extractUpstreamErrorMessage } from "./upstream-error.ts";
+
+/** A tool-call fragment inside an OpenAI streaming `delta`. */
+interface ToolCallDelta {
+  index: number;
+  id?: string;
+  type?: string;
+  function?: { name?: string; arguments?: string };
+}
+
+/** A single OpenAI chat-completions streaming chunk (the JSON after `data: `). */
+interface OpenAIStreamChunk {
+  choices?: {
+    index?: number;
+    delta?: {
+      content?: string;
+      /** Thinking models' chain-of-thought: DeepSeek and Volcengine spell it this way... */
+      reasoning_content?: string;
+      /** ...and OpenRouter this way. Both carry the same thing. */
+      reasoning?: string;
+      tool_calls?: ToolCallDelta[];
+    };
+    finish_reason?: string | null;
+  }[];
+  /** The count `stream_options.include_usage` asks for, normally a final chunk with no choices. */
+  usage?: {
+    prompt_tokens?: number;
+    completion_tokens?: number;
+    prompt_tokens_details?: { cached_tokens?: number } | null;
+    completion_tokens_details?: { reasoning_tokens?: number } | null;
+  } | null;
+}
+
+/**
+ * Normalize an upstream usage object into a `usage` frame, or null when it carries no count. The
+ * detail figures are included only when the provider sent them: absent is "not said", not zero.
+ *
+ * @param {OpenAIStreamChunk["usage"]} usage - The chunk's `usage` member
+ * @returns {StreamUsageEvent | null}
+ */
+function usageFrame(usage: OpenAIStreamChunk["usage"]): StreamUsageEvent | null {
+  if (!usage || typeof usage.prompt_tokens !== "number") {
+    return null;
+  }
+  const frame: StreamUsageEvent = {
+    type: "usage",
+    inputTokens: usage.prompt_tokens,
+    outputTokens: usage.completion_tokens ?? 0,
+  };
+  const cached = usage.prompt_tokens_details?.cached_tokens;
+  if (typeof cached === "number") {
+    frame.cachedInputTokens = cached;
+  }
+  const reasoning = usage.completion_tokens_details?.reasoning_tokens;
+  if (typeof reasoning === "number") {
+    frame.reasoningTokens = reasoning;
+  }
+  return frame;
+}
+
+/**
+ * The stop reasons the stream reports; any other `finish_reason` keeps it reading.
+ *
+ * @param {string | null | undefined} reason - The chunk's `finish_reason`
+ * @returns {string | null}
+ */
+function stopReasonOf(reason: string | null | undefined): string | null {
+  if (reason === "tool_calls" || reason === "length" || reason === "stop") {
+    return reason;
+  }
+  return null;
+}
+
+/**
+ * The frame for an upstream request that never got a response: `done: cancelled` when it was
+ * aborted, otherwise an `error` frame carrying an `upstreamFailure` problem.
+ *
+ * The frame carries a problem rather than being one: the response began with a 200 long before this
+ * failed, so nothing can change the status now. `message` stays for the readers that already show
+ * it (server.md §4.3).
+ *
+ * @param {unknown} error - What `fetch` rejected with
+ * @returns {StreamEvent}
+ */
+export function fetchFailureFrame(error: unknown): StreamEvent {
+  if ((error as Error).name === "AbortError") {
+    return { type: "done", stopReason: "cancelled" };
+  }
+  const message = `Network error: ${(error as Error).message}`;
+  return { message, problem: problemDetails("upstreamFailure", message), type: "error" };
+}
+
+/**
+ * Normalize an OpenAI-compatible chat-completions response into the chat route's frames.
+ *
+ * - A non-2xx response is one `error` frame: the sentence inside the provider's body as `message`,
+ *   the upstream's own status as `code`, and a problem naming the kind of failure.
+ * - Text is `delta`; chain-of-thought (`reasoning_content`, or `reasoning` at OpenRouter) is
+ *   `reasoning`, forwarded rather than dropped, because DeepSeek's thinking mode requires every
+ *   prior turn's reasoning back on any request carrying `tools`.
+ * - A tool call is `tool_call_start`, then `tool_call_delta` per fragment, then `tool_call_end`.
+ * - The finish reason is REMEMBERED, not acted on: with `include_usage` the count arrives in a chunk
+ *   of its own after the finish, so the stream ends at `[DONE]` or when the body closes. The last
+ *   frames are the open calls' ends, the `usage` count when there is one, then `done`.
+ * - An aborted read is `done: cancelled`; any other failed read is an `error` frame.
+ * - A consumer that stops reading early cancels the upstream body.
+ *
+ * @param {Response} response - The upstream's response to the chat-completions request
+ * @yields {StreamEvent} The route's frames, in wire order
+ */
+export async function* normalizeOpenAIStream(response: Response): AsyncGenerator<StreamEvent> {
+  if (!response.ok) {
+    let errorBody = "";
+    try {
+      errorBody = await response.text();
+    } catch {
+      /* Ignore */
+    }
+    const cleanMessage = extractUpstreamErrorMessage(errorBody, response.statusText);
+    yield {
+      code: String(response.status),
+      message: cleanMessage,
+      // The upstream's own status is preserved in `code`; the problem names the KIND.
+      problem: problemDetails(problemTypeForStatus(response.status), cleanMessage),
+      type: "error",
+    };
+    return;
+  }
+
+  const reader = response.body?.getReader();
+  if (!reader) {
+    yield {
+      message: "No response body from upstream",
+      problem: problemDetails("upstreamFailure", "No response body from upstream"),
+      type: "error",
+    };
+    return;
+  }
+
+  const decoder = new TextDecoder();
+  let buffer = "";
+
+  const pendingToolCalls = new Map<number, { id: string; name: string; args: string }>();
+  let stopReason: string | null = null;
+  let usage: StreamUsageEvent | null = null;
+
+  /**
+   * Close every open call.
+   *
+   * @yields {StreamEvent} A `tool_call_end` per open call
+   */
+  function* closePendingToolCalls(): Generator<StreamEvent> {
+    for (const [, tc] of pendingToolCalls) {
+      yield { type: "tool_call_end", id: tc.id };
+    }
+    pendingToolCalls.clear();
+  }
+
+  /**
+   * The stream's last frames: open calls, the count when there is one, then `done`. The count comes
+   * BEFORE `done`, because a reader stops at `done`.
+   *
+   * @yields {StreamEvent} The closing frames, in order
+   */
+  function* finish(): Generator<StreamEvent> {
+    yield* closePendingToolCalls();
+    if (usage) {
+      yield usage;
+    }
+    yield { type: "done", stopReason: stopReason ?? "stop" };
+  }
+
+  /* Whether the loop ended on its own terms. A consumer that stops reading ends a generator through
+     `return()`, which runs `finally` and nothing else; when the server wrote each frame itself, that
+     exit was the next write throwing inside this `try`, so the `catch` cancelled the upstream body.
+     `finally` keeps that: the upstream must not outlive the reader it streams to. */
+  let settled = false;
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) {
+        break;
+      }
+
+      buffer += decoder.decode(value, { stream: true });
+      const lines = buffer.split("\n");
+      buffer = lines.pop() || "";
+
+      for (const line of lines) {
+        const trimmed = line.trim();
+        if (!trimmed || !trimmed.startsWith("data: ")) {
+          continue;
+        }
+
+        const dataStr = trimmed.slice(6);
+        if (dataStr === "[DONE]") {
+          yield* finish();
+          settled = true;
+          return;
+        }
+
+        let parsed: OpenAIStreamChunk;
+        try {
+          parsed = JSON.parse(dataStr) as OpenAIStreamChunk;
+        } catch {
+          continue;
+        }
+
+        // Usually a final chunk with no choices, but a provider may put it beside the finish.
+        usage = usageFrame(parsed.usage) ?? usage;
+
+        const choice = parsed.choices?.[0];
+        if (!choice) {
+          continue;
+        }
+
+        const { delta } = choice;
+
+        // Text content
+        if (delta?.content) {
+          yield { type: "delta", content: delta.content };
+        }
+
+        /* Chain-of-thought, under either of the two names providers give it. Forwarded rather than
+           dropped: DeepSeek's thinking mode requires every prior turn's reasoning back on any
+           request carrying `tools`, so a proxy that swallows these frames makes the NEXT round a
+           400 the client cannot repair. */
+        const reasoning = delta?.reasoning_content ?? delta?.reasoning;
+        if (typeof reasoning === "string" && reasoning) {
+          yield { type: "reasoning", content: reasoning };
+        }
+
+        // Tool calls
+        if (delta?.tool_calls) {
+          for (const tc of delta.tool_calls) {
+            if (tc.id) {
+              // First appearance
+              const entry = {
+                id: tc.id,
+                name: tc.function?.name || "",
+                args: tc.function?.arguments || "",
+              };
+              pendingToolCalls.set(tc.index, entry);
+
+              yield { type: "tool_call_start", id: tc.id, name: entry.name };
+
+              if (entry.args) {
+                yield { type: "tool_call_delta", id: tc.id, args: entry.args };
+              }
+            } else if (tc.function?.arguments) {
+              // Subsequent fragment
+              const existing = pendingToolCalls.get(tc.index);
+              if (existing) {
+                existing.args += tc.function.arguments;
+                yield { type: "tool_call_delta", id: existing.id, args: tc.function.arguments };
+              }
+            }
+          }
+        }
+
+        // Finish reason: close the calls now, report the reason at the end of the stream.
+        const finished = stopReasonOf(choice.finish_reason);
+        if (finished) {
+          yield* closePendingToolCalls();
+          stopReason = finished;
+        }
+      }
+    }
+
+    // Stream ended without `[DONE]`
+    yield* finish();
+    settled = true;
+  } catch (error) {
+    settled = true;
+    void reader.cancel();
+    if ((error as Error).name === "AbortError") {
+      yield { type: "done", stopReason: "cancelled" };
+      return;
+    }
+    const message = `Stream error: ${(error as Error).message}`;
+    yield { message, problem: problemDetails("upstreamFailure", message), type: "error" };
+  } finally {
+    if (!settled) {
+      void reader.cancel();
+    }
+  }
+}

package/src/gateway/problem.ts

@@ -0,0 +1,80 @@
+/**
+ * How the gateway says no: an RFC 9457 problem response, and the problem type an upstream's status
+ * maps onto.
+ *
+ * @module @jxsuite/ai/gateway
+ */
+
+import { PROBLEM_MEDIA_TYPE } from "@jxsuite/protocol/problem";
+import { problemDetails } from "@jxsuite/protocol/problems";
+import type { ProblemTypeName } from "@jxsuite/protocol/problems";
+import type { GatewayRefusal } from "./types.ts";
+
+/**
+ * A refusal of one declared problem type, answered with that type's own status.
+ *
+ * @param {ProblemTypeName} name - A declared type from `PROBLEM_TYPES`
+ * @param {string} detail - What happened this time; the line a human reads
+ * @returns {GatewayRefusal}
+ */
+export function refusal(name: ProblemTypeName, detail: string): GatewayRefusal {
+  const problem = problemDetails(name, detail);
+  return { problem, status: problem.status };
+}
+
+/**
+ * The response a refusal is answered with: `application/problem+json` at the refusal's status.
+ *
+ * One body satisfies both kinds of client. A problem reader keys on `type` and shows `detail`; an
+ * older reader shows `error` and acts on `code`. So `error` is present whenever `detail` is (it
+ * already is on a problem built by `problemDetails`, and is added here to one built by hand), and
+ * the refusal's `code`, when it has one, rides beside them.
+ *
+ * @param {GatewayRefusal} refused - What was refused, and why
+ * @returns {Response}
+ */
+export function problemResponse(refused: GatewayRefusal): Response {
+  const { code, problem, status } = refused;
+  const body = {
+    ...problem,
+    ...(problem.error === undefined && problem.detail !== undefined
+      ? { error: problem.detail }
+      : {}),
+    ...(code === undefined ? {} : { code }),
+  };
+  // Response.json would send `application/json`; RFC 9457 §3 is what says this body is a problem.
+  return Response.json(body, { headers: { "Content-Type": PROBLEM_MEDIA_TYPE }, status });
+}
+
+/**
+ * The declared type for an upstream provider's status.
+ *
+ * The normalizer needs it for the one failure that arrives with a status of its own: an upstream
+ * that answered non-2xx. The status itself is kept (as the frame's `code`), and the problem names
+ * the KIND of failure, which is the part a client can act on. `@jxsuite/server`'s `problem.ts`
+ * carries the same table for its own routes; `packages/server/tests/ai-api-gateway.test.ts` holds
+ * the two to agreement.
+ *
+ * @param {number} status - The upstream's HTTP status
+ * @returns {ProblemTypeName}
+ */
+export function problemTypeForStatus(status: number): ProblemTypeName {
+  /* Any other status still becomes SOME type, and which side of 500 it falls on is what decides
+     whether the caller is being told it made a mistake or that the upstream did. */
+  return STATUS_TYPES[status] ?? (status >= 500 ? "internalError" : "invalidRequest");
+}
+
+/** The statuses the table distinguishes, each onto a type answered with that same status. */
+const STATUS_TYPES: Readonly<Record<number, ProblemTypeName>> = {
+  400: "invalidRequest",
+  401: "unauthorized",
+  403: "forbidden",
+  404: "notFound",
+  405: "methodNotAllowed",
+  409: "conflict",
+  413: "payloadTooLarge",
+  501: "capabilityUnavailable",
+  502: "upstreamFailure",
+  503: "upstreamFailure",
+  504: "upstreamFailure",
+};

package/src/gateway/sse.ts

@@ -0,0 +1,42 @@
+/**
+ * The chat route's framing: one Server-Sent Events `data:` line per frame.
+ *
+ * @module @jxsuite/ai/gateway
+ */
+
+import type { StreamEvent } from "../streaming-client.ts";
+
+/**
+ * The headers a chat stream is answered with. `no-cache` because a replayed stream is a different
+ * turn, and `keep-alive` because the answer is long-lived by design.
+ */
+export const SSE_HEADERS: Readonly<Record<string, string>> = {
+  "Content-Type": "text/event-stream",
+  "Cache-Control": "no-cache",
+  Connection: "keep-alive",
+};
+
+/**
+ * Frame `events` as a Server-Sent Events body: `data: <JSON>` and a blank line per frame, and
+ * nothing else (no `event:`, `id:` or `retry:` field), so the JSON's own `type` is the only
+ * discriminator a reader needs.
+ *
+ * The stream is PUSHED: iteration starts when the stream is constructed, not when it is first read,
+ * and every frame is enqueued as soon as it exists. That is how the proxy has always behaved (its
+ * upstream request is on the wire before the response is returned), and this extraction keeps it.
+ * An iterable that throws errors the stream.
+ *
+ * @param {AsyncIterable<StreamEvent>} events - The frames, in wire order
+ * @returns {ReadableStream<Uint8Array>}
+ */
+export function encodeSse(events: AsyncIterable<StreamEvent>): ReadableStream<Uint8Array> {
+  const encoder = new TextEncoder();
+  return new ReadableStream<Uint8Array>({
+    async start(controller) {
+      for await (const event of events) {
+        controller.enqueue(encoder.encode(`data: ${JSON.stringify(event)}\n\n`));
+      }
+      controller.close();
+    },
+  });
+}

package/src/gateway/types.ts

@@ -0,0 +1,80 @@
+/**
+ * The gateway's contract types: what a host hands `createChatHandler`, and what the handler reports
+ * back to it.
+ *
+ * The split these types draw is the point of the gateway. The WIRE (reading the body, shaping the
+ * upstream request, normalizing the upstream stream, framing SSE, rendering a refusal) is the same
+ * for every host, so it lives here once. The POLICY (whose key a request may use, which base URLs
+ * are safe to reach, how many requests a caller may make) differs per host, so the host supplies it
+ * as `admit` and `resolveUpstream` and the gateway never second-guesses the answer.
+ *
+ * Type-only, so it is on the allowlist in `scripts/check-coverage-manifest.ts`.
+ *
+ * @module @jxsuite/ai/gateway
+ */
+
+import type { ProblemDetails } from "@jxsuite/protocol/problem";
+
+/** Where one chat request is forwarded. A host decides it per request, in `resolveUpstream`. */
+export interface Upstream {
+  /** The wire the upstream speaks. The OpenAI-compatible chat-completions API is the only one. */
+  readonly family: "openai-compat";
+  /** The provider's base URL, without the `/chat/completions` the gateway appends. */
+  readonly baseUrl: string;
+  /** Sent as a bearer token in the `Authorization` header. */
+  readonly apiKey: string;
+  /** The model a request that names none is forwarded with. */
+  readonly defaultModel: string;
+  /** True when the platform brokers the credential, rather than the user supplying one. */
+  readonly managed: boolean;
+}
+
+/**
+ * A request the gateway (or the host's policy) declined, answered as `application/problem+json`
+ * before any upstream is contacted.
+ */
+export interface GatewayRefusal {
+  /** The HTTP status of the refusal. */
+  readonly status: number;
+  /** The RFC 9457 problem document the body carries. */
+  readonly problem: ProblemDetails;
+  /** A machine code for clients that key on `code` rather than on the problem type. */
+  readonly code?: string;
+}
+
+/** What a request the gateway accepted asked for, reported to `onAccepted` for audit. */
+export interface ChatAdmission {
+  /** The wire version of the accepted body. Version 1 is the only one. */
+  readonly wire: 1;
+  /** The model the request is forwarded with: the body's, or the upstream's default. */
+  readonly model: string;
+  /** How many messages the body carried, not counting the system prompt the gateway prepends. */
+  readonly messageCount: number;
+  /** The body's size in bytes, as read rather than as declared. */
+  readonly bytes: number;
+}
+
+/** A host's half of the chat route. Only `resolveUpstream` is required. */
+export interface ChatGatewayOptions<C> {
+  /** Runs first, before the body is read (a rate limit, say). A refusal ends the request. */
+  readonly admit?: (
+    request: Request,
+    context: C,
+  ) => Promise<GatewayRefusal | null> | GatewayRefusal | null;
+  /**
+   * Where the request goes, or why it goes nowhere. This is the host's policy: key provenance and
+   * any guard on a caller-supplied base URL belong here, not in the gateway.
+   */
+  readonly resolveUpstream: (
+    request: Request,
+    context: C,
+  ) => Promise<Upstream | GatewayRefusal> | Upstream | GatewayRefusal;
+  /** The largest body accepted, in bytes actually read. Reading stops once it is passed. */
+  readonly maxBodyBytes?: number;
+  /** The most messages one request may carry. */
+  readonly maxMessages?: number;
+  /** Called once per accepted request, after every check and before the upstream is contacted. */
+  readonly onAccepted?: (admission: ChatAdmission, upstream: Upstream, context: C) => void;
+  /** The fetch the upstream is called with. The global `fetch`, looked up per call, when absent. */
+  readonly fetch?: typeof fetch;
+}

package/src/gateway/upstream-error.ts

@@ -0,0 +1,54 @@
+/**
+ * The sentence inside an upstream provider's error body.
+ *
+ * @module @jxsuite/ai/gateway
+ */
+
+/**
+ * Extract a human-readable message from an upstream error body, falling back to the raw body (or
+ * `fallback` when the body is empty).
+ *
+ * OpenAI-compatible providers return `{ error: "..." }` or `{ error: { message: "..." } }`.
+ * Cloudflare's own REST API (what a BYOK base URL pointed at Workers AI actually returns) uses a
+ * different envelope, `{ success: false, errors: [{ code, message }] }`, with no top-level `error`
+ * key at all. Without this branch that shape parses as valid JSON with nothing matched, and the
+ * caller was left showing the whole raw body (e.g. the full `{"success":false,"errors":[...]}`
+ * blob) instead of the human sentence inside it.
+ *
+ * The chat stream's `error` frame reads it, and so does a host's model catalogue, which reports the
+ * same sentence as `upstreamMessage`.
+ *
+ * @param {string} rawBody - The upstream's response body, as text
+ * @param {string} fallback - What to say when the body is empty (the status text, usually)
+ * @returns {string}
+ */
+export function extractUpstreamErrorMessage(rawBody: string, fallback: string): string {
+  if (!rawBody) {
+    return fallback;
+  }
+  try {
+    const { error, errors } = JSON.parse(rawBody) as {
+      error?: string | { message?: string };
+      errors?: { code?: number; message?: string }[];
+    };
+    if (typeof error === "string") {
+      return error;
+    }
+    if (error) {
+      const { message } = error;
+      if (message) {
+        return message;
+      }
+    }
+    const [first] = errors ?? [];
+    if (first) {
+      const { message } = first;
+      if (message) {
+        return message;
+      }
+    }
+  } catch {
+    /* Not JSON: use the raw body. */
+  }
+  return rawBody;
+}

package/src/index.ts

@@ -13,6 +13,8 @@ export { createToolDefinition, createToolRegistry } from "./tools.js";
 
 export { createChatState } from "./chat-state.js";
 
+export type { ChatUsage } from "./chat-state.js";
+
 export {
   STREAM_EVENT_TYPES,
   createOpenAIStreamingClient,
@@ -33,4 +35,5 @@ export type {
   StreamToolCallDeltaEvent,
   StreamToolCallEndEvent,
   StreamToolCallStartEvent,
+  StreamUsageEvent,
 } from "./streaming-client.js";