@jxsuite/ai 0.37.2 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -14
- package/package.json +5 -3
- package/src/cache-hints.ts +111 -0
- package/src/chat-state.ts +100 -10
- package/src/core-types.ts +97 -0
- package/src/gateway/chat.ts +224 -0
- package/src/gateway/index.ts +33 -0
- package/src/gateway/models.ts +22 -0
- package/src/gateway/normalize.ts +302 -0
- package/src/gateway/problem.ts +80 -0
- package/src/gateway/sse.ts +42 -0
- package/src/gateway/types.ts +87 -0
- package/src/gateway/upstream-error.ts +54 -0
- package/src/index.ts +14 -2
- package/src/streaming-client.ts +182 -49
- package/src/tools.ts +153 -14
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The chat route: a request from a Jx client in, an SSE stream of normalized frames out.
|
|
3
|
+
*
|
|
4
|
+
* @module @jxsuite/ai/gateway
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { upstreamCacheHints } from "../cache-hints.ts";
|
|
8
|
+
import type { StreamEvent } from "../streaming-client.ts";
|
|
9
|
+
import { fetchFailureFrame, normalizeOpenAIStream } from "./normalize.ts";
|
|
10
|
+
import { problemResponse, refusal } from "./problem.ts";
|
|
11
|
+
import { encodeSse, SSE_HEADERS } from "./sse.ts";
|
|
12
|
+
import type { ChatGatewayOptions, GatewayRefusal, Upstream } from "./types.ts";
|
|
13
|
+
|
|
14
|
+
/** The refusal's detail for a body that is not JSON. */
|
|
15
|
+
const INVALID_JSON = "Invalid JSON body";
|
|
16
|
+
|
|
17
|
+
/** The version 1 chat body, as a client sends it. Every member is optional on the wire. */
|
|
18
|
+
interface ChatRequestBody {
|
|
19
|
+
messages?: unknown;
|
|
20
|
+
tools?: unknown;
|
|
21
|
+
systemPrompt?: unknown;
|
|
22
|
+
model?: unknown;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** The chat-completions body forwarded upstream. */
|
|
26
|
+
interface UpstreamBody {
|
|
27
|
+
model: unknown;
|
|
28
|
+
messages: unknown[];
|
|
29
|
+
stream: boolean;
|
|
30
|
+
stream_options: { include_usage: boolean };
|
|
31
|
+
tools?: unknown;
|
|
32
|
+
tool_choice?: string;
|
|
33
|
+
parallel_tool_calls?: boolean;
|
|
34
|
+
prompt_cache_key?: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Whether `resolveUpstream` answered with a refusal rather than an upstream.
|
|
39
|
+
*
|
|
40
|
+
* @param {Upstream | GatewayRefusal} value - What `resolveUpstream` returned
|
|
41
|
+
* @returns {boolean}
|
|
42
|
+
*/
|
|
43
|
+
function isRefusal(value: Upstream | GatewayRefusal): value is GatewayRefusal {
|
|
44
|
+
return "problem" in value;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Read the request body to the end, or until it passes `limit` bytes, in which case the rest is
|
|
49
|
+
* never read and the result is null. Measured on the bytes that actually arrive, because a
|
|
50
|
+
* `Content-Length` is only a claim.
|
|
51
|
+
*
|
|
52
|
+
* @param {Request} request - The client's request
|
|
53
|
+
* @param {number | undefined} limit - The most bytes accepted, or undefined for no limit
|
|
54
|
+
* @returns {Promise<Uint8Array | null>}
|
|
55
|
+
*/
|
|
56
|
+
async function readBody(request: Request, limit: number | undefined): Promise<Uint8Array | null> {
|
|
57
|
+
const chunks: Uint8Array[] = [];
|
|
58
|
+
let length = 0;
|
|
59
|
+
const reader = request.body?.getReader();
|
|
60
|
+
if (reader) {
|
|
61
|
+
for (let chunk = await reader.read(); !chunk.done; chunk = await reader.read()) {
|
|
62
|
+
length += chunk.value.byteLength;
|
|
63
|
+
if (limit !== undefined && length > limit) {
|
|
64
|
+
void reader.cancel();
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
chunks.push(chunk.value);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
const bytes = new Uint8Array(length);
|
|
71
|
+
let offset = 0;
|
|
72
|
+
for (const chunk of chunks) {
|
|
73
|
+
bytes.set(chunk, offset);
|
|
74
|
+
offset += chunk.byteLength;
|
|
75
|
+
}
|
|
76
|
+
return bytes;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* The frames of one upstream exchange: the request, then its normalized response, or the one frame
|
|
81
|
+
* that says why there is no response.
|
|
82
|
+
*
|
|
83
|
+
* @param {typeof fetch} upstreamFetch - The fetch to call the upstream with
|
|
84
|
+
* @param {string} url - The upstream's chat-completions URL
|
|
85
|
+
* @param {RequestInit} init - The upstream request
|
|
86
|
+
* @yields {StreamEvent} The route's frames, in wire order
|
|
87
|
+
*/
|
|
88
|
+
async function* exchange(
|
|
89
|
+
upstreamFetch: typeof fetch,
|
|
90
|
+
url: string,
|
|
91
|
+
init: RequestInit,
|
|
92
|
+
): AsyncGenerator<StreamEvent> {
|
|
93
|
+
let response: Response;
|
|
94
|
+
try {
|
|
95
|
+
response = await upstreamFetch(url, init);
|
|
96
|
+
} catch (error) {
|
|
97
|
+
yield fetchFailureFrame(error);
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
yield* normalizeOpenAIStream(response);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Build the chat route's handler around a host's policy.
|
|
105
|
+
*
|
|
106
|
+
* Each request goes through the same steps in the same order, and the first one that refuses
|
|
107
|
+
* answers it as `application/problem+json` without contacting the upstream:
|
|
108
|
+
*
|
|
109
|
+
* 1. `admit`, before the body is read.
|
|
110
|
+
* 2. `resolveUpstream`: where the request goes, or why it goes nowhere.
|
|
111
|
+
* 3. The body is read, stopping at `maxBodyBytes` (413), and parsed as JSON (400 when it is not).
|
|
112
|
+
* 4. `messages` must be an array (400); its absence reads as an empty one.
|
|
113
|
+
* 5. `maxMessages` (413).
|
|
114
|
+
* 6. `onAccepted`.
|
|
115
|
+
* 7. The stream: a `200 text/event-stream` whose frames are the upstream's response normalized by
|
|
116
|
+
* `normalizeOpenAIStream`, or the one frame saying why the upstream could not be reached.
|
|
117
|
+
*
|
|
118
|
+
* The version 1 body is forwarded as the OpenAI-compatible request it describes: the system prompt
|
|
119
|
+
* as a leading `system` message, `stream_options.include_usage` so the provider's own token count
|
|
120
|
+
* arrives, and `tool_choice: "auto"` with parallel calls whenever tools are offered. When the host
|
|
121
|
+
* resolved a `sessionAffinity`, it rides along as the prompt-cache hint the upstream's host
|
|
122
|
+
* understands, and not at all to any other host (`upstreamCacheHints`). The upstream request
|
|
123
|
+
* carries `context.signal`, so aborting it ends the stream with `done: cancelled`.
|
|
124
|
+
*
|
|
125
|
+
* @param {ChatGatewayOptions<C>} options - The host's policy
|
|
126
|
+
* @returns {(request: Request, context: C) => Promise<Response>}
|
|
127
|
+
*/
|
|
128
|
+
export function createChatHandler<C extends { readonly signal: AbortSignal }>(
|
|
129
|
+
options: ChatGatewayOptions<C>,
|
|
130
|
+
): (request: Request, context: C) => Promise<Response> {
|
|
131
|
+
return async (request, context) => {
|
|
132
|
+
const admitted = await options.admit?.(request, context);
|
|
133
|
+
if (admitted) {
|
|
134
|
+
return problemResponse(admitted);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
const upstream = await options.resolveUpstream(request, context);
|
|
138
|
+
if (isRefusal(upstream)) {
|
|
139
|
+
return problemResponse(upstream);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// A body that cannot be read is answered as one that cannot be parsed, as it always was.
|
|
143
|
+
let bytes: Uint8Array | null;
|
|
144
|
+
try {
|
|
145
|
+
bytes = await readBody(request, options.maxBodyBytes);
|
|
146
|
+
} catch {
|
|
147
|
+
return problemResponse(refusal("invalidRequest", INVALID_JSON));
|
|
148
|
+
}
|
|
149
|
+
if (bytes === null) {
|
|
150
|
+
return problemResponse(
|
|
151
|
+
refusal(
|
|
152
|
+
"payloadTooLarge",
|
|
153
|
+
`The request body is larger than this gateway accepts (${String(options.maxBodyBytes)} bytes).`,
|
|
154
|
+
),
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
let body: ChatRequestBody;
|
|
158
|
+
try {
|
|
159
|
+
/* `ignoreBOM` keeps a leading U+FEFF in the text, so a body that starts with one is invalid
|
|
160
|
+
JSON, as it was when the server read it with Bun's `req.json()`. The default decoder would
|
|
161
|
+
strip it and accept the body; this extraction changes no byte a client receives. */
|
|
162
|
+
const text = new TextDecoder("utf-8", { fatal: false, ignoreBOM: true }).decode(bytes);
|
|
163
|
+
body = JSON.parse(text) as ChatRequestBody;
|
|
164
|
+
} catch {
|
|
165
|
+
return problemResponse(refusal("invalidRequest", INVALID_JSON));
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/* Defaults apply to an ABSENT member only, so a `null` one is forwarded as sent (and a `null`
|
|
169
|
+
`messages` is refused below). A body of JSON `null` throws here, as it always has. */
|
|
170
|
+
const { messages = [], tools = [], systemPrompt = "", model = upstream.defaultModel } = body;
|
|
171
|
+
|
|
172
|
+
if (!Array.isArray(messages)) {
|
|
173
|
+
return problemResponse(refusal("invalidRequest", "messages must be an array"));
|
|
174
|
+
}
|
|
175
|
+
if (options.maxMessages !== undefined && messages.length > options.maxMessages) {
|
|
176
|
+
return problemResponse(
|
|
177
|
+
refusal(
|
|
178
|
+
"payloadTooLarge",
|
|
179
|
+
`The request carries ${String(messages.length)} messages; this gateway accepts at most ${String(options.maxMessages)}.`,
|
|
180
|
+
),
|
|
181
|
+
);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
options.onAccepted?.(
|
|
185
|
+
{
|
|
186
|
+
bytes: bytes.byteLength,
|
|
187
|
+
messageCount: messages.length,
|
|
188
|
+
model: String(model),
|
|
189
|
+
wire: 1,
|
|
190
|
+
},
|
|
191
|
+
upstream,
|
|
192
|
+
context,
|
|
193
|
+
);
|
|
194
|
+
|
|
195
|
+
const upstreamBody: UpstreamBody = {
|
|
196
|
+
model,
|
|
197
|
+
messages: [{ role: "system", content: systemPrompt }, ...(messages as unknown[])],
|
|
198
|
+
stream: true,
|
|
199
|
+
stream_options: { include_usage: true },
|
|
200
|
+
};
|
|
201
|
+
// The wire has always forwarded `tools` as sent, so a non-array with a length passes through.
|
|
202
|
+
if (tools && (tools as { length: number }).length > 0) {
|
|
203
|
+
upstreamBody.tools = tools;
|
|
204
|
+
upstreamBody.tool_choice = "auto";
|
|
205
|
+
upstreamBody.parallel_tool_calls = true;
|
|
206
|
+
}
|
|
207
|
+
const hints = upstreamCacheHints(upstream.baseUrl, upstream.sessionAffinity);
|
|
208
|
+
Object.assign(upstreamBody, hints.body);
|
|
209
|
+
|
|
210
|
+
// Looked up per request, not captured at construction: a host or a test may swap it.
|
|
211
|
+
const upstreamFetch = options.fetch ?? fetch;
|
|
212
|
+
const frames = exchange(upstreamFetch, `${upstream.baseUrl}/chat/completions`, {
|
|
213
|
+
method: "POST",
|
|
214
|
+
headers: {
|
|
215
|
+
"Content-Type": "application/json",
|
|
216
|
+
Authorization: `Bearer ${upstream.apiKey}`,
|
|
217
|
+
...hints.headers,
|
|
218
|
+
},
|
|
219
|
+
body: JSON.stringify(upstreamBody),
|
|
220
|
+
signal: context.signal,
|
|
221
|
+
});
|
|
222
|
+
return new Response(encodeSse(frames), { headers: SSE_HEADERS });
|
|
223
|
+
};
|
|
224
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@jxsuite/ai/gateway`: the server half of the `ai/chat` and `ai/models` routes, written once.
|
|
3
|
+
*
|
|
4
|
+
* Every Jx backend that proxies the assistant to a provider does the same wire work: read the
|
|
5
|
+
* client's body, forward it as an OpenAI-compatible request, normalize the provider's stream into
|
|
6
|
+
* the route's frames, and frame those as SSE. Before this module each backend carried its own copy,
|
|
7
|
+
* and the copies drifted. Now `@jxsuite/server` runs this one, and any other backend can.
|
|
8
|
+
*
|
|
9
|
+
* What stays with the host is policy: whose key a request may use, which base URLs are safe to
|
|
10
|
+
* reach, how requests are admitted, and what the model catalogue says. The host supplies those as
|
|
11
|
+
* `ChatGatewayOptions`, and the gateway never second-guesses them.
|
|
12
|
+
*
|
|
13
|
+
* Worker-safe: it imports `@jxsuite/protocol`'s problem modules and nothing host-bound
|
|
14
|
+
* (`tests/worker-safety.test.ts`, `tsconfig.worker.json`).
|
|
15
|
+
*
|
|
16
|
+
* @module @jxsuite/ai/gateway
|
|
17
|
+
* @docs extending/embedding/backend-protocol
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
export {
|
|
21
|
+
AI_SESSION_HEADER,
|
|
22
|
+
affinityKey,
|
|
23
|
+
isAiSessionId,
|
|
24
|
+
upstreamCacheHints,
|
|
25
|
+
} from "../cache-hints.ts";
|
|
26
|
+
export { createChatHandler } from "./chat.ts";
|
|
27
|
+
export { modelsResponse } from "./models.ts";
|
|
28
|
+
export { normalizeOpenAIStream } from "./normalize.ts";
|
|
29
|
+
export { problemResponse } from "./problem.ts";
|
|
30
|
+
export { encodeSse } from "./sse.ts";
|
|
31
|
+
export { extractUpstreamErrorMessage } from "./upstream-error.ts";
|
|
32
|
+
export type { UpstreamCacheHints } from "../cache-hints.ts";
|
|
33
|
+
export type { ChatAdmission, ChatGatewayOptions, GatewayRefusal, Upstream } from "./types.ts";
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The models route's answer.
|
|
3
|
+
*
|
|
4
|
+
* @module @jxsuite/ai/gateway
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import type { AiModelsResponse } from "@jxsuite/protocol/types";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Answer the models route with `body`, always at `200`.
|
|
11
|
+
*
|
|
12
|
+
* The models route is a capability probe (ai.md §2.1), so every credential state is a 200: "no
|
|
13
|
+
* credentials", "credentials lapsed" and "the upstream could not list" are all answers the client
|
|
14
|
+
* reads, and a non-2xx makes the probe throw. Which catalogue to answer with is the host's business
|
|
15
|
+
* (its defaults, the upstream's listing, a brokered one); this only fixes the envelope.
|
|
16
|
+
*
|
|
17
|
+
* @param {AiModelsResponse} body - The catalogue and the credential state
|
|
18
|
+
* @returns {Response}
|
|
19
|
+
*/
|
|
20
|
+
export function modelsResponse(body: AiModelsResponse): Response {
|
|
21
|
+
return Response.json(body, { headers: { "Content-Type": "application/json" }, status: 200 });
|
|
22
|
+
}
|
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The OpenAI-compatible stream normalizer: an upstream chat-completions response in, the chat
|
|
3
|
+
* route's frames out.
|
|
4
|
+
*
|
|
5
|
+
* This is `@jxsuite/server`'s normalizer, moved here verbatim (specs/ai.md §2.4), so that every
|
|
6
|
+
* backend serving the chat route can run the same one. Its frames are frozen by the server's
|
|
7
|
+
* upstream fixtures (`packages/server/tests/fixtures/ai-upstream/*.server.json`); a change to what
|
|
8
|
+
* this file yields is a behaviour change and belongs in a change whose spec fragment names it.
|
|
9
|
+
*
|
|
10
|
+
* @module @jxsuite/ai/gateway
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { problemDetails } from "@jxsuite/protocol/problems";
|
|
14
|
+
import type { StreamEvent, StreamUsageEvent } from "../streaming-client.ts";
|
|
15
|
+
import { problemTypeForStatus } from "./problem.ts";
|
|
16
|
+
import { extractUpstreamErrorMessage } from "./upstream-error.ts";
|
|
17
|
+
|
|
18
|
+
/** A tool-call fragment inside an OpenAI streaming `delta`. */
|
|
19
|
+
interface ToolCallDelta {
|
|
20
|
+
index: number;
|
|
21
|
+
id?: string;
|
|
22
|
+
type?: string;
|
|
23
|
+
function?: { name?: string; arguments?: string };
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** A single OpenAI chat-completions streaming chunk (the JSON after `data: `). */
|
|
27
|
+
interface OpenAIStreamChunk {
|
|
28
|
+
choices?: {
|
|
29
|
+
index?: number;
|
|
30
|
+
delta?: {
|
|
31
|
+
content?: string;
|
|
32
|
+
/** Thinking models' chain-of-thought: DeepSeek and Volcengine spell it this way... */
|
|
33
|
+
reasoning_content?: string;
|
|
34
|
+
/** ...and OpenRouter this way. Both carry the same thing. */
|
|
35
|
+
reasoning?: string;
|
|
36
|
+
tool_calls?: ToolCallDelta[];
|
|
37
|
+
};
|
|
38
|
+
finish_reason?: string | null;
|
|
39
|
+
}[];
|
|
40
|
+
/** The count `stream_options.include_usage` asks for, normally a final chunk with no choices. */
|
|
41
|
+
usage?: {
|
|
42
|
+
prompt_tokens?: number;
|
|
43
|
+
completion_tokens?: number;
|
|
44
|
+
prompt_tokens_details?: { cached_tokens?: number } | null;
|
|
45
|
+
completion_tokens_details?: { reasoning_tokens?: number } | null;
|
|
46
|
+
} | null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Normalize an upstream usage object into a `usage` frame, or null when it carries no count. The
|
|
51
|
+
* detail figures are included only when the provider sent them: absent is "not said", not zero.
|
|
52
|
+
*
|
|
53
|
+
* @param {OpenAIStreamChunk["usage"]} usage - The chunk's `usage` member
|
|
54
|
+
* @returns {StreamUsageEvent | null}
|
|
55
|
+
*/
|
|
56
|
+
function usageFrame(usage: OpenAIStreamChunk["usage"]): StreamUsageEvent | null {
|
|
57
|
+
if (!usage || typeof usage.prompt_tokens !== "number") {
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
const frame: StreamUsageEvent = {
|
|
61
|
+
type: "usage",
|
|
62
|
+
inputTokens: usage.prompt_tokens,
|
|
63
|
+
outputTokens: usage.completion_tokens ?? 0,
|
|
64
|
+
};
|
|
65
|
+
const cached = usage.prompt_tokens_details?.cached_tokens;
|
|
66
|
+
if (typeof cached === "number") {
|
|
67
|
+
frame.cachedInputTokens = cached;
|
|
68
|
+
}
|
|
69
|
+
const reasoning = usage.completion_tokens_details?.reasoning_tokens;
|
|
70
|
+
if (typeof reasoning === "number") {
|
|
71
|
+
frame.reasoningTokens = reasoning;
|
|
72
|
+
}
|
|
73
|
+
return frame;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The stop reasons the stream reports; any other `finish_reason` keeps it reading.
|
|
78
|
+
*
|
|
79
|
+
* @param {string | null | undefined} reason - The chunk's `finish_reason`
|
|
80
|
+
* @returns {string | null}
|
|
81
|
+
*/
|
|
82
|
+
function stopReasonOf(reason: string | null | undefined): string | null {
|
|
83
|
+
if (reason === "tool_calls" || reason === "length" || reason === "stop") {
|
|
84
|
+
return reason;
|
|
85
|
+
}
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* The frame for an upstream request that never got a response: `done: cancelled` when it was
|
|
91
|
+
* aborted, otherwise an `error` frame carrying an `upstreamFailure` problem.
|
|
92
|
+
*
|
|
93
|
+
* The frame carries a problem rather than being one: the response began with a 200 long before this
|
|
94
|
+
* failed, so nothing can change the status now. `message` stays for the readers that already show
|
|
95
|
+
* it (server.md §4.3).
|
|
96
|
+
*
|
|
97
|
+
* @param {unknown} error - What `fetch` rejected with
|
|
98
|
+
* @returns {StreamEvent}
|
|
99
|
+
*/
|
|
100
|
+
export function fetchFailureFrame(error: unknown): StreamEvent {
|
|
101
|
+
if ((error as Error).name === "AbortError") {
|
|
102
|
+
return { type: "done", stopReason: "cancelled" };
|
|
103
|
+
}
|
|
104
|
+
const message = `Network error: ${(error as Error).message}`;
|
|
105
|
+
return { message, problem: problemDetails("upstreamFailure", message), type: "error" };
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Normalize an OpenAI-compatible chat-completions response into the chat route's frames.
|
|
110
|
+
*
|
|
111
|
+
* - A non-2xx response is one `error` frame: the sentence inside the provider's body as `message`,
|
|
112
|
+
* the upstream's own status as `code`, and a problem naming the kind of failure.
|
|
113
|
+
* - Text is `delta`; chain-of-thought (`reasoning_content`, or `reasoning` at OpenRouter) is
|
|
114
|
+
* `reasoning`, forwarded rather than dropped, because DeepSeek's thinking mode requires every
|
|
115
|
+
* prior turn's reasoning back on any request carrying `tools`.
|
|
116
|
+
* - A tool call is `tool_call_start`, then `tool_call_delta` per fragment, then `tool_call_end`.
|
|
117
|
+
* - The finish reason is REMEMBERED, not acted on: with `include_usage` the count arrives in a chunk
|
|
118
|
+
* of its own after the finish, so the stream ends at `[DONE]` or when the body closes. The last
|
|
119
|
+
* frames are the open calls' ends, the `usage` count when there is one, then `done`.
|
|
120
|
+
* - An aborted read is `done: cancelled`; any other failed read is an `error` frame.
|
|
121
|
+
* - A consumer that stops reading early cancels the upstream body.
|
|
122
|
+
*
|
|
123
|
+
* @param {Response} response - The upstream's response to the chat-completions request
|
|
124
|
+
* @yields {StreamEvent} The route's frames, in wire order
|
|
125
|
+
*/
|
|
126
|
+
export async function* normalizeOpenAIStream(response: Response): AsyncGenerator<StreamEvent> {
|
|
127
|
+
if (!response.ok) {
|
|
128
|
+
let errorBody = "";
|
|
129
|
+
try {
|
|
130
|
+
errorBody = await response.text();
|
|
131
|
+
} catch {
|
|
132
|
+
/* Ignore */
|
|
133
|
+
}
|
|
134
|
+
const cleanMessage = extractUpstreamErrorMessage(errorBody, response.statusText);
|
|
135
|
+
yield {
|
|
136
|
+
code: String(response.status),
|
|
137
|
+
message: cleanMessage,
|
|
138
|
+
// The upstream's own status is preserved in `code`; the problem names the KIND.
|
|
139
|
+
problem: problemDetails(problemTypeForStatus(response.status), cleanMessage),
|
|
140
|
+
type: "error",
|
|
141
|
+
};
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const reader = response.body?.getReader();
|
|
146
|
+
if (!reader) {
|
|
147
|
+
yield {
|
|
148
|
+
message: "No response body from upstream",
|
|
149
|
+
problem: problemDetails("upstreamFailure", "No response body from upstream"),
|
|
150
|
+
type: "error",
|
|
151
|
+
};
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const decoder = new TextDecoder();
|
|
156
|
+
let buffer = "";
|
|
157
|
+
|
|
158
|
+
const pendingToolCalls = new Map<number, { id: string; name: string; args: string }>();
|
|
159
|
+
let stopReason: string | null = null;
|
|
160
|
+
let usage: StreamUsageEvent | null = null;
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Close every open call.
|
|
164
|
+
*
|
|
165
|
+
* @yields {StreamEvent} A `tool_call_end` per open call
|
|
166
|
+
*/
|
|
167
|
+
function* closePendingToolCalls(): Generator<StreamEvent> {
|
|
168
|
+
for (const [, tc] of pendingToolCalls) {
|
|
169
|
+
yield { type: "tool_call_end", id: tc.id };
|
|
170
|
+
}
|
|
171
|
+
pendingToolCalls.clear();
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* The stream's last frames: open calls, the count when there is one, then `done`. The count comes
|
|
176
|
+
* BEFORE `done`, because a reader stops at `done`.
|
|
177
|
+
*
|
|
178
|
+
* @yields {StreamEvent} The closing frames, in order
|
|
179
|
+
*/
|
|
180
|
+
function* finish(): Generator<StreamEvent> {
|
|
181
|
+
yield* closePendingToolCalls();
|
|
182
|
+
if (usage) {
|
|
183
|
+
yield usage;
|
|
184
|
+
}
|
|
185
|
+
yield { type: "done", stopReason: stopReason ?? "stop" };
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/* Whether the loop ended on its own terms. A consumer that stops reading ends a generator through
|
|
189
|
+
`return()`, which runs `finally` and nothing else; when the server wrote each frame itself, that
|
|
190
|
+
exit was the next write throwing inside this `try`, so the `catch` cancelled the upstream body.
|
|
191
|
+
`finally` keeps that: the upstream must not outlive the reader it streams to. */
|
|
192
|
+
let settled = false;
|
|
193
|
+
try {
|
|
194
|
+
while (true) {
|
|
195
|
+
const { done, value } = await reader.read();
|
|
196
|
+
if (done) {
|
|
197
|
+
break;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
buffer += decoder.decode(value, { stream: true });
|
|
201
|
+
const lines = buffer.split("\n");
|
|
202
|
+
buffer = lines.pop() || "";
|
|
203
|
+
|
|
204
|
+
for (const line of lines) {
|
|
205
|
+
const trimmed = line.trim();
|
|
206
|
+
if (!trimmed || !trimmed.startsWith("data: ")) {
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
const dataStr = trimmed.slice(6);
|
|
211
|
+
if (dataStr === "[DONE]") {
|
|
212
|
+
yield* finish();
|
|
213
|
+
settled = true;
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
let parsed: OpenAIStreamChunk;
|
|
218
|
+
try {
|
|
219
|
+
parsed = JSON.parse(dataStr) as OpenAIStreamChunk;
|
|
220
|
+
} catch {
|
|
221
|
+
continue;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// Usually a final chunk with no choices, but a provider may put it beside the finish.
|
|
225
|
+
usage = usageFrame(parsed.usage) ?? usage;
|
|
226
|
+
|
|
227
|
+
const choice = parsed.choices?.[0];
|
|
228
|
+
if (!choice) {
|
|
229
|
+
continue;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
const { delta } = choice;
|
|
233
|
+
|
|
234
|
+
// Text content
|
|
235
|
+
if (delta?.content) {
|
|
236
|
+
yield { type: "delta", content: delta.content };
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/* Chain-of-thought, under either of the two names providers give it. Forwarded rather than
|
|
240
|
+
dropped: DeepSeek's thinking mode requires every prior turn's reasoning back on any
|
|
241
|
+
request carrying `tools`, so a proxy that swallows these frames makes the NEXT round a
|
|
242
|
+
400 the client cannot repair. */
|
|
243
|
+
const reasoning = delta?.reasoning_content ?? delta?.reasoning;
|
|
244
|
+
if (typeof reasoning === "string" && reasoning) {
|
|
245
|
+
yield { type: "reasoning", content: reasoning };
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// Tool calls
|
|
249
|
+
if (delta?.tool_calls) {
|
|
250
|
+
for (const tc of delta.tool_calls) {
|
|
251
|
+
if (tc.id) {
|
|
252
|
+
// First appearance
|
|
253
|
+
const entry = {
|
|
254
|
+
id: tc.id,
|
|
255
|
+
name: tc.function?.name || "",
|
|
256
|
+
args: tc.function?.arguments || "",
|
|
257
|
+
};
|
|
258
|
+
pendingToolCalls.set(tc.index, entry);
|
|
259
|
+
|
|
260
|
+
yield { type: "tool_call_start", id: tc.id, name: entry.name };
|
|
261
|
+
|
|
262
|
+
if (entry.args) {
|
|
263
|
+
yield { type: "tool_call_delta", id: tc.id, args: entry.args };
|
|
264
|
+
}
|
|
265
|
+
} else if (tc.function?.arguments) {
|
|
266
|
+
// Subsequent fragment
|
|
267
|
+
const existing = pendingToolCalls.get(tc.index);
|
|
268
|
+
if (existing) {
|
|
269
|
+
existing.args += tc.function.arguments;
|
|
270
|
+
yield { type: "tool_call_delta", id: existing.id, args: tc.function.arguments };
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// Finish reason: close the calls now, report the reason at the end of the stream.
|
|
277
|
+
const finished = stopReasonOf(choice.finish_reason);
|
|
278
|
+
if (finished) {
|
|
279
|
+
yield* closePendingToolCalls();
|
|
280
|
+
stopReason = finished;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
// Stream ended without `[DONE]`
|
|
286
|
+
yield* finish();
|
|
287
|
+
settled = true;
|
|
288
|
+
} catch (error) {
|
|
289
|
+
settled = true;
|
|
290
|
+
void reader.cancel();
|
|
291
|
+
if ((error as Error).name === "AbortError") {
|
|
292
|
+
yield { type: "done", stopReason: "cancelled" };
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
const message = `Stream error: ${(error as Error).message}`;
|
|
296
|
+
yield { message, problem: problemDetails("upstreamFailure", message), type: "error" };
|
|
297
|
+
} finally {
|
|
298
|
+
if (!settled) {
|
|
299
|
+
void reader.cancel();
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How the gateway says no: an RFC 9457 problem response, and the problem type an upstream's status
|
|
3
|
+
* maps onto.
|
|
4
|
+
*
|
|
5
|
+
* @module @jxsuite/ai/gateway
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { PROBLEM_MEDIA_TYPE } from "@jxsuite/protocol/problem";
|
|
9
|
+
import { problemDetails } from "@jxsuite/protocol/problems";
|
|
10
|
+
import type { ProblemTypeName } from "@jxsuite/protocol/problems";
|
|
11
|
+
import type { GatewayRefusal } from "./types.ts";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* A refusal of one declared problem type, answered with that type's own status.
|
|
15
|
+
*
|
|
16
|
+
* @param {ProblemTypeName} name - A declared type from `PROBLEM_TYPES`
|
|
17
|
+
* @param {string} detail - What happened this time; the line a human reads
|
|
18
|
+
* @returns {GatewayRefusal}
|
|
19
|
+
*/
|
|
20
|
+
export function refusal(name: ProblemTypeName, detail: string): GatewayRefusal {
|
|
21
|
+
const problem = problemDetails(name, detail);
|
|
22
|
+
return { problem, status: problem.status };
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* The response a refusal is answered with: `application/problem+json` at the refusal's status.
|
|
27
|
+
*
|
|
28
|
+
* One body satisfies both kinds of client. A problem reader keys on `type` and shows `detail`; an
|
|
29
|
+
* older reader shows `error` and acts on `code`. So `error` is present whenever `detail` is (it
|
|
30
|
+
* already is on a problem built by `problemDetails`, and is added here to one built by hand), and
|
|
31
|
+
* the refusal's `code`, when it has one, rides beside them.
|
|
32
|
+
*
|
|
33
|
+
* @param {GatewayRefusal} refused - What was refused, and why
|
|
34
|
+
* @returns {Response}
|
|
35
|
+
*/
|
|
36
|
+
export function problemResponse(refused: GatewayRefusal): Response {
|
|
37
|
+
const { code, problem, status } = refused;
|
|
38
|
+
const body = {
|
|
39
|
+
...problem,
|
|
40
|
+
...(problem.error === undefined && problem.detail !== undefined
|
|
41
|
+
? { error: problem.detail }
|
|
42
|
+
: {}),
|
|
43
|
+
...(code === undefined ? {} : { code }),
|
|
44
|
+
};
|
|
45
|
+
// Response.json would send `application/json`; RFC 9457 §3 is what says this body is a problem.
|
|
46
|
+
return Response.json(body, { headers: { "Content-Type": PROBLEM_MEDIA_TYPE }, status });
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* The declared type for an upstream provider's status.
|
|
51
|
+
*
|
|
52
|
+
* The normalizer needs it for the one failure that arrives with a status of its own: an upstream
|
|
53
|
+
* that answered non-2xx. The status itself is kept (as the frame's `code`), and the problem names
|
|
54
|
+
* the KIND of failure, which is the part a client can act on. `@jxsuite/server`'s `problem.ts`
|
|
55
|
+
* carries the same table for its own routes; `packages/server/tests/ai-api-gateway.test.ts` holds
|
|
56
|
+
* the two to agreement.
|
|
57
|
+
*
|
|
58
|
+
* @param {number} status - The upstream's HTTP status
|
|
59
|
+
* @returns {ProblemTypeName}
|
|
60
|
+
*/
|
|
61
|
+
export function problemTypeForStatus(status: number): ProblemTypeName {
|
|
62
|
+
/* Any other status still becomes SOME type, and which side of 500 it falls on is what decides
|
|
63
|
+
whether the caller is being told it made a mistake or that the upstream did. */
|
|
64
|
+
return STATUS_TYPES[status] ?? (status >= 500 ? "internalError" : "invalidRequest");
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** The statuses the table distinguishes, each onto a type answered with that same status. */
|
|
68
|
+
const STATUS_TYPES: Readonly<Record<number, ProblemTypeName>> = {
|
|
69
|
+
400: "invalidRequest",
|
|
70
|
+
401: "unauthorized",
|
|
71
|
+
403: "forbidden",
|
|
72
|
+
404: "notFound",
|
|
73
|
+
405: "methodNotAllowed",
|
|
74
|
+
409: "conflict",
|
|
75
|
+
413: "payloadTooLarge",
|
|
76
|
+
501: "capabilityUnavailable",
|
|
77
|
+
502: "upstreamFailure",
|
|
78
|
+
503: "upstreamFailure",
|
|
79
|
+
504: "upstreamFailure",
|
|
80
|
+
};
|