@starci/ai-router 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/LICENSE +21 -0
- package/README.md +312 -2
- package/dist/builder.d.ts +33 -0
- package/dist/builder.js +49 -0
- package/dist/config.d.ts +46 -0
- package/dist/config.js +148 -0
- package/dist/errors.d.ts +47 -0
- package/dist/errors.js +66 -0
- package/dist/events.d.ts +11 -0
- package/dist/events.js +31 -0
- package/dist/fake.d.ts +67 -0
- package/dist/fake.js +153 -0
- package/dist/headers.d.ts +15 -0
- package/dist/headers.js +66 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +8 -0
- package/dist/pool.d.ts +90 -0
- package/dist/pool.js +421 -0
- package/dist/router.d.ts +3 -0
- package/dist/router.js +588 -0
- package/dist/sse.d.ts +10 -0
- package/dist/sse.js +48 -0
- package/dist/store.d.ts +56 -0
- package/dist/store.js +48 -0
- package/dist/types.d.ts +426 -0
- package/dist/types.js +1 -0
- package/package.json +66 -4
package/dist/router.js
ADDED
|
@@ -0,0 +1,588 @@
|
|
|
1
|
+
import { resolvePolicy, resolveSettings, validateConfig } from "./config.js";
|
|
2
|
+
import { AiRouterError, classifyProviderError, classifyStatus, isRetryable, scrub } from "./errors.js";
|
|
3
|
+
import { Emitter } from "./events.js";
|
|
4
|
+
import { readRateLimit } from "./headers.js";
|
|
5
|
+
import { PoolState } from "./pool.js";
|
|
6
|
+
import { parseSse } from "./sse.js";
|
|
7
|
+
import { MemoryStateStore } from "./store.js";
|
|
8
|
+
const BUSY_POLL_MS = 250;
|
|
9
|
+
const ERROR_BODY_LIMIT = 4096;
|
|
10
|
+
/** A failed attempt inside the router, before it becomes part of an `AiRouterError`. */
|
|
11
|
+
class AttemptError extends Error {
|
|
12
|
+
errorClass;
|
|
13
|
+
status;
|
|
14
|
+
info;
|
|
15
|
+
missingKey;
|
|
16
|
+
constructor(errorClass, message, options = {}) {
|
|
17
|
+
super(message);
|
|
18
|
+
this.errorClass = errorClass;
|
|
19
|
+
this.status = options.status;
|
|
20
|
+
this.info = options.info ?? null;
|
|
21
|
+
this.missingKey = options.missingKey ?? false;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
/** One call's lifetime: its deadline, its cancellation, the pairs it tried and the failures it met. */
|
|
25
|
+
class Call {
|
|
26
|
+
requestId;
|
|
27
|
+
operation;
|
|
28
|
+
policy;
|
|
29
|
+
need;
|
|
30
|
+
lanes;
|
|
31
|
+
started = Date.now();
|
|
32
|
+
deadlineAt;
|
|
33
|
+
controller = new AbortController();
|
|
34
|
+
tried = { lanes: new Set(), pairs: new Set(), providers: new Set() };
|
|
35
|
+
failures = [];
|
|
36
|
+
attempts = 0;
|
|
37
|
+
transientFailures = 0;
|
|
38
|
+
cause = null;
|
|
39
|
+
#timer;
|
|
40
|
+
#signal;
|
|
41
|
+
#onAbort = () => {
|
|
42
|
+
this.#abort("cancelled");
|
|
43
|
+
};
|
|
44
|
+
constructor(operation, requestId, policy, need, lanes, signal) {
|
|
45
|
+
this.operation = operation;
|
|
46
|
+
this.requestId = requestId;
|
|
47
|
+
this.policy = policy;
|
|
48
|
+
this.need = need;
|
|
49
|
+
this.lanes = lanes;
|
|
50
|
+
this.deadlineAt = this.started + policy.deadlineMs;
|
|
51
|
+
this.#timer = setTimeout(() => {
|
|
52
|
+
this.#abort("deadline");
|
|
53
|
+
}, policy.deadlineMs);
|
|
54
|
+
this.#signal = signal;
|
|
55
|
+
if (signal?.aborted === true)
|
|
56
|
+
this.#abort("cancelled");
|
|
57
|
+
else
|
|
58
|
+
signal?.addEventListener("abort", this.#onAbort, { once: true });
|
|
59
|
+
}
|
|
60
|
+
#abort(cause) {
|
|
61
|
+
if (this.controller.signal.aborted)
|
|
62
|
+
return;
|
|
63
|
+
this.cause = cause;
|
|
64
|
+
this.controller.abort(new AttemptError(cause, cause === "cancelled" ? "the call was cancelled" : "the call passed its deadline"));
|
|
65
|
+
}
|
|
66
|
+
/** The error a stopped call throws. */
|
|
67
|
+
stopped() {
|
|
68
|
+
const cause = this.cause ?? "cancelled";
|
|
69
|
+
return this.error(cause, cause === "cancelled" ? "the call was cancelled" : `the call passed its ${String(this.policy.deadlineMs)} ms deadline`);
|
|
70
|
+
}
|
|
71
|
+
/** Builds the call's error with every attempt it made. */
|
|
72
|
+
error(errorClass, message, extra = {}) {
|
|
73
|
+
const last = this.failures.at(-1);
|
|
74
|
+
return new AiRouterError({
|
|
75
|
+
errorClass,
|
|
76
|
+
message,
|
|
77
|
+
requestId: this.requestId,
|
|
78
|
+
attempts: [...this.failures],
|
|
79
|
+
...(last === undefined ? {} : { provider: last.provider, key: last.key, model: last.model }),
|
|
80
|
+
...extra,
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
/** Stops the timers and listeners of the call. */
|
|
84
|
+
dispose() {
|
|
85
|
+
clearTimeout(this.#timer);
|
|
86
|
+
this.#signal?.removeEventListener("abort", this.#onAbort);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
/** One attempt's abort controller: it follows the call and adds the attempt's own timeout. */
|
|
90
|
+
class AttemptScope {
|
|
91
|
+
controller = new AbortController();
|
|
92
|
+
cause = null;
|
|
93
|
+
/** The key value this attempt sends, held only to scrub it from error messages. */
|
|
94
|
+
secret;
|
|
95
|
+
#timer;
|
|
96
|
+
#call;
|
|
97
|
+
#follow = () => {
|
|
98
|
+
this.cause = this.#call.cause;
|
|
99
|
+
this.controller.abort(this.#call.controller.signal.reason);
|
|
100
|
+
};
|
|
101
|
+
constructor(call, timeoutMs) {
|
|
102
|
+
this.#call = call;
|
|
103
|
+
if (call.controller.signal.aborted)
|
|
104
|
+
this.#follow();
|
|
105
|
+
else
|
|
106
|
+
call.controller.signal.addEventListener("abort", this.#follow, { once: true });
|
|
107
|
+
this.arm(timeoutMs);
|
|
108
|
+
}
|
|
109
|
+
/** Restarts the attempt's timeout. */
|
|
110
|
+
arm(timeoutMs) {
|
|
111
|
+
clearTimeout(this.#timer);
|
|
112
|
+
this.#timer = setTimeout(() => {
|
|
113
|
+
if (this.controller.signal.aborted)
|
|
114
|
+
return;
|
|
115
|
+
this.cause = "timeout";
|
|
116
|
+
this.controller.abort(new AttemptError("timeout", `the attempt passed its ${String(timeoutMs)} ms timeout`));
|
|
117
|
+
}, timeoutMs);
|
|
118
|
+
}
|
|
119
|
+
/** Turns whatever an attempt threw into an `AttemptError`, naming an abort by its cause. */
|
|
120
|
+
classify(error) {
|
|
121
|
+
const failure = error instanceof AttemptError
|
|
122
|
+
? error
|
|
123
|
+
: this.cause !== null
|
|
124
|
+
? new AttemptError(this.cause, this.cause === "timeout" ? "the attempt timed out" : `the call was ${this.cause === "deadline" ? "past its deadline" : "cancelled"}`)
|
|
125
|
+
: new AttemptError("network", error instanceof Error ? error.message : "the request failed");
|
|
126
|
+
// Whatever a provider echoed, the key value never leaves the attempt.
|
|
127
|
+
if (this.secret === undefined || !failure.message.includes(this.secret))
|
|
128
|
+
return failure;
|
|
129
|
+
return new AttemptError(failure.errorClass, scrub(failure.message, [this.secret]), { ...statusOf(failure), info: failure.info, missingKey: failure.missingKey });
|
|
130
|
+
}
|
|
131
|
+
/** Stops the timer and the call listener. */
|
|
132
|
+
dispose() {
|
|
133
|
+
clearTimeout(this.#timer);
|
|
134
|
+
this.#call.controller.signal.removeEventListener("abort", this.#follow);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
function textOf(content) {
|
|
138
|
+
if (content === null)
|
|
139
|
+
return "";
|
|
140
|
+
if (typeof content === "string")
|
|
141
|
+
return content;
|
|
142
|
+
return content.map((part) => part.text ?? "").join("");
|
|
143
|
+
}
|
|
144
|
+
function estimateTokens(text) {
|
|
145
|
+
return Math.ceil(text.length / 4);
|
|
146
|
+
}
|
|
147
|
+
function chatNeed(request, streaming, require) {
|
|
148
|
+
const capabilities = new Set(["chat", ...require]);
|
|
149
|
+
if (streaming)
|
|
150
|
+
capabilities.add("streaming");
|
|
151
|
+
if (request.tools !== undefined && request.tools.length > 0)
|
|
152
|
+
capabilities.add("tools");
|
|
153
|
+
if (request.response_format !== undefined && request.response_format.type !== "text")
|
|
154
|
+
capabilities.add("json");
|
|
155
|
+
return {
|
|
156
|
+
capabilities: [...capabilities],
|
|
157
|
+
promptTokens: estimateTokens(request.messages.map((message) => textOf(message.content)).join("\n")),
|
|
158
|
+
outputTokens: request.max_completion_tokens ?? request.max_tokens ?? 0,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
function hasToken(chunk) {
|
|
162
|
+
return chunk.choices.some((choice) => Object.entries(choice.delta).some(([field, value]) => field !== "role" && value !== null && value !== undefined && value !== "" && !(Array.isArray(value) && value.length === 0)));
|
|
163
|
+
}
|
|
164
|
+
function isRecord(value) {
|
|
165
|
+
return typeof value === "object" && value !== null;
|
|
166
|
+
}
|
|
167
|
+
function usageOf(json) {
|
|
168
|
+
return isRecord(json.usage) && typeof json.usage.prompt_tokens === "number" ? json.usage : undefined;
|
|
169
|
+
}
|
|
170
|
+
function statusOf(error) {
|
|
171
|
+
return error.status === undefined ? {} : { status: error.status };
|
|
172
|
+
}
|
|
173
|
+
/** Parses one SSE data payload: a chunk, null for an event to skip, or a thrown error the provider streamed. */
|
|
174
|
+
function parseChunk(data) {
|
|
175
|
+
let json;
|
|
176
|
+
try {
|
|
177
|
+
json = JSON.parse(data);
|
|
178
|
+
}
|
|
179
|
+
catch {
|
|
180
|
+
throw new AttemptError("stream", "the provider streamed an event that is not JSON");
|
|
181
|
+
}
|
|
182
|
+
if (!isRecord(json))
|
|
183
|
+
return null;
|
|
184
|
+
if (json.error !== undefined && json.error !== null) {
|
|
185
|
+
const message = isRecord(json.error) && typeof json.error.message === "string" ? json.error.message : "the provider streamed an error";
|
|
186
|
+
throw new AttemptError(classifyProviderError(json.error), message);
|
|
187
|
+
}
|
|
188
|
+
return Array.isArray(json.choices) ? json : null;
|
|
189
|
+
}
|
|
190
|
+
async function resolveSecret(key) {
|
|
191
|
+
const value = "env" in key ? process.env[key.env] : await key.resolve();
|
|
192
|
+
return value === undefined || value.trim() === "" ? undefined : value.trim();
|
|
193
|
+
}
|
|
194
|
+
async function errorBody(response) {
|
|
195
|
+
let text;
|
|
196
|
+
try {
|
|
197
|
+
text = (await response.text()).slice(0, ERROR_BODY_LIMIT);
|
|
198
|
+
}
|
|
199
|
+
catch {
|
|
200
|
+
return `HTTP ${String(response.status)}`;
|
|
201
|
+
}
|
|
202
|
+
let message = text;
|
|
203
|
+
try {
|
|
204
|
+
const json = JSON.parse(text);
|
|
205
|
+
if (isRecord(json) && isRecord(json.error) && typeof json.error.message === "string")
|
|
206
|
+
message = json.error.message;
|
|
207
|
+
else if (isRecord(json) && typeof json.message === "string")
|
|
208
|
+
message = json.message;
|
|
209
|
+
}
|
|
210
|
+
catch {
|
|
211
|
+
// Not JSON: the raw text is the message.
|
|
212
|
+
}
|
|
213
|
+
return `HTTP ${String(response.status)}: ${message || response.statusText}`;
|
|
214
|
+
}
|
|
215
|
+
/** Creates a router over a pool of providers, keys and models. The pool is validated now; keys are read per attempt. */
|
|
216
|
+
export function createRouter(config) {
|
|
217
|
+
validateConfig(config);
|
|
218
|
+
const settings = resolveSettings(config);
|
|
219
|
+
const emitter = new Emitter();
|
|
220
|
+
const state = new PoolState(config.providers, settings, config.store ?? new MemoryStateStore(), emitter);
|
|
221
|
+
const fetchImpl = config.fetch ?? ((input, init) => fetch(input, init));
|
|
222
|
+
const random = config.random ?? Math.random;
|
|
223
|
+
// The policy is resolved at once, so a bad profile or model throws from the call itself; the clock starts at start().
|
|
224
|
+
function prepare(operation, model, options, need) {
|
|
225
|
+
const policy = resolvePolicy(config, options, model);
|
|
226
|
+
const callNeed = need(policy);
|
|
227
|
+
const lanes = state.lanes(policy, callNeed);
|
|
228
|
+
const requestId = options.requestId ?? globalThis.crypto.randomUUID();
|
|
229
|
+
return {
|
|
230
|
+
requestId,
|
|
231
|
+
start: () => {
|
|
232
|
+
state.totals.requests++;
|
|
233
|
+
return new Call(operation, requestId, policy, callNeed, lanes, options.signal);
|
|
234
|
+
},
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
async function next(call) {
|
|
238
|
+
for (;;) {
|
|
239
|
+
if (call.controller.signal.aborted)
|
|
240
|
+
throw call.stopped();
|
|
241
|
+
if (call.lanes.length === 0) {
|
|
242
|
+
throw call.error("no_candidate", `no provider serves ${call.policy.models.join(", ")} with ${call.need.capabilities.join(", ")} within its context window and budget`);
|
|
243
|
+
}
|
|
244
|
+
if (call.attempts >= call.policy.maxAttempts) {
|
|
245
|
+
const last = call.failures.at(-1);
|
|
246
|
+
throw call.error(last?.errorClass ?? "no_candidate", `all ${String(call.attempts)} attempts failed; last: ${last?.message ?? "none"}`, last?.status === undefined ? {} : { status: last.status });
|
|
247
|
+
}
|
|
248
|
+
const picked = await state.pick(call.lanes, call.need, call.tried, call.policy.timeoutMs + 1000);
|
|
249
|
+
if ("lane" in picked)
|
|
250
|
+
return picked;
|
|
251
|
+
const now = Date.now();
|
|
252
|
+
if (picked.waitUntil > call.deadlineAt) {
|
|
253
|
+
const last = call.failures.at(-1);
|
|
254
|
+
const message = `no candidate is usable before the deadline${last === undefined ? "" : `; last: ${last.message}`}`;
|
|
255
|
+
const retryAt = Number.isFinite(picked.waitUntil) ? { retryAt: picked.waitUntil } : {};
|
|
256
|
+
throw call.error(last?.errorClass ?? "no_candidate", message, retryAt);
|
|
257
|
+
}
|
|
258
|
+
try {
|
|
259
|
+
await state.waitForCapacity(picked.busy ? BUSY_POLL_MS : picked.waitUntil - now, call.controller.signal);
|
|
260
|
+
}
|
|
261
|
+
catch {
|
|
262
|
+
throw call.stopped();
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
async function backoff(call, errorClass) {
|
|
267
|
+
// Rate and auth failures belong to one key; the next key owes them no wait. Transient ones back off.
|
|
268
|
+
if (errorClass === "rate_limit" || errorClass === "auth")
|
|
269
|
+
return;
|
|
270
|
+
call.transientFailures++;
|
|
271
|
+
const { baseMs, factor, maxMs } = settings.backoff;
|
|
272
|
+
const delay = Math.min(maxMs, baseMs * factor ** (call.transientFailures - 1)) * (0.5 + random() * 0.5);
|
|
273
|
+
try {
|
|
274
|
+
await new Promise((resolve, reject) => {
|
|
275
|
+
const signal = call.controller.signal;
|
|
276
|
+
const timer = setTimeout(() => {
|
|
277
|
+
signal.removeEventListener("abort", stop);
|
|
278
|
+
resolve();
|
|
279
|
+
}, delay);
|
|
280
|
+
const stop = () => {
|
|
281
|
+
clearTimeout(timer);
|
|
282
|
+
reject(new Error("stopped"));
|
|
283
|
+
};
|
|
284
|
+
if (signal.aborted)
|
|
285
|
+
stop();
|
|
286
|
+
else
|
|
287
|
+
signal.addEventListener("abort", stop, { once: true });
|
|
288
|
+
});
|
|
289
|
+
}
|
|
290
|
+
catch {
|
|
291
|
+
throw call.stopped();
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
function headers(lane, secret, call, options, accept) {
|
|
295
|
+
const out = {};
|
|
296
|
+
for (const [name, value] of Object.entries({ ...lane.provider.headers, ...options.headers })) {
|
|
297
|
+
if (name.toLowerCase() !== "authorization")
|
|
298
|
+
out[name] = value;
|
|
299
|
+
}
|
|
300
|
+
return {
|
|
301
|
+
...out,
|
|
302
|
+
"content-type": "application/json",
|
|
303
|
+
accept,
|
|
304
|
+
authorization: `Bearer ${secret}`,
|
|
305
|
+
"x-request-id": call.requestId,
|
|
306
|
+
"idempotency-key": call.requestId,
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
async function open(call, pick, scope, path, body, options, accept) {
|
|
310
|
+
let secret;
|
|
311
|
+
try {
|
|
312
|
+
secret = await resolveSecret(pick.lane.key);
|
|
313
|
+
}
|
|
314
|
+
catch {
|
|
315
|
+
secret = undefined;
|
|
316
|
+
}
|
|
317
|
+
if (secret === undefined)
|
|
318
|
+
throw new AttemptError("auth", `key ${pick.lane.key.alias} resolved to no value`, { missingKey: true });
|
|
319
|
+
scope.secret = secret;
|
|
320
|
+
let response;
|
|
321
|
+
try {
|
|
322
|
+
response = await fetchImpl(`${pick.lane.provider.baseUrl.replace(/\/+$/, "")}${path}`, {
|
|
323
|
+
method: "POST",
|
|
324
|
+
headers: headers(pick.lane, secret, call, options, accept),
|
|
325
|
+
body: JSON.stringify({ ...body, model: pick.lane.model.id }),
|
|
326
|
+
signal: scope.controller.signal,
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
catch (error) {
|
|
330
|
+
throw scope.classify(error);
|
|
331
|
+
}
|
|
332
|
+
if (!response.ok) {
|
|
333
|
+
const info = readRateLimit(response.headers, Date.now());
|
|
334
|
+
throw new AttemptError(classifyStatus(response.status), await errorBody(response), { status: response.status, info });
|
|
335
|
+
}
|
|
336
|
+
return response;
|
|
337
|
+
}
|
|
338
|
+
function begin(call, pick) {
|
|
339
|
+
call.attempts++;
|
|
340
|
+
call.tried.lanes.add(pick.lane.id);
|
|
341
|
+
call.tried.pairs.add(pick.lane.pair);
|
|
342
|
+
call.tried.providers.add(pick.lane.provider.id);
|
|
343
|
+
state.count(pick.lane);
|
|
344
|
+
}
|
|
345
|
+
async function fail(call, pick, error, latencyMs, content) {
|
|
346
|
+
await state.failed(pick.lane, error.errorClass, error.info, error.missingKey);
|
|
347
|
+
call.failures.push({
|
|
348
|
+
provider: pick.lane.provider.id,
|
|
349
|
+
key: pick.lane.key.alias,
|
|
350
|
+
model: pick.lane.model.id,
|
|
351
|
+
errorClass: error.errorClass,
|
|
352
|
+
message: error.message,
|
|
353
|
+
...(error.status === undefined ? {} : { status: error.status }),
|
|
354
|
+
});
|
|
355
|
+
emitter.emit("attempt", {
|
|
356
|
+
type: "attempt",
|
|
357
|
+
requestId: call.requestId,
|
|
358
|
+
operation: call.operation,
|
|
359
|
+
attempt: call.attempts,
|
|
360
|
+
provider: pick.lane.provider.id,
|
|
361
|
+
key: pick.lane.key.alias,
|
|
362
|
+
model: pick.lane.model.id,
|
|
363
|
+
latencyMs,
|
|
364
|
+
outcome: "failure",
|
|
365
|
+
errorClass: error.errorClass,
|
|
366
|
+
...(error.status === undefined ? {} : { status: error.status }),
|
|
367
|
+
...(settings.includeContent ? { content: { request: content } } : {}),
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
async function succeed(call, pick, info, latencyMs, usage, content) {
|
|
371
|
+
const cost = await state.succeeded(pick.lane, info, latencyMs, usage);
|
|
372
|
+
emitter.emit("attempt", {
|
|
373
|
+
type: "attempt",
|
|
374
|
+
requestId: call.requestId,
|
|
375
|
+
operation: call.operation,
|
|
376
|
+
attempt: call.attempts,
|
|
377
|
+
provider: pick.lane.provider.id,
|
|
378
|
+
key: pick.lane.key.alias,
|
|
379
|
+
model: pick.lane.model.id,
|
|
380
|
+
latencyMs,
|
|
381
|
+
outcome: "success",
|
|
382
|
+
...(usage === undefined ? {} : { usage }),
|
|
383
|
+
...(cost === undefined ? {} : { cost }),
|
|
384
|
+
...(settings.includeContent ? { content } : {}),
|
|
385
|
+
});
|
|
386
|
+
return {
|
|
387
|
+
requestId: call.requestId,
|
|
388
|
+
provider: pick.lane.provider.id,
|
|
389
|
+
key: pick.lane.key.alias,
|
|
390
|
+
model: pick.lane.model.id,
|
|
391
|
+
attempts: call.attempts,
|
|
392
|
+
latencyMs: Date.now() - call.started,
|
|
393
|
+
...(cost === undefined ? {} : { cost }),
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
function settle(call, routing, error) {
|
|
397
|
+
if (routing !== null)
|
|
398
|
+
state.totals.succeeded++;
|
|
399
|
+
else
|
|
400
|
+
state.totals.failed++;
|
|
401
|
+
emitter.emit("request", {
|
|
402
|
+
type: "request",
|
|
403
|
+
requestId: call.requestId,
|
|
404
|
+
operation: call.operation,
|
|
405
|
+
outcome: routing === null ? "failure" : "success",
|
|
406
|
+
attempts: call.attempts,
|
|
407
|
+
latencyMs: Date.now() - call.started,
|
|
408
|
+
...(routing === null ? {} : { provider: routing.provider, key: routing.key, model: routing.model }),
|
|
409
|
+
...(error === null ? {} : { errorClass: error.errorClass }),
|
|
410
|
+
});
|
|
411
|
+
call.dispose();
|
|
412
|
+
}
|
|
413
|
+
function toRouterError(call, error) {
|
|
414
|
+
if (error instanceof AiRouterError)
|
|
415
|
+
return error;
|
|
416
|
+
return call.error("network", error instanceof Error ? error.message : "the call failed");
|
|
417
|
+
}
|
|
418
|
+
async function unary(call, path, body, options, parse) {
|
|
419
|
+
try {
|
|
420
|
+
for (;;) {
|
|
421
|
+
const pick = await next(call);
|
|
422
|
+
begin(call, pick);
|
|
423
|
+
const scope = new AttemptScope(call, call.policy.timeoutMs);
|
|
424
|
+
const started = Date.now();
|
|
425
|
+
try {
|
|
426
|
+
const response = await open(call, pick, scope, path, body, options, "application/json");
|
|
427
|
+
let json;
|
|
428
|
+
try {
|
|
429
|
+
json = await response.json();
|
|
430
|
+
}
|
|
431
|
+
catch (error) {
|
|
432
|
+
throw scope.cause === null ? new AttemptError("transient", "the provider answered a body that is not JSON") : scope.classify(error);
|
|
433
|
+
}
|
|
434
|
+
if (!isRecord(json))
|
|
435
|
+
throw new AttemptError("transient", "the provider answered a body that is not an object");
|
|
436
|
+
if (json.error !== undefined && json.error !== null) {
|
|
437
|
+
const message = isRecord(json.error) && typeof json.error.message === "string" ? json.error.message : "the provider answered an error";
|
|
438
|
+
throw new AttemptError(classifyProviderError(json.error), message);
|
|
439
|
+
}
|
|
440
|
+
const parsed = parse(json);
|
|
441
|
+
const latencyMs = Date.now() - started;
|
|
442
|
+
const routing = await succeed(call, pick, readRateLimit(response.headers, Date.now()), latencyMs, parsed.usage, { request: body, response: json });
|
|
443
|
+
settle(call, routing, null);
|
|
444
|
+
return { value: parsed.value, routing };
|
|
445
|
+
}
|
|
446
|
+
catch (error) {
|
|
447
|
+
const failure = scope.classify(error);
|
|
448
|
+
await fail(call, pick, failure, Date.now() - started, body);
|
|
449
|
+
if (failure.errorClass === "cancelled" || failure.errorClass === "deadline")
|
|
450
|
+
throw call.stopped();
|
|
451
|
+
if (!isRetryable(failure.errorClass)) {
|
|
452
|
+
throw call.error(failure.errorClass, failure.message, statusOf(failure));
|
|
453
|
+
}
|
|
454
|
+
await backoff(call, failure.errorClass);
|
|
455
|
+
}
|
|
456
|
+
finally {
|
|
457
|
+
scope.dispose();
|
|
458
|
+
await state.release(pick.lane, pick.leases);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
catch (error) {
|
|
463
|
+
const routerError = toRouterError(call, error);
|
|
464
|
+
settle(call, null, routerError);
|
|
465
|
+
throw routerError;
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
async function* streamCall(call, body, options) {
|
|
469
|
+
let settled = false;
|
|
470
|
+
try {
|
|
471
|
+
for (;;) {
|
|
472
|
+
const pick = await next(call);
|
|
473
|
+
begin(call, pick);
|
|
474
|
+
const scope = new AttemptScope(call, call.policy.timeoutMs);
|
|
475
|
+
const started = Date.now();
|
|
476
|
+
let firstToken = false;
|
|
477
|
+
let usage;
|
|
478
|
+
let info = null;
|
|
479
|
+
// "pending" in the finally block means the consumer stopped reading: no catch ran and no success did.
|
|
480
|
+
let outcome = "pending";
|
|
481
|
+
try {
|
|
482
|
+
const response = await open(call, pick, scope, "/chat/completions", { ...body, stream: true }, options, "text/event-stream");
|
|
483
|
+
info = readRateLimit(response.headers, Date.now());
|
|
484
|
+
if (response.body === null)
|
|
485
|
+
throw new AttemptError("transient", "the provider answered no stream body");
|
|
486
|
+
const held = [];
|
|
487
|
+
for await (const event of parseSse(response.body)) {
|
|
488
|
+
if (event.data === "[DONE]")
|
|
489
|
+
break;
|
|
490
|
+
const chunk = parseChunk(event.data);
|
|
491
|
+
if (chunk === null)
|
|
492
|
+
continue;
|
|
493
|
+
if (isRecord(chunk.usage))
|
|
494
|
+
usage = chunk.usage;
|
|
495
|
+
scope.arm(call.policy.timeoutMs);
|
|
496
|
+
if (firstToken) {
|
|
497
|
+
yield chunk;
|
|
498
|
+
continue;
|
|
499
|
+
}
|
|
500
|
+
held.push(chunk);
|
|
501
|
+
if (hasToken(chunk)) {
|
|
502
|
+
firstToken = true;
|
|
503
|
+
for (const early of held.splice(0))
|
|
504
|
+
yield early;
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
for (const early of held.splice(0))
|
|
508
|
+
yield early;
|
|
509
|
+
outcome = "done";
|
|
510
|
+
const routing = await succeed(call, pick, info, Date.now() - started, usage, { request: body });
|
|
511
|
+
settled = true;
|
|
512
|
+
settle(call, routing, null);
|
|
513
|
+
return;
|
|
514
|
+
}
|
|
515
|
+
catch (error) {
|
|
516
|
+
outcome = "failed";
|
|
517
|
+
const failure = scope.classify(error);
|
|
518
|
+
await fail(call, pick, failure, Date.now() - started, body);
|
|
519
|
+
if (failure.errorClass === "cancelled" || failure.errorClass === "deadline")
|
|
520
|
+
throw call.stopped();
|
|
521
|
+
if (firstToken) {
|
|
522
|
+
throw call.error(failure.errorClass, `the stream failed after its first token: ${failure.message}`, { afterFirstToken: true, ...statusOf(failure) });
|
|
523
|
+
}
|
|
524
|
+
if (!isRetryable(failure.errorClass))
|
|
525
|
+
throw call.error(failure.errorClass, failure.message, statusOf(failure));
|
|
526
|
+
await backoff(call, failure.errorClass);
|
|
527
|
+
}
|
|
528
|
+
finally {
|
|
529
|
+
scope.controller.abort();
|
|
530
|
+
scope.dispose();
|
|
531
|
+
if (outcome === "pending") {
|
|
532
|
+
const routing = await succeed(call, pick, info, Date.now() - started, usage, { request: body });
|
|
533
|
+
settled = true;
|
|
534
|
+
settle(call, routing, null);
|
|
535
|
+
}
|
|
536
|
+
await state.release(pick.lane, pick.leases);
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
catch (error) {
|
|
541
|
+
const routerError = toRouterError(call, error);
|
|
542
|
+
settled = true;
|
|
543
|
+
settle(call, null, routerError);
|
|
544
|
+
throw routerError;
|
|
545
|
+
}
|
|
546
|
+
finally {
|
|
547
|
+
if (!settled)
|
|
548
|
+
settle(call, null, call.error("cancelled", "the consumer stopped reading"));
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
return {
|
|
552
|
+
async chat(request, options = {}) {
|
|
553
|
+
const body = { ...request };
|
|
554
|
+
delete body.stream;
|
|
555
|
+
const { start } = prepare("chat", request.model, options, (policy) => chatNeed(request, false, policy.require));
|
|
556
|
+
const { value, routing } = await unary(start(), "/chat/completions", body, options, (json) => ({
|
|
557
|
+
value: json,
|
|
558
|
+
usage: usageOf(json),
|
|
559
|
+
}));
|
|
560
|
+
return { ...value, routing };
|
|
561
|
+
},
|
|
562
|
+
stream(request, options = {}) {
|
|
563
|
+
const body = { ...request };
|
|
564
|
+
delete body.stream;
|
|
565
|
+
const { requestId, start } = prepare("stream", request.model, options, (policy) => chatNeed(request, true, policy.require));
|
|
566
|
+
return {
|
|
567
|
+
requestId,
|
|
568
|
+
[Symbol.asyncIterator]: () => streamCall(start(), body, options),
|
|
569
|
+
};
|
|
570
|
+
},
|
|
571
|
+
async embeddings(request, options = {}) {
|
|
572
|
+
const { start } = prepare("embeddings", request.model, options, (policy) => ({
|
|
573
|
+
capabilities: ["embeddings", ...policy.require],
|
|
574
|
+
promptTokens: estimateTokens(typeof request.input === "string" ? request.input : request.input.join("\n")),
|
|
575
|
+
outputTokens: 0,
|
|
576
|
+
}));
|
|
577
|
+
const { value, routing } = await unary(start(), "/embeddings", { ...request }, options, (json) => ({
|
|
578
|
+
value: json,
|
|
579
|
+
usage: usageOf(json),
|
|
580
|
+
}));
|
|
581
|
+
return { ...value, routing };
|
|
582
|
+
},
|
|
583
|
+
health: () => state.health(),
|
|
584
|
+
stats: () => state.stats(),
|
|
585
|
+
on: (event, handler) => emitter.on(event, handler),
|
|
586
|
+
reset: (target = {}) => state.reset(target),
|
|
587
|
+
};
|
|
588
|
+
}
|
package/dist/sse.d.ts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/** One server-sent event: its `event` name (`message` when absent) and its joined `data` lines. */
|
|
2
|
+
export interface SseEvent {
|
|
3
|
+
readonly event: string;
|
|
4
|
+
readonly data: string;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Parses a server-sent-event byte stream. It reads the body only when the consumer asks for the next event, so a slow
|
|
8
|
+
* consumer holds the socket back (backpressure) and a consumer that stops cancels the body.
|
|
9
|
+
*/
|
|
10
|
+
export declare function parseSse(body: ReadableStream<Uint8Array>): AsyncGenerator<SseEvent, void, undefined>;
|
package/dist/sse.js
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parses a server-sent-event byte stream. It reads the body only when the consumer asks for the next event, so a slow
|
|
3
|
+
* consumer holds the socket back (backpressure) and a consumer that stops cancels the body.
|
|
4
|
+
*/
|
|
5
|
+
export async function* parseSse(body) {
|
|
6
|
+
const reader = body.getReader();
|
|
7
|
+
const decoder = new TextDecoder();
|
|
8
|
+
let buffer = "";
|
|
9
|
+
let event = "";
|
|
10
|
+
let data = [];
|
|
11
|
+
let done = false;
|
|
12
|
+
try {
|
|
13
|
+
while (!done) {
|
|
14
|
+
const chunk = await reader.read();
|
|
15
|
+
done = chunk.done;
|
|
16
|
+
buffer += done ? decoder.decode() : decoder.decode(chunk.value, { stream: true });
|
|
17
|
+
// A trailing carriage return may be the first half of a CRLF split across chunks; keep it until more arrives.
|
|
18
|
+
const lines = buffer.split(/\r\n|\r(?!$)|\n/);
|
|
19
|
+
buffer = done ? "" : (lines.pop() ?? "");
|
|
20
|
+
for (const raw of lines) {
|
|
21
|
+
const line = raw.endsWith("\r") ? raw.slice(0, -1) : raw;
|
|
22
|
+
if (line === "") {
|
|
23
|
+
if (data.length > 0)
|
|
24
|
+
yield { event: event === "" ? "message" : event, data: data.join("\n") };
|
|
25
|
+
event = "";
|
|
26
|
+
data = [];
|
|
27
|
+
continue;
|
|
28
|
+
}
|
|
29
|
+
if (line.startsWith(":"))
|
|
30
|
+
continue;
|
|
31
|
+
const colon = line.indexOf(":");
|
|
32
|
+
const field = colon === -1 ? line : line.slice(0, colon);
|
|
33
|
+
const value = colon === -1 ? "" : line.slice(colon + 1).replace(/^ /, "");
|
|
34
|
+
if (field === "data")
|
|
35
|
+
data.push(value);
|
|
36
|
+
else if (field === "event")
|
|
37
|
+
event = value;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (data.length > 0)
|
|
41
|
+
yield { event: event === "" ? "message" : event, data: data.join("\n") };
|
|
42
|
+
}
|
|
43
|
+
finally {
|
|
44
|
+
if (!done)
|
|
45
|
+
await reader.cancel().catch(() => undefined);
|
|
46
|
+
reader.releaseLock();
|
|
47
|
+
}
|
|
48
|
+
}
|