@starci/ai-router 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/LICENSE +21 -0
- package/README.md +312 -2
- package/dist/builder.d.ts +33 -0
- package/dist/builder.js +49 -0
- package/dist/config.d.ts +46 -0
- package/dist/config.js +148 -0
- package/dist/errors.d.ts +47 -0
- package/dist/errors.js +66 -0
- package/dist/events.d.ts +11 -0
- package/dist/events.js +31 -0
- package/dist/fake.d.ts +67 -0
- package/dist/fake.js +153 -0
- package/dist/headers.d.ts +15 -0
- package/dist/headers.js +66 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +8 -0
- package/dist/pool.d.ts +90 -0
- package/dist/pool.js +421 -0
- package/dist/router.d.ts +3 -0
- package/dist/router.js +588 -0
- package/dist/sse.d.ts +10 -0
- package/dist/sse.js +48 -0
- package/dist/store.d.ts +56 -0
- package/dist/store.js +48 -0
- package/dist/types.d.ts +426 -0
- package/dist/types.js +1 -0
- package/package.json +66 -4
package/dist/errors.d.ts
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { ErrorClass } .js";
|
|
2
|
+
/** One failed attempt as an error lists it: where it went and why it failed, never a key value. */
|
|
3
|
+
export interface AttemptFailure {
|
|
4
|
+
readonly provider: string;
|
|
5
|
+
readonly key: string;
|
|
6
|
+
readonly model: string;
|
|
7
|
+
readonly errorClass: ErrorClass;
|
|
8
|
+
readonly status?: number;
|
|
9
|
+
readonly message: string;
|
|
10
|
+
}
|
|
11
|
+
/** The fields an `AiRouterError` is built from. */
|
|
12
|
+
export interface AiRouterErrorInit {
|
|
13
|
+
readonly errorClass: ErrorClass;
|
|
14
|
+
readonly message: string;
|
|
15
|
+
readonly status?: number;
|
|
16
|
+
readonly provider?: string;
|
|
17
|
+
readonly key?: string;
|
|
18
|
+
readonly model?: string;
|
|
19
|
+
readonly requestId?: string;
|
|
20
|
+
readonly attempts?: readonly AttemptFailure[];
|
|
21
|
+
readonly afterFirstToken?: boolean;
|
|
22
|
+
/** Epoch milliseconds a 429 asked to wait until. */
|
|
23
|
+
readonly retryAt?: number;
|
|
24
|
+
}
|
|
25
|
+
/** True for a class another key, provider or model may succeed at. */
|
|
26
|
+
export declare function isRetryable(errorClass: ErrorClass): boolean;
|
|
27
|
+
/** The one error the router throws. `errorClass` says why; `attempts` lists every failed attempt of the call. */
|
|
28
|
+
export declare class AiRouterError extends Error {
|
|
29
|
+
readonly errorClass: ErrorClass;
|
|
30
|
+
readonly retryable: boolean;
|
|
31
|
+
readonly status: number | undefined;
|
|
32
|
+
readonly provider: string | undefined;
|
|
33
|
+
readonly key: string | undefined;
|
|
34
|
+
readonly model: string | undefined;
|
|
35
|
+
readonly requestId: string | undefined;
|
|
36
|
+
readonly attempts: readonly AttemptFailure[];
|
|
37
|
+
readonly afterFirstToken: boolean;
|
|
38
|
+
readonly retryAt: number | undefined;
|
|
39
|
+
/** Builds the error from its fields. */
|
|
40
|
+
constructor(init: AiRouterErrorInit);
|
|
41
|
+
}
|
|
42
|
+
/** Classifies an HTTP status: 429 rate, 401/403 auth, 408/409/425/5xx transient, any other 4xx an invalid request. */
|
|
43
|
+
export declare function classifyStatus(status: number): ErrorClass;
|
|
44
|
+
/** Classifies an error object a provider sends inside a stream or a 200 body. */
|
|
45
|
+
export declare function classifyProviderError(error: unknown): ErrorClass;
|
|
46
|
+
/** Replaces every occurrence of each secret with `[redacted]`, so an echoed key never leaves the router. */
|
|
47
|
+
export declare function scrub(text: string, secrets: readonly string[]): string;
|
package/dist/errors.js
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
const RETRYABLE = new Set(["rate_limit", "auth", "transient", "timeout", "network", "stream"]);
|
|
2
|
+
/** True for a class another key, provider or model may succeed at. */
|
|
3
|
+
export function isRetryable(errorClass) {
|
|
4
|
+
return RETRYABLE.has(errorClass);
|
|
5
|
+
}
|
|
6
|
+
/** The one error the router throws. `errorClass` says why; `attempts` lists every failed attempt of the call. */
|
|
7
|
+
export class AiRouterError extends Error {
|
|
8
|
+
errorClass;
|
|
9
|
+
retryable;
|
|
10
|
+
status;
|
|
11
|
+
provider;
|
|
12
|
+
key;
|
|
13
|
+
model;
|
|
14
|
+
requestId;
|
|
15
|
+
attempts;
|
|
16
|
+
afterFirstToken;
|
|
17
|
+
retryAt;
|
|
18
|
+
/** Builds the error from its fields. */
|
|
19
|
+
constructor(init) {
|
|
20
|
+
super(init.message);
|
|
21
|
+
this.name = "AiRouterError";
|
|
22
|
+
this.errorClass = init.errorClass;
|
|
23
|
+
this.retryable = isRetryable(init.errorClass);
|
|
24
|
+
this.status = init.status;
|
|
25
|
+
this.provider = init.provider;
|
|
26
|
+
this.key = init.key;
|
|
27
|
+
this.model = init.model;
|
|
28
|
+
this.requestId = init.requestId;
|
|
29
|
+
this.attempts = init.attempts ?? [];
|
|
30
|
+
this.afterFirstToken = init.afterFirstToken ?? false;
|
|
31
|
+
this.retryAt = init.retryAt;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/** Classifies an HTTP status: 429 rate, 401/403 auth, 408/409/425/5xx transient, any other 4xx an invalid request. */
|
|
35
|
+
export function classifyStatus(status) {
|
|
36
|
+
if (status === 429)
|
|
37
|
+
return "rate_limit";
|
|
38
|
+
if (status === 401 || status === 403)
|
|
39
|
+
return "auth";
|
|
40
|
+
if (status === 408 || status === 409 || status === 425 || status >= 500)
|
|
41
|
+
return "transient";
|
|
42
|
+
return "invalid_request";
|
|
43
|
+
}
|
|
44
|
+
/** Classifies an error object a provider sends inside a stream or a 200 body. */
|
|
45
|
+
export function classifyProviderError(error) {
|
|
46
|
+
if (typeof error !== "object" || error === null)
|
|
47
|
+
return "transient";
|
|
48
|
+
const record = error;
|
|
49
|
+
const words = [record.type, record.code].filter((value) => typeof value === "string").join(" ");
|
|
50
|
+
if (/rate|quota|too_many/i.test(words))
|
|
51
|
+
return "rate_limit";
|
|
52
|
+
if (/auth|api_key|permission/i.test(words))
|
|
53
|
+
return "auth";
|
|
54
|
+
if (/invalid|context_length|not_found/i.test(words))
|
|
55
|
+
return "invalid_request";
|
|
56
|
+
return "transient";
|
|
57
|
+
}
|
|
58
|
+
/** Replaces every occurrence of each secret with `[redacted]`, so an echoed key never leaves the router. */
|
|
59
|
+
export function scrub(text, secrets) {
|
|
60
|
+
let out = text;
|
|
61
|
+
for (const secret of secrets) {
|
|
62
|
+
if (secret.length > 0)
|
|
63
|
+
out = out.split(secret).join("[redacted]");
|
|
64
|
+
}
|
|
65
|
+
return out;
|
|
66
|
+
}
|
package/dist/events.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { RouterEvents } .js";
|
|
2
|
+
type Handler<E extends keyof RouterEvents> = (payload: RouterEvents[E]) => void;
|
|
3
|
+
/** A typed event fan-out. A handler that throws is isolated: observing a call never breaks it. */
|
|
4
|
+
export declare class Emitter {
|
|
5
|
+
#private;
|
|
6
|
+
/** Subscribes and returns the unsubscribe function. */
|
|
7
|
+
on<E extends keyof RouterEvents>(event: E, handler: Handler<E>): () => void;
|
|
8
|
+
/** Delivers one event to every handler of its name. */
|
|
9
|
+
emit<E extends keyof RouterEvents>(event: E, payload: RouterEvents[E]): void;
|
|
10
|
+
}
|
|
11
|
+
export {};
|
package/dist/events.js
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/** A typed event fan-out. A handler that throws is isolated: observing a call never breaks it. */
|
|
2
|
+
export class Emitter {
|
|
3
|
+
#handlers = new Map();
|
|
4
|
+
/** Subscribes and returns the unsubscribe function. */
|
|
5
|
+
on(event, handler) {
|
|
6
|
+
let set = this.#handlers.get(event);
|
|
7
|
+
if (set === undefined) {
|
|
8
|
+
set = new Set();
|
|
9
|
+
this.#handlers.set(event, set);
|
|
10
|
+
}
|
|
11
|
+
set.add(handler);
|
|
12
|
+
return () => {
|
|
13
|
+
set.delete(handler);
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
/** Delivers one event to every handler of its name. */
|
|
17
|
+
emit(event, payload) {
|
|
18
|
+
const set = this.#handlers.get(event);
|
|
19
|
+
if (set === undefined)
|
|
20
|
+
return;
|
|
21
|
+
for (const handler of set) {
|
|
22
|
+
const typed = handler;
|
|
23
|
+
try {
|
|
24
|
+
typed(payload);
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
// An observer's failure is the observer's: the call it watches goes on.
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
package/dist/fake.d.ts
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import type { FetchLike, Usage } .js";
|
|
2
|
+
/** How a scripted stream behaves. */
|
|
3
|
+
export interface FakeStream {
|
|
4
|
+
/** The content pieces, one chunk each. Default: the reply text split at spaces. */
|
|
5
|
+
readonly chunks?: readonly string[];
|
|
6
|
+
/** The pause before each chunk. Default 0. */
|
|
7
|
+
readonly delayMs?: number;
|
|
8
|
+
/** Fails after this many content chunks (0: before the first token). */
|
|
9
|
+
readonly failAfter?: number;
|
|
10
|
+
/** Goes silent for `stallMs` after this many content chunks (an idle provider). */
|
|
11
|
+
readonly stallAfter?: number;
|
|
12
|
+
readonly stallMs?: number;
|
|
13
|
+
/** `network` breaks the body; `error-event` streams an OpenAI error object. Default `network`. */
|
|
14
|
+
readonly failWith?: "network" | "error-event";
|
|
15
|
+
/** Ends with a chunk that carries usage. Default true. */
|
|
16
|
+
readonly includeUsage?: boolean;
|
|
17
|
+
}
|
|
18
|
+
/** One scripted answer. Everything is optional: the default is a 200 chat reply `ok`. */
|
|
19
|
+
export interface FakeReply {
|
|
20
|
+
readonly status?: number;
|
|
21
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
22
|
+
/** The pause before the response headers; an abort during it rejects like `fetch`. */
|
|
23
|
+
readonly latencyMs?: number;
|
|
24
|
+
/** `network` rejects like `fetch` does when the connection fails. */
|
|
25
|
+
readonly error?: "network";
|
|
26
|
+
/** The chat reply text. Default `ok`. */
|
|
27
|
+
readonly text?: string;
|
|
28
|
+
readonly usage?: Usage;
|
|
29
|
+
/** The vector of every embedding. Default `[0.1, 0.2, 0.3]`. */
|
|
30
|
+
readonly embedding?: readonly number[];
|
|
31
|
+
/** Replaces the whole JSON body. */
|
|
32
|
+
readonly body?: unknown;
|
|
33
|
+
readonly stream?: FakeStream;
|
|
34
|
+
}
|
|
35
|
+
/** Which requests a script applies to; an absent field matches anything. */
|
|
36
|
+
export interface FakeMatch {
|
|
37
|
+
/** The provider's base URL the request starts with. */
|
|
38
|
+
readonly baseUrl?: string;
|
|
39
|
+
/** The bearer key value (a test value). */
|
|
40
|
+
readonly key?: string;
|
|
41
|
+
readonly model?: string;
|
|
42
|
+
readonly path?: string;
|
|
43
|
+
}
|
|
44
|
+
/** One request the fake received. */
|
|
45
|
+
export interface FakeCall {
|
|
46
|
+
readonly url: string;
|
|
47
|
+
readonly path: string;
|
|
48
|
+
/** The bearer key value sent (a test value). */
|
|
49
|
+
readonly key: string;
|
|
50
|
+
readonly model: string;
|
|
51
|
+
readonly stream: boolean;
|
|
52
|
+
readonly headers: Readonly<Record<string, string>>;
|
|
53
|
+
readonly body: Readonly<Record<string, unknown>>;
|
|
54
|
+
}
|
|
55
|
+
/** A scriptable OpenAI-compatible provider that answers in process, for consumers' tests. */
|
|
56
|
+
export interface FakeProvider {
|
|
57
|
+
/** Pass as `PoolConfig.fetch`. */
|
|
58
|
+
readonly fetch: FetchLike;
|
|
59
|
+
/** Every request received, in order. */
|
|
60
|
+
readonly calls: readonly FakeCall[];
|
|
61
|
+
/** Adds a script: matching requests get the replies in order, and the last one repeats. A later script wins. */
|
|
62
|
+
on(match: FakeMatch, ...replies: FakeReply[]): FakeProvider;
|
|
63
|
+
/** Forgets every script and call. */
|
|
64
|
+
reset(): void;
|
|
65
|
+
}
|
|
66
|
+
/** Creates a fake OpenAI-compatible provider: deterministic replies, latency, errors, 429s and streams by script. */
|
|
67
|
+
export declare function createFakeProvider(): FakeProvider;
|
package/dist/fake.js
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
function wait(ms, signal) {
|
|
2
|
+
return new Promise((resolve, reject) => {
|
|
3
|
+
if (signal?.aborted === true) {
|
|
4
|
+
reject(signal.reason);
|
|
5
|
+
return;
|
|
6
|
+
}
|
|
7
|
+
const timer = setTimeout(() => {
|
|
8
|
+
signal?.removeEventListener("abort", stop);
|
|
9
|
+
resolve();
|
|
10
|
+
}, ms);
|
|
11
|
+
const stop = () => {
|
|
12
|
+
clearTimeout(timer);
|
|
13
|
+
reject(signal?.reason);
|
|
14
|
+
};
|
|
15
|
+
signal?.addEventListener("abort", stop, { once: true });
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
function matches(match, call) {
|
|
19
|
+
return ((match.baseUrl === undefined || call.url.startsWith(match.baseUrl)) &&
|
|
20
|
+
(match.key === undefined || match.key === call.key) &&
|
|
21
|
+
(match.model === undefined || match.model === call.model) &&
|
|
22
|
+
(match.path === undefined || call.path.endsWith(match.path)));
|
|
23
|
+
}
|
|
24
|
+
function headerRecord(init) {
|
|
25
|
+
const out = {};
|
|
26
|
+
new Headers(init.headers).forEach((value, name) => {
|
|
27
|
+
out[name] = value;
|
|
28
|
+
});
|
|
29
|
+
return out;
|
|
30
|
+
}
|
|
31
|
+
function streamBody(reply, model, signal) {
|
|
32
|
+
const options = reply.stream ?? {};
|
|
33
|
+
const pieces = options.chunks ?? (reply.text ?? "ok").split(/(?<= )/);
|
|
34
|
+
const encoder = new TextEncoder();
|
|
35
|
+
const base = { id: "fake-stream", object: "chat.completion.chunk", created: 0, model };
|
|
36
|
+
const frames = [JSON.stringify({ ...base, choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }] })];
|
|
37
|
+
pieces.forEach((piece, index) => {
|
|
38
|
+
if (options.stallAfter === index)
|
|
39
|
+
frames.push("stall");
|
|
40
|
+
if (options.failAfter === index)
|
|
41
|
+
frames.push("fail");
|
|
42
|
+
frames.push(JSON.stringify({ ...base, choices: [{ index: 0, delta: { content: piece }, finish_reason: null }] }));
|
|
43
|
+
});
|
|
44
|
+
if (options.failAfter !== undefined && options.failAfter >= pieces.length)
|
|
45
|
+
frames.push("fail");
|
|
46
|
+
frames.push(JSON.stringify({ ...base, choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }));
|
|
47
|
+
if (options.includeUsage !== false) {
|
|
48
|
+
const usage = reply.usage ?? { prompt_tokens: 10, completion_tokens: pieces.length, total_tokens: 10 + pieces.length };
|
|
49
|
+
frames.push(JSON.stringify({ ...base, choices: [], usage }));
|
|
50
|
+
}
|
|
51
|
+
frames.push("[DONE]");
|
|
52
|
+
let index = 0;
|
|
53
|
+
return new ReadableStream({
|
|
54
|
+
async pull(controller) {
|
|
55
|
+
let frame = frames[index++];
|
|
56
|
+
try {
|
|
57
|
+
if (frame === "stall") {
|
|
58
|
+
await wait(options.stallMs ?? 1000, signal);
|
|
59
|
+
frame = frames[index++];
|
|
60
|
+
}
|
|
61
|
+
if (options.delayMs !== undefined && options.delayMs > 0)
|
|
62
|
+
await wait(options.delayMs, signal);
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
controller.error(error);
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
if (frame === undefined) {
|
|
69
|
+
controller.close();
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
if (frame !== "fail") {
|
|
73
|
+
controller.enqueue(encoder.encode(`data: ${frame}\n\n`));
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
if (options.failWith === "error-event") {
|
|
77
|
+
controller.enqueue(encoder.encode(`data: ${JSON.stringify({ error: { message: "fake stream error", type: "server_error" } })}\n\n`));
|
|
78
|
+
index = frames.length;
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
controller.error(new TypeError("terminated"));
|
|
82
|
+
},
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
function jsonBody(reply, call) {
|
|
86
|
+
if (reply.body !== undefined)
|
|
87
|
+
return reply.body;
|
|
88
|
+
const status = reply.status ?? 200;
|
|
89
|
+
if (status >= 400)
|
|
90
|
+
return { error: { message: `fake ${String(status)}`, type: status === 429 ? "rate_limit_exceeded" : "fake_error" } };
|
|
91
|
+
if (call.path.endsWith("/embeddings")) {
|
|
92
|
+
const input = call.body.input;
|
|
93
|
+
const count = Array.isArray(input) ? input.length : 1;
|
|
94
|
+
return {
|
|
95
|
+
object: "list",
|
|
96
|
+
data: Array.from({ length: count }, (_, index) => ({ object: "embedding", index, embedding: reply.embedding ?? [0.1, 0.2, 0.3] })),
|
|
97
|
+
model: call.model,
|
|
98
|
+
usage: reply.usage ?? { prompt_tokens: 4 * count, total_tokens: 4 * count },
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
const text = reply.text ?? "ok";
|
|
102
|
+
return {
|
|
103
|
+
id: `fake-${String(Date.now())}`,
|
|
104
|
+
object: "chat.completion",
|
|
105
|
+
created: 0,
|
|
106
|
+
model: call.model,
|
|
107
|
+
choices: [{ index: 0, message: { role: "assistant", content: text }, finish_reason: "stop" }],
|
|
108
|
+
usage: reply.usage ?? { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 },
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
/** Creates a fake OpenAI-compatible provider: deterministic replies, latency, errors, 429s and streams by script. */
|
|
112
|
+
export function createFakeProvider() {
|
|
113
|
+
let scripts = [];
|
|
114
|
+
const calls = [];
|
|
115
|
+
const fake = {
|
|
116
|
+
calls,
|
|
117
|
+
on(match, ...replies) {
|
|
118
|
+
scripts.unshift({ match, replies: replies.length === 0 ? [{}] : [...replies] });
|
|
119
|
+
return fake;
|
|
120
|
+
},
|
|
121
|
+
reset() {
|
|
122
|
+
scripts = [];
|
|
123
|
+
calls.length = 0;
|
|
124
|
+
},
|
|
125
|
+
async fetch(input, init) {
|
|
126
|
+
const url = new URL(input);
|
|
127
|
+
const headers = headerRecord(init);
|
|
128
|
+
const body = (typeof init.body === "string" ? JSON.parse(init.body) : {});
|
|
129
|
+
const call = {
|
|
130
|
+
url: input,
|
|
131
|
+
path: url.pathname,
|
|
132
|
+
key: (headers.authorization ?? "").replace(/^Bearer /, ""),
|
|
133
|
+
model: typeof body.model === "string" ? body.model : "",
|
|
134
|
+
stream: body.stream === true,
|
|
135
|
+
headers,
|
|
136
|
+
body,
|
|
137
|
+
};
|
|
138
|
+
calls.push(call);
|
|
139
|
+
const script = scripts.find((candidate) => matches(candidate.match, call));
|
|
140
|
+
const reply = (script === undefined ? undefined : script.replies.length > 1 ? script.replies.shift() : script.replies[0]) ?? {};
|
|
141
|
+
if (reply.latencyMs !== undefined && reply.latencyMs > 0)
|
|
142
|
+
await wait(reply.latencyMs, init.signal);
|
|
143
|
+
if (reply.error === "network")
|
|
144
|
+
throw new TypeError("fetch failed");
|
|
145
|
+
const status = reply.status ?? 200;
|
|
146
|
+
if (call.stream && status < 400 && reply.body === undefined) {
|
|
147
|
+
return new Response(streamBody(reply, call.model, init.signal), { status, headers: { "content-type": "text/event-stream", ...reply.headers } });
|
|
148
|
+
}
|
|
149
|
+
return new Response(JSON.stringify(jsonBody(reply, call)), { status, headers: { "content-type": "application/json", ...reply.headers } });
|
|
150
|
+
},
|
|
151
|
+
};
|
|
152
|
+
return fake;
|
|
153
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** What a response's headers say about a key's headroom; times are epoch milliseconds, 0 when absent. */
|
|
2
|
+
export interface RateLimitInfo {
|
|
3
|
+
readonly remainingRequests: number | null;
|
|
4
|
+
readonly requestsResetAt: number;
|
|
5
|
+
readonly remainingTokens: number | null;
|
|
6
|
+
readonly tokensResetAt: number;
|
|
7
|
+
/** From `retry-after-ms` or `retry-after` (seconds or an HTTP date). */
|
|
8
|
+
readonly retryAt: number;
|
|
9
|
+
}
|
|
10
|
+
/** Parses an OpenAI reset duration (`1s`, `6m0s`, `20ms`, `1h2m`) or a plain number of seconds into milliseconds. */
|
|
11
|
+
export declare function parseDuration(value: string): number | null;
|
|
12
|
+
/** Reads `retry-after-ms`, then `retry-after` as seconds or an HTTP date. */
|
|
13
|
+
export declare function retryAt(headers: Headers, now: number): number;
|
|
14
|
+
/** Reads the `x-ratelimit-*`, `ratelimit-*` and `retry-after` headers of a response. */
|
|
15
|
+
export declare function readRateLimit(headers: Headers, now: number): RateLimitInfo;
|
package/dist/headers.js
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
const DURATION = /^(?:(\d+(?:\.\d+)?)h)?(?:(\d+(?:\.\d+)?)m(?!s))?(?:(\d+(?:\.\d+)?)s)?(?:(\d+(?:\.\d+)?)ms)?$/;
|
|
2
|
+
/** Parses an OpenAI reset duration (`1s`, `6m0s`, `20ms`, `1h2m`) or a plain number of seconds into milliseconds. */
|
|
3
|
+
export function parseDuration(value) {
|
|
4
|
+
const text = value.trim();
|
|
5
|
+
if (text === "")
|
|
6
|
+
return null;
|
|
7
|
+
if (/^\d+(?:\.\d+)?$/.test(text))
|
|
8
|
+
return Number(text) * 1000;
|
|
9
|
+
const match = DURATION.exec(text);
|
|
10
|
+
if (match === null)
|
|
11
|
+
return null;
|
|
12
|
+
const [, h, m, s, ms] = match;
|
|
13
|
+
if (h === undefined && m === undefined && s === undefined && ms === undefined)
|
|
14
|
+
return null;
|
|
15
|
+
return Number(h ?? 0) * 3_600_000 + Number(m ?? 0) * 60_000 + Number(s ?? 0) * 1000 + Number(ms ?? 0);
|
|
16
|
+
}
|
|
17
|
+
function count(headers, names) {
|
|
18
|
+
for (const name of names) {
|
|
19
|
+
const raw = headers.get(name);
|
|
20
|
+
if (raw === null)
|
|
21
|
+
continue;
|
|
22
|
+
const value = Number(raw.trim());
|
|
23
|
+
if (Number.isFinite(value) && value >= 0)
|
|
24
|
+
return value;
|
|
25
|
+
}
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
function resetAt(headers, names, now) {
|
|
29
|
+
for (const name of names) {
|
|
30
|
+
const raw = headers.get(name);
|
|
31
|
+
if (raw === null)
|
|
32
|
+
continue;
|
|
33
|
+
const plain = Number(raw.trim());
|
|
34
|
+
// A plain number past 1e9 is an epoch in seconds (x-ratelimit-reset of some providers); smaller is a delta.
|
|
35
|
+
if (Number.isFinite(plain) && plain > 1e9)
|
|
36
|
+
return plain * 1000;
|
|
37
|
+
const ms = parseDuration(raw);
|
|
38
|
+
if (ms !== null)
|
|
39
|
+
return now + ms;
|
|
40
|
+
}
|
|
41
|
+
return 0;
|
|
42
|
+
}
|
|
43
|
+
/** Reads `retry-after-ms`, then `retry-after` as seconds or an HTTP date. */
|
|
44
|
+
export function retryAt(headers, now) {
|
|
45
|
+
const ms = headers.get("retry-after-ms");
|
|
46
|
+
if (ms !== null && Number.isFinite(Number(ms)))
|
|
47
|
+
return now + Number(ms);
|
|
48
|
+
const after = headers.get("retry-after");
|
|
49
|
+
if (after === null)
|
|
50
|
+
return 0;
|
|
51
|
+
const seconds = Number(after.trim());
|
|
52
|
+
if (Number.isFinite(seconds))
|
|
53
|
+
return now + seconds * 1000;
|
|
54
|
+
const date = Date.parse(after);
|
|
55
|
+
return Number.isNaN(date) ? 0 : date;
|
|
56
|
+
}
|
|
57
|
+
/** Reads the `x-ratelimit-*`, `ratelimit-*` and `retry-after` headers of a response. */
|
|
58
|
+
export function readRateLimit(headers, now) {
|
|
59
|
+
return {
|
|
60
|
+
remainingRequests: count(headers, ["x-ratelimit-remaining-requests", "x-ratelimit-remaining", "ratelimit-remaining"]),
|
|
61
|
+
requestsResetAt: resetAt(headers, ["x-ratelimit-reset-requests", "x-ratelimit-reset", "ratelimit-reset"], now),
|
|
62
|
+
remainingTokens: count(headers, ["x-ratelimit-remaining-tokens"]),
|
|
63
|
+
tokensResetAt: resetAt(headers, ["x-ratelimit-reset-tokens"], now),
|
|
64
|
+
retryAt: retryAt(headers, now),
|
|
65
|
+
};
|
|
66
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export { createRouter } .js";
|
|
2
|
+
export { definePool, PoolBuilder } .js";
|
|
3
|
+
export type { KeyOptions, ModelOptions, ProviderOptions } .js";
|
|
4
|
+
export { validateConfig } .js";
|
|
5
|
+
export { AiRouterError, classifyStatus } .js";
|
|
6
|
+
export type { AiRouterErrorInit, AttemptFailure } .js";
|
|
7
|
+
export { parseDuration, readRateLimit } .js";
|
|
8
|
+
export type { RateLimitInfo } .js";
|
|
9
|
+
export { parseSse } .js";
|
|
10
|
+
export type { SseEvent } .js";
|
|
11
|
+
export { EMPTY_RECORD, MemoryStateStore } .js";
|
|
12
|
+
export type { HealthRecord, StateStore } .js";
|
|
13
|
+
export { createFakeProvider } .js";
|
|
14
|
+
export type { FakeCall, FakeMatch, FakeProvider, FakeReply, FakeStream } .js";
|
|
15
|
+
export type * .js";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export { createRouter } from "./router.js";
|
|
2
|
+
export { definePool, PoolBuilder } from "./builder.js";
|
|
3
|
+
export { validateConfig } from "./config.js";
|
|
4
|
+
export { AiRouterError, classifyStatus } from "./errors.js";
|
|
5
|
+
export { parseDuration, readRateLimit } from "./headers.js";
|
|
6
|
+
export { parseSse } from "./sse.js";
|
|
7
|
+
export { EMPTY_RECORD, MemoryStateStore } from "./store.js";
|
|
8
|
+
export { createFakeProvider } from "./fake.js";
|
package/dist/pool.d.ts
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import type { ResolvedPolicy, Settings } .js";
|
|
2
|
+
import type { Emitter } .js";
|
|
3
|
+
import type { RateLimitInfo } .js";
|
|
4
|
+
import type { StateStore } .js";
|
|
5
|
+
import type { Capability, ErrorClass, HealthSnapshot, KeyConfig, ModelConfig, ProviderConfig, ResetTarget, StatsSnapshot, Usage } .js";
|
|
6
|
+
/** One routable (provider, key, model) triple and the rank of its model in the call's policy. */
|
|
7
|
+
export interface Lane {
|
|
8
|
+
readonly provider: ProviderConfig;
|
|
9
|
+
readonly key: KeyConfig;
|
|
10
|
+
readonly model: ModelConfig;
|
|
11
|
+
readonly rank: number;
|
|
12
|
+
/** `provider/key/model`, the lane's name in local counters. */
|
|
13
|
+
readonly id: string;
|
|
14
|
+
/** `provider/key`, the pair a retry moves away from. */
|
|
15
|
+
readonly pair: string;
|
|
16
|
+
}
|
|
17
|
+
/** What one call needs from a lane. */
|
|
18
|
+
export interface Need {
|
|
19
|
+
readonly capabilities: readonly Capability[];
|
|
20
|
+
/** An estimate of the prompt's tokens (four characters a token). */
|
|
21
|
+
readonly promptTokens: number;
|
|
22
|
+
/** `max_tokens` or `max_completion_tokens`, 0 when unbounded. */
|
|
23
|
+
readonly outputTokens: number;
|
|
24
|
+
}
|
|
25
|
+
/** What a call has tried so far: lanes, key pairs and providers. */
|
|
26
|
+
export interface Tried {
|
|
27
|
+
readonly lanes: ReadonlySet<string>;
|
|
28
|
+
readonly pairs: ReadonlySet<string>;
|
|
29
|
+
readonly providers: ReadonlySet<string>;
|
|
30
|
+
}
|
|
31
|
+
/** A lane picked for an attempt, with the probe leases it holds while a breaker is half-open. */
|
|
32
|
+
export interface Pick {
|
|
33
|
+
readonly lane: Lane;
|
|
34
|
+
readonly leases: readonly string[];
|
|
35
|
+
}
|
|
36
|
+
/** No lane is usable now: `waitUntil` is when the earliest one may be (Infinity: never within reach). */
|
|
37
|
+
export interface Blocked {
|
|
38
|
+
readonly waitUntil: number;
|
|
39
|
+
/** True when a lane waits only on in-process concurrency or a probe another attempt holds. */
|
|
40
|
+
readonly busy: boolean;
|
|
41
|
+
}
|
|
42
|
+
/** The cost of a usage on a model, or undefined when the model has no cost or the provider reported no usage. */
|
|
43
|
+
export declare function costOf(model: ModelConfig, usage: Usage | undefined): number | undefined;
|
|
44
|
+
/**
|
|
45
|
+
* The pool's state: shared breaker, disabled-key and headroom records in the store; in-flight counts, latency EWMAs and
|
|
46
|
+
* round-robin weights in this process. It answers which lane an attempt takes and records how the attempt ended.
|
|
47
|
+
*/
|
|
48
|
+
export declare class PoolState {
|
|
49
|
+
#private;
|
|
50
|
+
readonly totals: {
|
|
51
|
+
requests: number;
|
|
52
|
+
succeeded: number;
|
|
53
|
+
failed: number;
|
|
54
|
+
attempts: number;
|
|
55
|
+
};
|
|
56
|
+
/** Binds the state to a pool's providers, settings, store and emitter. */
|
|
57
|
+
constructor(providers: readonly ProviderConfig[], settings: Settings, store: StateStore, emitter: Emitter);
|
|
58
|
+
/** Every lane that serves one of the policy's models with the capabilities, context window and budget the call needs. */
|
|
59
|
+
lanes(policy: ResolvedPolicy, need: Need): Lane[];
|
|
60
|
+
/**
|
|
61
|
+
* Picks the lane for the next attempt and holds its in-flight slot. Tiers, in order: a lane not yet tried by this
|
|
62
|
+
* call, the policy's model order, a key not yet tried, a provider not yet tried, enough headroom. So a retry moves
|
|
63
|
+
* to another key or provider of the same model first, then to the next model, and only then back to a tried lane.
|
|
64
|
+
* Within the first non-empty tier:
|
|
65
|
+
* the fewest in-flight requests, then latency within the tolerance of the best EWMA, then smooth weighted
|
|
66
|
+
* round-robin. The concurrency check and the hold run without an await between them, so concurrent calls never
|
|
67
|
+
* overshoot a cap.
|
|
68
|
+
*/
|
|
69
|
+
pick(lanes: readonly Lane[], need: Need, tried: Tried, leaseMs: number): Promise<Pick | Blocked>;
|
|
70
|
+
/** Counts an attempt on its lane (the in-flight slot is already held by `pick`). */
|
|
71
|
+
count(lane: Lane): void;
|
|
72
|
+
/** Ends an in-flight attempt, frees its probe leases and wakes every call waiting for capacity. */
|
|
73
|
+
release(lane: Lane, leases: readonly string[]): Promise<void>;
|
|
74
|
+
/** Resolves on the next release or after `ms`, whichever comes first, or rejects when the signal aborts. */
|
|
75
|
+
waitForCapacity(ms: number, signal: AbortSignal): Promise<void>;
|
|
76
|
+
/** Records a success: closes the lane's breakers, stores its headroom and feeds its latency EWMA. */
|
|
77
|
+
succeeded(lane: Lane, info: RateLimitInfo | null, latencyMs: number, usage: Usage | undefined): Promise<number | undefined>;
|
|
78
|
+
/**
|
|
79
|
+
* Records a failure by class: a 429 cools the lane down (retry-after or the default), an auth failure disables the
|
|
80
|
+
* key, a transient failure counts toward the key's and the provider's breakers (honouring retry-after). Any other
|
|
81
|
+
* class touches no shared state.
|
|
82
|
+
*/
|
|
83
|
+
failed(lane: Lane, errorClass: ErrorClass, info: RateLimitInfo | null, missingKey: boolean): Promise<void>;
|
|
84
|
+
/** Reads every record into a health snapshot. */
|
|
85
|
+
health(): Promise<HealthSnapshot>;
|
|
86
|
+
/** Returns this process's counters. */
|
|
87
|
+
stats(): StatsSnapshot;
|
|
88
|
+
/** Deletes the records of the target and forgets its local latency and round-robin state. */
|
|
89
|
+
reset(target: ResetTarget): Promise<void>;
|
|
90
|
+
}
|