@starci/ai-router 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/LICENSE +21 -0
- package/README.md +399 -2
- package/dist/ai-router-testing.module.d.ts +13 -0
- package/dist/ai-router-testing.module.js +8 -0
- package/dist/ai-router.module-definition.d.ts +13 -0
- package/dist/ai-router.module-definition.js +6 -0
- package/dist/ai-router.module.d.ts +10 -0
- package/dist/ai-router.module.js +36 -0
- package/dist/ai-router.options.d.ts +10 -0
- package/dist/ai-router.options.js +1 -0
- package/dist/ai-router.service.d.ts +29 -0
- package/dist/ai-router.service.js +65 -0
- package/dist/ai-router.tokens.d.ts +11 -0
- package/dist/ai-router.tokens.js +20 -0
- package/dist/config.d.ts +46 -0
- package/dist/config.js +148 -0
- package/dist/errors.d.ts +47 -0
- package/dist/errors.js +66 -0
- package/dist/events.d.ts +11 -0
- package/dist/events.js +31 -0
- package/dist/fake.d.ts +67 -0
- package/dist/fake.js +153 -0
- package/dist/headers.d.ts +15 -0
- package/dist/headers.js +66 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +4 -0
- package/dist/pool.d.ts +90 -0
- package/dist/pool.js +435 -0
- package/dist/router.d.ts +3 -0
- package/dist/router.js +588 -0
- package/dist/sse.d.ts +10 -0
- package/dist/sse.js +48 -0
- package/dist/store.d.ts +56 -0
- package/dist/store.js +48 -0
- package/dist/testing.d.ts +4 -0
- package/dist/testing.js +2 -0
- package/dist/types.d.ts +427 -0
- package/dist/types.js +1 -0
- package/package.json +81 -4
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The injectable facade over one router. Every call it makes also follows the application's shutdown, and every
|
|
3
|
+
* handler it subscribed is released then; calls made on the raw router (`AI_ROUTER`) are the caller's to cancel.
|
|
4
|
+
*/
|
|
5
|
+
export class AiRouterService {
|
|
6
|
+
/** The raw router this service delegates to. */
|
|
7
|
+
router;
|
|
8
|
+
#shutdown = new AbortController();
|
|
9
|
+
#releases = new Set();
|
|
10
|
+
constructor(router, handlers = {}) {
|
|
11
|
+
this.router = router;
|
|
12
|
+
if (handlers.attempt !== undefined)
|
|
13
|
+
this.on("attempt", handlers.attempt);
|
|
14
|
+
if (handlers.breaker !== undefined)
|
|
15
|
+
this.on("breaker", handlers.breaker);
|
|
16
|
+
if (handlers["key-disabled"] !== undefined)
|
|
17
|
+
this.on("key-disabled", handlers["key-disabled"]);
|
|
18
|
+
if (handlers.request !== undefined)
|
|
19
|
+
this.on("request", handlers.request);
|
|
20
|
+
}
|
|
21
|
+
#options(options) {
|
|
22
|
+
const signal = options.signal === undefined ? this.#shutdown.signal : AbortSignal.any([this.#shutdown.signal, options.signal]);
|
|
23
|
+
return { ...options, signal };
|
|
24
|
+
}
|
|
25
|
+
/** Sends a chat-completions request through the pool. */
|
|
26
|
+
chat(request, options = {}) {
|
|
27
|
+
return this.router.chat(request, this.#options(options));
|
|
28
|
+
}
|
|
29
|
+
/** Streams a chat completion; the stream ends `cancelled` when the application shuts down. */
|
|
30
|
+
stream(request, options = {}) {
|
|
31
|
+
return this.router.stream(request, this.#options(options));
|
|
32
|
+
}
|
|
33
|
+
/** Sends an embeddings request through the pool. */
|
|
34
|
+
embeddings(request, options = {}) {
|
|
35
|
+
return this.router.embeddings(request, this.#options(options));
|
|
36
|
+
}
|
|
37
|
+
/** Reads every breaker, disabled key and headroom from the store. */
|
|
38
|
+
health() {
|
|
39
|
+
return this.router.health();
|
|
40
|
+
}
|
|
41
|
+
/** Returns this process's counters. */
|
|
42
|
+
stats() {
|
|
43
|
+
return this.router.stats();
|
|
44
|
+
}
|
|
45
|
+
/** Subscribes to an event until the application shuts down; returns the unsubscribe function. */
|
|
46
|
+
on(event, handler) {
|
|
47
|
+
const unsubscribe = this.router.on(event, handler);
|
|
48
|
+
const release = () => {
|
|
49
|
+
unsubscribe();
|
|
50
|
+
this.#releases.delete(release);
|
|
51
|
+
};
|
|
52
|
+
this.#releases.add(release);
|
|
53
|
+
return release;
|
|
54
|
+
}
|
|
55
|
+
/** Closes breakers, re-enables keys and clears headroom for the target (everything by default). */
|
|
56
|
+
reset(target) {
|
|
57
|
+
return this.router.reset(target);
|
|
58
|
+
}
|
|
59
|
+
/** Aborts every call still in flight through this service and releases its handlers. */
|
|
60
|
+
onApplicationShutdown() {
|
|
61
|
+
this.#shutdown.abort();
|
|
62
|
+
for (const release of [...this.#releases])
|
|
63
|
+
release();
|
|
64
|
+
}
|
|
65
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { AiRouterService } .js";
|
|
2
|
+
/** The injection token of the default instance's raw router (`AiRouter`). */
|
|
3
|
+
export declare const AI_ROUTER = "AI_ROUTER";
|
|
4
|
+
/** The token of a raw router: `AI_ROUTER` for the default instance, `AI_ROUTER:<name>` for a named one. */
|
|
5
|
+
export declare function getAiRouterToken(name?: string): string;
|
|
6
|
+
/** The token of a service: the `AiRouterService` class for the default instance, `AiRouterService:<name>` for a named one. */
|
|
7
|
+
export declare function getAiRouterServiceToken(name?: string): typeof AiRouterService | string;
|
|
8
|
+
/** Injects the raw router of the default instance, or of the instance registered under `name`. */
|
|
9
|
+
export declare function InjectAiRouter(name?: string): PropertyDecorator & ParameterDecorator;
|
|
10
|
+
/** Injects the `AiRouterService` of the default instance, or of the instance registered under `name`. */
|
|
11
|
+
export declare function InjectAiRouterService(name?: string): PropertyDecorator & ParameterDecorator;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { Inject } from "@nestjs/common";
|
|
2
|
+
import { AiRouterService } from "./ai-router.service.js";
|
|
3
|
+
/** The injection token of the default instance's raw router (`AiRouter`). */
|
|
4
|
+
export const AI_ROUTER = "AI_ROUTER";
|
|
5
|
+
/** The token of a raw router: `AI_ROUTER` for the default instance, `AI_ROUTER:<name>` for a named one. */
|
|
6
|
+
export function getAiRouterToken(name) {
|
|
7
|
+
return name === undefined ? AI_ROUTER : `${AI_ROUTER}:${name}`;
|
|
8
|
+
}
|
|
9
|
+
/** The token of a service: the `AiRouterService` class for the default instance, `AiRouterService:<name>` for a named one. */
|
|
10
|
+
export function getAiRouterServiceToken(name) {
|
|
11
|
+
return name === undefined ? AiRouterService : `AiRouterService:${name}`;
|
|
12
|
+
}
|
|
13
|
+
/** Injects the raw router of the default instance, or of the instance registered under `name`. */
|
|
14
|
+
export function InjectAiRouter(name) {
|
|
15
|
+
return Inject(getAiRouterToken(name));
|
|
16
|
+
}
|
|
17
|
+
/** Injects the `AiRouterService` of the default instance, or of the instance registered under `name`. */
|
|
18
|
+
export function InjectAiRouterService(name) {
|
|
19
|
+
return Inject(getAiRouterServiceToken(name));
|
|
20
|
+
}
|
package/dist/config.d.ts
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { Capability, CallOptions, ModelConfig, PoolConfig } .js";
|
|
2
|
+
/** The capabilities of a model that declares none. */
|
|
3
|
+
export declare const DEFAULT_CAPABILITIES: readonly Capability[];
|
|
4
|
+
/** A breaker's thresholds with every default applied. */
|
|
5
|
+
export interface BreakerSettings {
|
|
6
|
+
readonly failureThreshold: number;
|
|
7
|
+
readonly cooldownMs: number;
|
|
8
|
+
readonly maxCooldownMs: number;
|
|
9
|
+
}
|
|
10
|
+
/** The pool-wide settings with every default applied. */
|
|
11
|
+
export interface Settings {
|
|
12
|
+
readonly keyBreaker: BreakerSettings;
|
|
13
|
+
readonly providerBreaker: BreakerSettings;
|
|
14
|
+
readonly backoff: {
|
|
15
|
+
readonly baseMs: number;
|
|
16
|
+
readonly factor: number;
|
|
17
|
+
readonly maxMs: number;
|
|
18
|
+
};
|
|
19
|
+
readonly authDisableMs: number | null;
|
|
20
|
+
readonly rateLimitCooldownMs: number;
|
|
21
|
+
readonly latencyTolerance: number;
|
|
22
|
+
readonly latencySlackMs: number;
|
|
23
|
+
readonly ewmaAlpha: number;
|
|
24
|
+
readonly lowHeadroom: number;
|
|
25
|
+
readonly includeContent: boolean;
|
|
26
|
+
readonly namespace: string;
|
|
27
|
+
}
|
|
28
|
+
/** One call's policy with every default applied and its model list expanded. */
|
|
29
|
+
export interface ResolvedPolicy {
|
|
30
|
+
/** The preferred model first, then the fallbacks when fallback is allowed, without repeats. */
|
|
31
|
+
readonly models: readonly string[];
|
|
32
|
+
readonly maxAttempts: number;
|
|
33
|
+
readonly timeoutMs: number;
|
|
34
|
+
readonly deadlineMs: number;
|
|
35
|
+
readonly maxCost: number | null;
|
|
36
|
+
readonly providers: ReadonlySet<string> | null;
|
|
37
|
+
readonly require: readonly Capability[];
|
|
38
|
+
}
|
|
39
|
+
/** True when the name is the model's id or one of its aliases. */
|
|
40
|
+
export declare function modelMatches(model: ModelConfig, name: string): boolean;
|
|
41
|
+
/** Refuses a pool that cannot route: no providers, duplicate ids, a key without a source, a bad URL or number. */
|
|
42
|
+
export declare function validateConfig(config: PoolConfig): void;
|
|
43
|
+
/** Applies the defaults of every pool-wide setting. */
|
|
44
|
+
export declare function resolveSettings(config: PoolConfig): Settings;
|
|
45
|
+
/** Merges defaults, the named profile and the call's policy (later wins, field by field) and expands the model list. */
|
|
46
|
+
export declare function resolvePolicy<P extends string>(config: PoolConfig<P>, options: CallOptions<P>, requestModel: string | undefined): ResolvedPolicy;
|
package/dist/config.js
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { AiRouterError } from "./errors.js";
|
|
2
|
+
/** The capabilities of a model that declares none. */
|
|
3
|
+
export const DEFAULT_CAPABILITIES = ["chat", "streaming", "tools", "json"];
|
|
4
|
+
function configError(message) {
|
|
5
|
+
return new AiRouterError({ errorClass: "config", message });
|
|
6
|
+
}
|
|
7
|
+
function positive(value, name, allowZero = false) {
|
|
8
|
+
if (value === undefined)
|
|
9
|
+
return;
|
|
10
|
+
if (!Number.isFinite(value) || value < 0 || (!allowZero && value === 0))
|
|
11
|
+
throw configError(`${name} must be a positive number`);
|
|
12
|
+
}
|
|
13
|
+
/** True when the name is the model's id or one of its aliases. */
|
|
14
|
+
export function modelMatches(model, name) {
|
|
15
|
+
return model.id === name || (model.aliases?.includes(name) ?? false);
|
|
16
|
+
}
|
|
17
|
+
function knownModel(config, name) {
|
|
18
|
+
return config.providers.some((provider) => provider.models.some((model) => modelMatches(model, name)));
|
|
19
|
+
}
|
|
20
|
+
function validatePolicy(config, policy, where) {
|
|
21
|
+
for (const name of [policy.model, ...(policy.fallbacks ?? [])]) {
|
|
22
|
+
if (name !== undefined && !knownModel(config, name))
|
|
23
|
+
throw configError(`${where} names model "${name}", which no provider serves`);
|
|
24
|
+
}
|
|
25
|
+
positive(policy.maxAttempts, `${where}.maxAttempts`);
|
|
26
|
+
positive(policy.timeoutMs, `${where}.timeoutMs`);
|
|
27
|
+
positive(policy.deadlineMs, `${where}.deadlineMs`);
|
|
28
|
+
positive(policy.budget?.maxCost, `${where}.budget.maxCost`, true);
|
|
29
|
+
for (const id of policy.providers ?? []) {
|
|
30
|
+
if (!config.providers.some((provider) => provider.id === id))
|
|
31
|
+
throw configError(`${where} names provider "${id}", which is not configured`);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/** Refuses a pool that cannot route: no providers, duplicate ids, a key without a source, a bad URL or number. */
|
|
35
|
+
export function validateConfig(config) {
|
|
36
|
+
if (config.providers.length === 0)
|
|
37
|
+
throw configError("the pool has no providers");
|
|
38
|
+
const ids = new Set();
|
|
39
|
+
for (const provider of config.providers) {
|
|
40
|
+
if (provider.id === "" || provider.id.includes("/"))
|
|
41
|
+
throw configError(`provider id "${provider.id}" must be non-empty and without "/"`);
|
|
42
|
+
if (ids.has(provider.id))
|
|
43
|
+
throw configError(`provider "${provider.id}" is declared twice`);
|
|
44
|
+
ids.add(provider.id);
|
|
45
|
+
let url;
|
|
46
|
+
try {
|
|
47
|
+
url = new URL(provider.baseUrl);
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
throw configError(`provider "${provider.id}" has an invalid baseUrl`);
|
|
51
|
+
}
|
|
52
|
+
if (url.protocol !== "https:" && url.protocol !== "http:")
|
|
53
|
+
throw configError(`provider "${provider.id}" baseUrl must be http or https`);
|
|
54
|
+
positive(provider.weight, `provider "${provider.id}" weight`);
|
|
55
|
+
positive(provider.maxConcurrency, `provider "${provider.id}" maxConcurrency`);
|
|
56
|
+
if (provider.keys.length === 0)
|
|
57
|
+
throw configError(`provider "${provider.id}" has no keys`);
|
|
58
|
+
if (provider.models.length === 0)
|
|
59
|
+
throw configError(`provider "${provider.id}" has no models`);
|
|
60
|
+
const aliases = new Set();
|
|
61
|
+
for (const key of provider.keys) {
|
|
62
|
+
if (key.alias === "" || key.alias.includes("/"))
|
|
63
|
+
throw configError(`key alias "${key.alias}" of provider "${provider.id}" must be non-empty and without "/"`);
|
|
64
|
+
if (aliases.has(key.alias))
|
|
65
|
+
throw configError(`provider "${provider.id}" declares key "${key.alias}" twice`);
|
|
66
|
+
aliases.add(key.alias);
|
|
67
|
+
const sources = ("env" in key ? 1 : 0) + ("resolve" in key ? 1 : 0);
|
|
68
|
+
if (sources !== 1)
|
|
69
|
+
throw configError(`key "${key.alias}" of provider "${provider.id}" needs exactly one of env or resolve`);
|
|
70
|
+
positive(key.weight, `key "${key.alias}" weight`);
|
|
71
|
+
positive(key.maxConcurrency, `key "${key.alias}" maxConcurrency`);
|
|
72
|
+
}
|
|
73
|
+
const models = new Set();
|
|
74
|
+
for (const model of provider.models) {
|
|
75
|
+
if (model.id === "")
|
|
76
|
+
throw configError(`provider "${provider.id}" has a model with an empty id`);
|
|
77
|
+
if (models.has(model.id))
|
|
78
|
+
throw configError(`provider "${provider.id}" declares model "${model.id}" twice`);
|
|
79
|
+
models.add(model.id);
|
|
80
|
+
positive(model.weight, `model "${model.id}" weight`);
|
|
81
|
+
positive(model.contextWindow, `model "${model.id}" contextWindow`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
if (config.defaults !== undefined)
|
|
85
|
+
validatePolicy(config, config.defaults, "defaults");
|
|
86
|
+
for (const [name, policy] of Object.entries(config.profiles ?? {}))
|
|
87
|
+
validatePolicy(config, policy, `profile "${name}"`);
|
|
88
|
+
}
|
|
89
|
+
/** Applies the defaults of every pool-wide setting. */
|
|
90
|
+
export function resolveSettings(config) {
|
|
91
|
+
const r = config.resilience ?? {};
|
|
92
|
+
const s = config.selection ?? {};
|
|
93
|
+
return {
|
|
94
|
+
keyBreaker: {
|
|
95
|
+
failureThreshold: r.keyBreaker?.failureThreshold ?? 3,
|
|
96
|
+
cooldownMs: r.keyBreaker?.cooldownMs ?? 15_000,
|
|
97
|
+
maxCooldownMs: r.keyBreaker?.maxCooldownMs ?? 300_000,
|
|
98
|
+
},
|
|
99
|
+
providerBreaker: {
|
|
100
|
+
failureThreshold: r.providerBreaker?.failureThreshold ?? 8,
|
|
101
|
+
cooldownMs: r.providerBreaker?.cooldownMs ?? 30_000,
|
|
102
|
+
maxCooldownMs: r.providerBreaker?.maxCooldownMs ?? 300_000,
|
|
103
|
+
},
|
|
104
|
+
backoff: { baseMs: r.backoff?.baseMs ?? 100, factor: r.backoff?.factor ?? 2, maxMs: r.backoff?.maxMs ?? 5_000 },
|
|
105
|
+
authDisableMs: r.authDisableMs === undefined ? null : r.authDisableMs,
|
|
106
|
+
rateLimitCooldownMs: r.rateLimitCooldownMs ?? 10_000,
|
|
107
|
+
latencyTolerance: s.latencyTolerance ?? 1.5,
|
|
108
|
+
latencySlackMs: s.latencySlackMs ?? 50,
|
|
109
|
+
ewmaAlpha: s.ewmaAlpha ?? 0.3,
|
|
110
|
+
lowHeadroom: s.lowHeadroom ?? 1,
|
|
111
|
+
includeContent: config.telemetry?.includeContent ?? false,
|
|
112
|
+
namespace: config.storeNamespace ?? "ai-router",
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
/** Merges defaults, the named profile and the call's policy (later wins, field by field) and expands the model list. */
|
|
116
|
+
export function resolvePolicy(config, options, requestModel) {
|
|
117
|
+
let profile = {};
|
|
118
|
+
if (options.profile !== undefined) {
|
|
119
|
+
const found = config.profiles?.[options.profile];
|
|
120
|
+
if (found === undefined)
|
|
121
|
+
throw configError(`profile "${options.profile}" is not configured`);
|
|
122
|
+
profile = found;
|
|
123
|
+
}
|
|
124
|
+
const merged = { ...config.defaults, ...profile, ...options.policy };
|
|
125
|
+
if (options.policy !== undefined)
|
|
126
|
+
validatePolicy(config, options.policy, "policy");
|
|
127
|
+
// A call's policy, then its chosen profile, then the request's own model, then the defaults.
|
|
128
|
+
const preferred = options.policy?.model ?? profile.model ?? requestModel ?? config.defaults?.model;
|
|
129
|
+
if (preferred === undefined)
|
|
130
|
+
throw configError("no model: set request.model, a policy model or a profile model");
|
|
131
|
+
if (!knownModel(config, preferred))
|
|
132
|
+
throw configError(`model "${preferred}" is not served by any provider`);
|
|
133
|
+
const models = [preferred];
|
|
134
|
+
if (merged.allowFallback !== false) {
|
|
135
|
+
for (const name of merged.fallbacks ?? [])
|
|
136
|
+
if (!models.includes(name))
|
|
137
|
+
models.push(name);
|
|
138
|
+
}
|
|
139
|
+
return {
|
|
140
|
+
models,
|
|
141
|
+
maxAttempts: merged.maxAttempts ?? 4,
|
|
142
|
+
timeoutMs: merged.timeoutMs ?? 60_000,
|
|
143
|
+
deadlineMs: merged.deadlineMs ?? 120_000,
|
|
144
|
+
maxCost: merged.budget?.maxCost ?? null,
|
|
145
|
+
providers: merged.providers === undefined ? null : new Set(merged.providers),
|
|
146
|
+
require: merged.require ?? [],
|
|
147
|
+
};
|
|
148
|
+
}
|
package/dist/errors.d.ts
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { ErrorClass } .js";
|
|
2
|
+
/** One failed attempt as an error lists it: where it went and why it failed, never a key value. */
|
|
3
|
+
export interface AttemptFailure {
|
|
4
|
+
readonly provider: string;
|
|
5
|
+
readonly key: string;
|
|
6
|
+
readonly model: string;
|
|
7
|
+
readonly errorClass: ErrorClass;
|
|
8
|
+
readonly status?: number;
|
|
9
|
+
readonly message: string;
|
|
10
|
+
}
|
|
11
|
+
/** The fields an `AiRouterError` is built from. */
|
|
12
|
+
export interface AiRouterErrorInit {
|
|
13
|
+
readonly errorClass: ErrorClass;
|
|
14
|
+
readonly message: string;
|
|
15
|
+
readonly status?: number;
|
|
16
|
+
readonly provider?: string;
|
|
17
|
+
readonly key?: string;
|
|
18
|
+
readonly model?: string;
|
|
19
|
+
readonly requestId?: string;
|
|
20
|
+
readonly attempts?: readonly AttemptFailure[];
|
|
21
|
+
readonly afterFirstToken?: boolean;
|
|
22
|
+
/** Epoch milliseconds a 429 asked to wait until. */
|
|
23
|
+
readonly retryAt?: number;
|
|
24
|
+
}
|
|
25
|
+
/** True for a class another key, provider or model may succeed at. */
|
|
26
|
+
export declare function isRetryable(errorClass: ErrorClass): boolean;
|
|
27
|
+
/** The one error the router throws. `errorClass` says why; `attempts` lists every failed attempt of the call. */
|
|
28
|
+
export declare class AiRouterError extends Error {
|
|
29
|
+
readonly errorClass: ErrorClass;
|
|
30
|
+
readonly retryable: boolean;
|
|
31
|
+
readonly status: number | undefined;
|
|
32
|
+
readonly provider: string | undefined;
|
|
33
|
+
readonly key: string | undefined;
|
|
34
|
+
readonly model: string | undefined;
|
|
35
|
+
readonly requestId: string | undefined;
|
|
36
|
+
readonly attempts: readonly AttemptFailure[];
|
|
37
|
+
readonly afterFirstToken: boolean;
|
|
38
|
+
readonly retryAt: number | undefined;
|
|
39
|
+
/** Builds the error from its fields. */
|
|
40
|
+
constructor(init: AiRouterErrorInit);
|
|
41
|
+
}
|
|
42
|
+
/** Classifies an HTTP status: 429 rate, 401/403 auth, 408/409/425/5xx transient, any other 4xx an invalid request. */
|
|
43
|
+
export declare function classifyStatus(status: number): ErrorClass;
|
|
44
|
+
/** Classifies an error object a provider sends inside a stream or a 200 body. */
|
|
45
|
+
export declare function classifyProviderError(error: unknown): ErrorClass;
|
|
46
|
+
/** Replaces every occurrence of each secret with `[redacted]`, so an echoed key never leaves the router. */
|
|
47
|
+
export declare function scrub(text: string, secrets: readonly string[]): string;
|
package/dist/errors.js
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
const RETRYABLE = new Set(["rate_limit", "auth", "transient", "timeout", "network", "stream"]);
|
|
2
|
+
/** True for a class another key, provider or model may succeed at. */
|
|
3
|
+
export function isRetryable(errorClass) {
|
|
4
|
+
return RETRYABLE.has(errorClass);
|
|
5
|
+
}
|
|
6
|
+
/** The one error the router throws. `errorClass` says why; `attempts` lists every failed attempt of the call. */
|
|
7
|
+
export class AiRouterError extends Error {
|
|
8
|
+
errorClass;
|
|
9
|
+
retryable;
|
|
10
|
+
status;
|
|
11
|
+
provider;
|
|
12
|
+
key;
|
|
13
|
+
model;
|
|
14
|
+
requestId;
|
|
15
|
+
attempts;
|
|
16
|
+
afterFirstToken;
|
|
17
|
+
retryAt;
|
|
18
|
+
/** Builds the error from its fields. */
|
|
19
|
+
constructor(init) {
|
|
20
|
+
super(init.message);
|
|
21
|
+
this.name = "AiRouterError";
|
|
22
|
+
this.errorClass = init.errorClass;
|
|
23
|
+
this.retryable = isRetryable(init.errorClass);
|
|
24
|
+
this.status = init.status;
|
|
25
|
+
this.provider = init.provider;
|
|
26
|
+
this.key = init.key;
|
|
27
|
+
this.model = init.model;
|
|
28
|
+
this.requestId = init.requestId;
|
|
29
|
+
this.attempts = init.attempts ?? [];
|
|
30
|
+
this.afterFirstToken = init.afterFirstToken ?? false;
|
|
31
|
+
this.retryAt = init.retryAt;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/** Classifies an HTTP status: 429 rate, 401/403 auth, 408/409/425/5xx transient, any other 4xx an invalid request. */
|
|
35
|
+
export function classifyStatus(status) {
|
|
36
|
+
if (status === 429)
|
|
37
|
+
return "rate_limit";
|
|
38
|
+
if (status === 401 || status === 403)
|
|
39
|
+
return "auth";
|
|
40
|
+
if (status === 408 || status === 409 || status === 425 || status >= 500)
|
|
41
|
+
return "transient";
|
|
42
|
+
return "invalid_request";
|
|
43
|
+
}
|
|
44
|
+
/** Classifies an error object a provider sends inside a stream or a 200 body. */
|
|
45
|
+
export function classifyProviderError(error) {
|
|
46
|
+
if (typeof error !== "object" || error === null)
|
|
47
|
+
return "transient";
|
|
48
|
+
const record = error;
|
|
49
|
+
const words = [record.type, record.code].filter((value) => typeof value === "string").join(" ");
|
|
50
|
+
if (/rate|quota|too_many/i.test(words))
|
|
51
|
+
return "rate_limit";
|
|
52
|
+
if (/auth|api_key|permission/i.test(words))
|
|
53
|
+
return "auth";
|
|
54
|
+
if (/invalid|context_length|not_found/i.test(words))
|
|
55
|
+
return "invalid_request";
|
|
56
|
+
return "transient";
|
|
57
|
+
}
|
|
58
|
+
/** Replaces every occurrence of each secret with `[redacted]`, so an echoed key never leaves the router. */
|
|
59
|
+
export function scrub(text, secrets) {
|
|
60
|
+
let out = text;
|
|
61
|
+
for (const secret of secrets) {
|
|
62
|
+
if (secret.length > 0)
|
|
63
|
+
out = out.split(secret).join("[redacted]");
|
|
64
|
+
}
|
|
65
|
+
return out;
|
|
66
|
+
}
|
package/dist/events.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { RouterEvents } .js";
|
|
2
|
+
type Handler<E extends keyof RouterEvents> = (payload: RouterEvents[E]) => void;
|
|
3
|
+
/** A typed event fan-out. A handler that throws is isolated: observing a call never breaks it. */
|
|
4
|
+
export declare class Emitter {
|
|
5
|
+
#private;
|
|
6
|
+
/** Subscribes and returns the unsubscribe function. */
|
|
7
|
+
on<E extends keyof RouterEvents>(event: E, handler: Handler<E>): () => void;
|
|
8
|
+
/** Delivers one event to every handler of its name. */
|
|
9
|
+
emit<E extends keyof RouterEvents>(event: E, payload: RouterEvents[E]): void;
|
|
10
|
+
}
|
|
11
|
+
export {};
|
package/dist/events.js
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/** A typed event fan-out. A handler that throws is isolated: observing a call never breaks it. */
|
|
2
|
+
export class Emitter {
|
|
3
|
+
#handlers = new Map();
|
|
4
|
+
/** Subscribes and returns the unsubscribe function. */
|
|
5
|
+
on(event, handler) {
|
|
6
|
+
let set = this.#handlers.get(event);
|
|
7
|
+
if (set === undefined) {
|
|
8
|
+
set = new Set();
|
|
9
|
+
this.#handlers.set(event, set);
|
|
10
|
+
}
|
|
11
|
+
set.add(handler);
|
|
12
|
+
return () => {
|
|
13
|
+
set.delete(handler);
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
/** Delivers one event to every handler of its name. */
|
|
17
|
+
emit(event, payload) {
|
|
18
|
+
const set = this.#handlers.get(event);
|
|
19
|
+
if (set === undefined)
|
|
20
|
+
return;
|
|
21
|
+
for (const handler of set) {
|
|
22
|
+
const typed = handler;
|
|
23
|
+
try {
|
|
24
|
+
typed(payload);
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
// An observer's failure is the observer's: the call it watches goes on.
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
package/dist/fake.d.ts
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import type { FetchLike, Usage } .js";
|
|
2
|
+
/** How a scripted stream behaves. */
|
|
3
|
+
export interface FakeStream {
|
|
4
|
+
/** The content pieces, one chunk each. Default: the reply text split at spaces. */
|
|
5
|
+
readonly chunks?: readonly string[];
|
|
6
|
+
/** The pause before each chunk. Default 0. */
|
|
7
|
+
readonly delayMs?: number;
|
|
8
|
+
/** Fails after this many content chunks (0: before the first token). */
|
|
9
|
+
readonly failAfter?: number;
|
|
10
|
+
/** Goes silent for `stallMs` after this many content chunks (an idle provider). */
|
|
11
|
+
readonly stallAfter?: number;
|
|
12
|
+
readonly stallMs?: number;
|
|
13
|
+
/** `network` breaks the body; `error-event` streams an OpenAI error object. Default `network`. */
|
|
14
|
+
readonly failWith?: "network" | "error-event";
|
|
15
|
+
/** Ends with a chunk that carries usage. Default true. */
|
|
16
|
+
readonly includeUsage?: boolean;
|
|
17
|
+
}
|
|
18
|
+
/** One scripted answer. Everything is optional: the default is a 200 chat reply `ok`. */
|
|
19
|
+
export interface FakeReply {
|
|
20
|
+
readonly status?: number;
|
|
21
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
22
|
+
/** The pause before the response headers; an abort during it rejects like `fetch`. */
|
|
23
|
+
readonly latencyMs?: number;
|
|
24
|
+
/** `network` rejects like `fetch` does when the connection fails. */
|
|
25
|
+
readonly error?: "network";
|
|
26
|
+
/** The chat reply text. Default `ok`. */
|
|
27
|
+
readonly text?: string;
|
|
28
|
+
readonly usage?: Usage;
|
|
29
|
+
/** The vector of every embedding. Default `[0.1, 0.2, 0.3]`. */
|
|
30
|
+
readonly embedding?: readonly number[];
|
|
31
|
+
/** Replaces the whole JSON body. */
|
|
32
|
+
readonly body?: unknown;
|
|
33
|
+
readonly stream?: FakeStream;
|
|
34
|
+
}
|
|
35
|
+
/** Which requests a script applies to; an absent field matches anything. */
|
|
36
|
+
export interface FakeMatch {
|
|
37
|
+
/** The provider's base URL the request starts with. */
|
|
38
|
+
readonly baseUrl?: string;
|
|
39
|
+
/** The bearer key value (a test value). */
|
|
40
|
+
readonly key?: string;
|
|
41
|
+
readonly model?: string;
|
|
42
|
+
readonly path?: string;
|
|
43
|
+
}
|
|
44
|
+
/** One request the fake received. */
|
|
45
|
+
export interface FakeCall {
|
|
46
|
+
readonly url: string;
|
|
47
|
+
readonly path: string;
|
|
48
|
+
/** The bearer key value sent (a test value). */
|
|
49
|
+
readonly key: string;
|
|
50
|
+
readonly model: string;
|
|
51
|
+
readonly stream: boolean;
|
|
52
|
+
readonly headers: Readonly<Record<string, string>>;
|
|
53
|
+
readonly body: Readonly<Record<string, unknown>>;
|
|
54
|
+
}
|
|
55
|
+
/** A scriptable OpenAI-compatible provider that answers in process, for consumers' tests. */
|
|
56
|
+
export interface FakeProvider {
|
|
57
|
+
/** Pass as `PoolConfig.fetch`. */
|
|
58
|
+
readonly fetch: FetchLike;
|
|
59
|
+
/** Every request received, in order. */
|
|
60
|
+
readonly calls: readonly FakeCall[];
|
|
61
|
+
/** Adds a script: matching requests get the replies in order, and the last one repeats. A later script wins. */
|
|
62
|
+
on(match: FakeMatch, ...replies: FakeReply[]): FakeProvider;
|
|
63
|
+
/** Forgets every script and call. */
|
|
64
|
+
reset(): void;
|
|
65
|
+
}
|
|
66
|
+
/** Creates a fake OpenAI-compatible provider: deterministic replies, latency, errors, 429s and streams by script. */
|
|
67
|
+
export declare function createFakeProvider(): FakeProvider;
|
package/dist/fake.js
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
function wait(ms, signal) {
|
|
2
|
+
return new Promise((resolve, reject) => {
|
|
3
|
+
if (signal?.aborted === true) {
|
|
4
|
+
reject(signal.reason);
|
|
5
|
+
return;
|
|
6
|
+
}
|
|
7
|
+
const timer = setTimeout(() => {
|
|
8
|
+
signal?.removeEventListener("abort", stop);
|
|
9
|
+
resolve();
|
|
10
|
+
}, ms);
|
|
11
|
+
const stop = () => {
|
|
12
|
+
clearTimeout(timer);
|
|
13
|
+
reject(signal?.reason);
|
|
14
|
+
};
|
|
15
|
+
signal?.addEventListener("abort", stop, { once: true });
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
function matches(match, call) {
|
|
19
|
+
return ((match.baseUrl === undefined || call.url.startsWith(match.baseUrl)) &&
|
|
20
|
+
(match.key === undefined || match.key === call.key) &&
|
|
21
|
+
(match.model === undefined || match.model === call.model) &&
|
|
22
|
+
(match.path === undefined || call.path.endsWith(match.path)));
|
|
23
|
+
}
|
|
24
|
+
function headerRecord(init) {
|
|
25
|
+
const out = {};
|
|
26
|
+
new Headers(init.headers).forEach((value, name) => {
|
|
27
|
+
out[name] = value;
|
|
28
|
+
});
|
|
29
|
+
return out;
|
|
30
|
+
}
|
|
31
|
+
function streamBody(reply, model, signal) {
|
|
32
|
+
const options = reply.stream ?? {};
|
|
33
|
+
const pieces = options.chunks ?? (reply.text ?? "ok").split(/(?<= )/);
|
|
34
|
+
const encoder = new TextEncoder();
|
|
35
|
+
const base = { id: "fake-stream", object: "chat.completion.chunk", created: 0, model };
|
|
36
|
+
const frames = [JSON.stringify({ ...base, choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }] })];
|
|
37
|
+
pieces.forEach((piece, index) => {
|
|
38
|
+
if (options.stallAfter === index)
|
|
39
|
+
frames.push("stall");
|
|
40
|
+
if (options.failAfter === index)
|
|
41
|
+
frames.push("fail");
|
|
42
|
+
frames.push(JSON.stringify({ ...base, choices: [{ index: 0, delta: { content: piece }, finish_reason: null }] }));
|
|
43
|
+
});
|
|
44
|
+
if (options.failAfter !== undefined && options.failAfter >= pieces.length)
|
|
45
|
+
frames.push("fail");
|
|
46
|
+
frames.push(JSON.stringify({ ...base, choices: [{ index: 0, delta: {}, finish_reason: "stop" }] }));
|
|
47
|
+
if (options.includeUsage !== false) {
|
|
48
|
+
const usage = reply.usage ?? { prompt_tokens: 10, completion_tokens: pieces.length, total_tokens: 10 + pieces.length };
|
|
49
|
+
frames.push(JSON.stringify({ ...base, choices: [], usage }));
|
|
50
|
+
}
|
|
51
|
+
frames.push("[DONE]");
|
|
52
|
+
let index = 0;
|
|
53
|
+
return new ReadableStream({
|
|
54
|
+
async pull(controller) {
|
|
55
|
+
let frame = frames[index++];
|
|
56
|
+
try {
|
|
57
|
+
if (frame === "stall") {
|
|
58
|
+
await wait(options.stallMs ?? 1000, signal);
|
|
59
|
+
frame = frames[index++];
|
|
60
|
+
}
|
|
61
|
+
if (options.delayMs !== undefined && options.delayMs > 0)
|
|
62
|
+
await wait(options.delayMs, signal);
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
controller.error(error);
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
if (frame === undefined) {
|
|
69
|
+
controller.close();
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
if (frame !== "fail") {
|
|
73
|
+
controller.enqueue(encoder.encode(`data: ${frame}\n\n`));
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
if (options.failWith === "error-event") {
|
|
77
|
+
controller.enqueue(encoder.encode(`data: ${JSON.stringify({ error: { message: "fake stream error", type: "server_error" } })}\n\n`));
|
|
78
|
+
index = frames.length;
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
controller.error(new TypeError("terminated"));
|
|
82
|
+
},
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
function jsonBody(reply, call) {
|
|
86
|
+
if (reply.body !== undefined)
|
|
87
|
+
return reply.body;
|
|
88
|
+
const status = reply.status ?? 200;
|
|
89
|
+
if (status >= 400)
|
|
90
|
+
return { error: { message: `fake ${String(status)}`, type: status === 429 ? "rate_limit_exceeded" : "fake_error" } };
|
|
91
|
+
if (call.path.endsWith("/embeddings")) {
|
|
92
|
+
const input = call.body.input;
|
|
93
|
+
const count = Array.isArray(input) ? input.length : 1;
|
|
94
|
+
return {
|
|
95
|
+
object: "list",
|
|
96
|
+
data: Array.from({ length: count }, (_, index) => ({ object: "embedding", index, embedding: reply.embedding ?? [0.1, 0.2, 0.3] })),
|
|
97
|
+
model: call.model,
|
|
98
|
+
usage: reply.usage ?? { prompt_tokens: 4 * count, total_tokens: 4 * count },
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
const text = reply.text ?? "ok";
|
|
102
|
+
return {
|
|
103
|
+
id: `fake-${String(Date.now())}`,
|
|
104
|
+
object: "chat.completion",
|
|
105
|
+
created: 0,
|
|
106
|
+
model: call.model,
|
|
107
|
+
choices: [{ index: 0, message: { role: "assistant", content: text }, finish_reason: "stop" }],
|
|
108
|
+
usage: reply.usage ?? { prompt_tokens: 10, completion_tokens: 2, total_tokens: 12 },
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
/** Creates a fake OpenAI-compatible provider: deterministic replies, latency, errors, 429s and streams by script. */
|
|
112
|
+
export function createFakeProvider() {
|
|
113
|
+
let scripts = [];
|
|
114
|
+
const calls = [];
|
|
115
|
+
const fake = {
|
|
116
|
+
calls,
|
|
117
|
+
on(match, ...replies) {
|
|
118
|
+
scripts.unshift({ match, replies: replies.length === 0 ? [{}] : [...replies] });
|
|
119
|
+
return fake;
|
|
120
|
+
},
|
|
121
|
+
reset() {
|
|
122
|
+
scripts = [];
|
|
123
|
+
calls.length = 0;
|
|
124
|
+
},
|
|
125
|
+
async fetch(input, init) {
|
|
126
|
+
const url = new URL(input);
|
|
127
|
+
const headers = headerRecord(init);
|
|
128
|
+
const body = (typeof init.body === "string" ? JSON.parse(init.body) : {});
|
|
129
|
+
const call = {
|
|
130
|
+
url: input,
|
|
131
|
+
path: url.pathname,
|
|
132
|
+
key: (headers.authorization ?? "").replace(/^Bearer /, ""),
|
|
133
|
+
model: typeof body.model === "string" ? body.model : "",
|
|
134
|
+
stream: body.stream === true,
|
|
135
|
+
headers,
|
|
136
|
+
body,
|
|
137
|
+
};
|
|
138
|
+
calls.push(call);
|
|
139
|
+
const script = scripts.find((candidate) => matches(candidate.match, call));
|
|
140
|
+
const reply = (script === undefined ? undefined : script.replies.length > 1 ? script.replies.shift() : script.replies[0]) ?? {};
|
|
141
|
+
if (reply.latencyMs !== undefined && reply.latencyMs > 0)
|
|
142
|
+
await wait(reply.latencyMs, init.signal);
|
|
143
|
+
if (reply.error === "network")
|
|
144
|
+
throw new TypeError("fetch failed");
|
|
145
|
+
const status = reply.status ?? 200;
|
|
146
|
+
if (call.stream && status < 400 && reply.body === undefined) {
|
|
147
|
+
return new Response(streamBody(reply, call.model, init.signal), { status, headers: { "content-type": "text/event-stream", ...reply.headers } });
|
|
148
|
+
}
|
|
149
|
+
return new Response(JSON.stringify(jsonBody(reply, call)), { status, headers: { "content-type": "application/json", ...reply.headers } });
|
|
150
|
+
},
|
|
151
|
+
};
|
|
152
|
+
return fake;
|
|
153
|
+
}
|