@starci/ai-router 0.4.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +133 -40
- package/dist/ai-router.service.d.ts +2 -2
- package/dist/ai-router.service.js +4 -2
- package/dist/attempt.d.ts +26 -0
- package/dist/attempt.js +29 -0
- package/dist/classify.d.ts +40 -0
- package/dist/classify.js +145 -0
- package/dist/config.d.ts +30 -5
- package/dist/config.js +184 -24
- package/dist/env.d.ts +70 -0
- package/dist/env.js +152 -0
- package/dist/errors.d.ts +1 -3
- package/dist/errors.js +4 -23
- package/dist/fake.d.ts +624 -5
- package/dist/fake.js +177 -21
- package/dist/gateway/config-file.d.ts +8 -3
- package/dist/gateway/config-file.js +53 -8
- package/dist/gateway/gateway.d.ts +2 -0
- package/dist/gateway/gateway.js +19 -3
- package/dist/gateway/index.d.ts +10 -1
- package/dist/gateway/index.js +5 -0
- package/dist/gateway/serve.js +7 -2
- package/dist/headers.d.ts +1 -1
- package/dist/headers.js +12 -6
- package/dist/pool.d.ts +37 -16
- package/dist/pool.js +173 -48
- package/dist/providers.d.ts +34 -0
- package/dist/providers.js +22 -0
- package/dist/router.js +108 -105
- package/dist/secrets.d.ts +8 -0
- package/dist/secrets.js +22 -0
- package/dist/shapes.d.ts +11 -0
- package/dist/shapes.js +124 -0
- package/dist/state-stores.d.ts +38 -0
- package/dist/state-stores.js +101 -0
- package/dist/store.d.ts +26 -7
- package/dist/store.js +64 -2
- package/dist/testing.d.ts +2 -2
- package/dist/testing.js +1 -1
- package/dist/types.d.ts +126 -38
- package/dist/wire/anthropic.d.ts +21 -0
- package/dist/wire/anthropic.js +322 -0
- package/dist/wire/index.d.ts +5 -0
- package/dist/wire/index.js +7 -0
- package/dist/wire/wire.d.ts +42 -0
- package/dist/wire/wire.js +35 -0
- package/docs/design.md +279 -0
- package/package.json +12 -3
package/dist/env.js
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { matchesOf } from "./config.js";
|
|
3
|
+
import { AiRouterError } from "./errors.js";
|
|
4
|
+
import { PROVIDER_PRESETS } from "./providers.js";
|
|
5
|
+
function configError(message) {
|
|
6
|
+
return new AiRouterError({ errorClass: "config", message });
|
|
7
|
+
}
|
|
8
|
+
function present(env, name) {
|
|
9
|
+
const value = env[name];
|
|
10
|
+
return value !== undefined && value.trim() !== "";
|
|
11
|
+
}
|
|
12
|
+
async function firstLine(path) {
|
|
13
|
+
try {
|
|
14
|
+
return (await readFile(path, "utf8")).split(/\r?\n/)[0];
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return undefined;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
/** The prefix a spec reads its keys under. */
|
|
21
|
+
export function envPrefixOf(spec) {
|
|
22
|
+
const prefix = spec.envPrefix ?? PROVIDER_PRESETS[spec.kind].envPrefix;
|
|
23
|
+
if (prefix === null)
|
|
24
|
+
throw configError(`a ${spec.kind} provider needs envPrefix: its kind has no canonical env names`);
|
|
25
|
+
if (!/^[A-Z][A-Z0-9_]*$/.test(prefix))
|
|
26
|
+
throw configError(`envPrefix "${prefix}" must be upper-case letters, digits and _`);
|
|
27
|
+
return prefix;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* The keys of one provider found in env under its prefix, as references read at each attempt: never a copied value.
|
|
31
|
+
* A key given both as a variable and as a `_FILE` twin is refused, since which one wins would be a guess.
|
|
32
|
+
*/
|
|
33
|
+
export function keysFromEnv(prefix, id, env = process.env, maxConcurrency) {
|
|
34
|
+
const pattern = new RegExp(`^${prefix}_API_KEY(?:_(\\d+))?(_FILE)?$`);
|
|
35
|
+
const found = new Map();
|
|
36
|
+
for (const name of Object.keys(env)) {
|
|
37
|
+
const match = pattern.exec(name);
|
|
38
|
+
if (match === null || !present(env, name))
|
|
39
|
+
continue;
|
|
40
|
+
const index = match[1] === undefined ? 1 : Number(match[1]);
|
|
41
|
+
if (index < 1 || (match[1] !== undefined && index === 1))
|
|
42
|
+
throw configError(`${name}: number keys from 2 (${prefix}_API_KEY is the first)`);
|
|
43
|
+
if (found.has(index))
|
|
44
|
+
throw configError(`${prefix} key ${String(index)} is set both as a variable and as a _FILE: keep one`);
|
|
45
|
+
found.set(index, { name, file: match[2] !== undefined });
|
|
46
|
+
}
|
|
47
|
+
const cap = maxConcurrency === undefined ? {} : { maxConcurrency };
|
|
48
|
+
const keys = [...found.entries()]
|
|
49
|
+
.sort(([a], [b]) => a - b)
|
|
50
|
+
.map(([index, { name, file }]) => ({ alias: `${id}-${String(index)}`, ...cap, resolve: file ? () => firstLine(env[name] ?? "") : () => env[name] }));
|
|
51
|
+
const list = `${prefix}_API_KEYS`;
|
|
52
|
+
if (present(env, list)) {
|
|
53
|
+
const count = (env[list] ?? "").split(/[\s,]+/).filter((value) => value !== "").length;
|
|
54
|
+
for (let at = 0; at < count; at++)
|
|
55
|
+
keys.push({ alias: `${id}-list-${String(at + 1)}`, ...cap, resolve: () => (env[list] ?? "").split(/[\s,]+/).filter((value) => value !== "")[at] });
|
|
56
|
+
}
|
|
57
|
+
return keys;
|
|
58
|
+
}
|
|
59
|
+
/** The providers of the specs that have at least one key in env; a provider with none is left out, not refused. */
|
|
60
|
+
export function providersFromEnv(specs, env = process.env) {
|
|
61
|
+
const out = [];
|
|
62
|
+
for (const spec of specs) {
|
|
63
|
+
const prefix = envPrefixOf(spec);
|
|
64
|
+
const id = spec.id ?? (spec.kind === "openai-compatible" ? undefined : spec.kind);
|
|
65
|
+
if (id === undefined)
|
|
66
|
+
throw configError(`an openai-compatible provider (prefix ${prefix}) needs an id`);
|
|
67
|
+
const keys = keysFromEnv(prefix, id, env, spec.keyMaxConcurrency);
|
|
68
|
+
if (keys.length === 0)
|
|
69
|
+
continue;
|
|
70
|
+
const baseUrl = spec.baseUrl ?? (present(env, `${prefix}_BASE_URL`) ? env[`${prefix}_BASE_URL`]?.trim() : undefined);
|
|
71
|
+
out.push({
|
|
72
|
+
id,
|
|
73
|
+
kind: spec.kind,
|
|
74
|
+
...(baseUrl === undefined ? {} : { baseUrl }),
|
|
75
|
+
...(spec.headers === undefined ? {} : { headers: spec.headers }),
|
|
76
|
+
...(spec.weight === undefined ? {} : { weight: spec.weight }),
|
|
77
|
+
...(spec.maxConcurrency === undefined ? {} : { maxConcurrency: spec.maxConcurrency }),
|
|
78
|
+
...(spec.defaultMaxTokens === undefined ? {} : { defaultMaxTokens: spec.defaultMaxTokens }),
|
|
79
|
+
...(spec.budget === undefined ? {} : { budget: spec.budget }),
|
|
80
|
+
keys,
|
|
81
|
+
models: spec.models,
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
return out;
|
|
85
|
+
}
|
|
86
|
+
function isTier(entry) {
|
|
87
|
+
return Array.isArray(entry);
|
|
88
|
+
}
|
|
89
|
+
/** A policy cut down to the candidates the present providers serve; null when nothing it names is left. */
|
|
90
|
+
function prune(config, policy) {
|
|
91
|
+
const served = (ref) => matchesOf(config, ref).length > 0;
|
|
92
|
+
if (policy.candidates !== undefined) {
|
|
93
|
+
const tiers = policy.candidates
|
|
94
|
+
.map((entry) => (isTier(entry) ? entry.filter(served) : served(entry) ? entry : null))
|
|
95
|
+
.filter((entry) => entry !== null && (!isTier(entry) || entry.length > 0));
|
|
96
|
+
return tiers.length === 0 ? null : { ...policy, candidates: tiers };
|
|
97
|
+
}
|
|
98
|
+
const fallbacks = policy.fallbacks?.filter((model) => served({ model }));
|
|
99
|
+
const kept = { ...policy, ...(fallbacks === undefined ? {} : { fallbacks }) };
|
|
100
|
+
if (policy.model === undefined || served({ model: policy.model }))
|
|
101
|
+
return kept;
|
|
102
|
+
// The preferred model's provider has no key: its first served fallback leads, unless the profile is pinned.
|
|
103
|
+
const [first, ...rest] = fallbacks ?? [];
|
|
104
|
+
if (first === undefined || policy.pinned === true)
|
|
105
|
+
return null;
|
|
106
|
+
return { ...kept, model: first, fallbacks: rest };
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* A pool from env: the providers that have keys, and every profile cut down to the candidates they serve, so an app
|
|
110
|
+
* declares every provider it may use and deploys with the keys it has. A profile left with nothing is refused, naming
|
|
111
|
+
* the variables that would give it a candidate.
|
|
112
|
+
*/
|
|
113
|
+
export function poolFromEnv(spec, env = process.env) {
|
|
114
|
+
const providers = providersFromEnv(spec.providers, env);
|
|
115
|
+
const expected = spec.providers.map((entry) => `${envPrefixOf(entry)}_API_KEY`).join(", ");
|
|
116
|
+
if (providers.length === 0)
|
|
117
|
+
throw configError(`no provider has a key in env: set one of ${expected}`);
|
|
118
|
+
const { defaults: declaredDefaults, profiles: declaredProfiles, ...settings } = { ...spec, providers };
|
|
119
|
+
const served = settings;
|
|
120
|
+
// A default that names nothing served is dropped, not kept to fail validation.
|
|
121
|
+
const defaults = declaredDefaults === undefined ? null : prune(served, declaredDefaults);
|
|
122
|
+
const pool = { ...settings, ...(defaults === null ? {} : { defaults }) };
|
|
123
|
+
if (declaredProfiles === undefined)
|
|
124
|
+
return pool;
|
|
125
|
+
const profiles = { ...declaredProfiles };
|
|
126
|
+
for (const [name, policy] of Object.entries(declaredProfiles)) {
|
|
127
|
+
const kept = prune(served, policy);
|
|
128
|
+
if (kept === null)
|
|
129
|
+
throw configError(`profile "${name}" has no candidate whose provider has a key: set one of ${expected}`);
|
|
130
|
+
Object.assign(profiles, { [name]: kept });
|
|
131
|
+
}
|
|
132
|
+
return { ...pool, profiles };
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* The app's virtual key as a mounted key, from `STARCI_AI_KEY` (or its `_FILE` twin, a sealed secret mounted as a
|
|
136
|
+
* file): the one key every client of the app (its LangChain integration, its agent runtime) sends to the gateway.
|
|
137
|
+
*/
|
|
138
|
+
export async function virtualKeyFromEnv(spec, env = process.env) {
|
|
139
|
+
const variable = spec.variable ?? "STARCI_AI_KEY";
|
|
140
|
+
const file = env[`${variable}_FILE`];
|
|
141
|
+
const secret = present(env, variable) ? (env[variable] ?? "").trim() : file !== undefined && file.trim() !== "" ? ((await firstLine(file.trim())) ?? "").trim() : "";
|
|
142
|
+
if (secret === "")
|
|
143
|
+
throw configError(`${variable} (or ${variable}_FILE) is not set: seal the app's virtual key`);
|
|
144
|
+
return {
|
|
145
|
+
label: spec.label,
|
|
146
|
+
secret,
|
|
147
|
+
profiles: spec.profiles ?? ["work", "embed"],
|
|
148
|
+
models: spec.models ?? [],
|
|
149
|
+
...(spec.quota === undefined ? {} : { quota: spec.quota }),
|
|
150
|
+
...(spec.owner === undefined ? {} : { owner: spec.owner }),
|
|
151
|
+
};
|
|
152
|
+
}
|
package/dist/errors.d.ts
CHANGED
|
@@ -39,9 +39,7 @@ export declare class AiRouterError extends Error {
|
|
|
39
39
|
/** Builds the error from its fields. */
|
|
40
40
|
constructor(init: AiRouterErrorInit);
|
|
41
41
|
}
|
|
42
|
-
/** Classifies an HTTP status: 429 rate, 401/403 auth, 408/409/425/5xx transient, any other 4xx an invalid request. */
|
|
42
|
+
/** Classifies an HTTP status alone, with no body: 402 quota, 429 rate, 401/403 auth, 408/409/425/498/5xx transient, any other 4xx an invalid request. */
|
|
43
43
|
export declare function classifyStatus(status: number): ErrorClass;
|
|
44
|
-
/** Classifies an error object a provider sends inside a stream or a 200 body. */
|
|
45
|
-
export declare function classifyProviderError(error: unknown): ErrorClass;
|
|
46
44
|
/** Replaces every occurrence of each secret with `[redacted]`, so an echoed key never leaves the router. */
|
|
47
45
|
export declare function scrub(text: string, secrets: readonly string[]): string;
|
package/dist/errors.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
|
|
1
|
+
import { classifyFailure } from "./classify.js";
|
|
2
|
+
const RETRYABLE = new Set(["rate_limit", "quota", "auth", "transient", "timeout", "network", "stream"]);
|
|
2
3
|
/** True for a class another key, provider or model may succeed at. */
|
|
3
4
|
export function isRetryable(errorClass) {
|
|
4
5
|
return RETRYABLE.has(errorClass);
|
|
@@ -31,29 +32,9 @@ export class AiRouterError extends Error {
|
|
|
31
32
|
this.retryAt = init.retryAt;
|
|
32
33
|
}
|
|
33
34
|
}
|
|
34
|
-
/** Classifies an HTTP status: 429 rate, 401/403 auth, 408/409/425/5xx transient, any other 4xx an invalid request. */
|
|
35
|
+
/** Classifies an HTTP status alone, with no body: 402 quota, 429 rate, 401/403 auth, 408/409/425/498/5xx transient, any other 4xx an invalid request. */
|
|
35
36
|
export function classifyStatus(status) {
|
|
36
|
-
|
|
37
|
-
return "rate_limit";
|
|
38
|
-
if (status === 401 || status === 403)
|
|
39
|
-
return "auth";
|
|
40
|
-
if (status === 408 || status === 409 || status === 425 || status >= 500)
|
|
41
|
-
return "transient";
|
|
42
|
-
return "invalid_request";
|
|
43
|
-
}
|
|
44
|
-
/** Classifies an error object a provider sends inside a stream or a 200 body. */
|
|
45
|
-
export function classifyProviderError(error) {
|
|
46
|
-
if (typeof error !== "object" || error === null)
|
|
47
|
-
return "transient";
|
|
48
|
-
const record = error;
|
|
49
|
-
const words = [record.type, record.code].filter((value) => typeof value === "string").join(" ");
|
|
50
|
-
if (/rate|quota|too_many/i.test(words))
|
|
51
|
-
return "rate_limit";
|
|
52
|
-
if (/auth|api_key|permission/i.test(words))
|
|
53
|
-
return "auth";
|
|
54
|
-
if (/invalid|context_length|not_found/i.test(words))
|
|
55
|
-
return "invalid_request";
|
|
56
|
-
return "transient";
|
|
37
|
+
return classifyFailure("openai-compatible", status, undefined).errorClass;
|
|
57
38
|
}
|
|
58
39
|
/** Replaces every occurrence of each secret with `[redacted]`, so an echoed key never leaves the router. */
|
|
59
40
|
export function scrub(text, secrets) {
|