@juspay/neurolink 9.76.0 → 9.77.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/browser/neurolink.min.js +341 -341
- package/dist/index.d.ts +30 -0
- package/dist/index.js +33 -0
- package/dist/lib/index.d.ts +30 -0
- package/dist/lib/index.js +33 -0
- package/dist/lib/neurolink.d.ts +21 -0
- package/dist/lib/neurolink.js +367 -90
- package/dist/lib/routing/index.d.ts +7 -0
- package/dist/lib/routing/index.js +8 -0
- package/dist/lib/routing/modelPool.d.ts +83 -0
- package/dist/lib/routing/modelPool.js +243 -0
- package/dist/lib/routing/requestRouter.d.ts +30 -0
- package/dist/lib/routing/requestRouter.js +81 -0
- package/dist/lib/types/config.d.ts +18 -0
- package/dist/lib/types/index.d.ts +2 -0
- package/dist/lib/types/index.js +4 -0
- package/dist/lib/types/modelPool.d.ts +47 -0
- package/dist/lib/types/modelPool.js +11 -0
- package/dist/lib/types/requestRouter.d.ts +75 -0
- package/dist/lib/types/requestRouter.js +16 -0
- package/dist/lib/utils/providerErrorClassification.d.ts +24 -0
- package/dist/lib/utils/providerErrorClassification.js +89 -0
- package/dist/neurolink.d.ts +21 -0
- package/dist/neurolink.js +367 -90
- package/dist/routing/index.d.ts +7 -0
- package/dist/routing/index.js +7 -0
- package/dist/routing/modelPool.d.ts +83 -0
- package/dist/routing/modelPool.js +242 -0
- package/dist/routing/requestRouter.d.ts +30 -0
- package/dist/routing/requestRouter.js +80 -0
- package/dist/types/config.d.ts +18 -0
- package/dist/types/index.d.ts +2 -0
- package/dist/types/index.js +4 -0
- package/dist/types/modelPool.d.ts +47 -0
- package/dist/types/modelPool.js +10 -0
- package/dist/types/requestRouter.d.ts +75 -0
- package/dist/types/requestRouter.js +15 -0
- package/dist/utils/providerErrorClassification.d.ts +24 -0
- package/dist/utils/providerErrorClassification.js +88 -0
- package/package.json +4 -2
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ModelPool runtime — multi-provider failover with per-member cooldown.
|
|
3
|
+
*
|
|
4
|
+
* This module is PURE (no provider imports, no circular dependencies).
|
|
5
|
+
* It works exclusively with provider NAMES + (provider, model, region) tuples;
|
|
6
|
+
* the existing AIProviderFactory creates actual provider instances.
|
|
7
|
+
*
|
|
8
|
+
* Usage:
|
|
9
|
+
* const pool = new ModelPool({ members: [...], strategy: "round-robin" });
|
|
10
|
+
* const member = pool.selectNext();
|
|
11
|
+
* try {
|
|
12
|
+
* await callProvider(member);
|
|
13
|
+
* pool.recordSuccess(member);
|
|
14
|
+
* } catch (err) {
|
|
15
|
+
* pool.recordFailure(member, classifyProviderError(err));
|
|
16
|
+
* // try next member ...
|
|
17
|
+
* }
|
|
18
|
+
*/
|
|
19
|
+
import { AuthenticationError, AuthorizationError, ModelAccessDeniedError, } from "../types/index.js";
|
|
20
|
+
import { looksLikeModelAccessDenied } from "../utils/providerErrorClassification.js";
|
|
21
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
22
|
+
// Error classification
|
|
23
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
24
|
+
/**
|
|
25
|
+
* Returns the HTTP status code embedded in an error object, or undefined.
|
|
26
|
+
*/
|
|
27
|
+
function extractHttpStatus(error) {
|
|
28
|
+
if (!error || typeof error !== "object") {
|
|
29
|
+
return undefined;
|
|
30
|
+
}
|
|
31
|
+
const e = error;
|
|
32
|
+
if (typeof e.status === "number") {
|
|
33
|
+
return e.status;
|
|
34
|
+
}
|
|
35
|
+
if (typeof e.statusCode === "number") {
|
|
36
|
+
return e.statusCode;
|
|
37
|
+
}
|
|
38
|
+
return undefined;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Classify a provider error into a coarse `ProviderErrorClass`.
|
|
42
|
+
*
|
|
43
|
+
* Rules (checked in order):
|
|
44
|
+
* 1. HTTP 429 or message pattern → "rate_limit"
|
|
45
|
+
* 2. HTTP 401/403, access-denied pattern, or auth keywords → "auth"
|
|
46
|
+
* 3. Context-window / token-limit message → "context_window"
|
|
47
|
+
* 4. HTTP 5xx or server-error message → "server"
|
|
48
|
+
* 5. Network connectivity error → "network"
|
|
49
|
+
* 6. Everything else → "unknown"
|
|
50
|
+
*/
|
|
51
|
+
export function classifyProviderError(error) {
|
|
52
|
+
const status = extractHttpStatus(error);
|
|
53
|
+
const msg = error instanceof Error
|
|
54
|
+
? error.message
|
|
55
|
+
: typeof error === "object" && error !== null
|
|
56
|
+
? String(error.message ?? error)
|
|
57
|
+
: String(error ?? "");
|
|
58
|
+
const lower = msg.toLowerCase();
|
|
59
|
+
// 1. Rate limit
|
|
60
|
+
if (status === 429 ||
|
|
61
|
+
/rate.?limit|quota exceeded|too many requests|requests per minute/.test(lower)) {
|
|
62
|
+
return "rate_limit";
|
|
63
|
+
}
|
|
64
|
+
// 2. Auth / access denied — also check typed error classes so that a
|
|
65
|
+
// typed AuthenticationError or ModelAccessDeniedError is classified as
|
|
66
|
+
// "auth" even when its message doesn't match the regex patterns.
|
|
67
|
+
if (status === 401 ||
|
|
68
|
+
status === 403 ||
|
|
69
|
+
error instanceof AuthenticationError ||
|
|
70
|
+
error instanceof AuthorizationError ||
|
|
71
|
+
error instanceof ModelAccessDeniedError ||
|
|
72
|
+
looksLikeModelAccessDenied(error) ||
|
|
73
|
+
/unauthori[sz]ed|forbidden|invalid api.?key|api key|access denied|authentication failed|permission denied|unauthenticated/.test(lower)) {
|
|
74
|
+
return "auth";
|
|
75
|
+
}
|
|
76
|
+
// 3. Context window
|
|
77
|
+
if (/context.?length|maximum context|token.*exceed|exceeds.*context|too.?long|input.*too.?large/.test(lower)) {
|
|
78
|
+
return "context_window";
|
|
79
|
+
}
|
|
80
|
+
// 4. Server error
|
|
81
|
+
if ((status !== undefined && status >= 500 && status < 600) ||
|
|
82
|
+
/server error|internal server|overloaded|service unavailable|bad gateway|gateway timeout/.test(lower)) {
|
|
83
|
+
return "server";
|
|
84
|
+
}
|
|
85
|
+
// 5. Network
|
|
86
|
+
if (/econnreset|etimedout|enotfound|network error|socket hang up|connection refused|connection reset/.test(lower)) {
|
|
87
|
+
return "network";
|
|
88
|
+
}
|
|
89
|
+
return "unknown";
|
|
90
|
+
}
|
|
91
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
92
|
+
// Cooldown constants
|
|
93
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
94
|
+
const DEFAULT_COOLDOWN_MS = 60_000; // 1 minute for retryable errors
|
|
95
|
+
// Non-retryable errors use a permanent-for-the-instance cooldown: 10 years.
|
|
96
|
+
// NOTE: Because ModelPool is stored as a long-lived field on the NeuroLink
|
|
97
|
+
// instance (neurolink.ts constructor), a "permanent" cooldown blocks the
|
|
98
|
+
// member for the entire lifetime of the NeuroLink instance, not just the
|
|
99
|
+
// current call. Only use PERMANENT_COOLDOWN_MS for error classes where the
|
|
100
|
+
// root cause is structural and will not resolve between calls (e.g. wrong API
|
|
101
|
+
// key, model not whitelisted for the team). Transient or unclassified errors
|
|
102
|
+
// must use the timed cooldown so a momentary failure doesn't permanently
|
|
103
|
+
// retire a healthy member.
|
|
104
|
+
const PERMANENT_COOLDOWN_MS = 10 * 365 * 24 * 60 * 60 * 1000;
|
|
105
|
+
/**
|
|
106
|
+
* Error classes that should receive a timed cooldown rather than a permanent
|
|
107
|
+
* ban. "unknown" is deliberately included: because the classification is
|
|
108
|
+
* uncertain, treating an unrecognised error as permanent could retire a
|
|
109
|
+
* healthy member due to a one-off network blip or a non-standard error
|
|
110
|
+
* message. A timed cooldown lets the member recover automatically.
|
|
111
|
+
*/
|
|
112
|
+
const RETRYABLE_ERROR_CLASSES = new Set([
|
|
113
|
+
"rate_limit",
|
|
114
|
+
"server",
|
|
115
|
+
"network",
|
|
116
|
+
"unknown",
|
|
117
|
+
]);
|
|
118
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
119
|
+
// ModelPool
|
|
120
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
121
|
+
/**
|
|
122
|
+
* Multi-provider pool with per-member cooldown and strategy-based selection.
|
|
123
|
+
*
|
|
124
|
+
* All state (cooldowns, cursor) is instance-local and resets on construction.
|
|
125
|
+
* Thread safety is not required — Node.js is single-threaded for async work.
|
|
126
|
+
*/
|
|
127
|
+
export class ModelPool {
|
|
128
|
+
config;
|
|
129
|
+
strategy;
|
|
130
|
+
cooldownMs;
|
|
131
|
+
state;
|
|
132
|
+
cursor;
|
|
133
|
+
now;
|
|
134
|
+
constructor(config, injectors) {
|
|
135
|
+
this.config = config;
|
|
136
|
+
this.strategy = config.strategy ?? "priority";
|
|
137
|
+
this.cooldownMs = config.cooldownMs ?? DEFAULT_COOLDOWN_MS;
|
|
138
|
+
this.now = injectors?.now ?? Date.now;
|
|
139
|
+
this.cursor = 0;
|
|
140
|
+
this.state = new Map();
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* The maximum number of attempts per call (pool config value or member count).
|
|
144
|
+
* Used by callers that drive the retry loop externally.
|
|
145
|
+
*/
|
|
146
|
+
get maxAttempts() {
|
|
147
|
+
return this.config.maxAttempts ?? this.config.members.length;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Returns a stable string key for a pool member.
|
|
151
|
+
* Format: `${provider}:${model ?? "*"}:${region ?? "*"}`
|
|
152
|
+
*/
|
|
153
|
+
memberKey(member) {
|
|
154
|
+
return `${member.provider}:${member.model ?? "*"}:${member.region ?? "*"}`;
|
|
155
|
+
}
|
|
156
|
+
/** Returns members whose cooldown has expired (or were never cooled). */
|
|
157
|
+
availableMembers() {
|
|
158
|
+
const now = this.now();
|
|
159
|
+
return this.config.members.filter((m) => {
|
|
160
|
+
const s = this.state.get(this.memberKey(m));
|
|
161
|
+
return !s || s.cooldownUntil <= now;
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
/**
|
|
165
|
+
* Selects the next member to try according to the configured strategy.
|
|
166
|
+
*
|
|
167
|
+
* @param excludedKeys — keys of members already attempted this call.
|
|
168
|
+
* @returns the chosen member, or undefined when all members are exhausted.
|
|
169
|
+
*/
|
|
170
|
+
selectNext(excludedKeys) {
|
|
171
|
+
const candidates = this.availableMembers().filter((m) => !excludedKeys?.has(this.memberKey(m)));
|
|
172
|
+
if (candidates.length === 0) {
|
|
173
|
+
return undefined;
|
|
174
|
+
}
|
|
175
|
+
switch (this.strategy) {
|
|
176
|
+
case "priority":
|
|
177
|
+
return candidates[0];
|
|
178
|
+
case "round-robin": {
|
|
179
|
+
// Advance cursor to the next candidate index in the ORIGINAL list order.
|
|
180
|
+
const allMembers = this.config.members;
|
|
181
|
+
const candidateSet = new Set(candidates.map((m) => this.memberKey(m)));
|
|
182
|
+
// Find the next member starting from cursor that is in candidateSet.
|
|
183
|
+
for (let i = 0; i < allMembers.length; i++) {
|
|
184
|
+
const idx = (this.cursor + i) % allMembers.length;
|
|
185
|
+
const member = allMembers[idx];
|
|
186
|
+
if (candidateSet.has(this.memberKey(member))) {
|
|
187
|
+
// Advance cursor past this pick for the next call.
|
|
188
|
+
this.cursor = (idx + 1) % allMembers.length;
|
|
189
|
+
return member;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
return candidates[0];
|
|
193
|
+
}
|
|
194
|
+
case "weighted": {
|
|
195
|
+
// Deterministic weighted selection: compute cumulative weights, pick
|
|
196
|
+
// by cursor position (mod total weight) so repeated calls rotate.
|
|
197
|
+
const totalWeight = candidates.reduce((sum, m) => sum + (m.weight ?? 1), 0);
|
|
198
|
+
if (totalWeight <= 0) {
|
|
199
|
+
return candidates[0];
|
|
200
|
+
}
|
|
201
|
+
const pick = this.cursor % totalWeight;
|
|
202
|
+
let accumulated = 0;
|
|
203
|
+
for (const member of candidates) {
|
|
204
|
+
accumulated += member.weight ?? 1;
|
|
205
|
+
if (pick < accumulated) {
|
|
206
|
+
this.cursor = (this.cursor + 1) % Math.max(totalWeight, 1);
|
|
207
|
+
return member;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
// Fallback (floating-point edge case)
|
|
211
|
+
this.cursor = (this.cursor + 1) % Math.max(totalWeight, 1);
|
|
212
|
+
return candidates[candidates.length - 1];
|
|
213
|
+
}
|
|
214
|
+
default:
|
|
215
|
+
return candidates[0];
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Records a provider failure, setting a cooldown appropriate for the error class.
|
|
220
|
+
*
|
|
221
|
+
* - Retryable classes (rate_limit, server, network, unknown): timed cooldown
|
|
222
|
+
* for `cooldownMs` so the member can recover and be retried later.
|
|
223
|
+
* - Non-retryable classes (auth, context_window): permanent cooldown for the
|
|
224
|
+
* lifetime of this ModelPool instance, because these errors are structural
|
|
225
|
+
* and will not resolve between calls (wrong credentials, model not available
|
|
226
|
+
* in team whitelist, payload exceeds the model's context window).
|
|
227
|
+
*/
|
|
228
|
+
recordFailure(member, errorClass) {
|
|
229
|
+
const key = this.memberKey(member);
|
|
230
|
+
const cooldown = RETRYABLE_ERROR_CLASSES.has(errorClass)
|
|
231
|
+
? this.now() + this.cooldownMs
|
|
232
|
+
: this.now() + PERMANENT_COOLDOWN_MS;
|
|
233
|
+
this.state.set(key, { cooldownUntil: cooldown });
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* Records a successful response, clearing any existing cooldown for this member
|
|
237
|
+
* so it remains fully available.
|
|
238
|
+
*/
|
|
239
|
+
recordSuccess(member) {
|
|
240
|
+
this.state.delete(this.memberKey(member));
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
//# sourceMappingURL=modelPool.js.map
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Default heuristic request router.
|
|
3
|
+
*
|
|
4
|
+
* `createDefaultRequestRouter` produces a `RequestRouter` function that maps
|
|
5
|
+
* lightweight request characteristics to a provider/model/region override.
|
|
6
|
+
* It is entirely optional — hosts may supply their own `RequestRouter` function
|
|
7
|
+
* instead, or disable routing entirely by not passing either.
|
|
8
|
+
*
|
|
9
|
+
* This PR adds `requestRouter` (and `modelPool`) as NeuroLink constructor
|
|
10
|
+
* options. The router is skipped only when BOTH options.provider AND
|
|
11
|
+
* options.model are explicitly set by the caller — a partial pin (e.g. only
|
|
12
|
+
* provider) still allows the router to run.
|
|
13
|
+
*
|
|
14
|
+
* Decision logic (checked in order):
|
|
15
|
+
* 1. Vision request → visionTier
|
|
16
|
+
* 2. Large input or tools → largeTier
|
|
17
|
+
* 3. Default → smallTier (only when config.smallTier is provided)
|
|
18
|
+
*
|
|
19
|
+
* When none of the configured tiers matches (or when the tier is not
|
|
20
|
+
* configured), the router returns `{}` which is a no-op — the caller keeps
|
|
21
|
+
* whatever provider/model was set by the user.
|
|
22
|
+
*/
|
|
23
|
+
import type { DefaultRequestRouterConfig, RequestRouter } from "../types/index.js";
|
|
24
|
+
/**
|
|
25
|
+
* Creates a heuristic `RequestRouter` from a `DefaultRequestRouterConfig`.
|
|
26
|
+
*
|
|
27
|
+
* @param config — optional tier overrides; built-in defaults apply when omitted.
|
|
28
|
+
* @returns a synchronous `RequestRouter` function (satisfies the async signature).
|
|
29
|
+
*/
|
|
30
|
+
export declare function createDefaultRequestRouter(config?: DefaultRequestRouterConfig): RequestRouter;
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Default heuristic request router.
|
|
3
|
+
*
|
|
4
|
+
* `createDefaultRequestRouter` produces a `RequestRouter` function that maps
|
|
5
|
+
* lightweight request characteristics to a provider/model/region override.
|
|
6
|
+
* It is entirely optional — hosts may supply their own `RequestRouter` function
|
|
7
|
+
* instead, or disable routing entirely by not passing either.
|
|
8
|
+
*
|
|
9
|
+
* This PR adds `requestRouter` (and `modelPool`) as NeuroLink constructor
|
|
10
|
+
* options. The router is skipped only when BOTH options.provider AND
|
|
11
|
+
* options.model are explicitly set by the caller — a partial pin (e.g. only
|
|
12
|
+
* provider) still allows the router to run.
|
|
13
|
+
*
|
|
14
|
+
* Decision logic (checked in order):
|
|
15
|
+
* 1. Vision request → visionTier
|
|
16
|
+
* 2. Large input or tools → largeTier
|
|
17
|
+
* 3. Default → smallTier (only when config.smallTier is provided)
|
|
18
|
+
*
|
|
19
|
+
* When none of the configured tiers matches (or when the tier is not
|
|
20
|
+
* configured), the router returns `{}` which is a no-op — the caller keeps
|
|
21
|
+
* whatever provider/model was set by the user.
|
|
22
|
+
*/
|
|
23
|
+
/** Token threshold above which we treat a request as "large". */
|
|
24
|
+
const DEFAULT_LARGE_TOKEN_THRESHOLD = 32_000;
|
|
25
|
+
/**
|
|
26
|
+
* Built-in defaults for each tier.
|
|
27
|
+
* These are intentionally conservative — hosts should override them to match
|
|
28
|
+
* their provider credentials and preferred models.
|
|
29
|
+
*/
|
|
30
|
+
const DEFAULT_VISION_TIER = {
|
|
31
|
+
provider: "vertex",
|
|
32
|
+
model: "gemini-2.5-flash",
|
|
33
|
+
};
|
|
34
|
+
const DEFAULT_LARGE_TIER = {
|
|
35
|
+
provider: "anthropic",
|
|
36
|
+
model: "claude-sonnet-4-5",
|
|
37
|
+
};
|
|
38
|
+
const DEFAULT_SMALL_TIER = {
|
|
39
|
+
provider: "anthropic",
|
|
40
|
+
model: "claude-haiku-3-5",
|
|
41
|
+
};
|
|
42
|
+
function tierToDecision(tier, reason) {
|
|
43
|
+
return {
|
|
44
|
+
provider: tier.provider,
|
|
45
|
+
model: tier.model,
|
|
46
|
+
region: tier.region,
|
|
47
|
+
reason,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Creates a heuristic `RequestRouter` from a `DefaultRequestRouterConfig`.
|
|
52
|
+
*
|
|
53
|
+
* @param config — optional tier overrides; built-in defaults apply when omitted.
|
|
54
|
+
* @returns a synchronous `RequestRouter` function (satisfies the async signature).
|
|
55
|
+
*/
|
|
56
|
+
export function createDefaultRequestRouter(config) {
|
|
57
|
+
const largeTokenThreshold = config?.largeInputTokenThreshold ?? DEFAULT_LARGE_TOKEN_THRESHOLD;
|
|
58
|
+
const visionTier = config?.visionTier ?? DEFAULT_VISION_TIER;
|
|
59
|
+
const largeTier = config?.largeTier ?? DEFAULT_LARGE_TIER;
|
|
60
|
+
const smallTier = config?.smallTier ?? DEFAULT_SMALL_TIER;
|
|
61
|
+
return function defaultRequestRouter(ctx) {
|
|
62
|
+
// 1. Vision
|
|
63
|
+
if (ctx.requiresVision) {
|
|
64
|
+
return tierToDecision(visionTier, "vision request detected");
|
|
65
|
+
}
|
|
66
|
+
// 2. Large input or tool-heavy
|
|
67
|
+
const tokenCount = ctx.estimatedInputTokens ?? 0;
|
|
68
|
+
if (tokenCount >= largeTokenThreshold || ctx.hasTools) {
|
|
69
|
+
return tierToDecision(largeTier, ctx.hasTools ? "tool-enabled request" : "large input detected");
|
|
70
|
+
}
|
|
71
|
+
// 3. Default small/fast tier — only when config.smallTier is explicitly
|
|
72
|
+
// provided. When not configured, return {} (no-op) so callers that did
|
|
73
|
+
// not opt into a small tier see no surprise provider override.
|
|
74
|
+
if (config?.smallTier !== undefined) {
|
|
75
|
+
return tierToDecision(smallTier, "default small tier");
|
|
76
|
+
}
|
|
77
|
+
// No override — let the caller keep its own provider/model.
|
|
78
|
+
return {};
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=requestRouter.js.map
|
|
@@ -9,6 +9,8 @@ import type { ConversationMemoryConfig } from "./conversation.js";
|
|
|
9
9
|
import type { ObservabilityConfig } from "./observability.js";
|
|
10
10
|
import type { AuthProvider, AuthProviderType, AuthProviderConfig, Auth0Config, ClerkConfig, FirebaseConfig, SupabaseConfig, WorkOSConfig, BetterAuthConfig, JWTConfig, OAuth2Config, CognitoConfig, KeycloakConfig, AuthenticatedContext } from "./auth.js";
|
|
11
11
|
import type { NeurolinkCredentials } from "./providers.js";
|
|
12
|
+
import type { ModelPoolConfig } from "./modelPool.js";
|
|
13
|
+
import type { RequestRouter } from "./requestRouter.js";
|
|
12
14
|
/**
|
|
13
15
|
* Main NeuroLink configuration type
|
|
14
16
|
*/
|
|
@@ -86,6 +88,22 @@ export type NeurolinkConstructorConfig = {
|
|
|
86
88
|
* See {@link ToolDedupConfig}.
|
|
87
89
|
*/
|
|
88
90
|
toolDedup?: ToolDedupConfig;
|
|
91
|
+
/**
|
|
92
|
+
* Multi-provider pool for error-class-aware failover with per-member
|
|
93
|
+
* cooldown. When set, generate() and stream() source their candidate
|
|
94
|
+
* provider sequence from the pool instead of (or in addition to) the
|
|
95
|
+
* static providerPriority fallback. Fails open: a pool error leaves
|
|
96
|
+
* existing behavior unchanged.
|
|
97
|
+
*/
|
|
98
|
+
modelPool?: ModelPoolConfig;
|
|
99
|
+
/**
|
|
100
|
+
* Pluggable pre-call router: inspects lightweight request characteristics
|
|
101
|
+
* (token estimate, tools, vision, thinkingLevel) and returns an optional
|
|
102
|
+
* provider/model/region override. Only runs when the caller did NOT
|
|
103
|
+
* explicitly set options.provider/options.model. Fails open: a router
|
|
104
|
+
* error proceeds unrouted.
|
|
105
|
+
*/
|
|
106
|
+
requestRouter?: RequestRouter;
|
|
89
107
|
};
|
|
90
108
|
/**
|
|
91
109
|
* Configuration for MCP enhancement modules wired into generate()/stream() paths.
|
package/dist/lib/types/index.js
CHANGED
|
@@ -76,4 +76,8 @@ export * from "./music.js";
|
|
|
76
76
|
export * from "./replicate.js";
|
|
77
77
|
// Safe-fetch helper types (SSRF-hardened download)
|
|
78
78
|
export * from "./safeFetch.js";
|
|
79
|
+
// ModelPool — multi-provider failover with per-member cooldown (M9.x+)
|
|
80
|
+
export * from "./modelPool.js";
|
|
81
|
+
// RequestRouter — pluggable pre-call provider/model selection (M9.x+)
|
|
82
|
+
export * from "./requestRouter.js";
|
|
79
83
|
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ModelPool types — multi-provider failover with per-member cooldown.
|
|
3
|
+
*
|
|
4
|
+
* A ModelPool holds an ordered list of (provider, model, region) members and
|
|
5
|
+
* selects among them per-turn using a configurable strategy. On provider
|
|
6
|
+
* failure, the pool cools down the failed member and retries with the next
|
|
7
|
+
* available one, using error-class-aware logic to distinguish retryable from
|
|
8
|
+
* permanent failures.
|
|
9
|
+
*/
|
|
10
|
+
/** One candidate provider/model/region tuple in a pool. */
|
|
11
|
+
export type ModelPoolMember = {
|
|
12
|
+
provider: string;
|
|
13
|
+
model?: string;
|
|
14
|
+
region?: string;
|
|
15
|
+
/** Relative weight used by the "weighted" strategy. Default: 1. */
|
|
16
|
+
weight?: number;
|
|
17
|
+
};
|
|
18
|
+
/** Member-selection strategy when picking the next candidate. */
|
|
19
|
+
export type ModelPoolStrategy = "priority" | "round-robin" | "weighted";
|
|
20
|
+
/**
|
|
21
|
+
* Coarse error class returned by `classifyProviderError`.
|
|
22
|
+
* Drives the cooldown decision inside ModelPool.
|
|
23
|
+
*/
|
|
24
|
+
export type ProviderErrorClass = "rate_limit" | "auth" | "context_window" | "server" | "network" | "unknown";
|
|
25
|
+
/** Constructor-level configuration for a ModelPool instance. */
|
|
26
|
+
export type ModelPoolConfig = {
|
|
27
|
+
/** Ordered list of provider/model/region candidates. */
|
|
28
|
+
members: ModelPoolMember[];
|
|
29
|
+
/**
|
|
30
|
+
* How to pick among available members.
|
|
31
|
+
* - "priority" — always try the first available member (default).
|
|
32
|
+
* - "round-robin" — rotate through members in order.
|
|
33
|
+
* - "weighted" — prefer members with higher weight, varies by cursor.
|
|
34
|
+
*/
|
|
35
|
+
strategy?: ModelPoolStrategy;
|
|
36
|
+
/**
|
|
37
|
+
* How long (ms) a failed member stays in cooldown before it is eligible
|
|
38
|
+
* again. Applies to retryable error classes (rate_limit, server, network).
|
|
39
|
+
* Default: 60_000 (1 minute).
|
|
40
|
+
*/
|
|
41
|
+
cooldownMs?: number;
|
|
42
|
+
/**
|
|
43
|
+
* Maximum total attempts across all pool members per call.
|
|
44
|
+
* Default: members.length (try every member once).
|
|
45
|
+
*/
|
|
46
|
+
maxAttempts?: number;
|
|
47
|
+
};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ModelPool types — multi-provider failover with per-member cooldown.
|
|
3
|
+
*
|
|
4
|
+
* A ModelPool holds an ordered list of (provider, model, region) members and
|
|
5
|
+
* selects among them per-turn using a configurable strategy. On provider
|
|
6
|
+
* failure, the pool cools down the failed member and retries with the next
|
|
7
|
+
* available one, using error-class-aware logic to distinguish retryable from
|
|
8
|
+
* permanent failures.
|
|
9
|
+
*/
|
|
10
|
+
export {};
|
|
11
|
+
//# sourceMappingURL=modelPool.js.map
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pluggable pre-call request router types.
|
|
3
|
+
*
|
|
4
|
+
* A RequestRouter is a host-supplied async function that inspects lightweight
|
|
5
|
+
* request characteristics (token estimate, tools, vision, thinking) and
|
|
6
|
+
* returns an optional provider/model/region override. The router runs only
|
|
7
|
+
* when the host opts in (by passing `requestRouter` or `modelPool` to
|
|
8
|
+
* the NeuroLink constructor) and only when the user did NOT explicitly set
|
|
9
|
+
* BOTH `options.provider` AND `options.model` on the call (a partial pin —
|
|
10
|
+
* e.g. only `options.provider` — still allows the router to run).
|
|
11
|
+
*
|
|
12
|
+
* Everything fails open: if the router throws or returns an empty decision the
|
|
13
|
+
* call proceeds with whatever provider/model was configured by the caller.
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* Lightweight characteristics of the incoming request available to the router
|
|
17
|
+
* without executing the full call.
|
|
18
|
+
*/
|
|
19
|
+
export type RouterInputContext = {
|
|
20
|
+
/** The text prompt (or first text segment of a multi-modal input). */
|
|
21
|
+
prompt: string;
|
|
22
|
+
/** Rough token estimate for the input, if known by the caller. */
|
|
23
|
+
estimatedInputTokens?: number;
|
|
24
|
+
/** True when the call includes at least one tool definition. */
|
|
25
|
+
hasTools?: boolean;
|
|
26
|
+
/** True when the call includes at least one image/vision attachment. */
|
|
27
|
+
requiresVision?: boolean;
|
|
28
|
+
/** Thinking level passed to the call ("minimal" | "low" | "medium" | "high"). */
|
|
29
|
+
thinkingLevel?: string;
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* The router's decision. Any field that is undefined means "keep whatever the
|
|
33
|
+
* caller already configured" — returning `{}` is a valid no-op.
|
|
34
|
+
*/
|
|
35
|
+
export type RequestRouterDecision = {
|
|
36
|
+
provider?: string;
|
|
37
|
+
model?: string;
|
|
38
|
+
region?: string;
|
|
39
|
+
/** Optional human-readable reason, emitted at debug log level. */
|
|
40
|
+
reason?: string;
|
|
41
|
+
};
|
|
42
|
+
/**
|
|
43
|
+
* A pluggable pre-call router function.
|
|
44
|
+
*
|
|
45
|
+
* Receives a lightweight context snapshot and returns provider/model/region
|
|
46
|
+
* overrides. May be async (e.g. to consult a remote config service).
|
|
47
|
+
*/
|
|
48
|
+
export type RequestRouter = (context: RouterInputContext) => RequestRouterDecision | Promise<RequestRouterDecision>;
|
|
49
|
+
/**
|
|
50
|
+
* Tier-to-(provider,model) mapping used by `createDefaultRequestRouter`.
|
|
51
|
+
* Each tier is optional; an unmatched tier produces an empty decision.
|
|
52
|
+
*/
|
|
53
|
+
export type ModelTierEntry = {
|
|
54
|
+
provider: string;
|
|
55
|
+
model: string;
|
|
56
|
+
region?: string;
|
|
57
|
+
};
|
|
58
|
+
/**
|
|
59
|
+
* Configuration for the built-in heuristic request router produced by
|
|
60
|
+
* `createDefaultRequestRouter`. All fields are optional; sensible defaults
|
|
61
|
+
* apply when omitted.
|
|
62
|
+
*/
|
|
63
|
+
export type DefaultRequestRouterConfig = {
|
|
64
|
+
/**
|
|
65
|
+
* Token threshold above which the "large" tier is selected.
|
|
66
|
+
* Default: 32_000.
|
|
67
|
+
*/
|
|
68
|
+
largeInputTokenThreshold?: number;
|
|
69
|
+
/** Provider/model to use for vision requests. */
|
|
70
|
+
visionTier?: ModelTierEntry;
|
|
71
|
+
/** Provider/model to use for large inputs or tool-heavy requests. */
|
|
72
|
+
largeTier?: ModelTierEntry;
|
|
73
|
+
/** Provider/model to use for fast/small requests. */
|
|
74
|
+
smallTier?: ModelTierEntry;
|
|
75
|
+
};
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pluggable pre-call request router types.
|
|
3
|
+
*
|
|
4
|
+
* A RequestRouter is a host-supplied async function that inspects lightweight
|
|
5
|
+
* request characteristics (token estimate, tools, vision, thinking) and
|
|
6
|
+
* returns an optional provider/model/region override. The router runs only
|
|
7
|
+
* when the host opts in (by passing `requestRouter` or `modelPool` to
|
|
8
|
+
* the NeuroLink constructor) and only when the user did NOT explicitly set
|
|
9
|
+
* BOTH `options.provider` AND `options.model` on the call (a partial pin —
|
|
10
|
+
* e.g. only `options.provider` — still allows the router to run).
|
|
11
|
+
*
|
|
12
|
+
* Everything fails open: if the router throws or returns an empty decision the
|
|
13
|
+
* call proceeds with whatever provider/model was configured by the caller.
|
|
14
|
+
*/
|
|
15
|
+
export {};
|
|
16
|
+
//# sourceMappingURL=requestRouter.js.map
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared provider-error classification utilities used by both neurolink.ts and
|
|
3
|
+
* the ModelPool routing subsystem.
|
|
4
|
+
*
|
|
5
|
+
* Extracted here to avoid duplication and to ensure that typed error classes
|
|
6
|
+
* (AuthenticationError, AuthorizationError, ModelAccessDeniedError) are
|
|
7
|
+
* checked consistently in both code paths.
|
|
8
|
+
*/
|
|
9
|
+
/**
|
|
10
|
+
* Detects whether an error object looks like a model-access-denied condition.
|
|
11
|
+
* Matches LiteLLM "team not allowed" / "team can only access models=[...]"
|
|
12
|
+
* plus typed-error name/code markers when the full typed class is not present.
|
|
13
|
+
*/
|
|
14
|
+
export declare function looksLikeModelAccessDenied(error: unknown): boolean;
|
|
15
|
+
/**
|
|
16
|
+
* Returns true when the error is definitively non-retryable: typed error
|
|
17
|
+
* classes (auth, access-denied, invalid-model), non-retryable HTTP status
|
|
18
|
+
* codes, or deterministic 400-class message patterns.
|
|
19
|
+
*
|
|
20
|
+
* NOTE: ContextBudgetExceededError is intentionally NOT non-retryable —
|
|
21
|
+
* each provider has its own context window, so a budget rejection on one
|
|
22
|
+
* provider does not preclude another provider accepting the same payload.
|
|
23
|
+
*/
|
|
24
|
+
export declare function isNonRetryableProviderError(error: unknown): boolean;
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared provider-error classification utilities used by both neurolink.ts and
|
|
3
|
+
* the ModelPool routing subsystem.
|
|
4
|
+
*
|
|
5
|
+
* Extracted here to avoid duplication and to ensure that typed error classes
|
|
6
|
+
* (AuthenticationError, AuthorizationError, ModelAccessDeniedError) are
|
|
7
|
+
* checked consistently in both code paths.
|
|
8
|
+
*/
|
|
9
|
+
import { AuthenticationError, AuthorizationError, InvalidModelError, ModelAccessDeniedError, } from "../types/index.js";
|
|
10
|
+
import { NON_RETRYABLE_HTTP_STATUS_CODES, isDeterministicClientErrorMessage, } from "./retryability.js";
|
|
11
|
+
/**
|
|
12
|
+
* Detects whether an error object looks like a model-access-denied condition.
|
|
13
|
+
* Matches LiteLLM "team not allowed" / "team can only access models=[...]"
|
|
14
|
+
* plus typed-error name/code markers when the full typed class is not present.
|
|
15
|
+
*/
|
|
16
|
+
export function looksLikeModelAccessDenied(error) {
|
|
17
|
+
if (!error) {
|
|
18
|
+
return false;
|
|
19
|
+
}
|
|
20
|
+
const e = error;
|
|
21
|
+
if (e.name === "ModelAccessDeniedError") {
|
|
22
|
+
return true;
|
|
23
|
+
}
|
|
24
|
+
if (e.code === "MODEL_ACCESS_DENIED") {
|
|
25
|
+
return true;
|
|
26
|
+
}
|
|
27
|
+
const msg = typeof e.message === "string"
|
|
28
|
+
? e.message
|
|
29
|
+
: error instanceof Error
|
|
30
|
+
? error.message
|
|
31
|
+
: String(error);
|
|
32
|
+
if (!msg) {
|
|
33
|
+
return false;
|
|
34
|
+
}
|
|
35
|
+
const lower = msg.toLowerCase();
|
|
36
|
+
return ((lower.includes("team") && lower.includes("not allowed")) ||
|
|
37
|
+
lower.includes("team can only access") ||
|
|
38
|
+
/not\s+allowed\s+to\s+access\s+(this\s+)?model/i.test(msg));
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Returns true when the error is definitively non-retryable: typed error
|
|
42
|
+
* classes (auth, access-denied, invalid-model), non-retryable HTTP status
|
|
43
|
+
* codes, or deterministic 400-class message patterns.
|
|
44
|
+
*
|
|
45
|
+
* NOTE: ContextBudgetExceededError is intentionally NOT non-retryable —
|
|
46
|
+
* each provider has its own context window, so a budget rejection on one
|
|
47
|
+
* provider does not preclude another provider accepting the same payload.
|
|
48
|
+
*/
|
|
49
|
+
export function isNonRetryableProviderError(error) {
|
|
50
|
+
if (error instanceof InvalidModelError) {
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
53
|
+
if (error instanceof AuthenticationError) {
|
|
54
|
+
return true;
|
|
55
|
+
}
|
|
56
|
+
if (error instanceof AuthorizationError) {
|
|
57
|
+
return true;
|
|
58
|
+
}
|
|
59
|
+
if (error instanceof ModelAccessDeniedError) {
|
|
60
|
+
return true;
|
|
61
|
+
}
|
|
62
|
+
if (error && typeof error === "object") {
|
|
63
|
+
const err = error;
|
|
64
|
+
const status = typeof err.status === "number"
|
|
65
|
+
? err.status
|
|
66
|
+
: typeof err.statusCode === "number"
|
|
67
|
+
? err.statusCode
|
|
68
|
+
: undefined;
|
|
69
|
+
if (status !== undefined &&
|
|
70
|
+
NON_RETRYABLE_HTTP_STATUS_CODES.includes(status)) {
|
|
71
|
+
return true;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
if (error instanceof Error) {
|
|
75
|
+
const msg = error.message;
|
|
76
|
+
if (msg.includes("NOT_FOUND") ||
|
|
77
|
+
msg.includes("Model Not Found") ||
|
|
78
|
+
msg.includes("model not found") ||
|
|
79
|
+
msg.includes("PERMISSION_DENIED") ||
|
|
80
|
+
msg.includes("UNAUTHENTICATED")) {
|
|
81
|
+
return true;
|
|
82
|
+
}
|
|
83
|
+
if (isDeterministicClientErrorMessage(msg)) {
|
|
84
|
+
return true;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return false;
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=providerErrorClassification.js.map
|