@juspay/neurolink 10.2.1 → 10.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/browser/neurolink.min.js +298 -298
- package/dist/constants/contextWindows.d.ts +12 -0
- package/dist/constants/contextWindows.js +38 -0
- package/dist/lib/constants/contextWindows.d.ts +12 -0
- package/dist/lib/constants/contextWindows.js +38 -0
- package/dist/lib/providers/litellm.d.ts +19 -0
- package/dist/lib/providers/litellm.js +92 -0
- package/dist/providers/litellm.d.ts +19 -0
- package/dist/providers/litellm.js +92 -0
- package/package.json +4 -3
|
@@ -21,12 +21,24 @@ export declare const DEFAULT_OUTPUT_RESERVE_RATIO = 0.35;
|
|
|
21
21
|
* The "_default" key is the fallback for unknown models within a provider.
|
|
22
22
|
*/
|
|
23
23
|
export declare const MODEL_CONTEXT_WINDOWS: Record<string, Record<string, number>>;
|
|
24
|
+
/**
|
|
25
|
+
* Register a runtime-discovered context window for a provider/model pair.
|
|
26
|
+
* Later registrations overwrite earlier ones (rediscovery refreshes values).
|
|
27
|
+
* Non-positive/non-finite windows are ignored so a malformed discovery source
|
|
28
|
+
* can never shrink a budget to zero.
|
|
29
|
+
*/
|
|
30
|
+
export declare function registerRuntimeContextWindow(provider: string, model: string, contextWindow: number): void;
|
|
31
|
+
/** Test hook: clear runtime-discovered windows (state is module-global). */
|
|
32
|
+
export declare function clearRuntimeContextWindows(): void;
|
|
24
33
|
/**
|
|
25
34
|
* Resolve context window size for a provider/model combination.
|
|
26
35
|
*
|
|
27
36
|
* Priority:
|
|
28
37
|
* 0. Dynamic model registry (DynamicModelProvider) — resolves cross-provider
|
|
29
38
|
* models (e.g. Claude on Vertex) that the static table cannot handle
|
|
39
|
+
* 0.5 Runtime-discovered windows (registerRuntimeContextWindow) — real
|
|
40
|
+
* per-model limits fetched from the serving infrastructure (LiteLLM
|
|
41
|
+
* `/model/info`)
|
|
30
42
|
* 1. Exact model match under provider in static registry
|
|
31
43
|
* 2. Prefix match under provider in static registry
|
|
32
44
|
* 3. Provider's _default in static registry
|
|
@@ -392,12 +392,43 @@ function normalizeProviderForLookup(provider) {
|
|
|
392
392
|
// to DEFAULT_CONTEXT_WINDOW.
|
|
393
393
|
return PROVIDER_ALIAS_MAP[stripped] ?? stripped;
|
|
394
394
|
}
|
|
395
|
+
/**
|
|
396
|
+
* Runtime-discovered context windows, keyed `${provider}:${model}`.
|
|
397
|
+
*
|
|
398
|
+
* Populated asynchronously by providers that can discover real per-model
|
|
399
|
+
* limits at runtime (e.g. the LiteLLM provider reads `max_input_tokens` from
|
|
400
|
+
* the proxy's `/model/info`), and read synchronously by
|
|
401
|
+
* {@link getContextWindowSize} — the same async-populate/sync-read contract as
|
|
402
|
+
* the DynamicModelProvider registry. Keys use the RAW provider string callers
|
|
403
|
+
* pass into budget calculations (see the alias-map comment above: normalized
|
|
404
|
+
* provider names never reach these lookups).
|
|
405
|
+
*/
|
|
406
|
+
const RUNTIME_CONTEXT_WINDOWS = new Map();
|
|
407
|
+
/**
|
|
408
|
+
* Register a runtime-discovered context window for a provider/model pair.
|
|
409
|
+
* Later registrations overwrite earlier ones (rediscovery refreshes values).
|
|
410
|
+
* Non-positive/non-finite windows are ignored so a malformed discovery source
|
|
411
|
+
* can never shrink a budget to zero.
|
|
412
|
+
*/
|
|
413
|
+
export function registerRuntimeContextWindow(provider, model, contextWindow) {
|
|
414
|
+
if (!Number.isFinite(contextWindow) || contextWindow <= 0) {
|
|
415
|
+
return;
|
|
416
|
+
}
|
|
417
|
+
RUNTIME_CONTEXT_WINDOWS.set(`${provider}:${model}`, contextWindow);
|
|
418
|
+
}
|
|
419
|
+
/** Test hook: clear runtime-discovered windows (state is module-global). */
|
|
420
|
+
export function clearRuntimeContextWindows() {
|
|
421
|
+
RUNTIME_CONTEXT_WINDOWS.clear();
|
|
422
|
+
}
|
|
395
423
|
/**
|
|
396
424
|
* Resolve context window size for a provider/model combination.
|
|
397
425
|
*
|
|
398
426
|
* Priority:
|
|
399
427
|
* 0. Dynamic model registry (DynamicModelProvider) — resolves cross-provider
|
|
400
428
|
* models (e.g. Claude on Vertex) that the static table cannot handle
|
|
429
|
+
* 0.5 Runtime-discovered windows (registerRuntimeContextWindow) — real
|
|
430
|
+
* per-model limits fetched from the serving infrastructure (LiteLLM
|
|
431
|
+
* `/model/info`)
|
|
401
432
|
* 1. Exact model match under provider in static registry
|
|
402
433
|
* 2. Prefix match under provider in static registry
|
|
403
434
|
* 3. Provider's _default in static registry
|
|
@@ -421,6 +452,13 @@ export function getContextWindowSize(provider, model) {
|
|
|
421
452
|
// Dynamic registry not initialized yet — fall through to static lookup
|
|
422
453
|
}
|
|
423
454
|
}
|
|
455
|
+
// Step 0.5: Runtime-discovered window for this exact provider/model.
|
|
456
|
+
if (model) {
|
|
457
|
+
const discovered = RUNTIME_CONTEXT_WINDOWS.get(`${provider}:${model}`);
|
|
458
|
+
if (discovered !== undefined) {
|
|
459
|
+
return discovered;
|
|
460
|
+
}
|
|
461
|
+
}
|
|
424
462
|
// Static fallback chain — normalize aliases first so "lmstudio" / "llama.cpp" /
|
|
425
463
|
// "nvidianim" find their canonical entries instead of falling back to default.
|
|
426
464
|
const canonical = normalizeProviderForLookup(provider);
|
|
@@ -21,12 +21,24 @@ export declare const DEFAULT_OUTPUT_RESERVE_RATIO = 0.35;
|
|
|
21
21
|
* The "_default" key is the fallback for unknown models within a provider.
|
|
22
22
|
*/
|
|
23
23
|
export declare const MODEL_CONTEXT_WINDOWS: Record<string, Record<string, number>>;
|
|
24
|
+
/**
|
|
25
|
+
* Register a runtime-discovered context window for a provider/model pair.
|
|
26
|
+
* Later registrations overwrite earlier ones (rediscovery refreshes values).
|
|
27
|
+
* Non-positive/non-finite windows are ignored so a malformed discovery source
|
|
28
|
+
* can never shrink a budget to zero.
|
|
29
|
+
*/
|
|
30
|
+
export declare function registerRuntimeContextWindow(provider: string, model: string, contextWindow: number): void;
|
|
31
|
+
/** Test hook: clear runtime-discovered windows (state is module-global). */
|
|
32
|
+
export declare function clearRuntimeContextWindows(): void;
|
|
24
33
|
/**
|
|
25
34
|
* Resolve context window size for a provider/model combination.
|
|
26
35
|
*
|
|
27
36
|
* Priority:
|
|
28
37
|
* 0. Dynamic model registry (DynamicModelProvider) — resolves cross-provider
|
|
29
38
|
* models (e.g. Claude on Vertex) that the static table cannot handle
|
|
39
|
+
* 0.5 Runtime-discovered windows (registerRuntimeContextWindow) — real
|
|
40
|
+
* per-model limits fetched from the serving infrastructure (LiteLLM
|
|
41
|
+
* `/model/info`)
|
|
30
42
|
* 1. Exact model match under provider in static registry
|
|
31
43
|
* 2. Prefix match under provider in static registry
|
|
32
44
|
* 3. Provider's _default in static registry
|
|
@@ -392,12 +392,43 @@ function normalizeProviderForLookup(provider) {
|
|
|
392
392
|
// to DEFAULT_CONTEXT_WINDOW.
|
|
393
393
|
return PROVIDER_ALIAS_MAP[stripped] ?? stripped;
|
|
394
394
|
}
|
|
395
|
+
/**
|
|
396
|
+
* Runtime-discovered context windows, keyed `${provider}:${model}`.
|
|
397
|
+
*
|
|
398
|
+
* Populated asynchronously by providers that can discover real per-model
|
|
399
|
+
* limits at runtime (e.g. the LiteLLM provider reads `max_input_tokens` from
|
|
400
|
+
* the proxy's `/model/info`), and read synchronously by
|
|
401
|
+
* {@link getContextWindowSize} — the same async-populate/sync-read contract as
|
|
402
|
+
* the DynamicModelProvider registry. Keys use the RAW provider string callers
|
|
403
|
+
* pass into budget calculations (see the alias-map comment above: normalized
|
|
404
|
+
* provider names never reach these lookups).
|
|
405
|
+
*/
|
|
406
|
+
const RUNTIME_CONTEXT_WINDOWS = new Map();
|
|
407
|
+
/**
|
|
408
|
+
* Register a runtime-discovered context window for a provider/model pair.
|
|
409
|
+
* Later registrations overwrite earlier ones (rediscovery refreshes values).
|
|
410
|
+
* Non-positive/non-finite windows are ignored so a malformed discovery source
|
|
411
|
+
* can never shrink a budget to zero.
|
|
412
|
+
*/
|
|
413
|
+
export function registerRuntimeContextWindow(provider, model, contextWindow) {
|
|
414
|
+
if (!Number.isFinite(contextWindow) || contextWindow <= 0) {
|
|
415
|
+
return;
|
|
416
|
+
}
|
|
417
|
+
RUNTIME_CONTEXT_WINDOWS.set(`${provider}:${model}`, contextWindow);
|
|
418
|
+
}
|
|
419
|
+
/** Test hook: clear runtime-discovered windows (state is module-global). */
|
|
420
|
+
export function clearRuntimeContextWindows() {
|
|
421
|
+
RUNTIME_CONTEXT_WINDOWS.clear();
|
|
422
|
+
}
|
|
395
423
|
/**
|
|
396
424
|
* Resolve context window size for a provider/model combination.
|
|
397
425
|
*
|
|
398
426
|
* Priority:
|
|
399
427
|
* 0. Dynamic model registry (DynamicModelProvider) — resolves cross-provider
|
|
400
428
|
* models (e.g. Claude on Vertex) that the static table cannot handle
|
|
429
|
+
* 0.5 Runtime-discovered windows (registerRuntimeContextWindow) — real
|
|
430
|
+
* per-model limits fetched from the serving infrastructure (LiteLLM
|
|
431
|
+
* `/model/info`)
|
|
401
432
|
* 1. Exact model match under provider in static registry
|
|
402
433
|
* 2. Prefix match under provider in static registry
|
|
403
434
|
* 3. Provider's _default in static registry
|
|
@@ -421,6 +452,13 @@ export function getContextWindowSize(provider, model) {
|
|
|
421
452
|
// Dynamic registry not initialized yet — fall through to static lookup
|
|
422
453
|
}
|
|
423
454
|
}
|
|
455
|
+
// Step 0.5: Runtime-discovered window for this exact provider/model.
|
|
456
|
+
if (model) {
|
|
457
|
+
const discovered = RUNTIME_CONTEXT_WINDOWS.get(`${provider}:${model}`);
|
|
458
|
+
if (discovered !== undefined) {
|
|
459
|
+
return discovered;
|
|
460
|
+
}
|
|
461
|
+
}
|
|
424
462
|
// Static fallback chain — normalize aliases first so "lmstudio" / "llama.cpp" /
|
|
425
463
|
// "nvidianim" find their canonical entries instead of falling back to default.
|
|
426
464
|
const canonical = normalizeProviderForLookup(provider);
|
|
@@ -17,6 +17,12 @@ export declare class LiteLLMProvider extends OpenAIChatCompletionsProvider {
|
|
|
17
17
|
private static modelsCache;
|
|
18
18
|
private static modelsCacheTime;
|
|
19
19
|
private static readonly MODELS_CACHE_DURATION;
|
|
20
|
+
/**
|
|
21
|
+
* Dedupes the fire-and-forget `/model/info` discovery per base URL, so a
|
|
22
|
+
* process constructing many LiteLLM providers fetches each proxy's limits
|
|
23
|
+
* once per cache period.
|
|
24
|
+
*/
|
|
25
|
+
private static modelInfoFetchTime;
|
|
20
26
|
constructor(modelName?: string, sdk?: unknown, _region?: string, credentials?: {
|
|
21
27
|
apiKey?: string;
|
|
22
28
|
baseURL?: string;
|
|
@@ -44,6 +50,19 @@ export declare class LiteLLMProvider extends OpenAIChatCompletionsProvider {
|
|
|
44
50
|
* minimal safe default if the API fetch fails.
|
|
45
51
|
*/
|
|
46
52
|
getAvailableModels(): Promise<string[]>;
|
|
53
|
+
/**
|
|
54
|
+
* Discover real per-model context windows from the LiteLLM proxy's
|
|
55
|
+
* `GET /model/info` (`data[].model_info.max_input_tokens`) and register
|
|
56
|
+
* them with the context-window resolver. Fire-and-forget with the same
|
|
57
|
+
* cache period as the models list; any failure (endpoint absent, auth,
|
|
58
|
+
* timeout) is logged at debug and leaves the static `_default` in force.
|
|
59
|
+
*/
|
|
60
|
+
private discoverModelContextWindows;
|
|
61
|
+
/**
|
|
62
|
+
* Fetch `GET /model/info` and map model group name → max_input_tokens.
|
|
63
|
+
* Same fetch/auth/abort scaffolding as {@link fetchModelsFromAPI}.
|
|
64
|
+
*/
|
|
65
|
+
private fetchModelInfoFromAPI;
|
|
47
66
|
private fetchModelsFromAPI;
|
|
48
67
|
/**
|
|
49
68
|
* Generate an embedding for a single text input via native /v1/embeddings.
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
|
|
2
|
+
import { registerRuntimeContextWindow } from "../constants/contextWindows.js";
|
|
2
3
|
import { createProxyFetch } from "../proxy/proxyFetch.js";
|
|
3
4
|
import { AuthenticationError, InvalidModelError, ModelAccessDeniedError, NetworkError, ProviderError, RateLimitError, isModelAccessDeniedMessage, parseAllowedModels, } from "../types/index.js";
|
|
4
5
|
import { isAbortError } from "../utils/errorHandling.js";
|
|
@@ -45,12 +46,24 @@ export class LiteLLMProvider extends OpenAIChatCompletionsProvider {
|
|
|
45
46
|
static modelsCache = [];
|
|
46
47
|
static modelsCacheTime = 0;
|
|
47
48
|
static MODELS_CACHE_DURATION = 10 * 60 * 1000; // 10 minutes
|
|
49
|
+
/**
|
|
50
|
+
* Dedupes the fire-and-forget `/model/info` discovery per base URL, so a
|
|
51
|
+
* process constructing many LiteLLM providers fetches each proxy's limits
|
|
52
|
+
* once per cache period.
|
|
53
|
+
*/
|
|
54
|
+
static modelInfoFetchTime = new Map();
|
|
48
55
|
constructor(modelName, sdk, _region, credentials) {
|
|
49
56
|
const envConfig = getLiteLLMConfig();
|
|
50
57
|
super("litellm", modelName, sdk, {
|
|
51
58
|
baseURL: credentials?.baseURL ?? envConfig.baseURL,
|
|
52
59
|
apiKey: credentials?.apiKey ?? envConfig.apiKey,
|
|
53
60
|
});
|
|
61
|
+
// Fire-and-forget: discover real per-model context windows from the
|
|
62
|
+
// proxy's /model/info. The static table only has a one-size litellm
|
|
63
|
+
// `_default` (128K), while proxied models range from 8K to 2M — budget
|
|
64
|
+
// checks and compaction need the real window. Failures degrade cleanly
|
|
65
|
+
// to the static default.
|
|
66
|
+
this.discoverModelContextWindows();
|
|
54
67
|
logger.debug("LiteLLM Provider initialized", {
|
|
55
68
|
modelName: this.modelName,
|
|
56
69
|
provider: this.providerName,
|
|
@@ -208,6 +221,85 @@ export class LiteLLMProvider extends OpenAIChatCompletionsProvider {
|
|
|
208
221
|
}
|
|
209
222
|
return this.getFallbackModels();
|
|
210
223
|
}
|
|
224
|
+
/**
|
|
225
|
+
* Discover real per-model context windows from the LiteLLM proxy's
|
|
226
|
+
* `GET /model/info` (`data[].model_info.max_input_tokens`) and register
|
|
227
|
+
* them with the context-window resolver. Fire-and-forget with the same
|
|
228
|
+
* cache period as the models list; any failure (endpoint absent, auth,
|
|
229
|
+
* timeout) is logged at debug and leaves the static `_default` in force.
|
|
230
|
+
*/
|
|
231
|
+
discoverModelContextWindows() {
|
|
232
|
+
const baseURL = stripTrailingSlash(this.config.baseURL);
|
|
233
|
+
const now = Date.now();
|
|
234
|
+
const lastFetch = LiteLLMProvider.modelInfoFetchTime.get(baseURL) ?? 0;
|
|
235
|
+
if (now - lastFetch < LiteLLMProvider.MODELS_CACHE_DURATION) {
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
LiteLLMProvider.modelInfoFetchTime.set(baseURL, now);
|
|
239
|
+
void this.fetchModelInfoFromAPI()
|
|
240
|
+
.then((windows) => {
|
|
241
|
+
for (const [model, contextWindow] of windows) {
|
|
242
|
+
registerRuntimeContextWindow("litellm", model, contextWindow);
|
|
243
|
+
}
|
|
244
|
+
if (windows.size > 0) {
|
|
245
|
+
logger.debug("[LiteLLMProvider] Registered runtime context windows from /model/info", { baseURL: redactUrlCredentials(baseURL), models: windows.size });
|
|
246
|
+
}
|
|
247
|
+
})
|
|
248
|
+
.catch((error) => {
|
|
249
|
+
// Allow a retry before the cache period when discovery failed.
|
|
250
|
+
LiteLLMProvider.modelInfoFetchTime.delete(baseURL);
|
|
251
|
+
logger.debug("[LiteLLMProvider] /model/info discovery failed; static context-window defaults remain in force", { error: error instanceof Error ? error.message : String(error) });
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Fetch `GET /model/info` and map model group name → max_input_tokens.
|
|
256
|
+
* Same fetch/auth/abort scaffolding as {@link fetchModelsFromAPI}.
|
|
257
|
+
*/
|
|
258
|
+
async fetchModelInfoFromAPI() {
|
|
259
|
+
const infoUrl = `${stripTrailingSlash(this.config.baseURL)}/model/info`;
|
|
260
|
+
const proxyFetch = createProxyFetch();
|
|
261
|
+
const controller = new AbortController();
|
|
262
|
+
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
|
263
|
+
try {
|
|
264
|
+
const response = await proxyFetch(infoUrl, {
|
|
265
|
+
method: "GET",
|
|
266
|
+
headers: {
|
|
267
|
+
Authorization: `Bearer ${this.config.apiKey}`,
|
|
268
|
+
"Content-Type": "application/json",
|
|
269
|
+
},
|
|
270
|
+
signal: controller.signal,
|
|
271
|
+
});
|
|
272
|
+
if (!response.ok) {
|
|
273
|
+
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
274
|
+
}
|
|
275
|
+
const data = (await response.json());
|
|
276
|
+
const windows = new Map();
|
|
277
|
+
for (const entry of data.data ?? []) {
|
|
278
|
+
const model = entry?.model_name;
|
|
279
|
+
const maxInput = entry?.model_info?.max_input_tokens;
|
|
280
|
+
if (typeof model === "string" &&
|
|
281
|
+
model.length > 0 &&
|
|
282
|
+
typeof maxInput === "number" &&
|
|
283
|
+
Number.isFinite(maxInput) &&
|
|
284
|
+
maxInput > 0) {
|
|
285
|
+
// A model group can appear once per underlying deployment; keep the
|
|
286
|
+
// smallest advertised window so budgets are safe for every replica.
|
|
287
|
+
const existing = windows.get(model);
|
|
288
|
+
windows.set(model, existing === undefined ? maxInput : Math.min(existing, maxInput));
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
return windows;
|
|
292
|
+
}
|
|
293
|
+
catch (error) {
|
|
294
|
+
if (isAbortError(error)) {
|
|
295
|
+
throw new NetworkError("Request timed out after 5 seconds", this.providerName);
|
|
296
|
+
}
|
|
297
|
+
throw error;
|
|
298
|
+
}
|
|
299
|
+
finally {
|
|
300
|
+
clearTimeout(timeoutId);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
211
303
|
async fetchModelsFromAPI() {
|
|
212
304
|
const modelsUrl = `${stripTrailingSlash(this.config.baseURL)}/v1/models`;
|
|
213
305
|
const proxyFetch = createProxyFetch();
|
|
@@ -17,6 +17,12 @@ export declare class LiteLLMProvider extends OpenAIChatCompletionsProvider {
|
|
|
17
17
|
private static modelsCache;
|
|
18
18
|
private static modelsCacheTime;
|
|
19
19
|
private static readonly MODELS_CACHE_DURATION;
|
|
20
|
+
/**
|
|
21
|
+
* Dedupes the fire-and-forget `/model/info` discovery per base URL, so a
|
|
22
|
+
* process constructing many LiteLLM providers fetches each proxy's limits
|
|
23
|
+
* once per cache period.
|
|
24
|
+
*/
|
|
25
|
+
private static modelInfoFetchTime;
|
|
20
26
|
constructor(modelName?: string, sdk?: unknown, _region?: string, credentials?: {
|
|
21
27
|
apiKey?: string;
|
|
22
28
|
baseURL?: string;
|
|
@@ -44,6 +50,19 @@ export declare class LiteLLMProvider extends OpenAIChatCompletionsProvider {
|
|
|
44
50
|
* minimal safe default if the API fetch fails.
|
|
45
51
|
*/
|
|
46
52
|
getAvailableModels(): Promise<string[]>;
|
|
53
|
+
/**
|
|
54
|
+
* Discover real per-model context windows from the LiteLLM proxy's
|
|
55
|
+
* `GET /model/info` (`data[].model_info.max_input_tokens`) and register
|
|
56
|
+
* them with the context-window resolver. Fire-and-forget with the same
|
|
57
|
+
* cache period as the models list; any failure (endpoint absent, auth,
|
|
58
|
+
* timeout) is logged at debug and leaves the static `_default` in force.
|
|
59
|
+
*/
|
|
60
|
+
private discoverModelContextWindows;
|
|
61
|
+
/**
|
|
62
|
+
* Fetch `GET /model/info` and map model group name → max_input_tokens.
|
|
63
|
+
* Same fetch/auth/abort scaffolding as {@link fetchModelsFromAPI}.
|
|
64
|
+
*/
|
|
65
|
+
private fetchModelInfoFromAPI;
|
|
47
66
|
private fetchModelsFromAPI;
|
|
48
67
|
/**
|
|
49
68
|
* Generate an embedding for a single text input via native /v1/embeddings.
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
|
|
2
|
+
import { registerRuntimeContextWindow } from "../constants/contextWindows.js";
|
|
2
3
|
import { createProxyFetch } from "../proxy/proxyFetch.js";
|
|
3
4
|
import { AuthenticationError, InvalidModelError, ModelAccessDeniedError, NetworkError, ProviderError, RateLimitError, isModelAccessDeniedMessage, parseAllowedModels, } from "../types/index.js";
|
|
4
5
|
import { isAbortError } from "../utils/errorHandling.js";
|
|
@@ -45,12 +46,24 @@ export class LiteLLMProvider extends OpenAIChatCompletionsProvider {
|
|
|
45
46
|
static modelsCache = [];
|
|
46
47
|
static modelsCacheTime = 0;
|
|
47
48
|
static MODELS_CACHE_DURATION = 10 * 60 * 1000; // 10 minutes
|
|
49
|
+
/**
|
|
50
|
+
* Dedupes the fire-and-forget `/model/info` discovery per base URL, so a
|
|
51
|
+
* process constructing many LiteLLM providers fetches each proxy's limits
|
|
52
|
+
* once per cache period.
|
|
53
|
+
*/
|
|
54
|
+
static modelInfoFetchTime = new Map();
|
|
48
55
|
constructor(modelName, sdk, _region, credentials) {
|
|
49
56
|
const envConfig = getLiteLLMConfig();
|
|
50
57
|
super("litellm", modelName, sdk, {
|
|
51
58
|
baseURL: credentials?.baseURL ?? envConfig.baseURL,
|
|
52
59
|
apiKey: credentials?.apiKey ?? envConfig.apiKey,
|
|
53
60
|
});
|
|
61
|
+
// Fire-and-forget: discover real per-model context windows from the
|
|
62
|
+
// proxy's /model/info. The static table only has a one-size litellm
|
|
63
|
+
// `_default` (128K), while proxied models range from 8K to 2M — budget
|
|
64
|
+
// checks and compaction need the real window. Failures degrade cleanly
|
|
65
|
+
// to the static default.
|
|
66
|
+
this.discoverModelContextWindows();
|
|
54
67
|
logger.debug("LiteLLM Provider initialized", {
|
|
55
68
|
modelName: this.modelName,
|
|
56
69
|
provider: this.providerName,
|
|
@@ -208,6 +221,85 @@ export class LiteLLMProvider extends OpenAIChatCompletionsProvider {
|
|
|
208
221
|
}
|
|
209
222
|
return this.getFallbackModels();
|
|
210
223
|
}
|
|
224
|
+
/**
|
|
225
|
+
* Discover real per-model context windows from the LiteLLM proxy's
|
|
226
|
+
* `GET /model/info` (`data[].model_info.max_input_tokens`) and register
|
|
227
|
+
* them with the context-window resolver. Fire-and-forget with the same
|
|
228
|
+
* cache period as the models list; any failure (endpoint absent, auth,
|
|
229
|
+
* timeout) is logged at debug and leaves the static `_default` in force.
|
|
230
|
+
*/
|
|
231
|
+
discoverModelContextWindows() {
|
|
232
|
+
const baseURL = stripTrailingSlash(this.config.baseURL);
|
|
233
|
+
const now = Date.now();
|
|
234
|
+
const lastFetch = LiteLLMProvider.modelInfoFetchTime.get(baseURL) ?? 0;
|
|
235
|
+
if (now - lastFetch < LiteLLMProvider.MODELS_CACHE_DURATION) {
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
LiteLLMProvider.modelInfoFetchTime.set(baseURL, now);
|
|
239
|
+
void this.fetchModelInfoFromAPI()
|
|
240
|
+
.then((windows) => {
|
|
241
|
+
for (const [model, contextWindow] of windows) {
|
|
242
|
+
registerRuntimeContextWindow("litellm", model, contextWindow);
|
|
243
|
+
}
|
|
244
|
+
if (windows.size > 0) {
|
|
245
|
+
logger.debug("[LiteLLMProvider] Registered runtime context windows from /model/info", { baseURL: redactUrlCredentials(baseURL), models: windows.size });
|
|
246
|
+
}
|
|
247
|
+
})
|
|
248
|
+
.catch((error) => {
|
|
249
|
+
// Allow a retry before the cache period when discovery failed.
|
|
250
|
+
LiteLLMProvider.modelInfoFetchTime.delete(baseURL);
|
|
251
|
+
logger.debug("[LiteLLMProvider] /model/info discovery failed; static context-window defaults remain in force", { error: error instanceof Error ? error.message : String(error) });
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Fetch `GET /model/info` and map model group name → max_input_tokens.
|
|
256
|
+
* Same fetch/auth/abort scaffolding as {@link fetchModelsFromAPI}.
|
|
257
|
+
*/
|
|
258
|
+
async fetchModelInfoFromAPI() {
|
|
259
|
+
const infoUrl = `${stripTrailingSlash(this.config.baseURL)}/model/info`;
|
|
260
|
+
const proxyFetch = createProxyFetch();
|
|
261
|
+
const controller = new AbortController();
|
|
262
|
+
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
|
263
|
+
try {
|
|
264
|
+
const response = await proxyFetch(infoUrl, {
|
|
265
|
+
method: "GET",
|
|
266
|
+
headers: {
|
|
267
|
+
Authorization: `Bearer ${this.config.apiKey}`,
|
|
268
|
+
"Content-Type": "application/json",
|
|
269
|
+
},
|
|
270
|
+
signal: controller.signal,
|
|
271
|
+
});
|
|
272
|
+
if (!response.ok) {
|
|
273
|
+
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
274
|
+
}
|
|
275
|
+
const data = (await response.json());
|
|
276
|
+
const windows = new Map();
|
|
277
|
+
for (const entry of data.data ?? []) {
|
|
278
|
+
const model = entry?.model_name;
|
|
279
|
+
const maxInput = entry?.model_info?.max_input_tokens;
|
|
280
|
+
if (typeof model === "string" &&
|
|
281
|
+
model.length > 0 &&
|
|
282
|
+
typeof maxInput === "number" &&
|
|
283
|
+
Number.isFinite(maxInput) &&
|
|
284
|
+
maxInput > 0) {
|
|
285
|
+
// A model group can appear once per underlying deployment; keep the
|
|
286
|
+
// smallest advertised window so budgets are safe for every replica.
|
|
287
|
+
const existing = windows.get(model);
|
|
288
|
+
windows.set(model, existing === undefined ? maxInput : Math.min(existing, maxInput));
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
return windows;
|
|
292
|
+
}
|
|
293
|
+
catch (error) {
|
|
294
|
+
if (isAbortError(error)) {
|
|
295
|
+
throw new NetworkError("Request timed out after 5 seconds", this.providerName);
|
|
296
|
+
}
|
|
297
|
+
throw error;
|
|
298
|
+
}
|
|
299
|
+
finally {
|
|
300
|
+
clearTimeout(timeoutId);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
211
303
|
async fetchModelsFromAPI() {
|
|
212
304
|
const modelsUrl = `${stripTrailingSlash(this.config.baseURL)}/v1/models`;
|
|
213
305
|
const proxyFetch = createProxyFetch();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@juspay/neurolink",
|
|
3
|
-
"version": "10.2.
|
|
3
|
+
"version": "10.2.2",
|
|
4
4
|
"packageManager": "pnpm@10.15.1",
|
|
5
5
|
"description": "TypeScript AI SDK with 24+ LLM providers behind one consistent API. MCP-native (connect any MCP server), voice TTS/STT/realtime, RAG, agents, memory, context compaction. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · DeepSeek · NVIDIA NIM and more.",
|
|
6
6
|
"author": {
|
|
@@ -147,7 +147,7 @@
|
|
|
147
147
|
"test:system-messages:vitest": "pnpm exec vitest run test/systemMessages.test.ts",
|
|
148
148
|
"test:tool-routing-semantic:vitest": "pnpm exec vitest run test/toolRoutingSemantic.test.ts",
|
|
149
149
|
"test:tool-routing-semantic": "pnpm run test:tool-routing-semantic:vitest && npx tsx test/continuous-test-suite-tool-routing-semantic.ts",
|
|
150
|
-
"test:unit": "pnpm run test:envguard && pnpm run test:bugfixes && pnpm run test:file-detector-extension && pnpm run test:mcp:infra && pnpm run test:mcp:bash && pnpm run test:mcp:limits && pnpm run test:mcp:spans && pnpm run test:autoresearch:redis && pnpm run test:unit:vitest && pnpm run test:tool-routing-cli:vitest && pnpm run test:tool-dedup:vitest && pnpm run test:model-pool:vitest && pnpm run test:system-messages:vitest && pnpm run test:tool-routing-semantic:vitest && pnpm run test:anthropic-tools-policy && pnpm run test:sagemaker-tools && pnpm run test:anthropic-multimodal && pnpm run test:excel-interop",
|
|
150
|
+
"test:unit": "pnpm run test:envguard && pnpm run test:bugfixes && pnpm run test:file-detector-extension && pnpm run test:mcp:infra && pnpm run test:mcp:bash && pnpm run test:mcp:limits && pnpm run test:mcp:spans && pnpm run test:autoresearch:redis && pnpm run test:unit:vitest && pnpm run test:tool-routing-cli:vitest && pnpm run test:tool-dedup:vitest && pnpm run test:model-pool:vitest && pnpm run test:litellm-context:vitest && pnpm run test:system-messages:vitest && pnpm run test:tool-routing-semantic:vitest && pnpm run test:anthropic-tools-policy && pnpm run test:sagemaker-tools && pnpm run test:anthropic-multimodal && pnpm run test:excel-interop",
|
|
151
151
|
"// CI tier — live providers, runs only when API keys are present (test:credentials and test:dynamic make real provider calls when keys are set, so they live here, not in test:unit)": "",
|
|
152
152
|
"test:live": "pnpm run test:providers && pnpm run test:mcp:http && pnpm run test:mcp:sdk && pnpm run test:mcp:cli && pnpm run test:observability && pnpm run test:context && pnpm run test:memory && pnpm run test:tool-reliability && pnpm run test:evaluation && pnpm run test:autoresearch && pnpm run test:credentials && pnpm run test:dynamic",
|
|
153
153
|
"// CI tier — product output (image/video/TTS/PPT) — costs $$ per run": "",
|
|
@@ -208,7 +208,8 @@
|
|
|
208
208
|
"quality:report": "pnpm run quality:metrics && echo 'Quality metrics saved to quality-metrics.json'",
|
|
209
209
|
"pre-commit": "lint-staged",
|
|
210
210
|
"pre-push": "pnpm run validate:commit && pnpm run validate:env && pnpm run validate && pnpm run test:ci",
|
|
211
|
-
"check:all": "pnpm run lint && pnpm run format --check && pnpm run validate && pnpm run validate:commit"
|
|
211
|
+
"check:all": "pnpm run lint && pnpm run format --check && pnpm run validate && pnpm run validate:commit",
|
|
212
|
+
"test:litellm-context:vitest": "pnpm exec vitest run test/litellmContextWindows.test.ts"
|
|
212
213
|
},
|
|
213
214
|
"files": [
|
|
214
215
|
"dist",
|