@juspay/neurolink 10.2.2 → 10.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/dist/browser/neurolink.min.js +425 -422
  3. package/dist/cli/loop/optionsSchema.js +4 -0
  4. package/dist/constants/contextWindows.d.ts +24 -0
  5. package/dist/constants/contextWindows.js +50 -0
  6. package/dist/context/errorDetection.d.ts +6 -0
  7. package/dist/context/errorDetection.js +18 -0
  8. package/dist/context/stepBudgetGuard.d.ts +61 -0
  9. package/dist/context/stepBudgetGuard.js +328 -0
  10. package/dist/core/baseProvider.d.ts +12 -0
  11. package/dist/core/baseProvider.js +18 -0
  12. package/dist/core/modules/GenerationHandler.js +257 -43
  13. package/dist/core/modules/ToolsManager.d.ts +17 -0
  14. package/dist/core/modules/ToolsManager.js +99 -1
  15. package/dist/lib/constants/contextWindows.d.ts +24 -0
  16. package/dist/lib/constants/contextWindows.js +50 -0
  17. package/dist/lib/context/errorDetection.d.ts +6 -0
  18. package/dist/lib/context/errorDetection.js +18 -0
  19. package/dist/lib/context/stepBudgetGuard.d.ts +61 -0
  20. package/dist/lib/context/stepBudgetGuard.js +329 -0
  21. package/dist/lib/core/baseProvider.d.ts +12 -0
  22. package/dist/lib/core/baseProvider.js +18 -0
  23. package/dist/lib/core/modules/GenerationHandler.js +257 -43
  24. package/dist/lib/core/modules/ToolsManager.d.ts +17 -0
  25. package/dist/lib/core/modules/ToolsManager.js +99 -1
  26. package/dist/lib/mcp/toolDiscoveryService.js +59 -20
  27. package/dist/lib/neurolink.js +30 -9
  28. package/dist/lib/providers/litellm.d.ts +27 -19
  29. package/dist/lib/providers/litellm.js +171 -92
  30. package/dist/lib/providers/openaiChatCompletionsBase.d.ts +37 -1
  31. package/dist/lib/providers/openaiChatCompletionsBase.js +201 -33
  32. package/dist/lib/providers/openaiChatCompletionsClient.d.ts +23 -5
  33. package/dist/lib/providers/openaiChatCompletionsClient.js +94 -14
  34. package/dist/lib/proxy/proxyFetch.d.ts +17 -0
  35. package/dist/lib/proxy/proxyFetch.js +42 -4
  36. package/dist/lib/types/context.d.ts +22 -0
  37. package/dist/lib/types/generate.d.ts +18 -2
  38. package/dist/lib/types/openaiCompatible.d.ts +2 -0
  39. package/dist/lib/types/providers.d.ts +9 -0
  40. package/dist/lib/utils/errorHandling.js +8 -2
  41. package/dist/lib/utils/schemaConversion.d.ts +16 -0
  42. package/dist/lib/utils/schemaConversion.js +165 -8
  43. package/dist/lib/utils/timeout.d.ts +22 -0
  44. package/dist/lib/utils/timeout.js +72 -12
  45. package/dist/lib/utils/tokenLimits.js +22 -0
  46. package/dist/lib/utils/toolCallRepair.d.ts +8 -0
  47. package/dist/lib/utils/toolCallRepair.js +4 -1
  48. package/dist/mcp/toolDiscoveryService.js +59 -20
  49. package/dist/neurolink.js +30 -9
  50. package/dist/providers/litellm.d.ts +27 -19
  51. package/dist/providers/litellm.js +171 -92
  52. package/dist/providers/openaiChatCompletionsBase.d.ts +37 -1
  53. package/dist/providers/openaiChatCompletionsBase.js +201 -33
  54. package/dist/providers/openaiChatCompletionsClient.d.ts +23 -5
  55. package/dist/providers/openaiChatCompletionsClient.js +94 -14
  56. package/dist/proxy/proxyFetch.d.ts +17 -0
  57. package/dist/proxy/proxyFetch.js +42 -4
  58. package/dist/types/context.d.ts +22 -0
  59. package/dist/types/generate.d.ts +18 -2
  60. package/dist/types/openaiCompatible.d.ts +2 -0
  61. package/dist/types/providers.d.ts +9 -0
  62. package/dist/utils/errorHandling.js +8 -2
  63. package/dist/utils/schemaConversion.d.ts +16 -0
  64. package/dist/utils/schemaConversion.js +165 -8
  65. package/dist/utils/timeout.d.ts +22 -0
  66. package/dist/utils/timeout.js +72 -12
  67. package/dist/utils/tokenLimits.js +22 -0
  68. package/dist/utils/toolCallRepair.d.ts +8 -0
  69. package/dist/utils/toolCallRepair.js +4 -1
  70. package/package.json +6 -3
@@ -5190,12 +5190,22 @@ Current user's request: ${currentInput}`;
5190
5190
  if (!generationContext) {
5191
5191
  return null;
5192
5192
  }
5193
+ // Provider construction runs BEFORE the budget check so runtime
5194
+ // model-limit discovery (ensureModelLimits) can register real context
5195
+ // windows first — ensureMCPGenerationBudget otherwise computes against
5196
+ // the static default window (see directProviderGeneration for the
5197
+ // lock-out this ordering prevents).
5198
+ const provider = await AIProviderFactory.createProvider(generationContext.providerName, options.model, !options.disableTools, this, options.region, this.resolveCredentials(options.credentials));
5199
+ provider.setTraceContext(this._metricsTraceContext);
5200
+ // Never rejects — discovery failure degrades to static defaults.
5201
+ await provider.ensureModelLimits?.();
5193
5202
  const conversationMessages = await this.ensureMCPGenerationBudget(options, requestId, generationContext.providerName, generationContext.enhancedSystemPrompt, generationContext.availableTools, generationContext.conversationMessages);
5194
5203
  return this.generateWithMCPProvider({
5195
5204
  options,
5196
5205
  requestId,
5197
5206
  functionTag,
5198
5207
  tryMCPStartTime,
5208
+ provider,
5199
5209
  providerName: generationContext.providerName,
5200
5210
  availableTools: generationContext.availableTools,
5201
5211
  enhancedSystemPrompt: generationContext.enhancedSystemPrompt,
@@ -5479,9 +5489,7 @@ Current user's request: ${currentInput}`;
5479
5489
  return compactedMessages;
5480
5490
  }
5481
5491
  async generateWithMCPProvider(context) {
5482
- const { options, requestId, functionTag, tryMCPStartTime, providerName, availableTools, enhancedSystemPrompt, conversationMessages, } = context;
5483
- const provider = await AIProviderFactory.createProvider(providerName, options.model, !options.disableTools, this, options.region, this.resolveCredentials(options.credentials));
5484
- provider.setTraceContext(this._metricsTraceContext);
5492
+ const { options, requestId, functionTag, tryMCPStartTime, provider, providerName, availableTools, enhancedSystemPrompt, conversationMessages, } = context;
5485
5493
  this.emitter.emit("connected");
5486
5494
  this.emitter.emit("message", `${providerName} provider initialized successfully`);
5487
5495
  provider.setupToolExecutor({
@@ -5760,6 +5768,21 @@ Current user's request: ${currentInput}`;
5760
5768
  ?.length
5761
5769
  ? optionsWithMessages.conversationMessages
5762
5770
  : await getConversationMessages(this.conversationMemory, options);
5771
+ // Provider construction runs BEFORE the budget check so runtime
5772
+ // model-limit discovery (ensureModelLimits) can register real
5773
+ // context windows first. The previous order created a lock-out:
5774
+ // checkContextBudget ran against the static default window, its
5775
+ // pre-dispatch hard cap threw before the provider (whose
5776
+ // constructor owns the discovery) ever existed — so a blocked call
5777
+ // prevented the very discovery that would have unblocked it.
5778
+ const provider = await AIProviderFactory.createProvider(providerName, options.model, !options.disableTools, // Pass disableTools as inverse of enableMCP
5779
+ this, // Pass SDK instance
5780
+ options.region, // Pass region parameter
5781
+ this.resolveCredentials(options.credentials));
5782
+ // Propagate trace context for parent-child span hierarchy
5783
+ provider.setTraceContext(this._metricsTraceContext);
5784
+ // Never rejects — discovery failure degrades to static defaults.
5785
+ await provider.ensureModelLimits?.();
5763
5786
  // Pre-generation budget check
5764
5787
  const budgetCheck = checkContextBudget({
5765
5788
  provider: providerName,
@@ -5915,12 +5938,6 @@ Current user's request: ${currentInput}`;
5915
5938
  }
5916
5939
  }
5917
5940
  }
5918
- const provider = await AIProviderFactory.createProvider(providerName, options.model, !options.disableTools, // Pass disableTools as inverse of enableMCP
5919
- this, // Pass SDK instance
5920
- options.region, // Pass region parameter
5921
- this.resolveCredentials(options.credentials));
5922
- // Propagate trace context for parent-child span hierarchy
5923
- provider.setTraceContext(this._metricsTraceContext);
5924
5941
  // ADD: Emit connection events for successful provider creation (Bedrock-compatible)
5925
5942
  this.emitter.emit("connected");
5926
5943
  this.emitter.emit("message", `${providerName} provider initialized successfully`);
@@ -7892,6 +7909,10 @@ Current user's request: ${currentInput}`;
7892
7909
  this, // Pass SDK instance
7893
7910
  options.region, // Pass region parameter
7894
7911
  this.resolveCredentials(options.credentials));
7912
+ // Runtime model-limit discovery must land BEFORE the stream budget
7913
+ // check below — otherwise it computes against the static default
7914
+ // window. Never rejects; failure degrades to static defaults.
7915
+ await provider.ensureModelLimits?.();
7895
7916
  // Propagate trace context for parent-child span hierarchy
7896
7917
  provider.setTraceContext(this._metricsTraceContext);
7897
7918
  // Enable tool execution for the provider using BaseProvider method
@@ -1,6 +1,26 @@
1
1
  import type { AIProviderName } from "../constants/enums.js";
2
2
  import type { OpenAICompatBuildBodyArgs, OpenAICompatStreamLifecycleListeners } from "../types/index.js";
3
3
  import { OpenAIChatCompletionsProvider } from "./openaiChatCompletionsBase.js";
4
+ /** Test hook: reset the discovery caches (state is module-global). */
5
+ export declare function clearLiteLLMModelLimitsCache(): void;
6
+ /**
7
+ * Discover real per-model limits from the LiteLLM proxy's `GET /model/info`
8
+ * and register them with the runtime resolvers: `max_input_tokens` → context
9
+ * window, `max_output_tokens` → output ceiling. The static table only has a
10
+ * one-size litellm `_default` (128K) while proxied models range from 8K to
11
+ * 2M — budget checks, compaction, and max_tokens clamping need the real
12
+ * numbers.
13
+ *
14
+ * Awaitable and deduped per base URL: generation pipelines await this (via
15
+ * `LiteLLMProvider.ensureModelLimits`) BEFORE any budget math, so even the
16
+ * FIRST call in a fresh process budgets against real windows — the previous
17
+ * constructor-scoped fire-and-forget lost that race every time, and a
18
+ * budget-blocked call then prevented the discovery that would have unblocked
19
+ * it. Never rejects: any failure (endpoint absent, auth, timeout) is logged
20
+ * at debug, leaves the static defaults in force, and allows a retry on the
21
+ * next call.
22
+ */
23
+ export declare function ensureLiteLLMModelLimits(baseURL: string, apiKey: string): Promise<void>;
4
24
  /**
5
25
  * LiteLLM Provider — direct HTTP, no AI SDK. Talks to a LiteLLM proxy
6
26
  * server (or any deployment that speaks OpenAI chat-completions + the
@@ -17,16 +37,17 @@ export declare class LiteLLMProvider extends OpenAIChatCompletionsProvider {
17
37
  private static modelsCache;
18
38
  private static modelsCacheTime;
19
39
  private static readonly MODELS_CACHE_DURATION;
20
- /**
21
- * Dedupes the fire-and-forget `/model/info` discovery per base URL, so a
22
- * process constructing many LiteLLM providers fetches each proxy's limits
23
- * once per cache period.
24
- */
25
- private static modelInfoFetchTime;
26
40
  constructor(modelName?: string, sdk?: unknown, _region?: string, credentials?: {
27
41
  apiKey?: string;
28
42
  baseURL?: string;
29
43
  });
44
+ /**
45
+ * Awaitable model-limit discovery — generation pipelines call this before
46
+ * budget checks so window/output-ceiling math uses the proxy's real
47
+ * numbers (see BaseProvider.ensureModelLimits). Deduped and cached per
48
+ * base URL; never rejects (failure degrades to static defaults).
49
+ */
50
+ ensureModelLimits(): Promise<void>;
30
51
  protected getProviderName(): AIProviderName;
31
52
  protected getDefaultModel(): string;
32
53
  protected getFallbackModelName(): string;
@@ -50,19 +71,6 @@ export declare class LiteLLMProvider extends OpenAIChatCompletionsProvider {
50
71
  * minimal safe default if the API fetch fails.
51
72
  */
52
73
  getAvailableModels(): Promise<string[]>;
53
- /**
54
- * Discover real per-model context windows from the LiteLLM proxy's
55
- * `GET /model/info` (`data[].model_info.max_input_tokens`) and register
56
- * them with the context-window resolver. Fire-and-forget with the same
57
- * cache period as the models list; any failure (endpoint absent, auth,
58
- * timeout) is logged at debug and leaves the static `_default` in force.
59
- */
60
- private discoverModelContextWindows;
61
- /**
62
- * Fetch `GET /model/info` and map model group name → max_input_tokens.
63
- * Same fetch/auth/abort scaffolding as {@link fetchModelsFromAPI}.
64
- */
65
- private fetchModelInfoFromAPI;
66
74
  private fetchModelsFromAPI;
67
75
  /**
68
76
  * Generate an embedding for a single text input via native /v1/embeddings.
@@ -1,5 +1,5 @@
1
1
  import { SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
2
- import { registerRuntimeContextWindow } from "../constants/contextWindows.js";
2
+ import { registerRuntimeContextWindow, registerRuntimeOutputCeiling, } from "../constants/contextWindows.js";
3
3
  import { createProxyFetch } from "../proxy/proxyFetch.js";
4
4
  import { AuthenticationError, InvalidModelError, ModelAccessDeniedError, NetworkError, ProviderError, RateLimitError, isModelAccessDeniedMessage, parseAllowedModels, } from "../types/index.js";
5
5
  import { isAbortError } from "../utils/errorHandling.js";
@@ -21,6 +21,163 @@ const getLiteLLMConfig = () => ({
21
21
  * LiteLLM uses a 'provider/model' format. Override via LITELLM_MODEL env var.
22
22
  */
23
23
  const getDefaultLiteLLMModel = () => getProviderModel("LITELLM_MODEL", FALLBACK_LITELLM_MODEL);
24
+ /** Cache period for `/model/info` limit discovery (same as the models list). */
25
+ const MODEL_INFO_CACHE_DURATION = 10 * 60 * 1000;
26
+ /** In-flight `/model/info` discovery per base URL — dedupes concurrent callers. */
27
+ const modelInfoInFlight = new Map();
28
+ /** Last successful `/model/info` discovery per base URL. */
29
+ const modelInfoLastSuccess = new Map();
30
+ /**
31
+ * Discovered limits per base URL. The runtime registries key by
32
+ * (provider, model) only — budget call sites carry no deployment identity —
33
+ * so when a process talks to MULTIPLE proxies that share a model-group name,
34
+ * registrations use the MIN across every discovered deployment: budgets stay
35
+ * safe for all of them instead of last-writer-wins silently overstating one.
36
+ * Single-proxy processes (the normal case) are unaffected.
37
+ */
38
+ const modelInfoLimitsByURL = new Map();
39
+ /** Test hook: reset the discovery caches (state is module-global). */
40
+ export function clearLiteLLMModelLimitsCache() {
41
+ modelInfoInFlight.clear();
42
+ modelInfoLastSuccess.clear();
43
+ modelInfoLimitsByURL.clear();
44
+ }
45
+ /**
46
+ * Fetch `GET /model/info` and map model group name → advertised limits
47
+ * (`model_info.max_input_tokens` / `model_info.max_output_tokens`).
48
+ * A model group can appear once per underlying deployment; the SMALLEST
49
+ * advertised value wins per field so budgets are safe for every replica.
50
+ */
51
+ async function fetchLiteLLMModelLimits(baseURL, apiKey) {
52
+ const infoUrl = `${baseURL}/model/info`;
53
+ const proxyFetch = createProxyFetch();
54
+ const controller = new AbortController();
55
+ const timeoutId = setTimeout(() => controller.abort(), 5000);
56
+ try {
57
+ const response = await proxyFetch(infoUrl, {
58
+ method: "GET",
59
+ headers: {
60
+ Authorization: `Bearer ${apiKey}`,
61
+ "Content-Type": "application/json",
62
+ },
63
+ signal: controller.signal,
64
+ });
65
+ if (!response.ok) {
66
+ throw new Error(`HTTP ${response.status}: ${response.statusText}`);
67
+ }
68
+ const data = (await response.json());
69
+ const isUsable = (v) => typeof v === "number" && Number.isFinite(v) && v > 0;
70
+ const limits = new Map();
71
+ for (const entry of data.data ?? []) {
72
+ const model = entry?.model_name;
73
+ if (typeof model !== "string" || model.length === 0) {
74
+ continue;
75
+ }
76
+ const maxInput = entry?.model_info?.max_input_tokens;
77
+ const maxOutput = entry?.model_info?.max_output_tokens;
78
+ if (!isUsable(maxInput) && !isUsable(maxOutput)) {
79
+ continue;
80
+ }
81
+ const existing = limits.get(model) ?? {};
82
+ if (isUsable(maxInput)) {
83
+ existing.maxInputTokens =
84
+ existing.maxInputTokens === undefined
85
+ ? maxInput
86
+ : Math.min(existing.maxInputTokens, maxInput);
87
+ }
88
+ if (isUsable(maxOutput)) {
89
+ existing.maxOutputTokens =
90
+ existing.maxOutputTokens === undefined
91
+ ? maxOutput
92
+ : Math.min(existing.maxOutputTokens, maxOutput);
93
+ }
94
+ limits.set(model, existing);
95
+ }
96
+ return limits;
97
+ }
98
+ catch (error) {
99
+ if (isAbortError(error)) {
100
+ throw new NetworkError("Request timed out after 5 seconds", "litellm");
101
+ }
102
+ throw error;
103
+ }
104
+ finally {
105
+ clearTimeout(timeoutId);
106
+ }
107
+ }
108
+ /**
109
+ * Discover real per-model limits from the LiteLLM proxy's `GET /model/info`
110
+ * and register them with the runtime resolvers: `max_input_tokens` → context
111
+ * window, `max_output_tokens` → output ceiling. The static table only has a
112
+ * one-size litellm `_default` (128K) while proxied models range from 8K to
113
+ * 2M — budget checks, compaction, and max_tokens clamping need the real
114
+ * numbers.
115
+ *
116
+ * Awaitable and deduped per base URL: generation pipelines await this (via
117
+ * `LiteLLMProvider.ensureModelLimits`) BEFORE any budget math, so even the
118
+ * FIRST call in a fresh process budgets against real windows — the previous
119
+ * constructor-scoped fire-and-forget lost that race every time, and a
120
+ * budget-blocked call then prevented the discovery that would have unblocked
121
+ * it. Never rejects: any failure (endpoint absent, auth, timeout) is logged
122
+ * at debug, leaves the static defaults in force, and allows a retry on the
123
+ * next call.
124
+ */
125
+ export function ensureLiteLLMModelLimits(baseURL, apiKey) {
126
+ const key = stripTrailingSlash(baseURL);
127
+ const lastSuccess = modelInfoLastSuccess.get(key) ?? 0;
128
+ if (Date.now() - lastSuccess < MODEL_INFO_CACHE_DURATION) {
129
+ return Promise.resolve();
130
+ }
131
+ const inFlight = modelInfoInFlight.get(key);
132
+ if (inFlight) {
133
+ return inFlight;
134
+ }
135
+ const discovery = fetchLiteLLMModelLimits(key, apiKey)
136
+ .then((limits) => {
137
+ modelInfoLimitsByURL.set(key, limits);
138
+ // Min-merge across every base URL discovered so far (see
139
+ // modelInfoLimitsByURL) before registering.
140
+ const merged = new Map();
141
+ for (const urlLimits of modelInfoLimitsByURL.values()) {
142
+ for (const [model, modelLimits] of urlLimits) {
143
+ const existing = merged.get(model) ?? {};
144
+ if (modelLimits.maxInputTokens !== undefined) {
145
+ existing.maxInputTokens =
146
+ existing.maxInputTokens === undefined
147
+ ? modelLimits.maxInputTokens
148
+ : Math.min(existing.maxInputTokens, modelLimits.maxInputTokens);
149
+ }
150
+ if (modelLimits.maxOutputTokens !== undefined) {
151
+ existing.maxOutputTokens =
152
+ existing.maxOutputTokens === undefined
153
+ ? modelLimits.maxOutputTokens
154
+ : Math.min(existing.maxOutputTokens, modelLimits.maxOutputTokens);
155
+ }
156
+ merged.set(model, existing);
157
+ }
158
+ }
159
+ for (const [model, modelLimits] of merged) {
160
+ if (modelLimits.maxInputTokens !== undefined) {
161
+ registerRuntimeContextWindow("litellm", model, modelLimits.maxInputTokens);
162
+ }
163
+ if (modelLimits.maxOutputTokens !== undefined) {
164
+ registerRuntimeOutputCeiling("litellm", model, modelLimits.maxOutputTokens);
165
+ }
166
+ }
167
+ modelInfoLastSuccess.set(key, Date.now());
168
+ if (limits.size > 0) {
169
+ logger.debug("[LiteLLM] Registered runtime model limits from /model/info", { baseURL: redactUrlCredentials(key), models: limits.size });
170
+ }
171
+ })
172
+ .catch((error) => {
173
+ logger.debug("[LiteLLM] /model/info discovery failed; static context-window defaults remain in force", { error: error instanceof Error ? error.message : String(error) });
174
+ })
175
+ .finally(() => {
176
+ modelInfoInFlight.delete(key);
177
+ });
178
+ modelInfoInFlight.set(key, discovery);
179
+ return discovery;
180
+ }
24
181
  // LiteLLM model ids come in `provider/model` form (e.g. "google/gemini-2.5-flash").
25
182
  // Strip the provider prefix and delegate to the canonical anchored-regex
26
183
  // check in src/lib/utils/modelDetection.ts so the truth lives in one place.
@@ -46,30 +203,31 @@ export class LiteLLMProvider extends OpenAIChatCompletionsProvider {
46
203
  static modelsCache = [];
47
204
  static modelsCacheTime = 0;
48
205
  static MODELS_CACHE_DURATION = 10 * 60 * 1000; // 10 minutes
49
- /**
50
- * Dedupes the fire-and-forget `/model/info` discovery per base URL, so a
51
- * process constructing many LiteLLM providers fetches each proxy's limits
52
- * once per cache period.
53
- */
54
- static modelInfoFetchTime = new Map();
55
206
  constructor(modelName, sdk, _region, credentials) {
56
207
  const envConfig = getLiteLLMConfig();
57
208
  super("litellm", modelName, sdk, {
58
209
  baseURL: credentials?.baseURL ?? envConfig.baseURL,
59
210
  apiKey: credentials?.apiKey ?? envConfig.apiKey,
60
211
  });
61
- // Fire-and-forget: discover real per-model context windows from the
62
- // proxy's /model/info. The static table only has a one-size litellm
63
- // `_default` (128K), while proxied models range from 8K to 2M — budget
64
- // checks and compaction need the real window. Failures degrade cleanly
65
- // to the static default.
66
- this.discoverModelContextWindows();
212
+ // Warm the model-limit discovery early (deduped per base URL). The
213
+ // generation pipelines still AWAIT ensureModelLimits() before budget
214
+ // math this fire-and-forget only shaves latency off that first await.
215
+ void ensureLiteLLMModelLimits(this.config.baseURL, this.config.apiKey);
67
216
  logger.debug("LiteLLM Provider initialized", {
68
217
  modelName: this.modelName,
69
218
  provider: this.providerName,
70
219
  baseURL: redactUrlCredentials(this.config.baseURL),
71
220
  });
72
221
  }
222
+ /**
223
+ * Awaitable model-limit discovery — generation pipelines call this before
224
+ * budget checks so window/output-ceiling math uses the proxy's real
225
+ * numbers (see BaseProvider.ensureModelLimits). Deduped and cached per
226
+ * base URL; never rejects (failure degrades to static defaults).
227
+ */
228
+ async ensureModelLimits() {
229
+ await ensureLiteLLMModelLimits(this.config.baseURL, this.config.apiKey);
230
+ }
73
231
  getProviderName() {
74
232
  return "litellm";
75
233
  }
@@ -221,85 +379,6 @@ export class LiteLLMProvider extends OpenAIChatCompletionsProvider {
221
379
  }
222
380
  return this.getFallbackModels();
223
381
  }
224
- /**
225
- * Discover real per-model context windows from the LiteLLM proxy's
226
- * `GET /model/info` (`data[].model_info.max_input_tokens`) and register
227
- * them with the context-window resolver. Fire-and-forget with the same
228
- * cache period as the models list; any failure (endpoint absent, auth,
229
- * timeout) is logged at debug and leaves the static `_default` in force.
230
- */
231
- discoverModelContextWindows() {
232
- const baseURL = stripTrailingSlash(this.config.baseURL);
233
- const now = Date.now();
234
- const lastFetch = LiteLLMProvider.modelInfoFetchTime.get(baseURL) ?? 0;
235
- if (now - lastFetch < LiteLLMProvider.MODELS_CACHE_DURATION) {
236
- return;
237
- }
238
- LiteLLMProvider.modelInfoFetchTime.set(baseURL, now);
239
- void this.fetchModelInfoFromAPI()
240
- .then((windows) => {
241
- for (const [model, contextWindow] of windows) {
242
- registerRuntimeContextWindow("litellm", model, contextWindow);
243
- }
244
- if (windows.size > 0) {
245
- logger.debug("[LiteLLMProvider] Registered runtime context windows from /model/info", { baseURL: redactUrlCredentials(baseURL), models: windows.size });
246
- }
247
- })
248
- .catch((error) => {
249
- // Allow a retry before the cache period when discovery failed.
250
- LiteLLMProvider.modelInfoFetchTime.delete(baseURL);
251
- logger.debug("[LiteLLMProvider] /model/info discovery failed; static context-window defaults remain in force", { error: error instanceof Error ? error.message : String(error) });
252
- });
253
- }
254
- /**
255
- * Fetch `GET /model/info` and map model group name → max_input_tokens.
256
- * Same fetch/auth/abort scaffolding as {@link fetchModelsFromAPI}.
257
- */
258
- async fetchModelInfoFromAPI() {
259
- const infoUrl = `${stripTrailingSlash(this.config.baseURL)}/model/info`;
260
- const proxyFetch = createProxyFetch();
261
- const controller = new AbortController();
262
- const timeoutId = setTimeout(() => controller.abort(), 5000);
263
- try {
264
- const response = await proxyFetch(infoUrl, {
265
- method: "GET",
266
- headers: {
267
- Authorization: `Bearer ${this.config.apiKey}`,
268
- "Content-Type": "application/json",
269
- },
270
- signal: controller.signal,
271
- });
272
- if (!response.ok) {
273
- throw new Error(`HTTP ${response.status}: ${response.statusText}`);
274
- }
275
- const data = (await response.json());
276
- const windows = new Map();
277
- for (const entry of data.data ?? []) {
278
- const model = entry?.model_name;
279
- const maxInput = entry?.model_info?.max_input_tokens;
280
- if (typeof model === "string" &&
281
- model.length > 0 &&
282
- typeof maxInput === "number" &&
283
- Number.isFinite(maxInput) &&
284
- maxInput > 0) {
285
- // A model group can appear once per underlying deployment; keep the
286
- // smallest advertised window so budgets are safe for every replica.
287
- const existing = windows.get(model);
288
- windows.set(model, existing === undefined ? maxInput : Math.min(existing, maxInput));
289
- }
290
- }
291
- return windows;
292
- }
293
- catch (error) {
294
- if (isAbortError(error)) {
295
- throw new NetworkError("Request timed out after 5 seconds", this.providerName);
296
- }
297
- throw error;
298
- }
299
- finally {
300
- clearTimeout(timeoutId);
301
- }
302
- }
303
382
  async fetchModelsFromAPI() {
304
383
  const modelsUrl = `${stripTrailingSlash(this.config.baseURL)}/v1/models`;
305
384
  const proxyFetch = createProxyFetch();
@@ -19,7 +19,7 @@
19
19
  */
20
20
  import type { AIProviderName } from "../constants/enums.js";
21
21
  import { BaseProvider } from "../core/baseProvider.js";
22
- import type { LanguageModel, OpenAICompatBuildBodyArgs, OpenAICompatChatRequest, OpenAICompatResponseFormat, OpenAICompatStreamLifecycleListeners, Schema, StreamOptions, StreamResult, ZodUnknownSchema } from "../types/index.js";
22
+ import type { LanguageModel, OpenAICompatBuildBodyArgs, OpenAICompatChatMessage, OpenAICompatChatRequest, OpenAICompatChatTool, OpenAICompatResponseFormat, OpenAICompatStreamLifecycleListeners, Schema, StreamOptions, StreamResult, ZodUnknownSchema } from "../types/index.js";
23
23
  /**
24
24
  * Abstract HTTP+SSE provider for OpenAI chat-completions-shaped endpoints.
25
25
  */
@@ -97,6 +97,42 @@ export declare abstract class OpenAIChatCompletionsProvider extends BaseProvider
97
97
  statusCode?: number;
98
98
  responseBody?: string;
99
99
  }): OpenAICompatChatRequest | undefined;
100
+ /**
101
+ * Fit the outgoing `max_tokens` to what the target deployment can actually
102
+ * accept, using ONLY runtime-discovered limits — static table values are
103
+ * guesses, and hard-enforcing a guess would falsely reject requests the
104
+ * real deployment accepts:
105
+ *
106
+ * effective = min(requested, discovered output ceiling,
107
+ * discovered window − estimated input − margin)
108
+ *
109
+ * Returns the caller's value untouched when nothing was discovered, and
110
+ * `undefined` when the caller sent nothing and no ceiling is known (the
111
+ * wire then omits max_tokens and the backend applies its own default — no
112
+ * invented numbers). Throws ContextBudgetExceededError when the estimated
113
+ * input ALONE exceeds a discovered window: that request cannot succeed,
114
+ * and failing fast with honest numbers beats a guaranteed provider 400
115
+ * (plus any proxy-side fallback cascade) after a full round-trip.
116
+ */
117
+ protected resolveWireMaxTokens(modelId: string, requested: number | undefined, messages: ReadonlyArray<OpenAICompatChatMessage>, tools: OpenAICompatChatTool[] | undefined): number | undefined;
118
+ /**
119
+ * Learn from a provider context-overflow 400 and, when possible, produce a
120
+ * corrected body for the one-shot retry slot. Two dynamic effects, zero
121
+ * static data:
122
+ *
123
+ * - the window stated in the error is registered with the runtime
124
+ * resolver, so every later budget check / compaction / max_tokens fit
125
+ * uses the backend's own number — self-healing even when a discovery
126
+ * endpoint (`/model/info`) is absent or unauthorized;
127
+ * - when the error also states the real input size (vllm/LiteLLM
128
+ * phrasing) and the body carried `max_tokens`, it is re-fit to
129
+ * `window − input − margin` and the request retried once.
130
+ *
131
+ * Returns undefined when the error is not an overflow, or when no smaller
132
+ * `max_tokens` can make the request fit (input alone too large) — the
133
+ * original error then propagates unchanged.
134
+ */
135
+ private correctBodyAfterContextOverflow;
100
136
  /**
101
137
  * Hook called once at the start of every `executeStream` invocation.
102
138
  * Return lifecycle listeners (onUsage / onFinish) to receive deferred