@juspay/neurolink 10.2.1 → 10.3.0

package/dist/providers/litellm.js

@@ -1,4 +1,5 @@
 import { SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
+import { registerRuntimeContextWindow } from "../constants/contextWindows.js";
 import { createProxyFetch } from "../proxy/proxyFetch.js";
 import { AuthenticationError, InvalidModelError, ModelAccessDeniedError, NetworkError, ProviderError, RateLimitError, isModelAccessDeniedMessage, parseAllowedModels, } from "../types/index.js";
 import { isAbortError } from "../utils/errorHandling.js";
@@ -45,12 +46,24 @@ export class LiteLLMProvider extends OpenAIChatCompletionsProvider {
     static modelsCache = [];
     static modelsCacheTime = 0;
     static MODELS_CACHE_DURATION = 10 * 60 * 1000; // 10 minutes
+    /**
+     * Dedupes the fire-and-forget `/model/info` discovery per base URL, so a
+     * process constructing many LiteLLM providers fetches each proxy's limits
+     * once per cache period.
+     */
+    static modelInfoFetchTime = new Map();
     constructor(modelName, sdk, _region, credentials) {
         const envConfig = getLiteLLMConfig();
         super("litellm", modelName, sdk, {
             baseURL: credentials?.baseURL ?? envConfig.baseURL,
             apiKey: credentials?.apiKey ?? envConfig.apiKey,
         });
+        // Fire-and-forget: discover real per-model context windows from the
+        // proxy's /model/info. The static table only has a one-size litellm
+        // `_default` (128K), while proxied models range from 8K to 2M — budget
+        // checks and compaction need the real window. Failures degrade cleanly
+        // to the static default.
+        this.discoverModelContextWindows();
         logger.debug("LiteLLM Provider initialized", {
             modelName: this.modelName,
             provider: this.providerName,
@@ -208,6 +221,85 @@ export class LiteLLMProvider extends OpenAIChatCompletionsProvider {
         }
         return this.getFallbackModels();
     }
+    /**
+     * Discover real per-model context windows from the LiteLLM proxy's
+     * `GET /model/info` (`data[].model_info.max_input_tokens`) and register
+     * them with the context-window resolver. Fire-and-forget with the same
+     * cache period as the models list; any failure (endpoint absent, auth,
+     * timeout) is logged at debug and leaves the static `_default` in force.
+     */
+    discoverModelContextWindows() {
+        const baseURL = stripTrailingSlash(this.config.baseURL);
+        const now = Date.now();
+        const lastFetch = LiteLLMProvider.modelInfoFetchTime.get(baseURL) ?? 0;
+        if (now - lastFetch < LiteLLMProvider.MODELS_CACHE_DURATION) {
+            return;
+        }
+        LiteLLMProvider.modelInfoFetchTime.set(baseURL, now);
+        void this.fetchModelInfoFromAPI()
+            .then((windows) => {
+            for (const [model, contextWindow] of windows) {
+                registerRuntimeContextWindow("litellm", model, contextWindow);
+            }
+            if (windows.size > 0) {
+                logger.debug("[LiteLLMProvider] Registered runtime context windows from /model/info", { baseURL: redactUrlCredentials(baseURL), models: windows.size });
+            }
+        })
+            .catch((error) => {
+            // Allow a retry before the cache period when discovery failed.
+            LiteLLMProvider.modelInfoFetchTime.delete(baseURL);
+            logger.debug("[LiteLLMProvider] /model/info discovery failed; static context-window defaults remain in force", { error: error instanceof Error ? error.message : String(error) });
+        });
+    }
+    /**
+     * Fetch `GET /model/info` and map model group name → max_input_tokens.
+     * Same fetch/auth/abort scaffolding as {@link fetchModelsFromAPI}.
+     */
+    async fetchModelInfoFromAPI() {
+        const infoUrl = `${stripTrailingSlash(this.config.baseURL)}/model/info`;
+        const proxyFetch = createProxyFetch();
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), 5000);
+        try {
+            const response = await proxyFetch(infoUrl, {
+                method: "GET",
+                headers: {
+                    Authorization: `Bearer ${this.config.apiKey}`,
+                    "Content-Type": "application/json",
+                },
+                signal: controller.signal,
+            });
+            if (!response.ok) {
+                throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+            }
+            const data = (await response.json());
+            const windows = new Map();
+            for (const entry of data.data ?? []) {
+                const model = entry?.model_name;
+                const maxInput = entry?.model_info?.max_input_tokens;
+                if (typeof model === "string" &&
+                    model.length > 0 &&
+                    typeof maxInput === "number" &&
+                    Number.isFinite(maxInput) &&
+                    maxInput > 0) {
+                    // A model group can appear once per underlying deployment; keep the
+                    // smallest advertised window so budgets are safe for every replica.
+                    const existing = windows.get(model);
+                    windows.set(model, existing === undefined ? maxInput : Math.min(existing, maxInput));
+                }
+            }
+            return windows;
+        }
+        catch (error) {
+            if (isAbortError(error)) {
+                throw new NetworkError("Request timed out after 5 seconds", this.providerName);
+            }
+            throw error;
+        }
+        finally {
+            clearTimeout(timeoutId);
+        }
+    }
     async fetchModelsFromAPI() {
         const modelsUrl = `${stripTrailingSlash(this.config.baseURL)}/v1/models`;
         const proxyFetch = createProxyFetch();

package/dist/types/context.d.ts

@@ -219,6 +219,28 @@ export type BudgetCheckResult = {
         fileAttachments: number;
     };
 };
+/**
+ * Configuration for the per-step context budget guard that compacts the
+ * AI-SDK tool loop's messages before they overflow the model window
+ * (context/stepBudgetGuard.ts).
+ */
+export type StepBudgetGuardConfig = {
+    provider: string;
+    model?: string;
+    /** The caller's requested output budget (reserved out of the window). */
+    maxTokens?: number;
+    /** Static token cost of the hoisted system prompt + tool definitions. */
+    fixedOverheadTokens?: number;
+    /**
+     * Dynamic overhead resolver, re-evaluated on EVERY guard invocation. Takes
+     * precedence over `fixedOverheadTokens`. Use when the tool set can grow
+     * mid-loop (search_tools hydration) so newly added definitions count toward
+     * the budget.
+     */
+    getFixedOverheadTokens?: () => number;
+    /** Override the trigger ratio; defaults to DEFAULT_CONTEXT_GUARD_RATIO. */
+    thresholdRatio?: number;
+};
 /** Parameters for budget checking. */
 export type BudgetCheckParams = {
     provider: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/neurolink",
-  "version": "10.2.1",
+  "version": "10.3.0",
   "packageManager": "pnpm@10.15.1",
   "description": "TypeScript AI SDK with 24+ LLM providers behind one consistent API. MCP-native (connect any MCP server), voice TTS/STT/realtime, RAG, agents, memory, context compaction. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · DeepSeek · NVIDIA NIM and more.",
   "author": {
@@ -147,7 +147,7 @@
     "test:system-messages:vitest": "pnpm exec vitest run test/systemMessages.test.ts",
     "test:tool-routing-semantic:vitest": "pnpm exec vitest run test/toolRoutingSemantic.test.ts",
     "test:tool-routing-semantic": "pnpm run test:tool-routing-semantic:vitest && npx tsx test/continuous-test-suite-tool-routing-semantic.ts",
-    "test:unit": "pnpm run test:envguard && pnpm run test:bugfixes && pnpm run test:file-detector-extension && pnpm run test:mcp:infra && pnpm run test:mcp:bash && pnpm run test:mcp:limits && pnpm run test:mcp:spans && pnpm run test:autoresearch:redis && pnpm run test:unit:vitest && pnpm run test:tool-routing-cli:vitest && pnpm run test:tool-dedup:vitest && pnpm run test:model-pool:vitest && pnpm run test:system-messages:vitest && pnpm run test:tool-routing-semantic:vitest && pnpm run test:anthropic-tools-policy && pnpm run test:sagemaker-tools && pnpm run test:anthropic-multimodal && pnpm run test:excel-interop",
+    "test:unit": "pnpm run test:envguard && pnpm run test:bugfixes && pnpm run test:file-detector-extension && pnpm run test:mcp:infra && pnpm run test:mcp:bash && pnpm run test:mcp:limits && pnpm run test:mcp:spans && pnpm run test:autoresearch:redis && pnpm run test:unit:vitest && pnpm run test:tool-routing-cli:vitest && pnpm run test:tool-dedup:vitest && pnpm run test:model-pool:vitest && pnpm run test:litellm-context:vitest && pnpm run test:step-budget-guard:vitest && pnpm run test:system-messages:vitest && pnpm run test:tool-routing-semantic:vitest && pnpm run test:anthropic-tools-policy && pnpm run test:sagemaker-tools && pnpm run test:anthropic-multimodal && pnpm run test:excel-interop",
     "// CI tier — live providers, runs only when API keys are present (test:credentials and test:dynamic make real provider calls when keys are set, so they live here, not in test:unit)": "",
     "test:live": "pnpm run test:providers && pnpm run test:mcp:http && pnpm run test:mcp:sdk && pnpm run test:mcp:cli && pnpm run test:observability && pnpm run test:context && pnpm run test:memory && pnpm run test:tool-reliability && pnpm run test:evaluation && pnpm run test:autoresearch && pnpm run test:credentials && pnpm run test:dynamic",
     "// CI tier — product output (image/video/TTS/PPT) — costs $$ per run": "",
@@ -208,7 +208,9 @@
     "quality:report": "pnpm run quality:metrics && echo 'Quality metrics saved to quality-metrics.json'",
     "pre-commit": "lint-staged",
     "pre-push": "pnpm run validate:commit && pnpm run validate:env && pnpm run validate && pnpm run test:ci",
-    "check:all": "pnpm run lint && pnpm run format --check && pnpm run validate && pnpm run validate:commit"
+    "check:all": "pnpm run lint && pnpm run format --check && pnpm run validate && pnpm run validate:commit",
+    "test:litellm-context:vitest": "pnpm exec vitest run test/litellmContextWindows.test.ts",
+    "test:step-budget-guard:vitest": "pnpm exec vitest run test/stepBudgetGuard.test.ts"
   },
   "files": [
     "dist",