@juspay/neurolink 10.3.0 → 10.4.0

package/dist/neurolink.js

@@ -32,6 +32,7 @@ import { SYSTEM_LIMITS, DEFAULT_TOOL_ROUTING_TIMEOUT_MS, MIN_RECOVERY_TURN_BUDGE
 import { ConversationMemoryManager } from "./core/conversationMemoryManager.js";
 import { buildToolRoutingCatalog, buildRoutingQueryFromHistory, resolveToolRoutingExclusions, } from "./core/toolRouting.js";
 import { ToolRoutingCache } from "./core/toolRoutingCache.js";
+import { DEFAULT_RECENT_TURNS, KnowledgeGroundingEngine, } from "./knowledge/index.js";
 import { AIProviderFactory } from "./core/factory.js";
 import { createToolEventPayload } from "./core/toolEvents.js";
 import { ProviderRegistry } from "./factories/providerRegistry.js";
@@ -405,6 +406,9 @@ export class NeuroLink {
     // vectors are computed once. Cleared when the catalog changes via
     // setToolRoutingServers() so stale vectors are never reused.
     toolRoutingVectorCache;
+    // Knowledge grounding: lexical-first host-supplied retrieval engine. Built
+    // once from constructor config and undefined unless grounding is enabled.
+    knowledgeGroundingEngine;
     // Opt-in tool-signature deduplication config.
     toolDedupConfig;
     toolsConfig;
@@ -837,6 +841,18 @@ export class NeuroLink {
             // multiple NeuroLink instances.
             this.toolRoutingConfig = { ...config.toolRouting };
         }
+        const knowledgeGroundingConfig = config?.knowledgeGrounding;
+        if (knowledgeGroundingConfig?.enabled) {
+            if (Array.isArray(knowledgeGroundingConfig.sources) &&
+                knowledgeGroundingConfig.sources.length > 0) {
+                // The engine builds its immutable index asynchronously; eligible
+                // generate() and stream() calls await readiness before retrieval.
+                this.knowledgeGroundingEngine = new KnowledgeGroundingEngine(knowledgeGroundingConfig);
+            }
+            else {
+                logger.warn("[KnowledgeGrounding] enabled but no sources were provided; grounding disabled for this instance");
+            }
+        }
         if (config?.toolDedup) {
             this.toolDedupConfig = { ...config.toolDedup };
         }
@@ -853,7 +869,9 @@ export class NeuroLink {
         // generate() (marked so it never recursively re-routes). Fails open.
         this.classifierRouter = config?.classifierRouter?.enabled
             ? new ClassifierRouter(config.classifierRouter, {
-                generate: (genOptions) => this.generate(genOptions),
+                generate: (genOptions) => this.generate({
+                    ...genOptions,
+                }),
                 logger: {
                     debug: (message, meta) => logger.debug(message, meta),
                     warn: (message, meta) => logger.warn(message, meta),
@@ -3045,6 +3063,7 @@ Current user's request: ${currentInput}`;
      * @param optionsOrPrompt.maxTokens - Maximum tokens to generate
      * @param optionsOrPrompt.thinkingConfig - Extended thinking configuration (thinkingLevel: 'minimal'|'low'|'medium'|'high')
      * @param optionsOrPrompt.context - Context with conversationId and userId for memory
+     * @param optionsOrPrompt.useKnowledgeGrounding - Whether to use the instance's configured knowledge for this call
      * @returns Promise resolving to generation result with content and metadata
      *
      * @example Basic text generation
@@ -3131,9 +3150,22 @@ Current user's request: ${currentInput}`;
         if (typeof optionsOrPrompt !== "string") {
             optionsOrPrompt = cloneOptionsForCallIsolation(optionsOrPrompt);
         }
+        // Retrieve once at the public call boundary so fallback attempts reuse the
+        // same grounding block and internal preparation cannot inject it twice.
+        const groundingOptions = typeof optionsOrPrompt === "string"
+            ? { input: { text: optionsOrPrompt } }
+            : optionsOrPrompt;
+        const knowledgeOutcome = await this.retrieveKnowledgeGrounding(groundingOptions);
+        if (knowledgeOutcome?.ephemeralContext) {
+            const block = knowledgeOutcome.ephemeralContext.content;
+            optionsOrPrompt = {
+                ...groundingOptions,
+                systemPrompt: this.appendKnowledgeGroundingBlockToSystemPrompt(groundingOptions.systemPrompt, block),
+            };
+        }
         const startedAt = Date.now();
         try {
-            return await this.runWithFallbackOrchestration(optionsOrPrompt, "generate", (opts) => {
+            const result = await this.runWithFallbackOrchestration(optionsOrPrompt, "generate", (opts) => {
                 // Capture root-ness before startActiveSpan makes generateSpan active.
                 // The actual guest-rescue stamp is deferred to executeGenerateRequest,
                 // AFTER prepareGenerateRequest merges auth/requestContext-derived
@@ -3142,6 +3174,10 @@ Current user's request: ${currentInput}`;
                 const generateIsRoot = !trace.getSpan(context.active());
                 return tracers.sdk.startActiveSpan("neurolink.generate", { kind: SpanKind.INTERNAL }, (generateSpan) => this.executeGenerateWithMetricsContext(opts, generateSpan, generateIsRoot));
             });
+            if (knowledgeOutcome) {
+                result.knowledge = knowledgeOutcome.metadata;
+            }
+            return result;
         }
         catch (error) {
             // Lifecycle middleware (wrapGenerate.catch in builtin/lifecycle.ts)
@@ -5190,12 +5226,22 @@ Current user's request: ${currentInput}`;
             if (!generationContext) {
                 return null;
             }
+            // Provider construction runs BEFORE the budget check so runtime
+            // model-limit discovery (ensureModelLimits) can register real context
+            // windows first — ensureMCPGenerationBudget otherwise computes against
+            // the static default window (see directProviderGeneration for the
+            // lock-out this ordering prevents).
+            const provider = await AIProviderFactory.createProvider(generationContext.providerName, options.model, !options.disableTools, this, options.region, this.resolveCredentials(options.credentials));
+            provider.setTraceContext(this._metricsTraceContext);
+            // Never rejects — discovery failure degrades to static defaults.
+            await provider.ensureModelLimits?.();
             const conversationMessages = await this.ensureMCPGenerationBudget(options, requestId, generationContext.providerName, generationContext.enhancedSystemPrompt, generationContext.availableTools, generationContext.conversationMessages);
             return this.generateWithMCPProvider({
                 options,
                 requestId,
                 functionTag,
                 tryMCPStartTime,
+                provider,
                 providerName: generationContext.providerName,
                 availableTools: generationContext.availableTools,
                 enhancedSystemPrompt: generationContext.enhancedSystemPrompt,
@@ -5479,9 +5525,7 @@ Current user's request: ${currentInput}`;
         return compactedMessages;
     }
     async generateWithMCPProvider(context) {
-        const { options, requestId, functionTag, tryMCPStartTime, providerName, availableTools, enhancedSystemPrompt, conversationMessages, } = context;
-        const provider = await AIProviderFactory.createProvider(providerName, options.model, !options.disableTools, this, options.region, this.resolveCredentials(options.credentials));
-        provider.setTraceContext(this._metricsTraceContext);
+        const { options, requestId, functionTag, tryMCPStartTime, provider, providerName, availableTools, enhancedSystemPrompt, conversationMessages, } = context;
         this.emitter.emit("connected");
         this.emitter.emit("message", `${providerName} provider initialized successfully`);
         provider.setupToolExecutor({
@@ -5760,6 +5804,21 @@ Current user's request: ${currentInput}`;
                     ?.length
                     ? optionsWithMessages.conversationMessages
                     : await getConversationMessages(this.conversationMemory, options);
+                // Provider construction runs BEFORE the budget check so runtime
+                // model-limit discovery (ensureModelLimits) can register real
+                // context windows first. The previous order created a lock-out:
+                // checkContextBudget ran against the static default window, its
+                // pre-dispatch hard cap threw before the provider (whose
+                // constructor owns the discovery) ever existed — so a blocked call
+                // prevented the very discovery that would have unblocked it.
+                const provider = await AIProviderFactory.createProvider(providerName, options.model, !options.disableTools, // Pass disableTools as inverse of enableMCP
+                this, // Pass SDK instance
+                options.region, // Pass region parameter
+                this.resolveCredentials(options.credentials));
+                // Propagate trace context for parent-child span hierarchy
+                provider.setTraceContext(this._metricsTraceContext);
+                // Never rejects — discovery failure degrades to static defaults.
+                await provider.ensureModelLimits?.();
                 // Pre-generation budget check
                 const budgetCheck = checkContextBudget({
                     provider: providerName,
@@ -5915,12 +5974,6 @@ Current user's request: ${currentInput}`;
                         }
                     }
                 }
-                const provider = await AIProviderFactory.createProvider(providerName, options.model, !options.disableTools, // Pass disableTools as inverse of enableMCP
-                this, // Pass SDK instance
-                options.region, // Pass region parameter
-                this.resolveCredentials(options.credentials));
-                // Propagate trace context for parent-child span hierarchy
-                provider.setTraceContext(this._metricsTraceContext);
                 // ADD: Emit connection events for successful provider creation (Bedrock-compatible)
                 this.emitter.emit("connected");
                 this.emitter.emit("message", `${providerName} provider initialized successfully`);
@@ -6187,6 +6240,7 @@ Current user's request: ${currentInput}`;
      * @param options.enableEvaluation - Whether to include response quality evaluation
      * @param options.context - Additional context for the request
      * @param options.evaluationDomain - Domain for specialized evaluation
+     * @param options.useKnowledgeGrounding - Whether to use the instance's configured knowledge for this call
      *
      * @returns Promise resolving to StreamResult with an async iterable stream
      *
@@ -6244,8 +6298,23 @@ Current user's request: ${currentInput}`;
         // top-level keys and left `options.input` shared with the caller.
         options = cloneOptionsForCallIsolation(options);
         const startedAt = Date.now();
+        // Retrieve once before provider/model fallback orchestration. Every fallback
+        // receives the same explicitly enriched options, without the retrieval helper
+        // mutating its input or injecting the same block more than once.
+        const knowledgeOutcome = await this.retrieveKnowledgeGrounding(options);
+        if (knowledgeOutcome?.ephemeralContext) {
+            const block = knowledgeOutcome.ephemeralContext.content;
+            options = {
+                ...options,
+                systemPrompt: this.appendKnowledgeGroundingBlockToSystemPrompt(options.systemPrompt, block),
+            };
+        }
         try {
-            return await this.streamWithIterationFallback(options);
+            const result = await this.streamWithIterationFallback(options);
+            if (knowledgeOutcome) {
+                result.knowledge = knowledgeOutcome.metadata;
+            }
+            return result;
         }
         catch (error) {
             // Mirror generate(): fire consumer onError for failures that
@@ -6913,6 +6982,63 @@ Current user's request: ${currentInput}`;
         // to recompute exclusions against the new server/tool set.
         this.toolRoutingCacheInstance = undefined;
     }
+    /** Knowledge-grounding engine health (null when it was not configured). */
+    getKnowledgeStatus() {
+        return this.knowledgeGroundingEngine?.getStatus() ?? null;
+    }
+    appendKnowledgeGroundingBlockToSystemPrompt(systemPrompt, block) {
+        if (typeof systemPrompt === "function") {
+            const originalSystemPrompt = systemPrompt;
+            return async (context) => {
+                const resolved = await resolveDynamicArgument(originalSystemPrompt, context);
+                return resolved.value ? `${resolved.value}\n\n${block}` : block;
+            };
+        }
+        return systemPrompt ? `${systemPrompt}\n\n${block}` : block;
+    }
+    /**
+     * Retrieve knowledge grounding for a public generate/stream call without
+     * mutating its options. The outer call boundary decides how to apply the
+     * returned context. Returns undefined when the call does not opt in,
+     * grounding is disabled/not applicable, or retrieval fails open.
+     */
+    async retrieveKnowledgeGrounding(options) {
+        const engine = this.knowledgeGroundingEngine;
+        if (!engine ||
+            !engine.isEnabled() ||
+            options.useKnowledgeGrounding !== true) {
+            return undefined;
+        }
+        const query = options.input?.text;
+        if (!query) {
+            return undefined;
+        }
+        try {
+            const conversationMessages = options.conversationMessages !== undefined
+                ? options.conversationMessages
+                : await this.fetchRecentRoutingHistory(options);
+            const recentTurns = conversationMessages
+                .filter((message) => message.role === "user" || message.role === "assistant")
+                .slice(-DEFAULT_RECENT_TURNS)
+                .map((message) => ({
+                role: message.role === "assistant"
+                    ? "assistant"
+                    : "user",
+                text: typeof message.content === "string" ? message.content : "",
+            }));
+            return await engine.ground({
+                query,
+                recentTurns,
+                scope: options.knowledgeContext,
+            });
+        }
+        catch (error) {
+            logger.warn("[KnowledgeGrounding] grounding hook failed open", {
+                error: String(error),
+            });
+            return undefined;
+        }
+    }
     async validateStreamRequestOptions(options, startTime) {
         await this.validateStreamInput(options);
         // Input validation for stream
@@ -7892,6 +8018,10 @@ Current user's request: ${currentInput}`;
         this, // Pass SDK instance
         options.region, // Pass region parameter
         this.resolveCredentials(options.credentials));
+        // Runtime model-limit discovery must land BEFORE the stream budget
+        // check below — otherwise it computes against the static default
+        // window. Never rejects; failure degrades to static defaults.
+        await provider.ensureModelLimits?.();
         // Propagate trace context for parent-child span hierarchy
         provider.setTraceContext(this._metricsTraceContext);
         // Enable tool execution for the provider using BaseProvider method

package/dist/providers/litellm.d.ts

@@ -1,6 +1,26 @@
 import type { AIProviderName } from "../constants/enums.js";
 import type { OpenAICompatBuildBodyArgs, OpenAICompatStreamLifecycleListeners } from "../types/index.js";
 import { OpenAIChatCompletionsProvider } from "./openaiChatCompletionsBase.js";
+/** Test hook: reset the discovery caches (state is module-global). */
+export declare function clearLiteLLMModelLimitsCache(): void;
+/**
+ * Discover real per-model limits from the LiteLLM proxy's `GET /model/info`
+ * and register them with the runtime resolvers: `max_input_tokens` → context
+ * window, `max_output_tokens` → output ceiling. The static table only has a
+ * one-size litellm `_default` (128K) while proxied models range from 8K to
+ * 2M — budget checks, compaction, and max_tokens clamping need the real
+ * numbers.
+ *
+ * Awaitable and deduped per base URL: generation pipelines await this (via
+ * `LiteLLMProvider.ensureModelLimits`) BEFORE any budget math, so even the
+ * FIRST call in a fresh process budgets against real windows — the previous
+ * constructor-scoped fire-and-forget lost that race every time, and a
+ * budget-blocked call then prevented the discovery that would have unblocked
+ * it. Never rejects: any failure (endpoint absent, auth, timeout) is logged
+ * at debug, leaves the static defaults in force, and allows a retry on the
+ * next call.
+ */
+export declare function ensureLiteLLMModelLimits(baseURL: string, apiKey: string): Promise<void>;
 /**
  * LiteLLM Provider — direct HTTP, no AI SDK. Talks to a LiteLLM proxy
  * server (or any deployment that speaks OpenAI chat-completions + the
@@ -17,16 +37,17 @@ export declare class LiteLLMProvider extends OpenAIChatCompletionsProvider {
     private static modelsCache;
     private static modelsCacheTime;
     private static readonly MODELS_CACHE_DURATION;
-    /**
-     * Dedupes the fire-and-forget `/model/info` discovery per base URL, so a
-     * process constructing many LiteLLM providers fetches each proxy's limits
-     * once per cache period.
-     */
-    private static modelInfoFetchTime;
     constructor(modelName?: string, sdk?: unknown, _region?: string, credentials?: {
         apiKey?: string;
         baseURL?: string;
     });
+    /**
+     * Awaitable model-limit discovery — generation pipelines call this before
+     * budget checks so window/output-ceiling math uses the proxy's real
+     * numbers (see BaseProvider.ensureModelLimits). Deduped and cached per
+     * base URL; never rejects (failure degrades to static defaults).
+     */
+    ensureModelLimits(): Promise<void>;
     protected getProviderName(): AIProviderName;
     protected getDefaultModel(): string;
     protected getFallbackModelName(): string;
@@ -50,19 +71,6 @@ export declare class LiteLLMProvider extends OpenAIChatCompletionsProvider {
      * minimal safe default if the API fetch fails.
      */
     getAvailableModels(): Promise<string[]>;
-    /**
-     * Discover real per-model context windows from the LiteLLM proxy's
-     * `GET /model/info` (`data[].model_info.max_input_tokens`) and register
-     * them with the context-window resolver. Fire-and-forget with the same
-     * cache period as the models list; any failure (endpoint absent, auth,
-     * timeout) is logged at debug and leaves the static `_default` in force.
-     */
-    private discoverModelContextWindows;
-    /**
-     * Fetch `GET /model/info` and map model group name → max_input_tokens.
-     * Same fetch/auth/abort scaffolding as {@link fetchModelsFromAPI}.
-     */
-    private fetchModelInfoFromAPI;
     private fetchModelsFromAPI;
     /**
      * Generate an embedding for a single text input via native /v1/embeddings.

package/dist/providers/litellm.js

@@ -1,5 +1,5 @@
 import { SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
-import { registerRuntimeContextWindow } from "../constants/contextWindows.js";
+import { registerRuntimeContextWindow, registerRuntimeOutputCeiling, } from "../constants/contextWindows.js";
 import { createProxyFetch } from "../proxy/proxyFetch.js";
 import { AuthenticationError, InvalidModelError, ModelAccessDeniedError, NetworkError, ProviderError, RateLimitError, isModelAccessDeniedMessage, parseAllowedModels, } from "../types/index.js";
 import { isAbortError } from "../utils/errorHandling.js";
@@ -21,6 +21,163 @@ const getLiteLLMConfig = () => ({
  * LiteLLM uses a 'provider/model' format. Override via LITELLM_MODEL env var.
  */
 const getDefaultLiteLLMModel = () => getProviderModel("LITELLM_MODEL", FALLBACK_LITELLM_MODEL);
+/** Cache period for `/model/info` limit discovery (same as the models list). */
+const MODEL_INFO_CACHE_DURATION = 10 * 60 * 1000;
+/** In-flight `/model/info` discovery per base URL — dedupes concurrent callers. */
+const modelInfoInFlight = new Map();
+/** Last successful `/model/info` discovery per base URL. */
+const modelInfoLastSuccess = new Map();
+/**
+ * Discovered limits per base URL. The runtime registries key by
+ * (provider, model) only — budget call sites carry no deployment identity —
+ * so when a process talks to MULTIPLE proxies that share a model-group name,
+ * registrations use the MIN across every discovered deployment: budgets stay
+ * safe for all of them instead of last-writer-wins silently overstating one.
+ * Single-proxy processes (the normal case) are unaffected.
+ */
+const modelInfoLimitsByURL = new Map();
+/** Test hook: reset the discovery caches (state is module-global). */
+export function clearLiteLLMModelLimitsCache() {
+    modelInfoInFlight.clear();
+    modelInfoLastSuccess.clear();
+    modelInfoLimitsByURL.clear();
+}
+/**
+ * Fetch `GET /model/info` and map model group name → advertised limits
+ * (`model_info.max_input_tokens` / `model_info.max_output_tokens`).
+ * A model group can appear once per underlying deployment; the SMALLEST
+ * advertised value wins per field so budgets are safe for every replica.
+ */
+async function fetchLiteLLMModelLimits(baseURL, apiKey) {
+    const infoUrl = `${baseURL}/model/info`;
+    const proxyFetch = createProxyFetch();
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 5000);
+    try {
+        const response = await proxyFetch(infoUrl, {
+            method: "GET",
+            headers: {
+                Authorization: `Bearer ${apiKey}`,
+                "Content-Type": "application/json",
+            },
+            signal: controller.signal,
+        });
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+        }
+        const data = (await response.json());
+        const isUsable = (v) => typeof v === "number" && Number.isFinite(v) && v > 0;
+        const limits = new Map();
+        for (const entry of data.data ?? []) {
+            const model = entry?.model_name;
+            if (typeof model !== "string" || model.length === 0) {
+                continue;
+            }
+            const maxInput = entry?.model_info?.max_input_tokens;
+            const maxOutput = entry?.model_info?.max_output_tokens;
+            if (!isUsable(maxInput) && !isUsable(maxOutput)) {
+                continue;
+            }
+            const existing = limits.get(model) ?? {};
+            if (isUsable(maxInput)) {
+                existing.maxInputTokens =
+                    existing.maxInputTokens === undefined
+                        ? maxInput
+                        : Math.min(existing.maxInputTokens, maxInput);
+            }
+            if (isUsable(maxOutput)) {
+                existing.maxOutputTokens =
+                    existing.maxOutputTokens === undefined
+                        ? maxOutput
+                        : Math.min(existing.maxOutputTokens, maxOutput);
+            }
+            limits.set(model, existing);
+        }
+        return limits;
+    }
+    catch (error) {
+        if (isAbortError(error)) {
+            throw new NetworkError("Request timed out after 5 seconds", "litellm");
+        }
+        throw error;
+    }
+    finally {
+        clearTimeout(timeoutId);
+    }
+}
+/**
+ * Discover real per-model limits from the LiteLLM proxy's `GET /model/info`
+ * and register them with the runtime resolvers: `max_input_tokens` → context
+ * window, `max_output_tokens` → output ceiling. The static table only has a
+ * one-size litellm `_default` (128K) while proxied models range from 8K to
+ * 2M — budget checks, compaction, and max_tokens clamping need the real
+ * numbers.
+ *
+ * Awaitable and deduped per base URL: generation pipelines await this (via
+ * `LiteLLMProvider.ensureModelLimits`) BEFORE any budget math, so even the
+ * FIRST call in a fresh process budgets against real windows — the previous
+ * constructor-scoped fire-and-forget lost that race every time, and a
+ * budget-blocked call then prevented the discovery that would have unblocked
+ * it. Never rejects: any failure (endpoint absent, auth, timeout) is logged
+ * at debug, leaves the static defaults in force, and allows a retry on the
+ * next call.
+ */
+export function ensureLiteLLMModelLimits(baseURL, apiKey) {
+    const key = stripTrailingSlash(baseURL);
+    const lastSuccess = modelInfoLastSuccess.get(key) ?? 0;
+    if (Date.now() - lastSuccess < MODEL_INFO_CACHE_DURATION) {
+        return Promise.resolve();
+    }
+    const inFlight = modelInfoInFlight.get(key);
+    if (inFlight) {
+        return inFlight;
+    }
+    const discovery = fetchLiteLLMModelLimits(key, apiKey)
+        .then((limits) => {
+        modelInfoLimitsByURL.set(key, limits);
+        // Min-merge across every base URL discovered so far (see
+        // modelInfoLimitsByURL) before registering.
+        const merged = new Map();
+        for (const urlLimits of modelInfoLimitsByURL.values()) {
+            for (const [model, modelLimits] of urlLimits) {
+                const existing = merged.get(model) ?? {};
+                if (modelLimits.maxInputTokens !== undefined) {
+                    existing.maxInputTokens =
+                        existing.maxInputTokens === undefined
+                            ? modelLimits.maxInputTokens
+                            : Math.min(existing.maxInputTokens, modelLimits.maxInputTokens);
+                }
+                if (modelLimits.maxOutputTokens !== undefined) {
+                    existing.maxOutputTokens =
+                        existing.maxOutputTokens === undefined
+                            ? modelLimits.maxOutputTokens
+                            : Math.min(existing.maxOutputTokens, modelLimits.maxOutputTokens);
+                }
+                merged.set(model, existing);
+            }
+        }
+        for (const [model, modelLimits] of merged) {
+            if (modelLimits.maxInputTokens !== undefined) {
+                registerRuntimeContextWindow("litellm", model, modelLimits.maxInputTokens);
+            }
+            if (modelLimits.maxOutputTokens !== undefined) {
+                registerRuntimeOutputCeiling("litellm", model, modelLimits.maxOutputTokens);
+            }
+        }
+        modelInfoLastSuccess.set(key, Date.now());
+        if (limits.size > 0) {
+            logger.debug("[LiteLLM] Registered runtime model limits from /model/info", { baseURL: redactUrlCredentials(key), models: limits.size });
+        }
+    })
+        .catch((error) => {
+        logger.debug("[LiteLLM] /model/info discovery failed; static context-window defaults remain in force", { error: error instanceof Error ? error.message : String(error) });
+    })
+        .finally(() => {
+        modelInfoInFlight.delete(key);
+    });
+    modelInfoInFlight.set(key, discovery);
+    return discovery;
+}
 // LiteLLM model ids come in `provider/model` form (e.g. "google/gemini-2.5-flash").
 // Strip the provider prefix and delegate to the canonical anchored-regex
 // check in src/lib/utils/modelDetection.ts so the truth lives in one place.
@@ -46,30 +203,31 @@ export class LiteLLMProvider extends OpenAIChatCompletionsProvider {
     static modelsCache = [];
     static modelsCacheTime = 0;
     static MODELS_CACHE_DURATION = 10 * 60 * 1000; // 10 minutes
-    /**
-     * Dedupes the fire-and-forget `/model/info` discovery per base URL, so a
-     * process constructing many LiteLLM providers fetches each proxy's limits
-     * once per cache period.
-     */
-    static modelInfoFetchTime = new Map();
     constructor(modelName, sdk, _region, credentials) {
         const envConfig = getLiteLLMConfig();
         super("litellm", modelName, sdk, {
             baseURL: credentials?.baseURL ?? envConfig.baseURL,
             apiKey: credentials?.apiKey ?? envConfig.apiKey,
         });
-        // Fire-and-forget: discover real per-model context windows from the
-        // proxy's /model/info. The static table only has a one-size litellm
-        // `_default` (128K), while proxied models range from 8K to 2M — budget
-        // checks and compaction need the real window. Failures degrade cleanly
-        // to the static default.
-        this.discoverModelContextWindows();
+        // Warm the model-limit discovery early (deduped per base URL). The
+        // generation pipelines still AWAIT ensureModelLimits() before budget
+        // math — this fire-and-forget only shaves latency off that first await.
+        void ensureLiteLLMModelLimits(this.config.baseURL, this.config.apiKey);
         logger.debug("LiteLLM Provider initialized", {
             modelName: this.modelName,
             provider: this.providerName,
             baseURL: redactUrlCredentials(this.config.baseURL),
         });
     }
+    /**
+     * Awaitable model-limit discovery — generation pipelines call this before
+     * budget checks so window/output-ceiling math uses the proxy's real
+     * numbers (see BaseProvider.ensureModelLimits). Deduped and cached per
+     * base URL; never rejects (failure degrades to static defaults).
+     */
+    async ensureModelLimits() {
+        await ensureLiteLLMModelLimits(this.config.baseURL, this.config.apiKey);
+    }
     getProviderName() {
         return "litellm";
     }
@@ -221,85 +379,6 @@ export class LiteLLMProvider extends OpenAIChatCompletionsProvider {
         }
         return this.getFallbackModels();
     }
-    /**
-     * Discover real per-model context windows from the LiteLLM proxy's
-     * `GET /model/info` (`data[].model_info.max_input_tokens`) and register
-     * them with the context-window resolver. Fire-and-forget with the same
-     * cache period as the models list; any failure (endpoint absent, auth,
-     * timeout) is logged at debug and leaves the static `_default` in force.
-     */
-    discoverModelContextWindows() {
-        const baseURL = stripTrailingSlash(this.config.baseURL);
-        const now = Date.now();
-        const lastFetch = LiteLLMProvider.modelInfoFetchTime.get(baseURL) ?? 0;
-        if (now - lastFetch < LiteLLMProvider.MODELS_CACHE_DURATION) {
-            return;
-        }
-        LiteLLMProvider.modelInfoFetchTime.set(baseURL, now);
-        void this.fetchModelInfoFromAPI()
-            .then((windows) => {
-            for (const [model, contextWindow] of windows) {
-                registerRuntimeContextWindow("litellm", model, contextWindow);
-            }
-            if (windows.size > 0) {
-                logger.debug("[LiteLLMProvider] Registered runtime context windows from /model/info", { baseURL: redactUrlCredentials(baseURL), models: windows.size });
-            }
-        })
-            .catch((error) => {
-            // Allow a retry before the cache period when discovery failed.
-            LiteLLMProvider.modelInfoFetchTime.delete(baseURL);
-            logger.debug("[LiteLLMProvider] /model/info discovery failed; static context-window defaults remain in force", { error: error instanceof Error ? error.message : String(error) });
-        });
-    }
-    /**
-     * Fetch `GET /model/info` and map model group name → max_input_tokens.
-     * Same fetch/auth/abort scaffolding as {@link fetchModelsFromAPI}.
-     */
-    async fetchModelInfoFromAPI() {
-        const infoUrl = `${stripTrailingSlash(this.config.baseURL)}/model/info`;
-        const proxyFetch = createProxyFetch();
-        const controller = new AbortController();
-        const timeoutId = setTimeout(() => controller.abort(), 5000);
-        try {
-            const response = await proxyFetch(infoUrl, {
-                method: "GET",
-                headers: {
-                    Authorization: `Bearer ${this.config.apiKey}`,
-                    "Content-Type": "application/json",
-                },
-                signal: controller.signal,
-            });
-            if (!response.ok) {
-                throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-            }
-            const data = (await response.json());
-            const windows = new Map();
-            for (const entry of data.data ?? []) {
-                const model = entry?.model_name;
-                const maxInput = entry?.model_info?.max_input_tokens;
-                if (typeof model === "string" &&
-                    model.length > 0 &&
-                    typeof maxInput === "number" &&
-                    Number.isFinite(maxInput) &&
-                    maxInput > 0) {
-                    // A model group can appear once per underlying deployment; keep the
-                    // smallest advertised window so budgets are safe for every replica.
-                    const existing = windows.get(model);
-                    windows.set(model, existing === undefined ? maxInput : Math.min(existing, maxInput));
-                }
-            }
-            return windows;
-        }
-        catch (error) {
-            if (isAbortError(error)) {
-                throw new NetworkError("Request timed out after 5 seconds", this.providerName);
-            }
-            throw error;
-        }
-        finally {
-            clearTimeout(timeoutId);
-        }
-    }
     async fetchModelsFromAPI() {
         const modelsUrl = `${stripTrailingSlash(this.config.baseURL)}/v1/models`;
         const proxyFetch = createProxyFetch();