@juspay/neurolink 10.2.0 → 10.2.2

package/dist/constants/contextWindows.d.ts

@@ -21,12 +21,24 @@ export declare const DEFAULT_OUTPUT_RESERVE_RATIO = 0.35;
  * The "_default" key is the fallback for unknown models within a provider.
  */
 export declare const MODEL_CONTEXT_WINDOWS: Record<string, Record<string, number>>;
+/**
+ * Register a runtime-discovered context window for a provider/model pair.
+ * Later registrations overwrite earlier ones (rediscovery refreshes values).
+ * Non-positive/non-finite windows are ignored so a malformed discovery source
+ * can never shrink a budget to zero.
+ */
+export declare function registerRuntimeContextWindow(provider: string, model: string, contextWindow: number): void;
+/** Test hook: clear runtime-discovered windows (state is module-global). */
+export declare function clearRuntimeContextWindows(): void;
 /**
  * Resolve context window size for a provider/model combination.
  *
  * Priority:
  *  0. Dynamic model registry (DynamicModelProvider) — resolves cross-provider
  *     models (e.g. Claude on Vertex) that the static table cannot handle
+ *  0.5 Runtime-discovered windows (registerRuntimeContextWindow) — real
+ *      per-model limits fetched from the serving infrastructure (LiteLLM
+ *      `/model/info`)
  *  1. Exact model match under provider in static registry
  *  2. Prefix match under provider in static registry
  *  3. Provider's _default in static registry

package/dist/constants/contextWindows.js

@@ -392,12 +392,43 @@ function normalizeProviderForLookup(provider) {
     // to DEFAULT_CONTEXT_WINDOW.
     return PROVIDER_ALIAS_MAP[stripped] ?? stripped;
 }
+/**
+ * Runtime-discovered context windows, keyed `${provider}:${model}`.
+ *
+ * Populated asynchronously by providers that can discover real per-model
+ * limits at runtime (e.g. the LiteLLM provider reads `max_input_tokens` from
+ * the proxy's `/model/info`), and read synchronously by
+ * {@link getContextWindowSize} — the same async-populate/sync-read contract as
+ * the DynamicModelProvider registry. Keys use the RAW provider string callers
+ * pass into budget calculations (see the alias-map comment above: normalized
+ * provider names never reach these lookups).
+ */
+const RUNTIME_CONTEXT_WINDOWS = new Map();
+/**
+ * Register a runtime-discovered context window for a provider/model pair.
+ * Later registrations overwrite earlier ones (rediscovery refreshes values).
+ * Non-positive/non-finite windows are ignored so a malformed discovery source
+ * can never shrink a budget to zero.
+ */
+export function registerRuntimeContextWindow(provider, model, contextWindow) {
+    if (!Number.isFinite(contextWindow) || contextWindow <= 0) {
+        return;
+    }
+    RUNTIME_CONTEXT_WINDOWS.set(`${provider}:${model}`, contextWindow);
+}
+/** Test hook: clear runtime-discovered windows (state is module-global). */
+export function clearRuntimeContextWindows() {
+    RUNTIME_CONTEXT_WINDOWS.clear();
+}
 /**
  * Resolve context window size for a provider/model combination.
  *
  * Priority:
  *  0. Dynamic model registry (DynamicModelProvider) — resolves cross-provider
  *     models (e.g. Claude on Vertex) that the static table cannot handle
+ *  0.5 Runtime-discovered windows (registerRuntimeContextWindow) — real
+ *      per-model limits fetched from the serving infrastructure (LiteLLM
+ *      `/model/info`)
  *  1. Exact model match under provider in static registry
  *  2. Prefix match under provider in static registry
  *  3. Provider's _default in static registry
@@ -421,6 +452,13 @@ export function getContextWindowSize(provider, model) {
             // Dynamic registry not initialized yet — fall through to static lookup
         }
     }
+    // Step 0.5: Runtime-discovered window for this exact provider/model.
+    if (model) {
+        const discovered = RUNTIME_CONTEXT_WINDOWS.get(`${provider}:${model}`);
+        if (discovered !== undefined) {
+            return discovered;
+        }
+    }
     // Static fallback chain — normalize aliases first so "lmstudio" / "llama.cpp" /
     // "nvidianim" find their canonical entries instead of falling back to default.
     const canonical = normalizeProviderForLookup(provider);

package/dist/lib/constants/contextWindows.d.ts

@@ -21,12 +21,24 @@ export declare const DEFAULT_OUTPUT_RESERVE_RATIO = 0.35;
  * The "_default" key is the fallback for unknown models within a provider.
  */
 export declare const MODEL_CONTEXT_WINDOWS: Record<string, Record<string, number>>;
+/**
+ * Register a runtime-discovered context window for a provider/model pair.
+ * Later registrations overwrite earlier ones (rediscovery refreshes values).
+ * Non-positive/non-finite windows are ignored so a malformed discovery source
+ * can never shrink a budget to zero.
+ */
+export declare function registerRuntimeContextWindow(provider: string, model: string, contextWindow: number): void;
+/** Test hook: clear runtime-discovered windows (state is module-global). */
+export declare function clearRuntimeContextWindows(): void;
 /**
  * Resolve context window size for a provider/model combination.
  *
  * Priority:
  *  0. Dynamic model registry (DynamicModelProvider) — resolves cross-provider
  *     models (e.g. Claude on Vertex) that the static table cannot handle
+ *  0.5 Runtime-discovered windows (registerRuntimeContextWindow) — real
+ *      per-model limits fetched from the serving infrastructure (LiteLLM
+ *      `/model/info`)
  *  1. Exact model match under provider in static registry
  *  2. Prefix match under provider in static registry
  *  3. Provider's _default in static registry

package/dist/lib/constants/contextWindows.js

@@ -392,12 +392,43 @@ function normalizeProviderForLookup(provider) {
     // to DEFAULT_CONTEXT_WINDOW.
     return PROVIDER_ALIAS_MAP[stripped] ?? stripped;
 }
+/**
+ * Runtime-discovered context windows, keyed `${provider}:${model}`.
+ *
+ * Populated asynchronously by providers that can discover real per-model
+ * limits at runtime (e.g. the LiteLLM provider reads `max_input_tokens` from
+ * the proxy's `/model/info`), and read synchronously by
+ * {@link getContextWindowSize} — the same async-populate/sync-read contract as
+ * the DynamicModelProvider registry. Keys use the RAW provider string callers
+ * pass into budget calculations (see the alias-map comment above: normalized
+ * provider names never reach these lookups).
+ */
+const RUNTIME_CONTEXT_WINDOWS = new Map();
+/**
+ * Register a runtime-discovered context window for a provider/model pair.
+ * Later registrations overwrite earlier ones (rediscovery refreshes values).
+ * Non-positive/non-finite windows are ignored so a malformed discovery source
+ * can never shrink a budget to zero.
+ */
+export function registerRuntimeContextWindow(provider, model, contextWindow) {
+    if (!Number.isFinite(contextWindow) || contextWindow <= 0) {
+        return;
+    }
+    RUNTIME_CONTEXT_WINDOWS.set(`${provider}:${model}`, contextWindow);
+}
+/** Test hook: clear runtime-discovered windows (state is module-global). */
+export function clearRuntimeContextWindows() {
+    RUNTIME_CONTEXT_WINDOWS.clear();
+}
 /**
  * Resolve context window size for a provider/model combination.
  *
  * Priority:
  *  0. Dynamic model registry (DynamicModelProvider) — resolves cross-provider
  *     models (e.g. Claude on Vertex) that the static table cannot handle
+ *  0.5 Runtime-discovered windows (registerRuntimeContextWindow) — real
+ *      per-model limits fetched from the serving infrastructure (LiteLLM
+ *      `/model/info`)
  *  1. Exact model match under provider in static registry
  *  2. Prefix match under provider in static registry
  *  3. Provider's _default in static registry
@@ -421,6 +452,13 @@ export function getContextWindowSize(provider, model) {
             // Dynamic registry not initialized yet — fall through to static lookup
         }
     }
+    // Step 0.5: Runtime-discovered window for this exact provider/model.
+    if (model) {
+        const discovered = RUNTIME_CONTEXT_WINDOWS.get(`${provider}:${model}`);
+        if (discovered !== undefined) {
+            return discovered;
+        }
+    }
     // Static fallback chain — normalize aliases first so "lmstudio" / "llama.cpp" /
     // "nvidianim" find their canonical entries instead of falling back to default.
     const canonical = normalizeProviderForLookup(provider);

package/dist/lib/providers/azureOpenai.d.ts

@@ -22,6 +22,12 @@ export declare class AzureOpenAIProvider extends OpenAIChatCompletionsProvider {
     protected readonly azureDeploymentPathPrefix: string;
     protected readonly useMaxCompletionTokensOverride?: boolean;
     constructor(modelName?: string, sdk?: unknown, _region?: string, credentials?: NeurolinkCredentials["azure"]);
+    /**
+     * Azure OpenAI natively supports `response_format: json_schema` together
+     * with tool calling in one request, so structured output stays
+     * wire-enforced mid-loop instead of deferring to post-hoc coercion.
+     */
+    protected suppressResponseFormatWithTools(): boolean;
     protected getProviderName(): AIProviderName;
     /**
      * The "default model" for Azure is the deployment name — it's the

package/dist/lib/providers/azureOpenai.js

@@ -139,6 +139,14 @@ export class AzureOpenAIProvider extends OpenAIChatCompletionsProvider {
     // ===========================================================================
     // Abstract-hook implementations
     // ===========================================================================
+    /**
+     * Azure OpenAI natively supports `response_format: json_schema` together
+     * with tool calling in one request, so structured output stays
+     * wire-enforced mid-loop instead of deferring to post-hoc coercion.
+     */
+    suppressResponseFormatWithTools() {
+        return false;
+    }
     getProviderName() {
         return "azure";
     }

package/dist/lib/providers/litellm.d.ts

@@ -17,6 +17,12 @@ export declare class LiteLLMProvider extends OpenAIChatCompletionsProvider {
     private static modelsCache;
     private static modelsCacheTime;
     private static readonly MODELS_CACHE_DURATION;
+    /**
+     * Dedupes the fire-and-forget `/model/info` discovery per base URL, so a
+     * process constructing many LiteLLM providers fetches each proxy's limits
+     * once per cache period.
+     */
+    private static modelInfoFetchTime;
     constructor(modelName?: string, sdk?: unknown, _region?: string, credentials?: {
         apiKey?: string;
         baseURL?: string;
@@ -44,6 +50,19 @@ export declare class LiteLLMProvider extends OpenAIChatCompletionsProvider {
      * minimal safe default if the API fetch fails.
      */
     getAvailableModels(): Promise<string[]>;
+    /**
+     * Discover real per-model context windows from the LiteLLM proxy's
+     * `GET /model/info` (`data[].model_info.max_input_tokens`) and register
+     * them with the context-window resolver. Fire-and-forget with the same
+     * cache period as the models list; any failure (endpoint absent, auth,
+     * timeout) is logged at debug and leaves the static `_default` in force.
+     */
+    private discoverModelContextWindows;
+    /**
+     * Fetch `GET /model/info` and map model group name → max_input_tokens.
+     * Same fetch/auth/abort scaffolding as {@link fetchModelsFromAPI}.
+     */
+    private fetchModelInfoFromAPI;
     private fetchModelsFromAPI;
     /**
      * Generate an embedding for a single text input via native /v1/embeddings.

package/dist/lib/providers/litellm.js

@@ -1,4 +1,5 @@
 import { SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
+import { registerRuntimeContextWindow } from "../constants/contextWindows.js";
 import { createProxyFetch } from "../proxy/proxyFetch.js";
 import { AuthenticationError, InvalidModelError, ModelAccessDeniedError, NetworkError, ProviderError, RateLimitError, isModelAccessDeniedMessage, parseAllowedModels, } from "../types/index.js";
 import { isAbortError } from "../utils/errorHandling.js";
@@ -45,12 +46,24 @@ export class LiteLLMProvider extends OpenAIChatCompletionsProvider {
     static modelsCache = [];
     static modelsCacheTime = 0;
     static MODELS_CACHE_DURATION = 10 * 60 * 1000; // 10 minutes
+    /**
+     * Dedupes the fire-and-forget `/model/info` discovery per base URL, so a
+     * process constructing many LiteLLM providers fetches each proxy's limits
+     * once per cache period.
+     */
+    static modelInfoFetchTime = new Map();
     constructor(modelName, sdk, _region, credentials) {
         const envConfig = getLiteLLMConfig();
         super("litellm", modelName, sdk, {
             baseURL: credentials?.baseURL ?? envConfig.baseURL,
             apiKey: credentials?.apiKey ?? envConfig.apiKey,
         });
+        // Fire-and-forget: discover real per-model context windows from the
+        // proxy's /model/info. The static table only has a one-size litellm
+        // `_default` (128K), while proxied models range from 8K to 2M — budget
+        // checks and compaction need the real window. Failures degrade cleanly
+        // to the static default.
+        this.discoverModelContextWindows();
         logger.debug("LiteLLM Provider initialized", {
             modelName: this.modelName,
             provider: this.providerName,
@@ -208,6 +221,85 @@ export class LiteLLMProvider extends OpenAIChatCompletionsProvider {
         }
         return this.getFallbackModels();
     }
+    /**
+     * Discover real per-model context windows from the LiteLLM proxy's
+     * `GET /model/info` (`data[].model_info.max_input_tokens`) and register
+     * them with the context-window resolver. Fire-and-forget with the same
+     * cache period as the models list; any failure (endpoint absent, auth,
+     * timeout) is logged at debug and leaves the static `_default` in force.
+     */
+    discoverModelContextWindows() {
+        const baseURL = stripTrailingSlash(this.config.baseURL);
+        const now = Date.now();
+        const lastFetch = LiteLLMProvider.modelInfoFetchTime.get(baseURL) ?? 0;
+        if (now - lastFetch < LiteLLMProvider.MODELS_CACHE_DURATION) {
+            return;
+        }
+        LiteLLMProvider.modelInfoFetchTime.set(baseURL, now);
+        void this.fetchModelInfoFromAPI()
+            .then((windows) => {
+            for (const [model, contextWindow] of windows) {
+                registerRuntimeContextWindow("litellm", model, contextWindow);
+            }
+            if (windows.size > 0) {
+                logger.debug("[LiteLLMProvider] Registered runtime context windows from /model/info", { baseURL: redactUrlCredentials(baseURL), models: windows.size });
+            }
+        })
+            .catch((error) => {
+            // Allow a retry before the cache period when discovery failed.
+            LiteLLMProvider.modelInfoFetchTime.delete(baseURL);
+            logger.debug("[LiteLLMProvider] /model/info discovery failed; static context-window defaults remain in force", { error: error instanceof Error ? error.message : String(error) });
+        });
+    }
+    /**
+     * Fetch `GET /model/info` and map model group name → max_input_tokens.
+     * Same fetch/auth/abort scaffolding as {@link fetchModelsFromAPI}.
+     */
+    async fetchModelInfoFromAPI() {
+        const infoUrl = `${stripTrailingSlash(this.config.baseURL)}/model/info`;
+        const proxyFetch = createProxyFetch();
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), 5000);
+        try {
+            const response = await proxyFetch(infoUrl, {
+                method: "GET",
+                headers: {
+                    Authorization: `Bearer ${this.config.apiKey}`,
+                    "Content-Type": "application/json",
+                },
+                signal: controller.signal,
+            });
+            if (!response.ok) {
+                throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+            }
+            const data = (await response.json());
+            const windows = new Map();
+            for (const entry of data.data ?? []) {
+                const model = entry?.model_name;
+                const maxInput = entry?.model_info?.max_input_tokens;
+                if (typeof model === "string" &&
+                    model.length > 0 &&
+                    typeof maxInput === "number" &&
+                    Number.isFinite(maxInput) &&
+                    maxInput > 0) {
+                    // A model group can appear once per underlying deployment; keep the
+                    // smallest advertised window so budgets are safe for every replica.
+                    const existing = windows.get(model);
+                    windows.set(model, existing === undefined ? maxInput : Math.min(existing, maxInput));
+                }
+            }
+            return windows;
+        }
+        catch (error) {
+            if (isAbortError(error)) {
+                throw new NetworkError("Request timed out after 5 seconds", this.providerName);
+            }
+            throw error;
+        }
+        finally {
+            clearTimeout(timeoutId);
+        }
+    }
     async fetchModelsFromAPI() {
         const modelsUrl = `${stripTrailingSlash(this.config.baseURL)}/v1/models`;
         const proxyFetch = createProxyFetch();

package/dist/lib/providers/openAI.d.ts

@@ -16,6 +16,12 @@ import { OpenAIChatCompletionsProvider } from "./openaiChatCompletionsBase.js";
  */
 export declare class OpenAIProvider extends OpenAIChatCompletionsProvider {
     constructor(modelName?: string, sdk?: unknown, _region?: string, credentials?: NeurolinkCredentials["openai"]);
+    /**
+     * OpenAI natively supports `response_format: json_schema` together with
+     * tool calling in one request, so structured output stays wire-enforced
+     * mid-loop instead of deferring to post-hoc coercion.
+     */
+    protected suppressResponseFormatWithTools(): boolean;
     protected getProviderName(): AIProviderName;
     protected getDefaultModel(): string;
     formatProviderError(error: unknown): Error;

package/dist/lib/providers/openAI.js

@@ -74,6 +74,14 @@ export class OpenAIProvider extends OpenAIChatCompletionsProvider {
     // ===========================================================================
     // Abstract hook implementations
     // ===========================================================================
+    /**
+     * OpenAI natively supports `response_format: json_schema` together with
+     * tool calling in one request, so structured output stays wire-enforced
+     * mid-loop instead of deferring to post-hoc coercion.
+     */
+    suppressResponseFormatWithTools() {
+        return false;
+    }
     getProviderName() {
         return AIProviderNameEnum.OPENAI;
     }

package/dist/lib/providers/openaiChatCompletionsBase.d.ts

@@ -62,6 +62,21 @@ export declare abstract class OpenAIChatCompletionsProvider extends BaseProvider
      * downgraded `json_schema` to `json_object`. Subclasses replicate that here.
      */
     protected adjustResponseFormat(rf: OpenAICompatResponseFormat | undefined, _modelId: string): OpenAICompatResponseFormat | undefined;
+    /**
+     * When true (default), `response_format` is NOT sent on requests that carry
+     * tools. The AI SDK sets responseFormat on EVERY step of a tool loop, and
+     * generic/proxy backends (LiteLLM→vllm/GLM, openai-compatible, local
+     * servers) may silently honor it over tool calling — answering with
+     * final-shape JSON on step 1 instead of running the agentic loop. No error
+     * is raised, so the runtime tools↔schema conflict detector cannot catch it.
+     * The schema is still enforced post-hoc (GenerationHandler coerces the final
+     * text against it) — the same contract as the Gemini tools↔schema exclusion.
+     * Mirrors the streaming path, which never sends response_format.
+     *
+     * Backends with first-party support for tools + json_schema in one request
+     * (OpenAI, Azure OpenAI) override this to false.
+     */
+    protected suppressResponseFormatWithTools(): boolean;
     /**
      * Hook to adjust the fully-built wire request body before it is sent, on
      * both the streaming and non-streaming paths. Default identity. Override for

package/dist/lib/providers/openaiChatCompletionsBase.js

@@ -76,6 +76,23 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
     adjustResponseFormat(rf, _modelId) {
         return rf;
     }
+    /**
+     * When true (default), `response_format` is NOT sent on requests that carry
+     * tools. The AI SDK sets responseFormat on EVERY step of a tool loop, and
+     * generic/proxy backends (LiteLLM→vllm/GLM, openai-compatible, local
+     * servers) may silently honor it over tool calling — answering with
+     * final-shape JSON on step 1 instead of running the agentic loop. No error
+     * is raised, so the runtime tools↔schema conflict detector cannot catch it.
+     * The schema is still enforced post-hoc (GenerationHandler coerces the final
+     * text against it) — the same contract as the Gemini tools↔schema exclusion.
+     * Mirrors the streaming path, which never sends response_format.
+     *
+     * Backends with first-party support for tools + json_schema in one request
+     * (OpenAI, Azure OpenAI) override this to false.
+     */
+    suppressResponseFormatWithTools() {
+        return true;
+    }
     /**
      * Hook to adjust the fully-built wire request body before it is sent, on
      * both the streaming and non-streaming paths. Default identity. Override for
@@ -221,6 +238,7 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
         const adjustResponseFormat = this.adjustResponseFormat.bind(this);
         const adjustRequestBody = this.adjustRequestBody.bind(this);
         const adjustBodyAfter400 = this.adjustBodyAfter400.bind(this);
+        const suppressResponseFormatWithTools = this.suppressResponseFormatWithTools.bind(this);
         const getTimeoutForOptions = (opts) => this.getTimeout((opts ?? {}));
         return {
             specificationVersion: "v3",
@@ -229,7 +247,9 @@ export class OpenAIChatCompletionsProvider extends BaseProvider {
             supportedUrls: {},
             doGenerate: async (options) => {
                 const baseMessages = messageBuilderToOpenAI(options.prompt);
-                const responseFormat = options.responseFormat
+                const hasTools = Array.isArray(options.tools) && options.tools.length > 0;
+                const responseFormat = options.responseFormat &&
+                    !(hasTools && suppressResponseFormatWithTools())
                     ? adjustResponseFormat(v3ResponseFormatToOpenAI(options.responseFormat), modelId)
                     : undefined;
                 // ensureJsonWordInBody runs LAST — on the body after adjustRequestBody —

package/dist/providers/azureOpenai.d.ts

@@ -22,6 +22,12 @@ export declare class AzureOpenAIProvider extends OpenAIChatCompletionsProvider {
     protected readonly azureDeploymentPathPrefix: string;
     protected readonly useMaxCompletionTokensOverride?: boolean;
     constructor(modelName?: string, sdk?: unknown, _region?: string, credentials?: NeurolinkCredentials["azure"]);
+    /**
+     * Azure OpenAI natively supports `response_format: json_schema` together
+     * with tool calling in one request, so structured output stays
+     * wire-enforced mid-loop instead of deferring to post-hoc coercion.
+     */
+    protected suppressResponseFormatWithTools(): boolean;
     protected getProviderName(): AIProviderName;
     /**
      * The "default model" for Azure is the deployment name — it's the

package/dist/providers/azureOpenai.js

@@ -139,6 +139,14 @@ export class AzureOpenAIProvider extends OpenAIChatCompletionsProvider {
     // ===========================================================================
     // Abstract-hook implementations
     // ===========================================================================
+    /**
+     * Azure OpenAI natively supports `response_format: json_schema` together
+     * with tool calling in one request, so structured output stays
+     * wire-enforced mid-loop instead of deferring to post-hoc coercion.
+     */
+    suppressResponseFormatWithTools() {
+        return false;
+    }
     getProviderName() {
         return "azure";
     }

package/dist/providers/litellm.d.ts

@@ -17,6 +17,12 @@ export declare class LiteLLMProvider extends OpenAIChatCompletionsProvider {
     private static modelsCache;
     private static modelsCacheTime;
     private static readonly MODELS_CACHE_DURATION;
+    /**
+     * Dedupes the fire-and-forget `/model/info` discovery per base URL, so a
+     * process constructing many LiteLLM providers fetches each proxy's limits
+     * once per cache period.
+     */
+    private static modelInfoFetchTime;
     constructor(modelName?: string, sdk?: unknown, _region?: string, credentials?: {
         apiKey?: string;
         baseURL?: string;
@@ -44,6 +50,19 @@ export declare class LiteLLMProvider extends OpenAIChatCompletionsProvider {
      * minimal safe default if the API fetch fails.
      */
     getAvailableModels(): Promise<string[]>;
+    /**
+     * Discover real per-model context windows from the LiteLLM proxy's
+     * `GET /model/info` (`data[].model_info.max_input_tokens`) and register
+     * them with the context-window resolver. Fire-and-forget with the same
+     * cache period as the models list; any failure (endpoint absent, auth,
+     * timeout) is logged at debug and leaves the static `_default` in force.
+     */
+    private discoverModelContextWindows;
+    /**
+     * Fetch `GET /model/info` and map model group name → max_input_tokens.
+     * Same fetch/auth/abort scaffolding as {@link fetchModelsFromAPI}.
+     */
+    private fetchModelInfoFromAPI;
     private fetchModelsFromAPI;
     /**
      * Generate an embedding for a single text input via native /v1/embeddings.

package/dist/providers/litellm.js

@@ -1,4 +1,5 @@
 import { SpanKind, SpanStatusCode, trace } from "@opentelemetry/api";
+import { registerRuntimeContextWindow } from "../constants/contextWindows.js";
 import { createProxyFetch } from "../proxy/proxyFetch.js";
 import { AuthenticationError, InvalidModelError, ModelAccessDeniedError, NetworkError, ProviderError, RateLimitError, isModelAccessDeniedMessage, parseAllowedModels, } from "../types/index.js";
 import { isAbortError } from "../utils/errorHandling.js";
@@ -45,12 +46,24 @@ export class LiteLLMProvider extends OpenAIChatCompletionsProvider {
     static modelsCache = [];
     static modelsCacheTime = 0;
     static MODELS_CACHE_DURATION = 10 * 60 * 1000; // 10 minutes
+    /**
+     * Dedupes the fire-and-forget `/model/info` discovery per base URL, so a
+     * process constructing many LiteLLM providers fetches each proxy's limits
+     * once per cache period.
+     */
+    static modelInfoFetchTime = new Map();
     constructor(modelName, sdk, _region, credentials) {
         const envConfig = getLiteLLMConfig();
         super("litellm", modelName, sdk, {
             baseURL: credentials?.baseURL ?? envConfig.baseURL,
             apiKey: credentials?.apiKey ?? envConfig.apiKey,
         });
+        // Fire-and-forget: discover real per-model context windows from the
+        // proxy's /model/info. The static table only has a one-size litellm
+        // `_default` (128K), while proxied models range from 8K to 2M — budget
+        // checks and compaction need the real window. Failures degrade cleanly
+        // to the static default.
+        this.discoverModelContextWindows();
         logger.debug("LiteLLM Provider initialized", {
             modelName: this.modelName,
             provider: this.providerName,
@@ -208,6 +221,85 @@ export class LiteLLMProvider extends OpenAIChatCompletionsProvider {
         }
         return this.getFallbackModels();
     }
+    /**
+     * Discover real per-model context windows from the LiteLLM proxy's
+     * `GET /model/info` (`data[].model_info.max_input_tokens`) and register
+     * them with the context-window resolver. Fire-and-forget with the same
+     * cache period as the models list; any failure (endpoint absent, auth,
+     * timeout) is logged at debug and leaves the static `_default` in force.
+     */
+    discoverModelContextWindows() {
+        const baseURL = stripTrailingSlash(this.config.baseURL);
+        const now = Date.now();
+        const lastFetch = LiteLLMProvider.modelInfoFetchTime.get(baseURL) ?? 0;
+        if (now - lastFetch < LiteLLMProvider.MODELS_CACHE_DURATION) {
+            return;
+        }
+        LiteLLMProvider.modelInfoFetchTime.set(baseURL, now);
+        void this.fetchModelInfoFromAPI()
+            .then((windows) => {
+            for (const [model, contextWindow] of windows) {
+                registerRuntimeContextWindow("litellm", model, contextWindow);
+            }
+            if (windows.size > 0) {
+                logger.debug("[LiteLLMProvider] Registered runtime context windows from /model/info", { baseURL: redactUrlCredentials(baseURL), models: windows.size });
+            }
+        })
+            .catch((error) => {
+            // Allow a retry before the cache period when discovery failed.
+            LiteLLMProvider.modelInfoFetchTime.delete(baseURL);
+            logger.debug("[LiteLLMProvider] /model/info discovery failed; static context-window defaults remain in force", { error: error instanceof Error ? error.message : String(error) });
+        });
+    }
+    /**
+     * Fetch `GET /model/info` and map model group name → max_input_tokens.
+     * Same fetch/auth/abort scaffolding as {@link fetchModelsFromAPI}.
+     */
+    async fetchModelInfoFromAPI() {
+        const infoUrl = `${stripTrailingSlash(this.config.baseURL)}/model/info`;
+        const proxyFetch = createProxyFetch();
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), 5000);
+        try {
+            const response = await proxyFetch(infoUrl, {
+                method: "GET",
+                headers: {
+                    Authorization: `Bearer ${this.config.apiKey}`,
+                    "Content-Type": "application/json",
+                },
+                signal: controller.signal,
+            });
+            if (!response.ok) {
+                throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+            }
+            const data = (await response.json());
+            const windows = new Map();
+            for (const entry of data.data ?? []) {
+                const model = entry?.model_name;
+                const maxInput = entry?.model_info?.max_input_tokens;
+                if (typeof model === "string" &&
+                    model.length > 0 &&
+                    typeof maxInput === "number" &&
+                    Number.isFinite(maxInput) &&
+                    maxInput > 0) {
+                    // A model group can appear once per underlying deployment; keep the
+                    // smallest advertised window so budgets are safe for every replica.
+                    const existing = windows.get(model);
+                    windows.set(model, existing === undefined ? maxInput : Math.min(existing, maxInput));
+                }
+            }
+            return windows;
+        }
+        catch (error) {
+            if (isAbortError(error)) {
+                throw new NetworkError("Request timed out after 5 seconds", this.providerName);
+            }
+            throw error;
+        }
+        finally {
+            clearTimeout(timeoutId);
+        }
+    }
     async fetchModelsFromAPI() {
         const modelsUrl = `${stripTrailingSlash(this.config.baseURL)}/v1/models`;
         const proxyFetch = createProxyFetch();

package/dist/providers/openAI.d.ts

@@ -16,6 +16,12 @@ import { OpenAIChatCompletionsProvider } from "./openaiChatCompletionsBase.js";
  */
 export declare class OpenAIProvider extends OpenAIChatCompletionsProvider {
     constructor(modelName?: string, sdk?: unknown, _region?: string, credentials?: NeurolinkCredentials["openai"]);
+    /**
+     * OpenAI natively supports `response_format: json_schema` together with
+     * tool calling in one request, so structured output stays wire-enforced
+     * mid-loop instead of deferring to post-hoc coercion.
+     */
+    protected suppressResponseFormatWithTools(): boolean;
     protected getProviderName(): AIProviderName;
     protected getDefaultModel(): string;
     formatProviderError(error: unknown): Error;

package/dist/providers/openAI.js

@@ -74,6 +74,14 @@ export class OpenAIProvider extends OpenAIChatCompletionsProvider {
     // ===========================================================================
     // Abstract hook implementations
     // ===========================================================================
+    /**
+     * OpenAI natively supports `response_format: json_schema` together with
+     * tool calling in one request, so structured output stays wire-enforced
+     * mid-loop instead of deferring to post-hoc coercion.
+     */
+    suppressResponseFormatWithTools() {
+        return false;
+    }
     getProviderName() {
         return AIProviderNameEnum.OPENAI;
     }