@librechat/agents 3.3.0 → 3.3.2

package/src/llm/anthropic/llm.spec.ts

@@ -976,6 +976,7 @@ test('Anthropic message_delta usage preserves cumulative input totals', () => {
       output_tokens: 375,
       cache_creation_input_tokens: 11,
       cache_read_input_tokens: 13,
+      fallback_credit: null,
       server_tool_use: null,
       iterations: null,
       output_tokens_details: null,
@@ -1018,6 +1019,7 @@ test('Anthropic stream usage does not double-count cumulative input tokens', asy
           cache_creation: null,
           cache_creation_input_tokens: 20,
           cache_read_input_tokens: 30,
+          fallback_credit: null,
           inference_geo: null,
           input_tokens: 243,
           iterations: null,
@@ -1043,6 +1045,7 @@ test('Anthropic stream usage does not double-count cumulative input tokens', asy
         output_tokens: 375,
         cache_creation_input_tokens: 20,
         cache_read_input_tokens: 30,
+        fallback_credit: null,
         server_tool_use: null,
         iterations: null,
         output_tokens_details: null,
@@ -1089,6 +1092,7 @@ test('Anthropic stream usage accepts input first reported at message_delta', asy
           cache_creation: null,
           cache_creation_input_tokens: 0,
           cache_read_input_tokens: 0,
+          fallback_credit: null,
           inference_geo: null,
           input_tokens: 0,
           iterations: null,
@@ -1114,6 +1118,7 @@ test('Anthropic stream usage accepts input first reported at message_delta', asy
         output_tokens: 42,
         cache_creation_input_tokens: 500,
         cache_read_input_tokens: 200,
+        fallback_credit: null,
         server_tool_use: null,
         iterations: null,
         output_tokens_details: null,
@@ -1160,6 +1165,7 @@ test('Anthropic stream usage handles multiple cumulative message_delta events',
           cache_creation: null,
           cache_creation_input_tokens: 0,
           cache_read_input_tokens: 0,
+          fallback_credit: null,
           inference_geo: null,
           input_tokens: 243,
           iterations: null,
@@ -1185,6 +1191,7 @@ test('Anthropic stream usage handles multiple cumulative message_delta events',
         output_tokens: 100,
         cache_creation_input_tokens: 0,
         cache_read_input_tokens: 0,
+        fallback_credit: null,
         server_tool_use: null,
         iterations: null,
         output_tokens_details: null,
@@ -1204,6 +1211,7 @@ test('Anthropic stream usage handles multiple cumulative message_delta events',
         output_tokens: 375,
         cache_creation_input_tokens: 0,
         cache_read_input_tokens: 0,
+        fallback_credit: null,
         server_tool_use: null,
         iterations: null,
         output_tokens_details: null,

package/src/llm/bedrock/index.ts

@@ -26,12 +26,14 @@ import { AIMessageChunk } from '@langchain/core/messages';
 import { ChatGenerationChunk, ChatResult } from '@langchain/core/outputs';
 import {
   ConverseStreamCommand,
+  type ConverseStreamOutput,
   type GuardrailConfiguration,
   type GuardrailStreamConfiguration,
 } from '@aws-sdk/client-bedrock-runtime';
 import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
 import type { BaseMessage, ResponseMetadata } from '@langchain/core/messages';
 import type { ChatBedrockConverseInput } from '@langchain/aws';
+import type { ContentBlockDeltaEvent } from './types';
 import {
   convertToConverseMessages,
   createConverseToolUseStopChunk,
@@ -39,7 +41,6 @@ import {
   handleConverseStreamContentBlockDelta,
   handleConverseStreamMetadata,
 } from './utils';
-import type { ContentBlockDeltaEvent } from './types';
 import {
   resolveBedrockPromptCacheTtl,
   supportsBedrockToolCache,
@@ -102,6 +103,22 @@ function splitStreamToken(text: string): string[] {
   return chunks;
 }
 
+/**
+ * Resolves the text a delta contributes to the smoothing cadence, preferring a
+ * text delta over a reasoning delta and ignoring non-string payloads.
+ */
+function resolveVisibleText(text?: string, reasoningText?: string): string {
+  if (typeof text === 'string') {
+    return text;
+  }
+
+  if (typeof reasoningText === 'string') {
+    return reasoningText;
+  }
+
+  return '';
+}
+
 function getCadencedStreamDelay({
   targetDelay,
   lastVisibleContentAt,
@@ -282,10 +299,10 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
     const toolConfig =
       this.promptCache === true && supportsBedrockToolCache(this.cacheModelId)
         ? insertBedrockToolCachePoint(
-            baseParams.toolConfig,
-            true,
-            resolveBedrockPromptCacheTtl(this.promptCacheTtl, this.cacheModelId)
-          )
+          baseParams.toolConfig,
+          true,
+          resolveBedrockPromptCacheTtl(this.promptCacheTtl, this.cacheModelId)
+        )
         : baseParams.toolConfig;
 
     /** Service tier from options or fall back to class-level setting */
@@ -372,7 +389,8 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
         abortSignal: streamAbortController.signal,
       });
 
-      const stream = response.stream;
+      const stream: AsyncIterable<ConverseStreamOutput> | undefined =
+        response.stream;
       if (!stream) {
         return;
       }
@@ -525,12 +543,7 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
         const text = delta.text;
         const reasoningContent = delta.reasoningContent;
         const reasoningText = reasoningContent?.text;
-        const visibleText =
-          typeof text === 'string'
-            ? text
-            : typeof reasoningText === 'string'
-              ? reasoningText
-              : '';
+        const visibleText = resolveVisibleText(text, reasoningText);
         const smooth = this._lc_stream_delay > 0 && visibleText !== '';
         const tokenChunks = smooth
           ? splitStreamToken(visibleText)

package/src/llm/contextOverflowRecovery.ts

@@ -0,0 +1,292 @@
+/**
+ * Recovery policy for provider context-overflow rejections.
+ *
+ * Detection (`@/utils/errors`) answers "was this an overflow, and what did
+ * the provider disclose?". This module answers the follow-up: "what budget
+ * should the retry target?" — deliberately kept as pure functions so the
+ * policy can be reasoned about and tested without a graph.
+ *
+ * On units: `maxContextTokens` is a **provider-space** budget. The pruner
+ * converts it into its own raw estimate space by dividing by the
+ * `calibrationRatio` it learns from reported usage. A provider-reported
+ * ceiling is therefore applied verbatim — converting it here as well would
+ * apply the same correction twice and prune toward roughly `limit / ratio²`,
+ * silently discarding far more history than the overflow called for.
+ *
+ * `observedCalibrationRatio` is returned separately so the caller can seed
+ * the pruner's conversion without folding the same correction into the
+ * provider-space budget.
+ */
+import type { ContextOverflowInfo } from '@/utils/errors';
+import type { Providers } from '@/common';
+import { getContextOverflowInfo } from '@/utils/errors';
+
+/** Fraction of the previous budget used when the provider named no ceiling. */
+const BLIND_SHRINK_RATIO = 0.7;
+
+/** Slack left below a known ceiling so the retry is not sized to the edge. */
+const CEILING_HEADROOM_RATIO = 0.95;
+
+/**
+ * Fallback floor, used only when the instruction size is unknown. When it is
+ * known the floor is derived from it instead, so a genuinely small model — a
+ * 4k window, where a 95%-of-ceiling budget lands below this constant — can
+ * still recover.
+ */
+const MIN_RECOVERY_BUDGET_TOKENS = 4_000;
+
+/**
+ * Room a corrected budget must leave above the instructions for the messages
+ * themselves. Without it, a budget that merely clears the system prompt and
+ * tool schemas is not a budget anything can be compacted into.
+ */
+const MIN_MESSAGE_HEADROOM_TOKENS = 2_000;
+
+/** Bound on forced-compaction retries per agent, per run. */
+export const DEFAULT_MAX_OVERFLOW_RECOVERIES = 2;
+
+export interface OverflowRecoveryPlan {
+  /** Budget the retry should target, in provider token units when known. */
+  budgetTokens?: number;
+  /** What the provider disclosed. Carried through for logging. */
+  info: ContextOverflowInfo;
+  /**
+   * Provider-reported message tokens divided by our own message estimate,
+   * when both are known. Greater than 1 means we under-count relative to this
+   * provider.
+   *
+   * Returned separately so the graph can seed the pruner's calibration;
+   * applying it to this plan's budget as well would double-count. Fixed
+   * instruction overhead is removed before deriving it from
+   * `info.promptTokens`, never `info.requestedTokens`, since several providers
+   * fold the completion allowance into the latter.
+   */
+  observedCalibrationRatio?: number;
+}
+
+export interface OverflowRecoveryParams {
+  error: unknown;
+  provider: Providers;
+  /** Budget in force when the rejected prompt was built. */
+  maxContextTokens?: number;
+  /** Our own estimate of the prompt we actually sent. */
+  estimatedPromptTokens?: number;
+  /** Provider/local calibration already applied to the prompt estimate. */
+  calibrationRatio?: number;
+  /**
+   * System prompt plus tool schemas — the part of the budget compaction
+   * cannot touch. A corrected budget at or below this leaves no room for
+   * messages, and the summarize node refuses to run, so recovery is declined
+   * rather than entered.
+   */
+  instructionTokens?: number;
+  /** Whether a model-backed summary can compact messages without a pruner. */
+  canSummarize?: boolean;
+  /**
+   * Completion allowance the caller configured. Providers count it against
+   * the same ceiling, so it has to come off the top when the error itself did
+   * not break the total down.
+   */
+  configuredCompletionTokens?: number;
+  /** Recoveries already attempted for this agent in this run. */
+  attemptsSoFar: number;
+  maxAttempts?: number;
+}
+
+function isUsable(value: number | undefined): value is number {
+  return value != null && Number.isFinite(value) && value > 0;
+}
+
+/**
+ * Converts a provider-space retry budget into the units consumed by a pruner
+ * calibrated for another provider while preserving the same raw-token limit.
+ */
+export function translateRecoveryBudget(
+  budgetTokens: number | undefined,
+  sourceCalibrationRatio: number | undefined,
+  targetCalibrationRatio: number | undefined
+): number | undefined {
+  if (
+    !isUsable(budgetTokens) ||
+    !isUsable(sourceCalibrationRatio) ||
+    !isUsable(targetCalibrationRatio)
+  ) {
+    return budgetTokens;
+  }
+  return Math.floor(
+    (budgetTokens * targetCalibrationRatio) / sourceCalibrationRatio
+  );
+}
+
+/** Applies the conservative shrink used when no provider-space ceiling is usable. */
+export function getBlindRecoveryBudget(
+  maxContextTokens: number | undefined
+): number | undefined {
+  return isUsable(maxContextTokens)
+    ? Math.floor(maxContextTokens * BLIND_SHRINK_RATIO)
+    : undefined;
+}
+
+/**
+ * The completion allowance the provider counted against the same ceiling,
+ * when it reported both the total and the prompt portion. The retry budget
+ * governs the prompt only, so this has to come off the ceiling first —
+ * otherwise a large `maxTokens` keeps the request over the limit no matter
+ * how far the prompt is compacted.
+ */
+function reservedForCompletion(
+  info: ContextOverflowInfo,
+  configuredCompletionTokens: number | undefined
+): number {
+  if (isUsable(info.requestedTokens) && isUsable(info.promptTokens)) {
+    const difference = info.requestedTokens - info.promptTokens;
+    if (difference > 0) {
+      return difference;
+    }
+  }
+  /**
+   * No breakdown on offer. Fall back to what the caller configured, because
+   * the provider still counts it: targeting the whole ceiling would leave the
+   * retry at `prompt + maxTokens` and over the limit however far the prompt
+   * is compacted.
+   */
+  return isUsable(configuredCompletionTokens) ? configuredCompletionTokens : 0;
+}
+
+function resolveTargetBudget(
+  info: ContextOverflowInfo,
+  maxContextTokens: number | undefined,
+  estimatedPromptTokens: number | undefined,
+  configuredCompletionTokens: number | undefined
+): number | null {
+  if (isUsable(info.limitTokens)) {
+    const promptCeiling =
+      info.limitTokens -
+      reservedForCompletion(info, configuredCompletionTokens);
+    /**
+     * A completion allowance at or above the ceiling leaves nothing for the
+     * prompt: even an empty one plus the requested output overruns the limit.
+     * Compaction cannot fix that, so declining surfaces the real problem
+     * instead of burning the recovery budget on retries that must fail.
+     */
+    if (promptCeiling <= 0) {
+      return null;
+    }
+
+    return promptCeiling * CEILING_HEADROOM_RATIO;
+  }
+  if (isUsable(estimatedPromptTokens)) {
+    return estimatedPromptTokens * BLIND_SHRINK_RATIO;
+  }
+  if (isUsable(maxContextTokens)) {
+    return maxContextTokens * BLIND_SHRINK_RATIO;
+  }
+  return null;
+}
+
+/**
+ * Decides whether a failed model call is a recoverable context overflow and,
+ * if so, what budget the retry should be re-pruned against.
+ *
+ * Returns `null` when the error is something compaction cannot fix, or when
+ * the per-run recovery budget is spent — in both cases the caller should let
+ * its normal failure handling proceed.
+ */
+export function planContextOverflowRecovery({
+  error,
+  provider,
+  maxContextTokens,
+  estimatedPromptTokens,
+  calibrationRatio,
+  instructionTokens,
+  canSummarize = false,
+  configuredCompletionTokens,
+  attemptsSoFar,
+  maxAttempts = DEFAULT_MAX_OVERFLOW_RECOVERIES,
+}: OverflowRecoveryParams): OverflowRecoveryPlan | null {
+  if (attemptsSoFar >= maxAttempts) {
+    return null;
+  }
+
+  const info = getContextOverflowInfo(error, {
+    provider,
+    estimatedPromptTokens,
+    maxContextTokens,
+  });
+  if (info == null) {
+    return null;
+  }
+
+  const currentCalibrationRatio = isUsable(calibrationRatio)
+    ? calibrationRatio
+    : 1;
+  const estimatedMessageTokens =
+    isUsable(estimatedPromptTokens) && isUsable(instructionTokens)
+      ? estimatedPromptTokens - instructionTokens
+      : estimatedPromptTokens;
+  const observedMessageTokens =
+    isUsable(info.promptTokens) && isUsable(instructionTokens)
+      ? info.promptTokens - instructionTokens
+      : info.promptTokens;
+  const observedCalibrationRatio =
+    isUsable(observedMessageTokens) && isUsable(estimatedMessageTokens)
+      ? (observedMessageTokens / estimatedMessageTokens) *
+        currentCalibrationRatio
+      : undefined;
+
+  const target = resolveTargetBudget(
+    info,
+    maxContextTokens,
+    estimatedPromptTokens,
+    configuredCompletionTokens
+  );
+  if (target == null) {
+    const hasNumericBasis =
+      isUsable(info.limitTokens) ||
+      isUsable(maxContextTokens) ||
+      isUsable(estimatedPromptTokens);
+    return hasNumericBasis
+      ? null
+      : { info, observedCalibrationRatio, budgetTokens: undefined };
+  }
+
+  /**
+   * A retry that does not actually shrink the prompt would just reproduce the
+   * same rejection, so a ceiling that lands at or above the budget we already
+   * had is replaced by a blind shrink.
+   */
+  const bounded =
+    isUsable(maxContextTokens) && target >= maxContextTokens
+      ? (getBlindRecoveryBudget(maxContextTokens) ?? target)
+      : target;
+
+  const budgetTokens = Math.floor(bounded);
+
+  /**
+   * Below the floor there is no usable budget left: either nothing survives
+   * pruning, or the instructions alone fill the window, in which case the
+   * summarize node refuses to run and the detour would bounce between the
+   * agent and summarize nodes without ever shrinking the prompt. Declining
+   * lets the existing "instructions exceed context budget" guidance surface
+   * instead.
+   */
+  let floorTokens = MIN_RECOVERY_BUDGET_TOKENS;
+  if (isUsable(instructionTokens)) {
+    floorTokens = instructionTokens + MIN_MESSAGE_HEADROOM_TOKENS;
+  } else if (canSummarize) {
+    floorTokens = 1;
+  }
+  if (budgetTokens < floorTokens) {
+    return null;
+  }
+
+  /**
+   * Refuse to report a "recovery" that changes nothing: when the budget in
+   * force is already at or below the target, re-pruning cannot free space.
+   */
+  if (isUsable(maxContextTokens) && budgetTokens >= maxContextTokens) {
+    return null;
+  }
+
+  return { budgetTokens, info, observedCalibrationRatio };
+}

package/src/llm/invoke.ts

@@ -4,11 +4,13 @@ import type { RunnableConfig } from '@langchain/core/runnables';
 import type { ToolCall } from '@langchain/core/messages/tool';
 import type { BaseMessage } from '@langchain/core/messages';
 import type { ToolOutputReferenceRegistry } from '@/tools/toolOutputReferences';
+import type { ContextOverflowContext } from '@/utils/errors';
 import type * as t from '@/types';
 import { annotateMessagesForLLM } from '@/tools/toolOutputReferences';
 import { assertNotTruncatedToolCall } from '@/llm/truncation';
 import { Constants, GraphEvents, Providers } from '@/common';
 import { manualToolStreamProviders } from '@/llm/providers';
+import { getContextOverflowInfo } from '@/utils/errors';
 import { modifyDeltaProperties } from '@/messages';
 import { ChatModelStreamHandler } from '@/stream';
 import { initializeModel } from '@/llm/init';
@@ -312,6 +314,59 @@ export async function attemptInvoke(
   return { messages: [finalMessage] };
 }
 
+/**
+ * Identifies which fallback produced an error, so a caller planning a
+ * recovery can reason about the client that actually failed rather than the
+ * primary's configuration — their context windows and output allowances
+ * differ, which is the whole reason a fallback exists.
+ */
+export interface FallbackErrorContext {
+  provider: Providers;
+  clientOptions?: t.ClientOptions;
+  maxContextTokens?: number;
+}
+
+export interface FallbackOverflowCandidate {
+  error: unknown;
+  context: FallbackErrorContext;
+}
+
+const fallbackErrorContexts = new WeakMap<object, FallbackErrorContext>();
+const fallbackOverflowCandidates = new WeakMap<
+  object,
+  FallbackOverflowCandidate[]
+>();
+
+function attachFallbackErrorContext(
+  error: unknown,
+  fallbackContext: FallbackErrorContext
+): void {
+  if (typeof error !== 'object' || error === null) {
+    return;
+  }
+  fallbackErrorContexts.set(error, fallbackContext);
+}
+
+/** Reads back the fallback attribution attached by `tryFallbackProviders`. */
+export function getFallbackErrorContext(
+  error: unknown
+): FallbackErrorContext | undefined {
+  if (typeof error !== 'object' || error === null) {
+    return undefined;
+  }
+  return fallbackErrorContexts.get(error);
+}
+
+/** Returns every fallback overflow retained from an exhausted provider chain. */
+export function getFallbackOverflowCandidates(
+  error: unknown
+): FallbackOverflowCandidate[] {
+  if (typeof error !== 'object' || error === null) {
+    return [];
+  }
+  return [...(fallbackOverflowCandidates.get(error) ?? [])];
+}
+
 /**
  * Best-effort read of the configured model name from client options.
  * Providers disagree on the key (`model` vs `modelName`).
@@ -333,7 +388,12 @@ function extractClientOptionsModel(
 
 /**
  * Attempts each fallback provider in order until one succeeds.
- * Throws the last error if all fallbacks fail.
+ *
+ * When every fallback fails, a context overflow among them is thrown in
+ * preference to whichever failure happened to come last. An overflow is the
+ * one failure the caller can act on — it compacts and retries — and losing it
+ * behind a later unrelated error would surface a dead end instead. Ordinary
+ * failures still throw last-error-wins.
  */
 export async function tryFallbackProviders({
   fallbacks,
@@ -343,16 +403,38 @@ export async function tryFallbackProviders({
   primaryError,
   context,
   onChunk,
+  overflowContext,
 }: {
-  fallbacks: Array<{ provider: Providers; clientOptions?: t.ClientOptions }>;
+  fallbacks: t.FallbackConfig[];
   tools?: t.GraphTools;
   messages: BaseMessage[];
   config?: RunnableConfig;
   primaryError: unknown;
   context?: InvokeContext;
   onChunk?: OnChunk;
+  /**
+   * Prompt-size corroboration for signatures that are not self-describing.
+   * Vertex AI's overflow is a bare `400` with no reason, so without this a
+   * fallback that overflows is indistinguishable from any other 400 and would
+   * be dropped in favour of whichever failure came last.
+   */
+  overflowContext?: ContextOverflowContext;
 }): Promise<Partial<t.BaseGraphState> | undefined> {
+  const isOverflow = (
+    error: unknown,
+    contextOverride = overflowContext
+  ): boolean => getContextOverflowInfo(error, contextOverride) != null;
   let lastError: unknown = primaryError;
+  /**
+   * Tracked apart from the primary's overflow. A caller reaching this
+   * function with an overflowing primary has already failed to recover from
+   * it, so a fallback overflow — which may sit against a different window and
+   * output allowance — is the more useful of the two to surface.
+   */
+  const overflowCandidates: FallbackOverflowCandidate[] = [];
+  const primaryOverflowError: unknown = isOverflow(primaryError)
+    ? primaryError
+    : undefined;
   for (const fb of fallbacks) {
     try {
       const fbModel = initializeModel({
@@ -391,11 +473,44 @@ export async function tryFallbackProviders({
       return result;
     } catch (e) {
       lastError = e;
+      const fallbackOverflowContext: ContextOverflowContext = {
+        provider: fb.provider,
+        maxContextTokens: fb.maxContextTokens,
+        ...(overflowContext?.provider === fb.provider
+          ? {
+            estimatedPromptTokens: overflowContext.estimatedPromptTokens,
+          }
+          : {}),
+      };
+      if (isOverflow(e, fallbackOverflowContext)) {
+        const errorContext: FallbackErrorContext = {
+          provider: fb.provider,
+          clientOptions: fb.clientOptions,
+          maxContextTokens: fb.maxContextTokens,
+        };
+        attachFallbackErrorContext(e, errorContext);
+        overflowCandidates.push({ error: e, context: errorContext });
+      }
       continue;
     }
   }
-  if (lastError !== undefined) {
-    throw lastError;
+  /**
+   * Preference order: a fallback overflow, then the primary's overflow, then
+   * whichever failure came last. An overflow is the only one of the three a
+   * caller can act on, and the fallback's carries the client attribution that
+   * makes a correct retry budget possible.
+   */
+  const preferred =
+    overflowCandidates[0]?.error ?? primaryOverflowError ?? lastError;
+  if (
+    overflowCandidates.length > 0 &&
+    typeof preferred === 'object' &&
+    preferred !== null
+  ) {
+    fallbackOverflowCandidates.set(preferred, overflowCandidates);
+  }
+  if (preferred !== undefined) {
+    throw preferred;
   }
   return undefined;
 }