@librechat/agents 3.3.0 → 3.3.1

package/src/agents/AgentContext.ts

@@ -18,6 +18,13 @@ import {
   cloneMessage,
   type PromptCacheTtl,
 } from '@/messages/cache';
+import {
+  DEFAULT_RESERVE_RATIO,
+  ORIGINAL_CONTENT_MAX_CHARS,
+  clampCalibrationRatio,
+  createPruneMessages,
+  syncBudgetDerivedFields,
+} from '@/messages';
 import {
   ANTHROPIC_TOOL_TOKEN_MULTIPLIER,
   DEFAULT_TOOL_TOKEN_MULTIPLIER,
@@ -25,11 +32,6 @@ import {
   Constants,
   Providers,
 } from '@/common';
-import {
-  DEFAULT_RESERVE_RATIO,
-  createPruneMessages,
-  syncBudgetDerivedFields,
-} from '@/messages';
 import { createSchemaOnlyTools } from '@/tools/schema';
 import { apportionTokenCounts } from '@/utils/tokens';
 import { isThinkingEnabled } from '@/llm/request';
@@ -220,8 +222,22 @@ export class AgentContext {
   calibrationRatio: number = 1;
   /** Provider-observed instruction overhead from the pruner's best-variance turn. */
   resolvedInstructionOverhead?: number;
+  private _pendingOriginalToolContent?: Map<number, string>;
+  private pendingOriginalToolContentChars = 0;
   /** Pre-masking tool content keyed by message index, consumed by the summarize node. */
-  pendingOriginalToolContent?: Map<number, string>;
+  get pendingOriginalToolContent(): Map<number, string> | undefined {
+    return this._pendingOriginalToolContent;
+  }
+  set pendingOriginalToolContent(value: Map<number, string> | undefined) {
+    this._pendingOriginalToolContent = value;
+    this.pendingOriginalToolContentChars = 0;
+    if (value != null) {
+      for (const content of value.values()) {
+        this.pendingOriginalToolContentChars += content.length;
+      }
+      this.enforcePendingOriginalContentCap();
+    }
+  }
 
   /** Total instruction overhead: system message + tool schemas + pending summary. */
   get instructionTokens(): number {
@@ -322,6 +338,25 @@ export class AgentContext {
    * Summarization is allowed to fire again only when new messages appear.
    */
   private _lastSummarizationMsgCount: number = 0;
+  /**
+   * Forced compactions performed after a provider rejected a prompt as too
+   * large. Bounds the recovery loop so a model that keeps refusing cannot
+   * make the run compact indefinitely.
+   */
+  private _overflowRecoveryAttempts: number = 0;
+  /**
+   * Budget in force before the first overflow correction of the current run.
+   * Recorded so `reset()` can undo the correction for the next run without
+   * disturbing a `maxContextTokens` that no correction ever touched.
+   */
+  private _preOverflowMaxContextTokens?: number;
+  /**
+   * Prompt size, normalized into the local counter's uncalibrated units, at
+   * the last overflow correction. Keeping both measurements in the same units
+   * lets a later overflow prove whether compaction changed anything even when
+   * the provider observation updated calibration between attempts.
+   */
+  private _lastOverflowPromptTokens?: number;
   /**
    * Handoff context when this agent receives control via handoff.
    * Contains source and parallel execution info for system message context.
@@ -964,7 +999,7 @@ export class AgentContext {
   /**
    * Reset context for a new run
    */
-  reset(): void {
+  reset(options?: { preserveOriginalToolContent?: boolean }): void {
     this.systemMessageTokens = 0;
     this.dynamicInstructionTokens = 0;
     this.toolSchemaTokens = 0;
@@ -982,12 +1017,16 @@ export class AgentContext {
     this.currentTokenType = ContentTypes.TEXT;
     this.discoveredToolNames.clear();
     this.handoffContext = undefined;
+    if (options?.preserveOriginalToolContent !== true) {
+      this.pendingOriginalToolContent = undefined;
+    }
 
     this.summaryText = this._durableSummaryText;
     this.summaryTokenCount = this._durableSummaryTokenCount;
     this._lastSummarizationMsgCount = 0;
     this.lastCallUsage = undefined;
     this.totalTokensFresh = false;
+    this.restoreContextBudgetAfterOverflow();
 
     if (this.tokenCounter) {
       this.initializeSystemRunnable();
@@ -1308,6 +1347,148 @@ export class AgentContext {
     this._lastSummarizationMsgCount = msgCount;
   }
 
+  get overflowRecoveryAttempts(): number {
+    return this._overflowRecoveryAttempts;
+  }
+
+  shouldSummarizeOverflow(): boolean {
+    return (
+      this.summarizationEnabled === true &&
+      (this.tokenCounter == null ||
+        this.maxContextTokens == null ||
+        this._overflowRecoveryAttempts > 0)
+    );
+  }
+
+  /** Preserves the earliest full tool output recorded for each message index. */
+  preserveOriginalToolContent(
+    originalToolContent: Map<number, string> | undefined
+  ): void {
+    if (originalToolContent == null || originalToolContent.size === 0) {
+      return;
+    }
+    if (this.pendingOriginalToolContent == null) {
+      this.pendingOriginalToolContent = new Map();
+    }
+    for (const [index, content] of originalToolContent) {
+      if (!this.pendingOriginalToolContent.has(index)) {
+        this.pendingOriginalToolContent.set(index, content);
+        this.pendingOriginalToolContentChars += content.length;
+      }
+    }
+    this.enforcePendingOriginalContentCap();
+  }
+
+  private enforcePendingOriginalContentCap(): void {
+    const pending = this._pendingOriginalToolContent;
+    if (pending == null) {
+      return;
+    }
+    while (
+      this.pendingOriginalToolContentChars > ORIGINAL_CONTENT_MAX_CHARS &&
+      pending.size > 0
+    ) {
+      const oldest = pending.keys().next();
+      if (oldest.done === true) {
+        break;
+      }
+      const removed = pending.get(oldest.value);
+      if (removed != null) {
+        this.pendingOriginalToolContentChars -= removed.length;
+      }
+      pending.delete(oldest.value);
+    }
+  }
+
+  /**
+   * Retargets the context budget after a provider rejected the prompt as too
+   * large, and clears the memoized pruner so the next call is planned against
+   * the corrected budget rather than the one that was evidently wrong.
+   *
+   * Also clears the "already summarized at this message count" guard: that
+   * guard exists to stop redundant summarization of an unchanged history, but
+   * here the history has not changed and compaction is exactly what is
+   * needed.
+   */
+  applyContextBudgetCorrection(
+    budgetTokens: number | undefined,
+    promptTokens?: number
+  ): void {
+    if (this._overflowRecoveryAttempts === 0) {
+      this._preOverflowMaxContextTokens = this.maxContextTokens;
+    }
+    if (budgetTokens != null) {
+      this.maxContextTokens = budgetTokens;
+    }
+    this.pruneMessages = undefined;
+    this._lastSummarizationMsgCount = 0;
+    this._lastOverflowPromptTokens =
+      promptTokens != null
+        ? this.normalizePromptTokens(promptTokens)
+        : promptTokens;
+    this._overflowRecoveryAttempts += 1;
+  }
+
+  /** Applies token calibration only when the observation came from this provider. */
+  applyObservedOverflowCalibration(
+    provider: Providers | undefined,
+    observedCalibrationRatio: number | undefined
+  ): void {
+    if (
+      provider !== this.provider ||
+      observedCalibrationRatio == null ||
+      observedCalibrationRatio <= 0
+    ) {
+      return;
+    }
+    this.calibrationRatio = clampCalibrationRatio(observedCalibrationRatio);
+  }
+
+  /**
+   * True when a previous correction failed to make the prompt any smaller —
+   * the signature of a state nothing can compact further (an emptied message
+   * list carrying its content in an injected summary, for example). Retrying
+   * from there resends a byte-identical prompt, so the caller should stop.
+   */
+  overflowRecoveryStalled(currentPromptTokens?: number): boolean {
+    const previous = this._lastOverflowPromptTokens;
+    if (
+      previous == null ||
+      currentPromptTokens == null ||
+      !Number.isFinite(currentPromptTokens)
+    ) {
+      return false;
+    }
+    const rawCurrent = this.normalizePromptTokens(currentPromptTokens);
+    return rawCurrent >= previous;
+  }
+
+  private normalizePromptTokens(promptTokens: number): number {
+    if (this.calibrationRatio <= 0) {
+      return promptTokens;
+    }
+    const messageTokens = Math.max(0, promptTokens - this.instructionTokens);
+    return this.instructionTokens + messageTokens / this.calibrationRatio;
+  }
+
+  /**
+   * Undoes overflow corrections so a reused context starts the next run with
+   * the budget it was configured with and a fresh recovery allowance.
+   *
+   * Without this, a single overflow would permanently shrink the budget for
+   * every later turn, and two would exhaust the per-run allowance for the
+   * lifetime of the context.
+   */
+  private restoreContextBudgetAfterOverflow(): void {
+    if (this._overflowRecoveryAttempts === 0) {
+      return;
+    }
+    this.maxContextTokens = this._preOverflowMaxContextTokens;
+    this._preOverflowMaxContextTokens = undefined;
+    this._lastOverflowPromptTokens = undefined;
+    this._overflowRecoveryAttempts = 0;
+  }
+
   clearSummary(): void {
     if (this.summaryText != null) {
       this.summaryText = undefined;

package/src/agents/__tests__/AgentContext.overflow.test.ts

@@ -0,0 +1,205 @@
+import type * as t from '@/types';
+import { AgentContext } from '@/agents/AgentContext';
+import { Providers } from '@/common';
+
+/**
+ * The overflow-recovery bookkeeping on AgentContext: the budget correction,
+ * its restoration between runs, and the stall detector that stops a recovery
+ * loop when a correction demonstrably changed nothing.
+ */
+describe('AgentContext overflow recovery state', () => {
+  const createContext = (maxContextTokens?: number): AgentContext =>
+    AgentContext.fromConfig({
+      agentId: 'overflow-agent',
+      provider: Providers.ANTHROPIC,
+      instructions: 'Test instructions',
+      maxContextTokens,
+    } as Partial<t.AgentInputs> as t.AgentInputs);
+
+  it('records the correction and counts the attempt', () => {
+    const context = createContext(1_000_000);
+    context.applyContextBudgetCorrection(190_000, 274_468);
+
+    expect(context.maxContextTokens).toBe(190_000);
+    expect(context.overflowRecoveryAttempts).toBe(1);
+    /** Forces the pruner to be rebuilt against the corrected budget. */
+    expect(context.pruneMessages).toBeUndefined();
+  });
+
+  it('summarizes the first overflow when deterministic pruning is unavailable', () => {
+    const context = createContext(1_000_000);
+    context.summarizationEnabled = true;
+
+    expect(context.shouldSummarizeOverflow()).toBe(true);
+  });
+
+  it('summarizes immediately when no pruning budget is configured', () => {
+    const context = AgentContext.fromConfig(
+      {
+        agentId: 'overflow-agent',
+        provider: Providers.ANTHROPIC,
+        instructions: 'Test instructions',
+        summarizationEnabled: true,
+      } as Partial<t.AgentInputs> as t.AgentInputs,
+      () => 1
+    );
+
+    expect(context.shouldSummarizeOverflow()).toBe(true);
+  });
+
+  it('stages deterministic pruning before summarization when a counter exists', () => {
+    const context = AgentContext.fromConfig(
+      {
+        agentId: 'overflow-agent',
+        provider: Providers.ANTHROPIC,
+        instructions: 'Test instructions',
+        maxContextTokens: 1_000_000,
+        summarizationEnabled: true,
+      } as Partial<t.AgentInputs> as t.AgentInputs,
+      () => 1
+    );
+
+    expect(context.shouldSummarizeOverflow()).toBe(false);
+    context.applyContextBudgetCorrection(190_000, 274_468);
+    expect(context.shouldSummarizeOverflow()).toBe(true);
+  });
+
+  it('preserves the earliest full tool output when masking records collide', () => {
+    const context = createContext(1_000_000);
+    context.preserveOriginalToolContent(
+      new Map([
+        [2, 'full output'],
+        [4, 'another output'],
+      ])
+    );
+    context.preserveOriginalToolContent(
+      new Map([
+        [2, 'truncated placeholder'],
+        [6, 'new output'],
+      ])
+    );
+
+    expect(context.pendingOriginalToolContent).toEqual(
+      new Map([
+        [2, 'full output'],
+        [4, 'another output'],
+        [6, 'new output'],
+      ])
+    );
+  });
+
+  it('releases index-keyed tool output snapshots on reset', () => {
+    const context = createContext(1_000_000);
+    context.preserveOriginalToolContent(new Map([[2, 'full output']]));
+    context.reset();
+
+    expect(context.pendingOriginalToolContent).toBeUndefined();
+  });
+
+  it('preserves tool output snapshots when checkpointed messages survive reset', () => {
+    const context = createContext(1_000_000);
+    context.preserveOriginalToolContent(new Map([[2, 'full output']]));
+    context.reset({ preserveOriginalToolContent: true });
+
+    expect(context.pendingOriginalToolContent).toEqual(
+      new Map([[2, 'full output']])
+    );
+  });
+
+  it('restores the pre-correction budget on reset', () => {
+    const context = createContext(1_000_000);
+    context.applyContextBudgetCorrection(190_000, 274_468);
+    context.applyContextBudgetCorrection(133_000, 180_000);
+    context.reset();
+
+    expect(context.maxContextTokens).toBe(1_000_000);
+    expect(context.overflowRecoveryAttempts).toBe(0);
+  });
+
+  it('leaves an untouched budget alone on reset', () => {
+    const context = createContext(1_000_000);
+    context.maxContextTokens = 500_000;
+    context.reset();
+
+    expect(context.maxContextTokens).toBe(500_000);
+  });
+
+  it('keeps fallback calibration out of the primary agent context', () => {
+    const context = createContext(1_000_000);
+    context.calibrationRatio = 1.5;
+
+    context.applyObservedOverflowCalibration(Providers.VERTEXAI, 2);
+    expect(context.calibrationRatio).toBe(1.5);
+
+    context.applyObservedOverflowCalibration(Providers.ANTHROPIC, 2);
+    expect(context.calibrationRatio).toBe(2);
+  });
+
+  it('clamps provider-observed calibration to the shared safe range', () => {
+    const context = createContext(1_000_000);
+
+    context.applyObservedOverflowCalibration(Providers.ANTHROPIC, 10);
+    expect(context.calibrationRatio).toBe(5);
+
+    context.applyObservedOverflowCalibration(Providers.ANTHROPIC, 0.1);
+    expect(context.calibrationRatio).toBe(0.5);
+  });
+
+  it('records a summary-only recovery without inventing a token budget', () => {
+    const context = createContext();
+    context.applyContextBudgetCorrection(undefined, undefined);
+
+    expect(context.maxContextTokens).toBeUndefined();
+    expect(context.overflowRecoveryAttempts).toBe(1);
+  });
+
+  it('reports a stall when the prompt did not shrink', () => {
+    const context = createContext(1_000_000);
+    context.applyContextBudgetCorrection(190_000, 250_000);
+
+    expect(context.overflowRecoveryStalled(250_000)).toBe(true);
+    expect(context.overflowRecoveryStalled(260_000)).toBe(true);
+  });
+
+  it('reports no stall while the prompt is still shrinking', () => {
+    const context = createContext(1_000_000);
+    context.applyContextBudgetCorrection(190_000, 250_000);
+
+    expect(context.overflowRecoveryStalled(180_000)).toBe(false);
+  });
+
+  it('compares stall measurements in uncalibrated token units', () => {
+    const context = createContext(1_000_000);
+    context.applyContextBudgetCorrection(190_000, 250_000);
+    context.calibrationRatio = 2;
+
+    expect(context.overflowRecoveryStalled(360_000)).toBe(false);
+    expect(context.overflowRecoveryStalled(500_000)).toBe(true);
+  });
+
+  it('leaves fixed instruction overhead out of calibration normalization', () => {
+    const context = createContext(1_000_000);
+    context.systemMessageTokens = 100_000;
+    context.applyContextBudgetCorrection(190_000, 250_000);
+    context.calibrationRatio = 0.5;
+
+    expect(context.overflowRecoveryStalled(160_000)).toBe(false);
+    expect(context.overflowRecoveryStalled(175_000)).toBe(true);
+  });
+
+  it('reports no stall before any correction, or without a measurement', () => {
+    const context = createContext(1_000_000);
+    expect(context.overflowRecoveryStalled(250_000)).toBe(false);
+
+    context.applyContextBudgetCorrection(190_000, 250_000);
+    expect(context.overflowRecoveryStalled(undefined)).toBe(false);
+  });
+
+  it('clears the stall measurement on reset', () => {
+    const context = createContext(1_000_000);
+    context.applyContextBudgetCorrection(190_000, 250_000);
+    context.reset();
+
+    expect(context.overflowRecoveryStalled(250_000)).toBe(false);
+  });
+});

package/src/common/enum.ts

@@ -186,6 +186,8 @@ export enum Constants {
   WEB_SEARCH = 'web_search',
   CONTENT_AND_ARTIFACT = 'content_and_artifact',
   LC_TRANSFER_TO_ = 'lc_transfer_to_',
+  HANDOFF_PARALLEL_BATCH = '__handoff_parallel_batch',
+  HANDOFF_GROUP_ID = '__handoff_group_id',
   /** Delimiter for MCP tools: toolName_mcp_serverName */
   MCP_DELIMITER = '_mcp_',
   /** Anthropic server tool ID prefix (web_search, code_execution, etc.) */