@librechat/agents 3.4.7 → 3.5.0

package/src/specs/context-accuracy.live.test.ts

@@ -1,409 +0,0 @@
-// src/specs/context-accuracy.live.test.ts
-/**
- * Live ACCURACY verification for ON_CONTEXT_USAGE against real provider
- * counts in the hard scenarios: tool loops (where calibration engages),
- * prompt caching (cache fields for cost math), and pruning under context
- * pressure (calibrated remaining-units math). Logs measured ratios.
- *
- * Run with:
- * RUN_CONTEXT_USAGE_LIVE_TESTS=1 ANTHROPIC_API_KEY=... npm test -- context-accuracy.live.test.ts --runInBand
- *
- * The google/bedrock matrix entries skip without GOOGLE_API_KEY /
- * BEDROCK_AWS_* creds; Bedrock's AWS SDK needs
- * NODE_OPTIONS='--experimental-vm-modules' under jest.
- */
-import { config as dotenvConfig } from 'dotenv';
-dotenvConfig();
-
-import { z } from 'zod';
-import { tool } from '@langchain/core/tools';
-import { AIMessage, HumanMessage } from '@langchain/core/messages';
-import { describe, expect, it, jest } from '@jest/globals';
-import type { BaseMessage } from '@langchain/core/messages';
-import type { RunnableConfig } from '@langchain/core/runnables';
-import type * as t from '@/types';
-import { createTokenCounter, TokenEncoderManager } from '@/utils/tokens';
-import { GraphEvents, Providers } from '@/common';
-import { ModelEndHandler } from '@/events';
-import { Run } from '@/run';
-
-const shouldRunLive =
-  process.env.RUN_CONTEXT_USAGE_LIVE_TESTS === '1' &&
-  process.env.ANTHROPIC_API_KEY != null &&
-  process.env.ANTHROPIC_API_KEY !== '';
-
-const describeIfLive = shouldRunLive ? describe : describe.skip;
-const modelName =
-  process.env.ANTHROPIC_CONTEXT_LIVE_MODEL ?? 'claude-haiku-4-5';
-
-function createStreamConfig(threadId: string): Partial<RunnableConfig> & {
-  version: 'v1' | 'v2';
-  streamMode: string;
-} {
-  return {
-    configurable: { thread_id: threadId },
-    streamMode: 'values',
-    version: 'v2',
-  };
-}
-
-interface Captured {
-  contextEvents: t.ContextUsageEvent[];
-  collectedUsage: Array<{
-    input_tokens?: number;
-    output_tokens?: number;
-    input_token_details?: { cache_creation?: number; cache_read?: number };
-  }>;
-  handlers: Record<string, t.EventHandler>;
-}
-
-function createCapture(): Captured {
-  const contextEvents: t.ContextUsageEvent[] = [];
-  const collectedUsage: Captured['collectedUsage'] = [];
-  const handlers: Record<string, t.EventHandler> = {
-    [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(collectedUsage as never),
-    [GraphEvents.ON_CONTEXT_USAGE]: {
-      handle: (_event, data): void => {
-        contextEvents.push(data as unknown as t.ContextUsageEvent);
-      },
-    },
-  };
-  return { contextEvents, collectedUsage, handlers };
-}
-
-function estimatedUsed(event: t.ContextUsageEvent): number {
-  return (event.contextBudget ?? 0) - (event.remainingContextTokens ?? 0);
-}
-
-const addTool = tool(
-  async ({ a, b }: { a: number; b: number }) => String(a + b),
-  {
-    name: 'add',
-    description: 'Add two numbers and return the sum',
-    schema: z.object({ a: z.number(), b: z.number() }),
-  }
-);
-
-/** ~5K tokens so the system prompt clears the haiku prompt-cache minimum;
- *  salted per run so the first call always writes a cold cache entry */
-function buildLongInstructions(salt: string): string {
-  return [
-    `Session ${salt}: you are a precise assistant. Use the add tool for any arithmetic, then reply with only the number.`,
-    ...Array.from(
-      { length: 200 },
-      (_, i) =>
-        `Rule ${i}: always answer precisely, verify arithmetic results twice, and keep every response to a single line without commentary.`
-    ),
-  ].join(' ');
-}
-
-/** Cross-provider accuracy matrix: different tokenizers and usage shapes */
-const providerMatrix: Array<{
-  name: string;
-  enabled: boolean;
-  llmConfig: Record<string, unknown>;
-}> = [
-  {
-    name: 'google',
-    enabled: !!process.env.GOOGLE_API_KEY,
-    llmConfig: {
-      provider: Providers.GOOGLE,
-      model: process.env.GOOGLE_CONTEXT_LIVE_MODEL ?? 'gemini-2.5-flash',
-      apiKey: process.env.GOOGLE_API_KEY,
-      temperature: 0,
-      streaming: true,
-      streamUsage: true,
-    },
-  },
-  {
-    name: 'bedrock',
-    enabled:
-      !!process.env.BEDROCK_AWS_ACCESS_KEY_ID &&
-      !!process.env.BEDROCK_AWS_SECRET_ACCESS_KEY,
-    llmConfig: {
-      provider: Providers.BEDROCK,
-      model:
-        process.env.BEDROCK_CONTEXT_LIVE_MODEL ??
-        'us.anthropic.claude-sonnet-4-6',
-      region:
-        process.env.BEDROCK_AWS_REGION ??
-        process.env.AWS_DEFAULT_REGION ??
-        'us-east-1',
-      credentials: {
-        accessKeyId: process.env.BEDROCK_AWS_ACCESS_KEY_ID,
-        secretAccessKey: process.env.BEDROCK_AWS_SECRET_ACCESS_KEY,
-      },
-      temperature: 0,
-      maxTokens: 128,
-      streaming: true,
-      streamUsage: true,
-    },
-  },
-];
-
-describeIfLive('Context accuracy live integration', () => {
-  jest.setTimeout(240_000);
-
-  let tokenCounter: t.TokenCounter;
-
-  beforeAll(async () => {
-    tokenCounter = await createTokenCounter();
-  });
-
-  afterAll(() => {
-    TokenEncoderManager.reset();
-  });
-
-  it('tracks provider counts through a cached tool loop and tightens with calibration', async () => {
-    const capture = createCapture();
-    const run = await Run.create<t.IState>({
-      runId: `ctx-acc-loop-${Date.now()}`,
-      graphConfig: {
-        type: 'standard',
-        llmConfig: {
-          provider: Providers.ANTHROPIC,
-          modelName,
-          apiKey: process.env.ANTHROPIC_API_KEY,
-          temperature: 0,
-          maxTokens: 128,
-          streaming: true,
-          streamUsage: true,
-          promptCache: true,
-        },
-        instructions: buildLongInstructions(`${Date.now()}`),
-        maxContextTokens: 16000,
-        tools: [addTool],
-      },
-      returnContent: true,
-      skipCleanup: true,
-      customHandlers: capture.handlers,
-      tokenCounter,
-      indexTokenCountMap: {},
-    });
-
-    await run.processStream(
-      {
-        messages: [
-          new HumanMessage(
-            'Use the add tool to compute 1742 + 2581, then reply with only the number.'
-          ),
-        ],
-      },
-      createStreamConfig(`ctx-acc-loop-${Date.now()}`)
-    );
-
-    expect(capture.contextEvents.length).toBeGreaterThanOrEqual(2);
-    expect(capture.collectedUsage.length).toBe(capture.contextEvents.length);
-
-    const ratios = capture.contextEvents.map((event, index) => {
-      const usage = capture.collectedUsage[index];
-      /** LangChain-normalized Anthropic usage reports cache fields as
-       *  subsets of input_tokens; only add them when genuinely additive
-       *  (same heuristic as calculateTotalTokens) */
-      const baseInput = usage.input_tokens ?? 0;
-      const cacheSum =
-        (usage.input_token_details?.cache_creation ?? 0) +
-        (usage.input_token_details?.cache_read ?? 0);
-      const providerInput = baseInput + (cacheSum > baseInput ? cacheSum : 0);
-      console.log(`[ctx-accuracy] call ${index + 1}:`, {
-        estimated: estimatedUsed(event),
-        providerInput,
-        providerParts: {
-          input: usage.input_tokens,
-          cacheWrite: usage.input_token_details?.cache_creation,
-          cacheRead: usage.input_token_details?.cache_read,
-        },
-        effectiveInstructionTokens: event.effectiveInstructionTokens,
-        systemMessageTokens: event.breakdown.systemMessageTokens,
-        toolSchemaTokens: event.breakdown.toolSchemaTokens,
-        messageTokens: event.breakdown.messageTokens,
-        remaining: event.remainingContextTokens,
-        budget: event.contextBudget,
-        calibrationRatio: event.calibrationRatio,
-      });
-      return estimatedUsed(event) / providerInput;
-    });
-    console.log('[ctx-accuracy] tool-loop estimate/provider ratios:', ratios);
-
-    /** Call 1: uncalibrated local estimate must be in the right ballpark */
-    expect(ratios[0]).toBeGreaterThan(0.5);
-    expect(ratios[0]).toBeLessThan(2);
-
-    /** Call 2+: calibration has real provider counts — tighter band, and
-     *  no further from truth than the uncalibrated call (noise epsilon) */
-    const last = ratios[ratios.length - 1];
-    expect(last).toBeGreaterThan(0.6);
-    expect(last).toBeLessThan(1.7);
-    expect(Math.abs(last - 1)).toBeLessThanOrEqual(
-      Math.abs(ratios[0] - 1) + 0.15
-    );
-
-    /** Prompt caching engaged: write on the first call, read on the next */
-    const cacheWrites = capture.collectedUsage.map(
-      (u) => u.input_token_details?.cache_creation ?? 0
-    );
-    const cacheReads = capture.collectedUsage.map(
-      (u) => u.input_token_details?.cache_read ?? 0
-    );
-    expect(Math.max(...cacheWrites)).toBeGreaterThan(0);
-    expect(Math.max(...cacheReads)).toBeGreaterThan(0);
-
-    const text = (run.getRunMessages() ?? [])
-      .filter((message) => message.getType() === 'ai')
-      .map((message) =>
-        typeof message.content === 'string'
-          ? message.content
-          : JSON.stringify(message.content)
-      )
-      .join(' ');
-    expect(text).toContain('4323');
-  });
-
-  for (const entry of providerMatrix) {
-    const itIfProvider = entry.enabled ? it : it.skip;
-    itIfProvider(
-      `${entry.name}: tool-loop estimates track provider counts across tokenizers`,
-      async () => {
-        const capture = createCapture();
-        const run = await Run.create<t.IState>({
-          runId: `ctx-acc-${entry.name}-${Date.now()}`,
-          graphConfig: {
-            type: 'standard',
-            llmConfig: entry.llmConfig as t.LLMConfig,
-            instructions:
-              'You are a precise assistant. Use the add tool for any arithmetic, then reply with only the number. ' +
-              Array.from(
-                { length: 40 },
-                (_, i) =>
-                  `Rule ${i}: answer precisely and keep responses to a single line.`
-              ).join(' '),
-            maxContextTokens: 16000,
-            tools: [addTool],
-          },
-          returnContent: true,
-          skipCleanup: true,
-          customHandlers: capture.handlers,
-          tokenCounter,
-          indexTokenCountMap: {},
-        });
-
-        await run.processStream(
-          {
-            messages: [
-              new HumanMessage(
-                'Use the add tool to compute 1742 + 2581, then reply with only the number.'
-              ),
-            ],
-          },
-          createStreamConfig(`ctx-acc-${entry.name}-${Date.now()}`)
-        );
-
-        expect(capture.contextEvents.length).toBeGreaterThanOrEqual(2);
-        expect(capture.collectedUsage.length).toBe(
-          capture.contextEvents.length
-        );
-
-        const ratios = capture.contextEvents.map((event, index) => {
-          const usage = capture.collectedUsage[index];
-          const baseInput = usage.input_tokens ?? 0;
-          const cacheSum =
-            (usage.input_token_details?.cache_creation ?? 0) +
-            (usage.input_token_details?.cache_read ?? 0);
-          const providerInput =
-            baseInput + (cacheSum > baseInput ? cacheSum : 0);
-          return estimatedUsed(event) / providerInput;
-        });
-        console.log(`[ctx-accuracy] ${entry.name} ratios:`, ratios, {
-          provider: capture.collectedUsage.map((u) => ({
-            input: u.input_tokens,
-            details: u.input_token_details,
-          })),
-        });
-
-        /** Foreign tokenizers diverge most on the uncalibrated first call */
-        expect(ratios[0]).toBeGreaterThan(0.3);
-        expect(ratios[0]).toBeLessThan(3);
-
-        /** Calibration has real counts from call 1 — tighter band */
-        const last = ratios[ratios.length - 1];
-        expect(last).toBeGreaterThan(0.5);
-        expect(last).toBeLessThan(2);
-        expect(Math.abs(last - 1)).toBeLessThanOrEqual(
-          Math.abs(ratios[0] - 1) + 0.2
-        );
-      }
-    );
-  }
-
-  it('stays accurate when pruning drops history under a small budget', async () => {
-    const capture = createCapture();
-    const history: BaseMessage[] = [];
-    for (let i = 0; i < 8; i++) {
-      history.push(
-        new HumanMessage(
-          `Question ${i}: please summarize the following filler passage. ` +
-            `${'The quick brown fox jumps over the lazy dog while counting tokens carefully. '.repeat(12)}`
-        ),
-        new AIMessage(
-          `Answer ${i}: the passage repeats a pangram about a fox and a dog while counting tokens. `.repeat(
-            4
-          )
-        )
-      );
-    }
-    history.push(new HumanMessage('Reply with exactly one word: pruned'));
-
-    const indexTokenCountMap: Record<string, number> = {};
-    for (let i = 0; i < history.length; i++) {
-      indexTokenCountMap[i] = tokenCounter(history[i]);
-    }
-
-    const run = await Run.create<t.IState>({
-      runId: `ctx-acc-prune-${Date.now()}`,
-      graphConfig: {
-        type: 'standard',
-        llmConfig: {
-          provider: Providers.ANTHROPIC,
-          modelName,
-          apiKey: process.env.ANTHROPIC_API_KEY,
-          temperature: 0,
-          maxTokens: 64,
-          streaming: true,
-          streamUsage: true,
-        },
-        instructions: 'You follow instructions exactly.',
-        maxContextTokens: 1500,
-      },
-      returnContent: true,
-      skipCleanup: true,
-      customHandlers: capture.handlers,
-      tokenCounter,
-      indexTokenCountMap,
-    });
-
-    await run.processStream(
-      { messages: history },
-      createStreamConfig(`ctx-acc-prune-${Date.now()}`)
-    );
-
-    expect(capture.contextEvents).toHaveLength(1);
-    const event = capture.contextEvents[0];
-    const usage = capture.collectedUsage[0];
-
-    /** Pruning engaged: the full history exceeds what was sent */
-    expect(event.prePruneContextTokens).toBeGreaterThan(estimatedUsed(event));
-    expect(event.remainingContextTokens).toBeGreaterThanOrEqual(0);
-    expect(event.remainingContextTokens).toBeLessThan(event.contextBudget ?? 0);
-
-    const providerInput = usage.input_tokens ?? 0;
-    const ratio = estimatedUsed(event) / providerInput;
-    console.log('[ctx-accuracy] pruned-run estimate/provider ratio:', ratio, {
-      estimated: estimatedUsed(event),
-      providerInput,
-      prePrune: event.prePruneContextTokens,
-      budget: event.contextBudget,
-    });
-    expect(ratio).toBeGreaterThan(0.4);
-    expect(ratio).toBeLessThan(2.5);
-  });
-});

package/src/specs/context-overflow-recovery.live.test.ts

@@ -1,213 +0,0 @@
-/**
- * Live end-to-end verification that a real provider rejection for an
- * over-limit prompt is absorbed by forced compaction instead of surfacing.
- *
- * Each case deliberately configures `maxContextTokens` far above the model's
- * real window, so the proactive pruner does not prevent the overflow and the
- * reactive recovery path is what gets exercised.
- *
- * Run with:
- * RUN_CONTEXT_OVERFLOW_LIVE_TESTS=1 OPENROUTER_API_KEY=... npm test -- context-overflow-recovery.live.test.ts --runInBand
- *
- * Cases self-skip when their provider credentials are absent. Models are the
- * smallest-window, cheapest option per provider — the prompt has to exceed
- * the window for the test to mean anything, so a large window is a large bill.
- *
- * The Bedrock case needs `NODE_OPTIONS='--experimental-vm-modules'`; the AWS
- * SDK uses a dynamic import that jest otherwise refuses.
- */
-import { config as dotenvConfig } from 'dotenv';
-dotenvConfig();
-
-import { HumanMessage } from '@langchain/core/messages';
-import { describe, expect, it, jest } from '@jest/globals';
-import type { BaseMessage } from '@langchain/core/messages';
-import type * as t from '@/types';
-import { createTokenCounter } from '@/utils/tokens';
-import { hasAnyEnv, hasEnv } from '@/specs/spec.utils';
-import { GraphEvents, Providers } from '@/common';
-import { Run } from '@/run';
-
-jest.setTimeout(300_000);
-
-const liveEnabled = process.env.RUN_CONTEXT_OVERFLOW_LIVE_TESTS === '1';
-
-interface LiveCase {
-  label: string;
-  provider: Providers;
-  model: string;
-  /** The model's real input window. */
-  contextWindow: number;
-  envKeys: readonly string[];
-  clientOptions: Record<string, unknown>;
-}
-
-const LIVE_CASES: readonly LiveCase[] = [
-  {
-    label: 'openrouter / qwen-2.5-7b (32k window)',
-    provider: Providers.OPENROUTER,
-    model: 'qwen/qwen-2.5-7b-instruct',
-    contextWindow: 32_768,
-    envKeys: ['OPENROUTER_API_KEY'],
-    clientOptions: {
-      apiKey: process.env.OPENROUTER_API_KEY,
-      configuration: {
-        baseURL:
-          process.env.OPENROUTER_BASE_URL ?? 'https://openrouter.ai/api/v1',
-      },
-    },
-  },
-  {
-    /**
-     * Google's text models all carry a ~1M window, which would make this the
-     * most expensive case in the file by an order of magnitude. This model
-     * accepts ordinary text and caps its input at 64k, so it exercises the
-     * same Gemini API rejection for a fraction of the tokens.
-     */
-    label: 'google / gemini-3.1-flash-image (64k window)',
-    provider: Providers.GOOGLE,
-    model: 'gemini-3.1-flash-image',
-    contextWindow: 65_536,
-    envKeys: ['GOOGLE_API_KEY', 'GEMINI_API_KEY'],
-    clientOptions: {
-      apiKey: process.env.GOOGLE_API_KEY ?? process.env.GEMINI_API_KEY,
-    },
-  },
-  {
-    label: 'anthropic / haiku (200k window)',
-    provider: Providers.ANTHROPIC,
-    model: 'claude-haiku-4-5-20251001',
-    contextWindow: 200_000,
-    envKeys: ['ANTHROPIC_API_KEY'],
-    clientOptions: { apiKey: process.env.ANTHROPIC_API_KEY },
-  },
-  {
-    label: 'bedrock / haiku (200k window)',
-    provider: Providers.BEDROCK,
-    model: 'us.anthropic.claude-haiku-4-5-20251001-v1:0',
-    contextWindow: 200_000,
-    envKeys: ['BEDROCK_AWS_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID'],
-    clientOptions: {
-      region:
-        process.env.BEDROCK_AWS_REGION ?? process.env.AWS_REGION ?? 'us-east-1',
-    },
-  },
-];
-
-/**
- * Single-token words, so an N-word message is at least N tokens under every
- * provider's tokenizer — the prompt must clear the window with certainty,
- * since a request that squeaks under it is billed in full and proves nothing.
- */
-const FILLER =
-  'the quick brown fox jumps over lazy dog and then runs past'.split(' ');
-
-function buildOversizedHistory(totalWords: number): BaseMessage[] {
-  const perMessage = Math.ceil(totalWords / 4);
-  const messages: BaseMessage[] = [];
-  for (let turn = 0; turn < 4; turn++) {
-    const words: string[] = new Array(perMessage);
-    for (let i = 0; i < perMessage; i++) {
-      words[i] = FILLER[(i + turn) % FILLER.length];
-    }
-    messages.push(new HumanMessage(`Notes part ${turn}: ${words.join(' ')}`));
-  }
-  messages.push(
-    new HumanMessage('In one short sentence, what were those notes about?')
-  );
-  return messages;
-}
-
-const describeIfLive = liveEnabled ? describe : describe.skip;
-
-describeIfLive('context overflow recovery (live)', () => {
-  for (const testCase of LIVE_CASES) {
-    const runnable = hasAnyEnv(testCase.envKeys);
-    const maybeIt = runnable ? it : it.skip;
-
-    maybeIt(
-      `recovers without surfacing an error — ${testCase.label}`,
-      async () => {
-        const summarizeEvents: t.SummarizeCompleteEvent[] = [];
-        const tokenCounter = await createTokenCounter();
-
-        const run = await Run.create<t.IState>({
-          runId: `overflow-live-${testCase.provider}-${Date.now()}`,
-          graphConfig: {
-            type: 'standard',
-            llmConfig: {
-              provider: testCase.provider,
-              ...testCase.clientOptions,
-              model: testCase.model,
-            } as t.LLMConfig,
-            /**
-             * Deliberately wrong, by a wide margin: the pruner will happily
-             * build a prompt the provider cannot accept, which is exactly the
-             * situation the recovery path exists for.
-             */
-            maxContextTokens: testCase.contextWindow * 2,
-            summarizationEnabled: true,
-            summarizationConfig: {
-              provider: testCase.provider,
-              model: testCase.model,
-            },
-          },
-          returnContent: true,
-          /**
-           * The assertions below read post-run `AgentContext` state, and the
-           * default cleanup path calls `clearHeavyState()` → `reset()`, which
-           * deliberately undoes the overflow correction for the next turn.
-           */
-          skipCleanup: true,
-          tokenCounter,
-          customHandlers: {
-            [GraphEvents.ON_SUMMARIZE_COMPLETE]: {
-              handle: (_event: string, data: t.StreamEventData): void => {
-                summarizeEvents.push(
-                  data as unknown as t.SummarizeCompleteEvent
-                );
-              },
-            },
-          },
-        });
-
-        const messages = buildOversizedHistory(
-          Math.ceil(testCase.contextWindow * 1.3)
-        );
-
-        const content = await run.processStream({ messages }, {
-          configurable: { thread_id: `overflow-live-${testCase.provider}` },
-          streamMode: 'values',
-          version: 'v2',
-        } as never);
-
-        expect(content).toBeDefined();
-        expect(Array.isArray(content)).toBe(true);
-        expect((content ?? []).length).toBeGreaterThan(0);
-
-        const agentContext = run.Graph?.agentContexts.get('default');
-        expect(agentContext?.overflowRecoveryAttempts).toBeGreaterThan(0);
-        /** The corrected budget must be below the fiction we started with. */
-        expect(agentContext?.maxContextTokens).toBeLessThan(
-          testCase.contextWindow * 2
-        );
-        /**
-         * No summarization assertion: the first recovery compacts by
-         * compressing tool output, so a run that fits after compression alone
-         * never spends a summarization call. Any summary event that does
-         * arrive must at least carry no error.
-         */
-        for (const event of summarizeEvents) {
-          expect(event.error).toBeUndefined();
-        }
-      }
-    );
-  }
-
-  it('has at least one runnable provider configured', () => {
-    const anyRunnable = LIVE_CASES.some((testCase) =>
-      testCase.envKeys.some(hasEnv)
-    );
-    expect(anyRunnable).toBe(true);
-  });
-});