@librechat/agents 3.4.6 → 3.5.0

package/src/specs/bedrock-truncation.live.test.ts

@@ -1,191 +0,0 @@
-/**
- * Live Bedrock tool-call truncation verification.
- *
- * Reproduces the production failure where a large tool argument (`content`)
- * gets cut off by the max output token limit, leaving the handler with a
- * well-formed-but-incomplete tool call and no truncation signal.
- *
- * Run with:
- * RUN_BEDROCK_TRUNCATION_LIVE=1 npm test -- bedrock-truncation.live.test.ts --runInBand
- */
-import { config as dotenvConfig } from 'dotenv';
-dotenvConfig();
-
-import { z } from 'zod';
-import { HumanMessage } from '@langchain/core/messages';
-import { DynamicStructuredTool } from '@langchain/core/tools';
-import { describe, expect, it, jest } from '@jest/globals';
-import type * as t from '@/types';
-import { GraphEvents, Providers } from '@/common';
-import { ChatModelStreamHandler } from '@/stream';
-import { ToolEndHandler, ModelEndHandler } from '@/events';
-import { Run } from '@/run';
-
-const shouldRunLive =
-  process.env.RUN_BEDROCK_TRUNCATION_LIVE === '1' &&
-  (process.env.BEDROCK_AWS_ACCESS_KEY_ID ?? '') !== '' &&
-  (process.env.BEDROCK_AWS_SECRET_ACCESS_KEY ?? '') !== '';
-
-const describeIfLive = shouldRunLive ? describe : describe.skip;
-
-const REGION =
-  [
-    process.env.BEDROCK_AWS_REGION,
-    process.env.BEDROCK_AWS_DEFAULT_REGION,
-    process.env.AWS_DEFAULT_REGION,
-  ].find(
-    (value): value is string => typeof value === 'string' && value !== ''
-  ) ?? 'us-east-1';
-const MODEL = 'us.anthropic.claude-sonnet-4-5-20250929-v1:0';
-const MAX_TOKENS = Number(process.env.REPRO_MAX_TOKENS ?? 350);
-
-const schema = z.object({
-  path: z.string().describe('Path to write.'),
-  content: z.string().describe('Complete file contents.'),
-  overwrite: z.boolean().optional(),
-});
-
-describeIfLive('Bedrock tool-call truncation (live)', () => {
-  jest.setTimeout(120_000);
-
-  it('fails fast with a truncation error instead of looping', async () => {
-    const received: Array<{ keys: string[]; contentLen: number | null }> = [];
-    const stepToolCalls: string[][] = [];
-    let invocations = 0;
-
-    const createFile = new DynamicStructuredTool({
-      name: 'create_file',
-      description: 'Create a new file. Requires path and content.',
-      schema,
-      func: async (input: z.infer<typeof schema>) => {
-        invocations += 1;
-        const hasContent =
-          typeof input.content === 'string' && input.content.length > 0;
-        received.push({
-          keys: Object.keys(input as Record<string, unknown>),
-          contentLen: hasContent ? input.content.length : null,
-        });
-        if (!hasContent) {
-          return 'Error: content is required\n Please fix your mistakes.';
-        }
-        return `Wrote ${input.content.length} bytes to ${input.path}`;
-      },
-    });
-
-    const llmConfig = {
-      provider: Providers.BEDROCK,
-      model: MODEL,
-      region: REGION,
-      credentials: {
-        accessKeyId: process.env.BEDROCK_AWS_ACCESS_KEY_ID!,
-        secretAccessKey: process.env.BEDROCK_AWS_SECRET_ACCESS_KEY!,
-      },
-      maxTokens: MAX_TOKENS,
-      streaming: true,
-      streamUsage: true,
-    } as t.LLMConfig;
-
-    const run = await Run.create<t.IState>({
-      runId: 'bedrock-truncation-live',
-      graphConfig: {
-        type: 'standard',
-        llmConfig,
-        tools: [createFile],
-        instructions: 'You are a ClickHouse assistant. Use tools when asked.',
-        maxContextTokens: 89000,
-      },
-      returnContent: true,
-      skipCleanup: true,
-      customHandlers: {
-        [GraphEvents.TOOL_END]: new ToolEndHandler(
-          async (toolEndData: t.ToolEndData) => {
-            const out = toolEndData.output as
-              | { content?: unknown; name?: string; status?: string }
-              | undefined;
-            const content =
-              typeof out?.content === 'string'
-                ? out.content.slice(0, 120)
-                : JSON.stringify(out?.content).slice(0, 120);
-            // eslint-disable-next-line no-console
-            console.log(
-              `TOOL_END name=${out?.name} status=${out?.status} content=${content}`
-            );
-          }
-        ),
-        [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(),
-        [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(),
-        [GraphEvents.ON_RUN_STEP]: {
-          handle: (_event: string, data: t.StreamEventData): void => {
-            const step = data as unknown as {
-              stepDetails?: {
-                type?: string;
-                tool_calls?: Array<{ name?: string; args?: unknown }>;
-              };
-            };
-            const tcs = step.stepDetails?.tool_calls;
-            if ((tcs?.length ?? 0) > 0 && tcs != null) {
-              for (const tc of tcs) {
-                if (tc.name === 'create_file') {
-                  stepToolCalls.push(
-                    typeof tc.args === 'string'
-                      ? ['<raw-string>']
-                      : Object.keys((tc.args ?? {}) as Record<string, unknown>)
-                  );
-                }
-              }
-              // eslint-disable-next-line no-console
-              console.log(
-                'ON_RUN_STEP tool_calls=',
-                JSON.stringify(
-                  tcs.map((tc) => ({
-                    name: tc.name,
-                    args:
-                      typeof tc.args === 'string'
-                        ? tc.args.slice(0, 80)
-                        : Object.keys(
-                            (tc.args ?? {}) as Record<string, unknown>
-                        ),
-                  }))
-                )
-              );
-            }
-          },
-        },
-      },
-    });
-
-    const prompt =
-      'Call create_file once: path="skills/clickhouse-demo/SKILL.md", content=a complete ' +
-      '3000-word ClickHouse guide (schema design, MergeTree, partitioning, 10 example queries). ' +
-      'Put the entire guide in the content argument.';
-
-    let loopError: string | null = null;
-    try {
-      await run.processStream(
-        { messages: [new HumanMessage(prompt)] },
-        {
-          configurable: { provider: Providers.BEDROCK, thread_id: 'trunc' },
-          version: 'v2',
-          recursionLimit: 8,
-        }
-      );
-    } catch (err) {
-      loopError = err instanceof Error ? err.message : String(err);
-    }
-
-    // eslint-disable-next-line no-console
-    console.log('LIVE RESULT', {
-      invocations,
-      received,
-      stepToolCalls,
-      loopError,
-    });
-
-    // The run fails fast with the actionable truncation error...
-    expect(loopError).toMatch(/truncated at the maximum output token limit/i);
-    // ...instead of looping until the recursion limit.
-    expect(loopError).not.toMatch(/Recursion limit/i);
-    // No truncated create_file call ever carried a usable `content` arg.
-    expect(stepToolCalls.every((keys) => !keys.includes('content'))).toBe(true);
-  });
-});

package/src/specs/cache.simple.test.ts

@@ -1,416 +0,0 @@
-/* eslint-disable no-console */
-/* eslint-disable @typescript-eslint/no-explicit-any */
-import { config } from 'dotenv';
-config();
-import { Calculator } from '@/tools/Calculator';
-import {
-  AIMessage,
-  BaseMessage,
-  HumanMessage,
-  UsageMetadata,
-} from '@langchain/core/messages';
-import type * as t from '@/types';
-import { ModelEndHandler, ToolEndHandler } from '@/events';
-import { capitalizeFirstLetter, hasEnv, hasEveryEnv } from './spec.utils';
-import { createContentAggregator } from '@/stream';
-import { GraphEvents, Providers } from '@/common';
-import { getLLMConfig } from '@/utils/llmConfig';
-import { getArgs } from '@/scripts/args';
-import { Run } from '@/run';
-
-/**
- * These tests verify that prompt caching works correctly across multi-turn
- * conversations and that messages are not mutated in place.
- */
-describe('Prompt Caching Integration Tests', () => {
-  jest.setTimeout(120000);
-
-  const setupTest = (): {
-    collectedUsage: UsageMetadata[];
-    contentParts: Array<t.MessageContentComplex | undefined>;
-    customHandlers: Record<string | GraphEvents, t.EventHandler>;
-  } => {
-    const collectedUsage: UsageMetadata[] = [];
-    const { contentParts, aggregateContent } = createContentAggregator();
-
-    const customHandlers: Record<string | GraphEvents, t.EventHandler> = {
-      [GraphEvents.TOOL_END]: new ToolEndHandler(),
-      [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(collectedUsage),
-      [GraphEvents.ON_RUN_STEP_COMPLETED]: {
-        handle: (
-          event: GraphEvents.ON_RUN_STEP_COMPLETED,
-          data: t.StreamEventData
-        ): void => {
-          aggregateContent({
-            event,
-            data: data as unknown as { result: t.ToolEndEvent },
-          });
-        },
-      },
-      [GraphEvents.ON_RUN_STEP]: {
-        handle: (
-          event: GraphEvents.ON_RUN_STEP,
-          data: t.StreamEventData
-        ): void => {
-          aggregateContent({ event, data: data as t.RunStep });
-        },
-      },
-      [GraphEvents.ON_RUN_STEP_DELTA]: {
-        handle: (
-          event: GraphEvents.ON_RUN_STEP_DELTA,
-          data: t.StreamEventData
-        ): void => {
-          aggregateContent({ event, data: data as t.RunStepDeltaEvent });
-        },
-      },
-      [GraphEvents.ON_MESSAGE_DELTA]: {
-        handle: (
-          event: GraphEvents.ON_MESSAGE_DELTA,
-          data: t.StreamEventData
-        ): void => {
-          aggregateContent({ event, data: data as t.MessageDeltaEvent });
-        },
-      },
-    };
-
-    return { collectedUsage, contentParts, customHandlers };
-  };
-
-  const streamConfig = {
-    configurable: { thread_id: 'cache-test-thread' },
-    streamMode: 'values',
-    version: 'v2' as const,
-  };
-
-  const requiredBedrockEnv = [
-    'BEDROCK_AWS_REGION',
-    'BEDROCK_AWS_ACCESS_KEY_ID',
-    'BEDROCK_AWS_SECRET_ACCESS_KEY',
-  ];
-  const hasAnthropicApiKey = hasEnv('ANTHROPIC_API_KEY');
-  const hasBedrockCredentials = hasEveryEnv(requiredBedrockEnv);
-
-  const describeIfAnthropic = hasAnthropicApiKey ? describe : describe.skip;
-  const describeIfBedrock = hasBedrockCredentials ? describe : describe.skip;
-  const describeIfAnthropicAndBedrock =
-    hasAnthropicApiKey && hasBedrockCredentials ? describe : describe.skip;
-
-  describeIfAnthropic('Anthropic Prompt Caching', () => {
-    const provider = Providers.ANTHROPIC;
-
-    test(`${capitalizeFirstLetter(provider)}: multi-turn conversation with caching should not corrupt messages`, async () => {
-      const { userName, location } = await getArgs();
-      const llmConfig = getLLMConfig(provider);
-      const { collectedUsage, customHandlers } = setupTest();
-
-      const run = await Run.create<t.IState>({
-        runId: 'cache-test-anthropic',
-        graphConfig: {
-          type: 'standard',
-          llmConfig: { ...llmConfig, promptCache: true } as t.LLMConfig,
-          tools: [new Calculator()],
-          instructions: 'You are a helpful assistant.',
-          additional_instructions: `User: ${userName}, Location: ${location}`,
-        },
-        returnContent: true,
-        skipCleanup: true,
-        customHandlers,
-      });
-
-      // Turn 1
-      const turn1Messages: BaseMessage[] = [
-        new HumanMessage('Hello, what is 2+2?'),
-      ];
-      const turn1ContentSnapshot = JSON.stringify(turn1Messages[0].content);
-
-      const turn1Result = await run.processStream(
-        { messages: turn1Messages },
-        streamConfig
-      );
-      expect(turn1Result).toBeDefined();
-
-      // Verify original message was NOT mutated
-      expect(JSON.stringify(turn1Messages[0].content)).toBe(
-        turn1ContentSnapshot
-      );
-      expect((turn1Messages[0] as any).content).not.toContain('cache_control');
-
-      const turn1RunMessages = run.getRunMessages();
-      expect(turn1RunMessages).toBeDefined();
-      expect(turn1RunMessages!.length).toBeGreaterThan(0);
-
-      // Turn 2 - build on conversation
-      const turn2Messages: BaseMessage[] = [
-        ...turn1Messages,
-        ...turn1RunMessages!,
-        new HumanMessage('Now multiply that by 10'),
-      ];
-      const turn2HumanContentSnapshot = JSON.stringify(
-        turn2Messages[turn2Messages.length - 1].content
-      );
-
-      const run2 = await Run.create<t.IState>({
-        runId: 'cache-test-anthropic-2',
-        graphConfig: {
-          type: 'standard',
-          llmConfig: { ...llmConfig, promptCache: true } as t.LLMConfig,
-          tools: [new Calculator()],
-          instructions: 'You are a helpful assistant.',
-          additional_instructions: `User: ${userName}, Location: ${location}`,
-        },
-        returnContent: true,
-        skipCleanup: true,
-        customHandlers,
-      });
-
-      const turn2Result = await run2.processStream(
-        { messages: turn2Messages },
-        streamConfig
-      );
-      expect(turn2Result).toBeDefined();
-
-      // Verify messages were NOT mutated
-      expect(
-        JSON.stringify(turn2Messages[turn2Messages.length - 1].content)
-      ).toBe(turn2HumanContentSnapshot);
-
-      // Check that we got cache read tokens (indicating caching worked)
-      console.log(`${provider} Usage:`, collectedUsage);
-      expect(collectedUsage.length).toBeGreaterThan(0);
-
-      console.log(
-        `${capitalizeFirstLetter(provider)} multi-turn caching test passed - messages not mutated`
-      );
-    });
-
-    test(`${capitalizeFirstLetter(provider)}: tool calls should work with caching enabled`, async () => {
-      const llmConfig = getLLMConfig(provider);
-      const { customHandlers } = setupTest();
-
-      const run = await Run.create<t.IState>({
-        runId: 'cache-test-anthropic-tools',
-        graphConfig: {
-          type: 'standard',
-          llmConfig: { ...llmConfig, promptCache: true } as t.LLMConfig,
-          tools: [new Calculator()],
-          instructions:
-            'You are a math assistant. Use the calculator tool for all calculations.',
-        },
-        returnContent: true,
-        skipCleanup: true,
-        customHandlers,
-      });
-
-      const messages: BaseMessage[] = [
-        new HumanMessage('Calculate 123 * 456 using the calculator'),
-      ];
-
-      const result = await run.processStream({ messages }, streamConfig);
-      expect(result).toBeDefined();
-
-      const runMessages = run.getRunMessages();
-      expect(runMessages).toBeDefined();
-
-      // Should have used the calculator tool
-      const hasToolUse = runMessages?.some(
-        (msg) =>
-          msg._getType() === 'ai' &&
-          ((msg as AIMessage).tool_calls?.length ?? 0) > 0
-      );
-      expect(hasToolUse).toBe(true);
-
-      console.log(
-        `${capitalizeFirstLetter(provider)} tool call with caching test passed`
-      );
-    });
-  });
-
-  describeIfBedrock('Bedrock Prompt Caching', () => {
-    const provider = Providers.BEDROCK;
-
-    test(`${capitalizeFirstLetter(provider)}: multi-turn conversation with caching should not corrupt messages`, async () => {
-      const { userName, location } = await getArgs();
-      const llmConfig = getLLMConfig(provider);
-      const { collectedUsage, customHandlers } = setupTest();
-
-      const run = await Run.create<t.IState>({
-        runId: 'cache-test-bedrock',
-        graphConfig: {
-          type: 'standard',
-          llmConfig: { ...llmConfig, promptCache: true } as t.LLMConfig,
-          tools: [new Calculator()],
-          instructions: 'You are a helpful assistant.',
-          additional_instructions: `User: ${userName}, Location: ${location}`,
-        },
-        returnContent: true,
-        skipCleanup: true,
-        customHandlers,
-      });
-
-      // Turn 1
-      const turn1Messages: BaseMessage[] = [
-        new HumanMessage('Hello, what is 5+5?'),
-      ];
-      const turn1ContentSnapshot = JSON.stringify(turn1Messages[0].content);
-
-      const turn1Result = await run.processStream(
-        { messages: turn1Messages },
-        streamConfig
-      );
-      expect(turn1Result).toBeDefined();
-
-      // Verify original message was NOT mutated
-      expect(JSON.stringify(turn1Messages[0].content)).toBe(
-        turn1ContentSnapshot
-      );
-
-      const turn1RunMessages = run.getRunMessages();
-      expect(turn1RunMessages).toBeDefined();
-      expect(turn1RunMessages!.length).toBeGreaterThan(0);
-
-      // Turn 2
-      const turn2Messages: BaseMessage[] = [
-        ...turn1Messages,
-        ...turn1RunMessages!,
-        new HumanMessage('Multiply that by 3'),
-      ];
-      const turn2HumanContentSnapshot = JSON.stringify(
-        turn2Messages[turn2Messages.length - 1].content
-      );
-
-      const run2 = await Run.create<t.IState>({
-        runId: 'cache-test-bedrock-2',
-        graphConfig: {
-          type: 'standard',
-          llmConfig: { ...llmConfig, promptCache: true } as t.LLMConfig,
-          tools: [new Calculator()],
-          instructions: 'You are a helpful assistant.',
-          additional_instructions: `User: ${userName}, Location: ${location}`,
-        },
-        returnContent: true,
-        skipCleanup: true,
-        customHandlers,
-      });
-
-      const turn2Result = await run2.processStream(
-        { messages: turn2Messages },
-        streamConfig
-      );
-      expect(turn2Result).toBeDefined();
-
-      // Verify messages were NOT mutated
-      expect(
-        JSON.stringify(turn2Messages[turn2Messages.length - 1].content)
-      ).toBe(turn2HumanContentSnapshot);
-
-      console.log(`${provider} Usage:`, collectedUsage);
-      expect(collectedUsage.length).toBeGreaterThan(0);
-
-      console.log(
-        `${capitalizeFirstLetter(provider)} multi-turn caching test passed - messages not mutated`
-      );
-    });
-
-    test(`${capitalizeFirstLetter(provider)}: tool calls should work with caching enabled`, async () => {
-      const llmConfig = getLLMConfig(provider);
-      const { customHandlers } = setupTest();
-
-      const run = await Run.create<t.IState>({
-        runId: 'cache-test-bedrock-tools',
-        graphConfig: {
-          type: 'standard',
-          llmConfig: { ...llmConfig, promptCache: true } as t.LLMConfig,
-          tools: [new Calculator()],
-          instructions:
-            'You are a math assistant. Use the calculator tool for all calculations.',
-        },
-        returnContent: true,
-        skipCleanup: true,
-        customHandlers,
-      });
-
-      const messages: BaseMessage[] = [
-        new HumanMessage('Calculate 789 * 123 using the calculator'),
-      ];
-
-      const result = await run.processStream({ messages }, streamConfig);
-      expect(result).toBeDefined();
-
-      const runMessages = run.getRunMessages();
-      expect(runMessages).toBeDefined();
-
-      // Should have used the calculator tool
-      const hasToolUse = runMessages?.some(
-        (msg) =>
-          msg._getType() === 'ai' &&
-          ((msg as AIMessage).tool_calls?.length ?? 0) > 0
-      );
-      expect(hasToolUse).toBe(true);
-
-      console.log(
-        `${capitalizeFirstLetter(provider)} tool call with caching test passed`
-      );
-    });
-  });
-
-  describeIfAnthropicAndBedrock('Cross-provider message isolation', () => {
-    test('Messages processed by Anthropic should not affect Bedrock processing', async () => {
-      const anthropicConfig = getLLMConfig(Providers.ANTHROPIC);
-      const bedrockConfig = getLLMConfig(Providers.BEDROCK);
-      const { customHandlers: handlers1 } = setupTest();
-      const { customHandlers: handlers2 } = setupTest();
-
-      // Create a shared message array
-      const sharedMessages: BaseMessage[] = [
-        new HumanMessage('Hello, what is the capital of France?'),
-      ];
-      const originalContent = JSON.stringify(sharedMessages[0].content);
-
-      // Process with Anthropic first
-      const anthropicRun = await Run.create<t.IState>({
-        runId: 'cross-provider-anthropic',
-        graphConfig: {
-          type: 'standard',
-          llmConfig: { ...anthropicConfig, promptCache: true } as t.LLMConfig,
-          instructions: 'You are a helpful assistant.',
-        },
-        returnContent: true,
-        skipCleanup: true,
-        customHandlers: handlers1,
-      });
-
-      const anthropicResult = await anthropicRun.processStream(
-        { messages: sharedMessages },
-        streamConfig
-      );
-      expect(anthropicResult).toBeDefined();
-
-      // Verify message not mutated
-      expect(JSON.stringify(sharedMessages[0].content)).toBe(originalContent);
-
-      // Now process with Bedrock using the SAME messages
-      const bedrockRun = await Run.create<t.IState>({
-        runId: 'cross-provider-bedrock',
-        graphConfig: {
-          type: 'standard',
-          llmConfig: { ...bedrockConfig, promptCache: true } as t.LLMConfig,
-          instructions: 'You are a helpful assistant.',
-        },
-        returnContent: true,
-        skipCleanup: true,
-        customHandlers: handlers2,
-      });
-
-      const bedrockResult = await bedrockRun.processStream(
-        { messages: sharedMessages },
-        streamConfig
-      );
-      expect(bedrockResult).toBeDefined();
-
-      // Verify message STILL not mutated after both providers processed
-      expect(JSON.stringify(sharedMessages[0].content)).toBe(originalContent);
-
-      console.log('Cross-provider message isolation test passed');
-    });
-  });
-});