@librechat/agents 2.2.5 → 2.2.7

package/src/messages/formatAgentMessages.tools.test.ts

@@ -0,0 +1,349 @@
+import { HumanMessage, AIMessage, SystemMessage, ToolMessage } from '@langchain/core/messages';
+import type { TPayload } from '@/types';
+import { formatAgentMessages } from './format';
+import { ContentTypes } from '@/common';
+
+describe('formatAgentMessages with tools parameter', () => {
+  it('should process messages normally when tools is not provided', () => {
+    const payload: TPayload = [
+      { role: 'user', content: 'Hello' },
+      {
+        role: 'assistant',
+        content: [
+          {
+            type: ContentTypes.TEXT,
+            [ContentTypes.TEXT]: 'Let me check that for you.',
+            tool_call_ids: ['123'],
+          },
+          {
+            type: ContentTypes.TOOL_CALL,
+            tool_call: {
+              id: '123',
+              name: 'search',
+              args: '{"query":"weather"}',
+              output: 'The weather is sunny.',
+            },
+          },
+        ],
+      },
+    ];
+    
+    const result = formatAgentMessages(payload);
+    
+    expect(result.messages).toHaveLength(3);
+    expect(result.messages[0]).toBeInstanceOf(HumanMessage);
+    expect(result.messages[1]).toBeInstanceOf(AIMessage);
+    expect(result.messages[2]).toBeInstanceOf(ToolMessage);
+    expect((result.messages[1] as AIMessage).tool_calls).toHaveLength(1);
+    expect((result.messages[2] as ToolMessage).tool_call_id).toBe('123');
+  });
+
+  it('should treat an empty tools set the same as disallowing all tools', () => {
+    const payload: TPayload = [
+      { role: 'user', content: 'What\'s the weather?' },
+      {
+        role: 'assistant',
+        content: [
+          {
+            type: ContentTypes.TEXT,
+            [ContentTypes.TEXT]: 'Let me check the weather for you.',
+            tool_call_ids: ['weather_1'],
+          },
+          {
+            type: ContentTypes.TOOL_CALL,
+            tool_call: {
+              id: 'weather_1',
+              name: 'check_weather',
+              args: '{"location":"New York"}',
+              output: 'Sunny, 75°F',
+            },
+          },
+        ],
+      },
+    ];
+    
+    // Provide an empty set of allowed tools
+    const allowedTools = new Set<string>();
+    
+    const result = formatAgentMessages(payload, undefined, allowedTools);
+    
+    // Should convert to a single AIMessage with string content
+    expect(result.messages).toHaveLength(2);
+    expect(result.messages[0]).toBeInstanceOf(HumanMessage);
+    expect(result.messages[1]).toBeInstanceOf(AIMessage);
+    
+    // The content should be a string representation of both messages
+    expect(typeof result.messages[1].content).toBe('string');
+    expect(result.messages[1].content).toEqual('AI: Let me check the weather for you.\nTool: check_weather, Sunny, 75°F');
+  });
+
+  it('should convert tool messages to string when tool is not in the allowed set', () => {
+    const payload: TPayload = [
+      { role: 'user', content: 'What\'s the weather?' },
+      {
+        role: 'assistant',
+        content: [
+          {
+            type: ContentTypes.TEXT,
+            [ContentTypes.TEXT]: 'Let me check the weather for you.',
+            tool_call_ids: ['weather_1'],
+          },
+          {
+            type: ContentTypes.TOOL_CALL,
+            tool_call: {
+              id: 'weather_1',
+              name: 'check_weather',
+              args: '{"location":"New York"}',
+              output: 'Sunny, 75°F',
+            },
+          },
+        ],
+      },
+    ];
+    
+    // Provide a set of allowed tools that doesn't include 'check_weather'
+    const allowedTools = new Set(['search', 'calculator']);
+    
+    const result = formatAgentMessages(payload, undefined, allowedTools);
+    
+    // Should convert to a single AIMessage with string content
+    expect(result.messages).toHaveLength(2);
+    expect(result.messages[0]).toBeInstanceOf(HumanMessage);
+    expect(result.messages[1]).toBeInstanceOf(AIMessage);
+    
+    // The content should be a string representation of both messages
+    expect(typeof result.messages[1].content).toBe('string');
+    expect(result.messages[1].content).toEqual('AI: Let me check the weather for you.\nTool: check_weather, Sunny, 75°F');
+  });
+
+  it('should not convert tool messages when tool is in the allowed set', () => {
+    const payload: TPayload = [
+      { role: 'user', content: 'What\'s the weather?' },
+      {
+        role: 'assistant',
+        content: [
+          {
+            type: ContentTypes.TEXT,
+            [ContentTypes.TEXT]: 'Let me check the weather for you.',
+            tool_call_ids: ['weather_1'],
+          },
+          {
+            type: ContentTypes.TOOL_CALL,
+            tool_call: {
+              id: 'weather_1',
+              name: 'check_weather',
+              args: '{"location":"New York"}',
+              output: 'Sunny, 75°F',
+            },
+          },
+        ],
+      },
+    ];
+    
+    // Provide a set of allowed tools that includes 'check_weather'
+    const allowedTools = new Set(['check_weather', 'search']);
+    
+    const result = formatAgentMessages(payload, undefined, allowedTools);
+    
+    // Should keep the original structure
+    expect(result.messages).toHaveLength(3);
+    expect(result.messages[0]).toBeInstanceOf(HumanMessage);
+    expect(result.messages[1]).toBeInstanceOf(AIMessage);
+    expect(result.messages[2]).toBeInstanceOf(ToolMessage);
+  });
+
+  it('should handle multiple tool calls with mixed allowed/disallowed tools', () => {
+    const payload: TPayload = [
+      { role: 'user', content: 'Tell me about the weather and calculate something' },
+      {
+        role: 'assistant',
+        content: [
+          {
+            type: ContentTypes.TEXT,
+            [ContentTypes.TEXT]: 'Let me check the weather first.',
+            tool_call_ids: ['weather_1'],
+          },
+          {
+            type: ContentTypes.TOOL_CALL,
+            tool_call: {
+              id: 'weather_1',
+              name: 'check_weather',
+              args: '{"location":"New York"}',
+              output: 'Sunny, 75°F',
+            },
+          },
+          {
+            type: ContentTypes.TEXT,
+            [ContentTypes.TEXT]: 'Now let me calculate something for you.',
+            tool_call_ids: ['calc_1'],
+          },
+          {
+            type: ContentTypes.TOOL_CALL,
+            tool_call: {
+              id: 'calc_1',
+              name: 'calculator',
+              args: '{"expression":"1+1"}',
+              output: '2',
+            },
+          },
+        ],
+      },
+    ];
+    
+    // Allow calculator but not check_weather
+    const allowedTools = new Set(['calculator', 'search']);
+    
+    const result = formatAgentMessages(payload, undefined, allowedTools);
+    
+    // Should convert the entire sequence to a single AIMessage
+    expect(result.messages).toHaveLength(2);
+    expect(result.messages[0]).toBeInstanceOf(HumanMessage);
+    expect(result.messages[1]).toBeInstanceOf(AIMessage);
+    
+    // The content should include all parts
+    expect(typeof result.messages[1].content).toBe('string');
+    expect(result.messages[1].content).toContain('Let me check the weather first.');
+    expect(result.messages[1].content).toContain('Sunny, 75°F');
+    expect(result.messages[1].content).toContain('Now let me calculate something for you.');
+    expect(result.messages[1].content).toContain('2');
+  });
+
+  it('should update indexTokenCountMap correctly when converting tool messages', () => {
+    const payload: TPayload = [
+      { role: 'user', content: 'What\'s the weather?' },
+      {
+        role: 'assistant',
+        content: [
+          {
+            type: ContentTypes.TEXT,
+            [ContentTypes.TEXT]: 'Let me check the weather for you.',
+            tool_call_ids: ['weather_1'],
+          },
+          {
+            type: ContentTypes.TOOL_CALL,
+            tool_call: {
+              id: 'weather_1',
+              name: 'check_weather',
+              args: '{"location":"New York"}',
+              output: 'Sunny, 75°F',
+            },
+          },
+        ],
+      },
+    ];
+    
+    const indexTokenCountMap = {
+      0: 10,  // 10 tokens for user message
+      1: 40,  // 40 tokens for assistant message with tool call
+    };
+    
+    // Provide a set of allowed tools that doesn't include 'check_weather'
+    const allowedTools = new Set(['search', 'calculator']);
+    
+    const result = formatAgentMessages(payload, indexTokenCountMap, allowedTools);
+    
+    // Should have 2 messages and 2 entries in the token count map
+    expect(result.messages).toHaveLength(2);
+    expect(Object.keys(result.indexTokenCountMap || {}).length).toBe(2);
+    
+    // User message token count should be unchanged
+    expect(result.indexTokenCountMap?.[0]).toBe(10);
+    
+    // All assistant message tokens should be assigned to the single AIMessage
+    expect(result.indexTokenCountMap?.[1]).toBe(40);
+  });
+
+  it('should handle complex sequences with multiple tool calls', () => {
+    const payload: TPayload = [
+      { role: 'user', content: 'Help me with a complex task' },
+      {
+        role: 'assistant',
+        content: [
+          {
+            type: ContentTypes.TEXT,
+            [ContentTypes.TEXT]: 'I\'ll search for information first.',
+            tool_call_ids: ['search_1'],
+          },
+          {
+            type: ContentTypes.TOOL_CALL,
+            tool_call: {
+              id: 'search_1',
+              name: 'search',
+              args: '{"query":"complex task"}',
+              output: 'Found information about complex tasks.',
+            },
+          },
+        ],
+      },
+      {
+        role: 'assistant',
+        content: [
+          {
+            type: ContentTypes.TEXT,
+            [ContentTypes.TEXT]: 'Now I\'ll check the weather.',
+            tool_call_ids: ['weather_1'],
+          },
+          {
+            type: ContentTypes.TOOL_CALL,
+            tool_call: {
+              id: 'weather_1',
+              name: 'check_weather',
+              args: '{"location":"New York"}',
+              output: 'Sunny, 75°F',
+            },
+          },
+        ],
+      },
+      {
+        role: 'assistant',
+        content: [
+          {
+            type: ContentTypes.TEXT,
+            [ContentTypes.TEXT]: 'Finally, I\'ll calculate something.',
+            tool_call_ids: ['calc_1'],
+          },
+          {
+            type: ContentTypes.TOOL_CALL,
+            tool_call: {
+              id: 'calc_1',
+              name: 'calculator',
+              args: '{"expression":"1+1"}',
+              output: '2',
+            },
+          },
+        ],
+      },
+      { role: 'assistant', content: 'Here\'s your answer based on all that information.' },
+    ];
+    
+    // Allow search and calculator but not check_weather
+    const allowedTools = new Set(['search', 'calculator']);
+    
+    const result = formatAgentMessages(payload, undefined, allowedTools);
+    
+    // Should have the user message, search tool sequence (2 messages),
+    // a combined message for weather and calculator (since one has an invalid tool),
+    // and final message
+    expect(result.messages).toHaveLength(5);
+    
+    // Check the types of messages
+    expect(result.messages[0]).toBeInstanceOf(HumanMessage);
+    expect(result.messages[1]).toBeInstanceOf(AIMessage);  // Search message
+    expect(result.messages[2]).toBeInstanceOf(ToolMessage); // Search tool response
+    expect(result.messages[3]).toBeInstanceOf(AIMessage); // Converted weather+calculator message
+    expect(result.messages[4]).toBeInstanceOf(AIMessage); // Final message
+    
+    // Check that the combined message was converted to a string
+    expect(typeof result.messages[3].content).toBe('string');
+    
+    // The format might vary based on the getBufferString implementation
+    // but we should check that all the key information is present
+    const content = result.messages[3].content as string;
+    expect(content).toContain('Now I\'ll check the weather');
+    expect(content).toContain('Sunny');
+    expect(content).toContain('75');
+    expect(content).toContain('Finally');
+    expect(content).toContain('calculate');
+    expect(content).toContain('2');
+  });
+});

package/src/scripts/image.ts

@@ -65,8 +65,8 @@ async function testCodeExecution(): Promise<void> {
     graphConfig: {
       type: 'standard',
       llmConfig,
-      // tools: [fetchRandomImageTool],
-      tools: [fetchRandomImageURL],
+      tools: [fetchRandomImageTool],
+      // tools: [fetchRandomImageURL],
       instructions: 'You are a friendly AI assistant with internet capabilities. Always address the user by their name.',
       additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
     },

package/src/specs/tool-error.test.ts

@@ -0,0 +1,156 @@
+/* eslint-disable no-console */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { z } from 'zod';
+import { config } from 'dotenv';
+config();
+import { tool } from '@langchain/core/tools';
+import { ToolCall } from '@langchain/core/messages/tool';
+import { HumanMessage, BaseMessage } from '@langchain/core/messages';
+import type { RunnableConfig } from '@langchain/core/runnables';
+import type * as t from '@/types';
+import { ChatModelStreamHandler, createContentAggregator } from '@/stream';
+import { ToolEndHandler, ModelEndHandler } from '@/events';
+import { GraphEvents, Providers } from '@/common';
+import { getLLMConfig } from '@/utils/llmConfig';
+import { getArgs } from '@/scripts/args';
+import { StandardGraph } from '@/graphs';
+import { Run } from '@/run';
+
+const errorTool = tool(
+  async () => {
+    throw new Error('this is a test error I threw on purpose');
+  },
+  {
+    name: 'errorTool',
+    description: 'A tool that always throws an error',
+    schema: z.object({ input: z.string().optional() }),
+  }
+);
+
+describe('Tool Error Handling Tests', () => {
+  jest.setTimeout(30000);
+  let run: Run<t.IState>;
+  let contentParts: t.MessageContentComplex[];
+  let conversationHistory: BaseMessage[];
+  let aggregateContent: t.ContentAggregator;
+  let handleToolCallErrorSpy: jest.SpyInstance;
+
+  const config: Partial<RunnableConfig> & { version: 'v1' | 'v2'; run_id?: string; streamMode: string } = {
+    configurable: {
+      thread_id: 'conversation-num-1',
+    },
+    streamMode: 'values',
+    version: 'v2' as const,
+  };
+
+  beforeEach(async () => {
+    conversationHistory = [];
+    const { contentParts: parts, aggregateContent: ac } = createContentAggregator();
+    aggregateContent = ac;
+    contentParts = parts as t.MessageContentComplex[];
+    handleToolCallErrorSpy = jest.spyOn(StandardGraph.prototype, 'handleToolCallError');
+  });
+
+  afterEach(() => {
+    handleToolCallErrorSpy.mockRestore();
+  });
+
+  const onMessageDeltaSpy = jest.fn();
+  const onRunStepSpy = jest.fn();
+  const onRunStepCompletedSpy = jest.fn();
+
+  afterAll(() => {
+    onMessageDeltaSpy.mockReset();
+    onRunStepSpy.mockReset();
+    onRunStepCompletedSpy.mockReset();
+  });
+
+  const setupCustomHandlers = (): Record<string | GraphEvents, t.EventHandler> => ({
+    [GraphEvents.TOOL_END]: new ToolEndHandler(),
+    [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(),
+    [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(),
+    [GraphEvents.ON_RUN_STEP_COMPLETED]: {
+      handle: (event: GraphEvents.ON_RUN_STEP_COMPLETED, data: t.StreamEventData): void => {
+        if ((data.result as t.MessageContentComplex)?.['type'] === 'tool_call') {
+          run.Graph?.overrideTestModel(['Looks like there was an error calling the tool.'], 5);
+        }
+        onRunStepCompletedSpy(event, data);
+        aggregateContent({ event, data: data as unknown as { result: t.ToolEndEvent; } });
+      }
+    },
+    [GraphEvents.ON_RUN_STEP]: {
+      handle: (event: GraphEvents.ON_RUN_STEP, data: t.StreamEventData, metadata, graph): void => {
+        const runStepData = data as t.RunStep;
+        onRunStepSpy(event, runStepData, metadata, graph);
+        aggregateContent({ event, data: runStepData });
+      }
+    },
+    [GraphEvents.ON_RUN_STEP_DELTA]: {
+      handle: (event: GraphEvents.ON_RUN_STEP_DELTA, data: t.StreamEventData): void => {
+        aggregateContent({ event, data: data as t.RunStepDeltaEvent });
+      }
+    },
+    [GraphEvents.ON_MESSAGE_DELTA]: {
+      handle: (event: GraphEvents.ON_MESSAGE_DELTA, data: t.StreamEventData, metadata, graph): void => {
+        onMessageDeltaSpy(event, data, metadata, graph);
+        aggregateContent({ event, data: data as t.MessageDeltaEvent });
+      }
+    },
+  });
+
+  test('should handle tool call errors correctly', async () => {
+    const { userName, location } = await getArgs();
+    const llmConfig = getLLMConfig(Providers.OPENAI);
+    const customHandlers = setupCustomHandlers();
+
+    // Create the run instance
+    run = await Run.create<t.IState>({
+      runId: 'test-run-id',
+      graphConfig: {
+        type: 'standard',
+        llmConfig,
+        tools: [errorTool],
+        instructions: 'You are a helpful AI assistant.',
+        additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
+      },
+      returnContent: true,
+      customHandlers,
+    });
+
+    const toolCalls: ToolCall[] = [
+        {
+          name: "errorTool",
+          args: {
+            input: "test input",
+          },
+          id: "call_test123",
+          type: "tool_call",
+        }
+      ];
+    
+    const firstResponse = 'Let me try calling the tool';
+    run.Graph?.overrideTestModel([firstResponse], 5, toolCalls);
+
+    const userMessage = 'Use the error tool';
+    conversationHistory.push(new HumanMessage(userMessage));
+
+    const inputs = {
+      messages: conversationHistory,
+    };
+
+    await run.processStream(inputs, config);
+
+    // Verify handleToolCallError was called
+    expect(handleToolCallErrorSpy).toHaveBeenCalled();
+    
+    // Find the tool call content part
+    const toolCallPart = contentParts.find(part => 
+      part?.type === 'tool_call'
+    ) as t.ToolCallContent | undefined;
+    
+    // Verify the error message in contentParts
+    expect(toolCallPart).toBeDefined();
+    expect(toolCallPart?.tool_call?.output).toContain('Error processing tool');
+    expect(toolCallPart?.tool_call?.output).toContain('this is a test error I threw on purpose');
+  });
+});

package/src/tools/ToolNode.ts

@@ -14,12 +14,14 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
   private loadRuntimeTools?: t.ToolRefGenerator;
   handleToolErrors = true;
   toolCallStepIds?: Map<string, string>;
+  errorHandler?: t.ToolNodeConstructorParams['errorHandler'];
 
   constructor({
     tools,
     toolMap,
     name,
     tags,
+    errorHandler,
     toolCallStepIds,
     handleToolErrors,
     loadRuntimeTools,
@@ -30,6 +32,7 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
     this.toolCallStepIds = toolCallStepIds;
     this.handleToolErrors = handleToolErrors ?? this.handleToolErrors;
     this.loadRuntimeTools = loadRuntimeTools;
+    this.errorHandler = errorHandler;
   }
 
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -83,6 +86,12 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
           if (isGraphInterrupt(e)) {
             throw e;
           }
+          this.errorHandler?.({
+            error: e,
+            id: call.id!,
+            name: call.name,
+            input: call.args,
+          }, config?.metadata);
           return new ToolMessage({
             content: `Error: ${e.message}\n Please fix your mistakes.`,
             name: call.name,

package/src/types/stream.ts

@@ -93,6 +93,11 @@ export type MessageCreationDetails = {
 };
 
 export type ToolEndData = { input: string | Record<string, unknown>, output?: ToolMessage };
+export type ToolErrorData = {
+  id: string,
+  name: string,
+  error?: Error,
+} & Pick<ToolEndData, 'input'>;
 export type ToolEndCallback = (data: ToolEndData, metadata?: Record<string, unknown>) => void;
 
 export type ProcessedToolCall = {
@@ -240,8 +245,35 @@ export type BedrockReasoningContentText = {
   reasoningText: { text?: string; signature?: string; }
 };
 
+/**
+ * A call to a tool.
+ */
+export type ToolCallPart = {
+  /** Type ("tool_call") according to Assistants Tool Call Structure */
+  type: ContentTypes.TOOL_CALL;
+  /** The name of the tool to be called */
+  name: string;
+  /** The arguments to the tool call */
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  args?: string | Record<string, any>;
+
+  /** If provided, an identifier associated with the tool call */
+  id?: string;
+  /** If provided, the output of the tool call */
+  output?: string;
+  /** Auth URL */
+  auth?: string;
+  /** Expiration time */
+  expires_at?: number;
+};
+
+export type ToolCallContent = {
+  type: ContentTypes.TOOL_CALL;
+  tool_call?: ToolCallPart;
+};
+
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-export type MessageContentComplex = (ThinkingContentText | AgentUpdate | ReasoningContentText | MessageContentText | MessageContentImageUrl | (Record<string, any> & {
+export type MessageContentComplex = (ThinkingContentText | AgentUpdate | ToolCallContent | ReasoningContentText | MessageContentText | MessageContentImageUrl | (Record<string, any> & {
   type?: 'text' | 'image_url' | 'think' | 'thinking' | string;
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 }) | (Record<string, any> & {
@@ -249,7 +281,14 @@ export type MessageContentComplex = (ThinkingContentText | AgentUpdate | Reasoni
 })) & {
   tool_call_ids?: string[];
 };
-// #new
+
+export interface TMessage {
+  role?: string;
+  content?: MessageContentComplex[] | string;
+  [key: string]: any;
+}
+
+export type TPayload = Array<Partial<TMessage>>;
 
 export type CustomChunk = Partial<OpenAITypes.ChatCompletionChunk> & {
   choices?: Partial<Array<Partial<OpenAITypes.Chat.Completions.ChatCompletionChunk.Choice> & {

package/src/types/tools.ts

@@ -2,7 +2,8 @@
 import type { RunnableToolLike } from '@langchain/core/runnables';
 import type { StructuredToolInterface } from '@langchain/core/tools';
 import type { ToolCall } from '@langchain/core/messages/tool';
-import { ContentTypes, EnvVar } from '@/common';
+import type { ToolErrorData } from './stream';
+import { EnvVar } from '@/common';
 
 /** Replacement type for `import type { ToolCall } from '@langchain/core/messages/tool'` in order to have stringified args typed */
 export type CustomToolCall = {
@@ -29,6 +30,7 @@ export type ToolNodeOptions = {
   handleToolErrors?: boolean;
   loadRuntimeTools?: ToolRefGenerator;
   toolCallStepIds?: Map<string, string>;
+  errorHandler?: (data: ToolErrorData, metadata?: Record<string, unknown>) => void
 };
 
 export type ToolNodeConstructorParams = ToolRefs & ToolNodeOptions;
@@ -42,11 +44,6 @@ export type ToolEndEvent = {
   index: number;
 };
 
-export type ToolCallContent = {
-  type: ContentTypes.TOOL_CALL;
-  tool_call: ToolCall;
-};
-
 export type CodeEnvFile = {
   id: string;
   name: string;