@librechat/agents 2.2.6 → 2.2.7

package/src/llm/fake.ts

@@ -1,7 +1,9 @@
-import type { BaseMessage } from '@langchain/core/messages';
-import type { ChatGenerationChunk } from '@langchain/core/outputs';
+import { ChatGenerationChunk } from '@langchain/core/outputs';
+import { AIMessageChunk } from '@langchain/core/messages';
 import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
+import type { BaseMessage } from '@langchain/core/messages';
 import { FakeListChatModel } from '@langchain/core/utils/testing';
+import { ToolCall, ToolCallChunk } from '@langchain/core/messages/tool';
 
 type SplitStrategy = {
   type: 'regex' | 'fixed';
@@ -10,20 +12,25 @@ type SplitStrategy = {
 
 export class FakeChatModel extends FakeListChatModel {
   private splitStrategy: SplitStrategy;
+  private toolCalls: ToolCall[] = [];
+  private addedToolCalls: boolean = false;
 
   constructor({
     responses,
     sleep,
     emitCustomEvent,
-    splitStrategy = { type: 'regex', value: /(?<=\s+)|(?=\s+)/ }
+    splitStrategy = { type: 'regex', value: /(?<=\s+)|(?=\s+)/ },
+    toolCalls = []
   }: {
     responses: string[];
     sleep?: number;
     emitCustomEvent?: boolean;
     splitStrategy?: SplitStrategy;
+    toolCalls?: ToolCall[];
   }) {
     super({ responses, sleep, emitCustomEvent });
     this.splitStrategy = splitStrategy;
+    this.toolCalls = toolCalls;
   }
 
   private splitText(text: string): string[] {
@@ -38,6 +45,26 @@ export class FakeChatModel extends FakeListChatModel {
       return chunks;
     }
   }
+  _createResponseChunk(text: string, tool_call_chunks?: ToolCallChunk[]): ChatGenerationChunk {
+    return new ChatGenerationChunk({
+      text,
+      generationInfo: {},
+      message: new AIMessageChunk({
+        content: text,
+        tool_call_chunks,
+        additional_kwargs: tool_call_chunks ? {
+          tool_calls: tool_call_chunks.map((toolCall) => ({
+            index: toolCall.index ?? 0,
+            id: toolCall.id ?? '',
+            type: 'function',
+            function: {
+              name: toolCall.name ?? '',
+              arguments: toolCall.args ?? '',
+            },
+          })),
+        } : undefined,
+    })});
+  }
 
   async *_streamResponseChunks(
     _messages: BaseMessage[],
@@ -54,7 +81,6 @@ export class FakeChatModel extends FakeListChatModel {
     }
 
     const chunks = this.splitText(response);
-
     for await (const chunk of chunks) {
       await this._sleepIfRequested();
 
@@ -62,22 +88,46 @@ export class FakeChatModel extends FakeListChatModel {
         throw new Error(options.thrownErrorString);
       }
 
-      const responseChunk = this._createResponseChunk(chunk);
+      const responseChunk = super._createResponseChunk(chunk);
       yield responseChunk;
       void runManager?.handleLLMNewToken(chunk);
     }
+
+    await this._sleepIfRequested();
+    if (this.toolCalls.length > 0 && !this.addedToolCalls) {
+      this.addedToolCalls = true;
+      const toolCallChunks = this.toolCalls.map((toolCall) => {;
+        return {
+          name: toolCall.name,
+          args: JSON.stringify(toolCall.args),
+          id: toolCall.id,
+          type: 'tool_call_chunk',
+          } as ToolCallChunk
+      });
+      const responseChunk = this._createResponseChunk('', toolCallChunks);
+      yield responseChunk;
+      void runManager?.handleLLMNewToken('');
+    }
   }
 }
 
-export function createFakeStreamingLLM(
+export function createFakeStreamingLLM({
+  responses,
+  sleep,
+  splitStrategy,
+  toolCalls,
+} : {
   responses: string[],
   sleep?: number,
-  splitStrategy?: SplitStrategy
+  splitStrategy?: SplitStrategy,
+  toolCalls?: ToolCall[]
+}
 ): FakeChatModel {
   return new FakeChatModel({
     sleep,
     responses,
     emitCustomEvent: true,
     splitStrategy,
+    toolCalls,
   });
-}
+}

package/src/scripts/image.ts

@@ -65,8 +65,8 @@ async function testCodeExecution(): Promise<void> {
     graphConfig: {
       type: 'standard',
       llmConfig,
-      // tools: [fetchRandomImageTool],
-      tools: [fetchRandomImageURL],
+      tools: [fetchRandomImageTool],
+      // tools: [fetchRandomImageURL],
       instructions: 'You are a friendly AI assistant with internet capabilities. Always address the user by their name.',
       additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
     },

package/src/specs/tool-error.test.ts

@@ -0,0 +1,156 @@
+/* eslint-disable no-console */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { z } from 'zod';
+import { config } from 'dotenv';
+config();
+import { tool } from '@langchain/core/tools';
+import { ToolCall } from '@langchain/core/messages/tool';
+import { HumanMessage, BaseMessage } from '@langchain/core/messages';
+import type { RunnableConfig } from '@langchain/core/runnables';
+import type * as t from '@/types';
+import { ChatModelStreamHandler, createContentAggregator } from '@/stream';
+import { ToolEndHandler, ModelEndHandler } from '@/events';
+import { GraphEvents, Providers } from '@/common';
+import { getLLMConfig } from '@/utils/llmConfig';
+import { getArgs } from '@/scripts/args';
+import { StandardGraph } from '@/graphs';
+import { Run } from '@/run';
+
+const errorTool = tool(
+  async () => {
+    throw new Error('this is a test error I threw on purpose');
+  },
+  {
+    name: 'errorTool',
+    description: 'A tool that always throws an error',
+    schema: z.object({ input: z.string().optional() }),
+  }
+);
+
+describe('Tool Error Handling Tests', () => {
+  jest.setTimeout(30000);
+  let run: Run<t.IState>;
+  let contentParts: t.MessageContentComplex[];
+  let conversationHistory: BaseMessage[];
+  let aggregateContent: t.ContentAggregator;
+  let handleToolCallErrorSpy: jest.SpyInstance;
+
+  const config: Partial<RunnableConfig> & { version: 'v1' | 'v2'; run_id?: string; streamMode: string } = {
+    configurable: {
+      thread_id: 'conversation-num-1',
+    },
+    streamMode: 'values',
+    version: 'v2' as const,
+  };
+
+  beforeEach(async () => {
+    conversationHistory = [];
+    const { contentParts: parts, aggregateContent: ac } = createContentAggregator();
+    aggregateContent = ac;
+    contentParts = parts as t.MessageContentComplex[];
+    handleToolCallErrorSpy = jest.spyOn(StandardGraph.prototype, 'handleToolCallError');
+  });
+
+  afterEach(() => {
+    handleToolCallErrorSpy.mockRestore();
+  });
+
+  const onMessageDeltaSpy = jest.fn();
+  const onRunStepSpy = jest.fn();
+  const onRunStepCompletedSpy = jest.fn();
+
+  afterAll(() => {
+    onMessageDeltaSpy.mockReset();
+    onRunStepSpy.mockReset();
+    onRunStepCompletedSpy.mockReset();
+  });
+
+  const setupCustomHandlers = (): Record<string | GraphEvents, t.EventHandler> => ({
+    [GraphEvents.TOOL_END]: new ToolEndHandler(),
+    [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(),
+    [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(),
+    [GraphEvents.ON_RUN_STEP_COMPLETED]: {
+      handle: (event: GraphEvents.ON_RUN_STEP_COMPLETED, data: t.StreamEventData): void => {
+        if ((data.result as t.MessageContentComplex)?.['type'] === 'tool_call') {
+          run.Graph?.overrideTestModel(['Looks like there was an error calling the tool.'], 5);
+        }
+        onRunStepCompletedSpy(event, data);
+        aggregateContent({ event, data: data as unknown as { result: t.ToolEndEvent; } });
+      }
+    },
+    [GraphEvents.ON_RUN_STEP]: {
+      handle: (event: GraphEvents.ON_RUN_STEP, data: t.StreamEventData, metadata, graph): void => {
+        const runStepData = data as t.RunStep;
+        onRunStepSpy(event, runStepData, metadata, graph);
+        aggregateContent({ event, data: runStepData });
+      }
+    },
+    [GraphEvents.ON_RUN_STEP_DELTA]: {
+      handle: (event: GraphEvents.ON_RUN_STEP_DELTA, data: t.StreamEventData): void => {
+        aggregateContent({ event, data: data as t.RunStepDeltaEvent });
+      }
+    },
+    [GraphEvents.ON_MESSAGE_DELTA]: {
+      handle: (event: GraphEvents.ON_MESSAGE_DELTA, data: t.StreamEventData, metadata, graph): void => {
+        onMessageDeltaSpy(event, data, metadata, graph);
+        aggregateContent({ event, data: data as t.MessageDeltaEvent });
+      }
+    },
+  });
+
+  test('should handle tool call errors correctly', async () => {
+    const { userName, location } = await getArgs();
+    const llmConfig = getLLMConfig(Providers.OPENAI);
+    const customHandlers = setupCustomHandlers();
+
+    // Create the run instance
+    run = await Run.create<t.IState>({
+      runId: 'test-run-id',
+      graphConfig: {
+        type: 'standard',
+        llmConfig,
+        tools: [errorTool],
+        instructions: 'You are a helpful AI assistant.',
+        additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
+      },
+      returnContent: true,
+      customHandlers,
+    });
+
+    const toolCalls: ToolCall[] = [
+        {
+          name: "errorTool",
+          args: {
+            input: "test input",
+          },
+          id: "call_test123",
+          type: "tool_call",
+        }
+      ];
+    
+    const firstResponse = 'Let me try calling the tool';
+    run.Graph?.overrideTestModel([firstResponse], 5, toolCalls);
+
+    const userMessage = 'Use the error tool';
+    conversationHistory.push(new HumanMessage(userMessage));
+
+    const inputs = {
+      messages: conversationHistory,
+    };
+
+    await run.processStream(inputs, config);
+
+    // Verify handleToolCallError was called
+    expect(handleToolCallErrorSpy).toHaveBeenCalled();
+    
+    // Find the tool call content part
+    const toolCallPart = contentParts.find(part => 
+      part?.type === 'tool_call'
+    ) as t.ToolCallContent | undefined;
+    
+    // Verify the error message in contentParts
+    expect(toolCallPart).toBeDefined();
+    expect(toolCallPart?.tool_call?.output).toContain('Error processing tool');
+    expect(toolCallPart?.tool_call?.output).toContain('this is a test error I threw on purpose');
+  });
+});

package/src/tools/ToolNode.ts

@@ -14,12 +14,14 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
   private loadRuntimeTools?: t.ToolRefGenerator;
   handleToolErrors = true;
   toolCallStepIds?: Map<string, string>;
+  errorHandler?: t.ToolNodeConstructorParams['errorHandler'];
 
   constructor({
     tools,
     toolMap,
     name,
     tags,
+    errorHandler,
     toolCallStepIds,
     handleToolErrors,
     loadRuntimeTools,
@@ -30,6 +32,7 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
     this.toolCallStepIds = toolCallStepIds;
     this.handleToolErrors = handleToolErrors ?? this.handleToolErrors;
     this.loadRuntimeTools = loadRuntimeTools;
+    this.errorHandler = errorHandler;
   }
 
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -83,6 +86,12 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
           if (isGraphInterrupt(e)) {
             throw e;
           }
+          this.errorHandler?.({
+            error: e,
+            id: call.id!,
+            name: call.name,
+            input: call.args,
+          }, config?.metadata);
           return new ToolMessage({
             content: `Error: ${e.message}\n Please fix your mistakes.`,
             name: call.name,

package/src/types/stream.ts

@@ -93,6 +93,11 @@ export type MessageCreationDetails = {
 };
 
 export type ToolEndData = { input: string | Record<string, unknown>, output?: ToolMessage };
+export type ToolErrorData = {
+  id: string,
+  name: string,
+  error?: Error,
+} & Pick<ToolEndData, 'input'>;
 export type ToolEndCallback = (data: ToolEndData, metadata?: Record<string, unknown>) => void;
 
 export type ProcessedToolCall = {

package/src/types/tools.ts

@@ -2,7 +2,8 @@
 import type { RunnableToolLike } from '@langchain/core/runnables';
 import type { StructuredToolInterface } from '@langchain/core/tools';
 import type { ToolCall } from '@langchain/core/messages/tool';
-import { ContentTypes, EnvVar } from '@/common';
+import type { ToolErrorData } from './stream';
+import { EnvVar } from '@/common';
 
 /** Replacement type for `import type { ToolCall } from '@langchain/core/messages/tool'` in order to have stringified args typed */
 export type CustomToolCall = {
@@ -29,6 +30,7 @@ export type ToolNodeOptions = {
   handleToolErrors?: boolean;
   loadRuntimeTools?: ToolRefGenerator;
   toolCallStepIds?: Map<string, string>;
+  errorHandler?: (data: ToolErrorData, metadata?: Record<string, unknown>) => void
 };
 
 export type ToolNodeConstructorParams = ToolRefs & ToolNodeOptions;