@librechat/agents 3.1.1 → 3.1.21

package/src/specs/azure.simple.test.ts

@@ -19,7 +19,6 @@ import { ContentTypes, GraphEvents, Providers, TitleMethod } from '@/common';
 import { ChatModelStreamHandler, createContentAggregator } from '@/stream';
 import { capitalizeFirstLetter } from './spec.utils';
 import { getLLMConfig } from '@/utils/llmConfig';
-import { getArgs } from '@/scripts/args';
 import { Run } from '@/run';
 
 // Auto-skip this suite if Azure env vars are not present
@@ -34,15 +33,24 @@ const hasAzure = requiredAzureEnv.every(
 );
 const describeIfAzure = hasAzure ? describe : describe.skip;
 
+const isContentFilterError = (error: unknown): boolean => {
+  const message = error instanceof Error ? error.message : String(error);
+  return (
+    message.includes('content management policy') ||
+    message.includes('content filtering')
+  );
+};
+
 const provider = Providers.AZURE;
+let contentFilterTriggered = false;
 describeIfAzure(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
   jest.setTimeout(30000);
   let run: Run<t.IState>;
-  let runningHistory: BaseMessage[];
   let collectedUsage: UsageMetadata[];
   let conversationHistory: BaseMessage[];
   let aggregateContent: t.ContentAggregator;
   let contentParts: t.MessageContentComplex[];
+  let runningHistory: BaseMessage[] | null = null;
 
   const config = {
     configurable: {
@@ -129,186 +137,217 @@ describeIfAzure(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
   });
 
   test(`${capitalizeFirstLetter(provider)}: should process a simple message, generate title`, async () => {
-    const { userName, location } = await getArgs();
-    const llmConfig = getLLMConfig(provider);
-    const customHandlers = setupCustomHandlers();
-
-    run = await Run.create<t.IState>({
-      runId: 'test-run-id',
-      graphConfig: {
-        type: 'standard',
-        llmConfig,
-        tools: [new Calculator()],
-        instructions:
-          'You are a friendly AI assistant. Always address the user by their name.',
-        additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
-      },
-      returnContent: true,
-      customHandlers,
-    });
-
-    const userMessage = 'hi';
-    conversationHistory.push(new HumanMessage(userMessage));
-
-    const inputs = {
-      messages: conversationHistory,
-    };
-
-    const finalContentParts = await run.processStream(inputs, config);
-    expect(finalContentParts).toBeDefined();
-    const allTextParts = finalContentParts?.every(
-      (part) => part.type === ContentTypes.TEXT
-    );
-    expect(allTextParts).toBe(true);
-    expect(collectedUsage.length).toBeGreaterThan(0);
-    expect(collectedUsage[0].input_tokens).toBeGreaterThan(0);
-    expect(collectedUsage[0].output_tokens).toBeGreaterThan(0);
-
-    const finalMessages = run.getRunMessages();
-    expect(finalMessages).toBeDefined();
-    conversationHistory.push(...(finalMessages ?? []));
-    expect(conversationHistory.length).toBeGreaterThan(1);
-    runningHistory = conversationHistory.slice();
-
-    expect(onMessageDeltaSpy).toHaveBeenCalled();
-    expect(onMessageDeltaSpy.mock.calls.length).toBeGreaterThan(1);
-    expect(onMessageDeltaSpy.mock.calls[0][3]).toBeDefined(); // Graph exists
-
-    expect(onRunStepSpy).toHaveBeenCalled();
-    expect(onRunStepSpy.mock.calls.length).toBeGreaterThan(0);
-    expect(onRunStepSpy.mock.calls[0][3]).toBeDefined(); // Graph exists
-
-    const { handleLLMEnd, collected } = createMetadataAggregator();
-    const titleResult = await run.generateTitle({
-      provider,
-      inputText: userMessage,
-      titleMethod: TitleMethod.STRUCTURED,
-      contentParts,
-      clientOptions: llmConfig,
-      chainOptions: {
-        callbacks: [
-          {
-            handleLLMEnd,
-          },
-        ],
-      },
-    });
+    try {
+      const llmConfig = getLLMConfig(provider);
+      const customHandlers = setupCustomHandlers();
+
+      run = await Run.create<t.IState>({
+        runId: 'test-run-id',
+        graphConfig: {
+          type: 'standard',
+          llmConfig,
+          tools: [new Calculator()],
+          instructions:
+            'You are a helpful AI assistant. Keep responses concise and friendly.',
+        },
+        returnContent: true,
+        customHandlers,
+      });
 
-    expect(titleResult).toBeDefined();
-    expect(titleResult.title).toBeDefined();
-    expect(titleResult.language).toBeDefined();
-    expect(collected).toBeDefined();
+      const userMessage = 'Hello, how are you today?';
+      conversationHistory.push(new HumanMessage(userMessage));
+
+      const inputs = {
+        messages: conversationHistory,
+      };
+
+      const finalContentParts = await run.processStream(inputs, config);
+      expect(finalContentParts).toBeDefined();
+      const allTextParts = finalContentParts?.every(
+        (part) => part.type === ContentTypes.TEXT
+      );
+      expect(allTextParts).toBe(true);
+      expect(collectedUsage.length).toBeGreaterThan(0);
+      expect(collectedUsage[0].input_tokens).toBeGreaterThan(0);
+      expect(collectedUsage[0].output_tokens).toBeGreaterThan(0);
+
+      const finalMessages = run.getRunMessages();
+      expect(finalMessages).toBeDefined();
+      conversationHistory.push(...(finalMessages ?? []));
+      expect(conversationHistory.length).toBeGreaterThan(1);
+      runningHistory = conversationHistory.slice();
+
+      expect(onMessageDeltaSpy).toHaveBeenCalled();
+      expect(onMessageDeltaSpy.mock.calls.length).toBeGreaterThan(1);
+      expect(onMessageDeltaSpy.mock.calls[0][3]).toBeDefined(); // Graph exists
+
+      expect(onRunStepSpy).toHaveBeenCalled();
+      expect(onRunStepSpy.mock.calls.length).toBeGreaterThan(0);
+      expect(onRunStepSpy.mock.calls[0][3]).toBeDefined(); // Graph exists
+
+      const { handleLLMEnd, collected } = createMetadataAggregator();
+      const titleResult = await run.generateTitle({
+        provider,
+        inputText: userMessage,
+        titleMethod: TitleMethod.STRUCTURED,
+        contentParts,
+        clientOptions: llmConfig,
+        chainOptions: {
+          callbacks: [
+            {
+              handleLLMEnd,
+            },
+          ],
+        },
+      });
+
+      expect(titleResult).toBeDefined();
+      expect(titleResult.title).toBeDefined();
+      expect(titleResult.language).toBeDefined();
+      expect(collected).toBeDefined();
+    } catch (error) {
+      if (isContentFilterError(error)) {
+        contentFilterTriggered = true;
+        console.warn('Skipping test: Azure content filter triggered');
+        return;
+      }
+      throw error;
+    }
   });
 
   test(`${capitalizeFirstLetter(provider)}: should generate title using completion method`, async () => {
-    const { userName, location } = await getArgs();
-    const llmConfig = getLLMConfig(provider);
-    const customHandlers = setupCustomHandlers();
-
-    run = await Run.create<t.IState>({
-      runId: 'test-run-id-completion',
-      graphConfig: {
-        type: 'standard',
-        llmConfig,
-        tools: [new Calculator()],
-        instructions:
-          'You are a friendly AI assistant. Always address the user by their name.',
-        additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
-      },
-      returnContent: true,
-      customHandlers,
-    });
-
-    const userMessage = 'What is the weather like today?';
-    conversationHistory = [];
-    conversationHistory.push(new HumanMessage(userMessage));
-
-    const inputs = {
-      messages: conversationHistory,
-    };
-
-    const finalContentParts = await run.processStream(inputs, config);
-    expect(finalContentParts).toBeDefined();
-
-    const { handleLLMEnd, collected } = createMetadataAggregator();
-    const titleResult = await run.generateTitle({
-      provider,
-      inputText: userMessage,
-      titleMethod: TitleMethod.COMPLETION, // Using completion method
-      contentParts,
-      clientOptions: llmConfig,
-      chainOptions: {
-        callbacks: [
-          {
-            handleLLMEnd,
-          },
-        ],
-      },
-    });
-
-    expect(titleResult).toBeDefined();
-    expect(titleResult.title).toBeDefined();
-    expect(titleResult.title).not.toBe('');
-    // Completion method doesn't return language
-    expect(titleResult.language).toBeUndefined();
-    expect(collected).toBeDefined();
-    console.log(`Completion method generated title: "${titleResult.title}"`);
+    if (contentFilterTriggered) {
+      console.warn(
+        'Skipping test: Azure content filter was triggered in previous test'
+      );
+      return;
+    }
+    try {
+      const llmConfig = getLLMConfig(provider);
+      const customHandlers = setupCustomHandlers();
+
+      run = await Run.create<t.IState>({
+        runId: 'test-run-id-completion',
+        graphConfig: {
+          type: 'standard',
+          llmConfig,
+          tools: [new Calculator()],
+          instructions:
+            'You are a helpful AI assistant. Keep responses concise and friendly.',
+        },
+        returnContent: true,
+        customHandlers,
+      });
+
+      const userMessage = 'What can you help me with today?';
+      conversationHistory = [];
+      conversationHistory.push(new HumanMessage(userMessage));
+
+      const inputs = {
+        messages: conversationHistory,
+      };
+
+      const finalContentParts = await run.processStream(inputs, config);
+      expect(finalContentParts).toBeDefined();
+
+      const { handleLLMEnd, collected } = createMetadataAggregator();
+      const titleResult = await run.generateTitle({
+        provider,
+        inputText: userMessage,
+        titleMethod: TitleMethod.COMPLETION,
+        contentParts,
+        clientOptions: llmConfig,
+        chainOptions: {
+          callbacks: [
+            {
+              handleLLMEnd,
+            },
+          ],
+        },
+      });
+
+      expect(titleResult).toBeDefined();
+      expect(titleResult.title).toBeDefined();
+      expect(titleResult.title).not.toBe('');
+      expect(titleResult.language).toBeUndefined();
+      expect(collected).toBeDefined();
+      console.log(`Completion method generated title: "${titleResult.title}"`);
+    } catch (error) {
+      if (isContentFilterError(error)) {
+        contentFilterTriggered = true;
+        console.warn('Skipping test: Azure content filter triggered');
+        return;
+      }
+      throw error;
+    }
   });
 
   test(`${capitalizeFirstLetter(provider)}: should follow-up`, async () => {
-    console.log('Previous conversation length:', runningHistory.length);
-    console.log(
-      'Last message:',
-      runningHistory[runningHistory.length - 1].content
-    );
-    const { userName, location } = await getArgs();
-    const llmConfig = getLLMConfig(provider);
-    const customHandlers = setupCustomHandlers();
-
-    run = await Run.create<t.IState>({
-      runId: 'test-run-id',
-      graphConfig: {
-        type: 'standard',
-        llmConfig,
-        tools: [new Calculator()],
-        instructions:
-          'You are a friendly AI assistant. Always address the user by their name.',
-        additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
-      },
-      returnContent: true,
-      customHandlers,
-    });
-
-    conversationHistory = runningHistory.slice();
-    conversationHistory.push(new HumanMessage('how are you?'));
-
-    const inputs = {
-      messages: conversationHistory,
-    };
-
-    const finalContentParts = await run.processStream(inputs, config);
-    expect(finalContentParts).toBeDefined();
-    const allTextParts = finalContentParts?.every(
-      (part) => part.type === ContentTypes.TEXT
-    );
-    expect(allTextParts).toBe(true);
-    expect(collectedUsage.length).toBeGreaterThan(0);
-    expect(collectedUsage[0].input_tokens).toBeGreaterThan(0);
-    expect(collectedUsage[0].output_tokens).toBeGreaterThan(0);
-
-    const finalMessages = run.getRunMessages();
-    expect(finalMessages).toBeDefined();
-    expect(finalMessages?.length).toBeGreaterThan(0);
-    console.log(
-      `${capitalizeFirstLetter(provider)} follow-up message:`,
-      finalMessages?.[finalMessages.length - 1]?.content
-    );
-
-    expect(onMessageDeltaSpy).toHaveBeenCalled();
-    expect(onMessageDeltaSpy.mock.calls.length).toBeGreaterThan(1);
-
-    expect(onRunStepSpy).toHaveBeenCalled();
-    expect(onRunStepSpy.mock.calls.length).toBeGreaterThan(0);
+    if (contentFilterTriggered || runningHistory == null) {
+      console.warn(
+        'Skipping test: Azure content filter was triggered or no conversation history'
+      );
+      return;
+    }
+    try {
+      console.log('Previous conversation length:', runningHistory.length);
+      console.log(
+        'Last message:',
+        runningHistory[runningHistory.length - 1].content
+      );
+      const llmConfig = getLLMConfig(provider);
+      const customHandlers = setupCustomHandlers();
+
+      run = await Run.create<t.IState>({
+        runId: 'test-run-id',
+        graphConfig: {
+          type: 'standard',
+          llmConfig,
+          tools: [new Calculator()],
+          instructions:
+            'You are a helpful AI assistant. Keep responses concise and friendly.',
+        },
+        returnContent: true,
+        customHandlers,
+      });
+
+      conversationHistory = runningHistory.slice();
+      conversationHistory.push(new HumanMessage('What else can you tell me?'));
+
+      const inputs = {
+        messages: conversationHistory,
+      };
+
+      const finalContentParts = await run.processStream(inputs, config);
+      expect(finalContentParts).toBeDefined();
+      const allTextParts = finalContentParts?.every(
+        (part) => part.type === ContentTypes.TEXT
+      );
+      expect(allTextParts).toBe(true);
+      expect(collectedUsage.length).toBeGreaterThan(0);
+      expect(collectedUsage[0].input_tokens).toBeGreaterThan(0);
+      expect(collectedUsage[0].output_tokens).toBeGreaterThan(0);
+
+      const finalMessages = run.getRunMessages();
+      expect(finalMessages).toBeDefined();
+      expect(finalMessages?.length).toBeGreaterThan(0);
+      console.log(
+        `${capitalizeFirstLetter(provider)} follow-up message:`,
+        finalMessages?.[finalMessages.length - 1]?.content
+      );
+
+      expect(onMessageDeltaSpy).toHaveBeenCalled();
+      expect(onMessageDeltaSpy.mock.calls.length).toBeGreaterThan(1);
+
+      expect(onRunStepSpy).toHaveBeenCalled();
+      expect(onRunStepSpy.mock.calls.length).toBeGreaterThan(0);
+    } catch (error) {
+      if (isContentFilterError(error)) {
+        console.warn('Skipping test: Azure content filter triggered');
+        return;
+      }
+      throw error;
+    }
   });
 
   test('should handle errors appropriately', async () => {

package/src/tools/ToolNode.ts

@@ -20,7 +20,8 @@ import type { BaseMessage, AIMessage } from '@langchain/core/messages';
 import type { StructuredToolInterface } from '@langchain/core/tools';
 import type * as t from '@/types';
 import { RunnableCallable } from '@/utils';
-import { Constants } from '@/common';
+import { safeDispatchCustomEvent } from '@/utils/events';
+import { Constants, GraphEvents } from '@/common';
 
 /**
  * Helper to check if a value is a Send object
@@ -44,6 +45,10 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
   private programmaticCache?: t.ProgrammaticCache;
   /** Reference to Graph's sessions map for automatic session injection */
   private sessions?: t.ToolSessionMap;
+  /** When true, dispatches ON_TOOL_EXECUTE events instead of invoking tools directly */
+  private eventDrivenMode: boolean = false;
+  /** Tool definitions for event-driven mode */
+  private toolDefinitions?: Map<string, t.LCTool>;
 
   constructor({
     tools,
@@ -56,6 +61,8 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
     loadRuntimeTools,
     toolRegistry,
     sessions,
+    eventDrivenMode,
+    toolDefinitions,
   }: t.ToolNodeConstructorParams) {
     super({ name, tags, func: (input, config) => this.run(input, config) });
     this.toolMap = toolMap ?? new Map(tools.map((tool) => [tool.name, tool]));
@@ -66,6 +73,8 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
     this.toolUsageCount = new Map<string, number>();
     this.toolRegistry = toolRegistry;
     this.sessions = sessions;
+    this.eventDrivenMode = eventDrivenMode ?? false;
+    this.toolDefinitions = toolDefinitions;
   }
 
   /**
@@ -243,11 +252,77 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
     }
   }
 
+  /**
+   * Execute all tool calls via ON_TOOL_EXECUTE event dispatch.
+   * Used in event-driven mode where the host handles actual tool execution.
+   */
+  private async executeViaEvent(
+    toolCalls: ToolCall[],
+    config: RunnableConfig,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    input: any
+  ): Promise<T> {
+    const requests: t.ToolCallRequest[] = toolCalls.map((call) => {
+      const turn = this.toolUsageCount.get(call.name) ?? 0;
+      this.toolUsageCount.set(call.name, turn + 1);
+      return {
+        id: call.id!,
+        name: call.name,
+        args: call.args as Record<string, unknown>,
+        stepId: this.toolCallStepIds?.get(call.id!),
+        turn,
+      };
+    });
+
+    const results = await new Promise<t.ToolExecuteResult[]>(
+      (resolve, reject) => {
+        const request: t.ToolExecuteBatchRequest = {
+          toolCalls: requests,
+          userId: config.configurable?.user_id as string | undefined,
+          agentId: config.configurable?.agent_id as string | undefined,
+          resolve,
+          reject,
+        };
+
+        safeDispatchCustomEvent(GraphEvents.ON_TOOL_EXECUTE, request, config);
+      }
+    );
+
+    const outputs: ToolMessage[] = results.map((result) => {
+      const toolName =
+        requests.find((r) => r.id === result.toolCallId)?.name ?? 'unknown';
+
+      if (result.status === 'error') {
+        return new ToolMessage({
+          status: 'error',
+          content: `Error: ${result.errorMessage ?? 'Unknown error'}\n Please fix your mistakes.`,
+          name: toolName,
+          tool_call_id: result.toolCallId,
+        });
+      }
+
+      return new ToolMessage({
+        status: 'success',
+        content:
+          typeof result.content === 'string'
+            ? result.content
+            : JSON.stringify(result.content),
+        name: toolName,
+        tool_call_id: result.toolCallId,
+      });
+    });
+
+    return (Array.isArray(input) ? outputs : { messages: outputs }) as T;
+  }
+
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   protected async run(input: any, config: RunnableConfig): Promise<T> {
     let outputs: (BaseMessage | Command)[];
 
     if (this.isSendInput(input)) {
+      if (this.eventDrivenMode) {
+        return this.executeViaEvent([input.lg_tool_call], config, input);
+      }
       outputs = [await this.runTool(input.lg_tool_call, config)];
     } else {
       let messages: BaseMessage[];
@@ -289,21 +364,26 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
         this.programmaticCache = undefined; // Invalidate cache on toolMap change
       }
 
+      const filteredCalls =
+        aiMessage.tool_calls?.filter((call) => {
+          /**
+           * Filter out:
+           * 1. Already processed tool calls (present in toolMessageIds)
+           * 2. Server tool calls (e.g., web_search with IDs starting with 'srvtoolu_')
+           *    which are executed by the provider's API and don't require invocation
+           */
+          return (
+            (call.id == null || !toolMessageIds.has(call.id)) &&
+            !(call.id?.startsWith('srvtoolu_') ?? false)
+          );
+        }) ?? [];
+
+      if (this.eventDrivenMode && filteredCalls.length > 0) {
+        return this.executeViaEvent(filteredCalls, config, input);
+      }
+
       outputs = await Promise.all(
-        aiMessage.tool_calls
-          ?.filter((call) => {
-            /**
-             * Filter out:
-             * 1. Already processed tool calls (present in toolMessageIds)
-             * 2. Server tool calls (e.g., web_search with IDs starting with 'srvtoolu_')
-             *    which are executed by the provider's API and don't require invocation
-             */
-            return (
-              (call.id == null || !toolMessageIds.has(call.id)) &&
-              !(call.id?.startsWith('srvtoolu_') ?? false)
-            );
-          })
-          .map((call) => this.runTool(call, config)) ?? []
+        filteredCalls.map((call) => this.runTool(call, config))
       );
     }
 

package/src/tools/__tests__/ProgrammaticToolCalling.integration.test.ts

@@ -4,6 +4,9 @@
  * These tests hit the LIVE Code API and verify end-to-end functionality.
  *
  * Run with: npm test -- ProgrammaticToolCalling.integration.test.ts
+ *
+ * Requires LIBRECHAT_CODE_API_KEY environment variable.
+ * Tests are skipped when the API key is not available.
  */
 import { config as dotenvConfig } from 'dotenv';
 dotenvConfig();
@@ -19,19 +22,17 @@ import {
   createProgrammaticToolRegistry,
 } from '@/test/mockTools';
 
-describe('ProgrammaticToolCalling - Live API Integration', () => {
+const apiKey = process.env.LIBRECHAT_CODE_API_KEY;
+const shouldSkip = apiKey == null || apiKey === '';
+
+const describeIfApiKey = shouldSkip ? describe.skip : describe;
+
+describeIfApiKey('ProgrammaticToolCalling - Live API Integration', () => {
   let ptcTool: ReturnType<typeof createProgrammaticToolCallingTool>;
   let toolMap: t.ToolMap;
   let toolDefinitions: t.LCTool[];
 
   beforeAll(() => {
-    const apiKey = process.env.LIBRECHAT_CODE_API_KEY;
-    if (apiKey == null || apiKey === '') {
-      throw new Error(
-        'LIBRECHAT_CODE_API_KEY not set. Required for integration tests.'
-      );
-    }
-
     const tools = [
       createGetTeamMembersTool(),
       createGetExpensesTool(),
@@ -42,7 +43,7 @@ describe('ProgrammaticToolCalling - Live API Integration', () => {
     toolMap = new Map(tools.map((t) => [t.name, t]));
     toolDefinitions = Array.from(createProgrammaticToolRegistry().values());
 
-    ptcTool = createProgrammaticToolCallingTool({ apiKey });
+    ptcTool = createProgrammaticToolCallingTool({ apiKey: apiKey! });
   });
 
   it('executes simple single tool call', async () => {

package/src/tools/__tests__/ToolSearch.integration.test.ts

@@ -4,6 +4,9 @@
  * These tests hit the LIVE Code API and verify end-to-end search functionality.
  *
  * Run with: npm test -- ToolSearch.integration.test.ts
+ *
+ * Requires LIBRECHAT_CODE_API_KEY environment variable.
+ * Tests are skipped when the API key is not available.
  */
 import { config as dotenvConfig } from 'dotenv';
 dotenvConfig();
@@ -12,19 +15,17 @@ import { describe, it, expect, beforeAll } from '@jest/globals';
 import { createToolSearch } from '../ToolSearch';
 import { createToolSearchToolRegistry } from '@/test/mockTools';
 
-describe('ToolSearch - Live API Integration', () => {
+const apiKey = process.env.LIBRECHAT_CODE_API_KEY;
+const shouldSkip = apiKey == null || apiKey === '';
+
+const describeIfApiKey = shouldSkip ? describe.skip : describe;
+
+describeIfApiKey('ToolSearch - Live API Integration', () => {
   let searchTool: ReturnType<typeof createToolSearch>;
   const toolRegistry = createToolSearchToolRegistry();
 
   beforeAll(() => {
-    const apiKey = process.env.LIBRECHAT_CODE_API_KEY;
-    if (apiKey == null || apiKey === '') {
-      throw new Error(
-        'LIBRECHAT_CODE_API_KEY not set. Required for integration tests.'
-      );
-    }
-
-    searchTool = createToolSearch({ apiKey, toolRegistry });
+    searchTool = createToolSearch({ apiKey: apiKey!, toolRegistry });
   });
 
   it('searches for expense-related tools', async () => {