@librechat/agents 2.4.322 → 3.0.0-rc1

package/src/specs/anthropic.simple.test.ts

@@ -4,12 +4,19 @@
 import { config } from 'dotenv';
 config();
 import { Calculator } from '@langchain/community/tools/calculator';
-import { HumanMessage, BaseMessage, UsageMetadata } from '@langchain/core/messages';
-import type { StandardGraph } from '@/graphs';
+import {
+  HumanMessage,
+  BaseMessage,
+  UsageMetadata,
+} from '@langchain/core/messages';
 import type * as t from '@/types';
-import { ToolEndHandler, ModelEndHandler, createMetadataAggregator } from '@/events';
+import {
+  ToolEndHandler,
+  ModelEndHandler,
+  createMetadataAggregator,
+} from '@/events';
+import { ContentTypes, GraphEvents, Providers, TitleMethod } from '@/common';
 import { ChatModelStreamHandler, createContentAggregator } from '@/stream';
-import { ContentTypes, GraphEvents, Providers } from '@/common';
 import { capitalizeFirstLetter } from './spec.utils';
 import { getLLMConfig } from '@/utils/llmConfig';
 import { getArgs } from '@/scripts/args';
@@ -17,7 +24,7 @@ import { Run } from '@/run';
 
 const provider = Providers.ANTHROPIC;
 describe(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
-  jest.setTimeout(30000);
+  jest.setTimeout(60000);
   let run: Run<t.IState>;
   let runningHistory: BaseMessage[];
   let collectedUsage: UsageMetadata[];
@@ -36,7 +43,8 @@ describe(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
   beforeEach(async () => {
     conversationHistory = [];
     collectedUsage = [];
-    const { contentParts: cp, aggregateContent: ac } = createContentAggregator();
+    const { contentParts: cp, aggregateContent: ac } =
+      createContentAggregator();
     contentParts = cp as t.MessageContentComplex[];
     aggregateContent = ac;
   });
@@ -49,36 +57,62 @@ describe(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
     onRunStepSpy.mockReset();
   });
 
-  const setupCustomHandlers = (): Record<string | GraphEvents, t.EventHandler> => ({
+  const setupCustomHandlers = (): Record<
+    string | GraphEvents,
+    t.EventHandler
+  > => ({
     [GraphEvents.TOOL_END]: new ToolEndHandler(),
     [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(collectedUsage),
     [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(),
     [GraphEvents.ON_RUN_STEP_COMPLETED]: {
-      handle: (event: GraphEvents.ON_RUN_STEP_COMPLETED, data: t.StreamEventData): void => {
-        aggregateContent({ event, data: data as unknown as { result: t.ToolEndEvent; } });
-      }
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_COMPLETED,
+        data: t.StreamEventData
+      ): void => {
+        aggregateContent({
+          event,
+          data: data as unknown as { result: t.ToolEndEvent },
+        });
+      },
     },
     [GraphEvents.ON_RUN_STEP]: {
-      handle: (event: GraphEvents.ON_RUN_STEP, data: t.StreamEventData, metadata, graph): void => {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP,
+        data: t.StreamEventData,
+        metadata,
+        graph
+      ): void => {
         onRunStepSpy(event, data, metadata, graph);
         aggregateContent({ event, data: data as t.RunStep });
-      }
+      },
     },
     [GraphEvents.ON_RUN_STEP_DELTA]: {
-      handle: (event: GraphEvents.ON_RUN_STEP_DELTA, data: t.StreamEventData): void => {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_DELTA,
+        data: t.StreamEventData
+      ): void => {
         aggregateContent({ event, data: data as t.RunStepDeltaEvent });
-      }
+      },
     },
     [GraphEvents.ON_MESSAGE_DELTA]: {
-      handle: (event: GraphEvents.ON_MESSAGE_DELTA, data: t.StreamEventData, metadata, graph): void => {
+      handle: (
+        event: GraphEvents.ON_MESSAGE_DELTA,
+        data: t.StreamEventData,
+        metadata,
+        graph
+      ): void => {
         onMessageDeltaSpy(event, data, metadata, graph);
         aggregateContent({ event, data: data as t.MessageDeltaEvent });
-      }
+      },
     },
     [GraphEvents.TOOL_START]: {
-      handle: (_event: string, _data: t.StreamEventData, _metadata?: Record<string, unknown>): void => {
+      handle: (
+        _event: string,
+        _data: t.StreamEventData,
+        _metadata?: Record<string, unknown>
+      ): void => {
         // Handle tool start
-      }
+      },
     },
   });
 
@@ -93,7 +127,8 @@ describe(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
         type: 'standard',
         llmConfig,
         tools: [new Calculator()],
-        instructions: 'You are a friendly AI assistant. Always address the user by their name.',
+        instructions:
+          'You are a friendly AI assistant. Always address the user by their name.',
         additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
       },
       returnContent: true,
@@ -109,7 +144,9 @@ describe(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
 
     const finalContentParts = await run.processStream(inputs, config);
     expect(finalContentParts).toBeDefined();
-    const allTextParts = finalContentParts?.every((part) => part.type === ContentTypes.TEXT);
+    const allTextParts = finalContentParts?.every(
+      (part) => part.type === ContentTypes.TEXT
+    );
     expect(allTextParts).toBe(true);
     expect(collectedUsage.length).toBeGreaterThan(0);
     expect(collectedUsage[0].input_tokens).toBeGreaterThan(0);
@@ -117,26 +154,34 @@ describe(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
 
     const finalMessages = run.getRunMessages();
     expect(finalMessages).toBeDefined();
-    conversationHistory.push(...finalMessages ?? []);
+    conversationHistory.push(...(finalMessages ?? []));
     expect(conversationHistory.length).toBeGreaterThan(1);
     runningHistory = conversationHistory.slice();
 
     expect(onMessageDeltaSpy).toHaveBeenCalled();
     expect(onMessageDeltaSpy.mock.calls.length).toBeGreaterThan(1);
-    expect((onMessageDeltaSpy.mock.calls[0][3] as StandardGraph).provider).toBeDefined();
+    expect(onMessageDeltaSpy.mock.calls[0][3]).toBeDefined(); // Graph exists
 
     expect(onRunStepSpy).toHaveBeenCalled();
     expect(onRunStepSpy.mock.calls.length).toBeGreaterThan(0);
-    expect((onRunStepSpy.mock.calls[0][3] as StandardGraph).provider).toBeDefined();
+    expect(onRunStepSpy.mock.calls[0][3]).toBeDefined(); // Graph exists
 
     const { handleLLMEnd, collected } = createMetadataAggregator();
     const titleResult = await run.generateTitle({
+      provider,
       inputText: userMessage,
+      titleMethod: TitleMethod.STRUCTURED,
       contentParts,
+      clientOptions: {
+        ...llmConfig,
+        model: 'claude-3-5-haiku-latest',
+      },
       chainOptions: {
-        callbacks: [{
-          handleLLMEnd,
-        }],
+        callbacks: [
+          {
+            handleLLMEnd,
+          },
+        ],
       },
     });
 
@@ -146,9 +191,71 @@ describe(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
     expect(collected).toBeDefined();
   });
 
+  test(`${capitalizeFirstLetter(provider)}: should generate title using completion method`, async () => {
+    const { userName, location } = await getArgs();
+    const llmConfig = getLLMConfig(provider);
+    const customHandlers = setupCustomHandlers();
+
+    run = await Run.create<t.IState>({
+      runId: 'test-run-id-completion',
+      graphConfig: {
+        type: 'standard',
+        llmConfig,
+        tools: [new Calculator()],
+        instructions:
+          'You are a friendly AI assistant. Always address the user by their name.',
+        additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
+      },
+      returnContent: true,
+      customHandlers,
+    });
+
+    const userMessage =
+      'Can you help me calculate the area of a circle with radius 5?';
+    conversationHistory = [];
+    conversationHistory.push(new HumanMessage(userMessage));
+
+    const inputs = {
+      messages: conversationHistory,
+    };
+
+    const finalContentParts = await run.processStream(inputs, config);
+    expect(finalContentParts).toBeDefined();
+
+    const { handleLLMEnd, collected } = createMetadataAggregator();
+    const titleResult = await run.generateTitle({
+      provider,
+      inputText: userMessage,
+      titleMethod: TitleMethod.COMPLETION, // Using completion method
+      contentParts,
+      clientOptions: {
+        ...llmConfig,
+        model: 'claude-3-5-haiku-latest',
+      },
+      chainOptions: {
+        callbacks: [
+          {
+            handleLLMEnd,
+          },
+        ],
+      },
+    });
+
+    expect(titleResult).toBeDefined();
+    expect(titleResult.title).toBeDefined();
+    expect(titleResult.title).not.toBe('');
+    // Completion method doesn't return language
+    expect(titleResult.language).toBeUndefined();
+    expect(collected).toBeDefined();
+    console.log(`Completion method generated title: "${titleResult.title}"`);
+  });
+
   test(`${capitalizeFirstLetter(provider)}: should follow-up`, async () => {
     console.log('Previous conversation length:', runningHistory.length);
-    console.log('Last message:', runningHistory[runningHistory.length - 1].content);
+    console.log(
+      'Last message:',
+      runningHistory[runningHistory.length - 1].content
+    );
     const { userName, location } = await getArgs();
     const llmConfig = getLLMConfig(provider);
     const customHandlers = setupCustomHandlers();
@@ -159,7 +266,8 @@ describe(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
         type: 'standard',
         llmConfig,
         tools: [new Calculator()],
-        instructions: 'You are a friendly AI assistant. Always address the user by their name.',
+        instructions:
+          'You are a friendly AI assistant. Always address the user by their name.',
         additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
       },
       returnContent: true,
@@ -175,7 +283,9 @@ describe(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
 
     const finalContentParts = await run.processStream(inputs, config);
     expect(finalContentParts).toBeDefined();
-    const allTextParts = finalContentParts?.every((part) => part.type === ContentTypes.TEXT);
+    const allTextParts = finalContentParts?.every(
+      (part) => part.type === ContentTypes.TEXT
+    );
     expect(allTextParts).toBe(true);
     expect(collectedUsage.length).toBeGreaterThan(0);
     expect(collectedUsage[0].input_tokens).toBeGreaterThan(0);
@@ -184,7 +294,10 @@ describe(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
     const finalMessages = run.getRunMessages();
     expect(finalMessages).toBeDefined();
     expect(finalMessages?.length).toBeGreaterThan(0);
-    console.log(`${capitalizeFirstLetter(provider)} follow-up message:`, finalMessages?.[finalMessages.length - 1]?.content);
+    console.log(
+      `${capitalizeFirstLetter(provider)} follow-up message:`,
+      finalMessages?.[finalMessages.length - 1]?.content
+    );
 
     expect(onMessageDeltaSpy).toHaveBeenCalled();
     expect(onMessageDeltaSpy.mock.calls.length).toBeGreaterThan(1);
@@ -196,9 +309,12 @@ describe(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
   test('should handle errors appropriately', async () => {
     // Test error scenarios
     await expect(async () => {
-      await run.processStream({
-        messages: [],
-      }, {} as any);
+      await run.processStream(
+        {
+          messages: [],
+        },
+        {} as any
+      );
     }).rejects.toThrow();
   });
-});
+});

package/src/specs/azure.simple.test.ts

@@ -0,0 +1,325 @@
+/* eslint-disable no-console */
+/* eslint-disable @typescript-eslint/no-explicit-any */
+// src/scripts/cli.test.ts
+import { config } from 'dotenv';
+config();
+import { Calculator } from '@langchain/community/tools/calculator';
+import {
+  HumanMessage,
+  BaseMessage,
+  UsageMetadata,
+} from '@langchain/core/messages';
+import type * as t from '@/types';
+import {
+  ToolEndHandler,
+  ModelEndHandler,
+  createMetadataAggregator,
+} from '@/events';
+import { ContentTypes, GraphEvents, Providers, TitleMethod } from '@/common';
+import { ChatModelStreamHandler, createContentAggregator } from '@/stream';
+import { capitalizeFirstLetter } from './spec.utils';
+import { getLLMConfig } from '@/utils/llmConfig';
+import { getArgs } from '@/scripts/args';
+import { Run } from '@/run';
+
+// Auto-skip this suite if Azure env vars are not present
+const requiredAzureEnv = [
+  'AZURE_OPENAI_API_KEY',
+  'AZURE_OPENAI_API_INSTANCE',
+  'AZURE_OPENAI_API_DEPLOYMENT',
+  'AZURE_OPENAI_API_VERSION',
+];
+const hasAzure = requiredAzureEnv.every(
+  (k) => (process.env[k] ?? '').trim() !== ''
+);
+const describeIfAzure = hasAzure ? describe : describe.skip;
+
+const provider = Providers.AZURE;
+describeIfAzure(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
+  jest.setTimeout(30000);
+  let run: Run<t.IState>;
+  let runningHistory: BaseMessage[];
+  let collectedUsage: UsageMetadata[];
+  let conversationHistory: BaseMessage[];
+  let aggregateContent: t.ContentAggregator;
+  let contentParts: t.MessageContentComplex[];
+
+  const config = {
+    configurable: {
+      thread_id: 'conversation-num-1',
+    },
+    streamMode: 'values',
+    version: 'v2' as const,
+  };
+
+  beforeEach(async () => {
+    conversationHistory = [];
+    collectedUsage = [];
+    const { contentParts: cp, aggregateContent: ac } =
+      createContentAggregator();
+    contentParts = cp as t.MessageContentComplex[];
+    aggregateContent = ac;
+  });
+
+  const onMessageDeltaSpy = jest.fn();
+  const onRunStepSpy = jest.fn();
+
+  afterAll(() => {
+    onMessageDeltaSpy.mockReset();
+    onRunStepSpy.mockReset();
+  });
+
+  const setupCustomHandlers = (): Record<
+    string | GraphEvents,
+    t.EventHandler
+  > => ({
+    [GraphEvents.TOOL_END]: new ToolEndHandler(),
+    [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(collectedUsage),
+    [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(),
+    [GraphEvents.ON_RUN_STEP_COMPLETED]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_COMPLETED,
+        data: t.StreamEventData
+      ): void => {
+        aggregateContent({
+          event,
+          data: data as unknown as { result: t.ToolEndEvent },
+        });
+      },
+    },
+    [GraphEvents.ON_RUN_STEP]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP,
+        data: t.StreamEventData,
+        metadata,
+        graph
+      ): void => {
+        onRunStepSpy(event, data, metadata, graph);
+        aggregateContent({ event, data: data as t.RunStep });
+      },
+    },
+    [GraphEvents.ON_RUN_STEP_DELTA]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_DELTA,
+        data: t.StreamEventData
+      ): void => {
+        aggregateContent({ event, data: data as t.RunStepDeltaEvent });
+      },
+    },
+    [GraphEvents.ON_MESSAGE_DELTA]: {
+      handle: (
+        event: GraphEvents.ON_MESSAGE_DELTA,
+        data: t.StreamEventData,
+        metadata,
+        graph
+      ): void => {
+        onMessageDeltaSpy(event, data, metadata, graph);
+        aggregateContent({ event, data: data as t.MessageDeltaEvent });
+      },
+    },
+    [GraphEvents.TOOL_START]: {
+      handle: (
+        _event: string,
+        _data: t.StreamEventData,
+        _metadata?: Record<string, unknown>
+      ): void => {
+        // Handle tool start
+      },
+    },
+  });
+
+  test(`${capitalizeFirstLetter(provider)}: should process a simple message, generate title`, async () => {
+    const { userName, location } = await getArgs();
+    const llmConfig = getLLMConfig(provider);
+    const customHandlers = setupCustomHandlers();
+
+    run = await Run.create<t.IState>({
+      runId: 'test-run-id',
+      graphConfig: {
+        type: 'standard',
+        llmConfig,
+        tools: [new Calculator()],
+        instructions:
+          'You are a friendly AI assistant. Always address the user by their name.',
+        additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
+      },
+      returnContent: true,
+      customHandlers,
+    });
+
+    const userMessage = 'hi';
+    conversationHistory.push(new HumanMessage(userMessage));
+
+    const inputs = {
+      messages: conversationHistory,
+    };
+
+    const finalContentParts = await run.processStream(inputs, config);
+    expect(finalContentParts).toBeDefined();
+    const allTextParts = finalContentParts?.every(
+      (part) => part.type === ContentTypes.TEXT
+    );
+    expect(allTextParts).toBe(true);
+    expect(collectedUsage.length).toBeGreaterThan(0);
+    expect(collectedUsage[0].input_tokens).toBeGreaterThan(0);
+    expect(collectedUsage[0].output_tokens).toBeGreaterThan(0);
+
+    const finalMessages = run.getRunMessages();
+    expect(finalMessages).toBeDefined();
+    conversationHistory.push(...(finalMessages ?? []));
+    expect(conversationHistory.length).toBeGreaterThan(1);
+    runningHistory = conversationHistory.slice();
+
+    expect(onMessageDeltaSpy).toHaveBeenCalled();
+    expect(onMessageDeltaSpy.mock.calls.length).toBeGreaterThan(1);
+    expect(onMessageDeltaSpy.mock.calls[0][3]).toBeDefined(); // Graph exists
+
+    expect(onRunStepSpy).toHaveBeenCalled();
+    expect(onRunStepSpy.mock.calls.length).toBeGreaterThan(0);
+    expect(onRunStepSpy.mock.calls[0][3]).toBeDefined(); // Graph exists
+
+    const { handleLLMEnd, collected } = createMetadataAggregator();
+    const titleResult = await run.generateTitle({
+      provider,
+      inputText: userMessage,
+      titleMethod: TitleMethod.STRUCTURED,
+      contentParts,
+      clientOptions: llmConfig,
+      chainOptions: {
+        callbacks: [
+          {
+            handleLLMEnd,
+          },
+        ],
+      },
+    });
+
+    expect(titleResult).toBeDefined();
+    expect(titleResult.title).toBeDefined();
+    expect(titleResult.language).toBeDefined();
+    expect(collected).toBeDefined();
+  });
+
+  test(`${capitalizeFirstLetter(provider)}: should generate title using completion method`, async () => {
+    const { userName, location } = await getArgs();
+    const llmConfig = getLLMConfig(provider);
+    const customHandlers = setupCustomHandlers();
+
+    run = await Run.create<t.IState>({
+      runId: 'test-run-id-completion',
+      graphConfig: {
+        type: 'standard',
+        llmConfig,
+        tools: [new Calculator()],
+        instructions:
+          'You are a friendly AI assistant. Always address the user by their name.',
+        additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
+      },
+      returnContent: true,
+      customHandlers,
+    });
+
+    const userMessage = 'What is the weather like today?';
+    conversationHistory = [];
+    conversationHistory.push(new HumanMessage(userMessage));
+
+    const inputs = {
+      messages: conversationHistory,
+    };
+
+    const finalContentParts = await run.processStream(inputs, config);
+    expect(finalContentParts).toBeDefined();
+
+    const { handleLLMEnd, collected } = createMetadataAggregator();
+    const titleResult = await run.generateTitle({
+      provider,
+      inputText: userMessage,
+      titleMethod: TitleMethod.COMPLETION, // Using completion method
+      contentParts,
+      clientOptions: llmConfig,
+      chainOptions: {
+        callbacks: [
+          {
+            handleLLMEnd,
+          },
+        ],
+      },
+    });
+
+    expect(titleResult).toBeDefined();
+    expect(titleResult.title).toBeDefined();
+    expect(titleResult.title).not.toBe('');
+    // Completion method doesn't return language
+    expect(titleResult.language).toBeUndefined();
+    expect(collected).toBeDefined();
+    console.log(`Completion method generated title: "${titleResult.title}"`);
+  });
+
+  test(`${capitalizeFirstLetter(provider)}: should follow-up`, async () => {
+    console.log('Previous conversation length:', runningHistory.length);
+    console.log(
+      'Last message:',
+      runningHistory[runningHistory.length - 1].content
+    );
+    const { userName, location } = await getArgs();
+    const llmConfig = getLLMConfig(provider);
+    const customHandlers = setupCustomHandlers();
+
+    run = await Run.create<t.IState>({
+      runId: 'test-run-id',
+      graphConfig: {
+        type: 'standard',
+        llmConfig,
+        tools: [new Calculator()],
+        instructions:
+          'You are a friendly AI assistant. Always address the user by their name.',
+        additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
+      },
+      returnContent: true,
+      customHandlers,
+    });
+
+    conversationHistory = runningHistory.slice();
+    conversationHistory.push(new HumanMessage('how are you?'));
+
+    const inputs = {
+      messages: conversationHistory,
+    };
+
+    const finalContentParts = await run.processStream(inputs, config);
+    expect(finalContentParts).toBeDefined();
+    const allTextParts = finalContentParts?.every(
+      (part) => part.type === ContentTypes.TEXT
+    );
+    expect(allTextParts).toBe(true);
+    expect(collectedUsage.length).toBeGreaterThan(0);
+    expect(collectedUsage[0].input_tokens).toBeGreaterThan(0);
+    expect(collectedUsage[0].output_tokens).toBeGreaterThan(0);
+
+    const finalMessages = run.getRunMessages();
+    expect(finalMessages).toBeDefined();
+    expect(finalMessages?.length).toBeGreaterThan(0);
+    console.log(
+      `${capitalizeFirstLetter(provider)} follow-up message:`,
+      finalMessages?.[finalMessages.length - 1]?.content
+    );
+
+    expect(onMessageDeltaSpy).toHaveBeenCalled();
+    expect(onMessageDeltaSpy.mock.calls.length).toBeGreaterThan(1);
+
+    expect(onRunStepSpy).toHaveBeenCalled();
+    expect(onRunStepSpy.mock.calls.length).toBeGreaterThan(0);
+  });
+
+  test('should handle errors appropriately', async () => {
+    // Test error scenarios
+    await expect(async () => {
+      await run.processStream(
+        {
+          messages: [],
+        },
+        {} as any
+      );
+    }).rejects.toThrow();
+  });
+});