@librechat/agents 3.0.0-rc11 → 3.0.0-rc13

package/src/messages/content.test.ts

@@ -0,0 +1,362 @@
+import {
+  HumanMessage,
+  AIMessage,
+  SystemMessage,
+} from '@langchain/core/messages';
+import { formatContentStrings } from './content';
+import { ContentTypes } from '@/common';
+
+describe('formatContentStrings', () => {
+  describe('Human messages', () => {
+    it('should convert human message with all text blocks to string', () => {
+      const messages = [
+        new HumanMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Hello' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'World' },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe('Hello\nWorld');
+    });
+
+    it('should not convert human message with mixed content types (text + image)', () => {
+      const messages = [
+        new HumanMessage({
+          content: [
+            { type: ContentTypes.TEXT, text: 'what do you see' },
+            {
+              type: 'image_url',
+              image_url: {
+                url: 'data:image/png;base64,iVBO_SOME_BASE64_DATA=',
+                detail: 'auto',
+              },
+            },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toEqual([
+        { type: ContentTypes.TEXT, text: 'what do you see' },
+        {
+          type: 'image_url',
+          image_url: {
+            url: 'data:image/png;base64,iVBO_SOME_BASE64_DATA=',
+            detail: 'auto',
+          },
+        },
+      ]);
+    });
+
+    it('should leave string content unchanged', () => {
+      const messages = [
+        new HumanMessage({
+          content: 'Hello World',
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe('Hello World');
+    });
+
+    it('should handle empty text blocks', () => {
+      const messages = [
+        new HumanMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Hello' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: '' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'World' },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe('Hello\n\nWorld');
+    });
+
+    it('should handle null/undefined text values', () => {
+      const messages = [
+        new HumanMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Hello' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: null },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: undefined },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'World' },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe('Hello\n\n\nWorld');
+    });
+  });
+
+  describe('AI messages', () => {
+    it('should convert AI message with all text blocks to string', () => {
+      const messages = [
+        new AIMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Hello' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'World' },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe('Hello\nWorld');
+      expect(result[0].getType()).toBe('ai');
+    });
+
+    it('should not convert AI message with mixed content types', () => {
+      const messages = [
+        new AIMessage({
+          content: [
+            {
+              type: ContentTypes.TEXT,
+              [ContentTypes.TEXT]: 'Here is an image',
+            },
+            {
+              type: ContentTypes.TOOL_CALL,
+              tool_call: { name: 'generate_image' },
+            },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toEqual([
+        { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Here is an image' },
+        { type: ContentTypes.TOOL_CALL, tool_call: { name: 'generate_image' } },
+      ]);
+    });
+  });
+
+  describe('System messages', () => {
+    it('should convert System message with all text blocks to string', () => {
+      const messages = [
+        new SystemMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'System' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Message' },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe('System\nMessage');
+      expect(result[0].getType()).toBe('system');
+    });
+  });
+
+  describe('Mixed message types', () => {
+    it('should process all valid message types in mixed array', () => {
+      const messages = [
+        new HumanMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Human' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Message' },
+          ],
+        }),
+        new AIMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'AI' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Response' },
+          ],
+        }),
+        new SystemMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'System' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Prompt' },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(3);
+      // All messages should be converted
+      expect(result[0].content).toBe('Human\nMessage');
+      expect(result[0].getType()).toBe('human');
+
+      expect(result[1].content).toBe('AI\nResponse');
+      expect(result[1].getType()).toBe('ai');
+
+      expect(result[2].content).toBe('System\nPrompt');
+      expect(result[2].getType()).toBe('system');
+    });
+  });
+
+  describe('Edge cases', () => {
+    it('should handle empty array', () => {
+      const result = formatContentStrings([]);
+      expect(result).toEqual([]);
+    });
+
+    it('should handle messages with non-array content', () => {
+      const messages = [
+        new HumanMessage({
+          content: 'This is a string content',
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe('This is a string content');
+    });
+
+    it('should trim the final concatenated string', () => {
+      const messages = [
+        new HumanMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: '  Hello  ' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: '  World  ' },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe('Hello  \n  World');
+    });
+  });
+
+  describe('Real-world scenarios', () => {
+    it('should handle the exact scenario from the issue', () => {
+      const messages = [
+        new HumanMessage({
+          content: [
+            {
+              type: 'text',
+              text: 'hi there',
+            },
+          ],
+        }),
+        new AIMessage({
+          content: [
+            {
+              type: 'text',
+              text: 'Hi Danny! How can I help you today?',
+            },
+          ],
+        }),
+        new HumanMessage({
+          content: [
+            {
+              type: 'text',
+              text: 'what do you see',
+            },
+            {
+              type: 'image_url',
+              image_url: {
+                url: 'data:image/png;base64,iVBO_SOME_BASE64_DATA=',
+                detail: 'auto',
+              },
+            },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(3);
+
+      // First human message (all text) should be converted
+      expect(result[0].content).toBe('hi there');
+      expect(result[0].getType()).toBe('human');
+
+      // AI message (all text) should now also be converted
+      expect(result[1].content).toBe('Hi Danny! How can I help you today?');
+      expect(result[1].getType()).toBe('ai');
+
+      // Third message (mixed content) should remain unchanged
+      expect(result[2].content).toEqual([
+        {
+          type: 'text',
+          text: 'what do you see',
+        },
+        {
+          type: 'image_url',
+          image_url: {
+            url: 'data:image/png;base64,iVBO_SOME_BASE64_DATA=',
+            detail: 'auto',
+          },
+        },
+      ]);
+    });
+
+    it('should handle messages with tool calls', () => {
+      const messages = [
+        new HumanMessage({
+          content: [
+            {
+              type: ContentTypes.TEXT,
+              [ContentTypes.TEXT]: 'Please use the calculator',
+            },
+            {
+              type: ContentTypes.TOOL_CALL,
+              tool_call: { name: 'calculator', args: '{"a": 1, "b": 2}' },
+            },
+          ],
+        }),
+        new AIMessage({
+          content: [
+            {
+              type: ContentTypes.TEXT,
+              [ContentTypes.TEXT]: 'I will calculate that for you',
+            },
+            {
+              type: ContentTypes.TOOL_CALL,
+              tool_call: { name: 'calculator', args: '{"a": 1, "b": 2}' },
+            },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(2);
+      // Should not convert because not all blocks are text
+      expect(result[0].content).toEqual([
+        {
+          type: ContentTypes.TEXT,
+          [ContentTypes.TEXT]: 'Please use the calculator',
+        },
+        {
+          type: ContentTypes.TOOL_CALL,
+          tool_call: { name: 'calculator', args: '{"a": 1, "b": 2}' },
+        },
+      ]);
+      expect(result[1].content).toEqual([
+        {
+          type: ContentTypes.TEXT,
+          [ContentTypes.TEXT]: 'I will calculate that for you',
+        },
+        {
+          type: ContentTypes.TOOL_CALL,
+          tool_call: { name: 'calculator', args: '{"a": 1, "b": 2}' },
+        },
+      ]);
+    });
+  });
+});

package/src/messages/content.ts

@@ -0,0 +1,63 @@
+import { ContentTypes } from '@/common';
+import type { BaseMessage } from '@langchain/core/messages';
+
+/**
+ * Formats an array of messages for LangChain, making sure all content fields are strings
+ * @param {Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>} payload - The array of messages to format.
+ * @returns {Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>} - The array of formatted LangChain messages, including ToolMessages for tool calls.
+ */
+export const formatContentStrings = (
+  payload: Array<BaseMessage>
+): Array<BaseMessage> => {
+  // Create a new array to store the processed messages
+  const result: Array<BaseMessage> = [];
+
+  for (const message of payload) {
+    const messageType = message.getType();
+    const isValidMessage =
+      messageType === 'human' ||
+      messageType === 'ai' ||
+      messageType === 'system';
+
+    if (!isValidMessage) {
+      result.push(message);
+      continue;
+    }
+
+    // If content is already a string, add as-is
+    if (typeof message.content === 'string') {
+      result.push(message);
+      continue;
+    }
+
+    // If content is not an array, add as-is
+    if (!Array.isArray(message.content)) {
+      result.push(message);
+      continue;
+    }
+
+    // Check if all content blocks are text type
+    const allTextBlocks = message.content.every(
+      (block) => block.type === ContentTypes.TEXT
+    );
+
+    // Only convert to string if all blocks are text type
+    if (!allTextBlocks) {
+      result.push(message);
+      continue;
+    }
+
+    // Reduce text types to a single string
+    const content = message.content.reduce((acc, curr) => {
+      if (curr.type === ContentTypes.TEXT) {
+        return `${acc}${curr[ContentTypes.TEXT] || ''}\n`;
+      }
+      return acc;
+    }, '');
+
+    message.content = content.trim();
+    result.push(message);
+  }
+
+  return result;
+};

package/src/messages/format.ts

@@ -565,40 +565,6 @@ export const formatAgentMessages = (
   };
 };
 
-/**
- * Formats an array of messages for LangChain, making sure all content fields are strings
- * @param payload - The array of messages to format.
- * @returns - The array of formatted LangChain messages, including ToolMessages for tool calls.
- */
-export const formatContentStrings = (
-  payload: Array<BaseMessage>
-): Array<BaseMessage> => {
-  // Create a copy of the payload to avoid modifying the original
-  const result = [...payload];
-
-  for (const message of result) {
-    if (typeof message.content === 'string') {
-      continue;
-    }
-
-    if (!Array.isArray(message.content)) {
-      continue;
-    }
-
-    // Reduce text types to a single string, ignore all other types
-    const content = message.content.reduce((acc, curr) => {
-      if (curr.type === ContentTypes.TEXT) {
-        return `${acc}${curr[ContentTypes.TEXT] || ''}\n`;
-      }
-      return acc;
-    }, '');
-
-    message.content = content.trim();
-  }
-
-  return result;
-};
-
 /**
  * Adds a value at key 0 for system messages and shifts all key indices by one in an indexTokenCountMap.
  * This is useful when adding a system message at the beginning of a conversation.

package/src/messages/index.ts

@@ -2,3 +2,5 @@ export * from './core';
 export * from './ids';
 export * from './prune';
 export * from './format';
+export * from './cache';
+export * from './content';

package/src/run.ts

@@ -1,10 +1,14 @@
 // src/run.ts
+import './instrumentation';
+import { CallbackHandler } from '@langfuse/langchain';
 import { PromptTemplate } from '@langchain/core/prompts';
+import { RunnableLambda } from '@langchain/core/runnables';
 import { AzureChatOpenAI, ChatOpenAI } from '@langchain/openai';
 import type {
-  BaseMessage,
   MessageContentComplex,
+  BaseMessage,
 } from '@langchain/core/messages';
+import type { StringPromptValue } from '@langchain/core/prompt_values';
 import type { RunnableConfig } from '@langchain/core/runnables';
 import type * as t from '@/types';
 import {
@@ -17,6 +21,7 @@ import { createTokenCounter } from '@/utils/tokens';
 import { StandardGraph } from '@/graphs/Graph';
 import { HandlerRegistry } from '@/events';
 import { isOpenAILike } from '@/utils/llm';
+import { isPresent } from '@/utils/misc';
 
 export const defaultOmitOptions = new Set([
   'stream',
@@ -239,6 +244,27 @@ export class Run<_T extends t.BaseGraphState> {
       [Callback.CUSTOM_EVENT]: customEventCallback,
     });
 
+    if (
+      isPresent(process.env.LANGFUSE_SECRET_KEY) &&
+      isPresent(process.env.LANGFUSE_PUBLIC_KEY) &&
+      isPresent(process.env.LANGFUSE_BASE_URL)
+    ) {
+      const userId = config.configurable?.user_id;
+      const sessionId = config.configurable?.thread_id;
+      const traceMetadata = {
+        messageId: this.id,
+        parentMessageId: config.configurable?.requestBody?.parentMessageId,
+      };
+      const handler = new CallbackHandler({
+        userId,
+        sessionId,
+        traceMetadata,
+      });
+      config.callbacks = (
+        (config.callbacks as t.ProvidedCallbacks) ?? []
+      ).concat([handler]);
+    }
+
     if (!this.id) {
       throw new Error('Run ID not provided');
     }
@@ -313,18 +339,38 @@ export class Run<_T extends t.BaseGraphState> {
     titleMethod = TitleMethod.COMPLETION,
     titlePromptTemplate,
   }: t.RunTitleOptions): Promise<{ language?: string; title?: string }> {
+    if (
+      chainOptions != null &&
+      isPresent(process.env.LANGFUSE_SECRET_KEY) &&
+      isPresent(process.env.LANGFUSE_PUBLIC_KEY) &&
+      isPresent(process.env.LANGFUSE_BASE_URL)
+    ) {
+      const userId = chainOptions.configurable?.user_id;
+      const sessionId = chainOptions.configurable?.thread_id;
+      const traceMetadata = {
+        messageId: 'title-' + this.id,
+      };
+      const handler = new CallbackHandler({
+        userId,
+        sessionId,
+        traceMetadata,
+      });
+      chainOptions.callbacks = (
+        (chainOptions.callbacks as t.ProvidedCallbacks) ?? []
+      ).concat([handler]);
+    }
+
     const convoTemplate = PromptTemplate.fromTemplate(
       titlePromptTemplate ?? 'User: {input}\nAI: {output}'
     );
+
     const response = contentParts
       .map((part) => {
         if (part?.type === 'text') return part.text;
         return '';
       })
       .join('\n');
-    const convo = (
-      await convoTemplate.invoke({ input: inputText, output: response })
-    ).value;
+
     const model = this.Graph?.getNewModel({
       provider,
       clientOptions,
@@ -349,25 +395,51 @@ export class Run<_T extends t.BaseGraphState> {
       model.n = (clientOptions as t.OpenAIClientOptions | undefined)
         ?.n as number;
     }
-    const chain =
+
+    const convoToTitleInput = new RunnableLambda({
+      func: (
+        promptValue: StringPromptValue
+      ): { convo: string; inputText: string; skipLanguage?: boolean } => ({
+        convo: promptValue.value,
+        inputText,
+        skipLanguage,
+      }),
+    }).withConfig({ runName: 'ConvoTransform' });
+
+    const titleChain =
       titleMethod === TitleMethod.COMPLETION
         ? await createCompletionTitleRunnable(model, titlePrompt)
         : await createTitleRunnable(model, titlePrompt);
+
+    /** Pipes `convoTemplate` -> `transformer` -> `titleChain` */
+    const fullChain = convoTemplate
+      .withConfig({ runName: 'ConvoTemplate' })
+      .pipe(convoToTitleInput)
+      .pipe(titleChain)
+      .withConfig({ runName: 'TitleChain' });
+
     const invokeConfig = Object.assign({}, chainOptions, {
       run_id: this.id,
       runId: this.id,
     });
+
     try {
-      return await chain.invoke(
-        { convo, inputText, skipLanguage },
+      return await fullChain.invoke(
+        { input: inputText, output: response },
         invokeConfig
       );
     } catch (_e) {
       // Fallback: strip callbacks to avoid EventStream tracer errors in certain environments
+      // But preserve langfuse handler if it exists
+      const langfuseHandler = (invokeConfig.callbacks as t.ProvidedCallbacks)?.find(
+        (cb) => cb instanceof CallbackHandler
+      );
       const { callbacks: _cb, ...rest } = invokeConfig;
-      const safeConfig = Object.assign({}, rest, { callbacks: [] });
-      return await chain.invoke(
-        { convo, inputText, skipLanguage },
+      const safeConfig = Object.assign({}, rest, {
+        callbacks: langfuseHandler ? [langfuseHandler] : [],
+      });
+      return await fullChain.invoke(
+        { input: inputText, output: response },
         safeConfig as Partial<RunnableConfig>
       );
     }

package/src/scripts/ant_web_search.ts

@@ -80,7 +80,7 @@ async function testStandardStreaming(): Promise<void> {
   const llmConfig = getLLMConfig(
     Providers.ANTHROPIC
   ) as t.AnthropicClientOptions & t.SharedLLMConfig;
-  llmConfig.model = 'claude-3-5-sonnet-latest';
+  llmConfig.model = 'claude-haiku-4-5';
 
   const run = await Run.create<t.IState>({
     runId: 'test-run-id',

package/src/scripts/handoff-test.ts

@@ -92,7 +92,7 @@ const transferToFlightAssistant = createHandoffTool({
 });
 
 const llm = new ChatAnthropic({
-  modelName: 'claude-3-5-sonnet-latest',
+  modelName: 'claude-haiku-4-5',
   apiKey: process.env.ANTHROPIC_API_KEY,
 });
 

package/src/scripts/multi-agent-chain.ts

@@ -71,7 +71,7 @@ async function testSequentialAgentChain() {
       agentId: 'researcher',
       provider: Providers.ANTHROPIC,
       clientOptions: {
-        modelName: 'claude-3-5-sonnet-latest',
+        modelName: 'claude-haiku-4-5',
         apiKey: process.env.ANTHROPIC_API_KEY,
       },
       instructions: `You are a Research Agent specializing in gathering initial information.
@@ -88,7 +88,7 @@ async function testSequentialAgentChain() {
       agentId: 'analyst',
       provider: Providers.ANTHROPIC,
       clientOptions: {
-        modelName: 'claude-3-5-sonnet-latest',
+        modelName: 'claude-haiku-4-5',
         apiKey: process.env.ANTHROPIC_API_KEY,
       },
       instructions: `You are an Analysis Agent that builds upon research findings.
@@ -105,7 +105,7 @@ async function testSequentialAgentChain() {
       agentId: 'reviewer',
       provider: Providers.ANTHROPIC,
       clientOptions: {
-        modelName: 'claude-3-5-sonnet-latest',
+        modelName: 'claude-haiku-4-5',
         apiKey: process.env.ANTHROPIC_API_KEY,
       },
       instructions: `You are a Critical Review Agent that evaluates the work done so far.
@@ -122,7 +122,7 @@ async function testSequentialAgentChain() {
       agentId: 'summarizer',
       provider: Providers.ANTHROPIC,
       clientOptions: {
-        modelName: 'claude-3-5-sonnet-latest',
+        modelName: 'claude-haiku-4-5',
         apiKey: process.env.ANTHROPIC_API_KEY,
       },
       instructions: `You are a Summary Agent that creates the final comprehensive output.