@librechat/agents 3.0.0-rc9 → 3.0.0

package/src/messages/content.test.ts

@@ -0,0 +1,362 @@
+import {
+  HumanMessage,
+  AIMessage,
+  SystemMessage,
+} from '@langchain/core/messages';
+import { formatContentStrings } from './content';
+import { ContentTypes } from '@/common';
+
+describe('formatContentStrings', () => {
+  describe('Human messages', () => {
+    it('should convert human message with all text blocks to string', () => {
+      const messages = [
+        new HumanMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Hello' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'World' },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe('Hello\nWorld');
+    });
+
+    it('should not convert human message with mixed content types (text + image)', () => {
+      const messages = [
+        new HumanMessage({
+          content: [
+            { type: ContentTypes.TEXT, text: 'what do you see' },
+            {
+              type: 'image_url',
+              image_url: {
+                url: 'data:image/png;base64,iVBO_SOME_BASE64_DATA=',
+                detail: 'auto',
+              },
+            },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toEqual([
+        { type: ContentTypes.TEXT, text: 'what do you see' },
+        {
+          type: 'image_url',
+          image_url: {
+            url: 'data:image/png;base64,iVBO_SOME_BASE64_DATA=',
+            detail: 'auto',
+          },
+        },
+      ]);
+    });
+
+    it('should leave string content unchanged', () => {
+      const messages = [
+        new HumanMessage({
+          content: 'Hello World',
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe('Hello World');
+    });
+
+    it('should handle empty text blocks', () => {
+      const messages = [
+        new HumanMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Hello' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: '' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'World' },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe('Hello\n\nWorld');
+    });
+
+    it('should handle null/undefined text values', () => {
+      const messages = [
+        new HumanMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Hello' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: null },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: undefined },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'World' },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe('Hello\n\n\nWorld');
+    });
+  });
+
+  describe('AI messages', () => {
+    it('should convert AI message with all text blocks to string', () => {
+      const messages = [
+        new AIMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Hello' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'World' },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe('Hello\nWorld');
+      expect(result[0].getType()).toBe('ai');
+    });
+
+    it('should not convert AI message with mixed content types', () => {
+      const messages = [
+        new AIMessage({
+          content: [
+            {
+              type: ContentTypes.TEXT,
+              [ContentTypes.TEXT]: 'Here is an image',
+            },
+            {
+              type: ContentTypes.TOOL_CALL,
+              tool_call: { name: 'generate_image' },
+            },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toEqual([
+        { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Here is an image' },
+        { type: ContentTypes.TOOL_CALL, tool_call: { name: 'generate_image' } },
+      ]);
+    });
+  });
+
+  describe('System messages', () => {
+    it('should convert System message with all text blocks to string', () => {
+      const messages = [
+        new SystemMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'System' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Message' },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe('System\nMessage');
+      expect(result[0].getType()).toBe('system');
+    });
+  });
+
+  describe('Mixed message types', () => {
+    it('should process all valid message types in mixed array', () => {
+      const messages = [
+        new HumanMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Human' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Message' },
+          ],
+        }),
+        new AIMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'AI' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Response' },
+          ],
+        }),
+        new SystemMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'System' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Prompt' },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(3);
+      // All messages should be converted
+      expect(result[0].content).toBe('Human\nMessage');
+      expect(result[0].getType()).toBe('human');
+
+      expect(result[1].content).toBe('AI\nResponse');
+      expect(result[1].getType()).toBe('ai');
+
+      expect(result[2].content).toBe('System\nPrompt');
+      expect(result[2].getType()).toBe('system');
+    });
+  });
+
+  describe('Edge cases', () => {
+    it('should handle empty array', () => {
+      const result = formatContentStrings([]);
+      expect(result).toEqual([]);
+    });
+
+    it('should handle messages with non-array content', () => {
+      const messages = [
+        new HumanMessage({
+          content: 'This is a string content',
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe('This is a string content');
+    });
+
+    it('should trim the final concatenated string', () => {
+      const messages = [
+        new HumanMessage({
+          content: [
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: '  Hello  ' },
+            { type: ContentTypes.TEXT, [ContentTypes.TEXT]: '  World  ' },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(1);
+      expect(result[0].content).toBe('Hello  \n  World');
+    });
+  });
+
+  describe('Real-world scenarios', () => {
+    it('should handle the exact scenario from the issue', () => {
+      const messages = [
+        new HumanMessage({
+          content: [
+            {
+              type: 'text',
+              text: 'hi there',
+            },
+          ],
+        }),
+        new AIMessage({
+          content: [
+            {
+              type: 'text',
+              text: 'Hi Danny! How can I help you today?',
+            },
+          ],
+        }),
+        new HumanMessage({
+          content: [
+            {
+              type: 'text',
+              text: 'what do you see',
+            },
+            {
+              type: 'image_url',
+              image_url: {
+                url: 'data:image/png;base64,iVBO_SOME_BASE64_DATA=',
+                detail: 'auto',
+              },
+            },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(3);
+
+      // First human message (all text) should be converted
+      expect(result[0].content).toBe('hi there');
+      expect(result[0].getType()).toBe('human');
+
+      // AI message (all text) should now also be converted
+      expect(result[1].content).toBe('Hi Danny! How can I help you today?');
+      expect(result[1].getType()).toBe('ai');
+
+      // Third message (mixed content) should remain unchanged
+      expect(result[2].content).toEqual([
+        {
+          type: 'text',
+          text: 'what do you see',
+        },
+        {
+          type: 'image_url',
+          image_url: {
+            url: 'data:image/png;base64,iVBO_SOME_BASE64_DATA=',
+            detail: 'auto',
+          },
+        },
+      ]);
+    });
+
+    it('should handle messages with tool calls', () => {
+      const messages = [
+        new HumanMessage({
+          content: [
+            {
+              type: ContentTypes.TEXT,
+              [ContentTypes.TEXT]: 'Please use the calculator',
+            },
+            {
+              type: ContentTypes.TOOL_CALL,
+              tool_call: { name: 'calculator', args: '{"a": 1, "b": 2}' },
+            },
+          ],
+        }),
+        new AIMessage({
+          content: [
+            {
+              type: ContentTypes.TEXT,
+              [ContentTypes.TEXT]: 'I will calculate that for you',
+            },
+            {
+              type: ContentTypes.TOOL_CALL,
+              tool_call: { name: 'calculator', args: '{"a": 1, "b": 2}' },
+            },
+          ],
+        }),
+      ];
+
+      const result = formatContentStrings(messages);
+
+      expect(result).toHaveLength(2);
+      // Should not convert because not all blocks are text
+      expect(result[0].content).toEqual([
+        {
+          type: ContentTypes.TEXT,
+          [ContentTypes.TEXT]: 'Please use the calculator',
+        },
+        {
+          type: ContentTypes.TOOL_CALL,
+          tool_call: { name: 'calculator', args: '{"a": 1, "b": 2}' },
+        },
+      ]);
+      expect(result[1].content).toEqual([
+        {
+          type: ContentTypes.TEXT,
+          [ContentTypes.TEXT]: 'I will calculate that for you',
+        },
+        {
+          type: ContentTypes.TOOL_CALL,
+          tool_call: { name: 'calculator', args: '{"a": 1, "b": 2}' },
+        },
+      ]);
+    });
+  });
+});

package/src/messages/content.ts

@@ -0,0 +1,63 @@
+import { ContentTypes } from '@/common';
+import type { BaseMessage } from '@langchain/core/messages';
+
+/**
+ * Formats an array of messages for LangChain, making sure all content fields are strings
+ * @param {Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>} payload - The array of messages to format.
+ * @returns {Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>} - The array of formatted LangChain messages, including ToolMessages for tool calls.
+ */
+export const formatContentStrings = (
+  payload: Array<BaseMessage>
+): Array<BaseMessage> => {
+  // Create a new array to store the processed messages
+  const result: Array<BaseMessage> = [];
+
+  for (const message of payload) {
+    const messageType = message.getType();
+    const isValidMessage =
+      messageType === 'human' ||
+      messageType === 'ai' ||
+      messageType === 'system';
+
+    if (!isValidMessage) {
+      result.push(message);
+      continue;
+    }
+
+    // If content is already a string, add as-is
+    if (typeof message.content === 'string') {
+      result.push(message);
+      continue;
+    }
+
+    // If content is not an array, add as-is
+    if (!Array.isArray(message.content)) {
+      result.push(message);
+      continue;
+    }
+
+    // Check if all content blocks are text type
+    const allTextBlocks = message.content.every(
+      (block) => block.type === ContentTypes.TEXT
+    );
+
+    // Only convert to string if all blocks are text type
+    if (!allTextBlocks) {
+      result.push(message);
+      continue;
+    }
+
+    // Reduce text types to a single string
+    const content = message.content.reduce((acc, curr) => {
+      if (curr.type === ContentTypes.TEXT) {
+        return `${acc}${curr[ContentTypes.TEXT] || ''}\n`;
+      }
+      return acc;
+    }, '');
+
+    message.content = content.trim();
+    result.push(message);
+  }
+
+  return result;
+};

package/src/messages/core.ts

@@ -321,12 +321,15 @@ export function convertMessagesToContent(
       const id = (message as ToolMessage).tool_call_id;
       const output = (message as ToolMessage).content;
       const tool_call = toolCallMap.get(id);
-
+      if (currentAIMessageIndex === -1) {
+        processedContent.push({ type: 'text', text: '' });
+        currentAIMessageIndex = processedContent.length - 1;
+      }
+      const contentPart = processedContent[currentAIMessageIndex];
       processedContent.push({
         type: 'tool_call',
         tool_call: Object.assign({}, tool_call, { output }),
       });
-      const contentPart = processedContent[currentAIMessageIndex];
       const tool_call_ids = contentPart.tool_call_ids || [];
       tool_call_ids.push(id);
       contentPart.tool_call_ids = tool_call_ids;

package/src/messages/format.ts

@@ -17,28 +17,28 @@ import type {
 } from '@/types';
 import { Providers, ContentTypes } from '@/common';
 
-interface VisionMessageParams {
+interface MediaMessageParams {
   message: {
     role: string;
     content: string;
     name?: string;
     [key: string]: any;
   };
-  image_urls: MessageContentImageUrl[];
+  mediaParts: MessageContentComplex[];
   endpoint?: Providers;
 }
 
 /**
- * Formats a message to OpenAI Vision API payload format.
+ * Formats a message with media content (images, documents, videos, audios) to API payload format.
  *
- * @param {VisionMessageParams} params - The parameters for formatting.
- * @returns {Object} - The formatted message.
+ * @param params - The parameters for formatting.
+ * @returns - The formatted message.
  */
-export const formatVisionMessage = ({
+export const formatMediaMessage = ({
   message,
-  image_urls,
   endpoint,
-}: VisionMessageParams): {
+  mediaParts,
+}: MediaMessageParams): {
   role: string;
   content: MessageContentComplex[];
   name?: string;
@@ -57,7 +57,7 @@ export const formatVisionMessage = ({
 
   if (endpoint === Providers.ANTHROPIC) {
     result.content = [
-      ...image_urls,
+      ...mediaParts,
       { type: ContentTypes.TEXT, text: message.content },
     ] as MessageContentComplex[];
     return result;
@@ -65,7 +65,7 @@ export const formatVisionMessage = ({
 
   result.content = [
     { type: ContentTypes.TEXT, text: message.content },
-    ...image_urls,
+    ...mediaParts,
   ] as MessageContentComplex[];
 
   return result;
@@ -78,6 +78,9 @@ interface MessageInput {
   text?: string;
   content?: string | MessageContentComplex[];
   image_urls?: MessageContentImageUrl[];
+  documents?: MessageContentComplex[];
+  videos?: MessageContentComplex[];
+  audios?: MessageContentComplex[];
   lc_id?: string[];
   [key: string]: any;
 }
@@ -100,14 +103,14 @@ interface FormattedMessage {
 /**
  * Formats a message to OpenAI payload format based on the provided options.
  *
- * @param {FormatMessageParams} params - The parameters for formatting.
- * @returns {FormattedMessage | HumanMessage | AIMessage | SystemMessage} - The formatted message.
+ * @param params - The parameters for formatting.
+ * @returns - The formatted message.
  */
 export const formatMessage = ({
   message,
   userName,
-  assistantName,
   endpoint,
+  assistantName,
   langChain = false,
 }: FormatMessageParams):
   | FormattedMessage
@@ -135,21 +138,7 @@ export const formatMessage = ({
     content,
   };
 
-  const { image_urls } = message;
-  if (Array.isArray(image_urls) && image_urls.length > 0 && role === 'user') {
-    return formatVisionMessage({
-      message: {
-        ...formattedMessage,
-        content:
-          typeof formattedMessage.content === 'string'
-            ? formattedMessage.content
-            : '',
-      },
-      image_urls,
-      endpoint,
-    });
-  }
-
+  // Set name fields first
   if (_name != null && _name) {
     formattedMessage.name = _name;
   }
@@ -179,6 +168,45 @@ export const formatMessage = ({
     }
   }
 
+  const { image_urls, documents, videos, audios } = message;
+  const mediaParts: MessageContentComplex[] = [];
+
+  if (Array.isArray(documents) && documents.length > 0) {
+    mediaParts.push(...documents);
+  }
+
+  if (Array.isArray(videos) && videos.length > 0) {
+    mediaParts.push(...videos);
+  }
+
+  if (Array.isArray(audios) && audios.length > 0) {
+    mediaParts.push(...audios);
+  }
+
+  if (Array.isArray(image_urls) && image_urls.length > 0) {
+    mediaParts.push(...image_urls);
+  }
+
+  if (mediaParts.length > 0 && role === 'user') {
+    const mediaMessage = formatMediaMessage({
+      message: {
+        ...formattedMessage,
+        content:
+          typeof formattedMessage.content === 'string'
+            ? formattedMessage.content
+            : '',
+      },
+      mediaParts,
+      endpoint,
+    });
+
+    if (!langChain) {
+      return mediaMessage;
+    }
+
+    return new HumanMessage(mediaMessage);
+  }
+
   if (!langChain) {
     return formattedMessage;
   }
@@ -195,9 +223,9 @@ export const formatMessage = ({
 /**
  * Formats an array of messages for LangChain.
  *
- * @param {Array<MessageInput>} messages - The array of messages to format.
- * @param {Omit<FormatMessageParams, 'message' | 'langChain'>} formatOptions - The options for formatting each message.
- * @returns {Array<HumanMessage | AIMessage | SystemMessage>} - The array of formatted LangChain messages.
+ * @param messages - The array of messages to format.
+ * @param formatOptions - The options for formatting each message.
+ * @returns - The array of formatted LangChain messages.
  */
 export const formatLangChainMessages = (
   messages: Array<MessageInput>,
@@ -228,8 +256,8 @@ interface LangChainMessage {
 /**
  * Formats a LangChain message object by merging properties from `lc_kwargs` or `kwargs` and `additional_kwargs`.
  *
- * @param {LangChainMessage} message - The message object to format.
- * @returns {Record<string, any>} The formatted LangChain message.
+ * @param message - The message object to format.
+ * @returns - The formatted LangChain message.
  */
 export const formatFromLangChain = (
   message: LangChainMessage
@@ -357,10 +385,10 @@ function formatAssistantMessage(
 /**
  * Formats an array of messages for LangChain, handling tool calls and creating ToolMessage instances.
  *
- * @param {TPayload} payload - The array of messages to format.
- * @param {Record<number, number>} [indexTokenCountMap] - Optional map of message indices to token counts.
- * @param {Set<string>} [tools] - Optional set of tool names that are allowed in the request.
- * @returns {Object} - Object containing formatted messages and updated indexTokenCountMap if provided.
+ * @param payload - The array of messages to format.
+ * @param indexTokenCountMap - Optional map of message indices to token counts.
+ * @param tools - Optional set of tool names that are allowed in the request.
+ * @returns - Object containing formatted messages and updated indexTokenCountMap if provided.
  */
 export const formatAgentMessages = (
   payload: TPayload,
@@ -537,40 +565,6 @@ export const formatAgentMessages = (
   };
 };
 
-/**
- * Formats an array of messages for LangChain, making sure all content fields are strings
- * @param {Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>} payload - The array of messages to format.
- * @returns {Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>} - The array of formatted LangChain messages, including ToolMessages for tool calls.
- */
-export const formatContentStrings = (
-  payload: Array<BaseMessage>
-): Array<BaseMessage> => {
-  // Create a copy of the payload to avoid modifying the original
-  const result = [...payload];
-
-  for (const message of result) {
-    if (typeof message.content === 'string') {
-      continue;
-    }
-
-    if (!Array.isArray(message.content)) {
-      continue;
-    }
-
-    // Reduce text types to a single string, ignore all other types
-    const content = message.content.reduce((acc, curr) => {
-      if (curr.type === ContentTypes.TEXT) {
-        return `${acc}${curr[ContentTypes.TEXT] || ''}\n`;
-      }
-      return acc;
-    }, '');
-
-    message.content = content.trim();
-  }
-
-  return result;
-};
-
 /**
  * Adds a value at key 0 for system messages and shifts all key indices by one in an indexTokenCountMap.
  * This is useful when adding a system message at the beginning of a conversation.