@librechat/agents 3.0.0-rc1 → 3.0.0-rc11

package/src/llm/openai/index.ts

@@ -30,6 +30,7 @@ import {
   _convertOpenAIResponsesDeltaToBaseMessageChunk,
   type ResponseReturnStreamEvents,
 } from './utils';
+import { sleep } from '@/utils';
 
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
 const iife = <T>(fn: () => T) => fn();
@@ -195,9 +196,23 @@ export class CustomAzureOpenAIClient extends AzureOpenAIClient {
 
 /** @ts-expect-error We are intentionally overriding `getReasoningParams` */
 export class ChatOpenAI extends OriginalChatOpenAI<t.ChatOpenAICallOptions> {
+  _lc_stream_delay?: number;
+
+  constructor(
+    fields?: t.ChatOpenAICallOptions & {
+      _lc_stream_delay?: number;
+    } & t.OpenAIChatInput['modelKwargs']
+  ) {
+    super(fields);
+    this._lc_stream_delay = fields?._lc_stream_delay;
+  }
+
   public get exposedClient(): CustomOpenAIClient {
     return this.client;
   }
+  static lc_name(): string {
+    return 'LibreChatOpenAI';
+  }
   protected _getClientOptions(
     options?: OpenAICoreRequestOptions
   ): OpenAICoreRequestOptions {
@@ -233,7 +248,8 @@ export class ChatOpenAI extends OriginalChatOpenAI<t.ChatOpenAICallOptions> {
   getReasoningParams(
     options?: this['ParsedCallOptions']
   ): OpenAIClient.Reasoning | undefined {
-    if (!isReasoningModel(this.model)) {
+    const lc_name = (this.constructor as typeof ChatOpenAI).lc_name();
+    if (lc_name === 'LibreChatOpenAI' && !isReasoningModel(this.model)) {
       return;
     }
 
@@ -288,6 +304,9 @@ export class ChatOpenAI extends OriginalChatOpenAI<t.ChatOpenAICallOptions> {
       );
       if (chunk == null) continue;
       yield chunk;
+      if (this._lc_stream_delay != null) {
+        await sleep(this._lc_stream_delay);
+      }
       await runManager?.handleLLMNewToken(
         chunk.text || '',
         undefined,
@@ -345,6 +364,10 @@ export class ChatOpenAI extends OriginalChatOpenAI<t.ChatOpenAICallOptions> {
       } else if ('reasoning' in delta) {
         chunk.additional_kwargs.reasoning_content = delta.reasoning;
       }
+      if ('provider_specific_fields' in delta) {
+        chunk.additional_kwargs.provider_specific_fields =
+          delta.provider_specific_fields;
+      }
       defaultRole = delta.role ?? defaultRole;
       const newTokenIndices = {
         prompt: options.promptIndex ?? 0,
@@ -376,6 +399,9 @@ export class ChatOpenAI extends OriginalChatOpenAI<t.ChatOpenAICallOptions> {
         generationInfo,
       });
       yield generationChunk;
+      if (this._lc_stream_delay != null) {
+        await sleep(this._lc_stream_delay);
+      }
       await runManager?.handleLLMNewToken(
         generationChunk.text || '',
         newTokenIndices,
@@ -423,6 +449,9 @@ export class ChatOpenAI extends OriginalChatOpenAI<t.ChatOpenAICallOptions> {
         text: '',
       });
       yield generationChunk;
+      if (this._lc_stream_delay != null) {
+        await sleep(this._lc_stream_delay);
+      }
     }
     if (options.signal?.aborted === true) {
       throw new Error('AbortError');
@@ -432,9 +461,19 @@ export class ChatOpenAI extends OriginalChatOpenAI<t.ChatOpenAICallOptions> {
 
 /** @ts-expect-error We are intentionally overriding `getReasoningParams` */
 export class AzureChatOpenAI extends OriginalAzureChatOpenAI {
+  _lc_stream_delay?: number;
+
+  constructor(fields?: t.AzureOpenAIInput & { _lc_stream_delay?: number }) {
+    super(fields);
+    this._lc_stream_delay = fields?._lc_stream_delay;
+  }
+
   public get exposedClient(): CustomOpenAIClient {
     return this.client;
   }
+  static lc_name(): 'LibreChatAzureOpenAI' {
+    return 'LibreChatAzureOpenAI';
+  }
   /**
    * Returns backwards compatible reasoning parameters from constructor params and call options
    * @internal
@@ -559,6 +598,9 @@ export class AzureChatOpenAI extends OriginalAzureChatOpenAI {
       );
       if (chunk == null) continue;
       yield chunk;
+      if (this._lc_stream_delay != null) {
+        await sleep(this._lc_stream_delay);
+      }
       await runManager?.handleLLMNewToken(
         chunk.text || '',
         undefined,
@@ -576,6 +618,9 @@ export class ChatDeepSeek extends OriginalChatDeepSeek {
   public get exposedClient(): CustomOpenAIClient {
     return this.client;
   }
+  static lc_name(): 'LibreChatDeepSeek' {
+    return 'LibreChatDeepSeek';
+  }
   protected _getClientOptions(
     options?: OpenAICoreRequestOptions
   ): OpenAICoreRequestOptions {
@@ -624,13 +669,17 @@ export interface XAIUsageMetadata
 }
 
 export class ChatXAI extends OriginalChatXAI {
+  _lc_stream_delay?: number;
+
   constructor(
     fields?: Partial<ChatXAIInput> & {
       configuration?: { baseURL?: string };
       clientConfig?: { baseURL?: string };
+      _lc_stream_delay?: number;
     }
   ) {
     super(fields);
+    this._lc_stream_delay = fields?._lc_stream_delay;
     const customBaseURL =
       fields?.configuration?.baseURL ?? fields?.clientConfig?.baseURL;
     if (customBaseURL != null && customBaseURL) {
@@ -644,6 +693,10 @@ export class ChatXAI extends OriginalChatXAI {
     }
   }
 
+  static lc_name(): 'LibreChatXAI' {
+    return 'LibreChatXAI';
+  }
+
   public get exposedClient(): CustomOpenAIClient {
     return this.client;
   }
@@ -759,6 +812,9 @@ export class ChatXAI extends OriginalChatXAI {
         generationInfo,
       });
       yield generationChunk;
+      if (this._lc_stream_delay != null) {
+        await sleep(this._lc_stream_delay);
+      }
       await runManager?.handleLLMNewToken(
         generationChunk.text || '',
         newTokenIndices,
@@ -832,6 +888,9 @@ export class ChatXAI extends OriginalChatXAI {
         text: '',
       });
       yield generationChunk;
+      if (this._lc_stream_delay != null) {
+        await sleep(this._lc_stream_delay);
+      }
     }
     if (options.signal?.aborted === true) {
       throw new Error('AbortError');

package/src/llm/openai/utils/index.ts

@@ -298,10 +298,16 @@ export function _convertMessagesToOpenAIParams(
       role = 'developer';
     }
 
+    let hasAnthropicThinkingBlock: boolean = false;
+
     const content =
       typeof message.content === 'string'
         ? message.content
         : message.content.map((m) => {
+          if ('type' in m && m.type === 'thinking') {
+            hasAnthropicThinkingBlock = true;
+            return m;
+          }
           if (isDataContentBlock(m)) {
             return convertToProviderContentBlock(
               m,
@@ -326,7 +332,7 @@ export function _convertMessagesToOpenAIParams(
       completionParam.tool_calls = message.tool_calls.map(
         convertLangChainToolCallToOpenAI
       );
-      completionParam.content = '';
+      completionParam.content = hasAnthropicThinkingBlock ? content : '';
     } else {
       if (message.additional_kwargs.tool_calls != null) {
         completionParam.tool_calls = message.additional_kwargs.tool_calls;

package/src/llm/openrouter/index.ts

@@ -1,27 +1,36 @@
-import type { ChatOpenAICallOptions, OpenAIClient } from '@langchain/openai';
+import { ChatOpenAI } from '@/llm/openai';
 import type {
-  AIMessageChunk,
-  HumanMessageChunk,
-  SystemMessageChunk,
   FunctionMessageChunk,
+  SystemMessageChunk,
+  HumanMessageChunk,
   ToolMessageChunk,
   ChatMessageChunk,
+  AIMessageChunk,
 } from '@langchain/core/messages';
-import { ChatOpenAI } from '@/llm/openai';
+import type {
+  ChatOpenAICallOptions,
+  OpenAIChatInput,
+  OpenAIClient,
+} from '@langchain/openai';
 
 export interface ChatOpenRouterCallOptions extends ChatOpenAICallOptions {
   include_reasoning?: boolean;
+  modelKwargs?: OpenAIChatInput['modelKwargs'];
 }
 export class ChatOpenRouter extends ChatOpenAI {
   constructor(_fields: Partial<ChatOpenRouterCallOptions>) {
-    const { include_reasoning, ...fields } = _fields;
+    const { include_reasoning, modelKwargs = {}, ...fields } = _fields;
     super({
       ...fields,
       modelKwargs: {
+        ...modelKwargs,
         include_reasoning,
       },
     });
   }
+  static lc_name(): 'LibreChatOpenRouter' {
+    return 'LibreChatOpenRouter';
+  }
   protected override _convertOpenAIDeltaToBaseMessageChunk(
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     delta: Record<string, any>,

package/src/llm/vertexai/index.ts

@@ -313,8 +313,8 @@ export class ChatVertexAI extends ChatGoogle {
   lc_namespace = ['langchain', 'chat_models', 'vertexai'];
   dynamicThinkingBudget = false;
 
-  static lc_name(): 'ChatVertexAI' {
-    return 'ChatVertexAI';
+  static lc_name(): 'LibreChatVertexAI' {
+    return 'LibreChatVertexAI';
   }
 
   constructor(fields?: VertexAIClientOptions) {

package/src/messages/core.ts

@@ -321,12 +321,15 @@ export function convertMessagesToContent(
       const id = (message as ToolMessage).tool_call_id;
       const output = (message as ToolMessage).content;
       const tool_call = toolCallMap.get(id);
-
+      if (currentAIMessageIndex === -1) {
+        processedContent.push({ type: 'text', text: '' });
+        currentAIMessageIndex = processedContent.length - 1;
+      }
+      const contentPart = processedContent[currentAIMessageIndex];
       processedContent.push({
         type: 'tool_call',
         tool_call: Object.assign({}, tool_call, { output }),
       });
-      const contentPart = processedContent[currentAIMessageIndex];
       const tool_call_ids = contentPart.tool_call_ids || [];
       tool_call_ids.push(id);
       contentPart.tool_call_ids = tool_call_ids;

package/src/messages/format.ts

@@ -17,28 +17,28 @@ import type {
 } from '@/types';
 import { Providers, ContentTypes } from '@/common';
 
-interface VisionMessageParams {
+interface MediaMessageParams {
   message: {
     role: string;
     content: string;
     name?: string;
     [key: string]: any;
   };
-  image_urls: MessageContentImageUrl[];
+  mediaParts: MessageContentComplex[];
   endpoint?: Providers;
 }
 
 /**
- * Formats a message to OpenAI Vision API payload format.
+ * Formats a message with media content (images, documents, videos, audios) to API payload format.
  *
- * @param {VisionMessageParams} params - The parameters for formatting.
- * @returns {Object} - The formatted message.
+ * @param params - The parameters for formatting.
+ * @returns - The formatted message.
  */
-export const formatVisionMessage = ({
+export const formatMediaMessage = ({
   message,
-  image_urls,
   endpoint,
-}: VisionMessageParams): {
+  mediaParts,
+}: MediaMessageParams): {
   role: string;
   content: MessageContentComplex[];
   name?: string;
@@ -57,7 +57,7 @@ export const formatVisionMessage = ({
 
   if (endpoint === Providers.ANTHROPIC) {
     result.content = [
-      ...image_urls,
+      ...mediaParts,
       { type: ContentTypes.TEXT, text: message.content },
     ] as MessageContentComplex[];
     return result;
@@ -65,7 +65,7 @@ export const formatVisionMessage = ({
 
   result.content = [
     { type: ContentTypes.TEXT, text: message.content },
-    ...image_urls,
+    ...mediaParts,
   ] as MessageContentComplex[];
 
   return result;
@@ -78,6 +78,9 @@ interface MessageInput {
   text?: string;
   content?: string | MessageContentComplex[];
   image_urls?: MessageContentImageUrl[];
+  documents?: MessageContentComplex[];
+  videos?: MessageContentComplex[];
+  audios?: MessageContentComplex[];
   lc_id?: string[];
   [key: string]: any;
 }
@@ -100,14 +103,14 @@ interface FormattedMessage {
 /**
  * Formats a message to OpenAI payload format based on the provided options.
  *
- * @param {FormatMessageParams} params - The parameters for formatting.
- * @returns {FormattedMessage | HumanMessage | AIMessage | SystemMessage} - The formatted message.
+ * @param params - The parameters for formatting.
+ * @returns - The formatted message.
  */
 export const formatMessage = ({
   message,
   userName,
-  assistantName,
   endpoint,
+  assistantName,
   langChain = false,
 }: FormatMessageParams):
   | FormattedMessage
@@ -135,21 +138,7 @@ export const formatMessage = ({
     content,
   };
 
-  const { image_urls } = message;
-  if (Array.isArray(image_urls) && image_urls.length > 0 && role === 'user') {
-    return formatVisionMessage({
-      message: {
-        ...formattedMessage,
-        content:
-          typeof formattedMessage.content === 'string'
-            ? formattedMessage.content
-            : '',
-      },
-      image_urls,
-      endpoint,
-    });
-  }
-
+  // Set name fields first
   if (_name != null && _name) {
     formattedMessage.name = _name;
   }
@@ -179,6 +168,45 @@ export const formatMessage = ({
     }
   }
 
+  const { image_urls, documents, videos, audios } = message;
+  const mediaParts: MessageContentComplex[] = [];
+
+  if (Array.isArray(documents) && documents.length > 0) {
+    mediaParts.push(...documents);
+  }
+
+  if (Array.isArray(videos) && videos.length > 0) {
+    mediaParts.push(...videos);
+  }
+
+  if (Array.isArray(audios) && audios.length > 0) {
+    mediaParts.push(...audios);
+  }
+
+  if (Array.isArray(image_urls) && image_urls.length > 0) {
+    mediaParts.push(...image_urls);
+  }
+
+  if (mediaParts.length > 0 && role === 'user') {
+    const mediaMessage = formatMediaMessage({
+      message: {
+        ...formattedMessage,
+        content:
+          typeof formattedMessage.content === 'string'
+            ? formattedMessage.content
+            : '',
+      },
+      mediaParts,
+      endpoint,
+    });
+
+    if (!langChain) {
+      return mediaMessage;
+    }
+
+    return new HumanMessage(mediaMessage);
+  }
+
   if (!langChain) {
     return formattedMessage;
   }
@@ -195,9 +223,9 @@ export const formatMessage = ({
 /**
  * Formats an array of messages for LangChain.
  *
- * @param {Array<MessageInput>} messages - The array of messages to format.
- * @param {Omit<FormatMessageParams, 'message' | 'langChain'>} formatOptions - The options for formatting each message.
- * @returns {Array<HumanMessage | AIMessage | SystemMessage>} - The array of formatted LangChain messages.
+ * @param messages - The array of messages to format.
+ * @param formatOptions - The options for formatting each message.
+ * @returns - The array of formatted LangChain messages.
  */
 export const formatLangChainMessages = (
   messages: Array<MessageInput>,
@@ -228,8 +256,8 @@ interface LangChainMessage {
 /**
  * Formats a LangChain message object by merging properties from `lc_kwargs` or `kwargs` and `additional_kwargs`.
  *
- * @param {LangChainMessage} message - The message object to format.
- * @returns {Record<string, any>} The formatted LangChain message.
+ * @param message - The message object to format.
+ * @returns - The formatted LangChain message.
  */
 export const formatFromLangChain = (
   message: LangChainMessage
@@ -357,10 +385,10 @@ function formatAssistantMessage(
 /**
  * Formats an array of messages for LangChain, handling tool calls and creating ToolMessage instances.
  *
- * @param {TPayload} payload - The array of messages to format.
- * @param {Record<number, number>} [indexTokenCountMap] - Optional map of message indices to token counts.
- * @param {Set<string>} [tools] - Optional set of tool names that are allowed in the request.
- * @returns {Object} - Object containing formatted messages and updated indexTokenCountMap if provided.
+ * @param payload - The array of messages to format.
+ * @param indexTokenCountMap - Optional map of message indices to token counts.
+ * @param tools - Optional set of tool names that are allowed in the request.
+ * @returns - Object containing formatted messages and updated indexTokenCountMap if provided.
  */
 export const formatAgentMessages = (
   payload: TPayload,
@@ -539,8 +567,8 @@ export const formatAgentMessages = (
 
 /**
  * Formats an array of messages for LangChain, making sure all content fields are strings
- * @param {Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>} payload - The array of messages to format.
- * @returns {Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>} - The array of formatted LangChain messages, including ToolMessages for tool calls.
+ * @param payload - The array of messages to format.
+ * @returns - The array of formatted LangChain messages, including ToolMessages for tool calls.
  */
 export const formatContentStrings = (
   payload: Array<BaseMessage>