@librechat/agents 3.1.75 → 3.1.77-dev.1

package/src/llm/openai/utils/index.ts

@@ -16,7 +16,7 @@ import {
   isAIMessage,
   type UsageMetadata,
   type BaseMessageFields,
-  type MessageContent,
+  type MessageContentComplex,
   type InvalidToolCall,
   type MessageContentImageUrl,
   StandardContentBlockConverter,
@@ -37,6 +37,7 @@ import type {
   OpenAIChatInput,
   ChatOpenAIReasoningSummary,
 } from '@langchain/openai';
+import { toLangChainContent } from '@/messages/langchain';
 
 export type { OpenAICallOptions, OpenAIChatInput };
 
@@ -302,6 +303,7 @@ export function _convertMessagesToOpenAIParams(
   model?: string,
   options?: ConvertMessagesOptions
 ): OpenAICompletionParam[] {
+  let hasReasoningToolCallContext = false;
   // TODO: Function messages do not support array content, fix cast
   return messages.flatMap((message) => {
     let role = messageToOpenAIRole(message);
@@ -332,6 +334,8 @@ export function _convertMessagesToOpenAIParams(
       role,
       content,
     };
+    let messageHasToolCalls = false;
+    let messageIsToolResult = false;
     if (message.name != null) {
       completionParam.name = message.name;
     }
@@ -340,17 +344,11 @@ export function _convertMessagesToOpenAIParams(
       completionParam.content = '';
     }
     if (isAIMessage(message) && !!message.tool_calls?.length) {
+      messageHasToolCalls = true;
       completionParam.tool_calls = message.tool_calls.map(
         convertLangChainToolCallToOpenAI
       );
       completionParam.content = hasAnthropicThinkingBlock ? content : '';
-      if (
-        options?.includeReasoningContent === true &&
-        message.additional_kwargs.reasoning_content != null
-      ) {
-        completionParam.reasoning_content =
-          message.additional_kwargs.reasoning_content;
-      }
       if (
         options?.includeReasoningDetails === true &&
         message.additional_kwargs.reasoning_details != null
@@ -398,14 +396,10 @@ export function _convertMessagesToOpenAIParams(
       }
     } else {
       if (message.additional_kwargs.tool_calls != null) {
+        messageHasToolCalls =
+          !Array.isArray(message.additional_kwargs.tool_calls) ||
+          message.additional_kwargs.tool_calls.length > 0;
         completionParam.tool_calls = message.additional_kwargs.tool_calls;
-        if (
-          options?.includeReasoningContent === true &&
-          message.additional_kwargs.reasoning_content != null
-        ) {
-          completionParam.reasoning_content =
-            message.additional_kwargs.reasoning_content;
-        }
         if (
           options?.includeReasoningDetails === true &&
           message.additional_kwargs.reasoning_details != null
@@ -453,10 +447,26 @@ export function _convertMessagesToOpenAIParams(
         }
       }
       if ((message as ToolMessage).tool_call_id != null) {
+        messageIsToolResult = true;
         completionParam.tool_call_id = (message as ToolMessage).tool_call_id;
       }
     }
 
+    if (
+      options?.includeReasoningContent === true &&
+      isAIMessage(message) &&
+      (hasReasoningToolCallContext || messageHasToolCalls) &&
+      typeof message.additional_kwargs.reasoning_content === 'string' &&
+      message.additional_kwargs.reasoning_content !== ''
+    ) {
+      completionParam.reasoning_content =
+        message.additional_kwargs.reasoning_content;
+    }
+
+    if (messageHasToolCalls || messageIsToolResult) {
+      hasReasoningToolCallContext = true;
+    }
+
     if (
       message.additional_kwargs.audio &&
       typeof message.additional_kwargs.audio === 'object' &&
@@ -521,6 +531,9 @@ export function _convertMessagesToOpenAIResponsesParams(
           type?: string;
           refusal?: string;
         };
+      const responseMetadata = lcMsg.response_metadata as {
+        output?: ResponsesInputItem[];
+      };
 
       let role = messageToOpenAIRole(lcMsg);
       if (role === 'system' && isReasoningModel(model)) role = 'developer';
@@ -589,12 +602,12 @@ export function _convertMessagesToOpenAIResponsesParams(
         // if we have the original response items, just reuse them
         if (
           !zdrEnabled &&
-          lcMsg.response_metadata.output != null &&
-          Array.isArray(lcMsg.response_metadata.output) &&
-          lcMsg.response_metadata.output.length > 0 &&
-          lcMsg.response_metadata.output.every((item) => 'type' in item)
+          responseMetadata.output != null &&
+          Array.isArray(responseMetadata.output) &&
+          responseMetadata.output.length > 0 &&
+          responseMetadata.output.every((item) => 'type' in item)
         ) {
-          return lcMsg.response_metadata.output;
+          return responseMetadata.output;
         }
 
         // otherwise, try to reconstruct the response from what we have
@@ -610,7 +623,13 @@ export function _convertMessagesToOpenAIResponsesParams(
         }
 
         // ai content
-        let { content } = lcMsg;
+        let content = lcMsg.content as
+          | string
+          | Array<
+              | MessageContentComplex
+              | OpenAIClient.Responses.ResponseOutputText
+              | OpenAIClient.Responses.ResponseOutputRefusal
+            >;
         if (additional_kwargs.refusal) {
           if (typeof content === 'string') {
             content = [{ type: 'output_text', text: content, annotations: [] }];
@@ -632,11 +651,13 @@ export function _convertMessagesToOpenAIResponsesParams(
               ? content
               : content.flatMap((item) => {
                 if (item.type === 'text') {
+                  const textItem = item as MessageContentComplex & {
+                      annotations?: unknown[];
+                    };
                   return {
                     type: 'output_text',
                     text: item.text,
-                    // @ts-expect-error TODO: add types for `annotations`
-                    annotations: item.annotations ?? [],
+                    annotations: textItem.annotations ?? [],
                   };
                 }
 
@@ -646,7 +667,7 @@ export function _convertMessagesToOpenAIResponsesParams(
 
                 return [];
               }),
-        });
+        } as ResponsesInputItem);
 
         const functionCallIds = additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY];
 
@@ -677,12 +698,9 @@ export function _convertMessagesToOpenAIResponsesParams(
         }
 
         const toolOutputs =
-          ((
-            lcMsg.response_metadata.output as
-              | Array<ResponsesInputItem>
-              | undefined
-          )?.length ?? 0) > 0
-            ? lcMsg.response_metadata.output
+          ((responseMetadata.output as Array<ResponsesInputItem> | undefined)
+            ?.length ?? 0) > 0
+            ? responseMetadata.output
             : additional_kwargs.tool_outputs;
 
         const fallthroughCallTypes: ResponsesInputItem['type'][] = [
@@ -712,52 +730,63 @@ export function _convertMessagesToOpenAIResponsesParams(
         }
 
         const messages: ResponsesInputItem[] = [];
-        const content = lcMsg.content.flatMap((item) => {
-          if (item.type === 'mcp_approval_response') {
-            messages.push({
-              // @ts-ignore
-              type: 'mcp_approval_response',
-              approval_request_id: item.approval_request_id,
-              approve: item.approve,
-            });
-          }
-          if (isDataContentBlock(item)) {
-            return convertToProviderContentBlock(
-              item,
-              completionsApiContentBlockConverter
-            );
-          }
-          if (item.type === 'text') {
-            return {
-              type: 'input_text',
-              text: item.text,
-            };
-          }
-          if (item.type === 'image_url') {
-            return {
-              type: 'input_image',
-              image_url:
-                typeof item.image_url === 'string'
-                  ? item.image_url
-                  : item.image_url.url,
-              detail:
-                typeof item.image_url === 'string'
-                  ? 'auto'
-                  : item.image_url.detail,
-            };
-          }
-          if (
-            item.type === 'input_text' ||
-            item.type === 'input_image' ||
-            item.type === 'input_file'
-          ) {
-            return item;
+        const content = (lcMsg.content as MessageContentComplex[]).flatMap(
+          (item) => {
+            if (item.type === 'mcp_approval_response') {
+              const approvalResponse = item as MessageContentComplex & {
+                approval_request_id: string;
+                approve: boolean;
+              };
+              messages.push({
+                // @ts-ignore
+                type: 'mcp_approval_response',
+                approval_request_id: approvalResponse.approval_request_id,
+                approve: approvalResponse.approve,
+              });
+            }
+            if (isDataContentBlock(item)) {
+              return convertToProviderContentBlock(
+                item,
+                completionsApiContentBlockConverter
+              );
+            }
+            if (item.type === 'text') {
+              return {
+                type: 'input_text',
+                text: item.text,
+              };
+            }
+            if (item.type === 'image_url') {
+              const imageItem = item as MessageContentImageUrl;
+              return {
+                type: 'input_image',
+                image_url:
+                  typeof imageItem.image_url === 'string'
+                    ? imageItem.image_url
+                    : imageItem.image_url.url,
+                detail:
+                  typeof imageItem.image_url === 'string'
+                    ? 'auto'
+                    : imageItem.image_url.detail,
+              };
+            }
+            if (
+              item.type === 'input_text' ||
+              item.type === 'input_image' ||
+              item.type === 'input_file'
+            ) {
+              return item;
+            }
+            return [];
           }
-          return [];
-        });
+        );
 
         if (content.length > 0) {
-          messages.push({ type: 'message', role, content });
+          messages.push({
+            type: 'message',
+            role,
+            content,
+          } as ResponsesInputItem);
         }
         return messages;
       }
@@ -785,7 +814,7 @@ function _convertOpenAIResponsesMessageToBaseMessage(
   }
 
   let messageId: string | undefined;
-  const content: MessageContent = [];
+  const content: MessageContentComplex[] = [];
   const tool_calls: ToolCall[] = [];
   const invalid_tool_calls: InvalidToolCall[] = [];
   const response_metadata: Record<string, unknown> = {
@@ -871,7 +900,7 @@ function _convertOpenAIResponsesMessageToBaseMessage(
 
   return new AIMessage({
     id: messageId,
-    content,
+    content: toLangChainContent(content),
     tool_calls,
     invalid_tool_calls,
     usage_metadata: response.usage,
@@ -883,7 +912,7 @@ function _convertOpenAIResponsesMessageToBaseMessage(
 export function _convertOpenAIResponsesDeltaToBaseMessageChunk(
   chunk: ResponseReturnStreamEvents
 ) {
-  const content: Record<string, unknown>[] = [];
+  const content: MessageContentComplex[] = [];
   let generationInfo: Record<string, unknown> = {};
   let usage_metadata: UsageMetadata | undefined;
   const tool_call_chunks: ToolCallChunk[] = [];
@@ -1021,10 +1050,10 @@ export function _convertOpenAIResponsesDeltaToBaseMessageChunk(
 
   return new ChatGenerationChunk({
     // Legacy reasons, `onLLMNewToken` should pulls this out
-    text: content.map((part) => part.text).join(''),
+    text: content.map((part) => ('text' in part ? part.text : '')).join(''),
     message: new AIMessageChunk({
       id,
-      content,
+      content: toLangChainContent(content),
       tool_call_chunks,
       usage_metadata,
       additional_kwargs,

package/src/llm/openai/utils/messages.test.ts

@@ -0,0 +1,159 @@
+import { AIMessage, HumanMessage, ToolMessage } from '@langchain/core/messages';
+import { _convertMessagesToOpenAIParams } from './index';
+
+describe('_convertMessagesToOpenAIParams', () => {
+  it('includes reasoning_content for assistant messages in tool-call context when requested', () => {
+    const messages = [
+      new AIMessage({
+        content: '',
+        tool_calls: [
+          {
+            id: 'call_1',
+            name: 'calculator',
+            args: { input: '127 * 453' },
+            type: 'tool_call',
+          },
+        ],
+        additional_kwargs: {
+          reasoning_content: 'Need calculator.',
+        },
+      }),
+      new ToolMessage({
+        content: '57531',
+        tool_call_id: 'call_1',
+      }),
+      new AIMessage({
+        content: '127 * 453 = 57531.',
+        additional_kwargs: {
+          reasoning_content: 'Calculator returned 57531.',
+        },
+      }),
+    ];
+
+    const params = _convertMessagesToOpenAIParams(messages, 'deepseek-v4-pro', {
+      includeReasoningContent: true,
+    });
+
+    expect(params).toHaveLength(3);
+    expect(params[0]).toEqual(
+      expect.objectContaining({
+        role: 'assistant',
+        content: '',
+        reasoning_content: 'Need calculator.',
+      })
+    );
+    expect(params[2]).toEqual(
+      expect.objectContaining({
+        role: 'assistant',
+        reasoning_content: 'Calculator returned 57531.',
+      })
+    );
+  });
+
+  it('does not include reasoning_content for no-tool assistant messages', () => {
+    const messages = [
+      new AIMessage({
+        content: '127 * 453 = 57531.',
+        additional_kwargs: {
+          reasoning_content: 'Mental calculation.',
+        },
+      }),
+    ];
+
+    const params = _convertMessagesToOpenAIParams(messages, 'deepseek-v4-pro', {
+      includeReasoningContent: true,
+    });
+
+    expect(params).toHaveLength(1);
+    expect(params[0]).not.toHaveProperty('reasoning_content');
+  });
+
+  it('does not include reasoning_content unless explicitly requested', () => {
+    const messages = [
+      new AIMessage({
+        content: '',
+        tool_calls: [
+          {
+            id: 'call_1',
+            name: 'calculator',
+            args: { input: '127 * 453' },
+            type: 'tool_call',
+          },
+        ],
+        additional_kwargs: {
+          reasoning_content: 'Need calculator.',
+        },
+      }),
+    ];
+
+    const params = _convertMessagesToOpenAIParams(messages, 'deepseek-v4-pro');
+
+    expect(params).toHaveLength(1);
+    expect(params[0]).not.toHaveProperty('reasoning_content');
+  });
+
+  it('keeps reasoning_content latched after tool-call context is established', () => {
+    const messages = [
+      new AIMessage({
+        content: 'No tool was needed.',
+        additional_kwargs: {
+          reasoning_content: 'Initial no-tool reasoning.',
+        },
+      }),
+      new HumanMessage('Use the calculator.'),
+      new AIMessage({
+        content: '',
+        tool_calls: [
+          {
+            id: 'call_1',
+            name: 'calculator',
+            args: { input: '127 * 453' },
+            type: 'tool_call',
+          },
+        ],
+        additional_kwargs: {
+          reasoning_content: 'Need calculator.',
+        },
+      }),
+      new ToolMessage({
+        content: '57531',
+        tool_call_id: 'call_1',
+      }),
+      new AIMessage({
+        content: '127 * 453 = 57531.',
+        additional_kwargs: {
+          reasoning_content: 'Calculator returned 57531.',
+        },
+      }),
+      new HumanMessage('Was that correct?'),
+      new AIMessage({
+        content: 'Yes.',
+        additional_kwargs: {
+          reasoning_content: 'The prior calculator result is available.',
+        },
+      }),
+    ];
+
+    const params = _convertMessagesToOpenAIParams(messages, 'deepseek-v4-pro', {
+      includeReasoningContent: true,
+    });
+
+    expect(params).toHaveLength(7);
+    expect(params[0]).not.toHaveProperty('reasoning_content');
+    expect(params[2]).toEqual(
+      expect.objectContaining({
+        reasoning_content: 'Need calculator.',
+      })
+    );
+    expect(params[4]).toEqual(
+      expect.objectContaining({
+        reasoning_content: 'Calculator returned 57531.',
+      })
+    );
+    expect(params[6]).toEqual(
+      expect.objectContaining({
+        reasoning_content: 'The prior calculator result is available.',
+      })
+    );
+  });
+});