@librechat/agents 3.2.21 → 3.2.32

package/src/messages/format.ts

@@ -11,6 +11,7 @@ import type {
   MessageContent,
   MessageContentImageUrl,
 } from '@langchain/core/messages';
+import type { RunnableConfig } from '@langchain/core/runnables';
 import type { ToolCall } from '@langchain/core/messages/tool';
 import type {
   BedrockReasoningContentText,
@@ -25,10 +26,10 @@ import type {
   TPayload,
   TMessage,
 } from '@/types';
-import type { RunnableConfig } from '@langchain/core/runnables';
-import { emitAgentLog } from '@/utils/events';
-import { Providers, ContentTypes, Constants } from '@/common';
+import { normalizeAnthropicToolCallId } from '@/llm/anthropic/utils/message_inputs';
 import { toLangChainContent, toLangChainMessageFields } from './langchain';
+import { Providers, ContentTypes, Constants } from '@/common';
+import { emitAgentLog } from '@/utils/events';
 
 interface MediaMessageParams {
   message: {
@@ -106,6 +107,29 @@ interface FormatMessageParams {
   langChain?: boolean;
 }
 
+export type LangChainMessageRole = 'system' | 'user' | 'assistant' | 'tool';
+
+export type RoleBearingMessage<T extends BaseMessage = BaseMessage> = T & {
+  role: LangChainMessageRole;
+};
+
+export function withMessageRole<T extends BaseMessage>(
+  message: T,
+  role: LangChainMessageRole
+): RoleBearingMessage<T> {
+  const roleMessage = message as T & { role?: LangChainMessageRole };
+  if (roleMessage.role === role) {
+    return roleMessage as RoleBearingMessage<T>;
+  }
+  Object.defineProperty(roleMessage, 'role', {
+    value: role,
+    writable: true,
+    enumerable: false,
+    configurable: true,
+  });
+  return roleMessage as RoleBearingMessage<T>;
+}
+
 interface FormattedMessage {
   role: string;
   content: string | MessageContentComplex[];
@@ -127,9 +151,9 @@ export const formatMessage = ({
   langChain = false,
 }: FormatMessageParams):
   | FormattedMessage
-  | HumanMessage
-  | AIMessage
-  | SystemMessage => {
+  | RoleBearingMessage<HumanMessage>
+  | RoleBearingMessage<AIMessage>
+  | RoleBearingMessage<SystemMessage> => {
   // eslint-disable-next-line prefer-const
   let { role: _role, _name, sender, text, content: _content, lc_id } = message;
   if (lc_id && lc_id[2] && !langChain) {
@@ -217,7 +241,10 @@ export const formatMessage = ({
       return mediaMessage;
     }
 
-    return new HumanMessage(toLangChainMessageFields(mediaMessage));
+    return withMessageRole(
+      new HumanMessage(toLangChainMessageFields(mediaMessage)),
+      'user'
+    );
   }
 
   if (!langChain) {
@@ -225,11 +252,20 @@ export const formatMessage = ({
   }
 
   if (role === 'user') {
-    return new HumanMessage(toLangChainMessageFields(formattedMessage));
+    return withMessageRole(
+      new HumanMessage(toLangChainMessageFields(formattedMessage)),
+      'user'
+    );
   } else if (role === 'assistant') {
-    return new AIMessage(toLangChainMessageFields(formattedMessage));
+    return withMessageRole(
+      new AIMessage(toLangChainMessageFields(formattedMessage)),
+      'assistant'
+    );
   } else {
-    return new SystemMessage(toLangChainMessageFields(formattedMessage));
+    return withMessageRole(
+      new SystemMessage(toLangChainMessageFields(formattedMessage)),
+      'system'
+    );
   }
 };
 
@@ -243,14 +279,21 @@ export const formatMessage = ({
 export const formatLangChainMessages = (
   messages: Array<MessageInput>,
   formatOptions: Omit<FormatMessageParams, 'message' | 'langChain'>
-): Array<HumanMessage | AIMessage | SystemMessage> => {
+): Array<
+  | RoleBearingMessage<HumanMessage>
+  | RoleBearingMessage<AIMessage>
+  | RoleBearingMessage<SystemMessage>
+> => {
   return messages.map((msg) => {
     const formatted = formatMessage({
       ...formatOptions,
       message: msg,
       langChain: true,
     });
-    return formatted as HumanMessage | AIMessage | SystemMessage;
+    return formatted as
+      | RoleBearingMessage<HumanMessage>
+      | RoleBearingMessage<AIMessage>
+      | RoleBearingMessage<SystemMessage>;
   });
 };
 
@@ -284,12 +327,17 @@ export const formatFromLangChain = (
 };
 
 interface FormatAssistantMessageOptions {
+  preserveUnpairedServerToolUses?: boolean;
   preserveReasoningContent?: boolean;
   provider?: Providers;
 }
 
 interface FormatAgentMessagesOptions {
   provider?: Providers;
+  /** Skill names already primed fresh this turn (manual/always-apply). Their
+   *  historical `skill` tool_calls are not reconstructed into a HumanMessage,
+   *  so the same SKILL.md body is not injected twice in one request. */
+  skipSkillBodyNames?: Set<string>;
 }
 
 function extractReasoningContent(
@@ -371,6 +419,40 @@ function getToolUseId(part: MessageContentComplex): string | undefined {
   return part.tool_use_id;
 }
 
+function isValidServerToolResult(part: MessageContentComplex): boolean {
+  const toolUseId = getToolUseId(part);
+  if (
+    toolUseId?.startsWith(Constants.ANTHROPIC_SERVER_TOOL_PREFIX) !== true ||
+    !('content' in part)
+  ) {
+    return false;
+  }
+  const { content } = part as { content?: unknown };
+  return (
+    Array.isArray(content) ||
+    (content != null &&
+      typeof content === 'object' &&
+      'type' in content &&
+      (content as { type?: unknown }).type === 'web_search_tool_result_error')
+  );
+}
+
+function getToolCallId(part: MessageContentComplex): string | undefined {
+  if (part.type !== ContentTypes.TOOL_CALL) {
+    return undefined;
+  }
+  const id = part.tool_call?.id;
+  return typeof id === 'string' && id !== '' ? id : undefined;
+}
+
+function hasToolCallOutput(part: MessageContentComplex): boolean {
+  if (part.type !== ContentTypes.TOOL_CALL) {
+    return false;
+  }
+  const output = part.tool_call?.output;
+  return output != null && output !== '';
+}
+
 /**
  * Helper function to format an assistant message
  * @param message The message to format
@@ -380,16 +462,20 @@ function getToolUseId(part: MessageContentComplex): string | undefined {
 function formatAssistantMessage(
   message: Partial<TMessage>,
   options?: FormatAssistantMessageOptions
-): Array<AIMessage | ToolMessage> {
-  const formattedMessages: Array<AIMessage | ToolMessage> = [];
+): Array<RoleBearingMessage<AIMessage> | RoleBearingMessage<ToolMessage>> {
+  const formattedMessages: Array<
+    RoleBearingMessage<AIMessage> | RoleBearingMessage<ToolMessage>
+  > = [];
   let currentContent: MessageContentComplex[] = [];
-  let lastAIMessage: AIMessage | null = null;
+  let lastAIMessage: RoleBearingMessage<AIMessage> | null = null;
   let hasReasoning = false;
   let pendingReasoningContent = '';
   const emittedServerToolUseIds = new Set<string>();
   const pendingServerToolUses = new Map<string, MessageContentComplex>();
   const shouldPreserveReasoningContent =
     options?.preserveReasoningContent === true;
+  const serverToolResultIds = new Set<string>();
+  const preferredToolCallParts = new Map<string, MessageContentComplex>();
 
   const takePendingReasoningContent = (): string | undefined => {
     if (!shouldPreserveReasoningContent || !pendingReasoningContent) {
@@ -400,14 +486,19 @@ function formatAssistantMessage(
     return reasoningContent;
   };
 
-  const createAIMessage = (content: MessageContent): AIMessage => {
+  const createAIMessage = (
+    content: MessageContent
+  ): RoleBearingMessage<AIMessage> => {
     const reasoningContent = takePendingReasoningContent();
-    return new AIMessage({
-      content,
-      ...(reasoningContent != null && {
-        additional_kwargs: { reasoning_content: reasoningContent },
+    return withMessageRole(
+      new AIMessage({
+        content,
+        ...(reasoningContent != null && {
+          additional_kwargs: { reasoning_content: reasoningContent },
+        }),
       }),
-    });
+      'assistant'
+    );
   };
 
   const attachPendingReasoningContent = (aiMessage: AIMessage): void => {
@@ -437,15 +528,52 @@ function formatAssistantMessage(
       MessageContentComplex | undefined | null
     >;
 
+    for (const part of contentParts) {
+      if (part == null) {
+        continue;
+      }
+      if (isValidServerToolResult(part)) {
+        serverToolResultIds.add(getToolUseId(part) ?? '');
+      }
+      if (options?.provider === Providers.ANTHROPIC) {
+        const toolCallId = getToolCallId(part);
+        if (toolCallId == null) {
+          continue;
+        }
+        const preferredPart = preferredToolCallParts.get(toolCallId);
+        if (
+          preferredPart == null ||
+          (!hasToolCallOutput(preferredPart) && hasToolCallOutput(part))
+        ) {
+          preferredToolCallParts.set(toolCallId, part);
+        }
+      }
+    }
+
     for (const part of contentParts) {
       if (part == null) {
         continue;
       }
       const toolUseId = getToolUseId(part);
       if (toolUseId != null) {
+        const isServerToolResult = isValidServerToolResult(part);
+        if (
+          toolUseId.startsWith(Constants.ANTHROPIC_SERVER_TOOL_PREFIX) &&
+          !isServerToolResult
+        ) {
+          continue;
+        }
         flushPendingServerToolUse(toolUseId);
+        if (isServerToolResult) {
+          currentContent.push(part);
+          continue;
+        }
       } else if (hasMeaningfulAssistantContent(part)) {
-        pendingServerToolUses.clear();
+        for (const id of pendingServerToolUses.keys()) {
+          if (!serverToolResultIds.has(id)) {
+            pendingServerToolUses.delete(id);
+          }
+        }
       }
       if (part.type === ContentTypes.TEXT && part.tool_call_ids) {
         /*
@@ -453,6 +581,15 @@ function formatAssistantMessage(
         For Anthropic models, the "tool_calls" field on a message is only respected if content is a string.
         */
         if (currentContent.length > 0) {
+          if (
+            currentContent.some((content) => content.type !== ContentTypes.TEXT)
+          ) {
+            currentContent.push(part);
+            lastAIMessage = createAIMessage(toLangChainContent(currentContent));
+            formattedMessages.push(lastAIMessage);
+            currentContent = [];
+            continue;
+          }
           let content = currentContent.reduce((acc, curr) => {
             if (curr.type === ContentTypes.TEXT) {
               return `${acc}${getTextContent(curr)}\n`;
@@ -473,6 +610,14 @@ function formatAssistantMessage(
         if (part.tool_call == null) {
           continue;
         }
+        const toolCallId = getToolCallId(part);
+        if (
+          options?.provider === Providers.ANTHROPIC &&
+          toolCallId != null &&
+          preferredToolCallParts.get(toolCallId) !== part
+        ) {
+          continue;
+        }
 
         // Note: `tool_calls` list is defined when constructed by `AIMessage` class, and outputs should be excluded from it
         const {
@@ -494,6 +639,12 @@ function formatAssistantMessage(
           typeof _tool_call.id === 'string' &&
           _tool_call.id.startsWith(Constants.ANTHROPIC_SERVER_TOOL_PREFIX)
         ) {
+          if (
+            !serverToolResultIds.has(_tool_call.id) &&
+            options.preserveUnpairedServerToolUses !== true
+          ) {
+            continue;
+          }
           if (
             emittedServerToolUseIds.has(_tool_call.id) ||
             pendingServerToolUses.has(_tool_call.id)
@@ -531,17 +682,34 @@ function formatAssistantMessage(
         }
 
         tool_call.args = args;
-        if (!lastAIMessage.tool_calls) {
-          lastAIMessage.tool_calls = [];
+        if (
+          options?.provider === Providers.ANTHROPIC &&
+          Array.isArray(lastAIMessage.content)
+        ) {
+          const content = lastAIMessage.content as MessageContentComplex[];
+          content.push({
+            type: 'tool_use',
+            id: normalizeAnthropicToolCallId(tool_call.id ?? ''),
+            name: tool_call.name,
+            input: args,
+          } as MessageContentComplex);
+          lastAIMessage.content = content as MessageContent;
+        } else {
+          if (!lastAIMessage.tool_calls) {
+            lastAIMessage.tool_calls = [];
+          }
+          lastAIMessage.tool_calls.push(tool_call as ToolCall);
         }
-        lastAIMessage.tool_calls.push(tool_call as ToolCall);
 
         formattedMessages.push(
-          new ToolMessage({
-            tool_call_id: tool_call.id ?? '',
-            name: tool_call.name,
-            content: output != null ? output : '',
-          })
+          withMessageRole(
+            new ToolMessage({
+              tool_call_id: tool_call.id ?? '',
+              name: tool_call.name,
+              content: output != null ? output : '',
+            }),
+            'tool'
+          )
         );
       } else if (
         part.type === ContentTypes.THINK ||
@@ -995,6 +1163,7 @@ function extractSkillName(args: unknown): string | undefined {
  * @param indexTokenCountMap - Optional map of message indices to token counts.
  * @param tools - Optional set of tool names that are allowed in the request.
  * @param skills - Optional map of skill name to body for reconstructing skill HumanMessages.
+ * @param options - Optional formatting options (provider, skipSkillBodyNames).
  * @returns - Object containing formatted messages and updated indexTokenCountMap if provided.
  */
 export const formatAgentMessages = (
@@ -1007,7 +1176,12 @@ export const formatAgentMessages = (
   skills?: Map<string, string>,
   options?: FormatAgentMessagesOptions
 ): {
-  messages: Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>;
+  messages: Array<
+    | RoleBearingMessage<HumanMessage>
+    | RoleBearingMessage<AIMessage>
+    | RoleBearingMessage<SystemMessage>
+    | RoleBearingMessage<ToolMessage>
+  >;
   indexTokenCountMap?: Record<number, number>;
   /** Cross-run summary extracted from the payload. Should be forwarded to the
    *  agent run so it can be included in the system message via AgentContext. */
@@ -1022,7 +1196,10 @@ export const formatAgentMessages = (
   };
 } => {
   const messages: Array<
-    HumanMessage | AIMessage | SystemMessage | ToolMessage
+    | RoleBearingMessage<HumanMessage>
+    | RoleBearingMessage<AIMessage>
+    | RoleBearingMessage<SystemMessage>
+    | RoleBearingMessage<ToolMessage>
   > = [];
   // If indexTokenCountMap is provided, create a new map to track the updated indices
   const updatedIndexTokenCountMap: Record<number, number> = {};
@@ -1078,7 +1255,10 @@ export const formatAgentMessages = (
       const formattedMessage = formatMessage({
         message: message as MessageInput,
         langChain: true,
-      }) as HumanMessage | AIMessage | SystemMessage;
+      }) as
+        | RoleBearingMessage<HumanMessage>
+        | RoleBearingMessage<AIMessage>
+        | RoleBearingMessage<SystemMessage>;
       if (sourceMessageId != null && sourceMessageId !== '') {
         formattedMessage.id = sourceMessageId;
       }
@@ -1254,6 +1434,7 @@ export const formatAgentMessages = (
     }
 
     const formattedMessages = formatAssistantMessage(processedMessage, {
+      preserveUnpairedServerToolUses: i === payload.length - 1,
       preserveReasoningContent: options?.provider === Providers.DEEPSEEK,
       provider: options?.provider,
     });
@@ -1269,19 +1450,26 @@ export const formatAgentMessages = (
     const endMessageIndex = messages.length;
 
     if (pendingSkillNames?.size != null && pendingSkillNames.size > 0) {
+      const skipSkillBodyNames = options?.skipSkillBodyNames;
       for (const skillName of pendingSkillNames) {
+        if (skipSkillBodyNames != null && skipSkillBodyNames.has(skillName)) {
+          continue;
+        }
         const body = skills?.get(skillName) ?? '';
         if (body) {
           messages.push(
-            new HumanMessage({
-              content: body,
-              additional_kwargs: {
-                role: 'user',
-                isMeta: true,
-                source: 'skill',
-                skillName,
-              },
-            })
+            withMessageRole(
+              new HumanMessage({
+                content: body,
+                additional_kwargs: {
+                  role: 'user',
+                  isMeta: true,
+                  source: 'skill',
+                  skillName,
+                },
+              }),
+              'user'
+            )
           );
         }
       }
@@ -1721,7 +1909,12 @@ export function ensureThinkingBlockInMessages(
         'ensureThinkingBlockInMessages: injecting [Previous agent context] HumanMessage' +
           ` (${parts.length} msgs at index ${i}, no thinking block in chain)`
       );
-      result.push(new HumanMessage({ content: toLangChainContent(parts) }));
+      result.push(
+        withMessageRole(
+          new HumanMessage({ content: toLangChainContent(parts) }),
+          'user'
+        )
+      );
       i = j;
     } else {
       // Keep the message as is