@librechat/agents 3.2.2 → 3.2.31

package/src/llm/vertexai/fixThoughtSignatures.test.ts

@@ -1,6 +1,6 @@
 import { expect, test, describe } from '@jest/globals';
-import type { GeminiContent } from '@langchain/google-common';
 import { AIMessage, HumanMessage, ToolMessage } from '@langchain/core/messages';
+import type { GeminiContent } from '@langchain/google-common';
 import { fixThoughtSignatures } from './index';
 
 const SIG_A = 'AY89a1/sigA==';

package/src/llm/vertexai/index.ts

@@ -1,5 +1,6 @@
 import { ChatGoogle } from '@langchain/google-gauth';
 import { ChatConnection } from '@langchain/google-common';
+import { AIMessageChunk, isAIMessage } from '@langchain/core/messages';
 import type {
   GeminiContent,
   GeminiRequest,
@@ -8,7 +9,6 @@ import type {
 } from '@langchain/google-common';
 import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
 import type { BaseMessage, UsageMetadata } from '@langchain/core/messages';
-import { AIMessageChunk, isAIMessage } from '@langchain/core/messages';
 import type { ChatGenerationChunk } from '@langchain/core/outputs';
 import type { GoogleThinkingConfig, VertexAIClientOptions } from '@/types';
 

package/src/messages/cache.test.ts

@@ -16,6 +16,7 @@ import {
 } from './cache';
 import { _convertMessagesToOpenAIParams } from '@/llm/openai/utils';
 import { toLangChainContent } from './langchain';
+import { formatAgentMessages } from './format';
 import { ContentTypes } from '@/common/enum';
 
 type CacheControlBlock = MessageContentComplex & {
@@ -1474,6 +1475,27 @@ describe('Multi-turn cache cleanup', () => {
 });
 
 describe('LangChain message type preservation', () => {
+  it('preserves direct roles for formatted LangChain messages after addCacheControl', () => {
+    const { messages } = formatAgentMessages([
+      { role: 'user', content: 'Hello' },
+      { role: 'assistant', content: 'Hi there' },
+      { role: 'user', content: 'Thanks' },
+    ]);
+
+    const result = addCacheControl(messages);
+
+    expect(result.map((message) => message.role)).toEqual([
+      'user',
+      'assistant',
+      'user',
+    ]);
+    expect(result[0]).toBeInstanceOf(HumanMessage);
+    expect(result[0]).not.toBe(messages[0]);
+    expect(Object.keys(result[0])).not.toContain('role');
+    expect(result[1]).toBe(messages[1]);
+    expect(result[2]).not.toBe(messages[2]);
+  });
+
   it('should preserve instanceof for LangChain messages after addCacheControl', () => {
     const messages: BaseMessage[] = [
       new HumanMessage({ content: [{ type: 'text', text: 'Hello' }] }),

package/src/messages/cache.ts

@@ -6,10 +6,11 @@ import {
   SystemMessage,
   MessageContentComplex,
 } from '@langchain/core/messages';
-import type { AnthropicMessage } from '@/types/messages';
 import type Anthropic from '@anthropic-ai/sdk';
-import { ContentTypes } from '@/common/enum';
+import type { AnthropicMessage } from '@/types/messages';
 import { toLangChainContent } from './langchain';
+import { ContentTypes } from '@/common/enum';
+import { withMessageRole } from './format';
 
 type MessageWithContent = {
   content?: string | MessageContentComplex[];
@@ -56,19 +57,31 @@ function cloneMessage<T extends MessageWithContent>(
     const msgType = message.getType();
     switch (msgType) {
     case 'ai':
-      return new AIMessage({
-        ...baseParams,
-        tool_calls: (message as unknown as AIMessage).tool_calls,
-      }) as unknown as T;
+      return withMessageRole(
+        new AIMessage({
+          ...baseParams,
+          tool_calls: (message as unknown as AIMessage).tool_calls,
+        }),
+        'assistant'
+      ) as unknown as T;
     case 'human':
-      return new HumanMessage(baseParams) as unknown as T;
+      return withMessageRole(
+        new HumanMessage(baseParams),
+        'user'
+      ) as unknown as T;
     case 'system':
-      return new SystemMessage(baseParams) as unknown as T;
+      return withMessageRole(
+        new SystemMessage(baseParams),
+        'system'
+      ) as unknown as T;
     case 'tool':
-      return new ToolMessage({
-        ...baseParams,
-        tool_call_id: (message as unknown as ToolMessage).tool_call_id,
-      }) as unknown as T;
+      return withMessageRole(
+        new ToolMessage({
+          ...baseParams,
+          tool_call_id: (message as unknown as ToolMessage).tool_call_id,
+        }),
+        'tool'
+      ) as unknown as T;
     default:
       break;
     }

package/src/messages/content.ts

@@ -1,5 +1,5 @@
-import { ContentTypes } from '@/common';
 import type { BaseMessage } from '@langchain/core/messages';
+import { ContentTypes } from '@/common';
 
 /**
  * Formats an array of messages for LangChain, making sure all content fields are strings

package/src/messages/contextPruning.ts

@@ -13,9 +13,9 @@
  */
 
 import { ToolMessage, type BaseMessage } from '@langchain/core/messages';
+import type { ContextPruningSettings } from './contextPruningSettings';
 import type { ContextPruningConfig } from '@/types/graph';
 import type { TokenCounter } from '@/types/run';
-import type { ContextPruningSettings } from './contextPruningSettings';
 import { resolveContextPruningSettings } from './contextPruningSettings';
 
 /**

package/src/messages/format.ts

@@ -11,6 +11,7 @@ import type {
   MessageContent,
   MessageContentImageUrl,
 } from '@langchain/core/messages';
+import type { RunnableConfig } from '@langchain/core/runnables';
 import type { ToolCall } from '@langchain/core/messages/tool';
 import type {
   BedrockReasoningContentText,
@@ -25,10 +26,10 @@ import type {
   TPayload,
   TMessage,
 } from '@/types';
-import type { RunnableConfig } from '@langchain/core/runnables';
-import { emitAgentLog } from '@/utils/events';
-import { Providers, ContentTypes, Constants } from '@/common';
+import { normalizeAnthropicToolCallId } from '@/llm/anthropic/utils/message_inputs';
 import { toLangChainContent, toLangChainMessageFields } from './langchain';
+import { Providers, ContentTypes, Constants } from '@/common';
+import { emitAgentLog } from '@/utils/events';
 
 interface MediaMessageParams {
   message: {
@@ -106,6 +107,29 @@ interface FormatMessageParams {
   langChain?: boolean;
 }
 
+export type LangChainMessageRole = 'system' | 'user' | 'assistant' | 'tool';
+
+export type RoleBearingMessage<T extends BaseMessage = BaseMessage> = T & {
+  role: LangChainMessageRole;
+};
+
+export function withMessageRole<T extends BaseMessage>(
+  message: T,
+  role: LangChainMessageRole
+): RoleBearingMessage<T> {
+  const roleMessage = message as T & { role?: LangChainMessageRole };
+  if (roleMessage.role === role) {
+    return roleMessage as RoleBearingMessage<T>;
+  }
+  Object.defineProperty(roleMessage, 'role', {
+    value: role,
+    writable: true,
+    enumerable: false,
+    configurable: true,
+  });
+  return roleMessage as RoleBearingMessage<T>;
+}
+
 interface FormattedMessage {
   role: string;
   content: string | MessageContentComplex[];
@@ -127,9 +151,9 @@ export const formatMessage = ({
   langChain = false,
 }: FormatMessageParams):
   | FormattedMessage
-  | HumanMessage
-  | AIMessage
-  | SystemMessage => {
+  | RoleBearingMessage<HumanMessage>
+  | RoleBearingMessage<AIMessage>
+  | RoleBearingMessage<SystemMessage> => {
   // eslint-disable-next-line prefer-const
   let { role: _role, _name, sender, text, content: _content, lc_id } = message;
   if (lc_id && lc_id[2] && !langChain) {
@@ -217,7 +241,10 @@ export const formatMessage = ({
       return mediaMessage;
     }
 
-    return new HumanMessage(toLangChainMessageFields(mediaMessage));
+    return withMessageRole(
+      new HumanMessage(toLangChainMessageFields(mediaMessage)),
+      'user'
+    );
   }
 
   if (!langChain) {
@@ -225,11 +252,20 @@ export const formatMessage = ({
   }
 
   if (role === 'user') {
-    return new HumanMessage(toLangChainMessageFields(formattedMessage));
+    return withMessageRole(
+      new HumanMessage(toLangChainMessageFields(formattedMessage)),
+      'user'
+    );
   } else if (role === 'assistant') {
-    return new AIMessage(toLangChainMessageFields(formattedMessage));
+    return withMessageRole(
+      new AIMessage(toLangChainMessageFields(formattedMessage)),
+      'assistant'
+    );
   } else {
-    return new SystemMessage(toLangChainMessageFields(formattedMessage));
+    return withMessageRole(
+      new SystemMessage(toLangChainMessageFields(formattedMessage)),
+      'system'
+    );
   }
 };
 
@@ -243,14 +279,21 @@ export const formatMessage = ({
 export const formatLangChainMessages = (
   messages: Array<MessageInput>,
   formatOptions: Omit<FormatMessageParams, 'message' | 'langChain'>
-): Array<HumanMessage | AIMessage | SystemMessage> => {
+): Array<
+  | RoleBearingMessage<HumanMessage>
+  | RoleBearingMessage<AIMessage>
+  | RoleBearingMessage<SystemMessage>
+> => {
   return messages.map((msg) => {
     const formatted = formatMessage({
       ...formatOptions,
       message: msg,
       langChain: true,
     });
-    return formatted as HumanMessage | AIMessage | SystemMessage;
+    return formatted as
+      | RoleBearingMessage<HumanMessage>
+      | RoleBearingMessage<AIMessage>
+      | RoleBearingMessage<SystemMessage>;
   });
 };
 
@@ -284,6 +327,7 @@ export const formatFromLangChain = (
 };
 
 interface FormatAssistantMessageOptions {
+  preserveUnpairedServerToolUses?: boolean;
   preserveReasoningContent?: boolean;
   provider?: Providers;
 }
@@ -371,6 +415,40 @@ function getToolUseId(part: MessageContentComplex): string | undefined {
   return part.tool_use_id;
 }
 
+function isValidServerToolResult(part: MessageContentComplex): boolean {
+  const toolUseId = getToolUseId(part);
+  if (
+    toolUseId?.startsWith(Constants.ANTHROPIC_SERVER_TOOL_PREFIX) !== true ||
+    !('content' in part)
+  ) {
+    return false;
+  }
+  const { content } = part as { content?: unknown };
+  return (
+    Array.isArray(content) ||
+    (content != null &&
+      typeof content === 'object' &&
+      'type' in content &&
+      (content as { type?: unknown }).type === 'web_search_tool_result_error')
+  );
+}
+
+function getToolCallId(part: MessageContentComplex): string | undefined {
+  if (part.type !== ContentTypes.TOOL_CALL) {
+    return undefined;
+  }
+  const id = part.tool_call?.id;
+  return typeof id === 'string' && id !== '' ? id : undefined;
+}
+
+function hasToolCallOutput(part: MessageContentComplex): boolean {
+  if (part.type !== ContentTypes.TOOL_CALL) {
+    return false;
+  }
+  const output = part.tool_call?.output;
+  return output != null && output !== '';
+}
+
 /**
  * Helper function to format an assistant message
  * @param message The message to format
@@ -380,16 +458,20 @@ function getToolUseId(part: MessageContentComplex): string | undefined {
 function formatAssistantMessage(
   message: Partial<TMessage>,
   options?: FormatAssistantMessageOptions
-): Array<AIMessage | ToolMessage> {
-  const formattedMessages: Array<AIMessage | ToolMessage> = [];
+): Array<RoleBearingMessage<AIMessage> | RoleBearingMessage<ToolMessage>> {
+  const formattedMessages: Array<
+    RoleBearingMessage<AIMessage> | RoleBearingMessage<ToolMessage>
+  > = [];
   let currentContent: MessageContentComplex[] = [];
-  let lastAIMessage: AIMessage | null = null;
+  let lastAIMessage: RoleBearingMessage<AIMessage> | null = null;
   let hasReasoning = false;
   let pendingReasoningContent = '';
   const emittedServerToolUseIds = new Set<string>();
   const pendingServerToolUses = new Map<string, MessageContentComplex>();
   const shouldPreserveReasoningContent =
     options?.preserveReasoningContent === true;
+  const serverToolResultIds = new Set<string>();
+  const preferredToolCallParts = new Map<string, MessageContentComplex>();
 
   const takePendingReasoningContent = (): string | undefined => {
     if (!shouldPreserveReasoningContent || !pendingReasoningContent) {
@@ -400,14 +482,19 @@ function formatAssistantMessage(
     return reasoningContent;
   };
 
-  const createAIMessage = (content: MessageContent): AIMessage => {
+  const createAIMessage = (
+    content: MessageContent
+  ): RoleBearingMessage<AIMessage> => {
     const reasoningContent = takePendingReasoningContent();
-    return new AIMessage({
-      content,
-      ...(reasoningContent != null && {
-        additional_kwargs: { reasoning_content: reasoningContent },
+    return withMessageRole(
+      new AIMessage({
+        content,
+        ...(reasoningContent != null && {
+          additional_kwargs: { reasoning_content: reasoningContent },
+        }),
       }),
-    });
+      'assistant'
+    );
   };
 
   const attachPendingReasoningContent = (aiMessage: AIMessage): void => {
@@ -437,15 +524,52 @@ function formatAssistantMessage(
       MessageContentComplex | undefined | null
     >;
 
+    for (const part of contentParts) {
+      if (part == null) {
+        continue;
+      }
+      if (isValidServerToolResult(part)) {
+        serverToolResultIds.add(getToolUseId(part) ?? '');
+      }
+      if (options?.provider === Providers.ANTHROPIC) {
+        const toolCallId = getToolCallId(part);
+        if (toolCallId == null) {
+          continue;
+        }
+        const preferredPart = preferredToolCallParts.get(toolCallId);
+        if (
+          preferredPart == null ||
+          (!hasToolCallOutput(preferredPart) && hasToolCallOutput(part))
+        ) {
+          preferredToolCallParts.set(toolCallId, part);
+        }
+      }
+    }
+
     for (const part of contentParts) {
       if (part == null) {
         continue;
       }
       const toolUseId = getToolUseId(part);
       if (toolUseId != null) {
+        const isServerToolResult = isValidServerToolResult(part);
+        if (
+          toolUseId.startsWith(Constants.ANTHROPIC_SERVER_TOOL_PREFIX) &&
+          !isServerToolResult
+        ) {
+          continue;
+        }
         flushPendingServerToolUse(toolUseId);
+        if (isServerToolResult) {
+          currentContent.push(part);
+          continue;
+        }
       } else if (hasMeaningfulAssistantContent(part)) {
-        pendingServerToolUses.clear();
+        for (const id of pendingServerToolUses.keys()) {
+          if (!serverToolResultIds.has(id)) {
+            pendingServerToolUses.delete(id);
+          }
+        }
       }
       if (part.type === ContentTypes.TEXT && part.tool_call_ids) {
         /*
@@ -453,6 +577,15 @@ function formatAssistantMessage(
         For Anthropic models, the "tool_calls" field on a message is only respected if content is a string.
         */
         if (currentContent.length > 0) {
+          if (
+            currentContent.some((content) => content.type !== ContentTypes.TEXT)
+          ) {
+            currentContent.push(part);
+            lastAIMessage = createAIMessage(toLangChainContent(currentContent));
+            formattedMessages.push(lastAIMessage);
+            currentContent = [];
+            continue;
+          }
           let content = currentContent.reduce((acc, curr) => {
             if (curr.type === ContentTypes.TEXT) {
               return `${acc}${getTextContent(curr)}\n`;
@@ -473,6 +606,14 @@ function formatAssistantMessage(
         if (part.tool_call == null) {
           continue;
         }
+        const toolCallId = getToolCallId(part);
+        if (
+          options?.provider === Providers.ANTHROPIC &&
+          toolCallId != null &&
+          preferredToolCallParts.get(toolCallId) !== part
+        ) {
+          continue;
+        }
 
         // Note: `tool_calls` list is defined when constructed by `AIMessage` class, and outputs should be excluded from it
         const {
@@ -494,6 +635,12 @@ function formatAssistantMessage(
           typeof _tool_call.id === 'string' &&
           _tool_call.id.startsWith(Constants.ANTHROPIC_SERVER_TOOL_PREFIX)
         ) {
+          if (
+            !serverToolResultIds.has(_tool_call.id) &&
+            options.preserveUnpairedServerToolUses !== true
+          ) {
+            continue;
+          }
           if (
             emittedServerToolUseIds.has(_tool_call.id) ||
             pendingServerToolUses.has(_tool_call.id)
@@ -531,17 +678,34 @@ function formatAssistantMessage(
         }
 
         tool_call.args = args;
-        if (!lastAIMessage.tool_calls) {
-          lastAIMessage.tool_calls = [];
+        if (
+          options?.provider === Providers.ANTHROPIC &&
+          Array.isArray(lastAIMessage.content)
+        ) {
+          const content = lastAIMessage.content as MessageContentComplex[];
+          content.push({
+            type: 'tool_use',
+            id: normalizeAnthropicToolCallId(tool_call.id ?? ''),
+            name: tool_call.name,
+            input: args,
+          } as MessageContentComplex);
+          lastAIMessage.content = content as MessageContent;
+        } else {
+          if (!lastAIMessage.tool_calls) {
+            lastAIMessage.tool_calls = [];
+          }
+          lastAIMessage.tool_calls.push(tool_call as ToolCall);
         }
-        lastAIMessage.tool_calls.push(tool_call as ToolCall);
 
         formattedMessages.push(
-          new ToolMessage({
-            tool_call_id: tool_call.id ?? '',
-            name: tool_call.name,
-            content: output != null ? output : '',
-          })
+          withMessageRole(
+            new ToolMessage({
+              tool_call_id: tool_call.id ?? '',
+              name: tool_call.name,
+              content: output != null ? output : '',
+            }),
+            'tool'
+          )
         );
       } else if (
         part.type === ContentTypes.THINK ||
@@ -1007,7 +1171,12 @@ export const formatAgentMessages = (
   skills?: Map<string, string>,
   options?: FormatAgentMessagesOptions
 ): {
-  messages: Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>;
+  messages: Array<
+    | RoleBearingMessage<HumanMessage>
+    | RoleBearingMessage<AIMessage>
+    | RoleBearingMessage<SystemMessage>
+    | RoleBearingMessage<ToolMessage>
+  >;
   indexTokenCountMap?: Record<number, number>;
   /** Cross-run summary extracted from the payload. Should be forwarded to the
    *  agent run so it can be included in the system message via AgentContext. */
@@ -1022,7 +1191,10 @@ export const formatAgentMessages = (
   };
 } => {
   const messages: Array<
-    HumanMessage | AIMessage | SystemMessage | ToolMessage
+    | RoleBearingMessage<HumanMessage>
+    | RoleBearingMessage<AIMessage>
+    | RoleBearingMessage<SystemMessage>
+    | RoleBearingMessage<ToolMessage>
   > = [];
   // If indexTokenCountMap is provided, create a new map to track the updated indices
   const updatedIndexTokenCountMap: Record<number, number> = {};
@@ -1078,7 +1250,10 @@ export const formatAgentMessages = (
       const formattedMessage = formatMessage({
         message: message as MessageInput,
         langChain: true,
-      }) as HumanMessage | AIMessage | SystemMessage;
+      }) as
+        | RoleBearingMessage<HumanMessage>
+        | RoleBearingMessage<AIMessage>
+        | RoleBearingMessage<SystemMessage>;
       if (sourceMessageId != null && sourceMessageId !== '') {
         formattedMessage.id = sourceMessageId;
       }
@@ -1254,6 +1429,7 @@ export const formatAgentMessages = (
     }
 
     const formattedMessages = formatAssistantMessage(processedMessage, {
+      preserveUnpairedServerToolUses: i === payload.length - 1,
       preserveReasoningContent: options?.provider === Providers.DEEPSEEK,
       provider: options?.provider,
     });
@@ -1273,15 +1449,18 @@ export const formatAgentMessages = (
         const body = skills?.get(skillName) ?? '';
         if (body) {
           messages.push(
-            new HumanMessage({
-              content: body,
-              additional_kwargs: {
-                role: 'user',
-                isMeta: true,
-                source: 'skill',
-                skillName,
-              },
-            })
+            withMessageRole(
+              new HumanMessage({
+                content: body,
+                additional_kwargs: {
+                  role: 'user',
+                  isMeta: true,
+                  source: 'skill',
+                  skillName,
+                },
+              }),
+              'user'
+            )
           );
         }
       }
@@ -1721,7 +1900,12 @@ export function ensureThinkingBlockInMessages(
         'ensureThinkingBlockInMessages: injecting [Previous agent context] HumanMessage' +
           ` (${parts.length} msgs at index ${i}, no thinking block in chain)`
       );
-      result.push(new HumanMessage({ content: toLangChainContent(parts) }));
+      result.push(
+        withMessageRole(
+          new HumanMessage({ content: toLangChainContent(parts) }),
+          'user'
+        )
+      );
       i = j;
     } else {
       // Keep the message as is