@librechat/agents 3.6.9 → 3.6.10

package/src/graphs/MultiAgentGraph.ts

@@ -17,8 +17,17 @@ import {
 import type { BaseMessage, AIMessageChunk } from '@langchain/core/messages';
 import type { LangGraphRunnableConfig } from '@langchain/langgraph';
 import type { ToolRuntime } from '@langchain/core/tools';
+import type { ProviderMessageProvenancePart } from '@/messages/provenance';
 import type { GraphFactoryDependencies } from '@/graphs/graphFactory';
 import type * as t from '@/types';
+import {
+  hasBijectiveProviderContentPartMapping,
+  inspectProviderMessageProvenance,
+  inspectProviderSourceMessageIds,
+  setInvalidProviderMessageProvenance,
+  setProviderMessageProvenance,
+  stampSyntheticProviderMessage,
+} from '@/messages/provenance';
 import { serializeToolContentBounded } from '@/utils/toolContent';
 import { Constants, MULTI_AGENT_GRAPH_RUN_NAME } from '@/common';
 import { HARD_MAX_TOOL_RESULT_CHARS } from '@/utils/truncation';
@@ -36,10 +45,12 @@ const HANDOFF_INSTRUCTIONS_KEY = 'handoff_instructions';
  * message replayed out of the payload.
  */
 function buildRoutingPrompt(content: string): HumanMessage {
-  return new HumanMessage({
-    content,
-    additional_kwargs: { role: 'user', isMeta: true, source: 'routing' },
-  });
+  return stampSyntheticProviderMessage(
+    new HumanMessage({
+      content,
+      additional_kwargs: { role: 'user', isMeta: true, source: 'routing' },
+    })
+  );
 }
 
 function getHandoffInstructions(
@@ -102,23 +113,137 @@ function isTransferToolName(name: unknown): boolean {
 function filterTransferToolUseBlocks(
   content: AIMessage['content'],
   transferToolCallIds: ReadonlySet<string>
-): AIMessage['content'] {
+): {
+  content: AIMessage['content'];
+  retainedContentPartIndices?: readonly number[];
+} {
   if (!Array.isArray(content)) {
-    return content;
+    return { content };
   }
-  return content.filter((block) => {
+  const filteredContent: typeof content = [];
+  const retainedContentPartIndices: number[] = [];
+  for (let index = 0; index < content.length; index++) {
+    const block = content[index];
     if (
-      typeof block !== 'object' ||
-      (block as { type?: string } | null)?.type !== 'tool_use'
+      typeof block === 'object' &&
+      (block as { type?: string } | null)?.type === 'tool_use'
     ) {
-      return true;
+      const toolUse = block as { id?: string; name?: string };
+      if (
+        (toolUse.id != null && transferToolCallIds.has(toolUse.id)) ||
+        isTransferToolName(toolUse.name)
+      ) {
+        continue;
+      }
+    }
+    filteredContent.push(block);
+    retainedContentPartIndices.push(index);
+  }
+  return { content: filteredContent, retainedContentPartIndices };
+}
+
+/** Rebuild a handoff AI message without losing authorship for retained bytes. */
+function copyFilteredAIMessageProvenance(
+  source: AIMessage | AIMessageChunk,
+  target: AIMessage,
+  retainedContentPartIndices?: readonly number[]
+): AIMessage {
+  const sourceMessageIdsState = inspectProviderSourceMessageIds(source);
+  const provenanceState = inspectProviderMessageProvenance(source);
+  if (
+    provenanceState.status === 'invalid' ||
+    sourceMessageIdsState.status === 'invalid'
+  ) {
+    setInvalidProviderMessageProvenance(target);
+    return target;
+  }
+  const sourceMessageIds =
+    sourceMessageIdsState.status === 'valid'
+      ? sourceMessageIdsState.sourceMessageIds
+      : [];
+  if (provenanceState.status === 'absent') {
+    if (sourceMessageIds.length > 0) {
+      setProviderMessageProvenance(
+        target,
+        sourceMessageIds.map((sourceMessageId) => ({
+          attribution: 'model',
+          sourceMessageId,
+        }))
+      );
+    }
+    return target;
+  }
+  const provenance = provenanceState.provenance;
+  const typedSourceMessageIds = new Set<string>();
+  for (const part of provenance.parts) {
+    if (part.sourceMessageId != null) {
+      typedSourceMessageIds.add(part.sourceMessageId);
+    }
+  }
+  const legacyParts: ProviderMessageProvenancePart[] = [];
+  for (const sourceMessageId of sourceMessageIds) {
+    if (!typedSourceMessageIds.has(sourceMessageId)) {
+      legacyParts.push({ attribution: 'model', sourceMessageId });
+    }
+  }
+  const setFilteredProvenance = (
+    parts: readonly ProviderMessageProvenancePart[]
+  ): void => {
+    if (legacyParts.length === 0) {
+      setProviderMessageProvenance(target, parts);
+      return;
+    }
+    const combinedParts = parts.slice();
+    for (const part of legacyParts) {
+      combinedParts.push(part);
     }
-    const toolUse = block as { id?: string; name?: string };
-    if (toolUse.id != null && transferToolCallIds.has(toolUse.id)) {
-      return false;
+    setProviderMessageProvenance(target, combinedParts);
+  };
+  if (
+    retainedContentPartIndices == null ||
+    !Array.isArray(source.content) ||
+    retainedContentPartIndices.length === source.content.length
+  ) {
+    setFilteredProvenance(provenance.parts);
+    return target;
+  }
+
+  if (
+    !hasBijectiveProviderContentPartMapping(
+      provenance.parts,
+      source.content.length
+    )
+  ) {
+    /** An older or aggregate envelope cannot be mapped to current blocks.
+     * Preserve its conservative attribution instead of dropping tool lineage. */
+    setFilteredProvenance(provenance.parts);
+    return target;
+  }
+
+  const retainedParts: ProviderMessageProvenancePart[] = [];
+  const retainedContentPartIndexSet = new Set(retainedContentPartIndices);
+  for (const part of provenance.parts) {
+    if (part.sourceContentPartIndices == null) {
+      retainedParts.push(part);
+      continue;
     }
-    return !isTransferToolName(toolUse.name);
-  });
+    const retainedSourceContentPartIndices: number[] = [];
+    for (const sourceContentPartIndex of part.sourceContentPartIndices) {
+      if (retainedContentPartIndexSet.has(sourceContentPartIndex)) {
+        retainedSourceContentPartIndices.push(sourceContentPartIndex);
+      }
+    }
+    if (retainedSourceContentPartIndices.length > 0) {
+      retainedParts.push({
+        ...part,
+        sourceContentPartIndices: retainedSourceContentPartIndices,
+      });
+    }
+  }
+  setFilteredProvenance(
+    retainedParts.length > 0 ? retainedParts : [{ attribution: 'model' }]
+  );
+  return target;
 }
 
 function isValidHandoffGroupId(value: unknown): value is number {
@@ -502,15 +627,22 @@ export class MultiAgentGraph extends StandardGraph {
       const handoffTools: t.GenericTool[] = [];
       const sourceAgentName = agentContext.name ?? agentId;
       for (const edge of edges) {
-        handoffTools.push(
-          ...this.createHandoffToolsForEdge(edge, agentId, sourceAgentName)
+        const edgeTools = this.createHandoffToolsForEdge(
+          edge,
+          agentId,
+          sourceAgentName
         );
+        for (const edgeTool of edgeTools) {
+          handoffTools.push(edgeTool);
+        }
       }
 
       if (!agentContext.graphTools) {
         agentContext.graphTools = [];
       }
-      agentContext.graphTools.push(...handoffTools);
+      for (const handoffTool of handoffTools) {
+        agentContext.graphTools.push(handoffTool);
+      }
     }
   }
 
@@ -701,11 +833,14 @@ export class MultiAgentGraph extends StandardGraph {
                    * Multiple tool calls - create filtered AIMessage with ONLY this call.
                    * This ensures valid message structure for parallel handoffs.
                    */
-                  const filteredAiMsg = new AIMessage({
-                    content: originalAiMsg.content,
-                    tool_calls: [thisToolCall],
-                    id: originalAiMsg.id,
-                  });
+                  const filteredAiMsg = copyFilteredAIMessageProvenance(
+                    originalAiMsg,
+                    new AIMessage({
+                      content: originalAiMsg.content,
+                      tool_calls: [thisToolCall],
+                      id: originalAiMsg.id,
+                    })
+                  );
 
                   filteredMessages = [
                     ...messages.slice(0, aiMessageIndex),
@@ -981,14 +1116,19 @@ export class MultiAgentGraph extends StandardGraph {
                * result in THIS recipient's state, so its id is never
                * collected, but its name still marks it.
                */
-              const filteredAiMsg = new AIMessage({
-                content: filterTransferToolUseBlocks(
-                  aiMsg.content,
-                  transferToolCallIds
-                ),
-                tool_calls: remainingToolCalls,
-                id: aiMsg.id,
-              });
+              const filteredContent = filterTransferToolUseBlocks(
+                aiMsg.content,
+                transferToolCallIds
+              );
+              const filteredAiMsg = copyFilteredAIMessageProvenance(
+                aiMsg,
+                new AIMessage({
+                  content: filteredContent.content,
+                  tool_calls: remainingToolCalls,
+                  id: aiMsg.id,
+                }),
+                filteredContent.retainedContentPartIndices
+              );
               filteredMessages.push(filteredAiMsg);
             }
             /** If no remaining content or tool calls, skip this message entirely */
@@ -1157,8 +1297,10 @@ export class MultiAgentGraph extends StandardGraph {
             if (lastMsg != null && lastMsg.getType() === 'tool') {
               messagesForAgent = [
                 ...filteredMessages,
-                new AIMessage(
-                  `[Processed tool result and transferring to ${agentId}]`
+                stampSyntheticProviderMessage(
+                  new AIMessage(
+                    `[Processed tool result and transferring to ${agentId}]`
+                  )
                 ),
                 buildRoutingPrompt(instructions),
               ];

package/src/llm/openai/index.ts

@@ -919,6 +919,48 @@ function attachResponsesReplayPosition(
   chunk.message.lc_kwargs.additional_kwargs = additionalKwargs;
 }
 
+type ResponsesAnnotationsBoundaryEvent = {
+  type: OpenAIClient.Responses.ResponseStreamEvent['type'];
+  response?: {
+    output?: Array<{
+      type: string;
+      content?: Array<{
+        type: string;
+        annotations?: object[] | null;
+      }>;
+    }>;
+  };
+};
+
+/**
+ * The Responses API spec declares `annotations` required on `output_text`
+ * content parts, but some OpenAI-compatible gateways omit the field on the
+ * terminal `response.completed`/`response.incomplete` events. LangChain's
+ * converter calls `part.annotations.map(...)` unconditionally, so a missing
+ * field crashes the whole stream. Default it to `[]` before conversion.
+ */
+export function ensureResponsesOutputAnnotations(
+  event: ResponsesAnnotationsBoundaryEvent
+): void {
+  if (event.type !== 'response.completed' && event.type !== 'response.incomplete') {
+    return;
+  }
+  const output = event.response?.output;
+  if (!Array.isArray(output)) {
+    return;
+  }
+  for (const item of output) {
+    if (item.type !== 'message' || !Array.isArray(item.content)) {
+      continue;
+    }
+    for (const part of item.content) {
+      if (part.type === 'output_text' && !Array.isArray(part.annotations)) {
+        part.annotations = [];
+      }
+    }
+  }
+}
+
 function getResponsesStreamError(
   event: OpenAIClient.Responses.ResponseStreamEvent
 ): Error | undefined {
@@ -960,6 +1002,7 @@ async function* convertLibreChatResponsesStream(
       if (streamError != null) {
         throw streamError;
       }
+      ensureResponsesOutputAnnotations(event);
       const convertedChunk =
         convertResponsesDeltaToChatGenerationChunk(event) ??
         convertDroppedResponsesReplayOutput(event);

package/src/messages/alternation.ts

@@ -1,6 +1,22 @@
 // src/messages/alternation.ts
 import { HumanMessage } from '@langchain/core/messages';
 import type { BaseMessage, MessageContent } from '@langchain/core/messages';
+import type { ProviderMessageProvenancePart } from './provenance';
+import type { ProviderToolCallIndex } from './toolResultTypes';
+import {
+  appendProviderToolCallDescriptor,
+  consumeProviderToolResultPair,
+  getBoundedProviderPairingArrayProperty,
+  getProviderAIMessageToolCallDescriptor,
+  getProviderToolCallPartDescriptor,
+  getProviderToolResultPartDescriptor,
+} from './toolResultTypes';
+import {
+  inspectProviderMessageProvenance,
+  inspectProviderSourceMessageIds,
+  setInvalidProviderMessageProvenance,
+  setProviderMessageProvenance,
+} from './provenance';
 import { Providers } from '@/common';
 
 /**
@@ -19,22 +35,57 @@ export const strictAlternationProviders: ReadonlySet<Providers> = new Set([
   Providers.MISTRALAI,
 ]);
 
-const TOOL_RESULT_TYPES = new Set(['tool_result', 'toolResult']);
-
 /**
  * True when every block is a tool result. Both vendored converters already
  * merge adjacent runs of these, and folding one into a text turn would break
  * the tool pairing they depend on — so they are left alone here.
  */
-function isToolResultMessage(message: BaseMessage): boolean {
-  const { content } = message;
-  if (typeof content === 'string' || content.length === 0) {
-    return false;
+function collectProviderToolCalls(message: BaseMessage): ProviderToolCallIndex {
+  const calls: ProviderToolCallIndex = new Map();
+  const content = getBoundedProviderPairingArrayProperty(message, 'content');
+  if (content != null) {
+    for (let index = 0; index < content.length; index++) {
+      const descriptor = getProviderToolCallPartDescriptor(content[index]);
+      if (descriptor != null) {
+        appendProviderToolCallDescriptor(calls, descriptor);
+      }
+    }
   }
-  return content.every(
-    (block) =>
-      typeof block.type === 'string' && TOOL_RESULT_TYPES.has(block.type)
+  const toolCalls = getBoundedProviderPairingArrayProperty(
+    message,
+    'tool_calls'
   );
+  if (toolCalls != null) {
+    for (let index = 0; index < toolCalls.length; index++) {
+      const descriptor = getProviderAIMessageToolCallDescriptor(
+        toolCalls[index]
+      );
+      if (descriptor != null) {
+        appendProviderToolCallDescriptor(calls, descriptor);
+      }
+    }
+  }
+  return calls;
+}
+
+function isToolResultMessage(
+  message: BaseMessage,
+  pairedToolCalls: ProviderToolCallIndex
+): boolean {
+  const content = getBoundedProviderPairingArrayProperty(message, 'content');
+  if (content == null || content.length === 0) {
+    return false;
+  }
+  for (let index = 0; index < content.length; index++) {
+    const descriptor = getProviderToolResultPartDescriptor(content[index]);
+    if (
+      descriptor?.allowHumanMessagePairing !== true ||
+      !consumeProviderToolResultPair(descriptor, pairedToolCalls)
+    ) {
+      return false;
+    }
+  }
+  return true;
 }
 
 function toBlocks(content: MessageContent): Exclude<MessageContent, string> {
@@ -44,14 +95,97 @@ function toBlocks(content: MessageContent): Exclude<MessageContent, string> {
   return content;
 }
 
-function joinContent(
-  left: MessageContent,
-  right: MessageContent
-): MessageContent {
-  if (typeof left === 'string' && typeof right === 'string') {
-    return left === '' ? right : `${left}\n\n${right}`;
+function joinStringContents(
+  messages: readonly BaseMessage[],
+  endIndex = messages.length
+): string {
+  const contents: string[] = [];
+  for (let index = 0; index < endIndex; index++) {
+    const content = messages[index].content as string;
+    if (contents.length === 0 && content === '') {
+      continue;
+    }
+    contents.push(content);
+  }
+  return contents.join('\n\n');
+}
+
+function joinContents(messages: readonly BaseMessage[]): MessageContent {
+  const firstArrayIndex = messages.findIndex((message) =>
+    Array.isArray(message.content)
+  );
+  if (firstArrayIndex === -1) {
+    return joinStringContents(messages);
+  }
+
+  const prefix = joinStringContents(messages, firstArrayIndex);
+  const blocks: Exclude<MessageContent, string> = [];
+  const appendBlocks = (content: MessageContent): void => {
+    const contentBlocks = toBlocks(content);
+    for (let index = 0; index < contentBlocks.length; index++) {
+      blocks.push(contentBlocks[index]);
+    }
+  };
+  appendBlocks(prefix);
+  for (let index = firstArrayIndex; index < messages.length; index++) {
+    appendBlocks(messages[index].content);
   }
-  return [...toBlocks(left), ...toBlocks(right)];
+  return blocks;
+}
+
+function getHumanMessageProvenanceParts(
+  message: BaseMessage
+): ProviderMessageProvenancePart[] | null {
+  const provenanceState = inspectProviderMessageProvenance(message);
+  const sourceMessageIdsState = inspectProviderSourceMessageIds(message);
+  if (
+    provenanceState.status === 'invalid' ||
+    sourceMessageIdsState.status === 'invalid'
+  ) {
+    return null;
+  }
+  const sourceMessageIds =
+    sourceMessageIdsState.status === 'valid'
+      ? sourceMessageIdsState.sourceMessageIds
+      : [];
+  if (provenanceState.status === 'valid') {
+    const parts = [...provenanceState.provenance.parts];
+    const representedSourceIds = new Set<string>();
+    for (const part of parts) {
+      if (part.sourceMessageId != null) {
+        representedSourceIds.add(part.sourceMessageId);
+      }
+    }
+    const missingSourceIds = sourceMessageIds.filter(
+      (sourceMessageId) => !representedSourceIds.has(sourceMessageId)
+    );
+    if (
+      parts.length === 1 &&
+      parts[0].sourceMessageId == null &&
+      missingSourceIds.length === 1
+    ) {
+      return missingSourceIds.map((sourceMessageId) => ({
+        ...parts[0],
+        sourceMessageId,
+      }));
+    }
+    for (const sourceMessageId of missingSourceIds) {
+      parts.push({ attribution: 'user', sourceMessageId });
+    }
+    return parts;
+  }
+  return sourceMessageIds.length > 0
+    ? sourceMessageIds.map((sourceMessageId) => ({
+      attribution: 'user' as const,
+      sourceMessageId,
+    }))
+    : [{ attribution: 'user' }];
+}
+
+function hasProviderVisibleContent(message: BaseMessage): boolean {
+  return typeof message.content === 'string'
+    ? message.content.length > 0
+    : message.content.length > 0;
 }
 
 /**
@@ -66,25 +200,66 @@ function joinContent(
 export function coalesceAdjacentUserTurns(
   messages: BaseMessage[]
 ): BaseMessage[] {
-  const result: BaseMessage[] = [];
-  let mergedAny = false;
-  for (const message of messages) {
-    const previous = result[result.length - 1];
-    const mergeable =
-      result.length > 0 &&
-      previous.getType() === 'human' &&
-      message.getType() === 'human' &&
-      !isToolResultMessage(previous) &&
-      !isToolResultMessage(message);
+  let result: BaseMessage[] | null = null;
+  let pairedToolCalls: ProviderToolCallIndex = new Map();
+  let index = 0;
+  while (index < messages.length) {
+    const first = messages[index];
+    if (first.getType() === 'ai') {
+      pairedToolCalls = collectProviderToolCalls(first);
+      result?.push(first);
+      index++;
+      continue;
+    }
+    if (first.getType() !== 'human') {
+      result?.push(first);
+      index++;
+      continue;
+    }
+    if (isToolResultMessage(first, pairedToolCalls)) {
+      result?.push(first);
+      index++;
+      continue;
+    }
 
-    if (!mergeable) {
-      result.push(message);
+    let endIndex = index + 1;
+    while (
+      endIndex < messages.length &&
+      messages[endIndex].getType() === 'human' &&
+      !isToolResultMessage(messages[endIndex], pairedToolCalls)
+    ) {
+      endIndex++;
+    }
+    if (endIndex === index + 1) {
+      result?.push(first);
+      pairedToolCalls = new Map();
+      index = endIndex;
       continue;
     }
-    mergedAny = true;
+    result ??= messages.slice(0, index);
+    const run = messages.slice(index, endIndex);
+    const last = run[run.length - 1];
+    const provenanceParts: ProviderMessageProvenancePart[] = [];
+    let hasInvalidProvenance = false;
+    for (const message of run) {
+      if (!hasProviderVisibleContent(message)) {
+        continue;
+      }
+      const messageProvenanceParts = getHumanMessageProvenanceParts(message);
+      if (messageProvenanceParts == null) {
+        hasInvalidProvenance = true;
+        continue;
+      }
+      for (const part of messageProvenanceParts) {
+        provenanceParts.push(part);
+      }
+    }
+    if (provenanceParts.length === 0) {
+      provenanceParts.push({ attribution: 'user' });
+    }
 
-    result[result.length - 1] = new HumanMessage({
-      content: joinContent(previous.content, message.content),
+    const mergedMessage = new HumanMessage({
+      content: joinContents(run),
       /**
        * The LATER turn's kwargs, deliberately. The one provider-path consumer
        * of these flags is the prompt-cache tail anchor, and it reasons
@@ -96,9 +271,17 @@ export function coalesceAdjacentUserTurns(
        * skill body must not pin the cache to the volatile body. The first
        * turn's id is kept so origin tracking can re-attach by key.
        */
-      additional_kwargs: message.additional_kwargs,
-      ...(previous.id != null && { id: previous.id }),
+      additional_kwargs: { ...last.additional_kwargs },
+      ...(first.id != null && { id: first.id }),
     });
+    if (hasInvalidProvenance) {
+      setInvalidProviderMessageProvenance(mergedMessage);
+    } else {
+      setProviderMessageProvenance(mergedMessage, provenanceParts);
+    }
+    result.push(mergedMessage);
+    pairedToolCalls = new Map();
+    index = endIndex;
   }
   /**
    * Identity on the no-merge path. The pass runs twice for a primary
@@ -108,5 +291,5 @@ export function coalesceAdjacentUserTurns(
    * the SAME array rather than reallocating a context-sized copy, and
    * callers can cheaply detect "nothing changed" by identity.
    */
-  return mergedAny ? result : messages;
+  return result ?? messages;
 }