@librechat/agents 3.7.17 → 3.7.19

package/src/graphs/Graph.ts

@@ -151,6 +151,7 @@ import { shouldTraceToolNodeForLangfuse } from '@/langfuseToolOutputTracing';
 import { createLocalCodingToolBundle } from '@/tools/local/LocalCodingTools';
 import { SUBAGENT_REPLAY_CONTROLLER } from '@/tools/subagent/SubagentReplay';
 import { applyGraphRuntimeConfig } from '@/graphs/applyGraphRuntimeConfig';
+import { isFadingTier, isInformativeFadingTier } from '@/messages/fading';
 import { createContextPressureMeter } from '@/llm/contextPressureMeter';
 import { safeDispatchCustomEvent, emitAgentLog } from '@/utils/events';
 import { prepareProviderRequest } from '@/llm/prepareProviderRequest';
@@ -163,7 +164,6 @@ import { initializeLangfuseTracing } from '@/instrumentation';
 import { shouldTriggerSummarization } from '@/summarization';
 import { isRunStepResumeState } from '@/tools/runStepResume';
 import { resolveLocalToolsForBinding } from '@/tools/local';
-import { isFadingTier, isInformativeFadingTier } from '@/messages/fading';
 import { createSummarizeNode } from '@/summarization/node';
 import { getTruncationStopReason } from '@/llm/truncation';
 import { messagesStateReducer } from '@/messages/reducer';
@@ -3992,6 +3992,7 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
         rootAgentName: this.agentContexts.get(this.defaultAgentId)?.name,
         activeAgentId: agentId,
         activeAgentName: agentContext.name,
+        endpoint: agentContext.endpoint,
       });
       let langfuseHandler: CallbackEntry | undefined;
       let invokeConfig = {

package/src/langfuse.ts

@@ -840,6 +840,10 @@ export function createLangfuseTraceMetadata({
   rootAgentName,
   activeAgentId,
   activeAgentName,
+  endpoint,
+  model,
+  provider,
+  resolvedProvider,
 }: {
   messageId?: unknown;
   parentMessageId?: unknown;
@@ -849,6 +853,10 @@ export function createLangfuseTraceMetadata({
   rootAgentName?: unknown;
   activeAgentId?: unknown;
   activeAgentName?: unknown;
+  endpoint?: unknown;
+  model?: unknown;
+  provider?: unknown;
+  resolvedProvider?: unknown;
 }): LangfuseTraceMetadata {
   return createTraceMetadata({
     messageId,
@@ -859,6 +867,10 @@ export function createLangfuseTraceMetadata({
     rootAgentName,
     activeAgentId,
     activeAgentName,
+    endpoint,
+    model,
+    provider,
+    resolvedProvider,
   });
 }
 

package/src/llm/bedrock/utils/message_inputs.ts

@@ -58,6 +58,15 @@ const FOREIGN_SERVER_TOOL_TYPES = ['toolCall', 'toolResponse'];
  * also has no tool calls, fall back to this placeholder text.
  */
 const BEDROCK_EMPTY_TEXT_PLACEHOLDER = '_';
+/**
+ * Whether a converted user message carries top-level text and document blocks, recorded
+ * during its single conversion pass and OR-ed through merging so finalization never
+ * rescans content.
+ */
+const userGroupShape = new WeakMap<
+  BedrockMessage,
+  { hasText: boolean; hasDocument: boolean }
+>();
 
 /**
  * Convert a LangChain reasoning block to a Bedrock reasoning block.
@@ -965,20 +974,72 @@ function convertFromV1ToChatBedrockConverseMessage(
 function convertHumanMessageToConverseMessage(
   msg: BaseMessage
 ): BedrockMessage {
-  const userMessage: BedrockMessage = {
-    role: 'user',
-    content: [],
+  const contentBlocks: BedrockContentBlock[] = [];
+  let hasText = false;
+  let hasDocument = false;
+  // Whitespace with nothing before it to fold into is held until the next text fragment,
+  // so `[' ', 'hello']` and `[image, ' ', 'hello']` keep their leading space instead of
+  // losing it; whitespace left over at the end has nothing to attach to and is dropped.
+  let pendingWhitespace = '';
+  const appendText = (text: string): void => {
+    const candidate = pendingWhitespace + text;
+    if (appendSerializableBedrockTextBlock(contentBlocks, candidate)) {
+      hasText = true;
+      pendingWhitespace = '';
+      return;
+    }
+    pendingWhitespace = candidate;
   };
 
   if (typeof msg.content === 'string') {
-    userMessage.content = [{ text: msg.content }];
+    appendText(msg.content);
   } else if (Array.isArray(msg.content)) {
-    userMessage.content = msg.content.map((block) =>
-      convertLangChainContentBlockToConverseContentBlock({ block })
-    );
+    for (const block of msg.content) {
+      if (
+        block.type === 'text' &&
+        typeof (block as { text?: unknown }).text === 'string'
+      ) {
+        appendText((block as { text: string }).text);
+        continue;
+      }
+      const converted = convertLangChainContentBlockToConverseContentBlock({
+        block,
+      });
+      if ('document' in converted) {
+        hasDocument = true;
+      }
+      contentBlocks.push(converted);
+    }
   }
 
-  return userMessage;
+  const message: BedrockMessage = { role: 'user', content: contentBlocks };
+  userGroupShape.set(message, { hasText, hasDocument });
+  return message;
+}
+
+/**
+ * Bedrock rejects an empty user message just as it rejects a blank text block
+ * (`The text field in the ContentBlock object at messages.N.content.0 is blank`), and the
+ * rejection fences the whole conversation on every retry. A promptless send — attachment, no
+ * typed text — serializes to exactly that. Applied only once a user group is complete: an
+ * empty human turn adjacent to a tool result merges into it below, and a per-message
+ * placeholder would inject a synthetic `_` into an otherwise valid message.
+ */
+function finalizeUserMessage(message: BedrockMessage): void {
+  if (message.role !== 'user') {
+    return;
+  }
+  if (message.content == null || message.content.length === 0) {
+    message.content = [{ text: BEDROCK_EMPTY_TEXT_PLACEHOLDER }];
+    return;
+  }
+  // Converse also requires a text block in any message that carries a document block. A
+  // promptless document upload arrives as blank text plus the document; the blank is
+  // dropped during conversion, so supply the placeholder if no text survived.
+  const shape = userGroupShape.get(message);
+  if (shape != null && shape.hasDocument && !shape.hasText) {
+    message.content.push({ text: BEDROCK_EMPTY_TEXT_PLACEHOLDER });
+  }
 }
 
 /**
@@ -1110,13 +1171,30 @@ export function convertToConverseMessages(messages: BaseMessage[]): {
       const lastMessage = acc[acc.length - 1];
       if (lastMessage.role === 'user' && curr.role === 'user') {
         lastMessage.content = lastMessage.content?.concat(curr.content ?? []);
+        const lastShape = userGroupShape.get(lastMessage);
+        const nextShape = userGroupShape.get(curr);
+        if (lastShape != null || nextShape != null) {
+          userGroupShape.set(lastMessage, {
+            hasText:
+              (lastShape?.hasText ?? false) || (nextShape?.hasText ?? false),
+            hasDocument:
+              (lastShape?.hasDocument ?? false) ||
+              (nextShape?.hasDocument ?? false),
+          });
+        }
       } else {
+        finalizeUserMessage(lastMessage);
         acc.push(curr);
       }
       return acc;
     },
     []
   );
+  if (combinedConverseMessages.length > 0) {
+    finalizeUserMessage(
+      combinedConverseMessages[combinedConverseMessages.length - 1]
+    );
+  }
 
   return { converseMessages: combinedConverseMessages, converseSystem };
 }

package/src/llm/invoke.ts

@@ -692,11 +692,32 @@ export async function attemptInvoke(
   config?: RunnableConfig
 ): Promise<Partial<t.BaseGraphState>> {
   const provider = resolveAttemptProvider(params);
-  const stampedConfig: RunnableConfig = {
+  const providerStampedConfig: RunnableConfig = {
     ...config,
     metadata: {
       ...(config?.metadata ?? {}),
       [Constants.INVOKED_PROVIDER]: provider,
+      provider,
+      resolvedProvider: provider,
+    },
+  };
+  const request = resolveAttemptRequest(params, providerStampedConfig);
+  const configuredModel = providerStampedConfig.metadata?.[
+    Constants.INVOKED_MODEL
+  ];
+  const modelId =
+    request.modelId ??
+    (typeof configuredModel === 'string' ? configuredModel : undefined);
+  const stampedConfig: RunnableConfig = {
+    ...providerStampedConfig,
+    metadata: {
+      ...providerStampedConfig.metadata,
+      ...(modelId == null
+        ? {}
+        : {
+          [Constants.INVOKED_MODEL]: modelId,
+          model: modelId,
+        }),
       /**
        * One `attemptInvoke` call is one model attempt; primary, fallback,
        * and retry attempts within a node otherwise share the same langgraph
@@ -727,7 +748,7 @@ export async function attemptInvoke(
   try {
     return await attemptInvokeBody(
       {
-        request: resolveAttemptRequest(params, stampedConfig),
+        request,
         context: params.context,
         onChunk: params.onChunk,
         preemptAgentId: params.preemptAgentId,

package/src/llm/prepareProviderRequest.ts

@@ -1,5 +1,10 @@
+import { Buffer } from 'node:buffer';
+import type {
+  BaseMessage,
+  Data,
+  MessageContentComplex,
+} from '@langchain/core/messages';
 import type { RunnableConfig } from '@langchain/core/runnables';
-import type { BaseMessage } from '@langchain/core/messages';
 import type { ToolOutputReferenceRegistry } from '@/tools/toolOutputReferences';
 import type * as t from '@/types';
 import {
@@ -20,14 +25,60 @@ import {
 import {
   stripAnthropicCacheControl,
   stripBedrockCacheControl,
+  cloneMessage,
 } from '@/messages/cache';
+import {
+  isAnthropicLike,
+  isGoogleLike,
+  isOpenAILike,
+} from '@/utils/llm';
 import { annotateMessagesForLLM } from '@/tools/toolOutputReferences';
 import { providerRequiresStrictAlternation } from '@/llm/providers';
-import { isAnthropicLike, isOpenAILike } from '@/utils/llm';
 import { getProviderFamily } from '@/llm/providerRegistry';
 import { Providers } from '@/common';
 
 const preparedProviderRequestBrand = Symbol('PreparedProviderRequest');
+const OMITTED_ATTACHMENT_TEXT =
+  '[Attachment omitted because its binary format is unsupported by this provider.]';
+
+const BEDROCK_DOCUMENT_MIME_TYPES: Readonly<Partial<Record<string, string>>> = {
+  csv: 'text/csv',
+  doc: 'application/msword',
+  docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+  html: 'text/html',
+  md: 'text/markdown',
+  pdf: 'application/pdf',
+  txt: 'text/plain',
+  xls: 'application/vnd.ms-excel',
+  xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+};
+
+interface SerializedBuffer {
+  type: 'Buffer';
+  data: number[];
+}
+
+interface BedrockDocumentBlock {
+  type: 'document';
+  document: {
+    name: string;
+    format: string;
+    source: {
+      bytes: Uint8Array | SerializedBuffer;
+    };
+  };
+}
+
+type StandardBase64FileBlock = Data.StandardFileBlock & {
+  source_type: 'base64';
+  mime_type: string;
+  data: string;
+};
+
+type ProviderContentBlock =
+  | MessageContentComplex
+  | BedrockDocumentBlock
+  | StandardBase64FileBlock;
 
 export type ProviderMessageProjectionMode =
   | 'chat-messages'
@@ -159,16 +210,216 @@ interface ProjectMessagesForProviderParams {
   callOptions?: unknown;
 }
 
+function isSerializedBuffer(value: object): value is SerializedBuffer {
+  return (
+    'type' in value &&
+    value.type === 'Buffer' &&
+    'data' in value &&
+    Array.isArray(value.data)
+  );
+}
+
+function isBedrockDocumentBlock(
+  block: ProviderContentBlock
+): block is BedrockDocumentBlock {
+  if (block.type !== 'document' || !('document' in block)) {
+    return false;
+  }
+  const document = block.document;
+  if (typeof document !== 'object' || document == null) {
+    return false;
+  }
+  if (
+    !('name' in document) ||
+    typeof document.name !== 'string' ||
+    !('format' in document) ||
+    typeof document.format !== 'string' ||
+    !('source' in document) ||
+    typeof document.source !== 'object' ||
+    document.source == null ||
+    !('bytes' in document.source)
+  ) {
+    return false;
+  }
+  const bytes = document.source.bytes;
+  return (
+    bytes instanceof Uint8Array ||
+    (typeof bytes === 'object' && bytes != null && isSerializedBuffer(bytes))
+  );
+}
+
+function toStandardFileBlock(
+  block: BedrockDocumentBlock
+): StandardBase64FileBlock | undefined {
+  const mimeType = BEDROCK_DOCUMENT_MIME_TYPES[block.document.format];
+  if (mimeType == null) {
+    return undefined;
+  }
+  const source = block.document.source.bytes;
+  const bytes = source instanceof Uint8Array ? source : source.data;
+  return {
+    type: 'file',
+    source_type: 'base64',
+    mime_type: mimeType,
+    data: Buffer.from(bytes).toString('base64'),
+    metadata: { name: block.document.name },
+  };
+}
+
+function isStandardFileBlock(
+  block: ProviderContentBlock
+): block is StandardBase64FileBlock {
+  return (
+    block.type === 'file' &&
+    'source_type' in block &&
+    block.source_type === 'base64' &&
+    'mime_type' in block &&
+    typeof block.mime_type === 'string' &&
+    'data' in block &&
+    typeof block.data === 'string'
+  );
+}
+
+function shouldRetainStandardFile(
+  provider: t.ProviderName,
+  mimeType: string
+): boolean {
+  const normalizedMimeType = mimeType.split(';', 1)[0].trim().toLowerCase();
+  if (
+    provider === Providers.ANTHROPIC ||
+    getProviderFamily(provider) === 'anthropic'
+  ) {
+    return (
+      normalizedMimeType === 'application/pdf' ||
+      normalizedMimeType === 'image/jpeg' ||
+      normalizedMimeType === 'image/png' ||
+      normalizedMimeType === 'image/gif' ||
+      normalizedMimeType === 'image/webp'
+    );
+  }
+  if (isOpenAILike(provider)) {
+    return normalizedMimeType === 'application/pdf';
+  }
+  return true;
+}
+
+function canProjectBedrockDocument(
+  provider: t.ProviderName,
+  mimeType: string
+): boolean {
+  if (isGoogleLike(provider)) {
+    return true;
+  }
+  if (
+    provider === Providers.ANTHROPIC ||
+    getProviderFamily(provider) === 'anthropic' ||
+    isOpenAILike(provider)
+  ) {
+    return mimeType === 'application/pdf';
+  }
+  return false;
+}
+
+function isBlankTextBlock(block: ProviderContentBlock): boolean {
+  return (
+    block.type === 'text' &&
+    'text' in block &&
+    typeof block.text === 'string' &&
+    block.text.trim() === ''
+  );
+}
+
+function copyUsableContentPrefix(
+  sourceContent: ProviderContentBlock[],
+  end: number
+): ProviderContentBlock[] {
+  const content: ProviderContentBlock[] = [];
+  for (let index = 0; index < end; index++) {
+    const block = sourceContent[index];
+    if (!isBlankTextBlock(block)) {
+      content.push(block);
+    }
+  }
+  return content;
+}
+
+/** Reprojects persisted provider-native attachments without changing history. */
+function projectAttachmentsForProvider(
+  messages: BaseMessage[],
+  provider: t.ProviderName
+): BaseMessage[] {
+  if (
+    provider === Providers.BEDROCK ||
+    getProviderFamily(provider) === 'bedrock'
+  ) {
+    return messages;
+  }
+  let projected: BaseMessage[] | undefined;
+  for (let messageIndex = 0; messageIndex < messages.length; messageIndex++) {
+    const message = messages[messageIndex];
+    if (!Array.isArray(message.content)) {
+      continue;
+    }
+    const sourceContent = message.content as ProviderContentBlock[];
+    let content: ProviderContentBlock[] | undefined;
+    for (let blockIndex = 0; blockIndex < sourceContent.length; blockIndex++) {
+      const block = sourceContent[blockIndex];
+      if (isStandardFileBlock(block)) {
+        if (shouldRetainStandardFile(provider, block.mime_type)) {
+          content?.push(block);
+          continue;
+        }
+        content ??= copyUsableContentPrefix(sourceContent, blockIndex);
+        continue;
+      }
+      if (!isBedrockDocumentBlock(block)) {
+        if (content != null && !isBlankTextBlock(block)) {
+          content.push(block);
+        }
+        continue;
+      }
+      content ??= copyUsableContentPrefix(sourceContent, blockIndex);
+      const mimeType = BEDROCK_DOCUMENT_MIME_TYPES[block.document.format];
+      if (
+        mimeType == null ||
+        !canProjectBedrockDocument(provider, mimeType)
+      ) {
+        continue;
+      }
+      const standardFile = toStandardFileBlock(block);
+      if (standardFile == null) {
+        continue;
+      }
+      content.push(standardFile);
+    }
+    if (content == null) {
+      continue;
+    }
+    if (content.length === 0) {
+      content.push({ type: 'text', text: OMITTED_ATTACHMENT_TEXT });
+    }
+    projected ??= [...messages];
+    projected[messageIndex] = cloneMessage(
+      message,
+      content as MessageContentComplex[]
+    );
+  }
+  return projected ?? messages;
+}
+
 function projectMessagesForProviderMode(
   { messages, provider, maxToolResultChars }: ProjectMessagesForProviderParams,
   projectionMode: ProviderMessageProjectionMode
 ): BaseMessage[] {
   const providerFamily = getProviderFamily(provider);
   const nativeOpenAIResponses = projectionMode === 'openai-responses';
-  const providerInputMessages = projectToolStreamContentForProvider(
-    messages,
-    nativeOpenAIResponses ? 'native' : 'fallback',
-    maxToolResultChars
+  const providerInputMessages = projectAttachmentsForProvider(
+    projectToolStreamContentForProvider(
+      messages,
+      nativeOpenAIResponses ? 'native' : 'fallback',
+      maxToolResultChars
+    ),
+    provider
   );
   if (nativeOpenAIResponses) {
     return projectOpenAIResponsesToolMessageContent(

package/src/tools/search/rerankers.ts

@@ -4,6 +4,7 @@ import { createDefaultLogger, formatErrorForLog } from './utils';
 import { createSearchMetrics } from './metrics';
 
 const DEFAULT_JINA_API_URL = 'https://api.jina.ai/v1/rerank';
+const DEFAULT_COHERE_API_URL = 'https://api.cohere.com/v2/rerank';
 
 /** Every other network call in the search pipeline is bounded (scrapers,
  * search providers); rerank requests must be too, or a hung rerank API
@@ -15,6 +16,11 @@ const getDefaultJinaApiUrl = (): string =>
     ? process.env.JINA_API_URL
     : DEFAULT_JINA_API_URL;
 
+const getDefaultCohereApiUrl = (): string =>
+  process.env.COHERE_API_URL != null && process.env.COHERE_API_URL !== ''
+    ? process.env.COHERE_API_URL
+    : DEFAULT_COHERE_API_URL;
+
 export abstract class BaseReranker {
   protected apiKey: string | undefined;
   protected logger: t.Logger;
@@ -219,23 +225,27 @@ export class JinaReranker extends BaseReranker {
 
 export class CohereReranker extends BaseReranker {
   readonly provider = 'cohere';
+  private apiUrl: string;
   private timeout: number;
   private httpAgent?: t.HttpAgent;
   private httpsAgent?: t.HttpsAgent;
 
   constructor({
     apiKey = process.env.COHERE_API_KEY,
+    apiUrl = getDefaultCohereApiUrl(),
     timeout = DEFAULT_RERANKER_TIMEOUT,
     logger,
     httpAgent,
     httpsAgent,
   }: {
     apiKey?: string;
+    apiUrl?: string;
     timeout?: number;
     logger?: t.Logger;
   } & t.HttpAgentConfig) {
     super(logger);
     this.apiKey = apiKey;
+    this.apiUrl = apiUrl;
     this.timeout = timeout;
     this.httpAgent = httpAgent;
     this.httpsAgent = httpsAgent;
@@ -263,7 +273,7 @@ export class CohereReranker extends BaseReranker {
       };
 
       const response = await axios.post<t.CohereRerankerResponse | undefined>(
-        'https://api.cohere.com/v2/rerank',
+        this.apiUrl,
         requestData,
         {
           headers: {
@@ -560,6 +570,7 @@ export const createReranker = (
     jinaApiKey?: string;
     jinaApiUrl?: string;
     cohereApiKey?: string;
+    cohereApiUrl?: string;
     ragApiUrl?: string;
     ragApiTokenSupplier?: t.RagApiTokenSupplier;
     ragApiProfile?: string;
@@ -572,6 +583,7 @@ export const createReranker = (
     jinaApiKey,
     jinaApiUrl,
     cohereApiKey,
+    cohereApiUrl,
     ragApiUrl,
     ragApiTokenSupplier,
     ragApiProfile,
@@ -597,6 +609,7 @@ export const createReranker = (
   case 'cohere':
     return new CohereReranker({
       apiKey: cohereApiKey,
+      apiUrl: cohereApiUrl,
       timeout: rerankerTimeout,
       logger: defaultLogger,
       httpAgent,

package/src/tools/search/tool.ts

@@ -20,8 +20,8 @@ import { createFirecrawlScraper } from './firecrawl';
 import { INTENT_PROPERTY } from '@/tools/intentArg';
 import { createCrwScraper } from './crw-scraper';
 import { expandHighlights } from './highlights';
-import { formatResultsForLLM } from './format';
 import { createSearchMetrics } from './metrics';
+import { formatResultsForLLM } from './format';
 import { createDefaultLogger } from './utils';
 import { createReranker } from './rerankers';
 import { Constants } from '@/common';
@@ -514,6 +514,7 @@ export const createSearchTool = (
     jinaApiKey,
     jinaApiUrl,
     cohereApiKey,
+    cohereApiUrl,
     ragApiUrl,
     ragApiTokenSupplier,
     ragApiProfile,
@@ -637,6 +638,7 @@ export const createSearchTool = (
     jinaApiKey,
     jinaApiUrl,
     cohereApiKey,
+    cohereApiUrl,
     ragApiUrl,
     ragApiTokenSupplier,
     ragApiProfile,

package/src/tools/search/types.ts

@@ -519,6 +519,7 @@ export interface SearchToolConfig
   jinaApiKey?: string;
   jinaApiUrl?: string;
   cohereApiKey?: string;
+  cohereApiUrl?: string;
   /** Base URL of the public `rag_api` service (`RAG_API_URL` env fallback).
    * Requests post to `${ragApiUrl}/v1/rerank`. */
   ragApiUrl?: string;

package/src/types/graph.ts

@@ -853,6 +853,8 @@ export interface LangfuseConfig {
 
 interface AgentInputFields {
   agentId: string;
+  /** Logical endpoint selected by the host before provider resolution. */
+  endpoint?: string;
   /**
    * Partition key for transient code-session ids and file refs. Agents with
    * the same key share those refs; different execution profiles/scopes must