@librechat/agents 3.7.18 → 3.7.20

package/src/llm/prepareProviderRequest.ts

@@ -1,5 +1,10 @@
+import { Buffer } from 'node:buffer';
+import type {
+  BaseMessage,
+  Data,
+  MessageContentComplex,
+} from '@langchain/core/messages';
 import type { RunnableConfig } from '@langchain/core/runnables';
-import type { BaseMessage } from '@langchain/core/messages';
 import type { ToolOutputReferenceRegistry } from '@/tools/toolOutputReferences';
 import type * as t from '@/types';
 import {
@@ -20,14 +25,60 @@ import {
 import {
   stripAnthropicCacheControl,
   stripBedrockCacheControl,
+  cloneMessage,
 } from '@/messages/cache';
+import {
+  isAnthropicLike,
+  isGoogleLike,
+  isOpenAILike,
+} from '@/utils/llm';
 import { annotateMessagesForLLM } from '@/tools/toolOutputReferences';
 import { providerRequiresStrictAlternation } from '@/llm/providers';
-import { isAnthropicLike, isOpenAILike } from '@/utils/llm';
 import { getProviderFamily } from '@/llm/providerRegistry';
 import { Providers } from '@/common';
 
 const preparedProviderRequestBrand = Symbol('PreparedProviderRequest');
+const OMITTED_ATTACHMENT_TEXT =
+  '[Attachment omitted because its binary format is unsupported by this provider.]';
+
+const BEDROCK_DOCUMENT_MIME_TYPES: Readonly<Partial<Record<string, string>>> = {
+  csv: 'text/csv',
+  doc: 'application/msword',
+  docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+  html: 'text/html',
+  md: 'text/markdown',
+  pdf: 'application/pdf',
+  txt: 'text/plain',
+  xls: 'application/vnd.ms-excel',
+  xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+};
+
+interface SerializedBuffer {
+  type: 'Buffer';
+  data: number[];
+}
+
+interface BedrockDocumentBlock {
+  type: 'document';
+  document: {
+    name: string;
+    format: string;
+    source: {
+      bytes: Uint8Array | SerializedBuffer;
+    };
+  };
+}
+
+type StandardBase64FileBlock = Data.StandardFileBlock & {
+  source_type: 'base64';
+  mime_type: string;
+  data: string;
+};
+
+type ProviderContentBlock =
+  | MessageContentComplex
+  | BedrockDocumentBlock
+  | StandardBase64FileBlock;
 
 export type ProviderMessageProjectionMode =
   | 'chat-messages'
@@ -159,16 +210,216 @@ interface ProjectMessagesForProviderParams {
   callOptions?: unknown;
 }
 
+function isSerializedBuffer(value: object): value is SerializedBuffer {
+  return (
+    'type' in value &&
+    value.type === 'Buffer' &&
+    'data' in value &&
+    Array.isArray(value.data)
+  );
+}
+
+function isBedrockDocumentBlock(
+  block: ProviderContentBlock
+): block is BedrockDocumentBlock {
+  if (block.type !== 'document' || !('document' in block)) {
+    return false;
+  }
+  const document = block.document;
+  if (typeof document !== 'object' || document == null) {
+    return false;
+  }
+  if (
+    !('name' in document) ||
+    typeof document.name !== 'string' ||
+    !('format' in document) ||
+    typeof document.format !== 'string' ||
+    !('source' in document) ||
+    typeof document.source !== 'object' ||
+    document.source == null ||
+    !('bytes' in document.source)
+  ) {
+    return false;
+  }
+  const bytes = document.source.bytes;
+  return (
+    bytes instanceof Uint8Array ||
+    (typeof bytes === 'object' && bytes != null && isSerializedBuffer(bytes))
+  );
+}
+
+function toStandardFileBlock(
+  block: BedrockDocumentBlock
+): StandardBase64FileBlock | undefined {
+  const mimeType = BEDROCK_DOCUMENT_MIME_TYPES[block.document.format];
+  if (mimeType == null) {
+    return undefined;
+  }
+  const source = block.document.source.bytes;
+  const bytes = source instanceof Uint8Array ? source : source.data;
+  return {
+    type: 'file',
+    source_type: 'base64',
+    mime_type: mimeType,
+    data: Buffer.from(bytes).toString('base64'),
+    metadata: { name: block.document.name },
+  };
+}
+
+function isStandardFileBlock(
+  block: ProviderContentBlock
+): block is StandardBase64FileBlock {
+  return (
+    block.type === 'file' &&
+    'source_type' in block &&
+    block.source_type === 'base64' &&
+    'mime_type' in block &&
+    typeof block.mime_type === 'string' &&
+    'data' in block &&
+    typeof block.data === 'string'
+  );
+}
+
+function shouldRetainStandardFile(
+  provider: t.ProviderName,
+  mimeType: string
+): boolean {
+  const normalizedMimeType = mimeType.split(';', 1)[0].trim().toLowerCase();
+  if (
+    provider === Providers.ANTHROPIC ||
+    getProviderFamily(provider) === 'anthropic'
+  ) {
+    return (
+      normalizedMimeType === 'application/pdf' ||
+      normalizedMimeType === 'image/jpeg' ||
+      normalizedMimeType === 'image/png' ||
+      normalizedMimeType === 'image/gif' ||
+      normalizedMimeType === 'image/webp'
+    );
+  }
+  if (isOpenAILike(provider)) {
+    return normalizedMimeType === 'application/pdf';
+  }
+  return true;
+}
+
+function canProjectBedrockDocument(
+  provider: t.ProviderName,
+  mimeType: string
+): boolean {
+  if (isGoogleLike(provider)) {
+    return true;
+  }
+  if (
+    provider === Providers.ANTHROPIC ||
+    getProviderFamily(provider) === 'anthropic' ||
+    isOpenAILike(provider)
+  ) {
+    return mimeType === 'application/pdf';
+  }
+  return false;
+}
+
+function isBlankTextBlock(block: ProviderContentBlock): boolean {
+  return (
+    block.type === 'text' &&
+    'text' in block &&
+    typeof block.text === 'string' &&
+    block.text.trim() === ''
+  );
+}
+
+function copyUsableContentPrefix(
+  sourceContent: ProviderContentBlock[],
+  end: number
+): ProviderContentBlock[] {
+  const content: ProviderContentBlock[] = [];
+  for (let index = 0; index < end; index++) {
+    const block = sourceContent[index];
+    if (!isBlankTextBlock(block)) {
+      content.push(block);
+    }
+  }
+  return content;
+}
+
+/** Reprojects persisted provider-native attachments without changing history. */
+function projectAttachmentsForProvider(
+  messages: BaseMessage[],
+  provider: t.ProviderName
+): BaseMessage[] {
+  if (
+    provider === Providers.BEDROCK ||
+    getProviderFamily(provider) === 'bedrock'
+  ) {
+    return messages;
+  }
+  let projected: BaseMessage[] | undefined;
+  for (let messageIndex = 0; messageIndex < messages.length; messageIndex++) {
+    const message = messages[messageIndex];
+    if (!Array.isArray(message.content)) {
+      continue;
+    }
+    const sourceContent = message.content as ProviderContentBlock[];
+    let content: ProviderContentBlock[] | undefined;
+    for (let blockIndex = 0; blockIndex < sourceContent.length; blockIndex++) {
+      const block = sourceContent[blockIndex];
+      if (isStandardFileBlock(block)) {
+        if (shouldRetainStandardFile(provider, block.mime_type)) {
+          content?.push(block);
+          continue;
+        }
+        content ??= copyUsableContentPrefix(sourceContent, blockIndex);
+        continue;
+      }
+      if (!isBedrockDocumentBlock(block)) {
+        if (content != null && !isBlankTextBlock(block)) {
+          content.push(block);
+        }
+        continue;
+      }
+      content ??= copyUsableContentPrefix(sourceContent, blockIndex);
+      const mimeType = BEDROCK_DOCUMENT_MIME_TYPES[block.document.format];
+      if (
+        mimeType == null ||
+        !canProjectBedrockDocument(provider, mimeType)
+      ) {
+        continue;
+      }
+      const standardFile = toStandardFileBlock(block);
+      if (standardFile == null) {
+        continue;
+      }
+      content.push(standardFile);
+    }
+    if (content == null) {
+      continue;
+    }
+    if (content.length === 0) {
+      content.push({ type: 'text', text: OMITTED_ATTACHMENT_TEXT });
+    }
+    projected ??= [...messages];
+    projected[messageIndex] = cloneMessage(
+      message,
+      content as MessageContentComplex[]
+    );
+  }
+  return projected ?? messages;
+}
+
 function projectMessagesForProviderMode(
   { messages, provider, maxToolResultChars }: ProjectMessagesForProviderParams,
   projectionMode: ProviderMessageProjectionMode
 ): BaseMessage[] {
   const providerFamily = getProviderFamily(provider);
   const nativeOpenAIResponses = projectionMode === 'openai-responses';
-  const providerInputMessages = projectToolStreamContentForProvider(
-    messages,
-    nativeOpenAIResponses ? 'native' : 'fallback',
-    maxToolResultChars
+  const providerInputMessages = projectAttachmentsForProvider(
+    projectToolStreamContentForProvider(
+      messages,
+      nativeOpenAIResponses ? 'native' : 'fallback',
+      maxToolResultChars
+    ),
+    provider
   );
   if (nativeOpenAIResponses) {
     return projectOpenAIResponsesToolMessageContent(

package/src/tools/search/format.ts

@@ -11,6 +11,9 @@ const DEFAULT_MAX_LLM_OUTPUT_CHARS = 50000;
  *  this we drop it whole rather than emit a useless sliver. */
 const MIN_PARTIAL_HIGHLIGHT_CHARS = 200;
 
+export const MISSING_SEARCH_RESULT_DATA_ERROR =
+  'Search provider returned no result data';
+
 /** Resolves the per-search highlight budget from config, the
  *  `SEARCH_MAX_LLM_OUTPUT_CHARS` env var, or the default (50,000 chars). */
 export function resolveMaxLLMOutputChars(maxOutputChars?: number): number {
@@ -219,9 +222,16 @@ function formatSource(
 
 export function formatResultsForLLM(
   turn: number,
-  results: t.SearchResultData,
+  results?: t.SearchResultData | null,
   maxOutputChars?: number
 ): { output: string; references: t.ResultReference[] } {
+  if (results == null) {
+    return {
+      output: `Search failed: ${MISSING_SEARCH_RESULT_DATA_ERROR}`,
+      references: [],
+    };
+  }
+
   /** Bound highlight content to the per-search budget before formatting */
   const trimmedHighlights = trimHighlightsToBudget(
     results,
@@ -239,6 +249,10 @@ export function formatResultsForLLM(
 
   const references: t.ResultReference[] = [];
 
+  if (results.error != null && results.error !== '') {
+    outputLines.push(`Search failed: ${results.error}`);
+  }
+
   // Organic (web) results
   if (results.organic?.length != null && results.organic.length > 0) {
     addSection(`Web Results, Turn ${turn}`);

package/src/tools/search/tool.ts

@@ -21,7 +21,10 @@ import { INTENT_PROPERTY } from '@/tools/intentArg';
 import { createCrwScraper } from './crw-scraper';
 import { expandHighlights } from './highlights';
 import { createSearchMetrics } from './metrics';
-import { formatResultsForLLM } from './format';
+import {
+  formatResultsForLLM,
+  MISSING_SEARCH_RESULT_DATA_ERROR,
+} from './format';
 import { createDefaultLogger } from './utils';
 import { createReranker } from './rerankers';
 import { Constants } from '@/common';
@@ -66,6 +69,12 @@ export function resolveSearchOutcome(
   return `Found ${count} result${count === 1 ? '' : 's'} for "${query}"`;
 }
 
+export function normalizeSearchResultData(
+  result: t.SearchResultData | null | undefined
+): t.SearchResultData {
+  return result ?? { error: MISSING_SEARCH_RESULT_DATA_ERROR };
+}
+
 /** Distinct rows across the main search's two collections. SearXNG derives
  * both from one result array — a row matching its news heuristic lands in
  * `organic` and `topStories` alike — so summing the lengths would report
@@ -431,12 +440,13 @@ function createTool({
         }),
       });
       const turn = runnableConfig.toolCall?.turn ?? 0;
+      const resultData = normalizeSearchResultData(searchResult);
       const { output, references } = formatResultsForLLM(
         turn,
-        searchResult,
+        resultData,
         maxOutputChars
       );
-      const data: t.SearchResultData = { turn, ...searchResult, references };
+      const data: t.SearchResultData = { turn, ...resultData, references };
       const outcome = resolveSearchOutcome(data, query);
       return [
         output,

package/src/types/graph.ts

@@ -853,6 +853,8 @@ export interface LangfuseConfig {
 
 interface AgentInputFields {
   agentId: string;
+  /** Logical endpoint selected by the host before provider resolution. */
+  endpoint?: string;
   /**
    * Partition key for transient code-session ids and file refs. Agents with
    * the same key share those refs; different execution profiles/scopes must

package/src/types/llm.ts

@@ -81,6 +81,29 @@ export type GoogleThinkingConfig = {
 export type ManagedRequestOptions = {
   promptCacheExplicit?: boolean;
   safety_identifier?: string;
+  /**
+   * Declares that this client talks to the first-party OpenAI or Azure surface.
+   * Gates the model-specific request *shaping* documented only for it —
+   * currently GPT-6 Astra's rejected sampling and logprob parameters, its
+   * unsupported reasoning efforts, and the encrypted reasoning it supports —
+   * and defaults to off.
+   *
+   * Shaping only. Which API serves the turn is not decided here: GPT-6 Astra
+   * serves tool calls only from the Responses API, and a caller wanting them
+   * must select it with `useResponsesApi`, alongside the rest of the request
+   * shaping that depends on which API is in use.
+   *
+   * The shaping runs inside this SDK's own request delegates. A caller that
+   * supplies its own `completions` or `responses` delegate replaces that
+   * construction and owns the request shaping for it — this flag cannot reach
+   * inside a delegate it did not build.
+   *
+   * Declared rather than inferred from a base URL: only the caller knows
+   * whether a URL is a faithful first-party route, a gateway, or a proxy with
+   * its own semantics, and every gate it controls removes capability, so
+   * guessing wrong silently degrades an endpoint the SDK cannot see.
+   */
+  firstPartyEndpoint?: boolean;
 };
 /**
  * Adaptive stream-smoothing configuration shared by every provider client.