@librechat/agents 3.3.3 → 3.3.4

package/src/llm/invoke.ts

@@ -6,6 +6,20 @@ import type { BaseMessage } from '@langchain/core/messages';
 import type { ToolOutputReferenceRegistry } from '@/tools/toolOutputReferences';
 import type { ContextOverflowContext } from '@/utils/errors';
 import type * as t from '@/types';
+import {
+  projectCacheControlledToolOutputsToText,
+  projectComputerCallOutputsToText,
+  projectOpenAIChatToolMessageContent,
+  projectOpenAIResponsesToolMessageContent,
+  projectOpenRouterToolMessageContent,
+  projectSingleTextToolOutputsToText,
+  projectStructuredToolOutputsToText,
+  projectToolStreamContentForProvider,
+} from '@/messages/core';
+import {
+  stripAnthropicCacheControl,
+  stripBedrockCacheControl,
+} from '@/messages/cache';
 import { annotateMessagesForLLM } from '@/tools/toolOutputReferences';
 import { assertNotTruncatedToolCall } from '@/llm/truncation';
 import { Constants, GraphEvents, Providers } from '@/common';
@@ -14,6 +28,7 @@ import { getContextOverflowInfo } from '@/utils/errors';
 import { modifyDeltaProperties } from '@/messages';
 import { ChatModelStreamHandler } from '@/stream';
 import { initializeModel } from '@/llm/init';
+import { isOpenAILike } from '@/utils/llm';
 
 /**
  * Context passed to `attemptInvoke`. Matches the subset of Graph that
@@ -54,6 +69,152 @@ export type InvokeContext = NonNullable<
  */
 export type OnChunk = (chunk: AIMessageChunk) => void | Promise<void>;
 
+export function usesNativeOpenAIResponses(
+  model: t.ChatModel,
+  provider: Providers,
+  callOptions?: unknown
+): boolean {
+  if (!isOpenAILike(provider)) {
+    return false;
+  }
+  let candidate: unknown = model;
+  let effectiveCallOptions = callOptions;
+  const seen = new Set<object>();
+  for (let depth = 0; depth < 20; depth++) {
+    if (candidate == null || typeof candidate !== 'object') {
+      return false;
+    }
+    if (seen.has(candidate)) {
+      return false;
+    }
+    seen.add(candidate);
+    const runnable = candidate as {
+      _useResponsesApi?: (options?: unknown) => boolean;
+      bound?: unknown;
+      defaultOptions?: unknown;
+      last?: unknown;
+      constructor?: { name?: unknown };
+    };
+    try {
+      if (
+        runnable.defaultOptions != null &&
+        typeof runnable.defaultOptions === 'object' &&
+        !Array.isArray(runnable.defaultOptions) &&
+        effectiveCallOptions != null &&
+        typeof effectiveCallOptions === 'object' &&
+        !Array.isArray(effectiveCallOptions)
+      ) {
+        effectiveCallOptions = {
+          ...(runnable.defaultOptions as Record<string, unknown>),
+          ...(effectiveCallOptions as Record<string, unknown>),
+        };
+      } else if (effectiveCallOptions == null) {
+        effectiveCallOptions = runnable.defaultOptions;
+      }
+      if (
+        runnable._useResponsesApi?.(effectiveCallOptions) === true ||
+        runnable._useResponsesApi?.(undefined) === true
+      ) {
+        return true;
+      }
+    } catch {
+      // Continue through RunnableSequence/RunnableBinding wrappers.
+    }
+    if (
+      typeof runnable.constructor?.name === 'string' &&
+      runnable.constructor.name.includes('Responses')
+    ) {
+      return true;
+    }
+    if (runnable.last != null && typeof runnable.last === 'object') {
+      candidate = runnable.last;
+      continue;
+    }
+    if (runnable.bound != null && typeof runnable.bound === 'object') {
+      candidate = runnable.bound;
+      continue;
+    }
+    return false;
+  }
+  return false;
+}
+
+/**
+ * Produces the exact provider-facing message representation before a model
+ * adapter serializes it. This is shared by invocation and Graph's final budget
+ * guard so structured tool output cannot grow after the payload was measured.
+ */
+export function projectMessagesForProvider({
+  model,
+  messages,
+  provider,
+  maxToolResultChars,
+  callOptions,
+}: {
+  model: t.ChatModel;
+  messages: BaseMessage[];
+  provider: Providers;
+  maxToolResultChars?: number;
+  callOptions?: unknown;
+}): BaseMessage[] {
+  const providerInputMessages = projectToolStreamContentForProvider(messages);
+  if (usesNativeOpenAIResponses(model, provider, callOptions)) {
+    return projectOpenAIResponsesToolMessageContent(
+      stripAnthropicCacheControl(
+        stripBedrockCacheControl(providerInputMessages)
+      ),
+      maxToolResultChars
+    );
+  }
+  if (provider === Providers.OPENROUTER) {
+    return projectComputerCallOutputsToText(
+      projectOpenRouterToolMessageContent(
+        stripBedrockCacheControl(providerInputMessages),
+        maxToolResultChars
+      )
+    );
+  }
+  if (isOpenAILike(provider)) {
+    return projectComputerCallOutputsToText(
+      projectOpenAIChatToolMessageContent(
+        stripAnthropicCacheControl(
+          stripBedrockCacheControl(providerInputMessages)
+        ),
+        maxToolResultChars
+      )
+    );
+  }
+  if (provider === Providers.ANTHROPIC) {
+    return projectComputerCallOutputsToText(
+      projectSingleTextToolOutputsToText(
+        stripBedrockCacheControl(providerInputMessages),
+        maxToolResultChars
+      )
+    );
+  }
+  if (provider === Providers.BEDROCK) {
+    return stripAnthropicCacheControl(
+      projectComputerCallOutputsToText(
+        projectCacheControlledToolOutputsToText(
+          providerInputMessages,
+          maxToolResultChars
+        )
+      )
+    );
+  }
+  return projectComputerCallOutputsToText(
+    projectStructuredToolOutputsToText(
+      projectSingleTextToolOutputsToText(
+        stripAnthropicCacheControl(
+          stripBedrockCacheControl(providerInputMessages)
+        ),
+        maxToolResultChars
+      ),
+      maxToolResultChars
+    )
+  );
+}
+
 function getRegisteredDefaultChatStreamHandler(
   context?: InvokeContext
 ): ChatModelStreamHandler | undefined {
@@ -207,9 +368,19 @@ export async function attemptInvoke(
    * untouched so the graph state never sees `[ref: …]` / `_ref`
    * payload.
    */
+  const invocationMessages = projectMessagesForProvider({
+    model,
+    messages,
+    provider,
+    callOptions: config,
+  });
   const registry = context?.getOrCreateToolOutputRegistry();
   const runId = config?.configurable?.run_id as string | undefined;
-  const messagesForProvider = annotateMessagesForLLM(messages, registry, runId);
+  const messagesForProvider = annotateMessagesForLLM(
+    invocationMessages,
+    registry,
+    runId
+  );
 
   /**
    * Stamp the provider that is ACTUALLY serving this invocation onto the
@@ -404,6 +575,7 @@ export async function tryFallbackProviders({
   context,
   onChunk,
   overflowContext,
+  prepareProviderMessages,
 }: {
   fallbacks: t.FallbackConfig[];
   tools?: t.GraphTools;
@@ -419,6 +591,19 @@ export async function tryFallbackProviders({
    * be dropped in favour of whichever failure came last.
    */
   overflowContext?: ContextOverflowContext;
+  /**
+   * Optional final payload guard used by Graph. It receives the initialized,
+   * tool-bound fallback model so Responses-vs-Chat projection is exact before
+   * the fallback request is measured and sent.
+   */
+  prepareProviderMessages?: (input: {
+    model: t.ChatModel;
+    messages: BaseMessage[];
+    provider: Providers;
+    clientOptions?: t.ClientOptions;
+    maxContextTokens?: number;
+    config?: RunnableConfig;
+  }) => BaseMessage[] | Promise<BaseMessage[]>;
 }): Promise<Partial<t.BaseGraphState> | undefined> {
   const isOverflow = (
     error: unknown,
@@ -460,10 +645,19 @@ export async function tryFallbackProviders({
               [Constants.INVOKED_MODEL]: fbModelName,
             },
           };
+      const fallbackMessages =
+        (await prepareProviderMessages?.({
+          model: fbModel as t.ChatModel,
+          messages,
+          provider: fb.provider,
+          clientOptions: fb.clientOptions,
+          maxContextTokens: fb.maxContextTokens,
+          config: fbConfig,
+        })) ?? messages;
       const result = await attemptInvoke(
         {
           model: fbModel as t.ChatModel,
-          messages,
+          messages: fallbackMessages,
           provider: fb.provider,
           context,
           onChunk,

package/src/llm/openai/index.ts

@@ -29,16 +29,28 @@ import type {
   BaseMessageChunk,
   UsageMetadata,
 } from '@langchain/core/messages';
+import type { ChatModelStreamEvent } from '@langchain/core/language_models/event';
 import type { BindToolsInput } from '@langchain/core/language_models/chat_models';
 import type { ChatGeneration, ChatResult } from '@langchain/core/outputs';
 import type { ChatXAIInput } from '@langchain/xai';
 import type * as t from '@langchain/openai';
 import type { SeenScalarMetadata } from './streamMetadata';
 import type { HeaderValue, HeadersLike } from './types';
+import type { PromptCacheTtl } from '@/messages/cache';
+import {
+  buildAnthropicCacheControl,
+  resolvePromptCacheTtl,
+  stripAnthropicCacheControl,
+  stripBedrockCacheControl,
+} from '@/messages/cache';
 import {
   STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY,
   OPENAI_CHAT_SEQUENTIAL_STREAMED_TOOL_CALL_ADAPTER,
 } from '@/tools/streamedToolCallSeals';
+import {
+  projectOpenAIResponsesToolMessageContent,
+  projectToolStreamContentForProvider,
+} from '@/messages/core';
 import { isReasoningModel, _convertMessagesToOpenAIParams } from './utils';
 import { dropRepeatedScalarMetadata } from './streamMetadata';
 
@@ -102,6 +114,9 @@ type LibreChatOpenAIFields = t.ChatOpenAIFields & {
   includeReasoningContent?: boolean;
   includeReasoningDetails?: boolean;
   convertReasoningDetailsToContent?: boolean;
+  preserveToolCacheControl?: boolean;
+  responsesPromptCache?: boolean;
+  responsesPromptCacheTtl?: PromptCacheTtl;
   promptCacheExplicit?: boolean;
   safety_identifier?: string;
 };
@@ -168,6 +183,40 @@ type OpenAIManagedRequestParams = {
   };
   safety_identifier?: string;
 };
+
+function stripResponsesToolCacheControl<T>(tools: T): T {
+  if (!Array.isArray(tools)) {
+    return tools;
+  }
+
+  return tools.map((tool) => {
+    if (tool == null || typeof tool !== 'object') {
+      return tool;
+    }
+    const clone = { ...(tool as Record<string, unknown>) };
+    delete clone.cache_control;
+    if (
+      clone.extras != null &&
+      typeof clone.extras === 'object' &&
+      !Array.isArray(clone.extras)
+    ) {
+      clone.extras = { ...(clone.extras as Record<string, unknown>) };
+      delete (clone.extras as Record<string, unknown>).cache_control;
+    }
+    return clone;
+  }) as T;
+}
+
+function projectOpenAIResponsesProviderMessages(
+  messages: BaseMessage[]
+): BaseMessage[] {
+  return projectOpenAIResponsesToolMessageContent(
+    stripAnthropicCacheControl(
+      stripBedrockCacheControl(projectToolStreamContentForProvider(messages))
+    )
+  );
+}
+
 type ResponsesRequest =
   | OpenAIClient.Responses.ResponseCreateParamsStreaming
   | OpenAIClient.Responses.ResponseCreateParamsNonStreaming;
@@ -1116,6 +1165,7 @@ class LibreChatOpenAICompletions extends OriginalChatOpenAICompletions {
   private includeReasoningContent?: boolean;
   private includeReasoningDetails?: boolean;
   private convertReasoningDetailsToContent?: boolean;
+  private preserveToolCacheControl?: boolean;
   private promptCacheExplicit?: boolean;
   private safetyIdentifier?: string;
 
@@ -1125,6 +1175,7 @@ class LibreChatOpenAICompletions extends OriginalChatOpenAICompletions {
     this.includeReasoningDetails = fields?.includeReasoningDetails;
     this.convertReasoningDetailsToContent =
       fields?.convertReasoningDetailsToContent;
+    this.preserveToolCacheControl = fields?.preserveToolCacheControl;
     this.promptCacheExplicit = fields?.promptCacheExplicit;
     this.safetyIdentifier = fields?.safety_identifier;
   }
@@ -1225,6 +1276,7 @@ class LibreChatOpenAICompletions extends OriginalChatOpenAICompletions {
         includeReasoningContent: this.includeReasoningContent,
         includeReasoningDetails: this.includeReasoningDetails,
         convertReasoningDetailsToContent: this.convertReasoningDetailsToContent,
+        preserveToolCacheControl: this.preserveToolCacheControl,
       }
     );
 
@@ -1393,6 +1445,7 @@ class LibreChatOpenAICompletions extends OriginalChatOpenAICompletions {
         includeReasoningContent: this.includeReasoningContent,
         includeReasoningDetails: this.includeReasoningDetails,
         convertReasoningDetailsToContent: this.convertReasoningDetailsToContent,
+        preserveToolCacheControl: this.preserveToolCacheControl,
       });
 
     const params = {
@@ -1533,21 +1586,52 @@ class LibreChatOpenAICompletions extends OriginalChatOpenAICompletions {
 
 class LibreChatOpenAIResponses extends OriginalChatOpenAIResponses {
   private promptCacheExplicit?: boolean;
+  private responsesPromptCache?: boolean;
+  private responsesPromptCacheTtl?: PromptCacheTtl;
   private safetyIdentifier?: string;
 
   constructor(fields?: LibreChatOpenAIFields) {
     super(fields);
     this.promptCacheExplicit = fields?.promptCacheExplicit;
+    this.responsesPromptCache = fields?.responsesPromptCache;
+    this.responsesPromptCacheTtl = fields?.responsesPromptCacheTtl;
     this.safetyIdentifier = fields?.safety_identifier;
   }
 
   invocationParams(
     options?: this['ParsedCallOptions']
   ): ReturnType<OriginalChatOpenAIResponses['invocationParams']> {
-    const params = applyManagedRequestParams(super.invocationParams(options), {
-      promptCacheExplicit: this.promptCacheExplicit,
-      safetyIdentifier: this.safetyIdentifier,
-    });
+    const cacheOptions = options as
+      | {
+          promptCache?: boolean;
+          promptCacheTtl?: PromptCacheTtl;
+        }
+      | undefined;
+    const promptCache = cacheOptions?.promptCache ?? this.responsesPromptCache;
+    const cacheControl =
+      promptCache === true
+        ? buildAnthropicCacheControl(
+          resolvePromptCacheTtl(
+            cacheOptions?.promptCacheTtl ?? this.responsesPromptCacheTtl
+          )
+        )
+        : undefined;
+    const baseParams = applyManagedRequestParams(
+      super.invocationParams(options),
+      {
+        promptCacheExplicit: this.promptCacheExplicit,
+        safetyIdentifier: this.safetyIdentifier,
+      }
+    );
+    const params = {
+      ...baseParams,
+      ...(baseParams.tools != null && {
+        tools: stripResponsesToolCacheControl(baseParams.tools),
+      }),
+      ...(cacheControl != null && {
+        cache_control: cacheControl,
+      }),
+    };
     if (shouldIncludeEncryptedReasoning(this.model, params)) {
       params.include = [
         ...new Set([
@@ -1592,7 +1676,11 @@ class LibreChatOpenAIResponses extends OriginalChatOpenAIResponses {
     options: this['ParsedCallOptions'],
     runManager?: CallbackManagerForLLMRun
   ): Promise<ChatResult> {
-    const result = await super._generate(messages, options, runManager);
+    const result = await super._generate(
+      projectOpenAIResponsesProviderMessages(messages),
+      options,
+      runManager
+    );
     for (const generation of result.generations) {
       attachCacheWriteUsage(generation.message);
     }
@@ -1605,7 +1693,7 @@ class LibreChatOpenAIResponses extends OriginalChatOpenAIResponses {
     runManager?: CallbackManagerForLLMRun
   ): AsyncGenerator<ChatGenerationChunk> {
     for await (const chunk of super._streamResponseChunks(
-      messages,
+      projectOpenAIResponsesProviderMessages(messages),
       options,
       runManager
     )) {
@@ -1614,6 +1702,18 @@ class LibreChatOpenAIResponses extends OriginalChatOpenAIResponses {
     }
   }
 
+  async *_streamChatModelEvents(
+    messages: BaseMessage[],
+    options: this['ParsedCallOptions'],
+    runManager?: CallbackManagerForLLMRun
+  ): AsyncGenerator<ChatModelStreamEvent> {
+    yield* super._streamChatModelEvents(
+      projectOpenAIResponsesProviderMessages(messages),
+      options,
+      runManager
+    );
+  }
+
   protected _getReasoningParams(
     options?: this['ParsedCallOptions']
   ): OpenAIClient.Reasoning | undefined {