@librechat/agents 3.7.4 → 3.7.5

package/src/graphs/Graph.ts

@@ -59,14 +59,12 @@ import {
   addTailCacheControl,
   resolvePromptCacheTtl,
   resolveBedrockPromptCacheTtl,
-  supportsBedrockToolCache,
   isSyntheticProviderContextMessage,
   compactSyntheticProviderContextMessage,
   getMessageId,
   getMessageCreationContentMetadata,
   splitAssistantTextContentByPhase,
   makeIsDeferred,
-  partitionAndMarkAnthropicToolCache,
   DEFAULT_RETAIN_RECENT_TURNS,
   resolveIntraTurnRetainTokens,
   splitAtRecencyBoundary,
@@ -149,14 +147,13 @@ import {
   findCallback,
   type CallbackEntry,
 } from '@/utils/callbacks';
-import { partitionAndMarkOpenRouterToolCache } from '@/llm/openrouter/toolCache';
 import { ToolNode as CustomToolNode, toolsCondition } from '@/tools/ToolNode';
 import { shouldTraceToolNodeForLangfuse } from '@/langfuseToolOutputTracing';
 import { createLocalCodingToolBundle } from '@/tools/local/LocalCodingTools';
 import { SUBAGENT_REPLAY_CONTROLLER } from '@/tools/subagent/SubagentReplay';
 import { applyGraphRuntimeConfig } from '@/graphs/applyGraphRuntimeConfig';
-import { partitionAndMarkBedrockToolCache } from '@/llm/bedrock/toolCache';
 import { createContextPressureMeter } from '@/llm/contextPressureMeter';
+import { prepareToolsForPromptCache } from '@/llm/promptCacheTools';
 import { safeDispatchCustomEvent, emitAgentLog } from '@/utils/events';
 import { prepareProviderRequest } from '@/llm/prepareProviderRequest';
 import { createCloudflareCodingToolBundle } from '@/tools/cloudflare';
@@ -2789,6 +2786,27 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
     };
   }
 
+  private getPreparedToolsForBinding(
+    agentContext: AgentContext,
+    provider: t.ProviderName = agentContext.provider,
+    clientOptions: t.ClientOptions | undefined = agentContext.clientOptions
+  ): t.GraphTools | undefined {
+    const tools = resolveLocalToolsForBinding({
+      tools: agentContext.getToolsForBinding(),
+      toolExecution: this.toolExecution,
+      toolRegistry: agentContext.toolRegistry,
+      discoveredToolNames: new Set(agentContext.getDiscoveredTools()),
+    });
+    return prepareToolsForPromptCache({
+      provider,
+      clientOptions,
+      tools,
+      isDeferred: makeIsDeferred(
+        agentContext.getEffectiveToolDefinitions()
+      ),
+    });
+  }
+
   createCallModel(agentId = 'default') {
     return async (
       state: t.AgentSubgraphState,
@@ -2846,13 +2864,6 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
         agentContext.markToolsAsDiscovered(discoveredNames);
       }
 
-      const rawToolsForBinding = resolveLocalToolsForBinding({
-        tools: agentContext.getToolsForBinding(),
-        toolExecution: this.toolExecution,
-        toolRegistry: agentContext.toolRegistry,
-        discoveredToolNames: new Set(agentContext.getDiscoveredTools()),
-      });
-
       /**
        * Anthropic prompt-cache breakpoint on the tool definitions.
        *
@@ -2866,69 +2877,7 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
        * Discovered deferred tools that arrive across turns sit *after*
        * the breakpoint and don't invalidate the prefix.
        */
-      let toolsForBinding = rawToolsForBinding;
-      const isDeferredTool = makeIsDeferred(
-        agentContext.getEffectiveToolDefinitions()
-      );
-      if (
-        agentContext.provider === Providers.ANTHROPIC &&
-        (agentContext.clientOptions as t.AnthropicClientOptions | undefined)
-          ?.promptCache === true
-      ) {
-        toolsForBinding =
-          partitionAndMarkAnthropicToolCache(
-            rawToolsForBinding,
-            isDeferredTool,
-            resolvePromptCacheTtl(
-              (
-                agentContext.clientOptions as
-                  | t.AnthropicClientOptions
-                  | undefined
-              )?.promptCacheTtl
-            )
-          ) ?? rawToolsForBinding;
-      } else if (
-        agentContext.provider === Providers.OPENROUTER &&
-        (
-          agentContext.clientOptions as
-            | t.ProviderOptionsMap[Providers.OPENROUTER]
-            | undefined
-        )?.promptCache === true
-      ) {
-        toolsForBinding =
-          partitionAndMarkOpenRouterToolCache(
-            rawToolsForBinding,
-            isDeferredTool,
-            resolvePromptCacheTtl(
-              (
-                agentContext.clientOptions as
-                  | t.ProviderOptionsMap[Providers.OPENROUTER]
-                  | undefined
-              )?.promptCacheTtl
-            )
-          ) ?? rawToolsForBinding;
-      } else if (
-        agentContext.provider === Providers.BEDROCK &&
-        (
-          agentContext.clientOptions as
-            | t.BedrockAnthropicClientOptions
-            | undefined
-        )?.promptCache === true
-      ) {
-        const bedrockModel = (
-          agentContext.clientOptions as { model?: string } | undefined
-        )?.model;
-        // An omitted model falls back to LangChain's default Claude model (which
-        // supports tool caching); only an explicit non-Claude model (e.g. Nova)
-        // skips tool marking so its stray marker never leaks into toolConfig.
-        if (bedrockModel == null || supportsBedrockToolCache(bedrockModel)) {
-          toolsForBinding =
-            partitionAndMarkBedrockToolCache(
-              rawToolsForBinding,
-              isDeferredTool
-            ) ?? rawToolsForBinding;
-        }
-      }
+      const toolsForBinding = this.getPreparedToolsForBinding(agentContext);
 
       let model =
         this.overrideModel ??
@@ -4924,6 +4873,15 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
             runId: this.runId,
             isMultiAgent: this.isMultiAgentGraph(),
             hookRegistry: this.hookRegistry,
+            getToolsForBinding: (
+              provider: t.ProviderName,
+              clientOptions: t.ClientOptions | undefined
+            ): t.GraphTools | undefined =>
+              this.getPreparedToolsForBinding(
+                agentContext,
+                provider,
+                clientOptions
+              ),
             /**
              * Live references (both maps are cleared in place, never
              * replaced), so summarization streams share the run's event

package/src/llm/promptCacheTools.ts

@@ -0,0 +1,58 @@
+import type * as t from '@/types';
+import {
+  resolvePromptCacheTtl,
+  supportsBedrockToolCache,
+} from '@/messages/cache';
+import { partitionAndMarkAnthropicToolCache } from '@/messages/anthropicToolCache';
+import { partitionAndMarkOpenRouterToolCache } from '@/llm/openrouter/toolCache';
+import { partitionAndMarkBedrockToolCache } from '@/llm/bedrock/toolCache';
+import { Providers } from '@/common';
+
+export function prepareToolsForPromptCache(params: {
+  provider: t.ProviderName;
+  clientOptions?: t.ClientOptions;
+  tools?: t.GraphTools;
+  isDeferred: (toolName: string) => boolean;
+}): t.GraphTools | undefined {
+  const { provider, clientOptions, tools, isDeferred } = params;
+  if (provider === Providers.ANTHROPIC) {
+    const options = clientOptions as t.AnthropicClientOptions | undefined;
+    if (options?.promptCache !== true) {
+      return tools;
+    }
+    return (
+      partitionAndMarkAnthropicToolCache(
+        tools,
+        isDeferred,
+        resolvePromptCacheTtl(options.promptCacheTtl)
+      ) ?? tools
+    );
+  }
+  if (provider === Providers.OPENROUTER) {
+    const options = clientOptions as
+      | t.ProviderOptionsMap[Providers.OPENROUTER]
+      | undefined;
+    if (options?.promptCache !== true) {
+      return tools;
+    }
+    return (
+      partitionAndMarkOpenRouterToolCache(
+        tools,
+        isDeferred,
+        resolvePromptCacheTtl(options.promptCacheTtl)
+      ) ?? tools
+    );
+  }
+  if (provider !== Providers.BEDROCK) {
+    return tools;
+  }
+  const options = clientOptions as t.BedrockAnthropicClientOptions | undefined;
+  if (options?.promptCache !== true) {
+    return tools;
+  }
+  const model = (options as { model?: string }).model;
+  if (model != null && !supportsBedrockToolCache(model)) {
+    return tools;
+  }
+  return partitionAndMarkBedrockToolCache(tools, isDeferred) ?? tools;
+}

package/src/summarization/node.ts

@@ -24,7 +24,9 @@ import {
 } from '@/llm/streamLimits';
 import {
   addTailCacheControl,
+  addBedrockTailCacheControl,
   resolvePromptCacheTtl,
+  resolveBedrockPromptCacheTtl,
   type PromptCacheTtl,
 } from '@/messages/cache';
 import {
@@ -41,7 +43,9 @@ import {
 } from '@/common';
 import { safeDispatchCustomEvent, emitAgentLog } from '@/utils/events';
 import { attemptInvoke, tryFallbackProviders } from '@/llm/invoke';
+import { prepareToolsForPromptCache } from '@/llm/promptCacheTools';
 import { calculateMaxToolResultChars } from '@/utils/truncation';
+import { makeIsDeferred } from '@/messages/anthropicToolCache';
 import { createRemoveAllMessage } from '@/messages/reducer';
 import { getMaxOutputTokensKey } from '@/llm/request';
 import { initializeModel } from '@/llm/init';
@@ -623,7 +627,13 @@ async function executeSummarizationWithFallback(params: {
   usePromptCache: boolean;
   log: LogFn;
   /** Carries the run's stream limits so the event cap covers summary streams. */
-  graph?: StreamLimitState & { getBreakerSignal?: () => AbortSignal };
+  graph?: StreamLimitState & {
+    getBreakerSignal?: () => AbortSignal;
+    getToolsForBinding?: (
+      provider: t.ProviderName,
+      clientOptions: t.ClientOptions | undefined
+    ) => t.GraphTools | undefined;
+  };
 }): Promise<{
   text: string;
   usage?: Partial<UsageMetadata>;
@@ -657,10 +667,24 @@ async function executeSummarizationWithFallback(params: {
      * (e.g. an unrecognized summarization.provider) surfaces through the
      * `log('error', ...)` path below rather than bubbling up silently.
      */
+    const summarizationTools = usePromptCache
+      ? (graph?.getToolsForBinding?.(
+          clientConfig.provider as t.ProviderName,
+          clientConfig.clientOptions as t.ClientOptions
+      ) ??
+        prepareToolsForPromptCache({
+          provider: clientConfig.provider as t.ProviderName,
+          clientOptions: clientConfig.clientOptions as t.ClientOptions,
+          tools: agentContext.getToolsForBinding(),
+          isDeferred: makeIsDeferred(
+            agentContext.getEffectiveToolDefinitions()
+          ),
+        }))
+      : agentContext.getToolsForBinding();
     const summarizationModel = initializeModel({
       provider: clientConfig.provider,
       clientOptions: clientConfig.clientOptions as t.ClientOptions,
-      tools: agentContext.getToolsForBinding(),
+      tools: summarizationTools,
     }) as t.ChatModel;
 
     const result = await summarizeWithCacheHit({
@@ -677,13 +701,23 @@ async function executeSummarizationWithFallback(params: {
       usePromptCache,
       promptCacheTtl:
         clientConfig.provider === Providers.ANTHROPIC ||
-        clientConfig.provider === Providers.OPENROUTER
-          ? resolvePromptCacheTtl(
-            (
+        clientConfig.provider === Providers.OPENROUTER ||
+        clientConfig.provider === Providers.BEDROCK
+          ? (
                 clientConfig.clientOptions as {
                   promptCacheTtl?: PromptCacheTtl;
                 }
-            ).promptCacheTtl
+          ).promptCacheTtl
+          : undefined,
+      bedrockModelId:
+        clientConfig.provider === Providers.BEDROCK
+          ? resolveBedrockCompactionCacheModel(
+            clientConfig.clientOptions as
+              | {
+                applicationInferenceProfile?: string;
+                model?: string;
+              }
+              | undefined
           )
           : undefined,
       log,
@@ -900,6 +934,10 @@ interface CreateSummarizeNodeParams {
     runId?: string;
     isMultiAgent: boolean;
     hookRegistry?: HookRegistry;
+    getToolsForBinding?: (
+      provider: t.ProviderName,
+      clientOptions: t.ClientOptions | undefined
+    ) => t.GraphTools | undefined;
     dispatchRunStep: (
       runStep: t.RunStep,
       config?: RunnableConfig
@@ -1221,7 +1259,7 @@ export function createSummarizeNode({
       clientConfig.provider === (agentContext.provider as string);
     const hasPromptCache =
       isSelfSummarizeModel &&
-      (agentContext.clientOptions as Record<string, unknown> | undefined)
+      (clientConfig.clientOptions as Record<string, unknown> | undefined)
         ?.promptCache === true;
 
     const log: LogFn = (level, message, data) => {
@@ -1615,6 +1653,45 @@ function traceConfig(
   };
 }
 
+export function applySummarizationHistoryCache(params: {
+  messages: BaseMessage[];
+  provider: t.ProviderName;
+  enabled: boolean;
+  promptCacheTtl?: PromptCacheTtl;
+  bedrockModelId?: string;
+}): BaseMessage[] {
+  if (!params.enabled) {
+    return params.messages;
+  }
+  if (params.provider === Providers.BEDROCK) {
+    return addBedrockTailCacheControl(
+      [...params.messages],
+      resolveBedrockPromptCacheTtl(
+        params.promptCacheTtl,
+        params.bedrockModelId
+      )
+    );
+  }
+  if (
+    params.provider !== Providers.ANTHROPIC &&
+    params.provider !== Providers.OPENROUTER
+  ) {
+    return params.messages;
+  }
+  return addTailCacheControl(
+    [...params.messages],
+    resolvePromptCacheTtl(params.promptCacheTtl)
+  );
+}
+
+export function resolveBedrockCompactionCacheModel(
+  options:
+    | { applicationInferenceProfile?: string; model?: string }
+    | undefined
+): string | undefined {
+  return options?.model;
+}
+
 /**
  * Cache-friendly compaction: sends raw conversation messages with the
  * summarization instruction appended as the final HumanMessage. Bound tool
@@ -1635,6 +1712,7 @@ async function summarizeWithCacheHit({
   graph,
   usePromptCache,
   promptCacheTtl,
+  bedrockModelId,
   log,
 }: {
   model: t.ChatModel;
@@ -1649,6 +1727,7 @@ async function summarizeWithCacheHit({
   graph?: StreamLimitState & { getBreakerSignal?: () => AbortSignal };
   usePromptCache?: boolean;
   promptCacheTtl?: PromptCacheTtl;
+  bedrockModelId?: string;
   log?: LogFn;
 }): Promise<{ text: string; usage?: Partial<UsageMetadata> }> {
   const instruction = buildSummarizationInstruction(
@@ -1657,11 +1736,17 @@ async function summarizeWithCacheHit({
     priorSummaryText
   );
 
-  const fullMessages = [...messages, new HumanMessage(instruction)];
-  const invokeMessages =
-    usePromptCache === true
-      ? addTailCacheControl(fullMessages, promptCacheTtl)
-      : fullMessages;
+  const cachedHistory = applySummarizationHistoryCache({
+    messages,
+    provider,
+    enabled: usePromptCache === true,
+    promptCacheTtl,
+    bedrockModelId,
+  });
+  const invokeMessages = [
+    ...cachedHistory,
+    new HumanMessage(instruction),
+  ];
 
   const result = await attemptInvoke(
     {