@librechat/agents 3.2.38 → 3.2.39

package/src/messages/anthropicToolCache.ts

@@ -26,6 +26,10 @@
  */
 
 import type { GraphTools } from '@/types';
+import {
+  buildAnthropicCacheControl,
+  type PromptCacheTtl,
+} from '@/messages/cache';
 
 const ANTHROPIC_BUILT_IN_TOOL_PREFIXES = [
   'text_editor_',
@@ -41,8 +45,6 @@ const ANTHROPIC_BUILT_IN_TOOL_PREFIXES = [
   'mcp_toolset',
 ] as const;
 
-const CACHE_CONTROL = { type: 'ephemeral' as const };
-
 type AnthropicToolCacheCandidate = {
   name?: unknown;
   type?: unknown;
@@ -60,29 +62,91 @@ function isAnthropicBuiltInTool(
   );
 }
 
+/**
+ * Whether a tool already carries a cache breakpoint. Built-ins use a direct
+ * `cache_control`; custom tools normally carry it under `extras`, but a
+ * caller-supplied Anthropic-native tool object can also put it directly on the
+ * block — check both so stale markers are never missed.
+ */
 function hasCacheControl(tool: AnthropicToolCacheCandidate): boolean {
   if (isAnthropicBuiltInTool(tool)) {
     return tool.cache_control != null;
   }
-  return tool.extras?.cache_control != null;
+  return tool.cache_control != null || tool.extras?.cache_control != null;
 }
 
-function markCacheControl(
+/**
+ * Return the `cache_control` from the location that actually reaches the
+ * Anthropic payload for this tool's shape: directly on the block for
+ * provider-shaped tools (built-ins and raw Anthropic tools), or under `extras`
+ * for LangChain custom tools (the adapter promotes only that). Returns
+ * `undefined` when no marker sits in the effective location — a marker in the
+ * wrong place (e.g. a direct marker on a custom tool) does not count.
+ */
+function getEffectiveCacheControl(
+  tool: AnthropicToolCacheCandidate
+): { ttl?: unknown } | undefined {
+  return (
+    isProviderShapedTool(tool) ? tool.cache_control : tool.extras?.cache_control
+  ) as { ttl?: unknown } | undefined;
+}
+
+/**
+ * Return a clone of `tool` with any `cache_control` removed — both the direct
+ * block marker and the `extras` marker — preserving the prototype chain. Used
+ * to clear stray markers off tools that must not anchor a competing breakpoint.
+ */
+function stripCacheControl(
   tool: AnthropicToolCacheCandidate
 ): AnthropicToolCacheCandidate {
   const prototype = Object.getPrototypeOf(tool) ?? Object.prototype;
-  if (isAnthropicBuiltInTool(tool)) {
+  const wrapped = { ...tool };
+  delete wrapped.cache_control;
+  if (wrapped.extras != null) {
+    wrapped.extras = { ...wrapped.extras };
+    delete wrapped.extras.cache_control;
+  }
+  return Object.assign(Object.create(prototype), wrapped);
+}
+
+/**
+ * Whether `tool` is already in the Anthropic provider payload shape — an
+ * Anthropic built-in or a raw Anthropic tool object (has `input_schema`). These
+ * carry `cache_control` directly on the block; the LangChain adapter does NOT
+ * promote `extras.cache_control` for them. LangChain StructuredTools, by
+ * contrast, expose the marker via `extras`.
+ */
+function isProviderShapedTool(tool: AnthropicToolCacheCandidate): boolean {
+  return (
+    isAnthropicBuiltInTool(tool) ||
+    'input_schema' in (tool as Record<string, unknown>)
+  );
+}
+
+function markCacheControl(
+  tool: AnthropicToolCacheCandidate,
+  ttl?: PromptCacheTtl
+): AnthropicToolCacheCandidate {
+  const cacheControl = buildAnthropicCacheControl(ttl);
+  const prototype = Object.getPrototypeOf(tool) ?? Object.prototype;
+  if (isProviderShapedTool(tool)) {
+    // Built-ins and raw Anthropic tool objects carry cache_control directly on
+    // the block; `extras` is not promoted onto the payload for these shapes.
     const wrapped = { ...tool };
     delete wrapped.extras;
     return Object.assign(Object.create(prototype), wrapped, {
-      cache_control: CACHE_CONTROL,
+      cache_control: cacheControl,
     });
   }
 
-  return Object.assign(Object.create(prototype), tool, {
+  // LangChain custom tools: drop any direct marker and expose the breakpoint via
+  // `extras`, which the Anthropic adapter promotes onto the payload.
+  const wrapped = { ...tool };
+  delete wrapped.cache_control;
+  return Object.assign(Object.create(prototype), wrapped, {
     extras: {
       ...(tool.extras ?? {}),
-      cache_control: CACHE_CONTROL,
+      cache_control: cacheControl,
     },
   });
 }
@@ -124,7 +188,8 @@ export function makeIsDeferred(
  */
 export function partitionAndMarkAnthropicToolCache(
   tools: GraphTools | undefined,
-  isDeferred: (toolName: string) => boolean
+  isDeferred: (toolName: string) => boolean,
+  ttl?: PromptCacheTtl
 ): GraphTools | undefined {
   if (tools == null || tools.length === 0) return tools;
 
@@ -143,19 +208,57 @@ export function partitionAndMarkAnthropicToolCache(
     }
   }
 
+  // Anthropic serializes ALL tools before system/messages, so a stray
+  // cache_control on any tool — static or deferred — that survives the resolved
+  // breakpoint would violate the longer-TTL-first ordering. Strip stale markers
+  // off the deferred tools first (they sit after the breakpoint but still before
+  // system/messages, and the all-deferred case has no breakpoint of its own).
+  let mutated = false;
+  for (let i = 0; i < deferredTools.length; i++) {
+    const candidate = deferredTools[i] as AnthropicToolCacheCandidate;
+    if (hasCacheControl(candidate)) {
+      deferredTools[i] = stripCacheControl(candidate);
+      mutated = true;
+    }
+  }
+
   if (staticTools.length === 0) {
-    return tools;
+    return mutated ? ([...deferredTools] as GraphTools) : tools;
+  }
+
+  // Strip any stray cache_control off the earlier static tools so a leftover
+  // 5-minute marker never sits ahead of the resolved breakpoint, then stamp (or
+  // re-stamp) only the last static tool with the resolved TTL.
+  for (let i = 0; i < staticTools.length - 1; i++) {
+    const candidate = staticTools[i] as AnthropicToolCacheCandidate;
+    if (hasCacheControl(candidate)) {
+      staticTools[i] = stripCacheControl(candidate);
+      mutated = true;
+    }
   }
 
   const last = staticTools[
     staticTools.length - 1
   ] as AnthropicToolCacheCandidate;
-  // Already marked? Don't double-clone.
-  if (hasCacheControl(last)) {
-    if (deferredTools.length === 0) return tools;
-    return [...staticTools, ...deferredTools] as GraphTools;
+  const desiredTtl: '1h' | undefined = ttl === '1h' ? '1h' : undefined;
+  // "Already correct" requires a marker IN the effective location (one that
+  // reaches the payload for this tool's shape) carrying the resolved TTL. A
+  // marker in the wrong place — e.g. a direct `cache_control` on a LangChain
+  // custom tool — is ineffective and must be re-stamped.
+  const effective = getEffectiveCacheControl(last);
+  const lastAlreadyCorrect =
+    effective != null &&
+    (effective.ttl === '1h' ? '1h' : undefined) === desiredTtl;
+  if (!lastAlreadyCorrect) {
+    staticTools[staticTools.length - 1] = markCacheControl(last, ttl);
+    mutated = true;
   }
 
-  staticTools[staticTools.length - 1] = markCacheControl(last);
+  // Return the original reference only when nothing changed AND partitioning
+  // moved nothing. When deferred tools exist they must end up after the cache
+  // breakpoint, so the partitioned array is returned even if no marker changed.
+  if (!mutated && deferredTools.length === 0) {
+    return tools;
+  }
   return [...staticTools, ...deferredTools] as GraphTools;
 }

package/src/messages/cache.test.ts

@@ -12,8 +12,14 @@ import {
   stripAnthropicCacheControl,
   stripBedrockCacheControl,
   addBedrockCacheControl,
+  addBedrockTailCacheControl,
   addCacheControl,
+  addTailCacheControl,
   addCacheControlToStablePrefixMessages,
+  buildAnthropicCacheControl,
+  buildBedrockCachePoint,
+  resolvePromptCacheTtl,
+  DEFAULT_PROMPT_CACHE_TTL,
 } from './cache';
 import { _convertMessagesToOpenAIParams } from '@/llm/openai/utils';
 import { toLangChainContent } from './langchain';
@@ -1813,3 +1819,172 @@ describe('OpenRouter prompt caching (reuses addCacheControl)', () => {
     expect('cache_control' in lastContent[0]).toBe(true);
   });
 });
+
+describe('prompt-cache TTL (1h default, 5m legacy)', () => {
+  describe('resolvePromptCacheTtl', () => {
+    it('defaults to the 1h extended cache when unset', () => {
+      expect(resolvePromptCacheTtl(undefined)).toBe('1h');
+      expect(DEFAULT_PROMPT_CACHE_TTL).toBe('1h');
+    });
+
+    it('passes an explicit value through unchanged', () => {
+      expect(resolvePromptCacheTtl('5m')).toBe('5m');
+      expect(resolvePromptCacheTtl('1h')).toBe('1h');
+    });
+  });
+
+  describe('marker builders', () => {
+    it('buildAnthropicCacheControl adds ttl only for 1h', () => {
+      expect(buildAnthropicCacheControl('1h')).toEqual({
+        type: 'ephemeral',
+        ttl: '1h',
+      });
+      // 5m / undefined stay byte-identical to the legacy marker (no ttl)
+      expect(buildAnthropicCacheControl('5m')).toEqual({ type: 'ephemeral' });
+      expect(buildAnthropicCacheControl()).toEqual({ type: 'ephemeral' });
+    });
+
+    it('buildBedrockCachePoint adds ttl only for 1h', () => {
+      expect(buildBedrockCachePoint('1h')).toEqual({
+        type: 'default',
+        ttl: '1h',
+      });
+      expect(buildBedrockCachePoint('5m')).toEqual({ type: 'default' });
+      expect(buildBedrockCachePoint()).toEqual({ type: 'default' });
+    });
+  });
+
+  describe('addTailCacheControl threads ttl', () => {
+    const baseMessages = (): AnthropicMessages => [
+      { role: 'user', content: 'Hello' },
+      { role: 'assistant', content: 'Hi there' },
+    ];
+
+    it('stamps a 1h cache_control on the tail block', () => {
+      const result = addTailCacheControl(baseMessages(), '1h');
+      const tail = result[result.length - 1].content as MessageContentComplex[];
+      expect(tail[tail.length - 1]).toEqual({
+        type: 'text',
+        text: 'Hi there',
+        cache_control: { type: 'ephemeral', ttl: '1h' },
+      });
+    });
+
+    it('omits ttl for 5m and for the unspecified (legacy) default', () => {
+      for (const ttl of ['5m', undefined] as const) {
+        const result = addTailCacheControl(baseMessages(), ttl);
+        const tail = result[result.length - 1]
+          .content as MessageContentComplex[];
+        expect(
+          (tail[tail.length - 1] as Anthropic.TextBlockParam).cache_control
+        ).toEqual({ type: 'ephemeral' });
+      }
+    });
+  });
+
+  describe('addCacheControl threads ttl', () => {
+    it('stamps a 1h cache_control on the latest user messages', () => {
+      const messages: AnthropicMessages = [
+        { role: 'user', content: 'first' },
+        { role: 'assistant', content: 'reply' },
+        { role: 'user', content: 'second' },
+      ];
+      const result = addCacheControl(messages, '1h');
+      const lastUser = result[2].content as MessageContentComplex[];
+      expect((lastUser[0] as Anthropic.TextBlockParam).cache_control).toEqual({
+        type: 'ephemeral',
+        ttl: '1h',
+      });
+    });
+  });
+
+  describe('addCacheControlToStablePrefixMessages threads ttl', () => {
+    it('stamps 1h on every stable-prefix marker', () => {
+      const messages: AnthropicMessages = [
+        { role: 'user', content: 'turn 1' },
+        { role: 'assistant', content: 'reply 1' },
+        { role: 'user', content: 'turn 2' },
+        { role: 'assistant', content: 'reply 2' },
+      ];
+      const result = addCacheControlToStablePrefixMessages(messages, 2, '1h');
+      const marked = result
+        .flatMap((m) =>
+          Array.isArray(m.content) ? (m.content as MessageContentComplex[]) : []
+        )
+        .filter((block) => 'cache_control' in block);
+      expect(marked.length).toBeGreaterThan(0);
+      for (const block of marked) {
+        expect((block as Anthropic.TextBlockParam).cache_control).toEqual({
+          type: 'ephemeral',
+          ttl: '1h',
+        });
+      }
+    });
+  });
+
+  describe('addBedrockTailCacheControl threads ttl', () => {
+    const messages = (): TestMsg[] => [
+      { role: 'user', content: 'Hello' },
+      { role: 'assistant', content: 'Hi' },
+    ];
+
+    it('stamps a 1h cachePoint on the tail message', () => {
+      const result = addBedrockTailCacheControl(messages(), '1h');
+      const last = result[result.length - 1].content as MessageContentComplex[];
+      expect(last[last.length - 1]).toEqual({
+        cachePoint: { type: 'default', ttl: '1h' },
+      });
+    });
+
+    it('omits ttl for 5m and the legacy default', () => {
+      for (const ttl of ['5m', undefined] as const) {
+        const result = addBedrockTailCacheControl(messages(), ttl);
+        const last = result[result.length - 1]
+          .content as MessageContentComplex[];
+        expect(last[last.length - 1]).toEqual({
+          cachePoint: { type: 'default' },
+        });
+      }
+    });
+
+    it('normalizes a stale 5m system cachePoint to the resolved tail ttl', () => {
+      const msgs: TestMsg[] = [
+        {
+          role: 'system',
+          content: [
+            { type: ContentTypes.TEXT, text: 'System' },
+            { cachePoint: { type: 'default' } } as MessageContentComplex,
+          ],
+        },
+        { role: 'user', content: 'Hello' },
+        { role: 'assistant', content: 'Hi' },
+      ];
+      const result = addBedrockTailCacheControl(msgs, '1h');
+      // Stale 5m system checkpoint is upgraded to 1h so it never precedes the
+      // 1h message tail (Bedrock requires longer-TTL checkpoints first).
+      const system = result[0].content as MessageContentComplex[];
+      expect(system[system.length - 1]).toEqual({
+        cachePoint: { type: 'default', ttl: '1h' },
+      });
+      const tail = result[result.length - 1].content as MessageContentComplex[];
+      expect(tail[tail.length - 1]).toEqual({
+        cachePoint: { type: 'default', ttl: '1h' },
+      });
+    });
+  });
+
+  describe('addBedrockCacheControl threads ttl', () => {
+    it('stamps a 1h cachePoint when configured', () => {
+      const messages: TestMsg[] = [
+        { role: 'user', content: 'Hello' },
+        { role: 'assistant', content: 'Hi' },
+      ];
+      const result = addBedrockCacheControl(messages, '1h');
+      // Only one user message present, so the cachePoint anchors on it.
+      const user = result[0].content as MessageContentComplex[];
+      expect(user[user.length - 1]).toEqual({
+        cachePoint: { type: 'default', ttl: '1h' },
+      });
+    });
+  });
+});