@librechat/agents 3.2.38 → 3.2.41

package/src/messages/__tests__/anthropicToolCache.test.ts

@@ -140,6 +140,174 @@ describe('partitionAndMarkAnthropicToolCache', () => {
     // is returned unchanged (same reference) so we don't churn the array.
     expect(partitionAndMarkAnthropicToolCache(input, () => false)).toBe(input);
   });
+
+  it('stamps the resolved 1h ttl on the last static tool', () => {
+    const out = partitionAndMarkAnthropicToolCache(
+      [fakeTool('a-static'), fakeTool('b-static')] as never,
+      () => false,
+      '1h'
+    ) as Array<{
+      extras?: { cache_control?: { type: string; ttl?: string } };
+    }>;
+    expect(out[1].extras?.cache_control).toEqual({
+      type: 'ephemeral',
+      ttl: '1h',
+    });
+    expect(out[0].extras?.cache_control).toBeUndefined();
+  });
+
+  it('re-stamps a pre-marked 5m tool to 1h so it does not precede a 1h breakpoint', () => {
+    const a = fakeTool('a-static') as {
+      extras?: { cache_control?: { type: string; ttl?: string } };
+    };
+    a.extras = { cache_control: { type: 'ephemeral' } };
+    const out = partitionAndMarkAnthropicToolCache(
+      [a] as never,
+      () => false,
+      '1h'
+    ) as Array<{
+      extras?: { cache_control?: { type: string; ttl?: string } };
+    }>;
+    expect(out[0].extras?.cache_control).toEqual({
+      type: 'ephemeral',
+      ttl: '1h',
+    });
+  });
+
+  it('strips a pre-marked earlier static tool so only the tail carries the 1h marker', () => {
+    const a = fakeTool('a-static') as {
+      extras?: { cache_control?: { type: string; ttl?: string } };
+    };
+    a.extras = { cache_control: { type: 'ephemeral' } };
+    const b = fakeTool('b-static');
+    const out = partitionAndMarkAnthropicToolCache(
+      [a, b] as never,
+      () => false,
+      '1h'
+    ) as Array<{
+      extras?: { cache_control?: { type: string; ttl?: string } };
+    }>;
+    // Earlier tool's stray 5m marker is removed so it can't precede the tail.
+    expect(out[0].extras?.cache_control).toBeUndefined();
+    expect(out[1].extras?.cache_control).toEqual({
+      type: 'ephemeral',
+      ttl: '1h',
+    });
+  });
+
+  it('strips stale markers off deferred tools so they do not precede the system/message breakpoint', () => {
+    const staticTool = fakeTool('a-static');
+    const deferred = fakeTool('b-deferred') as {
+      extras?: { cache_control?: { type: string } };
+    };
+    deferred.extras = { cache_control: { type: 'ephemeral' } };
+    const out = partitionAndMarkAnthropicToolCache(
+      [staticTool, deferred] as never,
+      (name) => name === 'b-deferred',
+      '1h'
+    ) as Array<{
+      extras?: { cache_control?: { type: string; ttl?: string } };
+    }>;
+    expect(out[0].extras?.cache_control).toEqual({
+      type: 'ephemeral',
+      ttl: '1h',
+    });
+    expect(out[1].extras?.cache_control).toBeUndefined();
+  });
+
+  it('strips stale markers in the all-deferred case', () => {
+    const deferred = fakeTool('only-deferred') as {
+      extras?: { cache_control?: { type: string } };
+    };
+    deferred.extras = { cache_control: { type: 'ephemeral' } };
+    const out = partitionAndMarkAnthropicToolCache(
+      [deferred] as never,
+      () => true,
+      '1h'
+    ) as Array<{ extras?: { cache_control?: unknown } }>;
+    expect(out[0].extras?.cache_control).toBeUndefined();
+  });
+
+  it('strips a direct cache_control on an earlier native (non-built-in) tool', () => {
+    const nativeWithMarker = {
+      name: 'native_a',
+      input_schema: { type: 'object', properties: {} },
+      cache_control: { type: 'ephemeral' },
+    };
+    const out = partitionAndMarkAnthropicToolCache(
+      [nativeWithMarker, fakeTool('native_b')] as never,
+      () => false,
+      '1h'
+    ) as Array<{
+      cache_control?: unknown;
+      extras?: { cache_control?: { type: string; ttl?: string } };
+    }>;
+    expect(out[0].cache_control).toBeUndefined();
+    expect(out[1].extras?.cache_control).toEqual({
+      type: 'ephemeral',
+      ttl: '1h',
+    });
+  });
+
+  it('upgrades a raw Anthropic tail tool to a direct 1h marker (not extras)', () => {
+    const nativeTail = {
+      name: 'native_tail',
+      input_schema: { type: 'object', properties: {} },
+      cache_control: { type: 'ephemeral' },
+    };
+    const out = partitionAndMarkAnthropicToolCache(
+      [nativeTail] as never,
+      () => false,
+      '1h'
+    ) as Array<{
+      cache_control?: { type: string; ttl?: string };
+      extras?: { cache_control?: unknown };
+    }>;
+    // Raw Anthropic tools carry cache_control directly (extras is not promoted
+    // for them), so the stale marker is upgraded in place to 1h — not moved.
+    expect(out[0].cache_control).toEqual({ type: 'ephemeral', ttl: '1h' });
+    expect(out[0].extras?.cache_control).toBeUndefined();
+  });
+
+  it('reorders deferred tools after a correctly pre-marked static tool', () => {
+    const deferred = fakeTool('z-deferred');
+    const staticTool = fakeTool('a-static') as {
+      extras?: { cache_control?: { type: string; ttl?: string } };
+    };
+    // Already carries the resolved 1h marker, so nothing is mutated...
+    staticTool.extras = { cache_control: { type: 'ephemeral', ttl: '1h' } };
+    const out = partitionAndMarkAnthropicToolCache(
+      [deferred, staticTool] as never, // deferred precedes static in the input
+      (name) => name === 'z-deferred',
+      '1h'
+    ) as Array<{ name?: string }>;
+    // ...but the static (cached) tool must still be hoisted ahead of the
+    // deferred tool so the breakpoint precedes discovered tools.
+    expect(out.map((t) => t.name)).toEqual(['a-static', 'z-deferred']);
+  });
+
+  it('re-stamps a LangChain tool whose marker sits only on the direct block', () => {
+    // A StructuredTool (custom) pre-marked directly — not under extras — does
+    // not reach the payload, so the breakpoint must be re-stamped under extras.
+    const t = fakeTool('a-static') as {
+      cache_control?: unknown;
+      extras?: { cache_control?: { type: string; ttl?: string } };
+    };
+    t.cache_control = { type: 'ephemeral', ttl: '1h' };
+    const out = partitionAndMarkAnthropicToolCache(
+      [t] as never,
+      () => false,
+      '1h'
+    ) as Array<{
+      cache_control?: unknown;
+      extras?: { cache_control?: { type: string; ttl?: string } };
+    }>;
+    expect(out[0].extras?.cache_control).toEqual({
+      type: 'ephemeral',
+      ttl: '1h',
+    });
+    expect(out[0].cache_control).toBeUndefined();
+  });
 });
 
 describe('makeIsDeferred', () => {

package/src/messages/anthropicToolCache.ts

@@ -26,6 +26,10 @@
  */
 
 import type { GraphTools } from '@/types';
+import {
+  buildAnthropicCacheControl,
+  type PromptCacheTtl,
+} from '@/messages/cache';
 
 const ANTHROPIC_BUILT_IN_TOOL_PREFIXES = [
   'text_editor_',
@@ -41,8 +45,6 @@ const ANTHROPIC_BUILT_IN_TOOL_PREFIXES = [
   'mcp_toolset',
 ] as const;
 
-const CACHE_CONTROL = { type: 'ephemeral' as const };
-
 type AnthropicToolCacheCandidate = {
   name?: unknown;
   type?: unknown;
@@ -60,29 +62,91 @@ function isAnthropicBuiltInTool(
   );
 }
 
+/**
+ * Whether a tool already carries a cache breakpoint. Built-ins use a direct
+ * `cache_control`; custom tools normally carry it under `extras`, but a
+ * caller-supplied Anthropic-native tool object can also put it directly on the
+ * block — check both so stale markers are never missed.
+ */
 function hasCacheControl(tool: AnthropicToolCacheCandidate): boolean {
   if (isAnthropicBuiltInTool(tool)) {
     return tool.cache_control != null;
   }
-  return tool.extras?.cache_control != null;
+  return tool.cache_control != null || tool.extras?.cache_control != null;
 }
 
-function markCacheControl(
+/**
+ * Return the `cache_control` from the location that actually reaches the
+ * Anthropic payload for this tool's shape: directly on the block for
+ * provider-shaped tools (built-ins and raw Anthropic tools), or under `extras`
+ * for LangChain custom tools (the adapter promotes only that). Returns
+ * `undefined` when no marker sits in the effective location — a marker in the
+ * wrong place (e.g. a direct marker on a custom tool) does not count.
+ */
+function getEffectiveCacheControl(
+  tool: AnthropicToolCacheCandidate
+): { ttl?: unknown } | undefined {
+  return (
+    isProviderShapedTool(tool) ? tool.cache_control : tool.extras?.cache_control
+  ) as { ttl?: unknown } | undefined;
+}
+
+/**
+ * Return a clone of `tool` with any `cache_control` removed — both the direct
+ * block marker and the `extras` marker — preserving the prototype chain. Used
+ * to clear stray markers off tools that must not anchor a competing breakpoint.
+ */
+function stripCacheControl(
   tool: AnthropicToolCacheCandidate
 ): AnthropicToolCacheCandidate {
   const prototype = Object.getPrototypeOf(tool) ?? Object.prototype;
-  if (isAnthropicBuiltInTool(tool)) {
+  const wrapped = { ...tool };
+  delete wrapped.cache_control;
+  if (wrapped.extras != null) {
+    wrapped.extras = { ...wrapped.extras };
+    delete wrapped.extras.cache_control;
+  }
+  return Object.assign(Object.create(prototype), wrapped);
+}
+
+/**
+ * Whether `tool` is already in the Anthropic provider payload shape — an
+ * Anthropic built-in or a raw Anthropic tool object (has `input_schema`). These
+ * carry `cache_control` directly on the block; the LangChain adapter does NOT
+ * promote `extras.cache_control` for them. LangChain StructuredTools, by
+ * contrast, expose the marker via `extras`.
+ */
+function isProviderShapedTool(tool: AnthropicToolCacheCandidate): boolean {
+  return (
+    isAnthropicBuiltInTool(tool) ||
+    'input_schema' in (tool as Record<string, unknown>)
+  );
+}
+
+function markCacheControl(
+  tool: AnthropicToolCacheCandidate,
+  ttl?: PromptCacheTtl
+): AnthropicToolCacheCandidate {
+  const cacheControl = buildAnthropicCacheControl(ttl);
+  const prototype = Object.getPrototypeOf(tool) ?? Object.prototype;
+  if (isProviderShapedTool(tool)) {
+    // Built-ins and raw Anthropic tool objects carry cache_control directly on
+    // the block; `extras` is not promoted onto the payload for these shapes.
     const wrapped = { ...tool };
     delete wrapped.extras;
     return Object.assign(Object.create(prototype), wrapped, {
-      cache_control: CACHE_CONTROL,
+      cache_control: cacheControl,
     });
   }
 
-  return Object.assign(Object.create(prototype), tool, {
+  // LangChain custom tools: drop any direct marker and expose the breakpoint via
+  // `extras`, which the Anthropic adapter promotes onto the payload.
+  const wrapped = { ...tool };
+  delete wrapped.cache_control;
+  return Object.assign(Object.create(prototype), wrapped, {
     extras: {
       ...(tool.extras ?? {}),
-      cache_control: CACHE_CONTROL,
+      cache_control: cacheControl,
     },
   });
 }
@@ -124,7 +188,8 @@ export function makeIsDeferred(
  */
 export function partitionAndMarkAnthropicToolCache(
   tools: GraphTools | undefined,
-  isDeferred: (toolName: string) => boolean
+  isDeferred: (toolName: string) => boolean,
+  ttl?: PromptCacheTtl
 ): GraphTools | undefined {
   if (tools == null || tools.length === 0) return tools;
 
@@ -143,19 +208,57 @@ export function partitionAndMarkAnthropicToolCache(
     }
   }
 
+  // Anthropic serializes ALL tools before system/messages, so a stray
+  // cache_control on any tool — static or deferred — that survives the resolved
+  // breakpoint would violate the longer-TTL-first ordering. Strip stale markers
+  // off the deferred tools first (they sit after the breakpoint but still before
+  // system/messages, and the all-deferred case has no breakpoint of its own).
+  let mutated = false;
+  for (let i = 0; i < deferredTools.length; i++) {
+    const candidate = deferredTools[i] as AnthropicToolCacheCandidate;
+    if (hasCacheControl(candidate)) {
+      deferredTools[i] = stripCacheControl(candidate);
+      mutated = true;
+    }
+  }
+
   if (staticTools.length === 0) {
-    return tools;
+    return mutated ? ([...deferredTools] as GraphTools) : tools;
+  }
+
+  // Strip any stray cache_control off the earlier static tools so a leftover
+  // 5-minute marker never sits ahead of the resolved breakpoint, then stamp (or
+  // re-stamp) only the last static tool with the resolved TTL.
+  for (let i = 0; i < staticTools.length - 1; i++) {
+    const candidate = staticTools[i] as AnthropicToolCacheCandidate;
+    if (hasCacheControl(candidate)) {
+      staticTools[i] = stripCacheControl(candidate);
+      mutated = true;
+    }
   }
 
   const last = staticTools[
     staticTools.length - 1
   ] as AnthropicToolCacheCandidate;
-  // Already marked? Don't double-clone.
-  if (hasCacheControl(last)) {
-    if (deferredTools.length === 0) return tools;
-    return [...staticTools, ...deferredTools] as GraphTools;
+  const desiredTtl: '1h' | undefined = ttl === '1h' ? '1h' : undefined;
+  // "Already correct" requires a marker IN the effective location (one that
+  // reaches the payload for this tool's shape) carrying the resolved TTL. A
+  // marker in the wrong place — e.g. a direct `cache_control` on a LangChain
+  // custom tool — is ineffective and must be re-stamped.
+  const effective = getEffectiveCacheControl(last);
+  const lastAlreadyCorrect =
+    effective != null &&
+    (effective.ttl === '1h' ? '1h' : undefined) === desiredTtl;
+  if (!lastAlreadyCorrect) {
+    staticTools[staticTools.length - 1] = markCacheControl(last, ttl);
+    mutated = true;
   }
 
-  staticTools[staticTools.length - 1] = markCacheControl(last);
+  // Return the original reference only when nothing changed AND partitioning
+  // moved nothing. When deferred tools exist they must end up after the cache
+  // breakpoint, so the partitioned array is returned even if no marker changed.
+  if (!mutated && deferredTools.length === 0) {
+    return tools;
+  }
   return [...staticTools, ...deferredTools] as GraphTools;
 }

package/src/messages/cache.test.ts

@@ -12,8 +12,14 @@ import {
   stripAnthropicCacheControl,
   stripBedrockCacheControl,
   addBedrockCacheControl,
+  addBedrockTailCacheControl,
   addCacheControl,
+  addTailCacheControl,
   addCacheControlToStablePrefixMessages,
+  buildAnthropicCacheControl,
+  buildBedrockCachePoint,
+  resolvePromptCacheTtl,
+  DEFAULT_PROMPT_CACHE_TTL,
 } from './cache';
 import { _convertMessagesToOpenAIParams } from '@/llm/openai/utils';
 import { toLangChainContent } from './langchain';
@@ -1813,3 +1819,172 @@ describe('OpenRouter prompt caching (reuses addCacheControl)', () => {
     expect('cache_control' in lastContent[0]).toBe(true);
   });
 });
+
+describe('prompt-cache TTL (1h default, 5m legacy)', () => {
+  describe('resolvePromptCacheTtl', () => {
+    it('defaults to the 1h extended cache when unset', () => {
+      expect(resolvePromptCacheTtl(undefined)).toBe('1h');
+      expect(DEFAULT_PROMPT_CACHE_TTL).toBe('1h');
+    });
+
+    it('passes an explicit value through unchanged', () => {
+      expect(resolvePromptCacheTtl('5m')).toBe('5m');
+      expect(resolvePromptCacheTtl('1h')).toBe('1h');
+    });
+  });
+
+  describe('marker builders', () => {
+    it('buildAnthropicCacheControl adds ttl only for 1h', () => {
+      expect(buildAnthropicCacheControl('1h')).toEqual({
+        type: 'ephemeral',
+        ttl: '1h',
+      });
+      // 5m / undefined stay byte-identical to the legacy marker (no ttl)
+      expect(buildAnthropicCacheControl('5m')).toEqual({ type: 'ephemeral' });
+      expect(buildAnthropicCacheControl()).toEqual({ type: 'ephemeral' });
+    });
+
+    it('buildBedrockCachePoint adds ttl only for 1h', () => {
+      expect(buildBedrockCachePoint('1h')).toEqual({
+        type: 'default',
+        ttl: '1h',
+      });
+      expect(buildBedrockCachePoint('5m')).toEqual({ type: 'default' });
+      expect(buildBedrockCachePoint()).toEqual({ type: 'default' });
+    });
+  });
+
+  describe('addTailCacheControl threads ttl', () => {
+    const baseMessages = (): AnthropicMessages => [
+      { role: 'user', content: 'Hello' },
+      { role: 'assistant', content: 'Hi there' },
+    ];
+
+    it('stamps a 1h cache_control on the tail block', () => {
+      const result = addTailCacheControl(baseMessages(), '1h');
+      const tail = result[result.length - 1].content as MessageContentComplex[];
+      expect(tail[tail.length - 1]).toEqual({
+        type: 'text',
+        text: 'Hi there',
+        cache_control: { type: 'ephemeral', ttl: '1h' },
+      });
+    });
+
+    it('omits ttl for 5m and for the unspecified (legacy) default', () => {
+      for (const ttl of ['5m', undefined] as const) {
+        const result = addTailCacheControl(baseMessages(), ttl);
+        const tail = result[result.length - 1]
+          .content as MessageContentComplex[];
+        expect(
+          (tail[tail.length - 1] as Anthropic.TextBlockParam).cache_control
+        ).toEqual({ type: 'ephemeral' });
+      }
+    });
+  });
+
+  describe('addCacheControl threads ttl', () => {
+    it('stamps a 1h cache_control on the latest user messages', () => {
+      const messages: AnthropicMessages = [
+        { role: 'user', content: 'first' },
+        { role: 'assistant', content: 'reply' },
+        { role: 'user', content: 'second' },
+      ];
+      const result = addCacheControl(messages, '1h');
+      const lastUser = result[2].content as MessageContentComplex[];
+      expect((lastUser[0] as Anthropic.TextBlockParam).cache_control).toEqual({
+        type: 'ephemeral',
+        ttl: '1h',
+      });
+    });
+  });
+
+  describe('addCacheControlToStablePrefixMessages threads ttl', () => {
+    it('stamps 1h on every stable-prefix marker', () => {
+      const messages: AnthropicMessages = [
+        { role: 'user', content: 'turn 1' },
+        { role: 'assistant', content: 'reply 1' },
+        { role: 'user', content: 'turn 2' },
+        { role: 'assistant', content: 'reply 2' },
+      ];
+      const result = addCacheControlToStablePrefixMessages(messages, 2, '1h');
+      const marked = result
+        .flatMap((m) =>
+          Array.isArray(m.content) ? (m.content as MessageContentComplex[]) : []
+        )
+        .filter((block) => 'cache_control' in block);
+      expect(marked.length).toBeGreaterThan(0);
+      for (const block of marked) {
+        expect((block as Anthropic.TextBlockParam).cache_control).toEqual({
+          type: 'ephemeral',
+          ttl: '1h',
+        });
+      }
+    });
+  });
+
+  describe('addBedrockTailCacheControl threads ttl', () => {
+    const messages = (): TestMsg[] => [
+      { role: 'user', content: 'Hello' },
+      { role: 'assistant', content: 'Hi' },
+    ];
+
+    it('stamps a 1h cachePoint on the tail message', () => {
+      const result = addBedrockTailCacheControl(messages(), '1h');
+      const last = result[result.length - 1].content as MessageContentComplex[];
+      expect(last[last.length - 1]).toEqual({
+        cachePoint: { type: 'default', ttl: '1h' },
+      });
+    });
+
+    it('omits ttl for 5m and the legacy default', () => {
+      for (const ttl of ['5m', undefined] as const) {
+        const result = addBedrockTailCacheControl(messages(), ttl);
+        const last = result[result.length - 1]
+          .content as MessageContentComplex[];
+        expect(last[last.length - 1]).toEqual({
+          cachePoint: { type: 'default' },
+        });
+      }
+    });
+
+    it('normalizes a stale 5m system cachePoint to the resolved tail ttl', () => {
+      const msgs: TestMsg[] = [
+        {
+          role: 'system',
+          content: [
+            { type: ContentTypes.TEXT, text: 'System' },
+            { cachePoint: { type: 'default' } } as MessageContentComplex,
+          ],
+        },
+        { role: 'user', content: 'Hello' },
+        { role: 'assistant', content: 'Hi' },
+      ];
+      const result = addBedrockTailCacheControl(msgs, '1h');
+      // Stale 5m system checkpoint is upgraded to 1h so it never precedes the
+      // 1h message tail (Bedrock requires longer-TTL checkpoints first).
+      const system = result[0].content as MessageContentComplex[];
+      expect(system[system.length - 1]).toEqual({
+        cachePoint: { type: 'default', ttl: '1h' },
+      });
+      const tail = result[result.length - 1].content as MessageContentComplex[];
+      expect(tail[tail.length - 1]).toEqual({
+        cachePoint: { type: 'default', ttl: '1h' },
+      });
+    });
+  });
+
+  describe('addBedrockCacheControl threads ttl', () => {
+    it('stamps a 1h cachePoint when configured', () => {
+      const messages: TestMsg[] = [
+        { role: 'user', content: 'Hello' },
+        { role: 'assistant', content: 'Hi' },
+      ];
+      const result = addBedrockCacheControl(messages, '1h');
+      // Only one user message present, so the cachePoint anchors on it.
+      const user = result[0].content as MessageContentComplex[];
+      expect(user[user.length - 1]).toEqual({
+        cachePoint: { type: 'default', ttl: '1h' },
+      });
+    });
+  });
+});