@librechat/agents 3.2.35 → 3.2.37

package/src/messages/cache.tail.test.ts

@@ -0,0 +1,340 @@
+import {
+  AIMessage,
+  HumanMessage,
+  SystemMessage,
+  ToolMessage,
+} from '@langchain/core/messages';
+import type {
+  BaseMessage,
+  MessageContentComplex,
+} from '@langchain/core/messages';
+import type Anthropic from '@anthropic-ai/sdk';
+import type { AnthropicMessages } from '@/types/messages';
+import { addTailCacheControl, addBedrockTailCacheControl } from './cache';
+import { toLangChainContent } from './langchain';
+
+type CacheControlBlock = MessageContentComplex & {
+  cache_control?: { type: 'ephemeral'; ttl?: '1h' };
+};
+
+/** Count every block across all messages that carries a cache_control marker. */
+function countCacheMarkers(
+  messages: ReadonlyArray<{ content: unknown }>
+): number {
+  let count = 0;
+  for (const message of messages) {
+    if (!Array.isArray(message.content)) {
+      continue;
+    }
+    for (const block of message.content) {
+      if (block && typeof block === 'object' && 'cache_control' in block) {
+        count++;
+      }
+    }
+  }
+  return count;
+}
+
+function blocksOf(message: { content: unknown }): CacheControlBlock[] {
+  return message.content as CacheControlBlock[];
+}
+
+describe('addTailCacheControl (single tail breakpoint)', () => {
+  test('places exactly one marker on the last message', () => {
+    const messages: AnthropicMessages = [
+      { role: 'user', content: [{ type: 'text', text: 'Hello' }] },
+      { role: 'assistant', content: [{ type: 'text', text: 'Hi there' }] },
+      { role: 'user', content: [{ type: 'text', text: 'How are you?' }] },
+      { role: 'assistant', content: [{ type: 'text', text: 'Doing well' }] },
+      { role: 'user', content: [{ type: 'text', text: 'Great!' }] },
+    ];
+
+    const result = addTailCacheControl(messages);
+
+    expect(countCacheMarkers(result)).toBe(1);
+    expect(
+      (result[4].content[0] as Anthropic.TextBlockParam).cache_control
+    ).toEqual({ type: 'ephemeral' });
+    expect(result[2].content[0]).not.toHaveProperty('cache_control');
+  });
+
+  test('anchors on a trailing tool_result block (tail is a tool turn)', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage('Run the tool'),
+      new AIMessage({
+        content: toLangChainContent([
+          { type: 'text', text: 'Calling it' },
+          { type: 'tool_use', id: 't1', name: 'search', input: {} },
+        ] as MessageContentComplex[]),
+        tool_calls: [{ id: 't1', name: 'search', args: {} }],
+      }),
+      new ToolMessage({
+        tool_call_id: 't1',
+        content: toLangChainContent([
+          {
+            type: 'tool_result',
+            tool_use_id: 't1',
+            content: 'result body',
+          },
+        ] as MessageContentComplex[]),
+      }),
+    ];
+
+    const result = addTailCacheControl(messages);
+
+    expect(countCacheMarkers(result)).toBe(1);
+    expect(blocksOf(result[2])[0].cache_control).toEqual({ type: 'ephemeral' });
+  });
+
+  test('strips ALL stale markers and re-anchors a single one at the tail', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage({
+        content: toLangChainContent([
+          {
+            type: 'text',
+            text: 'old marker',
+            cache_control: { type: 'ephemeral' },
+          },
+        ] as MessageContentComplex[]),
+      }),
+      new HumanMessage({
+        content: toLangChainContent([
+          {
+            type: 'text',
+            text: 'another old marker',
+            cache_control: { type: 'ephemeral' },
+          },
+        ] as MessageContentComplex[]),
+      }),
+      new AIMessage({ content: 'reply' }),
+    ];
+
+    const result = addTailCacheControl(messages);
+
+    expect(countCacheMarkers(result)).toBe(1);
+    expect(blocksOf(result[2])[0].cache_control).toEqual({ type: 'ephemeral' });
+    expect(blocksOf(result[0])[0]).not.toHaveProperty('cache_control');
+    expect(blocksOf(result[1])[0]).not.toHaveProperty('cache_control');
+  });
+
+  test('does not anchor on thinking blocks', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage('Hi'),
+      new AIMessage({
+        content: toLangChainContent([
+          { type: 'text', text: 'thought through it' },
+          { type: 'thinking', thinking: 'secret reasoning' },
+        ] as MessageContentComplex[]),
+      }),
+    ];
+
+    const result = addTailCacheControl(messages);
+
+    expect(countCacheMarkers(result)).toBe(1);
+    expect(blocksOf(result[1])[0].cache_control).toEqual({ type: 'ephemeral' });
+    expect(blocksOf(result[1])[1]).not.toHaveProperty('cache_control');
+  });
+
+  test.each(['reasoning_content', 'reasoning', 'think'])(
+    'does not anchor on a trailing foreign reasoning block (%s)',
+    (reasoningType) => {
+      // Foreign reasoning (Bedrock/Google/LibreChat) is dropped by the
+      // Anthropic converter on assistant turns; anchoring the only breakpoint
+      // there would silently lose tail caching on a cross-provider handoff.
+      const messages: BaseMessage[] = [
+        new HumanMessage('Hi'),
+        new AIMessage({
+          content: toLangChainContent([
+            { type: 'text', text: 'Here is my answer.' },
+            { type: reasoningType, text: 'foreign reasoning' },
+          ] as MessageContentComplex[]),
+        }),
+      ];
+
+      const result = addTailCacheControl(messages);
+
+      expect(countCacheMarkers(result)).toBe(1);
+      // Marker must land on the surviving text block, not the reasoning block.
+      expect(blocksOf(result[1])[0].cache_control).toEqual({
+        type: 'ephemeral',
+      });
+      expect(blocksOf(result[1])[1]).not.toHaveProperty('cache_control');
+    }
+  );
+
+  test('skips synthetic meta tail and anchors on the previous real message', () => {
+    const realTail = new AIMessage({ content: 'real answer' });
+    const metaTail = new HumanMessage({ content: 'reinjected skill body' });
+    (
+      metaTail as unknown as { additional_kwargs: Record<string, unknown> }
+    ).additional_kwargs = { isMeta: true };
+
+    const result = addTailCacheControl([
+      new HumanMessage({ content: 'question' }),
+      realTail,
+      metaTail,
+    ]);
+
+    expect(countCacheMarkers(result)).toBe(1);
+    expect(blocksOf(result[1])[0].cache_control).toEqual({ type: 'ephemeral' });
+  });
+
+  test('handles string content on the tail', () => {
+    const messages: AnthropicMessages = [
+      { role: 'user', content: 'Hello' },
+      { role: 'assistant', content: 'Final' },
+    ];
+
+    const result = addTailCacheControl(messages);
+
+    expect(result[0].content).toBe('Hello');
+    expect(result[1].content[0]).toEqual({
+      type: 'text',
+      text: 'Final',
+      cache_control: { type: 'ephemeral' },
+    });
+  });
+
+  test('does not mutate the original messages', () => {
+    const original: AnthropicMessages = [
+      { role: 'user', content: [{ type: 'text', text: 'Hello' }] },
+      { role: 'assistant', content: [{ type: 'text', text: 'World' }] },
+    ];
+
+    addTailCacheControl(original);
+
+    expect(original[1].content[0]).not.toHaveProperty('cache_control');
+  });
+
+  test('returns input unchanged for empty array', () => {
+    const messages: AnthropicMessages = [];
+    expect(addTailCacheControl(messages)).toEqual([]);
+  });
+});
+
+/** Count every Bedrock cachePoint block across all messages. */
+function countCachePoints(
+  messages: ReadonlyArray<{ content: unknown }>
+): number {
+  let count = 0;
+  for (const message of messages) {
+    if (!Array.isArray(message.content)) {
+      continue;
+    }
+    for (const block of message.content) {
+      if (block && typeof block === 'object' && 'cachePoint' in block) {
+        count++;
+      }
+    }
+  }
+  return count;
+}
+
+describe('addBedrockTailCacheControl (single tail cachePoint)', () => {
+  test('inserts exactly one cachePoint after the last text block of the tail', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage('First question'),
+      new AIMessage('First answer'),
+      new HumanMessage('Second question'),
+    ];
+
+    const result = addBedrockTailCacheControl(messages);
+
+    expect(countCachePoints(result)).toBe(1);
+    const tail = blocksOf(result[2]);
+    expect(tail[tail.length - 1]).toEqual({ cachePoint: { type: 'default' } });
+  });
+
+  test('strips stale cachePoints and re-anchors a single one at the tail', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage({
+        content: toLangChainContent([
+          { type: 'text', text: 'old' },
+          { cachePoint: { type: 'default' } },
+        ] as MessageContentComplex[]),
+      }),
+      new AIMessage('reply'),
+      new HumanMessage('newest'),
+    ];
+
+    const result = addBedrockTailCacheControl(messages);
+
+    expect(countCachePoints(result)).toBe(1);
+    const tail = blocksOf(result[2]);
+    expect(tail[tail.length - 1]).toEqual({ cachePoint: { type: 'default' } });
+    expect(blocksOf(result[0]).some((b) => 'cachePoint' in b)).toBe(false);
+  });
+
+  test('strips Anthropic cache_control from a system message but never anchors it', () => {
+    const messages: BaseMessage[] = [
+      new SystemMessage({
+        content: toLangChainContent([
+          {
+            type: 'text',
+            text: 'system rules',
+            cache_control: { type: 'ephemeral' },
+          },
+        ] as MessageContentComplex[]),
+      }),
+      new HumanMessage('hi'),
+    ];
+
+    const result = addBedrockTailCacheControl(messages);
+
+    expect(blocksOf(result[0])[0]).not.toHaveProperty('cache_control');
+    expect(countCachePoints(result)).toBe(1);
+    expect(blocksOf(result[1])[1]).toEqual({ cachePoint: { type: 'default' } });
+  });
+
+  test('skips synthetic meta tail and anchors on the previous real message', () => {
+    const metaTail = new HumanMessage({ content: 'reinjected skill body' });
+    (
+      metaTail as unknown as { additional_kwargs: Record<string, unknown> }
+    ).additional_kwargs = { source: 'skill' };
+
+    const result = addBedrockTailCacheControl([
+      new HumanMessage('question'),
+      new AIMessage('real answer'),
+      metaTail,
+    ]);
+
+    expect(countCachePoints(result)).toBe(1);
+    const realTail = blocksOf(result[1]);
+    expect(realTail[realTail.length - 1]).toEqual({
+      cachePoint: { type: 'default' },
+    });
+  });
+
+  test('handles string content on the tail', () => {
+    const result = addBedrockTailCacheControl([
+      new HumanMessage('Hello'),
+      new AIMessage('Final'),
+    ]);
+
+    expect(countCachePoints(result)).toBe(1);
+    expect(blocksOf(result[1])).toEqual([
+      { type: 'text', text: 'Final' },
+      { cachePoint: { type: 'default' } },
+    ]);
+  });
+
+  test('anchors on a trailing string tool result (agent-loop tail)', () => {
+    const result = addBedrockTailCacheControl([
+      new HumanMessage('Run the tool'),
+      new AIMessage({
+        content: 'Calling it',
+        tool_calls: [{ id: 't1', name: 'search', args: {} }],
+      }),
+      new ToolMessage({ tool_call_id: 't1', content: 'result body' }),
+    ]);
+
+    // The single cachePoint must land on the trailing tool result so the
+    // tool output is part of the cached prefix; the converter later hoists it
+    // out of toolResult.content (see toolResultCachePoint.test.ts).
+    expect(countCachePoints(result)).toBe(1);
+    expect(blocksOf(result[2])).toEqual([
+      { type: 'text', text: 'result body' },
+      { cachePoint: { type: 'default' } },
+    ]);
+  });
+});

package/src/messages/cache.ts

@@ -41,7 +41,7 @@ function deepCloneContent<T extends string | MessageContentComplex[]>(
  * in downstream code (e.g., ensureThinkingBlockInMessages).
  * For plain objects (AnthropicMessage), uses object spread.
  */
-function cloneMessage<T extends MessageWithContent>(
+export function cloneMessage<T extends MessageWithContent>(
   message: T,
   content: string | MessageContentComplex[]
 ): T {
@@ -254,6 +254,152 @@ function isCachePoint(block: MessageContentComplex): boolean {
   return 'cachePoint' in block && !('type' in block);
 }
 
+/**
+ * Block types that must never anchor the tail cache breakpoint, because the
+ * marker would not survive to the model call:
+ * - `thinking` / `redacted_thinking`: native Anthropic reasoning — the API
+ *   rejects `cache_control` on these blocks.
+ * - `reasoning_content` / `reasoning` / `think`: foreign reasoning (Bedrock,
+ *   Google, LibreChat) that `_convertMessagesToAnthropicPayload` DROPS on
+ *   assistant turns during a cross-provider handoff.
+ * - `input_json_delta`: persisted partial tool-input deltas, also DROPPED by
+ *   `_convertMessagesToAnthropicPayload` (the assembled input is restored onto
+ *   the tool_use block).
+ * Anchoring the only breakpoint on a block that is about to disappear silently
+ * loses tail caching, so all of these are excluded.
+ */
+const NON_ANCHORABLE_BLOCK_TYPES = new Set([
+  'thinking',
+  'redacted_thinking',
+  'reasoning_content',
+  'reasoning',
+  'think',
+  'input_json_delta',
+]);
+
+/**
+ * A block can anchor the tail cache breakpoint when it is a real content block
+ * that the Anthropic API accepts `cache_control` on and that survives provider
+ * conversion. Reasoning / dropped-delta blocks are excluded (see
+ * {@link NON_ANCHORABLE_BLOCK_TYPES}), and empty text blocks are not cacheable,
+ * so both are skipped.
+ */
+function isTailCacheableBlock(block: MessageContentComplex): boolean {
+  if (isCachePoint(block)) {
+    return false;
+  }
+  const type = (block as { type?: string }).type;
+  if (type == null || NON_ANCHORABLE_BLOCK_TYPES.has(type)) {
+    return false;
+  }
+  if (type === 'text') {
+    const text = (block as { text?: string }).text;
+    return text != null && text.trim() !== '';
+  }
+  return true;
+}
+
+/**
+ * Anthropic API: single tail cache breakpoint (default strategy).
+ *
+ * Places exactly ONE `cache_control` marker on the last cacheable block of the
+ * final non-synthetic message, mirroring the Claude Code strategy
+ * (`markerIndex = messages.length - 1`). Because the marker always rides the
+ * true tail, the entire conversation prefix is written once and read back on
+ * the next turn as the history grows append-only — instead of the rolling
+ * "last two user messages" markers, which leave freshly appended tool/assistant
+ * turns outside the cached prefix and re-write large spans every step.
+ *
+ * Stale markers (Anthropic `cache_control` and Bedrock cache points) are
+ * stripped from every message in a single backward pass so exactly one marker
+ * survives. Synthetic skill/meta messages are skipped as anchors (their volatile
+ * content must not pin the cache) but still have stale markers removed.
+ *
+ * Returns a new array; only messages that require modification are cloned.
+ */
+export function addTailCacheControl<T extends AnthropicMessage | BaseMessage>(
+  messages: T[]
+): T[] {
+  if (!Array.isArray(messages) || messages.length === 0) {
+    return messages;
+  }
+
+  const updatedMessages: T[] = [...messages];
+  let markerPlaced = false;
+
+  for (let i = updatedMessages.length - 1; i >= 0; i--) {
+    const originalMessage = updatedMessages[i];
+    const content = originalMessage.content;
+    const hasArrayContent = Array.isArray(content);
+    const canPlaceMarker =
+      !markerPlaced && !isSyntheticMetaMessage(originalMessage);
+
+    // Earlier string-content messages carry no markers to strip.
+    if (!canPlaceMarker && !hasArrayContent) {
+      continue;
+    }
+
+    let workingContent: MessageContentComplex[];
+    let modified = false;
+
+    if (hasArrayContent) {
+      const src = content as MessageContentComplex[];
+      workingContent = [];
+      let tailIndex = -1;
+      for (let j = 0; j < src.length; j++) {
+        const block = src[j];
+        if (isCachePoint(block)) {
+          modified = true;
+          continue;
+        }
+        const cloned = { ...block };
+        if ('cache_control' in cloned) {
+          delete (cloned as Record<string, unknown>).cache_control;
+          modified = true;
+        }
+        if (
+          canPlaceMarker &&
+          isTailCacheableBlock(cloned as MessageContentComplex)
+        ) {
+          tailIndex = workingContent.length;
+        }
+        workingContent.push(cloned as MessageContentComplex);
+      }
+
+      if (canPlaceMarker && tailIndex >= 0) {
+        (workingContent[tailIndex] as Anthropic.TextBlockParam).cache_control =
+          {
+            type: 'ephemeral',
+          };
+        markerPlaced = true;
+        modified = true;
+      }
+
+      if (!modified) {
+        continue;
+      }
+    } else if (
+      typeof content === 'string' &&
+      canPlaceMarker &&
+      content.trim() !== ''
+    ) {
+      workingContent = [
+        { type: 'text', text: content, cache_control: { type: 'ephemeral' } },
+      ] as unknown as MessageContentComplex[];
+      markerPlaced = true;
+    } else {
+      continue;
+    }
+
+    updatedMessages[i] = cloneMessage(
+      originalMessage as MessageWithContent,
+      workingContent
+    ) as T;
+  }
+
+  return updatedMessages;
+}
+
 function getMessageRole(message: MessageWithContent): string | undefined {
   if (message instanceof BaseMessage) {
     return message.getType();
@@ -625,3 +771,123 @@ export function addBedrockCacheControl<
 
   return updatedMessages;
 }
+
+/**
+ * Bedrock Converse API: single tail cache breakpoint (default strategy).
+ *
+ * The Bedrock counterpart of {@link addTailCacheControl}. Strips ALL existing
+ * cache control (Bedrock cache points and Anthropic `cache_control`) from every
+ * message, then inserts exactly ONE `{ cachePoint: { type: 'default' } }` block
+ * immediately after the last non-empty text block of the most recent
+ * non-synthetic, non-system message. Anchoring on the rolling tail keeps the
+ * cached prefix append-only as the conversation grows, instead of re-writing
+ * large spans every turn with the legacy "last two user messages" cache points.
+ *
+ * System messages are sanitized (Anthropic `cache_control` stripped) but never
+ * anchored. Synthetic skill/meta messages are skipped as anchors so their
+ * volatile content cannot pin the cache.
+ *
+ * Returns a new array - only clones messages that require modification.
+ */
+export function addBedrockTailCacheControl<
+  T extends MessageWithContent & { getType?: () => string; role?: string },
+>(messages: T[]): T[] {
+  if (!Array.isArray(messages) || messages.length === 0) {
+    return messages;
+  }
+
+  const updatedMessages: T[] = [...messages];
+  let cachePointPlaced = false;
+
+  for (let i = updatedMessages.length - 1; i >= 0; i--) {
+    const originalMessage = updatedMessages[i];
+    const messageType =
+      'getType' in originalMessage &&
+      typeof originalMessage.getType === 'function'
+        ? originalMessage.getType()
+        : undefined;
+    const messageRole =
+      'role' in originalMessage && typeof originalMessage.role === 'string'
+        ? originalMessage.role
+        : undefined;
+
+    const isSystemMessage =
+      messageType === 'system' || messageRole === 'system';
+    if (isSystemMessage) {
+      updatedMessages[i] = sanitizeBedrockSystemMessage(originalMessage);
+      continue;
+    }
+
+    const content = originalMessage.content;
+    const hasSerializationProps =
+      'lc_kwargs' in originalMessage ||
+      'lc_serializable' in originalMessage ||
+      'lc_namespace' in originalMessage;
+    const hasArrayContent = Array.isArray(content);
+    const isEmptyString = typeof content === 'string' && content === '';
+    const canPlaceCachePoint =
+      !cachePointPlaced &&
+      !isEmptyString &&
+      !isSyntheticMetaMessage(originalMessage) &&
+      (typeof content === 'string' || hasArrayContent);
+
+    if (!canPlaceCachePoint && !hasArrayContent && !hasSerializationProps) {
+      continue;
+    }
+
+    let workingContent: string | MessageContentComplex[];
+    let modified = hasSerializationProps;
+
+    if (hasArrayContent) {
+      const src = content as MessageContentComplex[];
+      workingContent = [];
+      let lastNonEmptyTextIndex = -1;
+      for (let j = 0; j < src.length; j++) {
+        const block = src[j];
+        if (isCachePoint(block)) {
+          modified = true;
+          continue;
+        }
+        const cloned = { ...block };
+        if ('cache_control' in cloned) {
+          delete (cloned as Record<string, unknown>).cache_control;
+          modified = true;
+        }
+        const type = (cloned as { type?: string }).type;
+        if (type === ContentTypes.TEXT || type === 'text') {
+          const text = (cloned as { text?: string }).text;
+          if (text != null && text.trim() !== '') {
+            lastNonEmptyTextIndex = workingContent.length;
+          }
+        }
+        workingContent.push(cloned as MessageContentComplex);
+      }
+
+      if (!modified && !canPlaceCachePoint) {
+        continue;
+      }
+
+      if (canPlaceCachePoint && lastNonEmptyTextIndex >= 0) {
+        workingContent.splice(lastNonEmptyTextIndex + 1, 0, {
+          cachePoint: { type: 'default' },
+        } as MessageContentComplex);
+        cachePointPlaced = true;
+        modified = true;
+      }
+    } else if (typeof content === 'string' && canPlaceCachePoint) {
+      workingContent = [
+        { type: ContentTypes.TEXT, text: content },
+        { cachePoint: { type: 'default' } } as MessageContentComplex,
+      ];
+      cachePointPlaced = true;
+    } else if (typeof content === 'string' && hasSerializationProps) {
+      workingContent = content;
+    } else {
+      continue;
+    }
+
+    updatedMessages[i] = cloneMessage(originalMessage, workingContent);
+  }
+
+  return updatedMessages;
+}

package/src/messages/index.ts

@@ -1,6 +1,7 @@
 export * from './core';
 export * from './ids';
 export * from './prune';
+export * from './budget';
 export * from './format';
 export * from './cache';
 export * from './anthropicToolCache';