@librechat/agents 3.2.36 → 3.2.38

package/src/messages/cache.ts

@@ -254,6 +254,152 @@ function isCachePoint(block: MessageContentComplex): boolean {
   return 'cachePoint' in block && !('type' in block);
 }
 
+/**
+ * Block types that must never anchor the tail cache breakpoint, because the
+ * marker would not survive to the model call:
+ * - `thinking` / `redacted_thinking`: native Anthropic reasoning — the API
+ *   rejects `cache_control` on these blocks.
+ * - `reasoning_content` / `reasoning` / `think`: foreign reasoning (Bedrock,
+ *   Google, LibreChat) that `_convertMessagesToAnthropicPayload` DROPS on
+ *   assistant turns during a cross-provider handoff.
+ * - `input_json_delta`: persisted partial tool-input deltas, also DROPPED by
+ *   `_convertMessagesToAnthropicPayload` (the assembled input is restored onto
+ *   the tool_use block).
+ * Anchoring the only breakpoint on a block that is about to disappear silently
+ * loses tail caching, so all of these are excluded.
+ */
+const NON_ANCHORABLE_BLOCK_TYPES = new Set([
+  'thinking',
+  'redacted_thinking',
+  'reasoning_content',
+  'reasoning',
+  'think',
+  'input_json_delta',
+]);
+
+/**
+ * A block can anchor the tail cache breakpoint when it is a real content block
+ * that the Anthropic API accepts `cache_control` on and that survives provider
+ * conversion. Reasoning / dropped-delta blocks are excluded (see
+ * {@link NON_ANCHORABLE_BLOCK_TYPES}), and empty text blocks are not cacheable,
+ * so both are skipped.
+ */
+function isTailCacheableBlock(block: MessageContentComplex): boolean {
+  if (isCachePoint(block)) {
+    return false;
+  }
+  const type = (block as { type?: string }).type;
+  if (type == null || NON_ANCHORABLE_BLOCK_TYPES.has(type)) {
+    return false;
+  }
+  if (type === 'text') {
+    const text = (block as { text?: string }).text;
+    return text != null && text.trim() !== '';
+  }
+  return true;
+}
+
+/**
+ * Anthropic API: single tail cache breakpoint (default strategy).
+ *
+ * Places exactly ONE `cache_control` marker on the last cacheable block of the
+ * final non-synthetic message, mirroring the Claude Code strategy
+ * (`markerIndex = messages.length - 1`). Because the marker always rides the
+ * true tail, the entire conversation prefix is written once and read back on
+ * the next turn as the history grows append-only — instead of the rolling
+ * "last two user messages" markers, which leave freshly appended tool/assistant
+ * turns outside the cached prefix and re-write large spans every step.
+ *
+ * Stale markers (Anthropic `cache_control` and Bedrock cache points) are
+ * stripped from every message in a single backward pass so exactly one marker
+ * survives. Synthetic skill/meta messages are skipped as anchors (their volatile
+ * content must not pin the cache) but still have stale markers removed.
+ *
+ * Returns a new array; only messages that require modification are cloned.
+ */
+export function addTailCacheControl<T extends AnthropicMessage | BaseMessage>(
+  messages: T[]
+): T[] {
+  if (!Array.isArray(messages) || messages.length === 0) {
+    return messages;
+  }
+
+  const updatedMessages: T[] = [...messages];
+  let markerPlaced = false;
+
+  for (let i = updatedMessages.length - 1; i >= 0; i--) {
+    const originalMessage = updatedMessages[i];
+    const content = originalMessage.content;
+    const hasArrayContent = Array.isArray(content);
+    const canPlaceMarker =
+      !markerPlaced && !isSyntheticMetaMessage(originalMessage);
+
+    // Earlier string-content messages carry no markers to strip.
+    if (!canPlaceMarker && !hasArrayContent) {
+      continue;
+    }
+
+    let workingContent: MessageContentComplex[];
+    let modified = false;
+
+    if (hasArrayContent) {
+      const src = content as MessageContentComplex[];
+      workingContent = [];
+      let tailIndex = -1;
+      for (let j = 0; j < src.length; j++) {
+        const block = src[j];
+        if (isCachePoint(block)) {
+          modified = true;
+          continue;
+        }
+        const cloned = { ...block };
+        if ('cache_control' in cloned) {
+          delete (cloned as Record<string, unknown>).cache_control;
+          modified = true;
+        }
+        if (
+          canPlaceMarker &&
+          isTailCacheableBlock(cloned as MessageContentComplex)
+        ) {
+          tailIndex = workingContent.length;
+        }
+        workingContent.push(cloned as MessageContentComplex);
+      }
+
+      if (canPlaceMarker && tailIndex >= 0) {
+        (workingContent[tailIndex] as Anthropic.TextBlockParam).cache_control =
+          {
+            type: 'ephemeral',
+          };
+        markerPlaced = true;
+        modified = true;
+      }
+
+      if (!modified) {
+        continue;
+      }
+    } else if (
+      typeof content === 'string' &&
+      canPlaceMarker &&
+      content.trim() !== ''
+    ) {
+      workingContent = [
+        { type: 'text', text: content, cache_control: { type: 'ephemeral' } },
+      ] as unknown as MessageContentComplex[];
+      markerPlaced = true;
+    } else {
+      continue;
+    }
+
+    updatedMessages[i] = cloneMessage(
+      originalMessage as MessageWithContent,
+      workingContent
+    ) as T;
+  }
+
+  return updatedMessages;
+}
+
 function getMessageRole(message: MessageWithContent): string | undefined {
   if (message instanceof BaseMessage) {
     return message.getType();
@@ -625,3 +771,123 @@ export function addBedrockCacheControl<
 
   return updatedMessages;
 }
+
+/**
+ * Bedrock Converse API: single tail cache breakpoint (default strategy).
+ *
+ * The Bedrock counterpart of {@link addTailCacheControl}. Strips ALL existing
+ * cache control (Bedrock cache points and Anthropic `cache_control`) from every
+ * message, then inserts exactly ONE `{ cachePoint: { type: 'default' } }` block
+ * immediately after the last non-empty text block of the most recent
+ * non-synthetic, non-system message. Anchoring on the rolling tail keeps the
+ * cached prefix append-only as the conversation grows, instead of re-writing
+ * large spans every turn with the legacy "last two user messages" cache points.
+ *
+ * System messages are sanitized (Anthropic `cache_control` stripped) but never
+ * anchored. Synthetic skill/meta messages are skipped as anchors so their
+ * volatile content cannot pin the cache.
+ *
+ * Returns a new array - only clones messages that require modification.
+ */
+export function addBedrockTailCacheControl<
+  T extends MessageWithContent & { getType?: () => string; role?: string },
+>(messages: T[]): T[] {
+  if (!Array.isArray(messages) || messages.length === 0) {
+    return messages;
+  }
+
+  const updatedMessages: T[] = [...messages];
+  let cachePointPlaced = false;
+
+  for (let i = updatedMessages.length - 1; i >= 0; i--) {
+    const originalMessage = updatedMessages[i];
+    const messageType =
+      'getType' in originalMessage &&
+      typeof originalMessage.getType === 'function'
+        ? originalMessage.getType()
+        : undefined;
+    const messageRole =
+      'role' in originalMessage && typeof originalMessage.role === 'string'
+        ? originalMessage.role
+        : undefined;
+
+    const isSystemMessage =
+      messageType === 'system' || messageRole === 'system';
+    if (isSystemMessage) {
+      updatedMessages[i] = sanitizeBedrockSystemMessage(originalMessage);
+      continue;
+    }
+
+    const content = originalMessage.content;
+    const hasSerializationProps =
+      'lc_kwargs' in originalMessage ||
+      'lc_serializable' in originalMessage ||
+      'lc_namespace' in originalMessage;
+    const hasArrayContent = Array.isArray(content);
+    const isEmptyString = typeof content === 'string' && content === '';
+    const canPlaceCachePoint =
+      !cachePointPlaced &&
+      !isEmptyString &&
+      !isSyntheticMetaMessage(originalMessage) &&
+      (typeof content === 'string' || hasArrayContent);
+
+    if (!canPlaceCachePoint && !hasArrayContent && !hasSerializationProps) {
+      continue;
+    }
+
+    let workingContent: string | MessageContentComplex[];
+    let modified = hasSerializationProps;
+
+    if (hasArrayContent) {
+      const src = content as MessageContentComplex[];
+      workingContent = [];
+      let lastNonEmptyTextIndex = -1;
+      for (let j = 0; j < src.length; j++) {
+        const block = src[j];
+        if (isCachePoint(block)) {
+          modified = true;
+          continue;
+        }
+        const cloned = { ...block };
+        if ('cache_control' in cloned) {
+          delete (cloned as Record<string, unknown>).cache_control;
+          modified = true;
+        }
+        const type = (cloned as { type?: string }).type;
+        if (type === ContentTypes.TEXT || type === 'text') {
+          const text = (cloned as { text?: string }).text;
+          if (text != null && text.trim() !== '') {
+            lastNonEmptyTextIndex = workingContent.length;
+          }
+        }
+        workingContent.push(cloned as MessageContentComplex);
+      }
+
+      if (!modified && !canPlaceCachePoint) {
+        continue;
+      }
+
+      if (canPlaceCachePoint && lastNonEmptyTextIndex >= 0) {
+        workingContent.splice(lastNonEmptyTextIndex + 1, 0, {
+          cachePoint: { type: 'default' },
+        } as MessageContentComplex);
+        cachePointPlaced = true;
+        modified = true;
+      }
+    } else if (typeof content === 'string' && canPlaceCachePoint) {
+      workingContent = [
+        { type: ContentTypes.TEXT, text: content },
+        { cachePoint: { type: 'default' } } as MessageContentComplex,
+      ];
+      cachePointPlaced = true;
+    } else if (typeof content === 'string' && hasSerializationProps) {
+      workingContent = content;
+    } else {
+      continue;
+    }
+
+    updatedMessages[i] = cloneMessage(originalMessage, workingContent);
+  }
+
+  return updatedMessages;
+}

package/src/messages/tailCacheConversion.test.ts

@@ -0,0 +1,161 @@
+import {
+  HumanMessage,
+  AIMessage,
+  ToolMessage,
+  type BaseMessage,
+  type MessageContentComplex,
+} from '@langchain/core/messages';
+import { _convertMessagesToAnthropicPayload } from '@/llm/anthropic/utils/message_inputs';
+import { ensureThinkingBlockInMessages } from './format';
+import { toLangChainContent } from './langchain';
+import { addTailCacheControl } from './cache';
+import { Providers } from '@/common';
+
+/**
+ * Regression coverage for the single tail prompt-cache breakpoint surviving all
+ * the way into the final Anthropic payload — i.e. the marker must land on a
+ * block that actually ships, not one that downstream conversion / folding
+ * removes. Two ways the breakpoint was silently lost:
+ *
+ *  - Foreign reasoning tail: addTailCacheControl anchored on a
+ *    `reasoning_content`/`reasoning`/`think` block, which the Anthropic
+ *    converter drops on assistant turns (cross-provider handoff).
+ *  - Thinking-fold ordering: marking before ensureThinkingBlockInMessages let
+ *    the fold rewrite the anchored AI→Tool tail into a `[Previous agent
+ *    context]` HumanMessage that copies text but not cache_control.
+ */
+
+type PayloadMessage = { content: unknown };
+
+function hasCacheControl(block: unknown): boolean {
+  return (
+    typeof block === 'object' && block !== null && 'cache_control' in block
+  );
+}
+
+/** Does any block (top-level or nested in tool_result) carry cache_control? */
+function breakpointSurvives(messages: PayloadMessage[]): boolean {
+  for (const m of messages) {
+    if (!Array.isArray(m.content)) {
+      continue;
+    }
+    for (const block of m.content as unknown[]) {
+      if (hasCacheControl(block)) {
+        return true;
+      }
+      const inner = (block as { content?: unknown }).content;
+      if (Array.isArray(inner) && inner.some(hasCacheControl)) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+describe('tail breakpoint survives Anthropic conversion', () => {
+  test('foreign reasoning tail keeps a usable breakpoint (anchored on text)', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage('hello'),
+      new AIMessage({
+        content: toLangChainContent([
+          { type: 'text', text: 'Here is my answer.' },
+          { type: 'reasoning_content', reasoningText: { text: 'r' } },
+        ] as MessageContentComplex[]),
+      }),
+    ];
+
+    const payload = _convertMessagesToAnthropicPayload(
+      addTailCacheControl(messages)
+    );
+
+    expect(breakpointSurvives(payload.messages as PayloadMessage[])).toBe(true);
+  });
+
+  test('string tool-result tail keeps a usable breakpoint on the tool_result block', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage('run it'),
+      new AIMessage({
+        content: 'calling',
+        tool_calls: [{ id: 't1', name: 'search', args: {} }],
+      }),
+      new ToolMessage({ tool_call_id: 't1', content: 'result body' }),
+    ];
+
+    const payload = _convertMessagesToAnthropicPayload(
+      addTailCacheControl(messages)
+    );
+
+    expect(breakpointSurvives(payload.messages as PayloadMessage[])).toBe(true);
+
+    // The marker must sit on the top-level tool_result block (the documented
+    // cacheable position), NOT nested inside tool_result.content.
+    const toolResult = (payload.messages as PayloadMessage[])
+      .flatMap((m) => (Array.isArray(m.content) ? m.content : []))
+      .find(
+        (b): b is Record<string, unknown> =>
+          b != null &&
+          typeof b === 'object' &&
+          'type' in b &&
+          (b as { type?: string }).type === 'tool_result'
+      ) as { cache_control?: unknown; content?: unknown } | undefined;
+    expect(toolResult?.cache_control).toEqual({ type: 'ephemeral' });
+    const inner = toolResult?.content;
+    if (Array.isArray(inner)) {
+      expect(
+        inner.some(
+          (b) => b != null && typeof b === 'object' && 'cache_control' in b
+        )
+      ).toBe(false);
+    }
+  });
+
+  test('marking AFTER the thinking fold preserves the breakpoint (Graph order)', () => {
+    // A historical non-thinking AI→Tool chain at the tail (no trailing human).
+    const messages: BaseMessage[] = [
+      new HumanMessage('do the thing'),
+      new AIMessage({
+        content: '',
+        tool_calls: [{ id: 't1', name: 'search', args: { q: 'x' } }],
+      }),
+      new ToolMessage({ tool_call_id: 't1', content: 'tool output text' }),
+    ];
+
+    // Graph applies the fold first, THEN the tail marker.
+    const folded = ensureThinkingBlockInMessages(
+      messages,
+      Providers.ANTHROPIC,
+      undefined,
+      messages.length
+    );
+    const payload = _convertMessagesToAnthropicPayload(
+      addTailCacheControl(folded)
+    );
+
+    expect(breakpointSurvives(payload.messages as PayloadMessage[])).toBe(true);
+  });
+
+  test('marking BEFORE the fold loses the breakpoint (guards the ordering)', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage('do the thing'),
+      new AIMessage({
+        content: '',
+        tool_calls: [{ id: 't1', name: 'search', args: { q: 'x' } }],
+      }),
+      new ToolMessage({ tool_call_id: 't1', content: 'tool output text' }),
+    ];
+
+    // The buggy order: mark first, then fold drops the marker.
+    const marked = addTailCacheControl(messages);
+    const folded = ensureThinkingBlockInMessages(
+      marked,
+      Providers.ANTHROPIC,
+      undefined,
+      messages.length
+    );
+    const payload = _convertMessagesToAnthropicPayload(folded);
+
+    expect(breakpointSurvives(payload.messages as PayloadMessage[])).toBe(
+      false
+    );
+  });
+});