@librechat/agents 3.2.36 → 3.2.37

package/src/llm/bedrock/utils/message_inputs.test.ts

@@ -0,0 +1,129 @@
+import { AIMessage, HumanMessage } from '@langchain/core/messages';
+import type { BaseMessage } from '@langchain/core/messages';
+import { convertToConverseMessages } from './message_inputs';
+
+/**
+ * Native-Bedrock reasoning serialization. A `reasoning_content` block whose
+ * `reasoningText.text` is null/empty (e.g. a signature-only block that never
+ * merged with its text) is invalid for Bedrock Converse — it rejects with
+ * `...reasoningContent.reasoningText.text ... Member must not be null`. Such a
+ * block must be dropped on replay rather than sent; a block carrying real text
+ * is still converted.
+ */
+type ConverseResult = ReturnType<typeof convertToConverseMessages>;
+
+/** Minimal view of a converted Bedrock Converse content block the assertions read. */
+interface ConverseBlock {
+  text?: string;
+  reasoningContent?: { reasoningText?: { text?: string; signature?: string } };
+  toolUse?: {
+    toolUseId?: string;
+    name?: string;
+    input?: Record<string, string>;
+  };
+}
+
+const assistantContent = (result: ConverseResult): ConverseBlock[] => {
+  const msg = result.converseMessages.find((m) => m.role === 'assistant');
+  return (msg?.content ?? []) as ConverseBlock[];
+};
+
+describe('convertToConverseMessages — native Bedrock reasoning serialization', () => {
+  it('drops a signature-only reasoning block, keeping text and tool calls', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage('what data do you have?'),
+      new AIMessage({
+        content: [
+          {
+            type: 'reasoning_content',
+            reasoningText: { signature: 'sig-abc' },
+          },
+          { type: 'text', text: 'Let me check your databases.' },
+        ],
+        tool_calls: [
+          {
+            id: 'tooluse_list',
+            name: 'list_databases',
+            args: {},
+            type: 'tool_call',
+          },
+        ],
+      }),
+    ];
+
+    expect(() => convertToConverseMessages(messages)).not.toThrow();
+    const content = assistantContent(convertToConverseMessages(messages));
+
+    expect(content.find((b) => b.reasoningContent != null)).toBeUndefined();
+    expect(JSON.stringify(content)).not.toContain('sig-abc');
+    expect(content.some((b) => b.text === 'Let me check your databases.')).toBe(
+      true
+    );
+    const toolUse = content.find((b) => b.toolUse != null);
+    expect(toolUse?.toolUse).toMatchObject({
+      toolUseId: 'tooluse_list',
+      name: 'list_databases',
+    });
+  });
+
+  it('drops a reasoning block whose text is empty', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage('hi'),
+      new AIMessage({
+        content: [
+          {
+            type: 'reasoning_content',
+            reasoningText: { text: '', signature: 'sig' },
+          },
+          { type: 'text', text: 'answer' },
+        ],
+      }),
+    ];
+
+    const content = assistantContent(convertToConverseMessages(messages));
+    expect(content.find((b) => b.reasoningContent != null)).toBeUndefined();
+    expect(content.some((b) => b.text === 'answer')).toBe(true);
+  });
+
+  it('emits a placeholder (not empty content) when the only block is a signature-only reasoning block', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage('hi'),
+      new AIMessage({
+        content: [
+          { type: 'reasoning_content', reasoningText: { signature: 'sig' } },
+        ],
+      }),
+    ];
+
+    expect(() => convertToConverseMessages(messages)).not.toThrow();
+    const content = assistantContent(convertToConverseMessages(messages));
+    expect(content.length).toBeGreaterThan(0);
+    expect(content.find((b) => b.reasoningContent != null)).toBeUndefined();
+    expect(content.every((b) => typeof b.text === 'string')).toBe(true);
+  });
+
+  it('still converts a reasoning block that carries text (not dropped)', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage('hi'),
+      new AIMessage({
+        content: [
+          {
+            type: 'reasoning_content',
+            reasoningText: {
+              text: 'native bedrock reasoning',
+              signature: 'sig',
+            },
+          },
+          { type: 'text', text: 'answer' },
+        ],
+      }),
+    ];
+
+    const content = assistantContent(convertToConverseMessages(messages));
+    const reasoning = content.find((b) => b.reasoningContent != null);
+    expect(reasoning).toBeDefined();
+    expect(reasoning?.reasoningContent?.reasoningText?.text).toBe(
+      'native bedrock reasoning'
+    );
+  });
+});

package/src/llm/bedrock/utils/message_inputs.ts

@@ -72,6 +72,22 @@ export function langchainReasoningBlockToBedrockReasoningBlock(
   throw new Error('Invalid reasoning content');
 }
 
+/**
+ * Whether a reasoning block can be serialized to a valid Bedrock
+ * `reasoningContent`. Bedrock Converse rejects `reasoningText` with a null/empty
+ * `text` (e.g. a signature-only block that never merged with its text), so such
+ * blocks must be dropped rather than sent.
+ */
+function isSerializableBedrockReasoningBlock(
+  content: MessageContentReasoningBlock
+): boolean {
+  if (content.reasoningText != null) {
+    const text = content.reasoningText.text;
+    return text != null && text !== '';
+  }
+  return content.redactedContent != null && content.redactedContent !== '';
+}
+
 /**
  * Concatenate consecutive reasoning blocks in content array.
  */
@@ -653,10 +669,17 @@ function convertAIMessageToConverseMessage(msg: BaseMessage): BedrockMessage {
           contentBlocks.push({ text });
         }
       } else if (block.type === 'reasoning_content') {
+        const reasoningBlock = block as MessageContentReasoningBlock;
+        // Bedrock Converse rejects reasoningContent whose reasoningText.text is
+        // null/empty (a signature-only block that never merged with its text).
+        // Drop it rather than emit an invalid request; the empty-turn
+        // placeholder below covers a turn left with no content.
+        if (!isSerializableBedrockReasoningBlock(reasoningBlock)) {
+          return;
+        }
         contentBlocks.push({
-          reasoningContent: langchainReasoningBlockToBedrockReasoningBlock(
-            block as MessageContentReasoningBlock
-          ),
+          reasoningContent:
+            langchainReasoningBlockToBedrockReasoningBlock(reasoningBlock),
         } as BedrockContentBlock);
       } else if (isDefaultCachePoint(block)) {
         contentBlocks.push({
@@ -832,15 +855,34 @@ function convertToolMessageToConverseMessage(msg: BaseMessage): BedrockMessage {
     content = [{ text: String(msg.content) }];
   }
 
+  // A `cachePoint` is a message-level ContentBlock — it is NOT a valid
+  // ToolResultContentBlock. A tail prompt-cache breakpoint that anchors on a
+  // tool result therefore ends up nested inside `toolResult.content`, which
+  // Bedrock silently ignores (no cache write, no cache read). Hoist any
+  // cachePoint(s) out of the tool result body so they sit as siblings after
+  // it, which is the only position Bedrock honors.
+  const toolResultContent: BedrockContentBlock[] = [];
+  const trailingCachePoints: BedrockContentBlock[] = [];
+  for (const block of content) {
+    if (isDefaultCachePoint(block)) {
+      trailingCachePoints.push({
+        cachePoint: { type: 'default' },
+      } as BedrockContentBlock);
+    } else {
+      toolResultContent.push(block);
+    }
+  }
+
   return {
     role: 'user',
     content: [
       {
         toolResult: {
           toolUseId: toolCallId,
-          content: content as { text: string }[],
+          content: toolResultContent as { text: string }[],
         },
       },
+      ...trailingCachePoints,
     ],
   };
 }

package/src/llm/bedrock/utils/toolResultCachePoint.test.ts

@@ -0,0 +1,103 @@
+import { HumanMessage, AIMessage, ToolMessage } from '@langchain/core/messages';
+import type {
+  BaseMessage,
+  MessageContentComplex,
+} from '@langchain/core/messages';
+import { addBedrockTailCacheControl } from '@/messages/cache';
+import { convertToConverseMessages } from './message_inputs';
+import { toLangChainContent } from '@/messages/langchain';
+
+/**
+ * A Bedrock `cachePoint` is a message-level ContentBlock and is NOT a valid
+ * `ToolResultContentBlock`. When the single tail prompt-cache breakpoint
+ * anchors on a tool result (the common agent-loop shape), the cachePoint must
+ * be hoisted out of `toolResult.content` to a message-level sibling — otherwise
+ * Bedrock silently drops the breakpoint (no cache write, no cache read),
+ * verified live against Bedrock Converse.
+ */
+
+interface ConverseBlock {
+  text?: string;
+  cachePoint?: { type?: string };
+  toolResult?: {
+    toolUseId?: string;
+    content?: Array<{ text?: string; cachePoint?: { type?: string } }>;
+  };
+}
+
+function toolUserMessage(
+  result: ReturnType<typeof convertToConverseMessages>
+): ConverseBlock[] {
+  const msg = result.converseMessages.find(
+    (m) =>
+      m.role === 'user' && m.content?.some((c) => 'toolResult' in c) === true
+  );
+  return (msg?.content ?? []) as ConverseBlock[];
+}
+
+describe('convertToConverseMessages — tool-result cachePoint hoisting', () => {
+  it('hoists a cachePoint out of toolResult.content to a message-level sibling', () => {
+    const toolMsg = new ToolMessage({
+      tool_call_id: 't1',
+      content: toLangChainContent([
+        { type: 'text', text: 'result body' },
+        { cachePoint: { type: 'default' } },
+      ] as MessageContentComplex[]),
+    });
+
+    const { converseMessages } = convertToConverseMessages([
+      new HumanMessage('go'),
+      toolMsg,
+    ]);
+
+    const content = toolUserMessage({ converseMessages, converseSystem: [] });
+
+    // toolResult body must NOT contain the cachePoint
+    const toolResult = content.find((c) => 'toolResult' in c)?.toolResult;
+    expect(toolResult?.content?.some((b) => 'cachePoint' in b)).toBe(false);
+    expect(toolResult?.content).toEqual([{ text: 'result body' }]);
+
+    // cachePoint must be a sibling AFTER the toolResult block
+    expect(content[content.length - 1]).toEqual({
+      cachePoint: { type: 'default' },
+    });
+  });
+
+  it('leaves tool results without a cachePoint untouched', () => {
+    const { converseMessages } = convertToConverseMessages([
+      new HumanMessage('go'),
+      new ToolMessage({ tool_call_id: 't1', content: 'plain result' }),
+    ]);
+
+    const content = toolUserMessage({ converseMessages, converseSystem: [] });
+    expect(content).toEqual([
+      { toolResult: { toolUseId: 't1', content: [{ text: 'plain result' }] } },
+    ]);
+  });
+
+  it('end-to-end: tail breakpoint on a string tool result renders as a valid sibling cachePoint', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage('What is 15 * 23? Use the calculator.'),
+      new AIMessage({
+        content: 'Calculating.',
+        tool_calls: [
+          { id: 't1', name: 'calculator', args: { expression: '15 * 23' } },
+        ],
+      }),
+      new ToolMessage({ tool_call_id: 't1', content: '345' }),
+    ];
+
+    const cached = addBedrockTailCacheControl(messages);
+    const { converseMessages } = convertToConverseMessages(cached);
+
+    const content = toolUserMessage({ converseMessages, converseSystem: [] });
+    const toolResult = content.find((c) => 'toolResult' in c)?.toolResult;
+
+    // Exactly one cachePoint, at the message level, never nested in the body.
+    expect(toolResult?.content?.some((b) => 'cachePoint' in b)).toBe(false);
+    expect(content.filter((c) => 'cachePoint' in c)).toHaveLength(1);
+    expect(content[content.length - 1]).toEqual({
+      cachePoint: { type: 'default' },
+    });
+  });
+});

package/src/messages/cache.tail.test.ts

@@ -0,0 +1,340 @@
+import {
+  AIMessage,
+  HumanMessage,
+  SystemMessage,
+  ToolMessage,
+} from '@langchain/core/messages';
+import type {
+  BaseMessage,
+  MessageContentComplex,
+} from '@langchain/core/messages';
+import type Anthropic from '@anthropic-ai/sdk';
+import type { AnthropicMessages } from '@/types/messages';
+import { addTailCacheControl, addBedrockTailCacheControl } from './cache';
+import { toLangChainContent } from './langchain';
+
+type CacheControlBlock = MessageContentComplex & {
+  cache_control?: { type: 'ephemeral'; ttl?: '1h' };
+};
+
+/** Count every block across all messages that carries a cache_control marker. */
+function countCacheMarkers(
+  messages: ReadonlyArray<{ content: unknown }>
+): number {
+  let count = 0;
+  for (const message of messages) {
+    if (!Array.isArray(message.content)) {
+      continue;
+    }
+    for (const block of message.content) {
+      if (block && typeof block === 'object' && 'cache_control' in block) {
+        count++;
+      }
+    }
+  }
+  return count;
+}
+
+function blocksOf(message: { content: unknown }): CacheControlBlock[] {
+  return message.content as CacheControlBlock[];
+}
+
+describe('addTailCacheControl (single tail breakpoint)', () => {
+  test('places exactly one marker on the last message', () => {
+    const messages: AnthropicMessages = [
+      { role: 'user', content: [{ type: 'text', text: 'Hello' }] },
+      { role: 'assistant', content: [{ type: 'text', text: 'Hi there' }] },
+      { role: 'user', content: [{ type: 'text', text: 'How are you?' }] },
+      { role: 'assistant', content: [{ type: 'text', text: 'Doing well' }] },
+      { role: 'user', content: [{ type: 'text', text: 'Great!' }] },
+    ];
+
+    const result = addTailCacheControl(messages);
+
+    expect(countCacheMarkers(result)).toBe(1);
+    expect(
+      (result[4].content[0] as Anthropic.TextBlockParam).cache_control
+    ).toEqual({ type: 'ephemeral' });
+    expect(result[2].content[0]).not.toHaveProperty('cache_control');
+  });
+
+  test('anchors on a trailing tool_result block (tail is a tool turn)', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage('Run the tool'),
+      new AIMessage({
+        content: toLangChainContent([
+          { type: 'text', text: 'Calling it' },
+          { type: 'tool_use', id: 't1', name: 'search', input: {} },
+        ] as MessageContentComplex[]),
+        tool_calls: [{ id: 't1', name: 'search', args: {} }],
+      }),
+      new ToolMessage({
+        tool_call_id: 't1',
+        content: toLangChainContent([
+          {
+            type: 'tool_result',
+            tool_use_id: 't1',
+            content: 'result body',
+          },
+        ] as MessageContentComplex[]),
+      }),
+    ];
+
+    const result = addTailCacheControl(messages);
+
+    expect(countCacheMarkers(result)).toBe(1);
+    expect(blocksOf(result[2])[0].cache_control).toEqual({ type: 'ephemeral' });
+  });
+
+  test('strips ALL stale markers and re-anchors a single one at the tail', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage({
+        content: toLangChainContent([
+          {
+            type: 'text',
+            text: 'old marker',
+            cache_control: { type: 'ephemeral' },
+          },
+        ] as MessageContentComplex[]),
+      }),
+      new HumanMessage({
+        content: toLangChainContent([
+          {
+            type: 'text',
+            text: 'another old marker',
+            cache_control: { type: 'ephemeral' },
+          },
+        ] as MessageContentComplex[]),
+      }),
+      new AIMessage({ content: 'reply' }),
+    ];
+
+    const result = addTailCacheControl(messages);
+
+    expect(countCacheMarkers(result)).toBe(1);
+    expect(blocksOf(result[2])[0].cache_control).toEqual({ type: 'ephemeral' });
+    expect(blocksOf(result[0])[0]).not.toHaveProperty('cache_control');
+    expect(blocksOf(result[1])[0]).not.toHaveProperty('cache_control');
+  });
+
+  test('does not anchor on thinking blocks', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage('Hi'),
+      new AIMessage({
+        content: toLangChainContent([
+          { type: 'text', text: 'thought through it' },
+          { type: 'thinking', thinking: 'secret reasoning' },
+        ] as MessageContentComplex[]),
+      }),
+    ];
+
+    const result = addTailCacheControl(messages);
+
+    expect(countCacheMarkers(result)).toBe(1);
+    expect(blocksOf(result[1])[0].cache_control).toEqual({ type: 'ephemeral' });
+    expect(blocksOf(result[1])[1]).not.toHaveProperty('cache_control');
+  });
+
+  test.each(['reasoning_content', 'reasoning', 'think'])(
+    'does not anchor on a trailing foreign reasoning block (%s)',
+    (reasoningType) => {
+      // Foreign reasoning (Bedrock/Google/LibreChat) is dropped by the
+      // Anthropic converter on assistant turns; anchoring the only breakpoint
+      // there would silently lose tail caching on a cross-provider handoff.
+      const messages: BaseMessage[] = [
+        new HumanMessage('Hi'),
+        new AIMessage({
+          content: toLangChainContent([
+            { type: 'text', text: 'Here is my answer.' },
+            { type: reasoningType, text: 'foreign reasoning' },
+          ] as MessageContentComplex[]),
+        }),
+      ];
+
+      const result = addTailCacheControl(messages);
+
+      expect(countCacheMarkers(result)).toBe(1);
+      // Marker must land on the surviving text block, not the reasoning block.
+      expect(blocksOf(result[1])[0].cache_control).toEqual({
+        type: 'ephemeral',
+      });
+      expect(blocksOf(result[1])[1]).not.toHaveProperty('cache_control');
+    }
+  );
+
+  test('skips synthetic meta tail and anchors on the previous real message', () => {
+    const realTail = new AIMessage({ content: 'real answer' });
+    const metaTail = new HumanMessage({ content: 'reinjected skill body' });
+    (
+      metaTail as unknown as { additional_kwargs: Record<string, unknown> }
+    ).additional_kwargs = { isMeta: true };
+
+    const result = addTailCacheControl([
+      new HumanMessage({ content: 'question' }),
+      realTail,
+      metaTail,
+    ]);
+
+    expect(countCacheMarkers(result)).toBe(1);
+    expect(blocksOf(result[1])[0].cache_control).toEqual({ type: 'ephemeral' });
+  });
+
+  test('handles string content on the tail', () => {
+    const messages: AnthropicMessages = [
+      { role: 'user', content: 'Hello' },
+      { role: 'assistant', content: 'Final' },
+    ];
+
+    const result = addTailCacheControl(messages);
+
+    expect(result[0].content).toBe('Hello');
+    expect(result[1].content[0]).toEqual({
+      type: 'text',
+      text: 'Final',
+      cache_control: { type: 'ephemeral' },
+    });
+  });
+
+  test('does not mutate the original messages', () => {
+    const original: AnthropicMessages = [
+      { role: 'user', content: [{ type: 'text', text: 'Hello' }] },
+      { role: 'assistant', content: [{ type: 'text', text: 'World' }] },
+    ];
+
+    addTailCacheControl(original);
+
+    expect(original[1].content[0]).not.toHaveProperty('cache_control');
+  });
+
+  test('returns input unchanged for empty array', () => {
+    const messages: AnthropicMessages = [];
+    expect(addTailCacheControl(messages)).toEqual([]);
+  });
+});
+
+/** Count every Bedrock cachePoint block across all messages. */
+function countCachePoints(
+  messages: ReadonlyArray<{ content: unknown }>
+): number {
+  let count = 0;
+  for (const message of messages) {
+    if (!Array.isArray(message.content)) {
+      continue;
+    }
+    for (const block of message.content) {
+      if (block && typeof block === 'object' && 'cachePoint' in block) {
+        count++;
+      }
+    }
+  }
+  return count;
+}
+
+describe('addBedrockTailCacheControl (single tail cachePoint)', () => {
+  test('inserts exactly one cachePoint after the last text block of the tail', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage('First question'),
+      new AIMessage('First answer'),
+      new HumanMessage('Second question'),
+    ];
+
+    const result = addBedrockTailCacheControl(messages);
+
+    expect(countCachePoints(result)).toBe(1);
+    const tail = blocksOf(result[2]);
+    expect(tail[tail.length - 1]).toEqual({ cachePoint: { type: 'default' } });
+  });
+
+  test('strips stale cachePoints and re-anchors a single one at the tail', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage({
+        content: toLangChainContent([
+          { type: 'text', text: 'old' },
+          { cachePoint: { type: 'default' } },
+        ] as MessageContentComplex[]),
+      }),
+      new AIMessage('reply'),
+      new HumanMessage('newest'),
+    ];
+
+    const result = addBedrockTailCacheControl(messages);
+
+    expect(countCachePoints(result)).toBe(1);
+    const tail = blocksOf(result[2]);
+    expect(tail[tail.length - 1]).toEqual({ cachePoint: { type: 'default' } });
+    expect(blocksOf(result[0]).some((b) => 'cachePoint' in b)).toBe(false);
+  });
+
+  test('strips Anthropic cache_control from a system message but never anchors it', () => {
+    const messages: BaseMessage[] = [
+      new SystemMessage({
+        content: toLangChainContent([
+          {
+            type: 'text',
+            text: 'system rules',
+            cache_control: { type: 'ephemeral' },
+          },
+        ] as MessageContentComplex[]),
+      }),
+      new HumanMessage('hi'),
+    ];
+
+    const result = addBedrockTailCacheControl(messages);
+
+    expect(blocksOf(result[0])[0]).not.toHaveProperty('cache_control');
+    expect(countCachePoints(result)).toBe(1);
+    expect(blocksOf(result[1])[1]).toEqual({ cachePoint: { type: 'default' } });
+  });
+
+  test('skips synthetic meta tail and anchors on the previous real message', () => {
+    const metaTail = new HumanMessage({ content: 'reinjected skill body' });
+    (
+      metaTail as unknown as { additional_kwargs: Record<string, unknown> }
+    ).additional_kwargs = { source: 'skill' };
+
+    const result = addBedrockTailCacheControl([
+      new HumanMessage('question'),
+      new AIMessage('real answer'),
+      metaTail,
+    ]);
+
+    expect(countCachePoints(result)).toBe(1);
+    const realTail = blocksOf(result[1]);
+    expect(realTail[realTail.length - 1]).toEqual({
+      cachePoint: { type: 'default' },
+    });
+  });
+
+  test('handles string content on the tail', () => {
+    const result = addBedrockTailCacheControl([
+      new HumanMessage('Hello'),
+      new AIMessage('Final'),
+    ]);
+
+    expect(countCachePoints(result)).toBe(1);
+    expect(blocksOf(result[1])).toEqual([
+      { type: 'text', text: 'Final' },
+      { cachePoint: { type: 'default' } },
+    ]);
+  });
+
+  test('anchors on a trailing string tool result (agent-loop tail)', () => {
+    const result = addBedrockTailCacheControl([
+      new HumanMessage('Run the tool'),
+      new AIMessage({
+        content: 'Calling it',
+        tool_calls: [{ id: 't1', name: 'search', args: {} }],
+      }),
+      new ToolMessage({ tool_call_id: 't1', content: 'result body' }),
+    ]);
+
+    // The single cachePoint must land on the trailing tool result so the
+    // tool output is part of the cached prefix; the converter later hoists it
+    // out of toolResult.content (see toolResultCachePoint.test.ts).
+    expect(countCachePoints(result)).toBe(1);
+    expect(blocksOf(result[2])).toEqual([
+      { type: 'text', text: 'result body' },
+      { cachePoint: { type: 'default' } },
+    ]);
+  });
+});