@librechat/agents 3.1.74 → 3.1.75

package/src/agents/__tests__/AgentContext.bedrock.live.test.ts

@@ -0,0 +1,149 @@
+// src/agents/__tests__/AgentContext.bedrock.live.test.ts
+/**
+ * Live Bedrock prompt-cache verification.
+ *
+ * Run with:
+ * RUN_BEDROCK_PROMPT_CACHE_LIVE_TESTS=1 BEDROCK_AWS_REGION=... BEDROCK_AWS_ACCESS_KEY_ID=... BEDROCK_AWS_SECRET_ACCESS_KEY=... npm test -- AgentContext.bedrock.live.test.ts --runInBand
+ *
+ * Standard AWS credential env vars or AWS_PROFILE can also be used.
+ */
+import { config as dotenvConfig } from 'dotenv';
+dotenvConfig();
+
+import { describe, expect, it } from '@jest/globals';
+import type * as t from '@/types';
+import {
+  runLiveTurn,
+  assertSystemPayloadShape,
+  buildDynamicInstructions,
+  buildStableInstructions,
+  waitForCachePropagation,
+} from './promptCacheLiveHelpers';
+import { Providers } from '@/common';
+
+const accessKeyId =
+  process.env.BEDROCK_AWS_ACCESS_KEY_ID ?? process.env.AWS_ACCESS_KEY_ID;
+const secretAccessKey =
+  process.env.BEDROCK_AWS_SECRET_ACCESS_KEY ??
+  process.env.AWS_SECRET_ACCESS_KEY;
+const sessionToken =
+  process.env.BEDROCK_AWS_SESSION_TOKEN ?? process.env.AWS_SESSION_TOKEN;
+const hasCredentialPair =
+  accessKeyId != null &&
+  accessKeyId !== '' &&
+  secretAccessKey != null &&
+  secretAccessKey !== '';
+const hasAmbientCredentials =
+  process.env.AWS_PROFILE != null ||
+  process.env.AWS_WEB_IDENTITY_TOKEN_FILE != null;
+
+const shouldRunLive =
+  process.env.RUN_BEDROCK_PROMPT_CACHE_LIVE_TESTS === '1' &&
+  (hasCredentialPair || hasAmbientCredentials);
+
+const describeIfLive = shouldRunLive ? describe : describe.skip;
+
+const model =
+  process.env.BEDROCK_PROMPT_CACHE_MODEL ??
+  'us.anthropic.claude-sonnet-4-5-20250929-v1:0';
+const region =
+  process.env.BEDROCK_AWS_REGION ?? process.env.AWS_REGION ?? 'us-east-1';
+const providerLabel = 'Bedrock';
+
+function getCredentials():
+  | t.BedrockAnthropicClientOptions['credentials']
+  | undefined {
+  if (!hasCredentialPair) {
+    return undefined;
+  }
+
+  return {
+    accessKeyId,
+    secretAccessKey,
+    ...(sessionToken != null && sessionToken !== '' ? { sessionToken } : {}),
+  };
+}
+
+function createClientOptions(): t.BedrockAnthropicClientOptions {
+  const credentials = getCredentials();
+  return {
+    model,
+    region,
+    maxTokens: 8,
+    streaming: true,
+    streamUsage: true,
+    promptCache: true,
+    ...(credentials != null ? { credentials } : {}),
+  };
+}
+
+describeIfLive('AgentContext Bedrock prompt cache live API', () => {
+  it('caches only the stable system prefix while dynamic tail changes', async () => {
+    const nonce = `agent-bedrock-cache-live-${Date.now()}`;
+    const clientOptions = createClientOptions();
+    const stableInstructions = buildStableInstructions({
+      nonce,
+      providerLabel,
+    });
+    const firstDynamicInstructions = buildDynamicInstructions({
+      marker: 'alpha',
+      tailDescription:
+        'The Dynamic Marker line is runtime context and must remain after the Bedrock cache point.',
+    });
+    const secondDynamicInstructions = buildDynamicInstructions({
+      marker: 'bravo',
+      tailDescription:
+        'The Dynamic Marker line is runtime context and must remain after the Bedrock cache point.',
+    });
+
+    await assertSystemPayloadShape({
+      agentId: 'live-bedrock-cache-shape-check',
+      provider: Providers.BEDROCK,
+      clientOptions,
+      stableInstructions,
+      dynamicInstructions: firstDynamicInstructions,
+      expectedContent: [
+        {
+          type: 'text',
+          text: stableInstructions,
+        },
+        {
+          cachePoint: { type: 'default' },
+        },
+        {
+          type: 'text',
+          text: firstDynamicInstructions,
+        },
+      ],
+    });
+
+    const first = await runLiveTurn({
+      provider: Providers.BEDROCK,
+      providerLabel,
+      clientOptions,
+      runId: `${nonce}-first`,
+      threadId: `${nonce}-thread`,
+      stableInstructions,
+      dynamicInstructions: firstDynamicInstructions,
+    });
+
+    expect(first.text.toLowerCase()).toContain('alpha');
+    expect(first.usage.input_token_details?.cache_creation).toBeGreaterThan(0);
+    expect(first.usage.input_token_details?.cache_read ?? 0).toBe(0);
+
+    await waitForCachePropagation();
+
+    const second = await runLiveTurn({
+      provider: Providers.BEDROCK,
+      providerLabel,
+      clientOptions,
+      runId: `${nonce}-second`,
+      threadId: `${nonce}-thread`,
+      stableInstructions,
+      dynamicInstructions: secondDynamicInstructions,
+    });
+
+    expect(second.text.toLowerCase()).toContain('bravo');
+    expect(second.usage.input_token_details?.cache_read).toBeGreaterThan(0);
+  }, 180_000);
+});

package/src/agents/__tests__/AgentContext.test.ts

@@ -1,9 +1,15 @@
 // src/agents/__tests__/AgentContext.test.ts
+import { HumanMessage } from '@langchain/core/messages';
 import { AgentContext } from '../AgentContext';
 import { Providers } from '@/common';
+import { addBedrockCacheControl } from '@/messages/cache';
 import type * as t from '@/types';
 
 describe('AgentContext', () => {
+  type TestSystemContentBlock =
+    | { type: 'text'; text: string; cache_control?: { type: 'ephemeral' } }
+    | { cachePoint: { type: 'default' } };
+
   type ContextOptions = {
     agentConfig?: Partial<t.AgentInputs>;
     tokenCounter?: t.TokenCounter;
@@ -59,14 +65,161 @@ describe('AgentContext', () => {
       expect(ctx.systemRunnable).toBeUndefined();
     });
 
-    it('includes additional_instructions in system message', () => {
+    it('keeps additional_instructions after stable instructions', async () => {
       const ctx = createBasicContext({
         agentConfig: {
           instructions: 'Base instructions',
           additional_instructions: 'Additional instructions',
         },
       });
-      expect(ctx.systemRunnable).toBeDefined();
+
+      const result = await ctx.systemRunnable!.invoke([]);
+      expect(result[0].content).toBe(
+        'Base instructions\n\nAdditional instructions'
+      );
+    });
+
+    it('marks only stable system text for Anthropic prompt caching', async () => {
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.ANTHROPIC,
+          clientOptions: { model: 'claude-3-5-sonnet', promptCache: true },
+          instructions: 'Stable instructions',
+          additional_instructions: 'Dynamic instructions',
+        },
+      });
+
+      const result = await ctx.systemRunnable!.invoke([]);
+      const content = result[0].content as TestSystemContentBlock[];
+      expect(content).toHaveLength(2);
+      expect(content[0]).toMatchObject({
+        type: 'text',
+        text: 'Stable instructions',
+        cache_control: { type: 'ephemeral' },
+      });
+      expect(content[1]).toEqual({
+        type: 'text',
+        text: 'Dynamic instructions',
+      });
+    });
+
+    it('omits Anthropic cache control when only dynamic system text exists', async () => {
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.ANTHROPIC,
+          clientOptions: { model: 'claude-3-5-sonnet', promptCache: true },
+          instructions: undefined,
+          additional_instructions: 'Dynamic only',
+        },
+      });
+
+      const result = await ctx.systemRunnable!.invoke([]);
+      const content = result[0].content as TestSystemContentBlock[];
+      expect(content).toEqual([{ type: 'text', text: 'Dynamic only' }]);
+      expect(content[0]).not.toHaveProperty('cache_control');
+    });
+
+    it('keeps cross-run summaries in the dynamic Anthropic system tail', async () => {
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.ANTHROPIC,
+          clientOptions: { model: 'claude-3-5-sonnet', promptCache: true },
+          instructions: 'Stable instructions',
+        },
+      });
+      ctx.setInitialSummary('Prior summary', 13);
+
+      const result = await ctx.systemRunnable!.invoke([]);
+      const content = result[0].content as TestSystemContentBlock[];
+      expect(content).toHaveLength(2);
+      expect(content[0]).toHaveProperty('cache_control');
+      expect(content[1]).toEqual({
+        type: 'text',
+        text: '## Conversation Summary\n\nPrior summary',
+      });
+    });
+
+    it('places the Bedrock cache point before dynamic system text', async () => {
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.BEDROCK,
+          clientOptions: {
+            model: 'anthropic.claude-3-5-sonnet',
+            promptCache: true,
+          },
+          instructions: 'Stable instructions',
+          additional_instructions: 'Dynamic instructions',
+        },
+      });
+
+      const result = await ctx.systemRunnable!.invoke([]);
+      const content = result[0].content as TestSystemContentBlock[];
+      expect(content).toEqual([
+        { type: 'text', text: 'Stable instructions' },
+        { cachePoint: { type: 'default' } },
+        { type: 'text', text: 'Dynamic instructions' },
+      ]);
+    });
+
+    it('uses plain Bedrock system text when only dynamic system text exists', async () => {
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.BEDROCK,
+          clientOptions: {
+            model: 'anthropic.claude-3-5-sonnet',
+            promptCache: true,
+          },
+          instructions: undefined,
+          additional_instructions: 'Dynamic only',
+        },
+      });
+
+      const result = await ctx.systemRunnable!.invoke([]);
+      expect(result[0].content).toBe('Dynamic only');
+    });
+
+    it('keeps non-cache providers as plain system text with promptCache-like options', async () => {
+      const clientOptions: t.OpenAIClientOptions & { promptCache: true } = {
+        modelName: 'gpt-4o-mini',
+        promptCache: true,
+      };
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.OPENAI,
+          clientOptions,
+          instructions: 'Stable instructions',
+          additional_instructions: 'Dynamic instructions',
+        },
+      });
+
+      const result = await ctx.systemRunnable!.invoke([]);
+      expect(result[0].content).toBe(
+        'Stable instructions\n\nDynamic instructions'
+      );
+    });
+
+    it('preserves the Bedrock system cache point through message cache-control pass', async () => {
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.BEDROCK,
+          clientOptions: {
+            model: 'anthropic.claude-3-5-sonnet',
+            promptCache: true,
+          },
+          instructions: 'Stable instructions',
+          additional_instructions: 'Dynamic instructions',
+        },
+      });
+
+      const result = await ctx.systemRunnable!.invoke([
+        new HumanMessage('Hello'),
+      ]);
+      const finalMessages = addBedrockCacheControl(result);
+      expect(finalMessages[0].content).toEqual([
+        { type: 'text', text: 'Stable instructions' },
+        { cachePoint: { type: 'default' } },
+        { type: 'text', text: 'Dynamic instructions' },
+      ]);
     });
   });
 

package/src/agents/__tests__/promptCacheLiveHelpers.ts

@@ -0,0 +1,165 @@
+import { expect } from '@jest/globals';
+import { HumanMessage } from '@langchain/core/messages';
+import type { UsageMetadata } from '@langchain/core/messages';
+import type * as t from '@/types';
+import { GraphEvents, Providers } from '@/common';
+import { AgentContext } from '../AgentContext';
+import { ModelEndHandler } from '@/events';
+import { Run } from '@/run';
+
+type LivePromptCacheProvider = Providers.ANTHROPIC | Providers.BEDROCK;
+
+type PromptCacheExpectedSystemBlock =
+  | { type: 'text'; text: string; cache_control?: { type: 'ephemeral' } }
+  | { cachePoint: { type: 'default' } };
+
+type LivePromptCacheClientOptions =
+  | t.ClientOptions
+  | t.BedrockAnthropicClientOptions;
+
+export function buildStableInstructions({
+  nonce,
+  providerLabel,
+}: {
+  nonce: string;
+  providerLabel: string;
+}): string {
+  const records = Array.from(
+    { length: 360 },
+    (_, index) =>
+      `Stable ${providerLabel} cache record ${index}: nonce ${nonce}; keep this reference in the cacheable prefix and do not use it as the dynamic marker.`
+  );
+  return [
+    `You are a ${providerLabel} prompt-cache verification assistant.`,
+    'When asked for the dynamic marker, answer with only the marker value from the Dynamic Marker line.',
+    ...records,
+  ].join('\n');
+}
+
+export function buildDynamicInstructions({
+  marker,
+  tailDescription,
+}: {
+  marker: string;
+  tailDescription: string;
+}): string {
+  return [`Dynamic Marker: ${marker}`, tailDescription].join('\n');
+}
+
+export function waitForCachePropagation(): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, 2000));
+}
+
+export async function assertSystemPayloadShape({
+  agentId,
+  provider,
+  clientOptions,
+  stableInstructions,
+  dynamicInstructions,
+  expectedContent,
+}: {
+  agentId: string;
+  provider: LivePromptCacheProvider;
+  clientOptions: LivePromptCacheClientOptions;
+  stableInstructions: string;
+  dynamicInstructions: string;
+  expectedContent: PromptCacheExpectedSystemBlock[];
+}): Promise<void> {
+  const ctx = AgentContext.fromConfig({
+    agentId,
+    provider,
+    clientOptions,
+    instructions: stableInstructions,
+    additional_instructions: dynamicInstructions,
+  });
+
+  const messages = await ctx.systemRunnable!.invoke([
+    new HumanMessage('What is the dynamic marker?'),
+  ]);
+
+  expect(messages[0].content).toEqual(expectedContent);
+}
+
+function latestUsage({
+  collectedUsage,
+  label,
+  providerLabel,
+}: {
+  collectedUsage: UsageMetadata[];
+  label: string;
+  providerLabel: string;
+}): UsageMetadata {
+  if (collectedUsage.length === 0) {
+    throw new Error(`Missing ${providerLabel} usage metadata for ${label}`);
+  }
+  return collectedUsage[collectedUsage.length - 1];
+}
+
+function collectText(parts: t.MessageContentComplex[] | undefined): string {
+  return (parts ?? []).reduce((text, part) => {
+    if (part.type === 'text') {
+      return text + part.text;
+    }
+    return text;
+  }, '');
+}
+
+export async function runLiveTurn({
+  provider,
+  providerLabel,
+  clientOptions,
+  runId,
+  threadId,
+  stableInstructions,
+  dynamicInstructions,
+}: {
+  provider: LivePromptCacheProvider;
+  providerLabel: string;
+  clientOptions: LivePromptCacheClientOptions;
+  runId: string;
+  threadId: string;
+  stableInstructions: string;
+  dynamicInstructions: string;
+}): Promise<{
+  text: string;
+  usage: UsageMetadata;
+}> {
+  const collectedUsage: UsageMetadata[] = [];
+  const run = await Run.create<t.IState>({
+    runId,
+    graphConfig: {
+      type: 'standard',
+      llmConfig: {
+        provider,
+        ...clientOptions,
+      } as t.LLMConfig,
+      instructions: stableInstructions,
+      additional_instructions: dynamicInstructions,
+    },
+    returnContent: true,
+    skipCleanup: true,
+    customHandlers: {
+      [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(collectedUsage),
+    },
+  });
+
+  const config = {
+    configurable: { thread_id: threadId },
+    streamMode: 'values',
+    version: 'v2' as const,
+  };
+
+  const contentParts = await run.processStream(
+    {
+      messages: [
+        new HumanMessage('What is the dynamic marker? Reply with only it.'),
+      ],
+    },
+    config
+  );
+
+  return {
+    text: collectText(contentParts),
+    usage: latestUsage({ collectedUsage, label: runId, providerLabel }),
+  };
+}

package/src/llm/anthropic/utils/message_inputs.ts

@@ -611,7 +611,12 @@ function _formatContent(message: BaseMessage) {
     return contentBlocks.filter(
       (block) =>
         block !== null &&
-        !(block.type === 'text' && 'text' in block && block.text === '')
+        !(
+          block.type === 'text' &&
+          'text' in block &&
+          typeof block.text === 'string' &&
+          block.text.trim() === ''
+        )
     );
   }
 }

package/src/llm/anthropic/utils/server-tool-inputs.test.ts

@@ -346,4 +346,81 @@ describe('_convertMessagesToAnthropicPayload — server tool use (web search) mu
     expect(textBlocks).toHaveLength(1);
     expect(textBlocks[0].text).toBe('Here are the results.');
   });
+
+  /**
+   * Regression for LibreChat discussion #12806.
+   *
+   * Anthropic web_search responses can include text blocks whose text is
+   * whitespace-only (e.g. ' ', '\n', '\t') alongside server_tool_use and
+   * web_search_tool_result blocks. On follow-up turns the API rejects these
+   * with: "messages: text content blocks must contain non-whitespace text".
+   *
+   * The empty-string check alone is insufficient — the filter must drop any
+   * text block whose trimmed content is empty.
+   */
+  it.each([
+    ['single space', ' '],
+    ['newline', '\n'],
+    ['tab', '\t'],
+    ['multiple spaces', '   '],
+    ['mixed whitespace', ' \n\t '],
+  ])(
+    'filters whitespace-only text blocks from array content (%s)',
+    (_label, whitespace) => {
+      const messageHistory: BaseMessage[] = [
+        new HumanMessage('search for X'),
+        new AIMessage({
+          content: [
+            { type: 'text', text: whitespace },
+            {
+              type: 'server_tool_use',
+              id: 'srvtoolu_1',
+              name: 'web_search',
+              input: { query: 'X' },
+            },
+            {
+              type: 'web_search_tool_result',
+              tool_use_id: 'srvtoolu_1',
+              content: [
+                {
+                  type: 'web_search_result',
+                  url: 'https://example.com',
+                  title: 'Result',
+                  encrypted_content: 'abc',
+                  page_age: '1d',
+                },
+              ],
+            },
+            { type: 'text', text: 'Here are the results.' },
+          ],
+          tool_calls: [
+            {
+              id: 'srvtoolu_1',
+              name: 'web_search',
+              args: { query: 'X' },
+              type: 'tool_call',
+            },
+          ],
+        }),
+        new HumanMessage('follow up'),
+      ];
+
+      const { messages } = _convertMessagesToAnthropicPayload(messageHistory);
+      const assistantContent = messages[1].content as any[];
+
+      const whitespaceTextBlocks = assistantContent.filter(
+        (b: any) =>
+          b.type === 'text' &&
+          typeof b.text === 'string' &&
+          b.text.trim() === ''
+      );
+      expect(whitespaceTextBlocks).toHaveLength(0);
+
+      const textBlocks = assistantContent.filter(
+        (b: any) => b.type === 'text'
+      );
+      expect(textBlocks).toHaveLength(1);
+      expect(textBlocks[0].text).toBe('Here are the results.');
+    }
+  );
 });