@librechat/agents 3.1.80 → 3.1.82

package/src/llm/vertexai/index.ts

@@ -65,50 +65,44 @@ type AdditionalKwargs =
  * - The signature for a functionCall part is an empty string
  *
  * This function correlates each "model" content block in the formatted request
- * back to its originating AI message, then re-attaches non-empty signatures
- * that the library failed to apply.
+ * back to its originating AI message by *position*, then re-attaches non-empty
+ * signatures that the library failed to apply. AI messages without signatures
+ * still consume their slot — filtering them out shifted later messages onto
+ * the wrong content block and dropped real signatures on the floor.
  */
-function fixThoughtSignatures(
+export function fixThoughtSignatures(
   contents: GeminiContent[],
   input: BaseMessage[]
 ): void {
-  // Collect AI messages that have signatures, in order
-  const aiMessages = input.filter(
-    (msg) =>
-      isAIMessage(msg) &&
-      Array.isArray((msg.additional_kwargs as AdditionalKwargs)?.signatures) &&
-      (msg.additional_kwargs.signatures as string[]).length > 0
-  );
-
-  // Collect "model" content blocks from the formatted request, in order
+  // All AI messages, in order — non-signature ones still consume positional
+  // slots so later messages line up with their model content blocks.
+  const aiMessages = input.filter(isAIMessage);
   const modelContents = contents.filter((c) => c.role === 'model');
 
-  // They should correspond 1:1 in order (both derived from the same input sequence)
   const count = Math.min(aiMessages.length, modelContents.length);
   for (let i = 0; i < count; i++) {
-    const msg = aiMessages[i];
-    const content = modelContents[i];
-    const signatures = (msg.additional_kwargs as AdditionalKwargs)?.signatures;
+    const signatures = (aiMessages[i].additional_kwargs as AdditionalKwargs)
+      ?.signatures;
+    if (!Array.isArray(signatures) || signatures.length === 0) continue;
 
-    // Collect non-empty signatures that aren't already attached to any part
+    const content = modelContents[i];
     const attachedSignatures = new Set(
       content.parts
         .map((p) => p.thoughtSignature)
         .filter((s): s is string => s != null && s !== '')
     );
-    const availableSignatures = signatures?.filter(
-      (s) => s != null && s !== '' && !attachedSignatures.has(s)
+    const availableSignatures = signatures.filter(
+      (s): s is string => s != null && s !== '' && !attachedSignatures.has(s)
     );
 
-    // Assign available signatures to functionCall parts missing one, in order
     let sigIdx = 0;
     for (const part of content.parts) {
       if (
         'functionCall' in part &&
         (part.thoughtSignature == null || part.thoughtSignature === '') &&
-        sigIdx < (availableSignatures?.length ?? 0)
+        sigIdx < availableSignatures.length
       ) {
-        part.thoughtSignature = availableSignatures?.[sigIdx];
+        part.thoughtSignature = availableSignatures[sigIdx];
         sigIdx++;
       }
     }

package/src/messages/cache.test.ts

@@ -14,9 +14,14 @@ import {
   addBedrockCacheControl,
   addCacheControl,
 } from './cache';
+import { _convertMessagesToOpenAIParams } from '@/llm/openai/utils';
 import { toLangChainContent } from './langchain';
 import { ContentTypes } from '@/common/enum';
 
+type CacheControlBlock = MessageContentComplex & {
+  cache_control?: { type: 'ephemeral'; ttl?: '1h' };
+};
+
 describe('addCacheControl', () => {
   test('should add cache control to the last two user messages with array content', () => {
     const messages: AnthropicMessages = [
@@ -1483,3 +1488,125 @@ describe('LangChain message type preservation', () => {
     expect((result[1] as AIMessage).tool_calls![0].name).toBe('navigate');
   });
 });
+
+describe('OpenRouter prompt caching (reuses addCacheControl)', () => {
+  it('adds cache_control to LangChain messages for OpenRouter (same format as Anthropic)', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage({ content: [{ type: 'text', text: 'System context' }] }),
+      new AIMessage({ content: [{ type: 'text', text: 'Acknowledged' }] }),
+      new HumanMessage({ content: [{ type: 'text', text: 'User query' }] }),
+    ];
+
+    const result = addCacheControl(messages);
+
+    const firstContent = result[0].content as MessageContentComplex[];
+    const lastContent = result[2].content as MessageContentComplex[];
+
+    expect((firstContent[0] as CacheControlBlock).cache_control).toEqual({
+      type: 'ephemeral',
+    });
+    expect((lastContent[0] as CacheControlBlock).cache_control).toEqual({
+      type: 'ephemeral',
+    });
+  });
+
+  it('preserves cache_control through OpenAI message conversion used by OpenRouter', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage({
+        content: [
+          {
+            type: 'text',
+            text: 'Hello',
+            cache_control: { type: 'ephemeral' },
+          },
+        ],
+      }),
+      new AIMessage({ content: 'Hi there' }),
+      new HumanMessage({
+        content: [
+          {
+            type: 'text',
+            text: 'Follow-up',
+            cache_control: { type: 'ephemeral' },
+          },
+        ],
+      }),
+    ];
+
+    const converted = _convertMessagesToOpenAIParams(messages);
+
+    const firstUserContent = converted[0].content as CacheControlBlock[];
+    const lastUserContent = converted[2].content as CacheControlBlock[];
+
+    expect(firstUserContent[0]).toHaveProperty('cache_control');
+    expect(firstUserContent[0].cache_control).toEqual({ type: 'ephemeral' });
+    expect(lastUserContent[0]).toHaveProperty('cache_control');
+    expect(lastUserContent[0].cache_control).toEqual({ type: 'ephemeral' });
+  });
+
+  it('end-to-end: addCacheControl then convert preserves breakpoints for OpenRouter', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage({ content: 'First message with context' }),
+      new AIMessage({ content: 'Response' }),
+      new HumanMessage({ content: 'Second question' }),
+    ];
+
+    const cached = addCacheControl(messages);
+    const converted = _convertMessagesToOpenAIParams(
+      cached,
+      'anthropic/claude-sonnet-4-20250514'
+    );
+
+    const firstUser = converted[0];
+    const lastUser = converted[2];
+
+    expect(Array.isArray(firstUser.content)).toBe(true);
+    expect(
+      (firstUser.content as CacheControlBlock[])[0]
+    ).toHaveProperty('cache_control');
+
+    expect(Array.isArray(lastUser.content)).toBe(true);
+    expect(
+      (lastUser.content as CacheControlBlock[])[0]
+    ).toHaveProperty('cache_control');
+  });
+
+  it('strips Bedrock cache before applying OpenRouter/Anthropic cache', () => {
+    const messages: TestMsg[] = [
+      {
+        role: 'user',
+        content: [
+          { type: ContentTypes.TEXT, text: 'First message' },
+          { cachePoint: { type: 'default' } },
+        ],
+      },
+      {
+        role: 'assistant',
+        content: [
+          { type: ContentTypes.TEXT, text: 'Response' },
+          { cachePoint: { type: 'default' } },
+        ],
+      },
+      {
+        role: 'user',
+        content: [{ type: ContentTypes.TEXT, text: 'Follow-up' }],
+      },
+    ];
+
+    /** @ts-expect-error - Testing cross-provider compatibility */
+    const result = addCacheControl(messages);
+
+    for (const msg of result) {
+      if (Array.isArray(msg.content)) {
+        expect(
+          (msg.content as MessageContentComplex[]).some(
+            (b) => 'cachePoint' in b
+          )
+        ).toBe(false);
+      }
+    }
+
+    const lastContent = result[2].content as MessageContentComplex[];
+    expect('cache_control' in lastContent[0]).toBe(true);
+  });
+});

package/src/scripts/openrouter_prompt_cache_live.ts

@@ -0,0 +1,310 @@
+import { config as loadEnv } from 'dotenv';
+import { HumanMessage, SystemMessage } from '@langchain/core/messages';
+import type { AIMessage, BaseMessage } from '@langchain/core/messages';
+import type { ClientOptions } from '@langchain/openai';
+import type { GraphTools } from '@/types';
+import type { ChatOpenRouterInput } from '@/llm/openrouter';
+import { addCacheControl } from '@/messages/cache';
+import { ChatOpenRouter } from '@/llm/openrouter';
+import { partitionAndMarkOpenRouterToolCache } from '@/llm/openrouter/toolCache';
+
+loadEnv({ path: process.env.DOTENV_CONFIG_PATH ?? '.env' });
+
+type ModelCase = {
+  label: string;
+  model: string;
+};
+
+type CacheUsage = {
+  cacheCreation: number;
+  cacheRead: number;
+  inputTokens: number;
+  outputTokens: number;
+  totalTokens: number;
+};
+
+type OpenRouterTool = {
+  type: 'function';
+  function: {
+    name: string;
+  };
+  cache_control?: { type: 'ephemeral' };
+};
+
+const DEFAULT_MODEL_CASES: ModelCase[] = [
+  { label: 'Anthropic Claude', model: 'anthropic/claude-haiku-4.5' },
+  { label: 'Google Gemini', model: 'google/gemini-2.5-flash' },
+  { label: 'Alibaba Qwen', model: 'qwen/qwen3-coder-flash' },
+];
+
+const apiKey = process.env.OPENROUTER_API_KEY;
+const baseURL =
+  process.env.OPENROUTER_BASE_URL ?? 'https://openrouter.ai/api/v1';
+const attempts = Number(process.env.OPENROUTER_PROMPT_CACHE_ATTEMPTS ?? '3');
+const modelCases = (
+  process.env.OPENROUTER_PROMPT_CACHE_MODELS?.split(',').map((model) => ({
+    label: 'Custom',
+    model: model.trim(),
+  })) ?? DEFAULT_MODEL_CASES
+).filter(({ model }) => model.length > 0);
+
+if (apiKey == null || apiKey.length === 0) {
+  throw new Error('OPENROUTER_API_KEY is required');
+}
+
+function buildStableReference(): string {
+  const paragraph =
+    'LibreChat OpenRouter prompt caching live validation reference. This paragraph is deliberately stable across repeated requests so OpenRouter can route the conversation to the same provider endpoint and reuse cached prompt tokens. It describes cache breakpoints, provider sticky routing, cache write metrics, cache read metrics, model-specific minimum prompt sizes, and the expected behavior of explicit per-message cache_control markers for supported OpenRouter providers.';
+
+  return Array.from({ length: 90 }, (_, index) => {
+    const section = index + 1;
+    return `Section ${section}. ${paragraph} Verification key ${section}: OPENROUTER_PROMPT_CACHE_LIVE_REFERENCE_${section}.`;
+  }).join('\n');
+}
+
+function buildStableToolDescription(): string {
+  const paragraph =
+    'Static OpenRouter tool contract for prompt cache validation. This tool description is stable across requests and intentionally verbose so provider-side prompt caching can write and then read a meaningful static tool-schema prefix while dynamic tools vary after the cache breakpoint.';
+
+  return Array.from({ length: 90 }, (_, index) => {
+    const section = index + 1;
+    return `Tool section ${section}. ${paragraph} Stable tool key ${section}: OPENROUTER_STATIC_TOOL_CACHE_REFERENCE_${section}.`;
+  }).join('\n');
+}
+
+function buildToolSet(attempt: number): GraphTools {
+  return [
+    {
+      type: 'function',
+      function: {
+        name: 'stable_reference_lookup',
+        description: buildStableToolDescription(),
+        parameters: {
+          type: 'object',
+          properties: {
+            query: {
+              type: 'string',
+              description: 'Stable lookup query.',
+            },
+          },
+          required: ['query'],
+          additionalProperties: false,
+        },
+      },
+    },
+    {
+      type: 'function',
+      function: {
+        name: `dynamic_runtime_tool_${attempt}`,
+        description: `Dynamic runtime tool ${attempt}; this varies between attempts and should sit after the cached static tool prefix.`,
+        parameters: {
+          type: 'object',
+          properties: {
+            value: {
+              type: 'string',
+            },
+          },
+          required: ['value'],
+          additionalProperties: false,
+        },
+      },
+    },
+  ] as GraphTools;
+}
+
+function buildMessages(model: string): BaseMessage[] {
+  const reference = buildStableReference();
+  const messages: BaseMessage[] = [
+    new SystemMessage(
+      'You are validating prompt caching. Answer with one concise sentence.'
+    ),
+    new HumanMessage(
+      [
+        `For model ${model}, reply with exactly this format: cache live check ok.`,
+        'Use the stable reference below only to make this request large enough to cache.',
+        reference,
+      ].join('\n\n')
+    ),
+  ];
+
+  return addCacheControl<BaseMessage>(messages);
+}
+
+function getCacheUsage(message: AIMessage): CacheUsage {
+  const usage = message.usage_metadata;
+  const inputDetails = usage?.input_token_details;
+
+  return {
+    inputTokens: usage?.input_tokens ?? 0,
+    outputTokens: usage?.output_tokens ?? 0,
+    totalTokens: usage?.total_tokens ?? 0,
+    cacheRead: inputDetails?.cache_read ?? 0,
+    cacheCreation: inputDetails?.cache_creation ?? 0,
+  };
+}
+
+function hasCacheHit(usages: CacheUsage[]): boolean {
+  return usages.some(({ cacheRead }) => cacheRead > 0);
+}
+
+function hasCacheActivity(usages: CacheUsage[]): boolean {
+  return usages.some(
+    ({ cacheCreation, cacheRead }) => cacheCreation > 0 || cacheRead > 0
+  );
+}
+
+function log(message = ''): void {
+  process.stdout.write(`${message}\n`);
+}
+
+function logError(message: string): void {
+  process.stderr.write(`${message}\n`);
+}
+
+async function runCase({ label, model }: ModelCase): Promise<CacheUsage[]> {
+  const llmInput: ChatOpenRouterInput & { configuration: ClientOptions } = {
+    model,
+    apiKey,
+    maxTokens: 12,
+    temperature: 0,
+    promptCache: true,
+    streamUsage: true,
+    configuration: {
+      baseURL,
+      defaultHeaders: {
+        'HTTP-Referer': 'https://librechat.ai',
+        'X-Title': 'LibreChat OpenRouter Prompt Cache Live Test',
+      },
+    },
+  };
+  const llm = new ChatOpenRouter(llmInput);
+  const messages = buildMessages(model);
+  const usages: CacheUsage[] = [];
+
+  log(`\n${label}: ${model}`);
+
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    const started = Date.now();
+    const response = (await llm.invoke(messages)) as AIMessage;
+    const usage = getCacheUsage(response);
+    usages.push(usage);
+
+    log(
+      [
+        `attempt=${attempt}`,
+        `ms=${Date.now() - started}`,
+        `input=${usage.inputTokens}`,
+        `output=${usage.outputTokens}`,
+        `write=${usage.cacheCreation}`,
+        `read=${usage.cacheRead}`,
+        `total=${usage.totalTokens}`,
+      ].join(' ')
+    );
+
+    if (hasCacheHit(usages)) {
+      return usages;
+    }
+  }
+
+  return usages;
+}
+
+async function runStaticToolCase(): Promise<CacheUsage[]> {
+  const model = 'anthropic/claude-haiku-4.5';
+  const usages: CacheUsage[] = [];
+
+  log(`\nStatic tools through OpenRouter: ${model}`);
+
+  for (let attempt = 1; attempt <= attempts; attempt++) {
+    const llmInput: ChatOpenRouterInput & { configuration: ClientOptions } = {
+      model,
+      apiKey,
+      maxTokens: 12,
+      temperature: 0,
+      promptCache: true,
+      streamUsage: true,
+      configuration: {
+        baseURL,
+        defaultHeaders: {
+          'HTTP-Referer': 'https://librechat.ai',
+          'X-Title': 'LibreChat OpenRouter Prompt Cache Live Test',
+        },
+      },
+    };
+    const llm = new ChatOpenRouter(llmInput);
+    const tools = partitionAndMarkOpenRouterToolCache(
+      buildToolSet(attempt),
+      (name) => name.startsWith('dynamic_runtime_tool_')
+    ) as OpenRouterTool[];
+    const markedTool = tools.find((entry) => entry.cache_control != null);
+    if (markedTool?.function.name !== 'stable_reference_lookup') {
+      throw new Error('Static tool cache marker was not applied as expected');
+    }
+
+    const modelWithTools = llm.bindTools(tools);
+    const started = Date.now();
+    const response = (await modelWithTools.invoke([
+      new SystemMessage('Reply with exactly: cache live check ok.'),
+      new HumanMessage(
+        `Attempt ${attempt}. Do not call tools; only answer with the requested text.`
+      ),
+    ])) as AIMessage;
+    const usage = getCacheUsage(response);
+    usages.push(usage);
+
+    log(
+      [
+        `attempt=${attempt}`,
+        `ms=${Date.now() - started}`,
+        `input=${usage.inputTokens}`,
+        `output=${usage.outputTokens}`,
+        `write=${usage.cacheCreation}`,
+        `read=${usage.cacheRead}`,
+        `total=${usage.totalTokens}`,
+      ].join(' ')
+    );
+
+    if (hasCacheHit(usages)) {
+      return usages;
+    }
+  }
+
+  return usages;
+}
+
+async function main(): Promise<void> {
+  const results: Array<ModelCase & { usages: CacheUsage[] }> = [];
+
+  for (const modelCase of modelCases) {
+    const usages = await runCase(modelCase);
+    results.push({ ...modelCase, usages });
+  }
+
+  const staticToolUsages = await runStaticToolCase();
+  results.push({
+    label: 'Static tools',
+    model: 'anthropic/claude-haiku-4.5',
+    usages: staticToolUsages,
+  });
+
+  const failures = results.filter(({ usages }) => {
+    return !hasCacheActivity(usages) || !hasCacheHit(usages);
+  });
+
+  log('\nSummary');
+  for (const { label, model, usages } of results) {
+    const writes = usages.map(({ cacheCreation }) => cacheCreation).join(',');
+    const reads = usages.map(({ cacheRead }) => cacheRead).join(',');
+    log(`${label} ${model}: writes=[${writes}] reads=[${reads}]`);
+  }
+
+  if (failures.length > 0) {
+    const failedModels = failures.map(({ model }) => model).join(', ');
+    throw new Error(`Prompt caching was not confirmed for: ${failedModels}`);
+  }
+}
+
+main().catch((error: Error) => {
+  logError(error.message);
+  process.exit(1);
+});

package/src/specs/agent-handoffs.live.test.ts

@@ -0,0 +1,140 @@
+// src/specs/agent-handoffs.live.test.ts
+/**
+ * Live handoff integration verification.
+ *
+ * Run with:
+ * RUN_HANDOFF_LIVE_TESTS=1 ANTHROPIC_API_KEY=... npm test -- agent-handoffs.live.test.ts --runInBand
+ */
+import { config as dotenvConfig } from 'dotenv';
+dotenvConfig();
+
+import { HumanMessage } from '@langchain/core/messages';
+import { describe, expect, it, jest } from '@jest/globals';
+import type { BaseMessage, ToolMessage } from '@langchain/core/messages';
+import type { RunnableConfig } from '@langchain/core/runnables';
+import type * as t from '@/types';
+import { Constants, Providers } from '@/common';
+import { Run } from '@/run';
+
+const shouldRunLive =
+  process.env.RUN_HANDOFF_LIVE_TESTS === '1' &&
+  process.env.ANTHROPIC_API_KEY != null &&
+  process.env.ANTHROPIC_API_KEY !== '';
+
+const describeIfLive = shouldRunLive ? describe : describe.skip;
+const modelName =
+  process.env.ANTHROPIC_HANDOFF_LIVE_MODEL ?? 'claude-sonnet-4-6';
+
+function createAnthropicAgent(
+  agentId: string,
+  instructions: string
+): t.AgentInputs {
+  return {
+    agentId,
+    provider: Providers.ANTHROPIC,
+    clientOptions: {
+      modelName,
+      apiKey: process.env.ANTHROPIC_API_KEY,
+      temperature: 0,
+      maxTokens: 256,
+      streaming: true,
+    },
+    instructions,
+    maxContextTokens: 8000,
+  };
+}
+
+function createStreamConfig(threadId: string): Partial<RunnableConfig> & {
+  version: 'v1' | 'v2';
+  streamMode: string;
+} {
+  return {
+    configurable: { thread_id: threadId },
+    streamMode: 'values',
+    version: 'v2',
+  };
+}
+
+function contentToText(message: BaseMessage): string {
+  if (typeof message.content === 'string') {
+    return message.content;
+  }
+  if (!Array.isArray(message.content)) {
+    return '';
+  }
+  return message.content
+    .map((part) => {
+      if (typeof part === 'string') {
+        return part;
+      }
+      if ('text' in part && typeof part.text === 'string') {
+        return part.text;
+      }
+      return '';
+    })
+    .join('');
+}
+
+describeIfLive('Agent handoffs live integration', () => {
+  jest.setTimeout(120_000);
+
+  it('routes through a real Anthropic handoff and preserves instructions', async () => {
+    const nonce = `live-handoff-${Date.now()}`;
+    const expectedReply = `${nonce}-specialist-confirmed`;
+    const handoffToolName = `${Constants.LC_TRANSFER_TO_}specialist`;
+    const agents: t.AgentInputs[] = [
+      createAnthropicAgent(
+        'router',
+        `You are a routing agent. For every user request, your only valid action is to call the handoff tool named ${handoffToolName}. Do not answer directly.
+
+When you call the handoff tool, include instructions telling the specialist to reply exactly with this marker and no extra words: ${expectedReply}`
+      ),
+      createAnthropicAgent(
+        'specialist',
+        'You are the specialist. When you receive handoff instructions with a marker, reply exactly with that marker and no extra words.'
+      ),
+    ];
+    const edges: t.GraphEdge[] = [
+      {
+        from: 'router',
+        to: 'specialist',
+        edgeType: 'handoff',
+        description: 'Transfer to the specialist for the final response',
+        prompt:
+          'Instructions for the specialist. Include any exact marker that must be returned.',
+        promptKey: 'instructions',
+      },
+    ];
+    const run = await Run.create({
+      runId: `${nonce}-run`,
+      graphConfig: { type: 'multi-agent', agents, edges },
+      returnContent: true,
+      skipCleanup: true,
+    });
+
+    await run.processStream(
+      {
+        messages: [
+          new HumanMessage(
+            `Please delegate this to the specialist. The final answer must be exactly: ${expectedReply}`
+          ),
+        ],
+      },
+      createStreamConfig(`${nonce}-thread`)
+    );
+
+    const messages = run.getRunMessages() ?? [];
+    const handoffMessage = messages.find(
+      (message): message is ToolMessage =>
+        message.getType() === 'tool' &&
+        (message as ToolMessage).name === handoffToolName
+    );
+    const finalText = messages
+      .filter((message) => message.getType() === 'ai')
+      .map(contentToText)
+      .join('\n');
+
+    expect(handoffMessage).toBeDefined();
+    expect(finalText).toContain(expectedReply);
+  });
+});