@librechat/agents 3.1.81 → 3.1.83

package/src/agents/__tests__/AgentContext.test.ts

@@ -1,5 +1,5 @@
 // src/agents/__tests__/AgentContext.test.ts
-import { HumanMessage } from '@langchain/core/messages';
+import { AIMessage, HumanMessage, ToolMessage } from '@langchain/core/messages';
 import { AgentContext } from '../AgentContext';
 import { Providers } from '@/common';
 import { addBedrockCacheControl } from '@/messages/cache';
@@ -79,7 +79,7 @@ describe('AgentContext', () => {
       );
     });
 
-    it('marks only stable system text for Anthropic prompt caching', async () => {
+    it('moves Anthropic dynamic instructions behind stable history', async () => {
       const ctx = createBasicContext({
         agentConfig: {
           provider: Providers.ANTHROPIC,
@@ -89,18 +89,39 @@ describe('AgentContext', () => {
         },
       });
 
-      const result = await ctx.systemRunnable!.invoke([]);
+      const result = await ctx.systemRunnable!.invoke([
+        new HumanMessage('Hello'),
+        new HumanMessage('Second'),
+      ]);
       const content = result[0].content as TestSystemContentBlock[];
-      expect(content).toHaveLength(2);
-      expect(content[0]).toMatchObject({
-        type: 'text',
-        text: 'Stable instructions',
-        cache_control: { type: 'ephemeral' },
-      });
-      expect(content[1]).toEqual({
-        type: 'text',
-        text: 'Dynamic instructions',
+      expect(content).toEqual([
+        {
+          type: 'text',
+          text: 'Stable instructions',
+          cache_control: { type: 'ephemeral' },
+        },
+      ]);
+      expect(result[1].content).toBe('Hello');
+      expect(result[2].content).toBe('Dynamic instructions');
+      expect(result[3].content).toBe('Second');
+    });
+
+    it('places Anthropic dynamic instructions before a single latest user prompt', async () => {
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.ANTHROPIC,
+          clientOptions: { model: 'claude-3-5-sonnet', promptCache: true },
+          instructions: 'Stable instructions',
+          additional_instructions: 'Dynamic instructions',
+        },
       });
+
+      const result = await ctx.systemRunnable!.invoke([
+        new HumanMessage('Latest'),
+      ]);
+
+      expect(result[1].content).toBe('Dynamic instructions');
+      expect(result[2].content).toBe('Latest');
     });
 
     it('omits Anthropic cache control when only dynamic system text exists', async () => {
@@ -119,7 +140,7 @@ describe('AgentContext', () => {
       expect(content[0]).not.toHaveProperty('cache_control');
     });
 
-    it('keeps cross-run summaries in the dynamic Anthropic system tail', async () => {
+    it('keeps cross-run summaries in the dynamic Anthropic tail', async () => {
       const ctx = createBasicContext({
         agentConfig: {
           provider: Providers.ANTHROPIC,
@@ -131,12 +152,11 @@ describe('AgentContext', () => {
 
       const result = await ctx.systemRunnable!.invoke([]);
       const content = result[0].content as TestSystemContentBlock[];
-      expect(content).toHaveLength(2);
+      expect(content).toHaveLength(1);
       expect(content[0]).toHaveProperty('cache_control');
-      expect(content[1]).toEqual({
-        type: 'text',
-        text: '## Conversation Summary\n\nPrior summary',
-      });
+      expect(result[1].content).toBe(
+        '## Conversation Summary\n\nPrior summary'
+      );
     });
 
     it('places the Bedrock cache point before dynamic system text', async () => {
@@ -198,6 +218,270 @@ describe('AgentContext', () => {
       );
     });
 
+    it('moves OpenRouter dynamic instructions behind stable history', async () => {
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.OPENROUTER,
+          clientOptions: {
+            model: 'anthropic/claude-haiku-4.5',
+            promptCache: true,
+          },
+          instructions: 'Stable instructions',
+          additional_instructions: 'Dynamic instructions',
+        },
+      });
+
+      const result = await ctx.systemRunnable!.invoke([
+        new HumanMessage('Hello'),
+        new HumanMessage('Second'),
+      ]);
+      const content = result[0].content as TestSystemContentBlock[];
+      expect(content).toEqual([
+        {
+          type: 'text',
+          text: 'Stable instructions',
+          cache_control: { type: 'ephemeral' },
+        },
+      ]);
+      expect(result[1].content).toBe('Hello');
+      expect(result[2].content).toBe('Dynamic instructions');
+      expect(result[3].content).toBe('Second');
+    });
+
+    it('keeps dynamic-only OpenRouter instructions as system text', async () => {
+      const tokenCounter = (msg: { content: unknown }): number => {
+        const content =
+          typeof msg.content === 'string'
+            ? msg.content
+            : JSON.stringify(msg.content);
+        return content.length;
+      };
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.OPENROUTER,
+          clientOptions: {
+            model: 'anthropic/claude-haiku-4.5',
+            promptCache: true,
+          },
+          instructions: undefined,
+          additional_instructions: 'Dynamic only',
+        },
+        tokenCounter,
+      });
+
+      ctx.initializeSystemRunnable();
+      const result = await ctx.systemRunnable!.invoke([
+        new HumanMessage('First'),
+        new HumanMessage('Second'),
+      ]);
+      const firstContent = result[1].content as TestSystemContentBlock[];
+      const secondContent = result[2].content as TestSystemContentBlock[];
+
+      expect(result).toHaveLength(3);
+      expect(result[0].content).toBe('Dynamic only');
+      expect(firstContent[0]).toMatchObject({
+        type: 'text',
+        text: 'First',
+        cache_control: { type: 'ephemeral' },
+      });
+      expect(secondContent[0]).toMatchObject({
+        type: 'text',
+        text: 'Second',
+        cache_control: { type: 'ephemeral' },
+      });
+      expect(ctx.systemMessageTokens).toBeGreaterThan(0);
+      expect(ctx.dynamicInstructionTokens).toBe(0);
+      expect(ctx.instructionTokens).toBe(ctx.systemMessageTokens);
+    });
+
+    it('does not cache OpenRouter body messages after dynamic instructions', async () => {
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.OPENROUTER,
+          clientOptions: {
+            model: 'google/gemini-2.5-flash',
+            promptCache: true,
+          },
+          instructions: 'Stable instructions',
+          additional_instructions: 'Dynamic instructions',
+        },
+      });
+
+      const result = await ctx.systemRunnable!.invoke([
+        new HumanMessage('First'),
+        new HumanMessage('Second'),
+      ]);
+
+      expect(result[1].content).toBe('First');
+      expect(result[2].content).toBe('Dynamic instructions');
+      expect(result[3].content).toBe('Second');
+    });
+
+    it('keeps the first OpenRouter user message before single-turn dynamic instructions', async () => {
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.OPENROUTER,
+          clientOptions: {
+            model: 'anthropic/claude-haiku-4.5',
+            promptCache: true,
+          },
+          instructions: 'Stable instructions',
+          additional_instructions: 'Dynamic instructions',
+        },
+      });
+
+      const result = await ctx.systemRunnable!.invoke([
+        new HumanMessage('Latest'),
+      ]);
+
+      expect(result[1].content).toBe('Latest');
+      expect(result[2].content).toBe('Dynamic instructions');
+    });
+
+    it('caches stable Anthropic history before dynamic instructions', async () => {
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.ANTHROPIC,
+          clientOptions: {
+            model: 'claude-3-5-sonnet',
+            promptCache: true,
+          },
+          instructions: 'Stable instructions',
+          additional_instructions: 'Dynamic instructions',
+        },
+      });
+
+      const result = await ctx.systemRunnable!.invoke([
+        new HumanMessage('First'),
+        new AIMessage('Stable assistant history'),
+        new HumanMessage('Latest'),
+      ]);
+      const stableHistory = result[2].content as TestSystemContentBlock[];
+
+      expect(result[1].content).toBe('First');
+      expect(stableHistory[0]).toMatchObject({
+        type: 'text',
+        text: 'Stable assistant history',
+        cache_control: { type: 'ephemeral' },
+      });
+      expect(result[3].content).toBe('Dynamic instructions');
+      expect(result[4].content).toBe('Latest');
+    });
+
+    it('does not place Anthropic dynamic instructions between tool calls and results', async () => {
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.ANTHROPIC,
+          clientOptions: {
+            model: 'claude-3-5-sonnet',
+            promptCache: true,
+          },
+          instructions: 'Stable instructions',
+          additional_instructions: 'Dynamic instructions',
+        },
+      });
+
+      const result = await ctx.systemRunnable!.invoke([
+        new HumanMessage('Use the tool'),
+        new AIMessage({
+          content: '',
+          tool_calls: [
+            {
+              id: 'call_1',
+              name: 'calculator',
+              args: { expression: '2+2' },
+              type: 'tool_call',
+            },
+          ],
+        }),
+        new ToolMessage({
+          content: '4',
+          name: 'calculator',
+          tool_call_id: 'call_1',
+        }),
+      ]);
+
+      expect(result[1].content).toBe('Use the tool');
+      expect((result[2] as AIMessage).tool_calls?.[0]?.id).toBe('call_1');
+      expect(result[3].getType()).toBe('tool');
+      expect(result[4].content).toBe('Dynamic instructions');
+    });
+
+    it('caches stable OpenRouter history before dynamic instructions', async () => {
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.OPENROUTER,
+          clientOptions: {
+            model: 'anthropic/claude-haiku-4.5',
+            promptCache: true,
+          },
+          instructions: 'Stable instructions',
+          additional_instructions: 'Dynamic instructions',
+        },
+      });
+
+      const result = await ctx.systemRunnable!.invoke([
+        new HumanMessage('First'),
+        new AIMessage('Stable assistant history'),
+        new HumanMessage('Latest'),
+      ]);
+      const stableHistory = result[2].content as TestSystemContentBlock[];
+
+      expect(result[1].content).toBe('First');
+      expect(stableHistory[0]).toMatchObject({
+        type: 'text',
+        text: 'Stable assistant history',
+        cache_control: { type: 'ephemeral' },
+      });
+      expect(result[3].content).toBe('Dynamic instructions');
+      expect(result[4].content).toBe('Latest');
+    });
+
+    it('adds OpenRouter body cache points when there is no dynamic tail', async () => {
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.OPENROUTER,
+          clientOptions: {
+            model: 'google/gemini-3.1-pro-preview',
+            promptCache: true,
+          },
+          instructions: 'Stable instructions',
+        },
+      });
+
+      const result = await ctx.systemRunnable!.invoke([
+        new HumanMessage('First'),
+        new HumanMessage('Second'),
+      ]);
+      const firstContent = result[1].content as TestSystemContentBlock[];
+      const secondContent = result[2].content as TestSystemContentBlock[];
+      expect(firstContent[0]).toHaveProperty('cache_control');
+      expect(secondContent[0]).toHaveProperty('cache_control');
+    });
+
+    it('places OpenRouter user-message summaries after the first stable message', async () => {
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.OPENROUTER,
+          clientOptions: {
+            model: 'google/gemini-3.1-pro-preview',
+            promptCache: true,
+          },
+          instructions: 'Stable instructions',
+        },
+      });
+      ctx.setSummary('Rotating summary', 7);
+
+      const result = await ctx.systemRunnable!.invoke([
+        new HumanMessage('First'),
+        new HumanMessage('Second'),
+      ]);
+
+      expect(result[1].content).toBe('First');
+      expect(result[2].content).toContain('Rotating summary');
+      expect(result[3].content).toBe('Second');
+    });
+
     it('preserves the Bedrock system cache point through message cache-control pass', async () => {
       const ctx = createBasicContext({
         agentConfig: {
@@ -557,6 +841,59 @@ describe('AgentContext', () => {
       expect(ctxWithDeferred.toolSchemaTokens).toBe(ctxBase.toolSchemaTokens);
     });
 
+    it('counts OpenRouter dynamic instructions outside the system message', () => {
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.OPENROUTER,
+          clientOptions: {
+            model: 'google/gemini-3.1-pro-preview',
+            promptCache: true,
+          },
+          instructions: 'Stable',
+          additional_instructions: 'Dynamic tail',
+        },
+        tokenCounter: mockTokenCounter,
+      });
+
+      ctx.initializeSystemRunnable();
+
+      expect(ctx.systemMessageTokens).toBeGreaterThan(0);
+      expect(ctx.dynamicInstructionTokens).toBeGreaterThan(0);
+      expect(ctx.instructionTokens).toBe(
+        ctx.systemMessageTokens + ctx.dynamicInstructionTokens
+      );
+      expect(ctx.getTokenBudgetBreakdown().dynamicInstructionTokens).toBe(
+        ctx.dynamicInstructionTokens
+      );
+    });
+
+    it('clears OpenRouter dynamic instruction tokens when no prompt remains', () => {
+      const ctx = createBasicContext({
+        agentConfig: {
+          provider: Providers.OPENROUTER,
+          clientOptions: {
+            model: 'google/gemini-3.1-pro-preview',
+            promptCache: true,
+          },
+          instructions: 'Stable instructions',
+        },
+        tokenCounter: mockTokenCounter,
+      });
+
+      ctx.setInitialSummary('Volatile summary', 8);
+      ctx.initializeSystemRunnable();
+      expect(ctx.dynamicInstructionTokens).toBeGreaterThan(0);
+
+      ctx.instructions = undefined;
+      ctx.clearSummary();
+      ctx.initializeSystemRunnable();
+
+      expect(ctx.systemRunnable).toBeUndefined();
+      expect(ctx.systemMessageTokens).toBe(0);
+      expect(ctx.dynamicInstructionTokens).toBe(0);
+      expect(ctx.instructionTokens).toBe(0);
+    });
+
     it('excludes programmatic-only toolDefinitions from toolSchemaTokens', async () => {
       // getEventDrivenToolsForBinding excludes definitions whose
       // allowed_callers omit 'direct'. Accounting must mirror that — a

package/src/agents/__tests__/promptCacheLiveHelpers.ts

@@ -1,13 +1,18 @@
 import { expect } from '@jest/globals';
 import { HumanMessage } from '@langchain/core/messages';
 import type { UsageMetadata } from '@langchain/core/messages';
+import type { ClientOptions } from '@langchain/openai';
 import type * as t from '@/types';
 import { GraphEvents, Providers } from '@/common';
 import { AgentContext } from '../AgentContext';
 import { ModelEndHandler } from '@/events';
 import { Run } from '@/run';
+import type { ChatOpenRouterInput } from '@/llm/openrouter';
 
-type LivePromptCacheProvider = Providers.ANTHROPIC | Providers.BEDROCK;
+type LivePromptCacheProvider =
+  | Providers.ANTHROPIC
+  | Providers.BEDROCK
+  | Providers.OPENROUTER;
 
 type PromptCacheExpectedSystemBlock =
   | { type: 'text'; text: string; cache_control?: { type: 'ephemeral' } }
@@ -15,7 +20,8 @@ type PromptCacheExpectedSystemBlock =
 
 type LivePromptCacheClientOptions =
   | t.ClientOptions
-  | t.BedrockAnthropicClientOptions;
+  | t.BedrockAnthropicClientOptions
+  | (ChatOpenRouterInput & { configuration?: ClientOptions });
 
 export function buildStableInstructions({
   nonce,

package/src/graphs/Graph.ts

@@ -62,6 +62,7 @@ import { isThinkingEnabled } from '@/llm/request';
 import { initializeModel } from '@/llm/init';
 import { HandlerRegistry } from '@/events';
 import { ChatOpenAI } from '@/llm/openai';
+import { partitionAndMarkOpenRouterToolCache } from '@/llm/openrouter/toolCache';
 import type { HookRegistry } from '@/hooks';
 
 const { AGENT, TOOLS, SUMMARIZE } = GraphNodeKeys;
@@ -817,6 +818,19 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
             rawToolsForBinding,
             makeIsDeferred(agentContext.toolDefinitions)
           ) ?? rawToolsForBinding;
+      } else if (
+        agentContext.provider === Providers.OPENROUTER &&
+        (
+          agentContext.clientOptions as
+            | t.ProviderOptionsMap[Providers.OPENROUTER]
+            | undefined
+        )?.promptCache === true
+      ) {
+        toolsForBinding =
+          partitionAndMarkOpenRouterToolCache(
+            rawToolsForBinding,
+            makeIsDeferred(agentContext.toolDefinitions)
+          ) ?? rawToolsForBinding;
       }
 
       let model =
@@ -1073,6 +1087,16 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
         if (bedrockOptions?.promptCache === true) {
           finalMessages = addBedrockCacheControl<BaseMessage>(finalMessages);
         }
+      } else if (agentContext.provider === Providers.OPENROUTER) {
+        const openRouterOptions = agentContext.clientOptions as
+          | t.ProviderOptionsMap[Providers.OPENROUTER]
+          | undefined;
+        if (
+          openRouterOptions?.promptCache === true &&
+          !agentContext.systemRunnable
+        ) {
+          finalMessages = addCacheControl<BaseMessage>(finalMessages);
+        }
       }
 
       if (

package/src/llm/custom-chat-models.smoke.test.ts

@@ -120,6 +120,17 @@ type OpenRouterReasoningStreamChoice = Omit<
 > & {
   delta: OpenRouterReasoningStreamDelta;
 };
+type PromptTokensDetailsWithCacheWrite = NonNullable<
+  OpenAIClient.Completions.CompletionUsage['prompt_tokens_details']
+> & {
+  cache_write_tokens?: number;
+};
+type CompletionUsageWithCacheWrite = Omit<
+  OpenAIClient.Completions.CompletionUsage,
+  'prompt_tokens_details'
+> & {
+  prompt_tokens_details?: PromptTokensDetailsWithCacheWrite;
+};
 type OpenAIStreamModel = ChatOpenAI | AzureChatOpenAI;
 
 const baseAzureFields = {
@@ -654,6 +665,71 @@ describe('custom chat model class smoke tests', () => {
     ]);
   });
 
+  it('maps OpenRouter cache write usage to cache_creation in streaming responses', async () => {
+    const model = new ChatOpenRouter({
+      model: 'anthropic/claude-sonnet-test',
+      apiKey: 'test-key',
+      streamUsage: true,
+    });
+    const completions = (model as unknown as StreamingCompletionBackedModel)
+      .completions;
+    const usage: CompletionUsageWithCacheWrite = {
+      prompt_tokens: 11,
+      completion_tokens: 7,
+      total_tokens: 18,
+      prompt_tokens_details: {
+        audio_tokens: 2,
+        cached_tokens: 3,
+        cache_write_tokens: 5,
+      },
+      completion_tokens_details: {
+        audio_tokens: 4,
+        reasoning_tokens: 6,
+      },
+    };
+
+    async function* streamChunks(): AsyncGenerator<OpenAIClient.Chat.Completions.ChatCompletionChunk> {
+      yield createOpenAIStreamChunk('answer', 'stop');
+      yield {
+        id: 'chatcmpl-openrouter-usage',
+        object: 'chat.completion.chunk',
+        created: 0,
+        model: 'anthropic/claude-sonnet-test',
+        choices: [],
+        usage,
+      } as OpenAIClient.Chat.Completions.ChatCompletionChunk;
+    }
+
+    completions.completionWithRetry = async (): Promise<
+      AsyncIterable<OpenAIClient.Chat.Completions.ChatCompletionChunk>
+    > => streamChunks();
+
+    const chunks: AIMessageChunk[] = [];
+    const stream = await model.stream([new HumanMessage('hi')]);
+    for await (const chunk of stream) {
+      chunks.push(chunk);
+    }
+
+    const usageChunk = chunks.find(
+      (chunk) =>
+        chunk.usage_metadata?.input_token_details?.cache_creation === 5
+    );
+    expect(usageChunk?.usage_metadata).toEqual({
+      input_tokens: 11,
+      output_tokens: 7,
+      total_tokens: 18,
+      input_token_details: {
+        audio: 2,
+        cache_read: 3,
+        cache_creation: 5,
+      },
+      output_token_details: {
+        audio: 4,
+        reasoning: 6,
+      },
+    });
+  });
+
   it('keeps Anthropic output, residency, compaction, and stream-delay options', () => {
     const contextManagement = {
       edits: [

package/src/llm/openai/deepseek.test.ts

@@ -11,6 +11,17 @@ type DeepSeekRequest =
 type OpenAIChatCompletion = OpenAIClient.Chat.Completions.ChatCompletion;
 type OpenAIChatCompletionChunk =
   OpenAIClient.Chat.Completions.ChatCompletionChunk;
+type PromptTokensDetailsWithCacheWrite = NonNullable<
+  OpenAIClient.Completions.CompletionUsage['prompt_tokens_details']
+> & {
+  cache_write_tokens?: number;
+};
+type CompletionUsageWithCacheWrite = Omit<
+  OpenAIClient.Completions.CompletionUsage,
+  'prompt_tokens_details'
+> & {
+  prompt_tokens_details?: PromptTokensDetailsWithCacheWrite;
+};
 type ReasoningAssistantMessageParam =
   OpenAIClient.Chat.Completions.ChatCompletionAssistantMessageParam & {
     reasoning_content?: string;
@@ -129,7 +140,7 @@ async function* createCompletionStream(
 }
 
 function createCompletion(
-  usage: OpenAIClient.Completions.CompletionUsage = {
+  usage: CompletionUsageWithCacheWrite = {
     prompt_tokens: 1,
     completion_tokens: 1,
     total_tokens: 2,
@@ -392,6 +403,7 @@ describe('ChatDeepSeek', () => {
         prompt_tokens_details: {
           audio_tokens: 2,
           cached_tokens: 3,
+          cache_write_tokens: 6,
         },
         completion_tokens_details: {
           audio_tokens: 4,
@@ -409,6 +421,7 @@ describe('ChatDeepSeek', () => {
       input_token_details: {
         audio: 2,
         cache_read: 3,
+        cache_creation: 6,
       },
       output_token_details: {
         audio: 4,