@librechat/agents 3.1.81 → 3.1.82

package/src/llm/custom-chat-models.smoke.test.ts

@@ -120,6 +120,17 @@ type OpenRouterReasoningStreamChoice = Omit<
 > & {
   delta: OpenRouterReasoningStreamDelta;
 };
+type PromptTokensDetailsWithCacheWrite = NonNullable<
+  OpenAIClient.Completions.CompletionUsage['prompt_tokens_details']
+> & {
+  cache_write_tokens?: number;
+};
+type CompletionUsageWithCacheWrite = Omit<
+  OpenAIClient.Completions.CompletionUsage,
+  'prompt_tokens_details'
+> & {
+  prompt_tokens_details?: PromptTokensDetailsWithCacheWrite;
+};
 type OpenAIStreamModel = ChatOpenAI | AzureChatOpenAI;
 
 const baseAzureFields = {
@@ -654,6 +665,71 @@ describe('custom chat model class smoke tests', () => {
     ]);
   });
 
+  it('maps OpenRouter cache write usage to cache_creation in streaming responses', async () => {
+    const model = new ChatOpenRouter({
+      model: 'anthropic/claude-sonnet-test',
+      apiKey: 'test-key',
+      streamUsage: true,
+    });
+    const completions = (model as unknown as StreamingCompletionBackedModel)
+      .completions;
+    const usage: CompletionUsageWithCacheWrite = {
+      prompt_tokens: 11,
+      completion_tokens: 7,
+      total_tokens: 18,
+      prompt_tokens_details: {
+        audio_tokens: 2,
+        cached_tokens: 3,
+        cache_write_tokens: 5,
+      },
+      completion_tokens_details: {
+        audio_tokens: 4,
+        reasoning_tokens: 6,
+      },
+    };
+
+    async function* streamChunks(): AsyncGenerator<OpenAIClient.Chat.Completions.ChatCompletionChunk> {
+      yield createOpenAIStreamChunk('answer', 'stop');
+      yield {
+        id: 'chatcmpl-openrouter-usage',
+        object: 'chat.completion.chunk',
+        created: 0,
+        model: 'anthropic/claude-sonnet-test',
+        choices: [],
+        usage,
+      } as OpenAIClient.Chat.Completions.ChatCompletionChunk;
+    }
+
+    completions.completionWithRetry = async (): Promise<
+      AsyncIterable<OpenAIClient.Chat.Completions.ChatCompletionChunk>
+    > => streamChunks();
+
+    const chunks: AIMessageChunk[] = [];
+    const stream = await model.stream([new HumanMessage('hi')]);
+    for await (const chunk of stream) {
+      chunks.push(chunk);
+    }
+
+    const usageChunk = chunks.find(
+      (chunk) =>
+        chunk.usage_metadata?.input_token_details?.cache_creation === 5
+    );
+    expect(usageChunk?.usage_metadata).toEqual({
+      input_tokens: 11,
+      output_tokens: 7,
+      total_tokens: 18,
+      input_token_details: {
+        audio: 2,
+        cache_read: 3,
+        cache_creation: 5,
+      },
+      output_token_details: {
+        audio: 4,
+        reasoning: 6,
+      },
+    });
+  });
+
   it('keeps Anthropic output, residency, compaction, and stream-delay options', () => {
     const contextManagement = {
       edits: [

package/src/llm/openai/deepseek.test.ts

@@ -11,6 +11,17 @@ type DeepSeekRequest =
 type OpenAIChatCompletion = OpenAIClient.Chat.Completions.ChatCompletion;
 type OpenAIChatCompletionChunk =
   OpenAIClient.Chat.Completions.ChatCompletionChunk;
+type PromptTokensDetailsWithCacheWrite = NonNullable<
+  OpenAIClient.Completions.CompletionUsage['prompt_tokens_details']
+> & {
+  cache_write_tokens?: number;
+};
+type CompletionUsageWithCacheWrite = Omit<
+  OpenAIClient.Completions.CompletionUsage,
+  'prompt_tokens_details'
+> & {
+  prompt_tokens_details?: PromptTokensDetailsWithCacheWrite;
+};
 type ReasoningAssistantMessageParam =
   OpenAIClient.Chat.Completions.ChatCompletionAssistantMessageParam & {
     reasoning_content?: string;
@@ -129,7 +140,7 @@ async function* createCompletionStream(
 }
 
 function createCompletion(
-  usage: OpenAIClient.Completions.CompletionUsage = {
+  usage: CompletionUsageWithCacheWrite = {
     prompt_tokens: 1,
     completion_tokens: 1,
     total_tokens: 2,
@@ -392,6 +403,7 @@ describe('ChatDeepSeek', () => {
         prompt_tokens_details: {
           audio_tokens: 2,
           cached_tokens: 3,
+          cache_write_tokens: 6,
         },
         completion_tokens_details: {
           audio_tokens: 4,
@@ -409,6 +421,7 @@ describe('ChatDeepSeek', () => {
       input_token_details: {
         audio: 2,
         cache_read: 3,
+        cache_creation: 6,
       },
       output_token_details: {
         audio: 4,

package/src/llm/openai/index.ts

@@ -134,6 +134,11 @@ type OpenAIChatCompletionRequest =
 type OpenAIChatCompletionResult =
   | AsyncIterable<OpenAIChatCompletionChunk>
   | OpenAIChatCompletion;
+type PromptTokensDetailsWithCacheWrite = NonNullable<
+  OpenAIClient.Completions.CompletionUsage['prompt_tokens_details']
+> & {
+  cache_write_tokens?: number;
+};
 type OpenAIChatCompletionRetry = (
   request: OpenAIChatCompletionRequest,
   requestOptions?: OpenAICoreRequestOptions
@@ -158,8 +163,12 @@ function createUsageMetadata(
   const outputTokenDetails: UsageMetadata['output_token_details'] = {};
   let hasInputTokenDetails = false;
   let hasOutputTokenDetails = false;
-  const audioInputTokens = usage.prompt_tokens_details?.audio_tokens;
-  const cachedInputTokens = usage.prompt_tokens_details?.cached_tokens;
+  const promptTokenDetails = usage.prompt_tokens_details as
+    | PromptTokensDetailsWithCacheWrite
+    | undefined;
+  const audioInputTokens = promptTokenDetails?.audio_tokens;
+  const cachedInputTokens = promptTokenDetails?.cached_tokens;
+  const cacheWriteInputTokens = promptTokenDetails?.cache_write_tokens;
   const audioOutputTokens = usage.completion_tokens_details?.audio_tokens;
   const reasoningOutputTokens =
     usage.completion_tokens_details?.reasoning_tokens;
@@ -172,6 +181,10 @@ function createUsageMetadata(
     inputTokenDetails.cache_read = cachedInputTokens;
     hasInputTokenDetails = true;
   }
+  if (cacheWriteInputTokens != null) {
+    inputTokenDetails.cache_creation = cacheWriteInputTokens;
+    hasInputTokenDetails = true;
+  }
   if (audioOutputTokens != null) {
     outputTokenDetails.audio = audioOutputTokens;
     hasOutputTokenDetails = true;
@@ -685,16 +698,23 @@ class LibreChatOpenAICompletions extends OriginalChatOpenAICompletions {
       usageMetadata.total_tokens =
         (usageMetadata.total_tokens ?? 0) + totalTokens;
     }
+    const promptTokensDetailsWithCacheWrite = promptTokensDetails as
+      | PromptTokensDetailsWithCacheWrite
+      | undefined;
     if (
-      promptTokensDetails?.audio_tokens != null ||
-      promptTokensDetails?.cached_tokens != null
+      promptTokensDetailsWithCacheWrite?.audio_tokens != null ||
+      promptTokensDetailsWithCacheWrite?.cached_tokens != null ||
+      promptTokensDetailsWithCacheWrite?.cache_write_tokens != null
     ) {
       usageMetadata.input_token_details = {
-        ...(promptTokensDetails.audio_tokens != null && {
-          audio: promptTokensDetails.audio_tokens,
+        ...(promptTokensDetailsWithCacheWrite.audio_tokens != null && {
+          audio: promptTokensDetailsWithCacheWrite.audio_tokens,
+        }),
+        ...(promptTokensDetailsWithCacheWrite.cached_tokens != null && {
+          cache_read: promptTokensDetailsWithCacheWrite.cached_tokens,
         }),
-        ...(promptTokensDetails.cached_tokens != null && {
-          cache_read: promptTokensDetails.cached_tokens,
+        ...(promptTokensDetailsWithCacheWrite.cache_write_tokens != null && {
+          cache_creation: promptTokensDetailsWithCacheWrite.cache_write_tokens,
         }),
       };
     }
@@ -846,12 +866,18 @@ class LibreChatOpenAICompletions extends OriginalChatOpenAICompletions {
       );
     }
     if (usage) {
+      const promptTokenDetails = usage.prompt_tokens_details as
+        | PromptTokensDetailsWithCacheWrite
+        | undefined;
       const inputTokenDetails = {
-        ...(usage.prompt_tokens_details?.audio_tokens != null && {
-          audio: usage.prompt_tokens_details.audio_tokens,
+        ...(promptTokenDetails?.audio_tokens != null && {
+          audio: promptTokenDetails.audio_tokens,
+        }),
+        ...(promptTokenDetails?.cached_tokens != null && {
+          cache_read: promptTokenDetails.cached_tokens,
         }),
-        ...(usage.prompt_tokens_details?.cached_tokens != null && {
-          cache_read: usage.prompt_tokens_details.cached_tokens,
+        ...(promptTokenDetails?.cache_write_tokens != null && {
+          cache_creation: promptTokenDetails.cache_write_tokens,
         }),
       };
       const outputTokenDetails = {

package/src/llm/openrouter/index.ts

@@ -29,6 +29,7 @@ export interface ChatOpenRouterCallOptions
   include_reasoning?: boolean;
   reasoning?: OpenRouterReasoning;
   modelKwargs?: OpenAIChatInput['modelKwargs'];
+  promptCache?: boolean;
 }
 
 export type ChatOpenRouterInput = Partial<
@@ -104,31 +105,45 @@ export class ChatOpenRouter extends ChatOpenAI {
   private includeReasoning?: boolean;
 
   constructor(_fields: ChatOpenRouterInput) {
+    const fieldsWithoutPromptCache: ChatOpenRouterInput = { ..._fields };
+    delete fieldsWithoutPromptCache.promptCache;
+
     const {
       include_reasoning,
       reasoning: openRouterReasoning,
       modelKwargs = {},
       ...fields
-    } = _fields;
+    } = fieldsWithoutPromptCache;
 
     // Extract reasoning from modelKwargs if provided there (e.g., from LLMConfig)
     const { reasoning: mkReasoning, ...restModelKwargs } = modelKwargs as {
       reasoning?: OpenRouterReasoning;
     } & Record<string, unknown>;
+    const mergedReasoning =
+      mkReasoning != null || openRouterReasoning != null
+        ? {
+          ...mkReasoning,
+          ...openRouterReasoning,
+        }
+        : undefined;
+    const runtimeReasoning =
+      mergedReasoning ??
+      (include_reasoning === true ? { enabled: true } : undefined);
+    const parentModelKwargs =
+      runtimeReasoning == null
+        ? restModelKwargs
+        : { ...restModelKwargs, reasoning: runtimeReasoning };
 
     super({
       ...fields,
-      modelKwargs: restModelKwargs,
+      modelKwargs: parentModelKwargs,
       includeReasoningDetails: true,
       convertReasoningDetailsToContent: true,
     });
 
     // Merge reasoning config: modelKwargs.reasoning < constructor reasoning
-    if (mkReasoning != null || openRouterReasoning != null) {
-      this.openRouterReasoning = {
-        ...mkReasoning,
-        ...openRouterReasoning,
-      };
+    if (mergedReasoning != null) {
+      this.openRouterReasoning = mergedReasoning;
     }
 
     this.includeReasoning = include_reasoning;

package/src/llm/openrouter/reasoning.test.ts

@@ -7,6 +7,17 @@ type CreateRouterOptions = Partial<
     Pick<OpenAIChatInput, 'model' | 'apiKey' | 'streamUsage'>
 >;
 
+type RuntimeInvocationParams = {
+  reasoning?: OpenRouterReasoning;
+  reasoning_effort?: string;
+};
+
+class RuntimeInspectableChatOpenRouter extends ChatOpenRouter {
+  getRuntimeInvocationParams(): RuntimeInvocationParams {
+    return this.completions.invocationParams() as RuntimeInvocationParams;
+  }
+}
+
 function createRouter(overrides: CreateRouterOptions = {}): ChatOpenRouter {
   return new ChatOpenRouter({
     model: 'openrouter/test-model',
@@ -91,6 +102,28 @@ describe('ChatOpenRouter reasoning handling', () => {
       expect(params.reasoning_effort).toBeUndefined();
     });
 
+    it('passes reasoning to the runtime completions delegate', () => {
+      const router = new RuntimeInspectableChatOpenRouter({
+        model: 'openrouter/test-model',
+        apiKey: 'test-key',
+        reasoning: { max_tokens: 1024 },
+      });
+      const params = router.getRuntimeInvocationParams();
+      expect(params.reasoning).toEqual({ max_tokens: 1024 });
+      expect(params.reasoning_effort).toBeUndefined();
+    });
+
+    it('passes legacy include_reasoning to the runtime completions delegate', () => {
+      const router = new RuntimeInspectableChatOpenRouter({
+        model: 'openrouter/test-model',
+        apiKey: 'test-key',
+        include_reasoning: true,
+      });
+      const params = router.getRuntimeInvocationParams();
+      expect(params.reasoning).toEqual({ enabled: true });
+      expect(params.reasoning_effort).toBeUndefined();
+    });
+
     it('does not include reasoning when none is configured', () => {
       const router = createRouter();
       const params = router.invocationParams();

package/src/llm/openrouter/toolCache.test.ts

@@ -0,0 +1,83 @@
+import { tool } from '@langchain/core/tools';
+import type { GraphTools } from '@/types';
+import { partitionAndMarkOpenRouterToolCache } from './toolCache';
+
+type OpenRouterTool = {
+  type: 'function';
+  function: {
+    name: string;
+    description?: string;
+    parameters?: object;
+  };
+  cache_control?: { type: 'ephemeral' };
+  defer_loading?: boolean;
+};
+
+function createOpenAITool(name: string): OpenRouterTool {
+  return {
+    type: 'function',
+    function: {
+      name,
+      description: `${name} description`,
+      parameters: {
+        type: 'object',
+        properties: {},
+      },
+    },
+  };
+}
+
+describe('partitionAndMarkOpenRouterToolCache', () => {
+  it('marks the last static OpenRouter tool before deferred tools', () => {
+    const tools = [
+      createOpenAITool('static_one'),
+      createOpenAITool('static_two'),
+      createOpenAITool('dynamic_one'),
+    ] as GraphTools;
+
+    const result = partitionAndMarkOpenRouterToolCache(
+      tools,
+      (name) => name === 'dynamic_one'
+    ) as OpenRouterTool[];
+
+    expect(result.map((entry) => entry.function.name)).toEqual([
+      'static_one',
+      'static_two',
+      'dynamic_one',
+    ]);
+    expect(result[0]).not.toHaveProperty('cache_control');
+    expect(result[1].cache_control).toEqual({ type: 'ephemeral' });
+    expect(result[2]).not.toHaveProperty('cache_control');
+  });
+
+  it('converts LangChain tools to OpenAI tools before adding cache control', () => {
+    const staticTool = tool(async () => 'static', {
+      name: 'static_tool',
+      description: 'Static tool',
+      schema: {
+        type: 'object',
+        properties: {},
+      },
+    });
+    const dynamicTool = tool(async () => 'dynamic', {
+      name: 'dynamic_tool',
+      description: 'Dynamic tool',
+      schema: {
+        type: 'object',
+        properties: {},
+      },
+    });
+
+    const result = partitionAndMarkOpenRouterToolCache(
+      [dynamicTool, staticTool] as GraphTools,
+      (name) => name === 'dynamic_tool'
+    ) as OpenRouterTool[];
+
+    expect(result.map((entry) => entry.function.name)).toEqual([
+      'static_tool',
+      'dynamic_tool',
+    ]);
+    expect(result[0].cache_control).toEqual({ type: 'ephemeral' });
+    expect(result[1]).not.toHaveProperty('cache_control');
+  });
+});

package/src/llm/openrouter/toolCache.ts

@@ -0,0 +1,89 @@
+import type { BindToolsInput } from '@langchain/core/language_models/chat_models';
+import type { OpenAIClient } from '@langchain/openai';
+import type { GraphTools } from '@/types';
+import { _convertToOpenAITool } from '@/llm/openai';
+
+const CACHE_CONTROL = { type: 'ephemeral' as const };
+
+type OpenRouterToolWithCacheControl = OpenAIClient.ChatCompletionTool & {
+  cache_control?: typeof CACHE_CONTROL;
+  defer_loading?: boolean;
+};
+
+type ToolNameCandidate = {
+  name?: unknown;
+  function?: {
+    name?: unknown;
+  };
+  defer_loading?: unknown;
+};
+
+function getToolName(tool: unknown): string | undefined {
+  const candidate = tool as ToolNameCandidate;
+  if (typeof candidate.name === 'string') {
+    return candidate.name;
+  }
+  if (typeof candidate.function?.name === 'string') {
+    return candidate.function.name;
+  }
+  return undefined;
+}
+
+function hasDeferredMarker(tool: unknown): boolean {
+  return (tool as ToolNameCandidate).defer_loading === true;
+}
+
+function toOpenRouterTool(tool: unknown): OpenRouterToolWithCacheControl {
+  const converted = _convertToOpenAITool(
+    tool as BindToolsInput
+  ) as OpenRouterToolWithCacheControl;
+
+  if (hasDeferredMarker(tool)) {
+    return { ...converted, defer_loading: true };
+  }
+
+  return converted;
+}
+
+function markCacheControl(
+  tool: OpenRouterToolWithCacheControl
+): OpenRouterToolWithCacheControl {
+  return {
+    ...tool,
+    cache_control: CACHE_CONTROL,
+  };
+}
+
+export function partitionAndMarkOpenRouterToolCache(
+  tools: GraphTools | undefined,
+  isDeferred: (toolName: string) => boolean
+): GraphTools | undefined {
+  if (tools == null || tools.length === 0) {
+    return tools;
+  }
+
+  const staticTools: OpenRouterToolWithCacheControl[] = [];
+  const deferredTools: OpenRouterToolWithCacheControl[] = [];
+
+  for (const tool of tools as readonly unknown[]) {
+    const converted = toOpenRouterTool(tool);
+    const name = getToolName(converted) ?? getToolName(tool);
+
+    if (name != null && isDeferred(name)) {
+      deferredTools.push(converted);
+      continue;
+    }
+
+    staticTools.push(converted);
+  }
+
+  if (staticTools.length === 0) {
+    return [...deferredTools] as GraphTools;
+  }
+
+  staticTools[staticTools.length - 1] = markCacheControl(
+    staticTools[staticTools.length - 1]
+  );
+
+  return [...staticTools, ...deferredTools] as GraphTools;
+}

package/src/messages/cache.test.ts

@@ -14,9 +14,14 @@ import {
   addBedrockCacheControl,
   addCacheControl,
 } from './cache';
+import { _convertMessagesToOpenAIParams } from '@/llm/openai/utils';
 import { toLangChainContent } from './langchain';
 import { ContentTypes } from '@/common/enum';
 
+type CacheControlBlock = MessageContentComplex & {
+  cache_control?: { type: 'ephemeral'; ttl?: '1h' };
+};
+
 describe('addCacheControl', () => {
   test('should add cache control to the last two user messages with array content', () => {
     const messages: AnthropicMessages = [
@@ -1483,3 +1488,125 @@ describe('LangChain message type preservation', () => {
     expect((result[1] as AIMessage).tool_calls![0].name).toBe('navigate');
   });
 });
+
+describe('OpenRouter prompt caching (reuses addCacheControl)', () => {
+  it('adds cache_control to LangChain messages for OpenRouter (same format as Anthropic)', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage({ content: [{ type: 'text', text: 'System context' }] }),
+      new AIMessage({ content: [{ type: 'text', text: 'Acknowledged' }] }),
+      new HumanMessage({ content: [{ type: 'text', text: 'User query' }] }),
+    ];
+
+    const result = addCacheControl(messages);
+
+    const firstContent = result[0].content as MessageContentComplex[];
+    const lastContent = result[2].content as MessageContentComplex[];
+
+    expect((firstContent[0] as CacheControlBlock).cache_control).toEqual({
+      type: 'ephemeral',
+    });
+    expect((lastContent[0] as CacheControlBlock).cache_control).toEqual({
+      type: 'ephemeral',
+    });
+  });
+
+  it('preserves cache_control through OpenAI message conversion used by OpenRouter', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage({
+        content: [
+          {
+            type: 'text',
+            text: 'Hello',
+            cache_control: { type: 'ephemeral' },
+          },
+        ],
+      }),
+      new AIMessage({ content: 'Hi there' }),
+      new HumanMessage({
+        content: [
+          {
+            type: 'text',
+            text: 'Follow-up',
+            cache_control: { type: 'ephemeral' },
+          },
+        ],
+      }),
+    ];
+
+    const converted = _convertMessagesToOpenAIParams(messages);
+
+    const firstUserContent = converted[0].content as CacheControlBlock[];
+    const lastUserContent = converted[2].content as CacheControlBlock[];
+
+    expect(firstUserContent[0]).toHaveProperty('cache_control');
+    expect(firstUserContent[0].cache_control).toEqual({ type: 'ephemeral' });
+    expect(lastUserContent[0]).toHaveProperty('cache_control');
+    expect(lastUserContent[0].cache_control).toEqual({ type: 'ephemeral' });
+  });
+
+  it('end-to-end: addCacheControl then convert preserves breakpoints for OpenRouter', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage({ content: 'First message with context' }),
+      new AIMessage({ content: 'Response' }),
+      new HumanMessage({ content: 'Second question' }),
+    ];
+
+    const cached = addCacheControl(messages);
+    const converted = _convertMessagesToOpenAIParams(
+      cached,
+      'anthropic/claude-sonnet-4-20250514'
+    );
+
+    const firstUser = converted[0];
+    const lastUser = converted[2];
+
+    expect(Array.isArray(firstUser.content)).toBe(true);
+    expect(
+      (firstUser.content as CacheControlBlock[])[0]
+    ).toHaveProperty('cache_control');
+
+    expect(Array.isArray(lastUser.content)).toBe(true);
+    expect(
+      (lastUser.content as CacheControlBlock[])[0]
+    ).toHaveProperty('cache_control');
+  });
+
+  it('strips Bedrock cache before applying OpenRouter/Anthropic cache', () => {
+    const messages: TestMsg[] = [
+      {
+        role: 'user',
+        content: [
+          { type: ContentTypes.TEXT, text: 'First message' },
+          { cachePoint: { type: 'default' } },
+        ],
+      },
+      {
+        role: 'assistant',
+        content: [
+          { type: ContentTypes.TEXT, text: 'Response' },
+          { cachePoint: { type: 'default' } },
+        ],
+      },
+      {
+        role: 'user',
+        content: [{ type: ContentTypes.TEXT, text: 'Follow-up' }],
+      },
+    ];
+
+    /** @ts-expect-error - Testing cross-provider compatibility */
+    const result = addCacheControl(messages);
+
+    for (const msg of result) {
+      if (Array.isArray(msg.content)) {
+        expect(
+          (msg.content as MessageContentComplex[]).some(
+            (b) => 'cachePoint' in b
+          )
+        ).toBe(false);
+      }
+    }
+
+    const lastContent = result[2].content as MessageContentComplex[];
+    expect('cache_control' in lastContent[0]).toBe(true);
+  });
+});