@librechat/agents 3.1.81 → 3.1.83

package/src/llm/openai/index.ts

@@ -134,6 +134,11 @@ type OpenAIChatCompletionRequest =
 type OpenAIChatCompletionResult =
   | AsyncIterable<OpenAIChatCompletionChunk>
   | OpenAIChatCompletion;
+type PromptTokensDetailsWithCacheWrite = NonNullable<
+  OpenAIClient.Completions.CompletionUsage['prompt_tokens_details']
+> & {
+  cache_write_tokens?: number;
+};
 type OpenAIChatCompletionRetry = (
   request: OpenAIChatCompletionRequest,
   requestOptions?: OpenAICoreRequestOptions
@@ -158,8 +163,12 @@ function createUsageMetadata(
   const outputTokenDetails: UsageMetadata['output_token_details'] = {};
   let hasInputTokenDetails = false;
   let hasOutputTokenDetails = false;
-  const audioInputTokens = usage.prompt_tokens_details?.audio_tokens;
-  const cachedInputTokens = usage.prompt_tokens_details?.cached_tokens;
+  const promptTokenDetails = usage.prompt_tokens_details as
+    | PromptTokensDetailsWithCacheWrite
+    | undefined;
+  const audioInputTokens = promptTokenDetails?.audio_tokens;
+  const cachedInputTokens = promptTokenDetails?.cached_tokens;
+  const cacheWriteInputTokens = promptTokenDetails?.cache_write_tokens;
   const audioOutputTokens = usage.completion_tokens_details?.audio_tokens;
   const reasoningOutputTokens =
     usage.completion_tokens_details?.reasoning_tokens;
@@ -172,6 +181,10 @@ function createUsageMetadata(
     inputTokenDetails.cache_read = cachedInputTokens;
     hasInputTokenDetails = true;
   }
+  if (cacheWriteInputTokens != null) {
+    inputTokenDetails.cache_creation = cacheWriteInputTokens;
+    hasInputTokenDetails = true;
+  }
   if (audioOutputTokens != null) {
     outputTokenDetails.audio = audioOutputTokens;
     hasOutputTokenDetails = true;
@@ -685,16 +698,23 @@ class LibreChatOpenAICompletions extends OriginalChatOpenAICompletions {
       usageMetadata.total_tokens =
         (usageMetadata.total_tokens ?? 0) + totalTokens;
     }
+    const promptTokensDetailsWithCacheWrite = promptTokensDetails as
+      | PromptTokensDetailsWithCacheWrite
+      | undefined;
     if (
-      promptTokensDetails?.audio_tokens != null ||
-      promptTokensDetails?.cached_tokens != null
+      promptTokensDetailsWithCacheWrite?.audio_tokens != null ||
+      promptTokensDetailsWithCacheWrite?.cached_tokens != null ||
+      promptTokensDetailsWithCacheWrite?.cache_write_tokens != null
     ) {
       usageMetadata.input_token_details = {
-        ...(promptTokensDetails.audio_tokens != null && {
-          audio: promptTokensDetails.audio_tokens,
+        ...(promptTokensDetailsWithCacheWrite.audio_tokens != null && {
+          audio: promptTokensDetailsWithCacheWrite.audio_tokens,
+        }),
+        ...(promptTokensDetailsWithCacheWrite.cached_tokens != null && {
+          cache_read: promptTokensDetailsWithCacheWrite.cached_tokens,
         }),
-        ...(promptTokensDetails.cached_tokens != null && {
-          cache_read: promptTokensDetails.cached_tokens,
+        ...(promptTokensDetailsWithCacheWrite.cache_write_tokens != null && {
+          cache_creation: promptTokensDetailsWithCacheWrite.cache_write_tokens,
         }),
       };
     }
@@ -846,12 +866,18 @@ class LibreChatOpenAICompletions extends OriginalChatOpenAICompletions {
       );
     }
     if (usage) {
+      const promptTokenDetails = usage.prompt_tokens_details as
+        | PromptTokensDetailsWithCacheWrite
+        | undefined;
       const inputTokenDetails = {
-        ...(usage.prompt_tokens_details?.audio_tokens != null && {
-          audio: usage.prompt_tokens_details.audio_tokens,
+        ...(promptTokenDetails?.audio_tokens != null && {
+          audio: promptTokenDetails.audio_tokens,
+        }),
+        ...(promptTokenDetails?.cached_tokens != null && {
+          cache_read: promptTokenDetails.cached_tokens,
         }),
-        ...(usage.prompt_tokens_details?.cached_tokens != null && {
-          cache_read: usage.prompt_tokens_details.cached_tokens,
+        ...(promptTokenDetails?.cache_write_tokens != null && {
+          cache_creation: promptTokenDetails.cache_write_tokens,
         }),
       };
       const outputTokenDetails = {

package/src/llm/openrouter/index.ts

@@ -29,6 +29,7 @@ export interface ChatOpenRouterCallOptions
   include_reasoning?: boolean;
   reasoning?: OpenRouterReasoning;
   modelKwargs?: OpenAIChatInput['modelKwargs'];
+  promptCache?: boolean;
 }
 
 export type ChatOpenRouterInput = Partial<
@@ -104,31 +105,45 @@ export class ChatOpenRouter extends ChatOpenAI {
   private includeReasoning?: boolean;
 
   constructor(_fields: ChatOpenRouterInput) {
+    const fieldsWithoutPromptCache: ChatOpenRouterInput = { ..._fields };
+    delete fieldsWithoutPromptCache.promptCache;
+
     const {
       include_reasoning,
       reasoning: openRouterReasoning,
       modelKwargs = {},
       ...fields
-    } = _fields;
+    } = fieldsWithoutPromptCache;
 
     // Extract reasoning from modelKwargs if provided there (e.g., from LLMConfig)
     const { reasoning: mkReasoning, ...restModelKwargs } = modelKwargs as {
       reasoning?: OpenRouterReasoning;
     } & Record<string, unknown>;
+    const mergedReasoning =
+      mkReasoning != null || openRouterReasoning != null
+        ? {
+          ...mkReasoning,
+          ...openRouterReasoning,
+        }
+        : undefined;
+    const runtimeReasoning =
+      mergedReasoning ??
+      (include_reasoning === true ? { enabled: true } : undefined);
+    const parentModelKwargs =
+      runtimeReasoning == null
+        ? restModelKwargs
+        : { ...restModelKwargs, reasoning: runtimeReasoning };
 
     super({
       ...fields,
-      modelKwargs: restModelKwargs,
+      modelKwargs: parentModelKwargs,
       includeReasoningDetails: true,
       convertReasoningDetailsToContent: true,
     });
 
     // Merge reasoning config: modelKwargs.reasoning < constructor reasoning
-    if (mkReasoning != null || openRouterReasoning != null) {
-      this.openRouterReasoning = {
-        ...mkReasoning,
-        ...openRouterReasoning,
-      };
+    if (mergedReasoning != null) {
+      this.openRouterReasoning = mergedReasoning;
     }
 
     this.includeReasoning = include_reasoning;

package/src/llm/openrouter/reasoning.test.ts

@@ -7,6 +7,17 @@ type CreateRouterOptions = Partial<
     Pick<OpenAIChatInput, 'model' | 'apiKey' | 'streamUsage'>
 >;
 
+type RuntimeInvocationParams = {
+  reasoning?: OpenRouterReasoning;
+  reasoning_effort?: string;
+};
+
+class RuntimeInspectableChatOpenRouter extends ChatOpenRouter {
+  getRuntimeInvocationParams(): RuntimeInvocationParams {
+    return this.completions.invocationParams() as RuntimeInvocationParams;
+  }
+}
+
 function createRouter(overrides: CreateRouterOptions = {}): ChatOpenRouter {
   return new ChatOpenRouter({
     model: 'openrouter/test-model',
@@ -91,6 +102,28 @@ describe('ChatOpenRouter reasoning handling', () => {
       expect(params.reasoning_effort).toBeUndefined();
     });
 
+    it('passes reasoning to the runtime completions delegate', () => {
+      const router = new RuntimeInspectableChatOpenRouter({
+        model: 'openrouter/test-model',
+        apiKey: 'test-key',
+        reasoning: { max_tokens: 1024 },
+      });
+      const params = router.getRuntimeInvocationParams();
+      expect(params.reasoning).toEqual({ max_tokens: 1024 });
+      expect(params.reasoning_effort).toBeUndefined();
+    });
+
+    it('passes legacy include_reasoning to the runtime completions delegate', () => {
+      const router = new RuntimeInspectableChatOpenRouter({
+        model: 'openrouter/test-model',
+        apiKey: 'test-key',
+        include_reasoning: true,
+      });
+      const params = router.getRuntimeInvocationParams();
+      expect(params.reasoning).toEqual({ enabled: true });
+      expect(params.reasoning_effort).toBeUndefined();
+    });
+
     it('does not include reasoning when none is configured', () => {
       const router = createRouter();
       const params = router.invocationParams();

package/src/llm/openrouter/toolCache.test.ts

@@ -0,0 +1,83 @@
+import { tool } from '@langchain/core/tools';
+import type { GraphTools } from '@/types';
+import { partitionAndMarkOpenRouterToolCache } from './toolCache';
+
+type OpenRouterTool = {
+  type: 'function';
+  function: {
+    name: string;
+    description?: string;
+    parameters?: object;
+  };
+  cache_control?: { type: 'ephemeral' };
+  defer_loading?: boolean;
+};
+
+function createOpenAITool(name: string): OpenRouterTool {
+  return {
+    type: 'function',
+    function: {
+      name,
+      description: `${name} description`,
+      parameters: {
+        type: 'object',
+        properties: {},
+      },
+    },
+  };
+}
+
+describe('partitionAndMarkOpenRouterToolCache', () => {
+  it('marks the last static OpenRouter tool before deferred tools', () => {
+    const tools = [
+      createOpenAITool('static_one'),
+      createOpenAITool('static_two'),
+      createOpenAITool('dynamic_one'),
+    ] as GraphTools;
+
+    const result = partitionAndMarkOpenRouterToolCache(
+      tools,
+      (name) => name === 'dynamic_one'
+    ) as OpenRouterTool[];
+
+    expect(result.map((entry) => entry.function.name)).toEqual([
+      'static_one',
+      'static_two',
+      'dynamic_one',
+    ]);
+    expect(result[0]).not.toHaveProperty('cache_control');
+    expect(result[1].cache_control).toEqual({ type: 'ephemeral' });
+    expect(result[2]).not.toHaveProperty('cache_control');
+  });
+
+  it('converts LangChain tools to OpenAI tools before adding cache control', () => {
+    const staticTool = tool(async () => 'static', {
+      name: 'static_tool',
+      description: 'Static tool',
+      schema: {
+        type: 'object',
+        properties: {},
+      },
+    });
+    const dynamicTool = tool(async () => 'dynamic', {
+      name: 'dynamic_tool',
+      description: 'Dynamic tool',
+      schema: {
+        type: 'object',
+        properties: {},
+      },
+    });
+
+    const result = partitionAndMarkOpenRouterToolCache(
+      [dynamicTool, staticTool] as GraphTools,
+      (name) => name === 'dynamic_tool'
+    ) as OpenRouterTool[];
+
+    expect(result.map((entry) => entry.function.name)).toEqual([
+      'static_tool',
+      'dynamic_tool',
+    ]);
+    expect(result[0].cache_control).toEqual({ type: 'ephemeral' });
+    expect(result[1]).not.toHaveProperty('cache_control');
+  });
+});

package/src/llm/openrouter/toolCache.ts

@@ -0,0 +1,89 @@
+import type { BindToolsInput } from '@langchain/core/language_models/chat_models';
+import type { OpenAIClient } from '@langchain/openai';
+import type { GraphTools } from '@/types';
+import { _convertToOpenAITool } from '@/llm/openai';
+
+const CACHE_CONTROL = { type: 'ephemeral' as const };
+
+type OpenRouterToolWithCacheControl = OpenAIClient.ChatCompletionTool & {
+  cache_control?: typeof CACHE_CONTROL;
+  defer_loading?: boolean;
+};
+
+type ToolNameCandidate = {
+  name?: unknown;
+  function?: {
+    name?: unknown;
+  };
+  defer_loading?: unknown;
+};
+
+function getToolName(tool: unknown): string | undefined {
+  const candidate = tool as ToolNameCandidate;
+  if (typeof candidate.name === 'string') {
+    return candidate.name;
+  }
+  if (typeof candidate.function?.name === 'string') {
+    return candidate.function.name;
+  }
+  return undefined;
+}
+
+function hasDeferredMarker(tool: unknown): boolean {
+  return (tool as ToolNameCandidate).defer_loading === true;
+}
+
+function toOpenRouterTool(tool: unknown): OpenRouterToolWithCacheControl {
+  const converted = _convertToOpenAITool(
+    tool as BindToolsInput
+  ) as OpenRouterToolWithCacheControl;
+
+  if (hasDeferredMarker(tool)) {
+    return { ...converted, defer_loading: true };
+  }
+
+  return converted;
+}
+
+function markCacheControl(
+  tool: OpenRouterToolWithCacheControl
+): OpenRouterToolWithCacheControl {
+  return {
+    ...tool,
+    cache_control: CACHE_CONTROL,
+  };
+}
+
+export function partitionAndMarkOpenRouterToolCache(
+  tools: GraphTools | undefined,
+  isDeferred: (toolName: string) => boolean
+): GraphTools | undefined {
+  if (tools == null || tools.length === 0) {
+    return tools;
+  }
+
+  const staticTools: OpenRouterToolWithCacheControl[] = [];
+  const deferredTools: OpenRouterToolWithCacheControl[] = [];
+
+  for (const tool of tools as readonly unknown[]) {
+    const converted = toOpenRouterTool(tool);
+    const name = getToolName(converted) ?? getToolName(tool);
+
+    if (name != null && isDeferred(name)) {
+      deferredTools.push(converted);
+      continue;
+    }
+
+    staticTools.push(converted);
+  }
+
+  if (staticTools.length === 0) {
+    return [...deferredTools] as GraphTools;
+  }
+
+  staticTools[staticTools.length - 1] = markCacheControl(
+    staticTools[staticTools.length - 1]
+  );
+
+  return [...staticTools, ...deferredTools] as GraphTools;
+}

package/src/messages/cache.test.ts

@@ -14,9 +14,14 @@ import {
   addBedrockCacheControl,
   addCacheControl,
 } from './cache';
+import { _convertMessagesToOpenAIParams } from '@/llm/openai/utils';
 import { toLangChainContent } from './langchain';
 import { ContentTypes } from '@/common/enum';
 
+type CacheControlBlock = MessageContentComplex & {
+  cache_control?: { type: 'ephemeral'; ttl?: '1h' };
+};
+
 describe('addCacheControl', () => {
   test('should add cache control to the last two user messages with array content', () => {
     const messages: AnthropicMessages = [
@@ -1483,3 +1488,125 @@ describe('LangChain message type preservation', () => {
     expect((result[1] as AIMessage).tool_calls![0].name).toBe('navigate');
   });
 });
+
+describe('OpenRouter prompt caching (reuses addCacheControl)', () => {
+  it('adds cache_control to LangChain messages for OpenRouter (same format as Anthropic)', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage({ content: [{ type: 'text', text: 'System context' }] }),
+      new AIMessage({ content: [{ type: 'text', text: 'Acknowledged' }] }),
+      new HumanMessage({ content: [{ type: 'text', text: 'User query' }] }),
+    ];
+
+    const result = addCacheControl(messages);
+
+    const firstContent = result[0].content as MessageContentComplex[];
+    const lastContent = result[2].content as MessageContentComplex[];
+
+    expect((firstContent[0] as CacheControlBlock).cache_control).toEqual({
+      type: 'ephemeral',
+    });
+    expect((lastContent[0] as CacheControlBlock).cache_control).toEqual({
+      type: 'ephemeral',
+    });
+  });
+
+  it('preserves cache_control through OpenAI message conversion used by OpenRouter', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage({
+        content: [
+          {
+            type: 'text',
+            text: 'Hello',
+            cache_control: { type: 'ephemeral' },
+          },
+        ],
+      }),
+      new AIMessage({ content: 'Hi there' }),
+      new HumanMessage({
+        content: [
+          {
+            type: 'text',
+            text: 'Follow-up',
+            cache_control: { type: 'ephemeral' },
+          },
+        ],
+      }),
+    ];
+
+    const converted = _convertMessagesToOpenAIParams(messages);
+
+    const firstUserContent = converted[0].content as CacheControlBlock[];
+    const lastUserContent = converted[2].content as CacheControlBlock[];
+
+    expect(firstUserContent[0]).toHaveProperty('cache_control');
+    expect(firstUserContent[0].cache_control).toEqual({ type: 'ephemeral' });
+    expect(lastUserContent[0]).toHaveProperty('cache_control');
+    expect(lastUserContent[0].cache_control).toEqual({ type: 'ephemeral' });
+  });
+
+  it('end-to-end: addCacheControl then convert preserves breakpoints for OpenRouter', () => {
+    const messages: BaseMessage[] = [
+      new HumanMessage({ content: 'First message with context' }),
+      new AIMessage({ content: 'Response' }),
+      new HumanMessage({ content: 'Second question' }),
+    ];
+
+    const cached = addCacheControl(messages);
+    const converted = _convertMessagesToOpenAIParams(
+      cached,
+      'anthropic/claude-sonnet-4-20250514'
+    );
+
+    const firstUser = converted[0];
+    const lastUser = converted[2];
+
+    expect(Array.isArray(firstUser.content)).toBe(true);
+    expect(
+      (firstUser.content as CacheControlBlock[])[0]
+    ).toHaveProperty('cache_control');
+
+    expect(Array.isArray(lastUser.content)).toBe(true);
+    expect(
+      (lastUser.content as CacheControlBlock[])[0]
+    ).toHaveProperty('cache_control');
+  });
+
+  it('strips Bedrock cache before applying OpenRouter/Anthropic cache', () => {
+    const messages: TestMsg[] = [
+      {
+        role: 'user',
+        content: [
+          { type: ContentTypes.TEXT, text: 'First message' },
+          { cachePoint: { type: 'default' } },
+        ],
+      },
+      {
+        role: 'assistant',
+        content: [
+          { type: ContentTypes.TEXT, text: 'Response' },
+          { cachePoint: { type: 'default' } },
+        ],
+      },
+      {
+        role: 'user',
+        content: [{ type: ContentTypes.TEXT, text: 'Follow-up' }],
+      },
+    ];
+
+    /** @ts-expect-error - Testing cross-provider compatibility */
+    const result = addCacheControl(messages);
+
+    for (const msg of result) {
+      if (Array.isArray(msg.content)) {
+        expect(
+          (msg.content as MessageContentComplex[]).some(
+            (b) => 'cachePoint' in b
+          )
+        ).toBe(false);
+      }
+    }
+
+    const lastContent = result[2].content as MessageContentComplex[];
+    expect('cache_control' in lastContent[0]).toBe(true);
+  });
+});

package/src/messages/cache.ts

@@ -240,6 +240,149 @@ function isCachePoint(block: MessageContentComplex): boolean {
   return 'cachePoint' in block && !('type' in block);
 }
 
+function getMessageRole(message: MessageWithContent): string | undefined {
+  if (message instanceof BaseMessage) {
+    return message.getType();
+  }
+  if ('role' in message && typeof message.role === 'string') {
+    return message.role;
+  }
+  return undefined;
+}
+
+function isCacheableConversationMessage(message: MessageWithContent): boolean {
+  const role = getMessageRole(message);
+  return (
+    role === 'human' || role === 'user' || role === 'ai' || role === 'assistant'
+  );
+}
+
+function isAssistantConversationMessage(message: MessageWithContent): boolean {
+  const role = getMessageRole(message);
+  return role === 'ai' || role === 'assistant';
+}
+
+function hasCacheMarker(message: MessageWithContent): boolean {
+  return (
+    Array.isArray(message.content) &&
+    message.content.some((block) => 'cache_control' in block)
+  );
+}
+
+function addCacheControlToRecentMessages<
+  T extends AnthropicMessage | BaseMessage,
+>(
+  messages: T[],
+  maxCachePoints: number,
+  canUseMessage: (message: MessageWithContent) => boolean
+): T[] {
+  if (
+    !Array.isArray(messages) ||
+    messages.length === 0 ||
+    maxCachePoints <= 0
+  ) {
+    return messages;
+  }
+
+  const updatedMessages: T[] = [...messages];
+  let cachePointsAdded = 0;
+
+  for (let i = updatedMessages.length - 1; i >= 0; i--) {
+    const originalMessage = updatedMessages[i];
+    const content = originalMessage.content;
+    const hasArrayContent = Array.isArray(content);
+    const canAddCache =
+      cachePointsAdded < maxCachePoints && canUseMessage(originalMessage);
+
+    if (!canAddCache && !hasArrayContent) {
+      continue;
+    }
+
+    let workingContent: MessageContentComplex[];
+    let modified = false;
+
+    if (hasArrayContent) {
+      const src = content as MessageContentComplex[];
+      workingContent = [];
+      let lastNonEmptyTextIndex = -1;
+
+      for (let j = 0; j < src.length; j++) {
+        const block = src[j];
+        if (isCachePoint(block)) {
+          modified = true;
+          continue;
+        }
+
+        const cloned = { ...block };
+        if ('cache_control' in cloned) {
+          delete (cloned as Record<string, unknown>).cache_control;
+          modified = true;
+        }
+
+        if ('type' in cloned && cloned.type === 'text') {
+          const text = (cloned as { text?: string }).text;
+          if (text != null && text.trim() !== '') {
+            lastNonEmptyTextIndex = workingContent.length;
+          }
+        }
+        workingContent.push(cloned as MessageContentComplex);
+      }
+
+      if (canAddCache && lastNonEmptyTextIndex >= 0) {
+        (
+          workingContent[lastNonEmptyTextIndex] as Anthropic.TextBlockParam
+        ).cache_control = {
+          type: 'ephemeral',
+        };
+        cachePointsAdded++;
+        modified = true;
+      }
+
+      if (!modified) {
+        continue;
+      }
+    } else if (
+      typeof content === 'string' &&
+      content.trim() !== '' &&
+      canAddCache
+    ) {
+      workingContent = [
+        { type: 'text', text: content, cache_control: { type: 'ephemeral' } },
+      ] as unknown as MessageContentComplex[];
+      cachePointsAdded++;
+    } else {
+      continue;
+    }
+
+    updatedMessages[i] = cloneMessage(
+      originalMessage as MessageWithContent,
+      workingContent
+    ) as T;
+  }
+
+  return updatedMessages;
+}
+
+export function addCacheControlToStablePrefixMessages<
+  T extends AnthropicMessage | BaseMessage,
+>(messages: T[], maxCachePoints: number): T[] {
+  const assistantMarked = addCacheControlToRecentMessages(
+    messages,
+    maxCachePoints,
+    isAssistantConversationMessage
+  );
+
+  if (assistantMarked.some(hasCacheMarker)) {
+    return assistantMarked;
+  }
+
+  return addCacheControlToRecentMessages(
+    messages,
+    maxCachePoints,
+    isCacheableConversationMessage
+  );
+}
+
 /**
  * Checks if a message's content has Anthropic cache_control fields.
  */