illuma-agents 1.0.9 → 1.0.11

package/src/llm/bedrock/index.ts

@@ -1,5 +1,6 @@
 /**
  * Optimized ChatBedrockConverse wrapper that fixes contentBlockIndex conflicts
+ * and adds prompt caching support for Bedrock Converse API.
  *
  * Bedrock sends the same contentBlockIndex for both text and tool_use content blocks,
  * causing LangChain's merge logic to fail with "field[contentBlockIndex] already exists"
@@ -9,28 +10,128 @@
  * The contentBlockIndex field is only used internally by Bedrock's streaming protocol
  * and isn't needed by application logic - the index field on tool_call_chunks serves
  * the purpose of tracking tool call ordering.
+ *
+ * PROMPT CACHING:
+ * When promptCache: true is set, this wrapper adds cachePoint markers to the tools array
+ * to enable Bedrock prompt caching for tool definitions. This allows tool schemas to be
+ * cached and reused across requests, reducing latency and costs.
+ *
+ * CACHE TOKEN EXTRACTION:
+ * LangChain AWS doesn't extract cacheReadInputTokens/cacheWriteInputTokens from Bedrock's
+ * response. This wrapper adds input_token_details to usage_metadata with cache information.
  */
 
 import { ChatBedrockConverse } from '@langchain/aws';
 import type { ChatBedrockConverseInput } from '@langchain/aws';
 import { AIMessageChunk } from '@langchain/core/messages';
-import type { BaseMessage } from '@langchain/core/messages';
+import type { BaseMessage, UsageMetadata } from '@langchain/core/messages';
 import { ChatGenerationChunk } from '@langchain/core/outputs';
 import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
 
+/** Extended input type with promptCache option */
+export interface CustomChatBedrockConverseInput extends ChatBedrockConverseInput {
+  promptCache?: boolean;
+}
+
 export class CustomChatBedrockConverse extends ChatBedrockConverse {
-  constructor(fields?: ChatBedrockConverseInput) {
+  promptCache: boolean;
+
+  constructor(fields?: CustomChatBedrockConverseInput) {
     super(fields);
+    this.promptCache = fields?.promptCache ?? false;
   }
 
   static lc_name(): string {
-    return 'IllumaBedrockConverse';
+    return 'LibreChatBedrockConverse';
   }
 
   /**
-   * Override _streamResponseChunks to strip contentBlockIndex from response_metadata
-   * This prevents LangChain's merge conflicts when the same index is used for
-   * different content types (text vs tool calls)
+   * Override invocationParams to add cachePoint to tools when promptCache is enabled.
+   * This enables Bedrock prompt caching for tool definitions.
+   * 
+   * STRATEGY: Separate cachePoints for core tools and MCP tools
+   * - Core tools (web_search, execute_code, etc.) are stable → cache first
+   * - MCP tools (have '_mcp_' in name) are dynamic → cache separately after
+   * - This allows core tools to stay cached when MCP selection changes
+   * 
+   * NOTE: Only Claude models support cachePoint - Nova and other models will reject it.
+   */
+  invocationParams(
+    options?: this['ParsedCallOptions']
+  ): ReturnType<ChatBedrockConverse['invocationParams']> {
+    const params = super.invocationParams(options);
+
+    // Add cachePoint to tools array if promptCache is enabled and tools exist
+    // Only Claude models support cachePoint - check model name
+    const modelId = this.model?.toLowerCase() ?? '';
+    const isClaudeModel = modelId.includes('claude') || modelId.includes('anthropic');
+
+    if (
+      this.promptCache &&
+      isClaudeModel &&
+      params.toolConfig?.tools &&
+      Array.isArray(params.toolConfig.tools) &&
+      params.toolConfig.tools.length > 0
+    ) {
+      // Separate core tools from MCP tools
+      // MCP tools have '_mcp_' in their name (e.g., 'search_emails_mcp_Google-Workspace')
+      const coreTools: typeof params.toolConfig.tools = [];
+      const mcpTools: typeof params.toolConfig.tools = [];
+      const coreToolNames: string[] = [];
+      const mcpToolNames: string[] = [];
+      
+      for (const tool of params.toolConfig.tools) {
+        // Check if tool has a name property with '_mcp_' pattern
+        const toolName = (tool as { toolSpec?: { name?: string } })?.toolSpec?.name ?? '';
+        if (toolName.includes('_mcp_')) {
+          mcpTools.push(tool);
+          mcpToolNames.push(toolName);
+        } else {
+          coreTools.push(tool);
+          coreToolNames.push(toolName);
+        }
+      }
+
+      // Always log cache structure (INFO level for tracking)
+      console.log(`[Cache] 🔧 Tools | Core: [${coreToolNames.join(', ')}] (${coreTools.length}) | MCP: [${mcpToolNames.join(', ')}] (${mcpTools.length})`);
+
+      // Build tools array with strategic cachePoints:
+      // [CoreTool1, CoreTool2, cachePoint] + [MCPTool1, MCPTool2, cachePoint]
+      const toolsWithCache: typeof params.toolConfig.tools = [];
+      let cachePointCount = 0;
+      
+      // Add core tools with cachePoint (if any)
+      if (coreTools.length > 0) {
+        toolsWithCache.push(...coreTools);
+        toolsWithCache.push({ cachePoint: { type: 'default' } });
+        cachePointCount++;
+      }
+      
+      // Add MCP tools with their own cachePoint (if any)
+      if (mcpTools.length > 0) {
+        toolsWithCache.push(...mcpTools);
+        toolsWithCache.push({ cachePoint: { type: 'default' } });
+        cachePointCount++;
+      }
+      
+      // If no tools at all (shouldn't happen but safety check)
+      if (toolsWithCache.length === 0) {
+        toolsWithCache.push({ cachePoint: { type: 'default' } });
+        cachePointCount++;
+      }
+      
+      console.log(`[Cache] 📍 Tool cachePoints: ${cachePointCount} | Order: [${coreToolNames.length > 0 ? 'CoreTools→CP' : ''}${mcpToolNames.length > 0 ? '→MCPTools→CP' : ''}]`);
+      
+      params.toolConfig.tools = toolsWithCache;
+    }
+
+    return params;
+  }
+
+  /**
+   * Override _streamResponseChunks to:
+   * 1. Strip contentBlockIndex from response_metadata to prevent merge conflicts
+   * 2. Extract cacheReadInputTokens/cacheWriteInputTokens and add to usage_metadata
    */
   async *_streamResponseChunks(
     messages: BaseMessage[],
@@ -50,21 +151,57 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
         (chunk.message as Partial<AIMessageChunk>).response_metadata &&
         typeof chunk.message.response_metadata === 'object'
       ) {
-        // Check if contentBlockIndex exists anywhere in response_metadata (top level or nested)
-        const hasContentBlockIndex = this.hasContentBlockIndex(
-          chunk.message.response_metadata
-        );
+        const responseMetadata = chunk.message.response_metadata as Record<string, unknown>;
+        let needsModification = false;
+        let cleanedMetadata = responseMetadata;
 
+        // Check if contentBlockIndex exists anywhere in response_metadata
+        const hasContentBlockIndex = this.hasContentBlockIndex(responseMetadata);
         if (hasContentBlockIndex) {
-          const cleanedMetadata = this.removeContentBlockIndex(
-            chunk.message.response_metadata
-          ) as Record<string, unknown>;
+          cleanedMetadata = this.removeContentBlockIndex(responseMetadata) as Record<string, unknown>;
+          needsModification = true;
+        }
+
+        // Extract cache tokens from metadata.usage (Bedrock streaming format)
+        // The metadata chunk contains usage with cacheReadInputTokens/cacheWriteInputTokens
+        const metadata = responseMetadata.metadata as Record<string, unknown> | undefined;
+        const usage = (metadata?.usage ?? responseMetadata.usage) as Record<string, unknown> | undefined;
+        
+        let enhancedUsageMetadata: UsageMetadata | undefined = chunk.message.usage_metadata;
+        
+        if (usage) {
+          const cacheRead = (usage.cacheReadInputTokens as number) ?? 0;
+          const cacheWrite = (usage.cacheWriteInputTokens as number) ?? 0;
+          const inputTokens = (usage.inputTokens as number) ?? 0;
+          const outputTokens = (usage.outputTokens as number) ?? 0;
+          
+          if (cacheRead > 0 || cacheWrite > 0) {
+            // Always log cache results for tracking
+            const cacheStatus = cacheRead > 0 && cacheWrite === 0 ? '✅ HIT' : 
+                               cacheWrite > 0 && cacheRead === 0 ? '📝 WRITE' :
+                               cacheRead > 0 && cacheWrite > 0 ? '🔄 PARTIAL' : '❌ MISS';
+            console.log(`[Cache] ${cacheStatus} | read=${cacheRead} | write=${cacheWrite} | input=${inputTokens} | output=${outputTokens}`);
+            
+            needsModification = true;
+            enhancedUsageMetadata = {
+              input_tokens: chunk.message.usage_metadata?.input_tokens ?? inputTokens,
+              output_tokens: chunk.message.usage_metadata?.output_tokens ?? outputTokens,
+              total_tokens: chunk.message.usage_metadata?.total_tokens ?? (usage.totalTokens as number) ?? 0,
+              input_token_details: {
+                cache_read: cacheRead,
+                cache_creation: cacheWrite,
+              },
+            };
+          }
+        }
 
+        if (needsModification) {
           yield new ChatGenerationChunk({
             text: chunk.text,
             message: new AIMessageChunk({
               ...chunk.message,
               response_metadata: cleanedMetadata,
+              usage_metadata: enhancedUsageMetadata,
             }),
             generationInfo: chunk.generationInfo,
           });

package/src/llm/google/index.ts

@@ -122,8 +122,8 @@ export class CustomChatGoogleGenerativeAI extends ChatGoogleGenerativeAI {
     this.streamUsage = fields.streamUsage ?? this.streamUsage;
   }
 
-  static lc_name(): 'IllumaGoogleGenerativeAI' {
-    return 'IllumaGoogleGenerativeAI';
+  static lc_name(): 'LibreChatGoogleGenerativeAI' {
+    return 'LibreChatGoogleGenerativeAI';
   }
 
   /**

package/src/llm/google/llm.spec.ts

@@ -1,6 +1,8 @@
 import { config } from 'dotenv';
 config();
-import { test } from '@jest/globals';
+import { test, jest } from '@jest/globals';
+
+jest.setTimeout(90000);
 import * as fs from 'node:fs/promises';
 import * as path from 'node:path';
 import {

package/src/llm/openai/index.ts

@@ -211,7 +211,7 @@ export class ChatOpenAI extends OriginalChatOpenAI<t.ChatOpenAICallOptions> {
     return this.client;
   }
   static lc_name(): string {
-    return 'IllumaOpenAI';
+    return 'LibreChatOpenAI';
   }
   protected _getClientOptions(
     options?: OpenAICoreRequestOptions
@@ -466,8 +466,8 @@ export class AzureChatOpenAI extends OriginalAzureChatOpenAI {
   public get exposedClient(): CustomOpenAIClient {
     return this.client;
   }
-  static lc_name(): 'IllumaAzureOpenAI' {
-    return 'IllumaAzureOpenAI';
+  static lc_name(): 'LibreChatAzureOpenAI' {
+    return 'LibreChatAzureOpenAI';
   }
   /**
    * Returns backwards compatible reasoning parameters from constructor params and call options
@@ -539,8 +539,8 @@ export class AzureChatOpenAI extends OriginalAzureChatOpenAI {
         ...params.defaultHeaders,
         'User-Agent':
           defaultHeaders['User-Agent'] != null
-            ? `${defaultHeaders['User-Agent']}: illuma-azure-openai-v2`
-            : 'illuma-azure-openai-v2',
+            ? `${defaultHeaders['User-Agent']}: librechat-azure-openai-v2`
+            : 'librechat-azure-openai-v2',
       };
 
       this.client = new CustomAzureOpenAIClient({
@@ -613,8 +613,8 @@ export class ChatDeepSeek extends OriginalChatDeepSeek {
   public get exposedClient(): CustomOpenAIClient {
     return this.client;
   }
-  static lc_name(): 'IllumaDeepSeek' {
-    return 'IllumaDeepSeek';
+  static lc_name(): 'LibreChatDeepSeek' {
+    return 'LibreChatDeepSeek';
   }
   protected _getClientOptions(
     options?: OpenAICoreRequestOptions
@@ -643,6 +643,132 @@ export class ChatDeepSeek extends OriginalChatDeepSeek {
     } as OpenAICoreRequestOptions;
     return requestOptions;
   }
+
+  async *_streamResponseChunks(
+    messages: BaseMessage[],
+    options: this['ParsedCallOptions'],
+    runManager?: CallbackManagerForLLMRun
+  ): AsyncGenerator<ChatGenerationChunk> {
+    const messagesMapped: OpenAICompletionParam[] =
+      _convertMessagesToOpenAIParams(messages, this.model, {
+        includeReasoningContent: true,
+      });
+
+    const params = {
+      ...this.invocationParams(options, {
+        streaming: true,
+      }),
+      messages: messagesMapped,
+      stream: true as const,
+    };
+    let defaultRole: OpenAIRoleEnum | undefined;
+
+    const streamIterable = await this.completionWithRetry(params, options);
+    let usage: OpenAIClient.Completions.CompletionUsage | undefined;
+    for await (const data of streamIterable) {
+      const choice = data.choices[0] as
+        | Partial<OpenAIClient.Chat.Completions.ChatCompletionChunk.Choice>
+        | undefined;
+      if (data.usage) {
+        usage = data.usage;
+      }
+      if (!choice) {
+        continue;
+      }
+
+      const { delta } = choice;
+      if (!delta) {
+        continue;
+      }
+      const chunk = this._convertOpenAIDeltaToBaseMessageChunk(
+        delta,
+        data,
+        defaultRole
+      );
+      if ('reasoning_content' in delta) {
+        chunk.additional_kwargs.reasoning_content = delta.reasoning_content;
+      }
+      defaultRole = delta.role ?? defaultRole;
+      const newTokenIndices = {
+        prompt: (options as OpenAIChatCallOptions).promptIndex ?? 0,
+        completion: choice.index ?? 0,
+      };
+      if (typeof chunk.content !== 'string') {
+        // eslint-disable-next-line no-console
+        console.log(
+          '[WARNING]: Received non-string content from OpenAI. This is currently not supported.'
+        );
+        continue;
+      }
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      const generationInfo: Record<string, any> = { ...newTokenIndices };
+      if (choice.finish_reason != null) {
+        generationInfo.finish_reason = choice.finish_reason;
+        generationInfo.system_fingerprint = data.system_fingerprint;
+        generationInfo.model_name = data.model;
+        generationInfo.service_tier = data.service_tier;
+      }
+      if (this.logprobs == true) {
+        generationInfo.logprobs = choice.logprobs;
+      }
+      const generationChunk = new ChatGenerationChunk({
+        message: chunk,
+        text: chunk.content,
+        generationInfo,
+      });
+      yield generationChunk;
+      await runManager?.handleLLMNewToken(
+        generationChunk.text || '',
+        newTokenIndices,
+        undefined,
+        undefined,
+        undefined,
+        { chunk: generationChunk }
+      );
+    }
+    if (usage) {
+      const inputTokenDetails = {
+        ...(usage.prompt_tokens_details?.audio_tokens != null && {
+          audio: usage.prompt_tokens_details.audio_tokens,
+        }),
+        ...(usage.prompt_tokens_details?.cached_tokens != null && {
+          cache_read: usage.prompt_tokens_details.cached_tokens,
+        }),
+      };
+      const outputTokenDetails = {
+        ...(usage.completion_tokens_details?.audio_tokens != null && {
+          audio: usage.completion_tokens_details.audio_tokens,
+        }),
+        ...(usage.completion_tokens_details?.reasoning_tokens != null && {
+          reasoning: usage.completion_tokens_details.reasoning_tokens,
+        }),
+      };
+      const generationChunk = new ChatGenerationChunk({
+        message: new AIMessageChunk({
+          content: '',
+          response_metadata: {
+            usage: { ...usage },
+          },
+          usage_metadata: {
+            input_tokens: usage.prompt_tokens,
+            output_tokens: usage.completion_tokens,
+            total_tokens: usage.total_tokens,
+            ...(Object.keys(inputTokenDetails).length > 0 && {
+              input_token_details: inputTokenDetails,
+            }),
+            ...(Object.keys(outputTokenDetails).length > 0 && {
+              output_token_details: outputTokenDetails,
+            }),
+          },
+        }),
+        text: '',
+      });
+      yield generationChunk;
+    }
+    if (options.signal?.aborted === true) {
+      throw new Error('AbortError');
+    }
+  }
 }
 
 /** xAI-specific usage metadata type */
@@ -688,8 +814,8 @@ export class ChatXAI extends OriginalChatXAI {
     }
   }
 
-  static lc_name(): 'IllumaXAI' {
-    return 'IllumaXAI';
+  static lc_name(): 'LibreChatXAI' {
+    return 'LibreChatXAI';
   }
 
   public get exposedClient(): CustomOpenAIClient {

package/src/llm/openai/utils/index.ts

@@ -286,10 +286,21 @@ const completionsApiContentBlockConverter: StandardContentBlockConverter<{
   },
 };
 
+/** Options for converting messages to OpenAI params */
+export interface ConvertMessagesOptions {
+  /** Include reasoning_content field for DeepSeek thinking mode with tool calls */
+  includeReasoningContent?: boolean;
+  /** Include reasoning_details field for OpenRouter/Gemini thinking mode with tool calls */
+  includeReasoningDetails?: boolean;
+  /** Convert reasoning_details to content blocks for Claude (requires content array format) */
+  convertReasoningDetailsToContent?: boolean;
+}
+
 // Used in LangSmith, export is important here
 export function _convertMessagesToOpenAIParams(
   messages: BaseMessage[],
-  model?: string
+  model?: string,
+  options?: ConvertMessagesOptions
 ): OpenAICompletionParam[] {
   // TODO: Function messages do not support array content, fix cast
   return messages.flatMap((message) => {
@@ -333,9 +344,113 @@ export function _convertMessagesToOpenAIParams(
         convertLangChainToolCallToOpenAI
       );
       completionParam.content = hasAnthropicThinkingBlock ? content : '';
+      if (
+        options?.includeReasoningContent === true &&
+        message.additional_kwargs.reasoning_content != null
+      ) {
+        completionParam.reasoning_content =
+          message.additional_kwargs.reasoning_content;
+      }
+      if (
+        options?.includeReasoningDetails === true &&
+        message.additional_kwargs.reasoning_details != null
+      ) {
+        // For Claude via OpenRouter, convert reasoning_details to content blocks
+        const isClaudeModel =
+          model?.includes('claude') === true ||
+          model?.includes('anthropic') === true;
+        if (
+          options.convertReasoningDetailsToContent === true &&
+          isClaudeModel
+        ) {
+          const reasoningDetails = message.additional_kwargs
+            .reasoning_details as Record<string, unknown>[];
+          const contentBlocks = [];
+
+          // Add thinking blocks from reasoning_details
+          for (const detail of reasoningDetails) {
+            if (detail.type === 'reasoning.text' && detail.text != null) {
+              contentBlocks.push({
+                type: 'thinking',
+                thinking: detail.text,
+              });
+            } else if (
+              detail.type === 'reasoning.encrypted' &&
+              detail.data != null
+            ) {
+              contentBlocks.push({
+                type: 'redacted_thinking',
+                data: detail.data,
+                id: detail.id,
+              });
+            }
+          }
+
+          // Set content to array with thinking blocks
+          if (contentBlocks.length > 0) {
+            completionParam.content = contentBlocks;
+          }
+        } else {
+          // For non-Claude models, pass as separate field
+          completionParam.reasoning_details =
+            message.additional_kwargs.reasoning_details;
+        }
+      }
     } else {
       if (message.additional_kwargs.tool_calls != null) {
         completionParam.tool_calls = message.additional_kwargs.tool_calls;
+        if (
+          options?.includeReasoningContent === true &&
+          message.additional_kwargs.reasoning_content != null
+        ) {
+          completionParam.reasoning_content =
+            message.additional_kwargs.reasoning_content;
+        }
+        if (
+          options?.includeReasoningDetails === true &&
+          message.additional_kwargs.reasoning_details != null
+        ) {
+          // For Claude via OpenRouter, convert reasoning_details to content blocks
+          const isClaudeModel =
+            model?.includes('claude') === true ||
+            model?.includes('anthropic') === true;
+          if (
+            options.convertReasoningDetailsToContent === true &&
+            isClaudeModel
+          ) {
+            const reasoningDetails = message.additional_kwargs
+              .reasoning_details as Record<string, unknown>[];
+            const contentBlocks = [];
+
+            // Add thinking blocks from reasoning_details
+            for (const detail of reasoningDetails) {
+              if (detail.type === 'reasoning.text' && detail.text != null) {
+                contentBlocks.push({
+                  type: 'thinking',
+                  thinking: detail.text,
+                });
+              } else if (
+                detail.type === 'reasoning.encrypted' &&
+                detail.data != null
+              ) {
+                contentBlocks.push({
+                  type: 'redacted_thinking',
+                  data: detail.data,
+                  id: detail.id,
+                });
+              }
+            }
+
+            // Set content to array with thinking blocks
+            if (contentBlocks.length > 0) {
+              completionParam.content = contentBlocks;
+            }
+          } else {
+            // For non-Claude models, pass as separate field
+            completionParam.reasoning_details =
+              message.additional_kwargs.reasoning_details;
+          }
+        }
       }
       if ((message as ToolMessage).tool_call_id != null) {
         completionParam.tool_call_id = (message as ToolMessage).tool_call_id;