illuma-agents 1.0.10 → 1.0.11

package/src/llm/bedrock/index.ts

@@ -1,5 +1,6 @@
 /**
  * Optimized ChatBedrockConverse wrapper that fixes contentBlockIndex conflicts
+ * and adds prompt caching support for Bedrock Converse API.
  *
  * Bedrock sends the same contentBlockIndex for both text and tool_use content blocks,
  * causing LangChain's merge logic to fail with "field[contentBlockIndex] already exists"
@@ -9,28 +10,128 @@
  * The contentBlockIndex field is only used internally by Bedrock's streaming protocol
  * and isn't needed by application logic - the index field on tool_call_chunks serves
  * the purpose of tracking tool call ordering.
+ *
+ * PROMPT CACHING:
+ * When promptCache: true is set, this wrapper adds cachePoint markers to the tools array
+ * to enable Bedrock prompt caching for tool definitions. This allows tool schemas to be
+ * cached and reused across requests, reducing latency and costs.
+ *
+ * CACHE TOKEN EXTRACTION:
+ * LangChain AWS doesn't extract cacheReadInputTokens/cacheWriteInputTokens from Bedrock's
+ * response. This wrapper adds input_token_details to usage_metadata with cache information.
  */
 
 import { ChatBedrockConverse } from '@langchain/aws';
 import type { ChatBedrockConverseInput } from '@langchain/aws';
 import { AIMessageChunk } from '@langchain/core/messages';
-import type { BaseMessage } from '@langchain/core/messages';
+import type { BaseMessage, UsageMetadata } from '@langchain/core/messages';
 import { ChatGenerationChunk } from '@langchain/core/outputs';
 import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
 
+/** Extended input type with promptCache option */
+export interface CustomChatBedrockConverseInput extends ChatBedrockConverseInput {
+  promptCache?: boolean;
+}
+
 export class CustomChatBedrockConverse extends ChatBedrockConverse {
-  constructor(fields?: ChatBedrockConverseInput) {
+  promptCache: boolean;
+
+  constructor(fields?: CustomChatBedrockConverseInput) {
     super(fields);
+    this.promptCache = fields?.promptCache ?? false;
   }
 
   static lc_name(): string {
-    return 'IllumaBedrockConverse';
+    return 'LibreChatBedrockConverse';
   }
 
   /**
-   * Override _streamResponseChunks to strip contentBlockIndex from response_metadata
-   * This prevents LangChain's merge conflicts when the same index is used for
-   * different content types (text vs tool calls)
+   * Override invocationParams to add cachePoint to tools when promptCache is enabled.
+   * This enables Bedrock prompt caching for tool definitions.
+   * 
+   * STRATEGY: Separate cachePoints for core tools and MCP tools
+   * - Core tools (web_search, execute_code, etc.) are stable → cache first
+   * - MCP tools (have '_mcp_' in name) are dynamic → cache separately after
+   * - This allows core tools to stay cached when MCP selection changes
+   * 
+   * NOTE: Only Claude models support cachePoint - Nova and other models will reject it.
+   */
+  invocationParams(
+    options?: this['ParsedCallOptions']
+  ): ReturnType<ChatBedrockConverse['invocationParams']> {
+    const params = super.invocationParams(options);
+
+    // Add cachePoint to tools array if promptCache is enabled and tools exist
+    // Only Claude models support cachePoint - check model name
+    const modelId = this.model?.toLowerCase() ?? '';
+    const isClaudeModel = modelId.includes('claude') || modelId.includes('anthropic');
+
+    if (
+      this.promptCache &&
+      isClaudeModel &&
+      params.toolConfig?.tools &&
+      Array.isArray(params.toolConfig.tools) &&
+      params.toolConfig.tools.length > 0
+    ) {
+      // Separate core tools from MCP tools
+      // MCP tools have '_mcp_' in their name (e.g., 'search_emails_mcp_Google-Workspace')
+      const coreTools: typeof params.toolConfig.tools = [];
+      const mcpTools: typeof params.toolConfig.tools = [];
+      const coreToolNames: string[] = [];
+      const mcpToolNames: string[] = [];
+      
+      for (const tool of params.toolConfig.tools) {
+        // Check if tool has a name property with '_mcp_' pattern
+        const toolName = (tool as { toolSpec?: { name?: string } })?.toolSpec?.name ?? '';
+        if (toolName.includes('_mcp_')) {
+          mcpTools.push(tool);
+          mcpToolNames.push(toolName);
+        } else {
+          coreTools.push(tool);
+          coreToolNames.push(toolName);
+        }
+      }
+
+      // Always log cache structure (INFO level for tracking)
+      console.log(`[Cache] 🔧 Tools | Core: [${coreToolNames.join(', ')}] (${coreTools.length}) | MCP: [${mcpToolNames.join(', ')}] (${mcpTools.length})`);
+
+      // Build tools array with strategic cachePoints:
+      // [CoreTool1, CoreTool2, cachePoint] + [MCPTool1, MCPTool2, cachePoint]
+      const toolsWithCache: typeof params.toolConfig.tools = [];
+      let cachePointCount = 0;
+      
+      // Add core tools with cachePoint (if any)
+      if (coreTools.length > 0) {
+        toolsWithCache.push(...coreTools);
+        toolsWithCache.push({ cachePoint: { type: 'default' } });
+        cachePointCount++;
+      }
+      
+      // Add MCP tools with their own cachePoint (if any)
+      if (mcpTools.length > 0) {
+        toolsWithCache.push(...mcpTools);
+        toolsWithCache.push({ cachePoint: { type: 'default' } });
+        cachePointCount++;
+      }
+      
+      // If no tools at all (shouldn't happen but safety check)
+      if (toolsWithCache.length === 0) {
+        toolsWithCache.push({ cachePoint: { type: 'default' } });
+        cachePointCount++;
+      }
+      
+      console.log(`[Cache] 📍 Tool cachePoints: ${cachePointCount} | Order: [${coreToolNames.length > 0 ? 'CoreTools→CP' : ''}${mcpToolNames.length > 0 ? '→MCPTools→CP' : ''}]`);
+      
+      params.toolConfig.tools = toolsWithCache;
+    }
+
+    return params;
+  }
+
+  /**
+   * Override _streamResponseChunks to:
+   * 1. Strip contentBlockIndex from response_metadata to prevent merge conflicts
+   * 2. Extract cacheReadInputTokens/cacheWriteInputTokens and add to usage_metadata
    */
   async *_streamResponseChunks(
     messages: BaseMessage[],
@@ -50,21 +151,57 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
         (chunk.message as Partial<AIMessageChunk>).response_metadata &&
         typeof chunk.message.response_metadata === 'object'
       ) {
-        // Check if contentBlockIndex exists anywhere in response_metadata (top level or nested)
-        const hasContentBlockIndex = this.hasContentBlockIndex(
-          chunk.message.response_metadata
-        );
+        const responseMetadata = chunk.message.response_metadata as Record<string, unknown>;
+        let needsModification = false;
+        let cleanedMetadata = responseMetadata;
 
+        // Check if contentBlockIndex exists anywhere in response_metadata
+        const hasContentBlockIndex = this.hasContentBlockIndex(responseMetadata);
         if (hasContentBlockIndex) {
-          const cleanedMetadata = this.removeContentBlockIndex(
-            chunk.message.response_metadata
-          ) as Record<string, unknown>;
+          cleanedMetadata = this.removeContentBlockIndex(responseMetadata) as Record<string, unknown>;
+          needsModification = true;
+        }
+
+        // Extract cache tokens from metadata.usage (Bedrock streaming format)
+        // The metadata chunk contains usage with cacheReadInputTokens/cacheWriteInputTokens
+        const metadata = responseMetadata.metadata as Record<string, unknown> | undefined;
+        const usage = (metadata?.usage ?? responseMetadata.usage) as Record<string, unknown> | undefined;
+        
+        let enhancedUsageMetadata: UsageMetadata | undefined = chunk.message.usage_metadata;
+        
+        if (usage) {
+          const cacheRead = (usage.cacheReadInputTokens as number) ?? 0;
+          const cacheWrite = (usage.cacheWriteInputTokens as number) ?? 0;
+          const inputTokens = (usage.inputTokens as number) ?? 0;
+          const outputTokens = (usage.outputTokens as number) ?? 0;
+          
+          if (cacheRead > 0 || cacheWrite > 0) {
+            // Always log cache results for tracking
+            const cacheStatus = cacheRead > 0 && cacheWrite === 0 ? '✅ HIT' : 
+                               cacheWrite > 0 && cacheRead === 0 ? '📝 WRITE' :
+                               cacheRead > 0 && cacheWrite > 0 ? '🔄 PARTIAL' : '❌ MISS';
+            console.log(`[Cache] ${cacheStatus} | read=${cacheRead} | write=${cacheWrite} | input=${inputTokens} | output=${outputTokens}`);
+            
+            needsModification = true;
+            enhancedUsageMetadata = {
+              input_tokens: chunk.message.usage_metadata?.input_tokens ?? inputTokens,
+              output_tokens: chunk.message.usage_metadata?.output_tokens ?? outputTokens,
+              total_tokens: chunk.message.usage_metadata?.total_tokens ?? (usage.totalTokens as number) ?? 0,
+              input_token_details: {
+                cache_read: cacheRead,
+                cache_creation: cacheWrite,
+              },
+            };
+          }
+        }
 
+        if (needsModification) {
           yield new ChatGenerationChunk({
             text: chunk.text,
             message: new AIMessageChunk({
               ...chunk.message,
               response_metadata: cleanedMetadata,
+              usage_metadata: enhancedUsageMetadata,
             }),
             generationInfo: chunk.generationInfo,
           });

package/src/llm/google/index.ts

@@ -122,8 +122,8 @@ export class CustomChatGoogleGenerativeAI extends ChatGoogleGenerativeAI {
     this.streamUsage = fields.streamUsage ?? this.streamUsage;
   }
 
-  static lc_name(): 'IllumaGoogleGenerativeAI' {
-    return 'IllumaGoogleGenerativeAI';
+  static lc_name(): 'LibreChatGoogleGenerativeAI' {
+    return 'LibreChatGoogleGenerativeAI';
   }
 
   /**

package/src/llm/openai/index.ts

@@ -211,7 +211,7 @@ export class ChatOpenAI extends OriginalChatOpenAI<t.ChatOpenAICallOptions> {
     return this.client;
   }
   static lc_name(): string {
-    return 'IllumaOpenAI';
+    return 'LibreChatOpenAI';
   }
   protected _getClientOptions(
     options?: OpenAICoreRequestOptions
@@ -466,8 +466,8 @@ export class AzureChatOpenAI extends OriginalAzureChatOpenAI {
   public get exposedClient(): CustomOpenAIClient {
     return this.client;
   }
-  static lc_name(): 'IllumaAzureOpenAI' {
-    return 'IllumaAzureOpenAI';
+  static lc_name(): 'LibreChatAzureOpenAI' {
+    return 'LibreChatAzureOpenAI';
   }
   /**
    * Returns backwards compatible reasoning parameters from constructor params and call options
@@ -539,8 +539,8 @@ export class AzureChatOpenAI extends OriginalAzureChatOpenAI {
         ...params.defaultHeaders,
         'User-Agent':
           defaultHeaders['User-Agent'] != null
-            ? `${defaultHeaders['User-Agent']}: illuma-azure-openai-v2`
-            : 'illuma-azure-openai-v2',
+            ? `${defaultHeaders['User-Agent']}: librechat-azure-openai-v2`
+            : 'librechat-azure-openai-v2',
       };
 
       this.client = new CustomAzureOpenAIClient({
@@ -613,8 +613,8 @@ export class ChatDeepSeek extends OriginalChatDeepSeek {
   public get exposedClient(): CustomOpenAIClient {
     return this.client;
   }
-  static lc_name(): 'IllumaDeepSeek' {
-    return 'IllumaDeepSeek';
+  static lc_name(): 'LibreChatDeepSeek' {
+    return 'LibreChatDeepSeek';
   }
   protected _getClientOptions(
     options?: OpenAICoreRequestOptions
@@ -814,8 +814,8 @@ export class ChatXAI extends OriginalChatXAI {
     }
   }
 
-  static lc_name(): 'IllumaXAI' {
-    return 'IllumaXAI';
+  static lc_name(): 'LibreChatXAI' {
+    return 'LibreChatXAI';
   }
 
   public get exposedClient(): CustomOpenAIClient {

package/src/llm/openrouter/index.ts

@@ -44,8 +44,8 @@ export class ChatOpenRouter extends ChatOpenAI {
       },
     });
   }
-  static lc_name(): 'IllumaOpenRouter' {
-    return 'IllumaOpenRouter';
+  static lc_name(): 'LibreChatOpenRouter' {
+    return 'LibreChatOpenRouter';
   }
   protected override _convertOpenAIDeltaToBaseMessageChunk(
     // eslint-disable-next-line @typescript-eslint/no-explicit-any