illuma-agents 1.0.10 → 1.0.12

package/src/messages/cache.ts

@@ -7,6 +7,18 @@ type MessageWithContent = {
   content?: string | MessageContentComplex[];
 };
 
+/** Always-on logger for cache operations */
+const logCache = (message: string) => {
+  console.log(`[Cache] ${message}`);
+};
+
+/** Debug logger for cache operations - set ILLUMA_DEBUG_CACHE=true to enable */
+const debugCache = (message: string, data?: unknown) => {
+  if (process.env.ILLUMA_DEBUG_CACHE === 'true') {
+    console.log(`[Cache] ${message}`, data !== undefined ? JSON.stringify(data, null, 2) : '');
+  }
+};
+
 /**
  * Anthropic API: Adds cache control to the appropriate user messages in the payload.
  * Strips ALL existing cache control (both Anthropic and Bedrock formats) from all messages,
@@ -139,39 +151,54 @@ export function stripBedrockCacheControl<T extends MessageWithContent>(
 }
 
 /**
- * Adds Bedrock Converse API cache points to the last two messages.
- * Inserts `{ cachePoint: { type: 'default' } }` as a separate content block
- * immediately after the last text block in each targeted message.
- * Strips ALL existing cache control (both Bedrock and Anthropic formats) from all messages,
- * then adds fresh cache points to the last 2 messages in a single backward pass.
- * This ensures we don't accumulate stale cache points across multiple turns.
- * @param messages - The array of message objects.
- * @returns - The updated array of message objects with cache points added.
+ * Adds Bedrock Converse API cache points using "Stable Prefix Caching" strategy.
+ * 
+ * STRATEGY: Place cache point after the LAST ASSISTANT message only.
+ * This ensures the prefix (everything before the cache point) remains STABLE
+ * as the conversation grows, maximizing cache hits.
+ * 
+ * Why this works:
+ * - System message has its own cachePoint (added in AgentContext)
+ * - Tools have their own cachePoint (added in CustomChatBedrockConverse)
+ * - Conversation history grows, but the PREFIX stays the same
+ * - Only the NEW user message is uncached (it's always different)
+ * 
+ * Example conversation flow:
+ * Request 1: [System+cachePoint][Tools+cachePoint][User1] → No conversation cache yet
+ * Request 2: [System][Tools][User1][Assistant1+cachePoint][User2] → Cache User1+Assistant1
+ * Request 3: [System][Tools][User1][Assistant1][User2][Assistant2+cachePoint][User3]
+ *            → Cache reads User1+A1+User2+A2, cache writes new portion
+ * 
+ * Claude's "Simplified Cache Management" automatically looks back up to 20 content
+ * blocks from the cache checkpoint to find the longest matching prefix.
+ * 
+ * @param messages - The array of message objects (excluding system message).
+ * @returns - The updated array with a single cache point after the last assistant message.
  */
 export function addBedrockCacheControl<
   T extends Partial<BaseMessage> & MessageWithContent,
 >(messages: T[]): T[] {
-  if (!Array.isArray(messages) || messages.length < 2) {
+  if (!Array.isArray(messages) || messages.length < 1) {
+    debugCache('addBedrockCacheControl: Skipping - no messages', { count: messages?.length });
     return messages;
   }
 
+  debugCache('addBedrockCacheControl: Processing messages with stable prefix strategy', { 
+    count: messages.length 
+  });
+  
   const updatedMessages: T[] = messages.slice();
-  let messagesModified = 0;
-
-  for (let i = updatedMessages.length - 1; i >= 0; i--) {
-    const message = updatedMessages[i];
-    const isToolMessage =
-      'getType' in message &&
-      typeof message.getType === 'function' &&
-      message.getType() === 'tool';
-
+  
+  // First pass: Remove ALL existing cache points to ensure clean state
+  // This prevents accumulation of stale cache points
+  for (const message of updatedMessages) {
     const content = message.content;
-
     if (Array.isArray(content)) {
       message.content = content.filter(
         (block) => !isCachePoint(block)
       ) as typeof content;
 
+      // Also remove Anthropic-style cache_control
       for (let j = 0; j < message.content.length; j++) {
         const block = message.content[j] as Record<string, unknown>;
         if ('cache_control' in block) {
@@ -179,63 +206,138 @@ export function addBedrockCacheControl<
         }
       }
     }
+  }
 
-    if (messagesModified >= 2 || isToolMessage) {
-      continue;
+  // Helper function to check if a message contains reasoning/thinking blocks
+  const hasReasoningBlock = (message: T): boolean => {
+    const content = message.content;
+    if (!Array.isArray(content)) {
+      return false;
     }
-
-    if (typeof content === 'string' && content === '') {
-      continue;
+    for (const block of content) {
+      const type = (block as { type?: string }).type;
+      // Check for all reasoning/thinking block types:
+      // - reasoning_content: Bedrock Anthropic extended thinking
+      // - reasoning: Generic reasoning format  
+      // - thinking: Anthropic direct API thinking
+      // - redacted_thinking: Anthropic redacted thinking blocks
+      if (
+        type === 'reasoning_content' ||
+        type === 'reasoning' ||
+        type === 'thinking' ||
+        type === 'redacted_thinking'
+      ) {
+        return true;
+      }
     }
-
-    if (typeof content === 'string') {
-      message.content = [
-        { type: ContentTypes.TEXT, text: content },
-        { cachePoint: { type: 'default' } },
-      ] as MessageContentComplex[];
-      messagesModified++;
-      continue;
+    return false;
+  };
+
+  // Second pass: Find the LAST assistant message WITHOUT reasoning blocks and add a cache point there
+  // Messages with reasoning/thinking blocks cannot have cache points after them (Bedrock limitation)
+  let lastAssistantIndex = -1;
+  let skippedWithReasoning = 0;
+  
+  // Count message types for logging
+  const messageTypes: Record<string, number> = {};
+  for (const message of updatedMessages) {
+    const msgType = 'getType' in message && typeof message.getType === 'function' 
+      ? message.getType() 
+      : 'unknown';
+    messageTypes[msgType] = (messageTypes[msgType] || 0) + 1;
+  }
+  
+  for (let i = updatedMessages.length - 1; i >= 0; i--) {
+    const message = updatedMessages[i];
+    const messageType = 'getType' in message && typeof message.getType === 'function' 
+      ? message.getType() 
+      : 'unknown';
+    
+    if (messageType === 'ai') {
+      // Skip assistant messages with reasoning blocks - cache points not allowed after them
+      if (hasReasoningBlock(message)) {
+        skippedWithReasoning++;
+        debugCache('addBedrockCacheControl: Skipping assistant message with reasoning block', { index: i });
+        continue;
+      }
+      lastAssistantIndex = i;
+      break;
     }
+  }
 
-    if (Array.isArray(message.content)) {
-      let hasCacheableContent = false;
-      for (const block of message.content) {
-        if (block.type === ContentTypes.TEXT) {
-          if (typeof block.text === 'string' && block.text !== '') {
-            hasCacheableContent = true;
-            break;
-          }
-        }
-      }
+  // Log message summary
+  logCache(`📨 Messages | total=${updatedMessages.length} | ${Object.entries(messageTypes).map(([k,v]) => `${k}:${v}`).join(' ')} | skippedReasoning=${skippedWithReasoning}`);
 
-      if (!hasCacheableContent) {
-        continue;
-      }
+  // If no suitable assistant message found, skip conversation caching
+  // (System and Tools caching are still handled separately)
+  if (lastAssistantIndex === -1) {
+    logCache('📨 Messages | No suitable assistant message for cachePoint (first turn or all have reasoning)');
+    return updatedMessages;
+  }
 
-      let inserted = false;
-      for (let j = message.content.length - 1; j >= 0; j--) {
-        const block = message.content[j] as MessageContentComplex;
-        const type = (block as { type?: string }).type;
-        if (type === ContentTypes.TEXT || type === 'text') {
-          const text = (block as { text?: string }).text;
-          if (text === '' || text === undefined) {
-            continue;
-          }
-          message.content.splice(j + 1, 0, {
+  // Add cache point to the last assistant message (without reasoning blocks)
+  const assistantMessage = updatedMessages[lastAssistantIndex];
+  const content = assistantMessage.content;
+
+  if (typeof content === 'string' && content !== '') {
+    assistantMessage.content = [
+      { type: ContentTypes.TEXT, text: content },
+      { cachePoint: { type: 'default' } },
+    ] as MessageContentComplex[];
+    logCache(`📍 Message cachePoint at index ${lastAssistantIndex} (string, ${content.length} chars)`);
+    debugCache('addBedrockCacheControl: Added cachePoint to assistant message (string content)', {
+      index: lastAssistantIndex,
+      contentLength: content.length,
+    });
+  } else if (Array.isArray(assistantMessage.content) && assistantMessage.content.length > 0) {
+    // Double-check: If this message has reasoning blocks, skip adding cache point entirely
+    // This handles edge cases where the initial skip check might have missed it
+    if (hasReasoningBlock(assistantMessage)) {
+      logCache(`⚠️ Message cachePoint SKIPPED at index ${lastAssistantIndex} (has reasoning blocks)`);
+      debugCache('addBedrockCacheControl: Skipping - assistant message has reasoning blocks (safety check)', {
+        index: lastAssistantIndex,
+      });
+      return updatedMessages;
+    }
+    
+    // Find the last text block and insert cache point after it
+    let inserted = false;
+    for (let j = assistantMessage.content.length - 1; j >= 0; j--) {
+      const block = assistantMessage.content[j] as MessageContentComplex;
+      const type = (block as { type?: string }).type;
+      if (type === ContentTypes.TEXT || type === 'text') {
+        const text = (block as { text?: string }).text;
+        if (text && text !== '') {
+          assistantMessage.content.splice(j + 1, 0, {
             cachePoint: { type: 'default' },
           } as MessageContentComplex);
           inserted = true;
+          logCache(`📍 Message cachePoint at index ${lastAssistantIndex} (array, block ${j}, ${text.length} chars)`);
+          debugCache('addBedrockCacheControl: Added cachePoint after text block in assistant message', {
+            index: lastAssistantIndex,
+            textBlockIndex: j,
+            contentLength: text.length,
+          });
           break;
         }
       }
-      if (!inserted) {
-        message.content.push({
-          cachePoint: { type: 'default' },
-        } as MessageContentComplex);
-      }
-      messagesModified++;
+    }
+    
+    // If no text block found, don't append cache point as the message structure is unexpected
+    if (!inserted) {
+      const contentTypes = assistantMessage.content.map((b) => (b as { type?: string }).type);
+      logCache(`⚠️ Message cachePoint SKIPPED at index ${lastAssistantIndex} (no text block, types: ${contentTypes.join(',')})`);
+      debugCache('addBedrockCacheControl: No suitable text block found, skipping cache point', {
+        index: lastAssistantIndex,
+        contentTypes,
+      });
     }
   }
 
+  debugCache('addBedrockCacheControl: Complete - stable prefix caching applied', { 
+    lastAssistantIndex,
+    totalMessages: updatedMessages.length,
+  });
+
   return updatedMessages;
 }

package/src/messages/index.ts

@@ -4,3 +4,4 @@ export * from './prune';
 export * from './format';
 export * from './cache';
 export * from './content';
+export * from './tools';

package/src/messages/tools.ts

@@ -0,0 +1,99 @@
+// src/messages/toolDiscovery.ts
+import { AIMessageChunk, ToolMessage } from '@langchain/core/messages';
+import type { BaseMessage } from '@langchain/core/messages';
+import { Constants } from '@/common';
+import { findLastIndex } from './core';
+
+type ToolSearchArtifact = {
+  tool_references?: Array<{ tool_name: string }>;
+};
+
+/**
+ * Extracts discovered tool names from tool search results in the current turn.
+ * Only processes tool search messages after the latest AI message with tool calls.
+ *
+ * Similar pattern to formatArtifactPayload - finds relevant messages efficiently
+ * by identifying the latest AI parent and only processing subsequent tool messages.
+ *
+ * @param messages - All messages in the conversation
+ * @returns Array of discovered tool names (empty if no new discoveries)
+ */
+export function extractToolDiscoveries(messages: BaseMessage[]): string[] {
+  const lastMessage = messages[messages.length - 1];
+  if (!(lastMessage instanceof ToolMessage)) return [];
+
+  // Find the latest AIMessage with tool_calls that this tool message belongs to
+  const latestAIParentIndex = findLastIndex(
+    messages,
+    (msg) =>
+      (msg instanceof AIMessageChunk &&
+        (msg.tool_calls?.length ?? 0) > 0 &&
+        msg.tool_calls?.some((tc) => tc.id === lastMessage.tool_call_id)) ??
+      false
+  );
+
+  if (latestAIParentIndex === -1) return [];
+
+  // Collect tool_call_ids from the AI message
+  const aiMessage = messages[latestAIParentIndex] as AIMessageChunk;
+  const toolCallIds = new Set(aiMessage.tool_calls?.map((tc) => tc.id) ?? []);
+
+  // Only process tool search results after the AI message that belong to this turn
+  const discoveredNames: string[] = [];
+  for (let i = latestAIParentIndex + 1; i < messages.length; i++) {
+    const msg = messages[i];
+    if (!(msg instanceof ToolMessage)) continue;
+    if (msg.name !== Constants.TOOL_SEARCH_REGEX) continue;
+    if (!toolCallIds.has(msg.tool_call_id)) continue;
+
+    // This is a tool search result from the current turn
+    if (typeof msg.artifact === 'object' && msg.artifact != null) {
+      const artifact = msg.artifact as ToolSearchArtifact;
+      if (artifact.tool_references && artifact.tool_references.length > 0) {
+        for (const ref of artifact.tool_references) {
+          discoveredNames.push(ref.tool_name);
+        }
+      }
+    }
+  }
+
+  return discoveredNames;
+}
+
+/**
+ * Checks if the current turn has any tool search results.
+ * Quick check to avoid full extraction when not needed.
+ */
+export function hasToolSearchInCurrentTurn(messages: BaseMessage[]): boolean {
+  const lastMessage = messages[messages.length - 1];
+  if (!(lastMessage instanceof ToolMessage)) return false;
+
+  // Find the latest AIMessage with tool_calls
+  const latestAIParentIndex = findLastIndex(
+    messages,
+    (msg) =>
+      (msg instanceof AIMessageChunk &&
+        (msg.tool_calls?.length ?? 0) > 0 &&
+        msg.tool_calls?.some((tc) => tc.id === lastMessage.tool_call_id)) ??
+      false
+  );
+
+  if (latestAIParentIndex === -1) return false;
+
+  const aiMessage = messages[latestAIParentIndex] as AIMessageChunk;
+  const toolCallIds = new Set(aiMessage.tool_calls?.map((tc) => tc.id) ?? []);
+
+  // Check if any tool search results exist after the AI message
+  for (let i = latestAIParentIndex + 1; i < messages.length; i++) {
+    const msg = messages[i];
+    if (
+      msg instanceof ToolMessage &&
+      msg.name === Constants.TOOL_SEARCH_REGEX &&
+      toolCallIds.has(msg.tool_call_id)
+    ) {
+      return true;
+    }
+  }
+
+  return false;
+}

package/src/run.ts

@@ -292,15 +292,6 @@ export class Run<_T extends t.BaseGraphState> {
 
       const eventName: t.EventName = info.event;
 
-      // Debug: log tool-related events
-      if (eventName.includes('tool')) {
-        console.log('[AgentRun] Tool event received:', eventName, { 
-          hasData: !!data, 
-          dataKeys: data ? Object.keys(data) : [],
-          outputType: data?.output ? typeof data.output : 'none',
-        });
-      }
-
       /** Skip custom events as they're handled by our callback */
       if (eventName === GraphEvents.ON_CUSTOM_EVENT) {
         continue;