@illuma-ai/agents 1.0.81

package/src/messages/cache.ts

@@ -0,0 +1,518 @@
+import { BaseMessage, MessageContentComplex } from '@langchain/core/messages';
+import type { AnthropicMessage } from '@/types/messages';
+import type Anthropic from '@anthropic-ai/sdk';
+import { ContentTypes } from '@/common/enum';
+
+type MessageWithContent = {
+  content?: string | MessageContentComplex[];
+};
+
+/** Debug logger for cache operations - set ILLUMA_DEBUG_CACHE=true to enable */
+const debugCache = (message: string, data?: unknown): void => {
+  if (process.env.ILLUMA_DEBUG_CACHE === 'true') {
+    // eslint-disable-next-line no-console
+    console.log(
+      `[Cache] ${message}`,
+      data !== undefined ? JSON.stringify(data, null, 2) : ''
+    );
+  }
+};
+
+/**
+ * Deep clones a message's content to prevent mutation of the original.
+ */
+function deepCloneContent<T extends string | MessageContentComplex[]>(
+  content: T
+): T {
+  if (typeof content === 'string') {
+    return content;
+  }
+  if (Array.isArray(content)) {
+    return content.map((block) => ({ ...block })) as T;
+  }
+  return content;
+}
+
+/**
+ * Clones a message with deep-cloned content, explicitly excluding LangChain
+ * serialization metadata to prevent coercion issues.
+ * Keeps lc_kwargs in sync with content to prevent LangChain serialization issues.
+ */
+function cloneMessage<T extends MessageWithContent>(
+  message: T,
+  content: string | MessageContentComplex[]
+): T {
+  const {
+    lc_kwargs: _lc_kwargs,
+    lc_serializable: _lc_serializable,
+    lc_namespace: _lc_namespace,
+    ...rest
+  } = message as T & {
+    lc_kwargs?: unknown;
+    lc_serializable?: unknown;
+    lc_namespace?: unknown;
+  };
+
+  const cloned = { ...rest, content } as T;
+
+  // Sync lc_kwargs.content with the new content to prevent LangChain coercion issues
+  const lcKwargs = (message as Record<string, unknown>).lc_kwargs as
+    | Record<string, unknown>
+    | undefined;
+  if (lcKwargs != null) {
+    (cloned as Record<string, unknown>).lc_kwargs = {
+      ...lcKwargs,
+      content: content,
+    };
+  }
+
+  // LangChain messages don't have a direct 'role' property - derive it from getType()
+  if (
+    'getType' in message &&
+    typeof message.getType === 'function' &&
+    !('role' in cloned)
+  ) {
+    const msgType = (message as unknown as BaseMessage).getType();
+    const roleMap: Record<string, string> = {
+      human: 'user',
+      ai: 'assistant',
+      system: 'system',
+      tool: 'tool',
+    };
+    (cloned as Record<string, unknown>).role = roleMap[msgType] || msgType;
+  }
+
+  return cloned;
+}
+
+/**
+ * Checks if a content block is a cache point
+ */
+function isCachePoint(block: MessageContentComplex): boolean {
+  return 'cachePoint' in block && !('type' in block);
+}
+
+/**
+ * Checks if a message's content needs cache control stripping.
+ * Returns true if content has cachePoint blocks or cache_control fields.
+ */
+function needsCacheStripping(content: MessageContentComplex[]): boolean {
+  for (let i = 0; i < content.length; i++) {
+    const block = content[i];
+    if (isCachePoint(block)) return true;
+    if ('cache_control' in block) return true;
+  }
+  return false;
+}
+
+/**
+ * Checks if a message's content has Anthropic cache_control fields.
+ */
+function hasAnthropicCacheControl(content: MessageContentComplex[]): boolean {
+  for (let i = 0; i < content.length; i++) {
+    if ('cache_control' in content[i]) return true;
+  }
+  return false;
+}
+
+/**
+ * Checks if a message's content has Bedrock cachePoint blocks.
+ */
+function hasBedrockCachePoint(content: MessageContentComplex[]): boolean {
+  for (let i = 0; i < content.length; i++) {
+    if (isCachePoint(content[i])) return true;
+  }
+  return false;
+}
+
+/**
+ * Anthropic API: Adds cache control to the appropriate user messages in the payload.
+ * Strips ALL existing cache control (both Anthropic and Bedrock formats) from all messages,
+ * then adds fresh cache control to the last 2 user messages in a single backward pass.
+ * This ensures we don't accumulate stale cache points across multiple turns.
+ * Returns a new array - only clones messages that require modification.
+ * @param messages - The array of message objects.
+ * @returns - A new array of message objects with cache control added.
+ */
+export function addCacheControl<T extends AnthropicMessage | BaseMessage>(
+  messages: T[]
+): T[] {
+  if (!Array.isArray(messages) || messages.length < 2) {
+    return messages;
+  }
+
+  const updatedMessages: T[] = [...messages];
+  let userMessagesModified = 0;
+
+  for (let i = updatedMessages.length - 1; i >= 0; i--) {
+    const originalMessage = updatedMessages[i];
+    const content = originalMessage.content;
+    const isUserMessage =
+      ('getType' in originalMessage && originalMessage.getType() === 'human') ||
+      ('role' in originalMessage && originalMessage.role === 'user');
+
+    const hasArrayContent = Array.isArray(content);
+    const needsStripping =
+      hasArrayContent &&
+      needsCacheStripping(content as MessageContentComplex[]);
+    const needsCacheAdd =
+      userMessagesModified < 2 &&
+      isUserMessage &&
+      (typeof content === 'string' || hasArrayContent);
+
+    if (!needsStripping && !needsCacheAdd) {
+      continue;
+    }
+
+    let workingContent: MessageContentComplex[];
+
+    if (hasArrayContent) {
+      workingContent = deepCloneContent(
+        content as MessageContentComplex[]
+      ).filter((block) => !isCachePoint(block as MessageContentComplex));
+
+      for (let j = 0; j < workingContent.length; j++) {
+        const block = workingContent[j] as Record<string, unknown>;
+        if ('cache_control' in block) {
+          delete block.cache_control;
+        }
+      }
+    } else if (typeof content === 'string') {
+      workingContent = [
+        { type: 'text', text: content },
+      ] as MessageContentComplex[];
+    } else {
+      workingContent = [];
+    }
+
+    if (userMessagesModified >= 2 || !isUserMessage) {
+      updatedMessages[i] = cloneMessage(
+        originalMessage as MessageWithContent,
+        workingContent
+      ) as T;
+      continue;
+    }
+
+    for (let j = workingContent.length - 1; j >= 0; j--) {
+      const contentPart = workingContent[j];
+      if ('type' in contentPart && contentPart.type === 'text') {
+        (contentPart as Anthropic.TextBlockParam).cache_control = {
+          type: 'ephemeral',
+        };
+        userMessagesModified++;
+        break;
+      }
+    }
+
+    updatedMessages[i] = cloneMessage(
+      originalMessage as MessageWithContent,
+      workingContent
+    ) as T;
+  }
+
+  return updatedMessages;
+}
+
+/**
+ * Removes all Anthropic cache_control fields from messages
+ * Used when switching from Anthropic to Bedrock provider
+ * Returns a new array - only clones messages that require modification.
+ */
+export function stripAnthropicCacheControl<T extends MessageWithContent>(
+  messages: T[]
+): T[] {
+  if (!Array.isArray(messages)) {
+    return messages;
+  }
+
+  const updatedMessages: T[] = [...messages];
+
+  for (let i = 0; i < updatedMessages.length; i++) {
+    const originalMessage = updatedMessages[i];
+    const content = originalMessage.content;
+
+    if (!Array.isArray(content) || !hasAnthropicCacheControl(content)) {
+      continue;
+    }
+
+    const clonedContent = deepCloneContent(content);
+    for (let j = 0; j < clonedContent.length; j++) {
+      const block = clonedContent[j] as Record<string, unknown>;
+      if ('cache_control' in block) {
+        delete block.cache_control;
+      }
+    }
+    updatedMessages[i] = cloneMessage(originalMessage, clonedContent) as T;
+  }
+
+  return updatedMessages;
+}
+
+/**
+ * Removes all Bedrock cachePoint blocks from messages
+ * Used when switching from Bedrock to Anthropic provider
+ * Returns a new array - only clones messages that require modification.
+ */
+export function stripBedrockCacheControl<T extends MessageWithContent>(
+  messages: T[]
+): T[] {
+  if (!Array.isArray(messages)) {
+    return messages;
+  }
+
+  const updatedMessages: T[] = [...messages];
+
+  for (let i = 0; i < updatedMessages.length; i++) {
+    const originalMessage = updatedMessages[i];
+    const content = originalMessage.content;
+
+    if (!Array.isArray(content) || !hasBedrockCachePoint(content)) {
+      continue;
+    }
+
+    const clonedContent = deepCloneContent(content).filter(
+      (block) => !isCachePoint(block as MessageContentComplex)
+    );
+    updatedMessages[i] = cloneMessage(originalMessage, clonedContent) as T;
+  }
+
+  return updatedMessages;
+}
+
+/**
+ * Adds Bedrock Converse API cache points using "Stable Prefix Caching" strategy.
+ *
+ * STRATEGY: Place cache point after the LAST ASSISTANT message only.
+ * This ensures the prefix (everything before the cache point) remains STABLE
+ * as the conversation grows, maximizing cache hits.
+ *
+ * Why this works:
+ * - System message has its own cachePoint (added in AgentContext)
+ * - Tools have their own cachePoint (added in CustomChatBedrockConverse)
+ * - Conversation history grows, but the PREFIX stays the same
+ * - Only the NEW user message is uncached (it's always different)
+ *
+ * Example conversation flow:
+ * Request 1: [System+cachePoint][Tools+cachePoint][User1] → No conversation cache yet
+ * Request 2: [System][Tools][User1][Assistant1+cachePoint][User2] → Cache User1+Assistant1
+ * Request 3: [System][Tools][User1][Assistant1][User2][Assistant2+cachePoint][User3]
+ *            → Cache reads User1+A1+User2+A2, cache writes new portion
+ *
+ * Claude's "Simplified Cache Management" automatically looks back up to 20 content
+ * blocks from the cache checkpoint to find the longest matching prefix.
+ *
+ * @param messages - The array of message objects (excluding system message).
+ * @returns - The updated array with a single cache point after the last assistant message.
+ */
+export function addBedrockCacheControl<
+  T extends Partial<BaseMessage> & MessageWithContent,
+>(messages: T[]): T[] {
+  if (!Array.isArray(messages) || messages.length < 1) {
+    debugCache('addBedrockCacheControl: Skipping - no messages', {
+      count: messages.length,
+    });
+    return messages;
+  }
+
+  debugCache(
+    'addBedrockCacheControl: Processing messages with stable prefix strategy',
+    {
+      count: messages.length,
+    }
+  );
+
+  const updatedMessages: T[] = messages.slice();
+
+  // First pass: Remove ALL existing cache points to ensure clean state
+  // This prevents accumulation of stale cache points
+  for (const message of updatedMessages) {
+    const content = message.content;
+    if (Array.isArray(content)) {
+      message.content = content.filter(
+        (block) => !isCachePoint(block)
+      ) as typeof content;
+
+      // Also remove Anthropic-style cache_control
+      for (let j = 0; j < message.content.length; j++) {
+        const block = message.content[j] as Record<string, unknown>;
+        if ('cache_control' in block) {
+          delete block.cache_control;
+        }
+      }
+    }
+  }
+
+  // Helper function to check if a message contains reasoning/thinking blocks
+  const hasReasoningBlock = (message: T): boolean => {
+    const content = message.content;
+    if (!Array.isArray(content)) {
+      return false;
+    }
+    for (const block of content) {
+      const type = (block as { type?: string }).type;
+      // Check for all reasoning/thinking block types:
+      // - reasoning_content: Bedrock Anthropic extended thinking
+      // - reasoning: Generic reasoning format
+      // - thinking: Anthropic direct API thinking
+      // - redacted_thinking: Anthropic redacted thinking blocks
+      if (
+        type === 'reasoning_content' ||
+        type === 'reasoning' ||
+        type === 'thinking' ||
+        type === 'redacted_thinking'
+      ) {
+        return true;
+      }
+    }
+    return false;
+  };
+
+  // Second pass: Find the LAST assistant message WITHOUT reasoning blocks and add a cache point there
+  // Messages with reasoning/thinking blocks cannot have cache points after them (Bedrock limitation)
+  let lastAssistantIndex = -1;
+  let skippedWithReasoning = 0;
+
+  // Count message types for logging
+  const messageTypes: Record<string, number> = {};
+  for (const message of updatedMessages) {
+    const msgType =
+      'getType' in message && typeof message.getType === 'function'
+        ? message.getType()
+        : 'unknown';
+    messageTypes[msgType] = (messageTypes[msgType] || 0) + 1;
+  }
+
+  for (let i = updatedMessages.length - 1; i >= 0; i--) {
+    const message = updatedMessages[i];
+    const messageType =
+      'getType' in message && typeof message.getType === 'function'
+        ? message.getType()
+        : 'unknown';
+
+    if (messageType === 'ai') {
+      // Skip assistant messages with reasoning blocks - cache points not allowed after them
+      if (hasReasoningBlock(message)) {
+        skippedWithReasoning++;
+        debugCache(
+          'addBedrockCacheControl: Skipping assistant message with reasoning block',
+          { index: i }
+        );
+        continue;
+      }
+      lastAssistantIndex = i;
+      break;
+    }
+  }
+
+  // Log message summary
+  debugCache(
+    `📨 Messages | total=${updatedMessages.length} | ${Object.entries(
+      messageTypes
+    )
+      .map(([k, v]) => `${k}:${v}`)
+      .join(' ')} | skippedReasoning=${skippedWithReasoning}`
+  );
+
+  // If no suitable assistant message found, skip conversation caching
+  // (System and Tools caching are still handled separately)
+  if (lastAssistantIndex === -1) {
+    debugCache(
+      '📨 Messages | No suitable assistant message for cachePoint (first turn or all have reasoning)'
+    );
+    return updatedMessages;
+  }
+
+  // Add cache point to the last assistant message (without reasoning blocks)
+  const assistantMessage = updatedMessages[lastAssistantIndex];
+  const content = assistantMessage.content;
+
+  if (typeof content === 'string' && content !== '') {
+    assistantMessage.content = [
+      { type: ContentTypes.TEXT, text: content },
+      { cachePoint: { type: 'default' } },
+    ] as MessageContentComplex[];
+    debugCache(
+      `📍 Message cachePoint at index ${lastAssistantIndex} (string, ${content.length} chars)`
+    );
+    debugCache(
+      'addBedrockCacheControl: Added cachePoint to assistant message (string content)',
+      {
+        index: lastAssistantIndex,
+        contentLength: content.length,
+      }
+    );
+  } else if (
+    Array.isArray(assistantMessage.content) &&
+    assistantMessage.content.length > 0
+  ) {
+    // Double-check: If this message has reasoning blocks, skip adding cache point entirely
+    // This handles edge cases where the initial skip check might have missed it
+    if (hasReasoningBlock(assistantMessage)) {
+      debugCache(
+        `⚠️ Message cachePoint SKIPPED at index ${lastAssistantIndex} (has reasoning blocks)`
+      );
+      debugCache(
+        'addBedrockCacheControl: Skipping - assistant message has reasoning blocks (safety check)',
+        {
+          index: lastAssistantIndex,
+        }
+      );
+      return updatedMessages;
+    }
+
+    // Find the last text block and insert cache point after it
+    let inserted = false;
+    for (let j = assistantMessage.content.length - 1; j >= 0; j--) {
+      const block = assistantMessage.content[j] as MessageContentComplex;
+      const type = (block as { type?: string }).type;
+      if (type === ContentTypes.TEXT || type === 'text') {
+        const text = (block as { text?: string }).text;
+        if (text != null && text !== '') {
+          assistantMessage.content.splice(j + 1, 0, {
+            cachePoint: { type: 'default' },
+          } as MessageContentComplex);
+          inserted = true;
+          debugCache(
+            `📍 Message cachePoint at index ${lastAssistantIndex} (array, block ${j}, ${text.length} chars)`
+          );
+          debugCache(
+            'addBedrockCacheControl: Added cachePoint after text block in assistant message',
+            {
+              index: lastAssistantIndex,
+              textBlockIndex: j,
+              contentLength: text.length,
+            }
+          );
+          break;
+        }
+      }
+    }
+
+    // If no text block found, don't append cache point as the message structure is unexpected
+    if (!inserted) {
+      const contentTypes = assistantMessage.content.map(
+        (b) => (b as { type?: string }).type
+      );
+      debugCache(
+        `⚠️ Message cachePoint SKIPPED at index ${lastAssistantIndex} (no text block, types: ${contentTypes.join(',')})`
+      );
+      debugCache(
+        'addBedrockCacheControl: No suitable text block found, skipping cache point',
+        {
+          index: lastAssistantIndex,
+          contentTypes,
+        }
+      );
+    }
+  }
+
+  debugCache(
+    'addBedrockCacheControl: Complete - stable prefix caching applied',
+    {
+      lastAssistantIndex,
+      totalMessages: updatedMessages.length,
+    }
+  );
+
+  return updatedMessages;
+}