illuma-agents 1.0.37 → 1.0.39

package/src/llm/bedrock/index.ts

@@ -1,6 +1,10 @@
 /**
  * Optimized ChatBedrockConverse wrapper that fixes contentBlockIndex conflicts
- * and adds prompt caching support for Bedrock Converse API.
+ * and adds support for:
+ *
+ * - Prompt caching support for Bedrock Converse API (Illuma feature)
+ * - Application Inference Profiles (PR #9129)
+ * - Service Tiers (Priority/Standard/Flex) (PR #9785) - requires AWS SDK 3.966.0+
  *
  * Bedrock sends the same contentBlockIndex for both text and tool_use content blocks,
  * causing LangChain's merge logic to fail with "field[contentBlockIndex] already exists"
@@ -22,24 +26,81 @@
  */
 
 import { ChatBedrockConverse } from '@langchain/aws';
-import type { ChatBedrockConverseInput } from '@langchain/aws';
 import { AIMessageChunk } from '@langchain/core/messages';
 import type { BaseMessage, UsageMetadata } from '@langchain/core/messages';
-import { ChatGenerationChunk } from '@langchain/core/outputs';
+import { ChatGenerationChunk, ChatResult } from '@langchain/core/outputs';
 import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
+import type { ChatBedrockConverseInput } from '@langchain/aws';
+
+/**
+ * Service tier type for Bedrock invocations.
+ * Requires AWS SDK >= 3.966.0 to actually work.
+ * @see https://docs.aws.amazon.com/bedrock/latest/userguide/service-tiers-inference.html
+ */
+export type ServiceTierType = 'priority' | 'default' | 'flex' | 'reserved';
 
-/** Extended input type with promptCache option */
+/**
+ * Extended input interface with additional features:
+ * - promptCache: Enable Bedrock prompt caching for tool definitions
+ * - applicationInferenceProfile: Use an inference profile ARN instead of model ID
+ * - serviceTier: Specify service tier (Priority, Standard, Flex, Reserved)
+ */
 export interface CustomChatBedrockConverseInput
   extends ChatBedrockConverseInput {
+  /**
+   * Enable Bedrock prompt caching for tool definitions.
+   * When true, adds cachePoint markers to tools array.
+   */
   promptCache?: boolean;
+
+  /**
+   * Application Inference Profile ARN to use for the model.
+   * For example, "arn:aws:bedrock:eu-west-1:123456789102:application-inference-profile/fm16bt65tzgx"
+   * When provided, this ARN will be used for the actual inference calls instead of the model ID.
+   * Must still provide `model` as normal modelId to benefit from all the metadata.
+   * @see https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-create.html
+   */
+  applicationInferenceProfile?: string;
+
+  /**
+   * Service tier for model invocation.
+   * Specifies the processing tier type used for serving the request.
+   * Supported values are 'priority', 'default', 'flex', and 'reserved'.
+   *
+   * - 'priority': Prioritized processing for lower latency
+   * - 'default': Standard processing tier
+   * - 'flex': Flexible processing tier with lower cost
+   * - 'reserved': Reserved capacity for consistent performance
+   *
+   * If not provided, AWS uses the default tier.
+   * Note: Requires AWS SDK >= 3.966.0 to work.
+   * @see https://docs.aws.amazon.com/bedrock/latest/userguide/service-tiers-inference.html
+   */
+  serviceTier?: ServiceTierType;
+}
+
+/**
+ * Extended call options with serviceTier override support.
+ */
+export interface CustomChatBedrockConverseCallOptions {
+  serviceTier?: ServiceTierType;
 }
 
 export class CustomChatBedrockConverse extends ChatBedrockConverse {
+  /** Enable Bedrock prompt caching for tool definitions */
   promptCache: boolean;
 
+  /** Application Inference Profile ARN to use instead of model ID */
+  applicationInferenceProfile?: string;
+
+  /** Service tier for model invocation */
+  serviceTier?: ServiceTierType;
+
   constructor(fields?: CustomChatBedrockConverseInput) {
     super(fields);
     this.promptCache = fields?.promptCache ?? false;
+    this.applicationInferenceProfile = fields?.applicationInferenceProfile;
+    this.serviceTier = fields?.serviceTier;
   }
 
   static lc_name(): string {
@@ -47,19 +108,30 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
   }
 
   /**
-   * Override invocationParams to add cachePoint to tools when promptCache is enabled.
-   * This enables Bedrock prompt caching for tool definitions.
+   * Get the model ID to use for API calls.
+   * Returns applicationInferenceProfile if set, otherwise returns this.model.
+   */
+  protected getModelId(): string {
+    return this.applicationInferenceProfile ?? this.model;
+  }
+
+  /**
+   * Override invocationParams to:
+   * 1. Add cachePoint to tools when promptCache is enabled
+   * 2. Add serviceTier support
    *
-   * STRATEGY: Separate cachePoints for core tools and MCP tools
+   * CACHING STRATEGY: Separate cachePoints for core tools and MCP tools
    * - Core tools (web_search, execute_code, etc.) are stable → cache first
    * - MCP tools (have '_mcp_' in name) are dynamic → cache separately after
    * - This allows core tools to stay cached when MCP selection changes
    *
    * NOTE: Only Claude models support cachePoint - Nova and other models will reject it.
    */
-  invocationParams(
-    options?: this['ParsedCallOptions']
-  ): ReturnType<ChatBedrockConverse['invocationParams']> {
+  override invocationParams(
+    options?: this['ParsedCallOptions'] & CustomChatBedrockConverseCallOptions
+  ): ReturnType<ChatBedrockConverse['invocationParams']> & {
+    serviceTier?: { type: ServiceTierType };
+  } {
     const params = super.invocationParams(options);
 
     // Add cachePoint to tools array if promptCache is enabled and tools exist
@@ -79,8 +151,6 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
       // MCP tools have '_mcp_' in their name (e.g., 'search_emails_mcp_Google-Workspace')
       const coreTools: typeof params.toolConfig.tools = [];
       const mcpTools: typeof params.toolConfig.tools = [];
-      const coreToolNames: string[] = [];
-      const mcpToolNames: string[] = [];
 
       for (const tool of params.toolConfig.tools) {
         // Check if tool has a name property with '_mcp_' pattern
@@ -88,10 +158,8 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
           (tool as { toolSpec?: { name?: string } }).toolSpec?.name ?? '';
         if (toolName.includes('_mcp_')) {
           mcpTools.push(tool);
-          mcpToolNames.push(toolName);
         } else {
           coreTools.push(tool);
-          coreToolNames.push(toolName);
         }
       }
 
@@ -119,104 +187,158 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
       params.toolConfig.tools = toolsWithCache;
     }
 
-    return params;
+    // Add serviceTier support
+    const serviceTierType = options?.serviceTier ?? this.serviceTier;
+
+    return {
+      ...params,
+      serviceTier: serviceTierType ? { type: serviceTierType } : undefined,
+    };
+  }
+
+  /**
+   * Override _generateNonStreaming to use applicationInferenceProfile as modelId.
+   * Uses the same model-swapping pattern as streaming for consistency.
+   */
+  override async _generateNonStreaming(
+    messages: BaseMessage[],
+    options: this['ParsedCallOptions'] & CustomChatBedrockConverseCallOptions,
+    runManager?: CallbackManagerForLLMRun
+  ): Promise<ChatResult> {
+    // Temporarily swap model for applicationInferenceProfile support
+    const originalModel = this.model;
+    if (
+      this.applicationInferenceProfile != null &&
+      this.applicationInferenceProfile !== ''
+    ) {
+      this.model = this.applicationInferenceProfile;
+    }
+
+    try {
+      return await super._generateNonStreaming(messages, options, runManager);
+    } finally {
+      // Restore original model
+      this.model = originalModel;
+    }
   }
 
   /**
    * Override _streamResponseChunks to:
-   * 1. Strip contentBlockIndex from response_metadata to prevent merge conflicts
-   * 2. Extract cacheReadInputTokens/cacheWriteInputTokens and add to usage_metadata
+   * 1. Use applicationInferenceProfile as modelId (by temporarily swapping this.model)
+   * 2. Strip contentBlockIndex from response_metadata to prevent merge conflicts
+   * 3. Extract cacheReadInputTokens/cacheWriteInputTokens and add to usage_metadata
+   *
+   * Note: We delegate to super._streamResponseChunks() to preserve @langchain/aws's
+   * internal chunk handling which correctly preserves array content for reasoning blocks.
    */
-  async *_streamResponseChunks(
+  override async *_streamResponseChunks(
     messages: BaseMessage[],
-    options: this['ParsedCallOptions'],
+    options: this['ParsedCallOptions'] & CustomChatBedrockConverseCallOptions,
     runManager?: CallbackManagerForLLMRun
   ): AsyncGenerator<ChatGenerationChunk> {
-    const baseStream = super._streamResponseChunks(
-      messages,
-      options,
-      runManager
-    );
-
-    for await (const chunk of baseStream) {
-      // Only process if we have response_metadata
-      if (
-        chunk.message instanceof AIMessageChunk &&
-        (chunk.message as Partial<AIMessageChunk>).response_metadata &&
-        typeof chunk.message.response_metadata === 'object'
-      ) {
-        const responseMetadata = chunk.message.response_metadata as Record<
-          string,
-          unknown
-        >;
-        let needsModification = false;
-        let cleanedMetadata = responseMetadata;
-
-        // Check if contentBlockIndex exists anywhere in response_metadata
-        const hasContentBlockIndex =
-          this.hasContentBlockIndex(responseMetadata);
-        if (hasContentBlockIndex) {
-          cleanedMetadata = this.removeContentBlockIndex(
-            responseMetadata
-          ) as Record<string, unknown>;
-          needsModification = true;
-        }
+    // Temporarily swap model for applicationInferenceProfile support
+    const originalModel = this.model;
+    if (
+      this.applicationInferenceProfile != null &&
+      this.applicationInferenceProfile !== ''
+    ) {
+      this.model = this.applicationInferenceProfile;
+    }
 
-        // Extract cache tokens from metadata.usage (Bedrock streaming format)
-        // The metadata chunk contains usage with cacheReadInputTokens/cacheWriteInputTokens
-        const metadata = responseMetadata.metadata as
-          | Record<string, unknown>
-          | undefined;
-        const usage = (metadata?.usage ?? responseMetadata.usage) as
-          | Record<string, unknown>
-          | undefined;
-
-        let enhancedUsageMetadata: UsageMetadata | undefined =
-          chunk.message.usage_metadata;
-
-        if (usage) {
-          const cacheRead =
-            (usage.cacheReadInputTokens as number | undefined) ?? 0;
-          const cacheWrite =
-            (usage.cacheWriteInputTokens as number | undefined) ?? 0;
-          const inputTokens = (usage.inputTokens as number | undefined) ?? 0;
-          const outputTokens = (usage.outputTokens as number | undefined) ?? 0;
-
-          if (cacheRead > 0 || cacheWrite > 0) {
-            needsModification = true;
-            enhancedUsageMetadata = {
-              input_tokens:
-                chunk.message.usage_metadata?.input_tokens ?? inputTokens,
-              output_tokens:
-                chunk.message.usage_metadata?.output_tokens ?? outputTokens,
-              total_tokens:
-                chunk.message.usage_metadata?.total_tokens ??
-                (usage.totalTokens as number | undefined) ??
-                0,
-              input_token_details: {
-                cache_read: cacheRead,
-                cache_creation: cacheWrite,
-              },
-            };
-          }
-        }
+    try {
+      // Use parent's streaming logic which correctly handles reasoning content
+      const baseStream = super._streamResponseChunks(
+        messages,
+        options,
+        runManager
+      );
+
+      for await (const chunk of baseStream) {
+        // Clean and enhance chunk
+        yield this.processChunk(chunk);
+      }
+    } finally {
+      // Restore original model
+      this.model = originalModel;
+    }
+  }
 
-        if (needsModification) {
-          yield new ChatGenerationChunk({
-            text: chunk.text,
-            message: new AIMessageChunk({
-              ...chunk.message,
-              response_metadata: cleanedMetadata,
-              usage_metadata: enhancedUsageMetadata,
-            }),
-            generationInfo: chunk.generationInfo,
-          });
-          continue;
-        }
+  /**
+   * Process a chunk by:
+   * 1. Removing contentBlockIndex from response_metadata
+   * 2. Extracting cache token information from Bedrock's usage data
+   */
+  private processChunk(chunk: ChatGenerationChunk): ChatGenerationChunk {
+    const message = chunk.message;
+    if (!(message instanceof AIMessageChunk)) {
+      return chunk;
+    }
+
+    const responseMetadata = message.response_metadata as Record<
+      string,
+      unknown
+    >;
+    let needsModification = false;
+    let cleanedMetadata = responseMetadata;
+
+    // Check if contentBlockIndex exists anywhere in response_metadata
+    const hasContentBlockIndex = this.hasContentBlockIndex(responseMetadata);
+    if (hasContentBlockIndex) {
+      cleanedMetadata = this.removeContentBlockIndex(
+        responseMetadata
+      ) as Record<string, unknown>;
+      needsModification = true;
+    }
+
+    // Extract cache tokens from metadata.usage (Bedrock streaming format)
+    // The metadata chunk contains usage with cacheReadInputTokens/cacheWriteInputTokens
+    const metadata = responseMetadata.metadata as
+      | Record<string, unknown>
+      | undefined;
+    const usage = (metadata?.usage ?? responseMetadata.usage) as
+      | Record<string, unknown>
+      | undefined;
+
+    let enhancedUsageMetadata: UsageMetadata | undefined =
+      message.usage_metadata;
+
+    if (usage) {
+      const cacheRead = (usage.cacheReadInputTokens as number | undefined) ?? 0;
+      const cacheWrite =
+        (usage.cacheWriteInputTokens as number | undefined) ?? 0;
+      const inputTokens = (usage.inputTokens as number | undefined) ?? 0;
+      const outputTokens = (usage.outputTokens as number | undefined) ?? 0;
+
+      if (cacheRead > 0 || cacheWrite > 0) {
+        needsModification = true;
+        enhancedUsageMetadata = {
+          input_tokens: message.usage_metadata?.input_tokens ?? inputTokens,
+          output_tokens: message.usage_metadata?.output_tokens ?? outputTokens,
+          total_tokens:
+            message.usage_metadata?.total_tokens ??
+            (usage.totalTokens as number | undefined) ??
+            0,
+          input_token_details: {
+            cache_read: cacheRead,
+            cache_creation: cacheWrite,
+          },
+        };
       }
+    }
 
-      yield chunk;
+    if (needsModification) {
+      return new ChatGenerationChunk({
+        text: chunk.text,
+        message: new AIMessageChunk({
+          ...message,
+          response_metadata: cleanedMetadata,
+          usage_metadata: enhancedUsageMetadata,
+        }),
+        generationInfo: chunk.generationInfo,
+      });
     }
+
+    return chunk;
   }
 
   /**