@librechat/agents 3.0.80 → 3.0.771

package/src/llm/bedrock/index.ts

@@ -1,9 +1,5 @@
 /**
  * Optimized ChatBedrockConverse wrapper that fixes contentBlockIndex conflicts
- * and adds support for latest @langchain/aws features:
- *
- * - Application Inference Profiles (PR #9129)
- * - Service Tiers (Priority/Standard/Flex) (PR #9785) - requires AWS SDK 3.966.0+
  *
  * Bedrock sends the same contentBlockIndex for both text and tool_use content blocks,
  * causing LangChain's merge logic to fail with "field[contentBlockIndex] already exists"
@@ -16,74 +12,15 @@
  */
 
 import { ChatBedrockConverse } from '@langchain/aws';
-import { AIMessageChunk } from '@langchain/core/messages';
-import { ChatGenerationChunk, ChatResult } from '@langchain/core/outputs';
-import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
 import type { ChatBedrockConverseInput } from '@langchain/aws';
+import { AIMessageChunk } from '@langchain/core/messages';
 import type { BaseMessage } from '@langchain/core/messages';
-
-/**
- * Service tier type for Bedrock invocations.
- * Requires AWS SDK >= 3.966.0 to actually work.
- * @see https://docs.aws.amazon.com/bedrock/latest/userguide/service-tiers-inference.html
- */
-export type ServiceTierType = 'priority' | 'default' | 'flex' | 'reserved';
-
-/**
- * Extended input interface with additional features:
- * - applicationInferenceProfile: Use an inference profile ARN instead of model ID
- * - serviceTier: Specify service tier (Priority, Standard, Flex, Reserved)
- */
-export interface CustomChatBedrockConverseInput
-  extends ChatBedrockConverseInput {
-  /**
-   * Application Inference Profile ARN to use for the model.
-   * For example, "arn:aws:bedrock:eu-west-1:123456789102:application-inference-profile/fm16bt65tzgx"
-   * When provided, this ARN will be used for the actual inference calls instead of the model ID.
-   * Must still provide `model` as normal modelId to benefit from all the metadata.
-   * @see https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-create.html
-   */
-  applicationInferenceProfile?: string;
-
-  /**
-   * Service tier for model invocation.
-   * Specifies the processing tier type used for serving the request.
-   * Supported values are 'priority', 'default', 'flex', and 'reserved'.
-   *
-   * - 'priority': Prioritized processing for lower latency
-   * - 'default': Standard processing tier
-   * - 'flex': Flexible processing tier with lower cost
-   * - 'reserved': Reserved capacity for consistent performance
-   *
-   * If not provided, AWS uses the default tier.
-   * Note: Requires AWS SDK >= 3.966.0 to work.
-   * @see https://docs.aws.amazon.com/bedrock/latest/userguide/service-tiers-inference.html
-   */
-  serviceTier?: ServiceTierType;
-}
-
-/**
- * Extended call options with serviceTier override support.
- */
-export interface CustomChatBedrockConverseCallOptions {
-  serviceTier?: ServiceTierType;
-}
+import { ChatGenerationChunk } from '@langchain/core/outputs';
+import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
 
 export class CustomChatBedrockConverse extends ChatBedrockConverse {
-  /**
-   * Application Inference Profile ARN to use instead of model ID.
-   */
-  applicationInferenceProfile?: string;
-
-  /**
-   * Service tier for model invocation.
-   */
-  serviceTier?: ServiceTierType;
-
-  constructor(fields?: CustomChatBedrockConverseInput) {
+  constructor(fields?: ChatBedrockConverseInput) {
     super(fields);
-    this.applicationInferenceProfile = fields?.applicationInferenceProfile;
-    this.serviceTier = fields?.serviceTier;
   }
 
   static lc_name(): string {
@@ -91,126 +28,52 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
   }
 
   /**
-   * Get the model ID to use for API calls.
-   * Returns applicationInferenceProfile if set, otherwise returns this.model.
-   */
-  protected getModelId(): string {
-    return this.applicationInferenceProfile ?? this.model;
-  }
-
-  /**
-   * Override invocationParams to add serviceTier support.
-   */
-  override invocationParams(
-    options?: this['ParsedCallOptions'] & CustomChatBedrockConverseCallOptions
-  ): ReturnType<ChatBedrockConverse['invocationParams']> & {
-    serviceTier?: { type: ServiceTierType };
-  } {
-    const baseParams = super.invocationParams(options);
-
-    /** Service tier from options or fall back to class-level setting */
-    const serviceTierType = options?.serviceTier ?? this.serviceTier;
-
-    return {
-      ...baseParams,
-      serviceTier: serviceTierType ? { type: serviceTierType } : undefined,
-    };
-  }
-
-  /**
-   * Override _generateNonStreaming to use applicationInferenceProfile as modelId.
-   * Uses the same model-swapping pattern as streaming for consistency.
+   * Override _streamResponseChunks to strip contentBlockIndex from response_metadata
+   * This prevents LangChain's merge conflicts when the same index is used for
+   * different content types (text vs tool calls)
    */
-  override async _generateNonStreaming(
+  async *_streamResponseChunks(
     messages: BaseMessage[],
-    options: this['ParsedCallOptions'] & CustomChatBedrockConverseCallOptions,
-    runManager?: CallbackManagerForLLMRun
-  ): Promise<ChatResult> {
-    // Temporarily swap model for applicationInferenceProfile support
-    const originalModel = this.model;
-    if (
-      this.applicationInferenceProfile != null &&
-      this.applicationInferenceProfile !== ''
-    ) {
-      this.model = this.applicationInferenceProfile;
-    }
-
-    try {
-      return await super._generateNonStreaming(messages, options, runManager);
-    } finally {
-      // Restore original model
-      this.model = originalModel;
-    }
-  }
-
-  /**
-   * Override _streamResponseChunks to:
-   * 1. Use applicationInferenceProfile as modelId (by temporarily swapping this.model)
-   * 2. Strip contentBlockIndex from response_metadata to prevent merge conflicts
-   *
-   * Note: We delegate to super._streamResponseChunks() to preserve @langchain/aws's
-   * internal chunk handling which correctly preserves array content for reasoning blocks.
-   */
-  override async *_streamResponseChunks(
-    messages: BaseMessage[],
-    options: this['ParsedCallOptions'] & CustomChatBedrockConverseCallOptions,
+    options: this['ParsedCallOptions'],
     runManager?: CallbackManagerForLLMRun
   ): AsyncGenerator<ChatGenerationChunk> {
-    // Temporarily swap model for applicationInferenceProfile support
-    const originalModel = this.model;
-    if (
-      this.applicationInferenceProfile != null &&
-      this.applicationInferenceProfile !== ''
-    ) {
-      this.model = this.applicationInferenceProfile;
-    }
-
-    try {
-      // Use parent's streaming logic which correctly handles reasoning content
-      const baseStream = super._streamResponseChunks(
-        messages,
-        options,
-        runManager
-      );
-
-      for await (const chunk of baseStream) {
-        // Clean contentBlockIndex from response_metadata to prevent merge conflicts
-        yield this.cleanChunk(chunk);
+    const baseStream = super._streamResponseChunks(
+      messages,
+      options,
+      runManager
+    );
+
+    for await (const chunk of baseStream) {
+      // Only process if we have response_metadata
+      if (
+        chunk.message instanceof AIMessageChunk &&
+        (chunk.message as Partial<AIMessageChunk>).response_metadata &&
+        typeof chunk.message.response_metadata === 'object'
+      ) {
+        // Check if contentBlockIndex exists anywhere in response_metadata (top level or nested)
+        const hasContentBlockIndex = this.hasContentBlockIndex(
+          chunk.message.response_metadata
+        );
+
+        if (hasContentBlockIndex) {
+          const cleanedMetadata = this.removeContentBlockIndex(
+            chunk.message.response_metadata
+          ) as Record<string, unknown>;
+
+          yield new ChatGenerationChunk({
+            text: chunk.text,
+            message: new AIMessageChunk({
+              ...chunk.message,
+              response_metadata: cleanedMetadata,
+            }),
+            generationInfo: chunk.generationInfo,
+          });
+          continue;
+        }
       }
-    } finally {
-      // Restore original model
-      this.model = originalModel;
-    }
-  }
 
-  /**
-   * Clean a chunk by removing contentBlockIndex from response_metadata.
-   */
-  private cleanChunk(chunk: ChatGenerationChunk): ChatGenerationChunk {
-    const message = chunk.message;
-    if (!(message instanceof AIMessageChunk)) {
-      return chunk;
+      yield chunk;
     }
-
-    const metadata = message.response_metadata as Record<string, unknown>;
-    const hasContentBlockIndex = this.hasContentBlockIndex(metadata);
-    if (!hasContentBlockIndex) {
-      return chunk;
-    }
-
-    const cleanedMetadata = this.removeContentBlockIndex(metadata) as Record<
-      string,
-      unknown
-    >;
-
-    return new ChatGenerationChunk({
-      text: chunk.text,
-      message: new AIMessageChunk({
-        ...message,
-        response_metadata: cleanedMetadata,
-      }),
-      generationInfo: chunk.generationInfo,
-    });
   }
 
   /**

package/src/messages/cache.test.ts

@@ -835,6 +835,221 @@ describe('Multi-agent provider interoperability', () => {
   });
 });
 
+describe('Immutability - addCacheControl does not mutate original messages', () => {
+  it('should not mutate original messages when adding cache control to string content', () => {
+    const originalMessages: TestMsg[] = [
+      { role: 'user', content: 'Hello' },
+      { role: 'assistant', content: 'Hi there' },
+      { role: 'user', content: 'How are you?' },
+    ];
+
+    const originalFirstContent = originalMessages[0].content;
+    const originalThirdContent = originalMessages[2].content;
+
+    const result = addCacheControl(originalMessages as never);
+
+    expect(originalMessages[0].content).toBe(originalFirstContent);
+    expect(originalMessages[2].content).toBe(originalThirdContent);
+    expect(typeof originalMessages[0].content).toBe('string');
+    expect(typeof originalMessages[2].content).toBe('string');
+
+    expect(Array.isArray(result[0].content)).toBe(true);
+    expect(Array.isArray(result[2].content)).toBe(true);
+  });
+
+  it('should not mutate original messages when adding cache control to array content', () => {
+    const originalMessages: TestMsg[] = [
+      {
+        role: 'user',
+        content: [{ type: ContentTypes.TEXT, text: 'Hello' }],
+      },
+      { role: 'assistant', content: 'Hi there' },
+      {
+        role: 'user',
+        content: [{ type: ContentTypes.TEXT, text: 'How are you?' }],
+      },
+    ];
+
+    const originalFirstBlock = {
+      ...(originalMessages[0].content as MessageContentComplex[])[0],
+    };
+    const originalThirdBlock = {
+      ...(originalMessages[2].content as MessageContentComplex[])[0],
+    };
+
+    const result = addCacheControl(originalMessages as never);
+
+    const firstContent = originalMessages[0].content as MessageContentComplex[];
+    const thirdContent = originalMessages[2].content as MessageContentComplex[];
+
+    expect('cache_control' in firstContent[0]).toBe(false);
+    expect('cache_control' in thirdContent[0]).toBe(false);
+    expect(firstContent[0]).toEqual(originalFirstBlock);
+    expect(thirdContent[0]).toEqual(originalThirdBlock);
+
+    const resultFirstContent = result[0].content as MessageContentComplex[];
+    const resultThirdContent = result[2].content as MessageContentComplex[];
+    expect('cache_control' in resultFirstContent[0]).toBe(true);
+    expect('cache_control' in resultThirdContent[0]).toBe(true);
+  });
+
+  it('should not mutate original messages when stripping existing cache control', () => {
+    const originalMessages: TestMsg[] = [
+      {
+        role: 'user',
+        content: [
+          {
+            type: ContentTypes.TEXT,
+            text: 'Hello',
+            cache_control: { type: 'ephemeral' },
+          } as MessageContentComplex,
+        ],
+      },
+      { role: 'assistant', content: 'Hi there' },
+      {
+        role: 'user',
+        content: [{ type: ContentTypes.TEXT, text: 'How are you?' }],
+      },
+    ];
+
+    const originalFirstBlock = (
+      originalMessages[0].content as MessageContentComplex[]
+    )[0];
+
+    addCacheControl(originalMessages as never);
+
+    expect('cache_control' in originalFirstBlock).toBe(true);
+  });
+});
+
+describe('Immutability - addBedrockCacheControl does not mutate original messages', () => {
+  it('should not mutate original messages when adding cache points to string content', () => {
+    const originalMessages: TestMsg[] = [
+      { role: 'user', content: 'Hello' },
+      { role: 'assistant', content: 'Hi there' },
+    ];
+
+    const originalFirstContent = originalMessages[0].content;
+    const originalSecondContent = originalMessages[1].content;
+
+    const result = addBedrockCacheControl(originalMessages);
+
+    expect(originalMessages[0].content).toBe(originalFirstContent);
+    expect(originalMessages[1].content).toBe(originalSecondContent);
+    expect(typeof originalMessages[0].content).toBe('string');
+    expect(typeof originalMessages[1].content).toBe('string');
+
+    expect(Array.isArray(result[0].content)).toBe(true);
+    expect(Array.isArray(result[1].content)).toBe(true);
+  });
+
+  it('should not mutate original messages when adding cache points to array content', () => {
+    const originalMessages: TestMsg[] = [
+      {
+        role: 'user',
+        content: [{ type: ContentTypes.TEXT, text: 'Hello' }],
+      },
+      {
+        role: 'assistant',
+        content: [{ type: ContentTypes.TEXT, text: 'Hi there' }],
+      },
+    ];
+
+    const originalFirstContentLength = (
+      originalMessages[0].content as MessageContentComplex[]
+    ).length;
+    const originalSecondContentLength = (
+      originalMessages[1].content as MessageContentComplex[]
+    ).length;
+
+    const result = addBedrockCacheControl(originalMessages);
+
+    const firstContent = originalMessages[0].content as MessageContentComplex[];
+    const secondContent = originalMessages[1]
+      .content as MessageContentComplex[];
+
+    expect(firstContent.length).toBe(originalFirstContentLength);
+    expect(secondContent.length).toBe(originalSecondContentLength);
+    expect(firstContent.some((b) => 'cachePoint' in b)).toBe(false);
+    expect(secondContent.some((b) => 'cachePoint' in b)).toBe(false);
+
+    const resultFirstContent = result[0].content as MessageContentComplex[];
+    const resultSecondContent = result[1].content as MessageContentComplex[];
+    expect(resultFirstContent.length).toBe(originalFirstContentLength + 1);
+    expect(resultSecondContent.length).toBe(originalSecondContentLength + 1);
+    expect(resultFirstContent.some((b) => 'cachePoint' in b)).toBe(true);
+    expect(resultSecondContent.some((b) => 'cachePoint' in b)).toBe(true);
+  });
+
+  it('should not mutate original messages when stripping existing cache control', () => {
+    const originalMessages: TestMsg[] = [
+      {
+        role: 'user',
+        content: [
+          {
+            type: ContentTypes.TEXT,
+            text: 'Hello',
+            cache_control: { type: 'ephemeral' },
+          } as MessageContentComplex,
+        ],
+      },
+      {
+        role: 'assistant',
+        content: [
+          { type: ContentTypes.TEXT, text: 'Hi there' },
+          { cachePoint: { type: 'default' } },
+        ],
+      },
+    ];
+
+    const originalFirstBlock = (
+      originalMessages[0].content as MessageContentComplex[]
+    )[0];
+    const originalSecondContentLength = (
+      originalMessages[1].content as MessageContentComplex[]
+    ).length;
+
+    addBedrockCacheControl(originalMessages);
+
+    expect('cache_control' in originalFirstBlock).toBe(true);
+    expect(
+      (originalMessages[1].content as MessageContentComplex[]).length
+    ).toBe(originalSecondContentLength);
+  });
+
+  it('should allow different providers to process same messages without cross-contamination', () => {
+    const sharedMessages: TestMsg[] = [
+      {
+        role: 'user',
+        content: [{ type: ContentTypes.TEXT, text: 'Shared message 1' }],
+      },
+      {
+        role: 'assistant',
+        content: [{ type: ContentTypes.TEXT, text: 'Shared response 1' }],
+      },
+    ];
+
+    const bedrockResult = addBedrockCacheControl(sharedMessages);
+
+    const anthropicResult = addCacheControl(sharedMessages as never);
+
+    const originalFirstContent = sharedMessages[0]
+      .content as MessageContentComplex[];
+    expect(originalFirstContent.some((b) => 'cachePoint' in b)).toBe(false);
+    expect('cache_control' in originalFirstContent[0]).toBe(false);
+
+    const bedrockFirstContent = bedrockResult[0]
+      .content as MessageContentComplex[];
+    expect(bedrockFirstContent.some((b) => 'cachePoint' in b)).toBe(true);
+    expect('cache_control' in bedrockFirstContent[0]).toBe(false);
+
+    const anthropicFirstContent = anthropicResult[0]
+      .content as MessageContentComplex[];
+    expect(anthropicFirstContent.some((b) => 'cachePoint' in b)).toBe(false);
+    expect('cache_control' in anthropicFirstContent[0]).toBe(true);
+  });
+});
+
 describe('Multi-turn cache cleanup', () => {
   it('strips stale Bedrock cache points from previous turns before applying new ones', () => {
     const messages: TestMsg[] = [