@librechat/agents 3.2.60 → 3.2.62

package/src/llm/bedrock/index.ts

@@ -39,6 +39,7 @@ import {
   handleConverseStreamContentBlockDelta,
   handleConverseStreamMetadata,
 } from './utils';
+import type { ContentBlockDeltaEvent } from './types';
 import {
   resolveBedrockPromptCacheTtl,
   supportsBedrockToolCache,
@@ -57,6 +58,96 @@ export type ServiceTierType = 'priority' | 'default' | 'flex' | 'reserved';
 export type CustomGuardrailConfiguration = GuardrailConfiguration &
   Pick<GuardrailStreamConfiguration, 'streamProcessingMode'>;
 
+const MAX_STREAM_QUEUE_CHUNKS = 256;
+const MAX_STREAM_QUEUE_TEXT_CHARS = 8192;
+const STREAM_CHUNK_MIN_SIZE = 4;
+const STREAM_BOUNDARIES = new Set([' ', '.', ',', '!', '?', ';', ':']);
+
+type QueuedGenerationChunk = {
+  chunk: ChatGenerationChunk;
+  callbackChunk?: ChatGenerationChunk;
+  callbackToken: string;
+  smooth: boolean;
+  textLength: number;
+};
+
+function findStreamChunkBoundary(text: string, minSize: number): number {
+  if (minSize >= text.length) {
+    return text.length;
+  }
+
+  for (let position = minSize; position < text.length; position++) {
+    if (STREAM_BOUNDARIES.has(text[position])) {
+      return position + 1;
+    }
+  }
+
+  return text.length;
+}
+
+function splitStreamToken(text: string): string[] {
+  const chunks: string[] = [];
+  let currentIndex = 0;
+
+  while (currentIndex < text.length) {
+    const remainingText = text.slice(currentIndex);
+    const chunkSize = findStreamChunkBoundary(
+      remainingText,
+      STREAM_CHUNK_MIN_SIZE
+    );
+    chunks.push(text.slice(currentIndex, currentIndex + chunkSize));
+    currentIndex += chunkSize;
+  }
+
+  return chunks;
+}
+
+function getCadencedStreamDelay({
+  targetDelay,
+  lastVisibleContentAt,
+  now,
+}: {
+  targetDelay: number;
+  lastVisibleContentAt?: number;
+  now: number;
+}): number {
+  if (targetDelay <= 0 || lastVisibleContentAt == null) {
+    return 0;
+  }
+  return Math.max(0, targetDelay - (now - lastVisibleContentAt));
+}
+
+async function waitForStreamDelay(
+  delay: number,
+  signal?: AbortSignal
+): Promise<void> {
+  if (delay <= 0 || isSignalAborted(signal)) {
+    return;
+  }
+  await new Promise<void>((resolve) => {
+    const timeoutRef: { current?: ReturnType<typeof setTimeout> } = {};
+    const onAbort = (): void => {
+      if (timeoutRef.current) {
+        clearTimeout(timeoutRef.current);
+      }
+      signal?.removeEventListener('abort', onAbort);
+      resolve();
+    };
+    timeoutRef.current = setTimeout(() => {
+      signal?.removeEventListener('abort', onAbort);
+      resolve();
+    }, delay);
+    signal?.addEventListener('abort', onAbort, { once: true });
+    if (isSignalAborted(signal)) {
+      onAbort();
+    }
+  });
+}
+
+function isSignalAborted(signal?: AbortSignal): boolean {
+  return signal?.aborted === true;
+}
+
 /**
  * Extended input interface with additional features:
  * - applicationInferenceProfile: Use an inference profile ARN instead of model ID
@@ -78,6 +169,11 @@ export interface CustomChatBedrockConverseInput
    */
   promptCacheTtl?: PromptCacheTtl;
 
+  /**
+   * Minimum delay in milliseconds between visible streamed content deltas.
+   */
+  _lc_stream_delay?: number;
+
   /**
    * Guardrail configuration for Converse and ConverseStream invocations.
    * `streamProcessingMode` is only used by ConverseStream.
@@ -119,6 +215,8 @@ export interface CustomChatBedrockConverseCallOptions {
 }
 
 export class CustomChatBedrockConverse extends ChatBedrockConverse {
+  _lc_stream_delay: number;
+
   /**
    * Whether to insert Bedrock prompt cache checkpoints when available.
    */
@@ -151,6 +249,7 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
     super(fields);
     this.promptCache = fields?.promptCache;
     this.promptCacheTtl = fields?.promptCacheTtl;
+    this._lc_stream_delay = Math.max(0, fields?._lc_stream_delay ?? 0);
     this.applicationInferenceProfile = fields?.applicationInferenceProfile;
     this.serviceTier = fields?.serviceTier;
     // `super(fields)` initializes `this.model` to LangChain's default Claude
@@ -183,10 +282,10 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
     const toolConfig =
       this.promptCache === true && supportsBedrockToolCache(this.cacheModelId)
         ? insertBedrockToolCachePoint(
-          baseParams.toolConfig,
-          true,
-          resolveBedrockPromptCacheTtl(this.promptCacheTtl, this.cacheModelId)
-        )
+            baseParams.toolConfig,
+            true,
+            resolveBedrockPromptCacheTtl(this.promptCacheTtl, this.cacheModelId)
+          )
         : baseParams.toolConfig;
 
     /** Service tier from options or fall back to class-level setting */
@@ -261,102 +360,354 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
       ...(params as Record<string, unknown>),
     });
 
-    const response = await this.client.send(command, {
-      abortSignal: options.signal,
-    });
-
-    if (!response.stream) {
-      return;
+    const streamAbortController = new AbortController();
+    const abortStream = (): void => streamAbortController.abort();
+    options.signal?.addEventListener('abort', abortStream, { once: true });
+    if (isSignalAborted(options.signal)) {
+      abortStream();
     }
 
-    const seenBlockIndices = new Set<number>();
-    const toolUseBlockIndices = new Set<number>();
-    /**
-     * Guardrails can reject an already-streamed toolUse block at
-     * `messageStop` (`guardrail_intervened`), after `contentBlockStop` has
-     * passed. Only emit eager-execution seals when no guardrails are
-     * configured, so a later intervention can't race an eagerly started tool.
-     */
-    const sealToolUseOnStop =
-      options.guardrailConfig == null && this.guardrailConfig == null;
-
-    for await (const event of response.stream) {
-      if (event.contentBlockStart != null) {
-        const startChunk = handleConverseStreamContentBlockStart(
-          event.contentBlockStart
-        );
-        if (startChunk != null) {
-          const idx = event.contentBlockStart.contentBlockIndex;
-          if (idx != null) {
-            seenBlockIndices.add(idx);
-            if (event.contentBlockStart.start?.toolUse != null) {
-              toolUseBlockIndices.add(idx);
+    try {
+      const response = await this.client.send(command, {
+        abortSignal: streamAbortController.signal,
+      });
+
+      const stream = response.stream;
+      if (!stream) {
+        return;
+      }
+
+      const seenBlockIndices = new Set<number>();
+      const toolUseBlockIndices = new Set<number>();
+      const queuedChunks: QueuedGenerationChunk[] = [];
+      const producerState: { done: boolean; error?: unknown } = { done: false };
+      let queuedChunkIndex = 0;
+      let bufferedTextLength = 0;
+      let consumerClosed = false;
+      let notifyConsumer: (() => void) | undefined;
+      let notifyProducer: (() => void) | undefined;
+      let hasEmittedVisibleContent = false;
+      let lastVisibleContentAt: number | undefined;
+
+      /**
+       * Guardrails can reject an already-streamed toolUse block at
+       * `messageStop` (`guardrail_intervened`), after `contentBlockStop` has
+       * passed. Only emit eager-execution seals when no guardrails are
+       * configured, so a later intervention can't race an eagerly started tool.
+       */
+      const sealToolUseOnStop =
+        options.guardrailConfig == null && this.guardrailConfig == null;
+
+      const notifyConsumerForChunk = (): void => {
+        notifyConsumer?.();
+        notifyConsumer = undefined;
+      };
+
+      const notifyProducerForSpace = (): void => {
+        notifyProducer?.();
+        notifyProducer = undefined;
+      };
+
+      const hasQueuedChunks = (): boolean =>
+        queuedChunkIndex < queuedChunks.length;
+
+      const getQueuedChunkCount = (): number =>
+        queuedChunks.length - queuedChunkIndex;
+
+      const isQueueAtCapacity = (): boolean =>
+        getQueuedChunkCount() >= MAX_STREAM_QUEUE_CHUNKS ||
+        bufferedTextLength >= MAX_STREAM_QUEUE_TEXT_CHARS;
+
+      const waitForNextChunk = async (): Promise<void> => {
+        if (
+          hasQueuedChunks() ||
+          producerState.done ||
+          producerState.error != null
+        ) {
+          return;
+        }
+        await new Promise<void>((resolve) => {
+          notifyConsumer = resolve;
+        });
+      };
+
+      const waitForQueueSpace = async (): Promise<void> => {
+        while (
+          isQueueAtCapacity() &&
+          !consumerClosed &&
+          !isSignalAborted(options.signal)
+        ) {
+          await new Promise<void>((resolve) => {
+            const signal = options.signal;
+            const onAbort = (): void => {
+              signal?.removeEventListener('abort', onAbort);
+              resolve();
+            };
+            const onSpace = (): void => {
+              signal?.removeEventListener('abort', onAbort);
+              resolve();
+            };
+            notifyProducer = onSpace;
+            signal?.addEventListener('abort', onAbort, { once: true });
+            if (isSignalAborted(signal)) {
+              onAbort();
             }
-          }
-          yield this.enrichChunk(startChunk, seenBlockIndices);
-
-          // Registered stream handlers receive chunks through callback
-          // events, not the yielded generator — dispatch the start chunk so
-          // they see the tool call's id/name (eager chunk state needs both).
-          await runManager?.handleLLMNewToken(
-            startChunk.text,
-            undefined,
-            undefined,
-            undefined,
-            undefined,
-            { chunk: startChunk }
-          );
+          });
         }
-      } else if (event.contentBlockDelta != null) {
-        const deltaChunk = handleConverseStreamContentBlockDelta(
-          event.contentBlockDelta
-        );
+      };
 
-        const idx = event.contentBlockDelta.contentBlockIndex;
+      const dequeue = (): QueuedGenerationChunk | undefined => {
+        if (!hasQueuedChunks()) {
+          return undefined;
+        }
+        const queuedChunk = queuedChunks[queuedChunkIndex];
+        queuedChunkIndex++;
+        if (
+          queuedChunkIndex > 128 &&
+          queuedChunkIndex * 2 >= queuedChunks.length
+        ) {
+          queuedChunks.splice(0, queuedChunkIndex);
+          queuedChunkIndex = 0;
+        }
+        return queuedChunk;
+      };
+
+      const enqueue = async (
+        queuedChunk: QueuedGenerationChunk
+      ): Promise<void> => {
+        await waitForQueueSpace();
+        if (consumerClosed || isSignalAborted(options.signal)) {
+          abortStream();
+          throw new Error('AbortError: User aborted the request.');
+        }
+        queuedChunks.push(queuedChunk);
+        if (queuedChunk.smooth) {
+          bufferedTextLength += queuedChunk.textLength;
+        }
+        notifyConsumerForChunk();
+      };
+
+      const enqueueChunk = async ({
+        chunk,
+        callbackChunk,
+        callbackToken = '',
+        smooth = false,
+        textLength = 0,
+      }: {
+        chunk: ChatGenerationChunk;
+        callbackChunk?: ChatGenerationChunk;
+        callbackToken?: string;
+        smooth?: boolean;
+        textLength?: number;
+      }): Promise<void> => {
+        await enqueue({
+          chunk,
+          callbackChunk,
+          callbackToken,
+          smooth,
+          textLength: smooth ? textLength : 0,
+        });
+      };
+
+      const enqueueDelta = async (
+        contentBlockDelta: ContentBlockDeltaEvent
+      ): Promise<void> => {
+        const delta = contentBlockDelta.delta;
+        if (delta == null) {
+          throw new Error('No delta found in content block.');
+        }
+
+        const idx = contentBlockDelta.contentBlockIndex;
         if (idx != null) {
           seenBlockIndices.add(idx);
         }
 
-        yield this.enrichChunk(deltaChunk, seenBlockIndices);
-
-        await runManager?.handleLLMNewToken(
-          deltaChunk.text,
-          undefined,
-          undefined,
-          undefined,
-          undefined,
-          { chunk: deltaChunk }
-        );
-      } else if (event.metadata != null) {
-        yield handleConverseStreamMetadata(event.metadata, { streamUsage });
-      } else if (event.contentBlockStop != null) {
-        const stopIdx = event.contentBlockStop.contentBlockIndex;
-        if (stopIdx != null) {
-          seenBlockIndices.add(stopIdx);
-          if (sealToolUseOnStop && toolUseBlockIndices.has(stopIdx)) {
-            // Converse guarantees the block's input is complete at stop, so
-            // emit an explicit seal chunk for eager tool execution — through
-            // the callback path too, for registered stream handlers.
-            const sealChunk = createConverseToolUseStopChunk(stopIdx);
-            yield sealChunk;
+        const text = delta.text;
+        const reasoningContent = delta.reasoningContent;
+        const reasoningText = reasoningContent?.text;
+        const visibleText =
+          typeof text === 'string'
+            ? text
+            : typeof reasoningText === 'string'
+              ? reasoningText
+              : '';
+        const smooth = this._lc_stream_delay > 0 && visibleText !== '';
+        const tokenChunks = smooth
+          ? splitStreamToken(visibleText)
+          : [visibleText];
+
+        for (const token of tokenChunks) {
+          let splitDelta = contentBlockDelta;
+          if (typeof text === 'string') {
+            splitDelta = {
+              ...contentBlockDelta,
+              delta: { text: token },
+            };
+          } else if (
+            typeof reasoningText === 'string' &&
+            reasoningContent != null
+          ) {
+            splitDelta = {
+              ...contentBlockDelta,
+              delta: {
+                reasoningContent: {
+                  ...reasoningContent,
+                  text: token,
+                },
+              },
+            };
+          }
+
+          const deltaChunk = handleConverseStreamContentBlockDelta(splitDelta);
+          await enqueueChunk({
+            chunk: this.enrichChunk(deltaChunk, seenBlockIndices),
+            callbackChunk: deltaChunk,
+            callbackToken: deltaChunk.text,
+            smooth,
+            textLength: token.length,
+          });
+        }
+      };
+
+      const producer = (async (): Promise<void> => {
+        try {
+          for await (const event of stream) {
+            if (isSignalAborted(options.signal)) {
+              abortStream();
+              throw new Error('AbortError: User aborted the request.');
+            }
+
+            if (event.contentBlockStart != null) {
+              const startChunk = handleConverseStreamContentBlockStart(
+                event.contentBlockStart
+              );
+              if (startChunk != null) {
+                const idx = event.contentBlockStart.contentBlockIndex;
+                if (idx != null) {
+                  seenBlockIndices.add(idx);
+                  if (event.contentBlockStart.start?.toolUse != null) {
+                    toolUseBlockIndices.add(idx);
+                  }
+                }
+                await enqueueChunk({
+                  chunk: this.enrichChunk(startChunk, seenBlockIndices),
+                  callbackChunk: startChunk,
+                  callbackToken: startChunk.text,
+                });
+              }
+            } else if (event.contentBlockDelta != null) {
+              await enqueueDelta(event.contentBlockDelta);
+            } else if (event.metadata != null) {
+              await enqueueChunk({
+                chunk: handleConverseStreamMetadata(event.metadata, {
+                  streamUsage,
+                }),
+              });
+            } else if (event.contentBlockStop != null) {
+              const stopIdx = event.contentBlockStop.contentBlockIndex;
+              if (stopIdx != null) {
+                seenBlockIndices.add(stopIdx);
+                if (sealToolUseOnStop && toolUseBlockIndices.has(stopIdx)) {
+                  const sealChunk = createConverseToolUseStopChunk(stopIdx);
+                  await enqueueChunk({
+                    chunk: sealChunk,
+                    callbackChunk: sealChunk,
+                    callbackToken: sealChunk.text,
+                  });
+                }
+              }
+            } else {
+              await enqueueChunk({
+                chunk: new ChatGenerationChunk({
+                  text: '',
+                  message: new AIMessageChunk({
+                    content: '',
+                    response_metadata: { ...event } as ResponseMetadata,
+                  }),
+                }),
+              });
+            }
+          }
+        } catch (error) {
+          producerState.error = error;
+        } finally {
+          producerState.done = true;
+          notifyConsumerForChunk();
+        }
+      })();
+
+      try {
+        let keepStreaming = true;
+        while (keepStreaming) {
+          if (isSignalAborted(options.signal)) {
+            abortStream();
+            throw new Error('AbortError: User aborted the request.');
+          }
+
+          await waitForNextChunk();
+          const queuedChunk = dequeue();
+
+          if (!queuedChunk) {
+            if (producerState.error != null) {
+              throw producerState.error;
+            }
+            if (producerState.done) {
+              keepStreaming = false;
+            }
+            continue;
+          }
+
+          if (queuedChunk.smooth) {
+            bufferedTextLength = Math.max(
+              0,
+              bufferedTextLength - queuedChunk.textLength
+            );
+            notifyProducerForSpace();
+            await waitForStreamDelay(
+              getCadencedStreamDelay({
+                targetDelay: hasEmittedVisibleContent
+                  ? this._lc_stream_delay
+                  : 0,
+                lastVisibleContentAt,
+                now: Date.now(),
+              }),
+              options.signal
+            );
+            if (isSignalAborted(options.signal)) {
+              abortStream();
+              throw new Error('AbortError: User aborted the request.');
+            }
+            hasEmittedVisibleContent = true;
+            lastVisibleContentAt = Date.now();
+          } else {
+            notifyProducerForSpace();
+          }
+
+          yield queuedChunk.chunk;
+
+          if (queuedChunk.callbackChunk != null) {
             await runManager?.handleLLMNewToken(
-              sealChunk.text,
+              queuedChunk.callbackToken,
               undefined,
               undefined,
               undefined,
               undefined,
-              { chunk: sealChunk }
+              { chunk: queuedChunk.callbackChunk }
             );
           }
         }
-      } else {
-        yield new ChatGenerationChunk({
-          text: '',
-          message: new AIMessageChunk({
-            content: '',
-            response_metadata: { ...event } as ResponseMetadata,
-          }),
-        });
+      } finally {
+        consumerClosed = true;
+        if (!producerState.done) {
+          abortStream();
+          notifyProducerForSpace();
+        }
+        await producer;
+      }
+    } finally {
+      options.signal?.removeEventListener('abort', abortStream);
+      if (!streamAbortController.signal.aborted) {
+        streamAbortController.abort();
       }
     }
   }

package/src/llm/bedrock/streamSealDispatch.test.ts

@@ -155,4 +155,101 @@ describe('Converse stream seal dispatch', () => {
     expect(dispatched.some(hasSeal)).toBe(false);
     expect(dispatched).toHaveLength(1);
   });
+
+  test('paces visible text deltas with configured stream delay', async () => {
+    const setTimeoutSpy = jest.spyOn(globalThis, 'setTimeout');
+
+    try {
+      const { yielded, dispatched } = await runStream(
+        [
+          {
+            contentBlockDelta: {
+              contentBlockIndex: 0,
+              delta: { text: 'hello' },
+            },
+          },
+          {
+            contentBlockDelta: {
+              contentBlockIndex: 0,
+              delta: { text: ' world' },
+            },
+          },
+        ],
+        { _lc_stream_delay: 35 }
+      );
+
+      expect(yielded.map((m) => m.content)).toEqual(['hello', ' world']);
+      expect(dispatched.map((m) => m.content)).toEqual(['hello', ' world']);
+
+      const timeoutDelays = setTimeoutSpy.mock.calls
+        .map(([, delay]) => delay)
+        .filter((delay): delay is number => typeof delay === 'number');
+      expect(timeoutDelays.some((delay) => delay > 0 && delay <= 35)).toBe(
+        true
+      );
+    } finally {
+      setTimeoutSpy.mockRestore();
+    }
+  });
+
+  test('splits large text deltas at Anthropic stream boundaries', async () => {
+    const { yielded, dispatched } = await runStream(
+      [
+        {
+          contentBlockDelta: {
+            contentBlockIndex: 0,
+            delta: { text: 'alpha beta gamma' },
+          },
+        },
+      ],
+      { _lc_stream_delay: 1 }
+    );
+
+    expect(yielded.map((m) => m.content)).toEqual(['alpha ', 'beta ', 'gamma']);
+    expect(dispatched.map((m) => m.content)).toEqual([
+      'alpha ',
+      'beta ',
+      'gamma',
+    ]);
+  });
+
+  test('paces visible reasoning deltas with configured stream delay', async () => {
+    const setTimeoutSpy = jest.spyOn(globalThis, 'setTimeout');
+
+    try {
+      const { yielded, dispatched } = await runStream(
+        [
+          {
+            contentBlockDelta: {
+              contentBlockIndex: 0,
+              delta: { reasoningContent: { text: 'One' } },
+            },
+          },
+          {
+            contentBlockDelta: {
+              contentBlockIndex: 0,
+              delta: { reasoningContent: { text: 'Two' } },
+            },
+          },
+        ],
+        { _lc_stream_delay: 35 }
+      );
+
+      expect(yielded.map((m) => m.additional_kwargs.reasoning_content)).toEqual(
+        ['One', 'Two']
+      );
+      expect(
+        dispatched.map((m) => m.additional_kwargs.reasoning_content)
+      ).toEqual(['One', 'Two']);
+
+      const timeoutDelays = setTimeoutSpy.mock.calls
+        .map(([, delay]) => delay)
+        .filter((delay): delay is number => typeof delay === 'number');
+      expect(timeoutDelays.some((delay) => delay > 0 && delay <= 35)).toBe(
+        true
+      );
+    } finally {
+      setTimeoutSpy.mockRestore();
+    }
+  });
 });