@librechat/agents 3.3.13 → 3.4.0

package/src/llm/bedrock/streamSealDispatch.test.ts

@@ -252,4 +252,80 @@ describe('Converse stream seal dispatch', () => {
       setTimeoutSpy.mockRestore();
     }
   });
+
+  test('emits toolUse seal chunks without pacing delay while text is paced', async () => {
+    const model = new CustomChatBedrockConverse({
+      model: 'anthropic.claude-3-5-sonnet-20240620-v1:0',
+      region: 'us-east-1',
+      credentials: { accessKeyId: 'test', secretAccessKey: 'test' },
+      _lc_stream_delay: 60,
+    });
+
+    (model as unknown as { client: { send: unknown } }).client.send = jest.fn(
+      async () => ({
+        stream: (async function* () {
+          yield {
+            contentBlockDelta: {
+              contentBlockIndex: 0,
+              delta: { text: 'alpha beta' },
+            },
+          };
+          yield {
+            contentBlockStart: {
+              contentBlockIndex: 1,
+              start: { toolUse: { toolUseId: 'call_1', name: 'weather' } },
+            },
+          };
+          yield {
+            contentBlockDelta: {
+              contentBlockIndex: 1,
+              delta: { toolUse: { input: '{"city":"NYC"}' } },
+            },
+          };
+          yield { contentBlockStop: { contentBlockIndex: 1 } };
+        })(),
+      })
+    );
+
+    const arrivals: { isSeal: boolean; hasText: boolean; at: number }[] = [];
+    for await (const chunk of model._streamResponseChunks(
+      [new HumanMessage('hi')],
+      {} as Parameters<CustomChatBedrockConverse['_streamResponseChunks']>[1],
+      undefined
+    )) {
+      const message = chunk.message as AIMessageChunk;
+      arrivals.push({
+        isSeal:
+          (message.response_metadata as Record<string, unknown>)[
+            STREAMED_TOOL_CALL_SEAL_METADATA_KEY
+          ] != null,
+        hasText: chunk.text !== '',
+        at: Date.now(),
+      });
+    }
+
+    const lastText = [...arrivals].reverse().find((entry) => entry.hasText);
+    const seal = arrivals.find((entry) => entry.isSeal);
+    expect(lastText).toBeDefined();
+    expect(seal).toBeDefined();
+    expect((seal as { at: number }).at).toBeGreaterThanOrEqual(
+      (lastText as { at: number }).at
+    );
+    expect(
+      (seal as { at: number }).at - (lastText as { at: number }).at
+    ).toBeLessThan(45);
+  });
+
+  test('defaults to 25ms adaptive smoothing with 0 disabling', () => {
+    const base = {
+      model: 'anthropic.claude-3-5-sonnet-20240620-v1:0',
+      region: 'us-east-1',
+      credentials: { accessKeyId: 'test', secretAccessKey: 'test' },
+    };
+    expect(new CustomChatBedrockConverse(base)._lc_stream_delay).toBe(25);
+    expect(
+      new CustomChatBedrockConverse({ ...base, _lc_stream_delay: 0 })
+        ._lc_stream_delay
+    ).toBe(0);
+  });
 });

package/src/llm/custom-chat-models.smoke.test.ts

@@ -1836,7 +1836,7 @@ describe('custom chat model class smoke tests', () => {
     );
     expect(model.applicationInferenceProfile).toBe(applicationInferenceProfile);
     expect(model._lc_stream_delay).toBe(12);
-    expect(defaultStreamDelayModel._lc_stream_delay).toBe(0);
+    expect(defaultStreamDelayModel._lc_stream_delay).toBe(25);
     expect(model.invocationParams({}).serviceTier).toEqual({
       type: 'priority',
     });
@@ -1949,3 +1949,18 @@ describe('custom chat model class smoke tests', () => {
     expect(capturedFetch.getSignal()?.aborted).toBe(true);
   });
 });
+
+describe('StreamSmoothingOptions type surface', () => {
+  it('accepts _lc_stream_delay: 0 across the typed provider options', () => {
+    const azure: import('@/types').AzureClientOptions = { _lc_stream_delay: 0 };
+    const openrouter: ChatOpenRouterCallOptions &
+      import('@/types').StreamSmoothingOptions = { _lc_stream_delay: 0 };
+    const mistral: import('@/types').MistralAIClientOptions = {
+      model: 'mistral-large-latest',
+      _lc_stream_delay: 0,
+    };
+    expect(azure._lc_stream_delay).toBe(0);
+    expect(openrouter._lc_stream_delay).toBe(0);
+    expect(mistral._lc_stream_delay).toBe(0);
+  });
+});

package/src/llm/google/index.ts

@@ -16,6 +16,8 @@ import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager
 import type { BaseMessage, UsageMetadata } from '@langchain/core/messages';
 import type { GeminiApiUsageMetadata, InputTokenDetails } from './types';
 import type { GoogleClientOptions, GoogleThinkingConfig } from '@/types';
+import { smoothGenerationChunks } from '@/llm/stream/chunkAdapters';
+import { resolveStreamDelay } from '@/llm/stream/smoother';
 import {
   convertResponseContentToChatGenerationChunk,
   convertBaseMessagesToContent,
@@ -36,6 +38,7 @@ type GoogleToolConfigWithServerSideInvocations = ToolConfig & {
 };
 
 export class CustomChatGoogleGenerativeAI extends ChatGoogleGenerativeAI {
+  _lc_stream_delay: number;
   thinkingConfig?: GoogleThinkingConfig;
   includeServerSideToolInvocations?: boolean;
 
@@ -55,6 +58,7 @@ export class CustomChatGoogleGenerativeAI extends ChatGoogleGenerativeAI {
   constructor(fields: GoogleClientOptions) {
     super(fields);
 
+    this._lc_stream_delay = resolveStreamDelay(fields._lc_stream_delay);
     this.model = fields.model.replace(/^models\//, '');
 
     this.maxOutputTokens = fields.maxOutputTokens ?? this.maxOutputTokens;
@@ -296,6 +300,18 @@ export class CustomChatGoogleGenerativeAI extends ChatGoogleGenerativeAI {
     messages: BaseMessage[],
     options: this['ParsedCallOptions'],
     runManager?: CallbackManagerForLLMRun
+  ): AsyncGenerator<ChatGenerationChunk> {
+    yield* smoothGenerationChunks({
+      chunks: this._streamProviderChunks(messages, options),
+      delayMs: this._lc_stream_delay,
+      signal: options.signal,
+      runManager,
+    });
+  }
+
+  private async *_streamProviderChunks(
+    messages: BaseMessage[],
+    options: this['ParsedCallOptions']
   ): AsyncGenerator<ChatGenerationChunk> {
     const prompt = convertBaseMessagesToContent(
       messages,
@@ -349,33 +365,16 @@ export class CustomChatGoogleGenerativeAI extends ChatGoogleGenerativeAI {
       }
 
       yield chunk;
-      await runManager?.handleLLMNewToken(
-        chunk.text || '',
-        undefined,
-        undefined,
-        undefined,
-        undefined,
-        { chunk }
-      );
     }
 
     if (lastUsageMetadata) {
-      const finalChunk = new ChatGenerationChunk({
+      yield new ChatGenerationChunk({
         text: '',
         message: new AIMessageChunk({
           content: '',
           usage_metadata: lastUsageMetadata,
         }),
       });
-      yield finalChunk;
-      await runManager?.handleLLMNewToken(
-        finalChunk.text || '',
-        undefined,
-        undefined,
-        undefined,
-        undefined,
-        { chunk: finalChunk }
-      );
     }
   }
 }

package/src/llm/google/streamSmoothing.test.ts

@@ -0,0 +1,121 @@
+import { expect, test, describe, jest } from '@jest/globals';
+import { HumanMessage, AIMessageChunk } from '@langchain/core/messages';
+import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
+import type { ChatGenerationChunk } from '@langchain/core/outputs';
+import { CustomChatGoogleGenerativeAI } from './index';
+
+describe('Google stream smoothing', () => {
+  function textResponse(text: string): Record<string, unknown> {
+    return {
+      candidates: [
+        {
+          content: { role: 'model', parts: [{ text }] },
+          index: 0,
+        },
+      ],
+    };
+  }
+
+  async function runStream(
+    responses: Record<string, unknown>[],
+    modelFields: Record<string, unknown> = {}
+  ): Promise<{
+    yielded: ChatGenerationChunk[];
+    dispatchedTokens: string[];
+  }> {
+    const model = new CustomChatGoogleGenerativeAI({
+      model: 'gemini-2.5-flash',
+      apiKey: 'test-key',
+      ...modelFields,
+    });
+
+    (
+      model as unknown as {
+        client: { generateContentStream: unknown };
+      }
+    ).client.generateContentStream = jest.fn(async () => ({
+      stream: (async function* () {
+        yield* responses;
+      })(),
+    }));
+
+    const dispatchedTokens: string[] = [];
+    const runManager = {
+      handleLLMNewToken: jest.fn(async (token: string) => {
+        dispatchedTokens.push(token);
+      }),
+    } as unknown as CallbackManagerForLLMRun;
+
+    const yielded: ChatGenerationChunk[] = [];
+    for await (const chunk of model._streamResponseChunks(
+      [new HumanMessage('hi')],
+      {} as Parameters<
+        CustomChatGoogleGenerativeAI['_streamResponseChunks']
+      >[1],
+      runManager
+    )) {
+      yielded.push(chunk);
+    }
+    return { yielded, dispatchedTokens };
+  }
+
+  test('splits large text responses at stream boundaries with pacing', async () => {
+    const { yielded, dispatchedTokens } = await runStream(
+      [textResponse('alpha beta gamma')],
+      { _lc_stream_delay: 1 }
+    );
+
+    const texts = yielded.map((chunk) => chunk.text).filter(Boolean);
+    expect(texts).toEqual(['alpha ', 'beta ', 'gamma']);
+    expect(dispatchedTokens.filter(Boolean)).toEqual([
+      'alpha ',
+      'beta ',
+      'gamma',
+    ]);
+  });
+
+  test('passes chunks through unsplit when smoothing is disabled', async () => {
+    const { yielded } = await runStream([textResponse('alpha beta gamma')], {
+      _lc_stream_delay: 0,
+    });
+
+    expect(yielded.map((chunk) => chunk.text).filter(Boolean)).toEqual([
+      'alpha beta gamma',
+    ]);
+  });
+
+  test('emits the final usage chunk without pacing delay', async () => {
+    const start = Date.now();
+    const { yielded } = await runStream(
+      [
+        {
+          ...textResponse('short text here'),
+          usageMetadata: {
+            promptTokenCount: 3,
+            candidatesTokenCount: 4,
+            totalTokenCount: 7,
+          },
+        },
+      ],
+      { _lc_stream_delay: 30 }
+    );
+    const elapsed = Date.now() - start;
+
+    const usageChunk = yielded.find(
+      (chunk) =>
+        chunk.text === '' &&
+        (chunk.message as AIMessageChunk).usage_metadata != null
+    );
+    expect(usageChunk).toBeDefined();
+    expect(elapsed).toBeLessThan(1000);
+  });
+
+  test('defaults to 25ms adaptive smoothing with 0 disabling', () => {
+    const base = { model: 'gemini-2.5-flash', apiKey: 'test-key' };
+    expect(new CustomChatGoogleGenerativeAI(base)._lc_stream_delay).toBe(25);
+    expect(
+      new CustomChatGoogleGenerativeAI({ ...base, _lc_stream_delay: 0 })
+        ._lc_stream_delay
+    ).toBe(0);
+  });
+});

package/src/llm/mistral/index.ts

@@ -0,0 +1,33 @@
+import { ChatMistralAI } from '@langchain/mistralai';
+import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
+import type { ChatGenerationChunk } from '@langchain/core/outputs';
+import type { BaseMessage } from '@langchain/core/messages';
+import type { MistralAIClientOptions } from '@/types';
+import { smoothGenerationChunks } from '@/llm/stream/chunkAdapters';
+import { resolveStreamDelay } from '@/llm/stream/smoother';
+
+export class CustomChatMistralAI extends ChatMistralAI {
+  _lc_stream_delay: number;
+
+  static lc_name(): 'LibreChatMistralAI' {
+    return 'LibreChatMistralAI';
+  }
+
+  constructor(fields?: MistralAIClientOptions) {
+    super(fields);
+    this._lc_stream_delay = resolveStreamDelay(fields?._lc_stream_delay);
+  }
+
+  async *_streamResponseChunks(
+    messages: BaseMessage[],
+    options: this['ParsedCallOptions'],
+    runManager?: CallbackManagerForLLMRun
+  ): AsyncGenerator<ChatGenerationChunk> {
+    yield* smoothGenerationChunks({
+      chunks: super._streamResponseChunks(messages, options, undefined),
+      delayMs: this._lc_stream_delay,
+      signal: options.signal,
+      runManager,
+    });
+  }
+}

package/src/llm/mistral/streamSmoothing.test.ts

@@ -0,0 +1,97 @@
+import { expect, test, describe, jest, afterEach } from '@jest/globals';
+import { AIMessageChunk, HumanMessage } from '@langchain/core/messages';
+import { ChatGenerationChunk } from '@langchain/core/outputs';
+import { ChatMistralAI } from '@langchain/mistralai';
+import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
+import { CustomChatMistralAI } from './index';
+
+describe('Mistral stream smoothing', () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  function stubParentStream(texts: string[]): void {
+    jest
+      .spyOn(
+        ChatMistralAI.prototype as unknown as {
+          _streamResponseChunks: (
+            ...args: unknown[]
+          ) => AsyncGenerator<ChatGenerationChunk>;
+        },
+        '_streamResponseChunks'
+      )
+      .mockImplementation(async function* () {
+        for (const text of texts) {
+          yield new ChatGenerationChunk({
+            text,
+            message: new AIMessageChunk({ content: text }),
+          });
+        }
+      });
+  }
+
+  async function collect(
+    model: CustomChatMistralAI,
+    runManager?: CallbackManagerForLLMRun
+  ): Promise<string[]> {
+    const texts: string[] = [];
+    for await (const chunk of model._streamResponseChunks(
+      [new HumanMessage('hi')],
+      {} as Parameters<CustomChatMistralAI['_streamResponseChunks']>[1],
+      runManager
+    )) {
+      if (chunk.text) {
+        texts.push(chunk.text);
+      }
+    }
+    return texts;
+  }
+
+  test('splits large text chunks at stream boundaries with pacing', async () => {
+    stubParentStream(['alpha beta gamma']);
+    const model = new CustomChatMistralAI({
+      model: 'mistral-large-latest',
+      apiKey: 'test-key',
+      _lc_stream_delay: 1,
+    });
+
+    const dispatchedTokens: string[] = [];
+    const runManager = {
+      handleLLMNewToken: jest.fn(async (token: string) => {
+        dispatchedTokens.push(token);
+      }),
+    } as unknown as CallbackManagerForLLMRun;
+
+    expect(await collect(model, runManager)).toEqual([
+      'alpha ',
+      'beta ',
+      'gamma',
+    ]);
+    expect(dispatchedTokens.filter(Boolean)).toEqual([
+      'alpha ',
+      'beta ',
+      'gamma',
+    ]);
+  });
+
+  test('passes chunks through unsplit when smoothing is disabled', async () => {
+    stubParentStream(['alpha beta gamma']);
+    const model = new CustomChatMistralAI({
+      model: 'mistral-large-latest',
+      apiKey: 'test-key',
+      _lc_stream_delay: 0,
+    });
+
+    expect(await collect(model)).toEqual(['alpha beta gamma']);
+  });
+
+  test('defaults to 25ms adaptive smoothing with 0 disabling', () => {
+    const base = { model: 'mistral-large-latest', apiKey: 'test-key' };
+    expect(new CustomChatMistralAI(base)._lc_stream_delay).toBe(25);
+    expect(
+      new CustomChatMistralAI({ ...base, _lc_stream_delay: 0 })
+        ._lc_stream_delay
+    ).toBe(0);
+    expect(CustomChatMistralAI.lc_name()).toBe('LibreChatMistralAI');
+  });
+});

package/src/llm/openai/deepseek.test.ts

@@ -557,6 +557,62 @@ describe('ChatDeepSeek', () => {
     expect(textChunks).toEqual(['alpha ', 'beta ', 'gamma']);
   });
 
+  it('does not delay tool-call and finish chunks between paced text', async () => {
+    const toolCallChunk: OpenAIChatCompletionChunk = {
+      id: 'chatcmpl-deepseek-test',
+      object: 'chat.completion.chunk',
+      created: 0,
+      model: 'deepseek-v4-pro',
+      choices: [
+        {
+          index: 0,
+          delta: {
+            role: 'assistant',
+            tool_calls: [
+              {
+                index: 0,
+                id: 'call_1',
+                type: 'function',
+                function: { name: 'lookup', arguments: '{}' },
+              },
+            ],
+          },
+          finish_reason: null,
+          logprobs: null,
+        },
+      ],
+    };
+    const model = new CapturingChatDeepSeek(
+      {
+        apiKey: 'test-key',
+        model: 'deepseek-v4-pro',
+        streaming: true,
+        _lc_stream_delay: 150,
+      },
+      [createContentChunk('alpha beta'), toolCallChunk]
+    );
+    const arrivals: { text: string; hasToolCall: boolean; at: number }[] = [];
+
+    for await (const chunk of model.streamChunksWithSignal(
+      new AbortController().signal
+    )) {
+      const message = chunk.message as Partial<AIMessage>;
+      arrivals.push({
+        text: chunk.text,
+        hasToolCall: (message.tool_calls?.length ?? 0) > 0,
+        at: Date.now(),
+      });
+    }
+
+    const toolArrival = arrivals.find((entry) => entry.hasToolCall);
+    const lastText = [...arrivals].reverse().find((entry) => entry.text !== '');
+    expect(toolArrival).toBeDefined();
+    expect(lastText).toBeDefined();
+    expect(
+      (toolArrival as { at: number }).at - (lastText as { at: number }).at
+    ).toBeLessThan(100);
+  });
+
   it('keeps delayed DeepSeek logprob chunks intact', async () => {
     const logprobs = { content: [], refusal: null } as NonNullable<
       OpenAIChatCompletionChunk['choices'][number]['logprobs']