@librechat/agents 3.1.86 → 3.1.88

package/src/llm/openai/index.ts

@@ -35,11 +35,13 @@ import type { ChatGeneration, ChatResult } from '@langchain/core/outputs';
 import type { ChatXAIInput } from '@langchain/xai';
 import type * as t from '@langchain/openai';
 import { isReasoningModel, _convertMessagesToOpenAIParams } from './utils';
-import { sleep } from '@/utils';
 
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
 const iife = <T>(fn: () => T) => fn();
 
+const STREAM_CHUNK_MIN_SIZE = 4;
+const STREAM_BOUNDARIES = new Set([' ', '.', ',', '!', '?', ';', ':']);
+
 export function isHeaders(headers: unknown): headers is Headers {
   return (
     typeof Headers !== 'undefined' &&
@@ -403,18 +405,160 @@ function getCustomOpenAIClientOptions(
   return requestOptions;
 }
 
-async function* delayStreamChunks<T>(
-  chunks: AsyncGenerator<T>,
-  delay?: number
-): AsyncGenerator<T> {
+function findStreamChunkBoundary(text: string, minSize: number): number {
+  if (minSize >= text.length) {
+    return text.length;
+  }
+
+  for (let position = minSize; position < text.length; position++) {
+    if (STREAM_BOUNDARIES.has(text[position])) {
+      return position + 1;
+    }
+  }
+
+  return text.length;
+}
+
+function splitStreamToken(text: string): string[] {
+  const chunks: string[] = [];
+  let currentIndex = 0;
+
+  while (currentIndex < text.length) {
+    const remainingText = text.slice(currentIndex);
+    const chunkSize = findStreamChunkBoundary(
+      remainingText,
+      STREAM_CHUNK_MIN_SIZE
+    );
+    chunks.push(text.slice(currentIndex, currentIndex + chunkSize));
+    currentIndex += chunkSize;
+  }
+
+  return chunks;
+}
+
+function splitTextGenerationChunk(
+  chunk: ChatGenerationChunk
+): ChatGenerationChunk[] {
+  const { message } = chunk;
+  if (
+    !chunk.text ||
+    !(message instanceof AIMessageChunk) ||
+    typeof message.content !== 'string' ||
+    message.content !== chunk.text ||
+    chunk.generationInfo?.logprobs != null ||
+    chunk.generationInfo?.finish_reason != null
+  ) {
+    return [chunk];
+  }
+
+  const tokenChunks = splitStreamToken(chunk.text);
+  if (tokenChunks.length <= 1) {
+    return [chunk];
+  }
+
+  let emittedUsage = false;
+  return tokenChunks.map((token) => {
+    const usageMetadata =
+      emittedUsage && message.usage_metadata != null
+        ? undefined
+        : message.usage_metadata;
+    if (message.usage_metadata != null && !emittedUsage) {
+      emittedUsage = true;
+    }
+
+    return new ChatGenerationChunk({
+      text: token,
+      generationInfo: chunk.generationInfo,
+      message: new AIMessageChunk(
+        Object.assign({}, message, {
+          content: token,
+          usage_metadata: usageMetadata,
+        })
+      ),
+    });
+  });
+}
+
+export async function emitStreamChunkCallback(
+  chunk: ChatGenerationChunk,
+  runManager?: CallbackManagerForLLMRun
+): Promise<void> {
+  await runManager?.handleLLMNewToken(
+    chunk.text,
+    getStreamChunkTokenIndices(chunk),
+    undefined,
+    undefined,
+    undefined,
+    { chunk }
+  );
+}
+
+function getStreamChunkTokenIndices(
+  chunk: ChatGenerationChunk
+): { prompt: number; completion: number } | undefined {
+  const prompt = chunk.generationInfo?.prompt;
+  const completion = chunk.generationInfo?.completion;
+
+  if (typeof prompt === 'number' && typeof completion === 'number') {
+    return { prompt, completion };
+  }
+
+  return undefined;
+}
+
+async function* delayStreamChunks(
+  chunks: AsyncGenerator<ChatGenerationChunk>,
+  delay?: number,
+  signal?: AbortSignal,
+  runManager?: CallbackManagerForLLMRun
+): AsyncGenerator<ChatGenerationChunk> {
+  let lastYieldedAt: number | undefined;
   for await (const chunk of chunks) {
-    yield chunk;
-    if (delay != null) {
-      await sleep(delay);
+    const outputChunks =
+      delay != null && delay > 0 ? splitTextGenerationChunk(chunk) : [chunk];
+    for (const outputChunk of outputChunks) {
+      signal?.throwIfAborted();
+      if (delay != null && delay > 0 && lastYieldedAt != null) {
+        const timeSinceLastYield = Date.now() - lastYieldedAt;
+        const timeToWait = Math.max(0, delay - timeSinceLastYield);
+        if (timeToWait > 0) {
+          await sleepWithAbort(timeToWait, signal);
+        }
+      }
+      signal?.throwIfAborted();
+      lastYieldedAt = Date.now();
+      await emitStreamChunkCallback(outputChunk, runManager);
+      signal?.throwIfAborted();
+      yield outputChunk;
     }
   }
 }
 
+async function sleepWithAbort(
+  delay: number,
+  signal?: AbortSignal
+): Promise<void> {
+  if (delay <= 0) {
+    return;
+  }
+  signal?.throwIfAborted();
+  await new Promise<void>((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      signal?.removeEventListener('abort', onAbort);
+      resolve();
+    }, delay);
+    const onAbort = (): void => {
+      clearTimeout(timeout);
+      signal?.removeEventListener('abort', onAbort);
+      reject(signal?.reason ?? new Error('AbortError: User aborted request.'));
+    };
+    signal?.addEventListener('abort', onAbort, { once: true });
+    if (signal?.aborted === true) {
+      onAbort();
+    }
+  });
+}
+
 function createAbortHandler(controller: AbortController): () => void {
   return function (): void {
     controller.abort();
@@ -468,7 +612,7 @@ export class CustomOpenAIClient extends OpenAIClient {
     this.abortHandler = handler;
     if (signal) signal.addEventListener('abort', handler, { once: true });
 
-    const timeout = setTimeout(() => handler, ms);
+    const timeout = setTimeout(handler, ms);
 
     const fetchOptions = {
       signal: controller.signal as AbortSignal,
@@ -503,7 +647,7 @@ export class CustomAzureOpenAIClient extends AzureOpenAIClient {
     this.abortHandler = handler;
     if (signal) signal.addEventListener('abort', handler, { once: true });
 
-    const timeout = setTimeout(() => handler, ms);
+    const timeout = setTimeout(handler, ms);
 
     const fetchOptions = {
       signal: controller.signal as AbortSignal,
@@ -1184,8 +1328,10 @@ export class ChatOpenAI extends OriginalChatOpenAI<t.ChatOpenAICallOptions> {
     runManager?: CallbackManagerForLLMRun
   ): AsyncGenerator<ChatGenerationChunk> {
     yield* delayStreamChunks(
-      super._streamResponseChunks(messages, options, runManager),
-      this._lc_stream_delay
+      super._streamResponseChunks(messages, options, undefined),
+      this._lc_stream_delay,
+      options.signal,
+      runManager
     );
   }
 }
@@ -1294,8 +1440,10 @@ export class AzureChatOpenAI extends OriginalAzureChatOpenAI {
     runManager?: CallbackManagerForLLMRun
   ): AsyncGenerator<ChatGenerationChunk> {
     yield* delayStreamChunks(
-      super._streamResponseChunks(messages, options, runManager),
-      this._lc_stream_delay
+      super._streamResponseChunks(messages, options, undefined),
+      this._lc_stream_delay,
+      options.signal,
+      runManager
     );
   }
 }
@@ -1425,8 +1573,10 @@ export class ChatDeepSeek extends OriginalChatDeepSeek {
     runManager?: CallbackManagerForLLMRun
   ): AsyncGenerator<ChatGenerationChunk> {
     yield* delayStreamChunks(
-      this._streamResponseChunksWithReasoning(messages, options, runManager),
-      this._lc_stream_delay
+      this._streamResponseChunksWithReasoning(messages, options, undefined),
+      this._lc_stream_delay,
+      options.signal,
+      runManager
     );
   }
 
@@ -1767,14 +1917,7 @@ export class ChatDeepSeek extends OriginalChatDeepSeek {
   protected _getDeepSeekTokenIndices(
     chunk: ChatGenerationChunk
   ): { prompt: number; completion: number } | undefined {
-    const prompt = chunk.generationInfo?.prompt;
-    const completion = chunk.generationInfo?.completion;
-
-    if (typeof prompt === 'number' && typeof completion === 'number') {
-      return { prompt, completion };
-    }
-
-    return undefined;
+    return getStreamChunkTokenIndices(chunk);
   }
 
   protected _getDeepSeekPartialTagSplitIndex(
@@ -1891,8 +2034,10 @@ export class ChatXAI extends OriginalChatXAI {
     runManager?: CallbackManagerForLLMRun
   ): AsyncGenerator<ChatGenerationChunk> {
     yield* delayStreamChunks(
-      super._streamResponseChunks(messages, options, runManager),
-      this._lc_stream_delay
+      super._streamResponseChunks(messages, options, undefined),
+      this._lc_stream_delay,
+      options.signal,
+      runManager
     );
   }
 }

package/src/llm/openai/utils/index.ts

@@ -38,6 +38,11 @@ import type {
   ChatOpenAIReasoningSummary,
 } from '@langchain/openai';
 import { toLangChainContent } from '@/messages/langchain';
+import {
+  STREAMED_TOOL_CALL_SEAL_METADATA_KEY,
+  STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY,
+  OPENAI_RESPONSES_STREAMED_TOOL_CALL_ADAPTER,
+} from '@/tools/streamedToolCallSeals';
 
 export type { OpenAICallOptions, OpenAIChatInput };
 
@@ -948,6 +953,8 @@ export function _convertOpenAIResponsesDeltaToBaseMessageChunk(
     chunk.type === 'response.output_item.added' &&
     chunk.item.type === 'function_call'
   ) {
+    response_metadata[STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY] =
+      OPENAI_RESPONSES_STREAMED_TOOL_CALL_ADAPTER;
     tool_call_chunks.push({
       type: 'tool_call_chunk',
       name: chunk.item.name,
@@ -988,11 +995,26 @@ export function _convertOpenAIResponsesDeltaToBaseMessageChunk(
       if (key !== 'id') response_metadata[key] = value;
     }
   } else if (chunk.type === 'response.function_call_arguments.delta') {
+    response_metadata[STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY] =
+      OPENAI_RESPONSES_STREAMED_TOOL_CALL_ADAPTER;
     tool_call_chunks.push({
       type: 'tool_call_chunk',
       args: chunk.delta,
       index: chunk.output_index,
     });
+  } else if (chunk.type === 'response.function_call_arguments.done') {
+    response_metadata[STREAMED_TOOL_CALL_ADAPTER_METADATA_KEY] =
+      OPENAI_RESPONSES_STREAMED_TOOL_CALL_ADAPTER;
+    response_metadata[STREAMED_TOOL_CALL_SEAL_METADATA_KEY] = {
+      kind: 'single',
+      index: chunk.output_index,
+    };
+    tool_call_chunks.push({
+      type: 'tool_call_chunk',
+      name: chunk.name,
+      args: chunk.arguments,
+      index: chunk.output_index,
+    });
   } else if (
     chunk.type === 'response.web_search_call.completed' ||
     chunk.type === 'response.file_search_call.completed'

package/src/llm/openrouter/index.ts

@@ -1,4 +1,4 @@
-import { ChatOpenAI } from '@/llm/openai';
+import { ChatOpenAI, emitStreamChunkCallback } from '@/llm/openai';
 import type { BaseMessage } from '@langchain/core/messages';
 import type { ChatGenerationChunk } from '@langchain/core/outputs';
 import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
@@ -237,7 +237,7 @@ export class ChatOpenRouter extends ChatOpenAI {
     for await (const generationChunk of super._streamResponseChunks(
       messages,
       options,
-      runManager
+      undefined
     )) {
       let currentReasoningText = '';
       const reasoningDetails = getReasoningDetails(
@@ -283,11 +283,13 @@ export class ChatOpenRouter extends ChatOpenAI {
         } else {
           delete generationChunk.message.additional_kwargs.reasoning_details;
         }
+        await emitStreamChunkCallback(generationChunk, runManager);
         yield generationChunk;
         continue;
       }
 
       delete generationChunk.message.additional_kwargs.reasoning_details;
+      await emitStreamChunkCallback(generationChunk, runManager);
       yield generationChunk;
     }
   }

package/src/openai/__tests__/openai.test.ts

@@ -0,0 +1,337 @@
+import { GraphEvents } from '@/common';
+import {
+  createChatCompletionChunk,
+  createOpenAIHandlers,
+  createOpenAIStreamTracker,
+  sendOpenAIFinalChunk,
+} from '@/openai';
+import type * as t from '@/types';
+
+describe('OpenAI-compatible adapters', () => {
+  it('creates chunks and streams message deltas as SSE data', async () => {
+    const writes: string[] = [];
+    const handlers = createOpenAIHandlers({
+      writer: { write: (data) => void writes.push(data) },
+      context: { requestId: 'chatcmpl_1', model: 'agent', created: 1 },
+      tracker: createOpenAIStreamTracker(),
+    });
+
+    await handlers[GraphEvents.ON_MESSAGE_DELTA].handle(
+      GraphEvents.ON_MESSAGE_DELTA,
+      {
+        id: 'msg',
+        delta: { content: [{ type: 'text', text: 'hello' }] },
+      } satisfies t.MessageDeltaEvent
+    );
+
+    expect(writes).toHaveLength(2);
+    expect(writes[0]).toContain('"role":"assistant"');
+    expect(writes[1]).toContain('"content":"hello"');
+  });
+
+  it('sends a final usage chunk and done marker', async () => {
+    const writes: string[] = [];
+    const tracker = createOpenAIStreamTracker();
+    tracker.usage.promptTokens = 3;
+    tracker.usage.completionTokens = 5;
+
+    await sendOpenAIFinalChunk({
+      writer: { write: (data) => void writes.push(data) },
+      context: { requestId: 'chatcmpl_2', model: 'agent', created: 1 },
+      tracker,
+    });
+
+    expect(writes).toHaveLength(4);
+    expect(writes[0]).toContain('"role":"assistant"');
+    expect(writes[1]).toContain('"finish_reason":"stop"');
+    expect(writes[1]).not.toContain('"usage"');
+    expect(writes[2]).toContain('"choices":[]');
+    expect(writes[2]).toContain('"total_tokens":8');
+    expect(writes[3]).toBe('data: [DONE]\n\n');
+  });
+
+  it('uses tool_calls finish reason after streaming tool deltas', async () => {
+    const writes: string[] = [];
+    const tracker = createOpenAIStreamTracker();
+    const handlers = createOpenAIHandlers({
+      writer: { write: (data) => void writes.push(data) },
+      context: { requestId: 'chatcmpl_tools', model: 'agent', created: 1 },
+      tracker,
+    });
+
+    await handlers[GraphEvents.ON_RUN_STEP_DELTA].handle(
+      GraphEvents.ON_RUN_STEP_DELTA,
+      {
+        id: 'step_1',
+        delta: {
+          type: 'tool_calls',
+          tool_calls: [
+            {
+              index: 0,
+              id: 'call_1',
+              name: 'search',
+              args: '{"query":"sessions"}',
+            },
+          ],
+        },
+      } as t.RunStepDeltaEvent
+    );
+    await sendOpenAIFinalChunk({
+      writer: { write: (data) => void writes.push(data) },
+      context: { requestId: 'chatcmpl_tools', model: 'agent', created: 1 },
+      tracker,
+    });
+
+    expect(writes[0]).toContain('"role":"assistant"');
+    expect(writes[1]).toContain('"tool_calls"');
+    expect(writes.at(-3)).toContain('"finish_reason":"tool_calls"');
+    expect(writes.at(-2)).toContain('"choices":[]');
+  });
+
+  it('uses stop finish reason when assistant text follows tool calls', async () => {
+    const writes: string[] = [];
+    const tracker = createOpenAIStreamTracker();
+    const handlers = createOpenAIHandlers({
+      writer: { write: (data) => void writes.push(data) },
+      context: { requestId: 'chatcmpl_tools_done', model: 'agent', created: 1 },
+      tracker,
+    });
+
+    await handlers[GraphEvents.ON_RUN_STEP_DELTA].handle(
+      GraphEvents.ON_RUN_STEP_DELTA,
+      {
+        id: 'step_1',
+        delta: {
+          type: 'tool_calls',
+          tool_calls: [{ index: 0, id: 'call_1', name: 'search' }],
+        },
+      } as t.RunStepDeltaEvent
+    );
+    await handlers[GraphEvents.ON_MESSAGE_DELTA].handle(
+      GraphEvents.ON_MESSAGE_DELTA,
+      {
+        id: 'msg',
+        delta: { content: [{ type: 'text', text: 'done' }] },
+      } satisfies t.MessageDeltaEvent
+    );
+    await handlers[GraphEvents.ON_RUN_STEP_DELTA].handle(
+      GraphEvents.ON_RUN_STEP_DELTA,
+      {
+        id: 'step_1',
+        delta: {
+          type: 'tool_calls',
+          tool_calls: [{ index: 0, id: 'call_1', name: 'search' }],
+        },
+      } as t.RunStepDeltaEvent
+    );
+    await sendOpenAIFinalChunk({
+      writer: { write: (data) => void writes.push(data) },
+      context: { requestId: 'chatcmpl_tools_done', model: 'agent', created: 1 },
+      tracker,
+    });
+
+    expect(writes.at(-3)).toContain('"finish_reason":"stop"');
+    expect(writes.at(-2)).toContain('"choices":[]');
+  });
+
+  it('scopes tool-call argument state by run step', async () => {
+    const writes: string[] = [];
+    const tracker = createOpenAIStreamTracker();
+    const handlers = createOpenAIHandlers({
+      writer: { write: (data) => void writes.push(data) },
+      context: { requestId: 'chatcmpl_step_tools', model: 'agent', created: 1 },
+      tracker,
+    });
+
+    await handlers[GraphEvents.ON_RUN_STEP_DELTA].handle(
+      GraphEvents.ON_RUN_STEP_DELTA,
+      {
+        id: 'step_1',
+        delta: {
+          type: 'tool_calls',
+          tool_calls: [
+            {
+              index: 0,
+              id: 'call_1',
+              name: 'search',
+              args: '{"query":"first"}',
+            },
+          ],
+        },
+      } as t.RunStepDeltaEvent
+    );
+    await handlers[GraphEvents.ON_RUN_STEP_DELTA].handle(
+      GraphEvents.ON_RUN_STEP_DELTA,
+      {
+        id: 'step_2',
+        delta: {
+          type: 'tool_calls',
+          tool_calls: [
+            {
+              index: 0,
+              id: 'call_2',
+              name: 'search',
+              args: '{"query":"second"}',
+            },
+          ],
+        },
+      } as t.RunStepDeltaEvent
+    );
+
+    const toolCallDeltas = writes
+      .map(
+        (data) =>
+          JSON.parse(data.slice(6)) as {
+            choices: Array<{
+              delta: {
+                tool_calls?: Array<{
+                  id?: string;
+                  function?: { name?: string; arguments?: string };
+                }>;
+              };
+            }>;
+          }
+      )
+      .flatMap((chunk) => chunk.choices[0].delta.tool_calls ?? []);
+
+    expect(toolCallDeltas).toHaveLength(2);
+    expect(toolCallDeltas[1]).toMatchObject({
+      id: 'call_2',
+      function: { name: 'search', arguments: '{"query":"second"}' },
+    });
+    expect(tracker.toolCalls.get(0)?.function.arguments).toBe(
+      '{"query":"second"}'
+    );
+  });
+
+  it('streams completed tool-call run steps without deltas', async () => {
+    const writes: string[] = [];
+    const tracker = createOpenAIStreamTracker();
+    const handlers = createOpenAIHandlers({
+      writer: { write: (data) => void writes.push(data) },
+      context: {
+        requestId: 'chatcmpl_complete_tools',
+        model: 'agent',
+        created: 1,
+      },
+      tracker,
+    });
+
+    await handlers[GraphEvents.ON_RUN_STEP].handle(GraphEvents.ON_RUN_STEP, {
+      id: 'step_complete',
+      index: 2,
+      type: 'tool_calls',
+      stepDetails: {
+        type: 'tool_calls',
+        tool_calls: [
+          {
+            id: 'call_complete',
+            type: 'function',
+            function: {
+              name: 'search',
+              arguments: { query: 'sessions' },
+            },
+          },
+        ],
+      },
+    } as t.RunStep);
+    await sendOpenAIFinalChunk({
+      writer: { write: (data) => void writes.push(data) },
+      context: {
+        requestId: 'chatcmpl_complete_tools',
+        model: 'agent',
+        created: 1,
+      },
+      tracker,
+    });
+
+    expect(writes[0]).toContain('"role":"assistant"');
+    expect(writes[1]).toContain('"tool_calls"');
+    expect(writes[1]).toContain('"id":"call_complete"');
+    expect(writes[1]).toContain('"name":"search"');
+    expect(writes[1]).toContain('"{\\"query\\":\\"sessions\\"}"');
+    expect(writes.at(-3)).toContain('"finish_reason":"tool_calls"');
+    expect(writes.at(-2)).toContain('"choices":[]');
+  });
+
+  it('tracks partial usage metadata without NaN totals', async () => {
+    const tracker = createOpenAIStreamTracker();
+    const handlers = createOpenAIHandlers({
+      writer: { write: jest.fn() },
+      context: { requestId: 'chatcmpl_usage', model: 'agent', created: 1 },
+      tracker,
+    });
+
+    await handlers[GraphEvents.CHAT_MODEL_END].handle(
+      GraphEvents.CHAT_MODEL_END,
+      {
+        output: { usage_metadata: { input_tokens: 3 } },
+      } as t.ModelEndData
+    );
+    await handlers[GraphEvents.CHAT_MODEL_END].handle(
+      GraphEvents.CHAT_MODEL_END,
+      {
+        output: { usage_metadata: { output_tokens: 5 } },
+      } as t.ModelEndData
+    );
+
+    expect(tracker.usage.promptTokens).toBe(3);
+    expect(tracker.usage.completionTokens).toBe(5);
+  });
+
+  it('includes reasoning token usage in the final chunk', async () => {
+    const writes: string[] = [];
+    const tracker = createOpenAIStreamTracker();
+    const handlers = createOpenAIHandlers({
+      writer: { write: (data) => void writes.push(data) },
+      context: {
+        requestId: 'chatcmpl_reasoning_usage',
+        model: 'agent',
+        created: 1,
+      },
+      tracker,
+    });
+
+    await handlers[GraphEvents.CHAT_MODEL_END].handle(
+      GraphEvents.CHAT_MODEL_END,
+      {
+        output: {
+          usage_metadata: {
+            input_tokens: 3,
+            output_tokens: 5,
+            output_token_details: { reasoning: 2 },
+          },
+        },
+      } as t.ModelEndData
+    );
+    await sendOpenAIFinalChunk({
+      writer: { write: (data) => void writes.push(data) },
+      context: {
+        requestId: 'chatcmpl_reasoning_usage',
+        model: 'agent',
+        created: 1,
+      },
+      tracker,
+    });
+
+    expect(writes.at(-2)).toContain('"choices":[]');
+    expect(writes.at(-2)).toContain(
+      '"completion_tokens_details":{"reasoning_tokens":2}'
+    );
+  });
+
+  it('builds a chat completion chunk without transport dependencies', () => {
+    expect(
+      createChatCompletionChunk(
+        { requestId: 'chatcmpl_3', model: 'agent', created: 1 },
+        { content: 'x' }
+      )
+    ).toEqual({
+      id: 'chatcmpl_3',
+      object: 'chat.completion.chunk',
+      created: 1,
+      model: 'agent',
+      choices: [{ index: 0, delta: { content: 'x' }, finish_reason: null }],
+    });
+  });
+});