@librechat/agents 3.3.13 → 3.4.0

package/dist/types/types/llm.d.ts

@@ -28,7 +28,7 @@ export type AzureClientOptions = Partial<OpenAIChatInput> & Partial<AzureOpenAII
     deploymentName?: string;
 } & BaseChatModelParams & {
     configuration?: OAIClientOptions;
-} & ManagedRequestOptions;
+} & ManagedRequestOptions & StreamSmoothingOptions;
 /**
  * Controls whether Claude's reasoning content is returned in adaptive
  * thinking responses. Added for Claude Opus 4.7, which omits thinking by
@@ -58,8 +58,19 @@ export type ManagedRequestOptions = {
     promptCacheExplicit?: boolean;
     safety_identifier?: string;
 };
-export type OpenAIClientOptions = ChatOpenAIFields & ManagedRequestOptions;
-export type AnthropicClientOptions = Omit<AnthropicInput, 'thinking'> & {
+/**
+ * Adaptive stream-smoothing configuration shared by every provider client.
+ */
+export type StreamSmoothingOptions = {
+    /**
+     * Minimum delay in milliseconds between visible streamed content deltas.
+     * Defaults to 25; piece sizes adapt to the backlog so render lag stays
+     * bounded regardless of provider chunk size. Set 0 to disable smoothing.
+     */
+    _lc_stream_delay?: number;
+};
+export type OpenAIClientOptions = ChatOpenAIFields & ManagedRequestOptions & StreamSmoothingOptions;
+export type AnthropicClientOptions = Omit<AnthropicInput, 'thinking'> & StreamSmoothingOptions & {
     thinking?: ThinkingConfig;
     promptCache?: boolean;
     /**
@@ -69,12 +80,12 @@ export type AnthropicClientOptions = Omit<AnthropicInput, 'thinking'> & {
      */
     promptCacheTtl?: PromptCacheTtl;
 };
-export type MistralAIClientOptions = ChatMistralAIInput;
-export type VertexAIClientOptions = ChatVertexAIInput & {
+export type MistralAIClientOptions = ChatMistralAIInput & StreamSmoothingOptions;
+export type VertexAIClientOptions = ChatVertexAIInput & StreamSmoothingOptions & {
     includeThoughts?: boolean;
     thinkingConfig?: GoogleThinkingConfig;
 };
-export type BedrockAnthropicInput = ChatBedrockConverseInput & {
+export type BedrockAnthropicInput = ChatBedrockConverseInput & StreamSmoothingOptions & {
     additionalModelRequestFields?: ChatBedrockConverseInput['additionalModelRequestFields'] & AnthropicReasoning;
     promptCache?: boolean;
     /**
@@ -84,20 +95,16 @@ export type BedrockAnthropicInput = ChatBedrockConverseInput & {
      * server-side, so the default is safe to leave on.
      */
     promptCacheTtl?: PromptCacheTtl;
-    /**
-     * Minimum delay in milliseconds between visible streamed content deltas.
-     */
-    _lc_stream_delay?: number;
 };
 export type BedrockConverseClientOptions = BedrockAnthropicInput;
 export type BedrockAnthropicClientOptions = BedrockAnthropicInput;
-export type GoogleClientOptions = GoogleGenerativeAIChatInput & {
+export type GoogleClientOptions = GoogleGenerativeAIChatInput & StreamSmoothingOptions & {
     customHeaders?: RequestOptions['customHeaders'];
     thinkingConfig?: GoogleThinkingConfig;
     includeServerSideToolInvocations?: boolean;
 };
-export type DeepSeekClientOptions = Partial<ChatDeepSeekInput>;
-export type XAIClientOptions = ChatXAIInput;
+export type DeepSeekClientOptions = Partial<ChatDeepSeekInput> & StreamSmoothingOptions;
+export type XAIClientOptions = ChatXAIInput & StreamSmoothingOptions;
 export type ClientOptions = OpenAIClientOptions | AzureClientOptions | AnthropicClientOptions | MistralAIClientOptions | VertexAIClientOptions | BedrockConverseClientOptions | GoogleClientOptions | DeepSeekClientOptions | XAIClientOptions;
 export type SharedLLMConfig = {
     provider: Providers;
@@ -122,7 +129,7 @@ export type ProviderOptionsMap = {
     [Providers.ANTHROPIC]: AnthropicClientOptions;
     [Providers.MISTRALAI]: MistralAIClientOptions;
     [Providers.MISTRAL]: MistralAIClientOptions;
-    [Providers.OPENROUTER]: ChatOpenRouterCallOptions;
+    [Providers.OPENROUTER]: ChatOpenRouterCallOptions & StreamSmoothingOptions;
     [Providers.BEDROCK]: BedrockAnthropicClientOptions;
     [Providers.XAI]: XAIClientOptions;
     [Providers.MOONSHOT]: OpenAIClientOptions;

package/dist/types/utils/tokens.d.ts

@@ -1,5 +1,15 @@
 import type { BaseMessage } from '@langchain/core/messages';
 export type EncodingName = 'o200k_base' | 'claude';
+export type UnsafeTokenMeasurementReason = 'message_proxy' | 'content_proxy' | 'metadata_proxy' | 'metadata_accessor' | 'invalid_count';
+export declare class UnsafeTokenMeasurementError extends Error {
+    readonly type = "unsafe_token_measurement";
+    readonly reason: UnsafeTokenMeasurementReason;
+    readonly path: string;
+    constructor({ reason, path, }: {
+        reason: UnsafeTokenMeasurementReason;
+        path: string;
+    });
+}
 /** Safety margin for image and document token estimates (5% overestimate). */
 export declare const IMAGE_TOKEN_SAFETY_MARGIN = 1.05;
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@librechat/agents",
-  "version": "3.3.13",
+  "version": "3.4.0",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/esm/main.mjs",
   "types": "./dist/types/index.d.ts",
@@ -248,7 +248,7 @@
     "nanoid": "^3.3.7",
     "okapibm25": "^1.4.1",
     "openai": "^6.46.0",
-    "reo-census": "^1.2.9",
+    "reo-census": "^1.2.10",
     "socks-proxy-agent": "^8.0.5",
     "uuid": "^11.1.1"
   },

package/src/graphs/Graph.ts

@@ -998,6 +998,7 @@ export abstract class Graph<
 
 export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
   overrideModel?: t.ChatModel;
+  private subagentModelOverride?: t.ChatModel;
   /** Optional compile options passed into workflow.compile() */
   compileOptions?: t.CompileOptions | undefined;
   /** Whether the workflow was actually compiled with a checkpointer. */
@@ -1320,6 +1321,7 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
     super.clearHeavyState();
     this.messages = [];
     this.overrideModel = undefined;
+    this.subagentModelOverride = undefined;
     /** Stream-limit accounting (argument tallies, event counts, charge
      * credits) deliberately SURVIVES cleanup: this runs in `processStream`'s
      * finally, which an ordinary parallel-branch failure reaches while
@@ -1944,6 +1946,11 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
     });
   }
 
+  /** Explicitly overrides the model used by isolated descendant subagent graphs. */
+  setSubagentModelOverride(model: t.ChatModel): void {
+    this.subagentModelOverride = model;
+  }
+
   getUsageMetadata(
     finalMessage?: BaseMessage
   ): Partial<UsageMetadata> | undefined {
@@ -4057,6 +4064,10 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
           maxDepth: effectiveSubagentDepth,
           createChildGraph: (input): StandardGraph => {
             const childGraph = new StandardGraph(input);
+            if (this.subagentModelOverride != null) {
+              childGraph.overrideModel = this.subagentModelOverride;
+              childGraph.setSubagentModelOverride(this.subagentModelOverride);
+            }
             const toolHandlerRegistry = createToolHandlerRegistry(
               getParentHandlerRegistry()
             );

package/src/index.ts

@@ -77,6 +77,14 @@ export type {
   ChatOpenRouterCallOptions,
 } from './llm/openrouter';
 export { getChatModelClass } from './llm/providers';
+export { CustomChatMistralAI } from './llm/mistral';
+export {
+  smoothStream,
+  resolveStreamDelay,
+  DEFAULT_STREAM_DELAY,
+  computeAdaptivePieceSize,
+} from './llm/stream/smoother';
+export type { SmoothItem, SmoothPiece } from './llm/stream/smoother';
 export { FakeChatModel, createFakeStreamingLLM } from './llm/fake';
 export { initializeModel } from './llm/init';
 export { attemptInvoke, tryFallbackProviders } from './llm/invoke';

package/src/llm/anthropic/index.ts

@@ -21,21 +21,17 @@ import type {
   AnthropicRequestOptions,
   AnthropicMessageStreamEvent,
 } from '@/llm/anthropic/types';
+import type { SmoothItem } from '@/llm/stream/smoother';
 import type { AnthropicUsageData } from './utils/message_outputs';
 import {
   _convertMessagesToAnthropicPayload,
   stripUnsupportedAssistantPrefill,
 } from './utils/message_inputs';
 import { _makeMessageChunkFromAnthropicEvent } from './utils/message_outputs';
+import { smoothStream, resolveStreamDelay, isSignalAborted } from '@/llm/stream/smoother';
 import { convertAnthropicStream } from './utils/stream_events';
 import { handleToolChoice } from './utils/tools';
 
-const DEFAULT_STREAM_DELAY = 25;
-const MAX_STREAM_QUEUE_CHUNKS = 256;
-const MAX_STREAM_QUEUE_TEXT_CHARS = 8192;
-const STREAM_CHUNK_MIN_SIZE = 4;
-const STREAM_BOUNDARIES = new Set([' ', '.', ',', '!', '?', ';', ':']);
-
 type StreamTokenType = 'string' | 'input' | 'content';
 
 interface AnthropicStreamUsage {
@@ -268,83 +264,6 @@ function getSamplingParams({
   };
 }
 
-function findStreamChunkBoundary(text: string, minSize: number): number {
-  if (minSize >= text.length) {
-    return text.length;
-  }
-
-  for (let position = minSize; position < text.length; position++) {
-    if (STREAM_BOUNDARIES.has(text[position])) {
-      return position + 1;
-    }
-  }
-
-  return text.length;
-}
-
-function splitStreamToken(text: string): string[] {
-  const chunks: string[] = [];
-  let currentIndex = 0;
-
-  while (currentIndex < text.length) {
-    const remainingText = text.slice(currentIndex);
-    const chunkSize = findStreamChunkBoundary(
-      remainingText,
-      STREAM_CHUNK_MIN_SIZE
-    );
-    chunks.push(text.slice(currentIndex, currentIndex + chunkSize));
-    currentIndex += chunkSize;
-  }
-
-  return chunks;
-}
-
-function getCadencedStreamDelay({
-  targetDelay,
-  lastVisibleTextAt,
-  now,
-}: {
-  targetDelay: number;
-  lastVisibleTextAt?: number;
-  now: number;
-}): number {
-  if (targetDelay <= 0 || lastVisibleTextAt == null) {
-    return 0;
-  }
-  return Math.max(0, targetDelay - (now - lastVisibleTextAt));
-}
-
-async function waitForStreamDelay(
-  delay: number,
-  signal?: AbortSignal
-): Promise<void> {
-  if (delay <= 0 || isSignalAborted(signal)) {
-    return;
-  }
-  await new Promise<void>((resolve) => {
-    const timeoutRef: { current?: ReturnType<typeof setTimeout> } = {};
-    const onAbort = (): void => {
-      if (timeoutRef.current) {
-        clearTimeout(timeoutRef.current);
-      }
-      signal?.removeEventListener('abort', onAbort);
-      resolve();
-    };
-    timeoutRef.current = setTimeout(() => {
-      signal?.removeEventListener('abort', onAbort);
-      resolve();
-    }, delay);
-    signal?.addEventListener('abort', onAbort, { once: true });
-    if (isSignalAborted(signal)) {
-      onAbort();
-    }
-  });
-}
-
-function isSignalAborted(signal?: AbortSignal): boolean {
-  return signal?.aborted === true;
-}
-
 async function* abortableAnthropicStream(
   source: AnthropicEventStream,
   signal?: AbortSignal
@@ -525,11 +444,9 @@ type CustomAnthropicInvocationParams = {
   output_config?: AnthropicOutputConfig;
 };
 
-type QueuedGenerationChunk = {
+type AnthropicEmittedChunk = {
   chunk: ChatGenerationChunk;
   token: string;
-  smooth: boolean;
-  textLength: number;
 };
 
 export class CustomAnthropic extends ChatAnthropicMessages {
@@ -543,10 +460,7 @@ export class CustomAnthropic extends ChatAnthropicMessages {
     super(fields);
     this.resetTokenEvents();
     this.setDirectFields(fields);
-    this._lc_stream_delay = Math.max(
-      0,
-      fields?._lc_stream_delay ?? DEFAULT_STREAM_DELAY
-    );
+    this._lc_stream_delay = resolveStreamDelay(fields?._lc_stream_delay);
     this.outputConfig = fields?.outputConfig;
     this.inferenceGeo = fields?.inferenceGeo;
     this.contextManagement = fields?.contextManagement;
@@ -764,292 +678,109 @@ export class CustomAnthropic extends ChatAnthropicMessages {
       cacheReadInputTokens: 0,
       outputTokens: 0,
     };
-    const queuedChunks: QueuedGenerationChunk[] = [];
-    const producerState: {
-      done: boolean;
-      error?: unknown;
-    } = { done: false };
-    let queuedChunkIndex = 0;
-    let bufferedTextLength = 0;
-    let consumerClosed = false;
-    let notifyConsumer: (() => void) | undefined;
-    let notifyProducer: (() => void) | undefined;
-
-    const notifyConsumerForChunk = (): void => {
-      notifyConsumer?.();
-      notifyConsumer = undefined;
-    };
-
-    const notifyProducerForSpace = (): void => {
-      notifyProducer?.();
-      notifyProducer = undefined;
-    };
-
-    const hasQueuedChunks = (): boolean =>
-      queuedChunkIndex < queuedChunks.length;
-
-    const getQueuedChunkCount = (): number =>
-      queuedChunks.length - queuedChunkIndex;
-
-    const isQueueAtCapacity = (): boolean =>
-      getQueuedChunkCount() >= MAX_STREAM_QUEUE_CHUNKS ||
-      bufferedTextLength >= MAX_STREAM_QUEUE_TEXT_CHARS;
-
-    const waitForNextChunk = async (): Promise<void> => {
-      if (
-        hasQueuedChunks() ||
-        producerState.done ||
-        producerState.error != null
-      ) {
-        return;
-      }
-      await new Promise<void>((resolve) => {
-        notifyConsumer = resolve;
-      });
-    };
-
-    const waitForQueueSpace = async (): Promise<void> => {
-      while (
-        isQueueAtCapacity() &&
-        !consumerClosed &&
-        !isSignalAborted(options.signal)
-      ) {
-        await new Promise<void>((resolve) => {
-          const signal = options.signal;
-          const onAbort = (): void => {
-            signal?.removeEventListener('abort', onAbort);
-            resolve();
-          };
-          const onSpace = (): void => {
-            signal?.removeEventListener('abort', onAbort);
-            resolve();
-          };
-          notifyProducer = onSpace;
-          signal?.addEventListener('abort', onAbort, { once: true });
-          if (isSignalAborted(signal)) {
-            onAbort();
-          }
-        });
-      }
-    };
-
-    const dequeue = (): QueuedGenerationChunk | undefined => {
-      if (!hasQueuedChunks()) {
-        return undefined;
-      }
-      const queuedChunk = queuedChunks[queuedChunkIndex];
-      queuedChunkIndex++;
-      if (
-        queuedChunkIndex > 128 &&
-        queuedChunkIndex * 2 >= queuedChunks.length
-      ) {
-        queuedChunks.splice(0, queuedChunkIndex);
-        queuedChunkIndex = 0;
-      }
-      return queuedChunk;
-    };
-
-    const enqueue = async (
-      queuedChunk: QueuedGenerationChunk
-    ): Promise<void> => {
-      await waitForQueueSpace();
-      if (consumerClosed || isSignalAborted(options.signal)) {
-        stream.controller.abort();
-        throw new Error('AbortError: User aborted the request.');
-      }
-      queuedChunks.push(queuedChunk);
-      if (queuedChunk.smooth) {
-        bufferedTextLength += queuedChunk.textLength;
-      }
-      notifyConsumerForChunk();
-    };
-
-    const enqueueChunk = async ({
-      token,
-      chunk,
-      smooth,
-    }: {
-      token: string;
-      chunk: AIMessageChunk;
-      smooth: boolean;
-    }): Promise<void> => {
-      await enqueue({
-        token,
-        smooth,
-        textLength: smooth ? token.length : 0,
-        chunk: this.createGenerationChunk({
-          token,
-          chunk,
-          shouldStreamUsage,
-        }),
-      });
-    };
-
-    const enqueueTextChunks = (
+    const toEmittedChunk = (
       token: string,
-      tokenType: StreamTokenType,
       chunk: AIMessageChunk
-    ): Promise<void> => {
-      if (token === '') {
-        return Promise.resolve();
-      }
-      if (this._lc_stream_delay <= 0) {
-        return enqueueChunk({ token, chunk, smooth: false });
-      }
-
-      const tokenChunks = splitStreamToken(token);
-      if (tokenChunks.length <= 1) {
-        return enqueueChunk({ token, chunk, smooth: true });
-      }
-
-      let emittedUsage = false;
-      return tokenChunks.reduce(async (previous, currentToken) => {
-        await previous;
-        const newChunk = cloneChunk(currentToken, tokenType, chunk);
-        const chunkForToken =
-          emittedUsage && newChunk.usage_metadata != null
-            ? new AIMessageChunk(
-              Object.assign({}, newChunk, { usage_metadata: undefined })
-            )
-            : newChunk;
-
-        await enqueueChunk({
-          token: currentToken,
-          chunk: chunkForToken,
-          smooth: true,
-        });
-
-        if (newChunk.usage_metadata != null && !emittedUsage) {
-          emittedUsage = true;
-        }
-      }, Promise.resolve());
-    };
+    ): AnthropicEmittedChunk => ({
+      token,
+      chunk: this.createGenerationChunk({ token, chunk, shouldStreamUsage }),
+    });
 
-    const producer = (async (): Promise<void> => {
-      try {
-        for await (const data of stream) {
-          if (isSignalAborted(options.signal)) {
-            stream.controller.abort();
-            throw new Error('AbortError: User aborted the request.');
+    const source = (async function* (): AsyncGenerator<
+      SmoothItem<AnthropicEmittedChunk>
+      > {
+      for await (const data of stream) {
+        const result = _makeMessageChunkFromAnthropicEvent(
+          data as Anthropic.Beta.Messages.BetaRawMessageStreamEvent,
+          {
+            streamUsage: shouldStreamUsage,
+            coerceContentToString,
           }
+        );
+        if (!result) {
+          continue;
+        }
 
-          const result = _makeMessageChunkFromAnthropicEvent(
-            data as Anthropic.Beta.Messages.BetaRawMessageStreamEvent,
-            {
-              streamUsage: shouldStreamUsage,
-              coerceContentToString,
-            }
+        let { chunk } = result;
+        if (data.type === 'message_start') {
+          streamUsage = {
+            ...streamUsage,
+            inputTokens: data.message.usage.input_tokens,
+            outputTokens: data.message.usage.output_tokens,
+            cacheCreationInputTokens:
+              data.message.usage.cache_creation_input_tokens ?? 0,
+            cacheReadInputTokens:
+              data.message.usage.cache_read_input_tokens ?? 0,
+          };
+        }
+        if (data.type === 'message_delta') {
+          const incremental = withIncrementalMessageDeltaUsage(
+            chunk,
+            streamUsage,
+            data.usage
           );
-          if (!result) {
-            continue;
-          }
-
-          let { chunk } = result;
-          if (data.type === 'message_start') {
-            streamUsage = {
-              ...streamUsage,
-              inputTokens: data.message.usage.input_tokens,
-              outputTokens: data.message.usage.output_tokens,
-              cacheCreationInputTokens:
-                data.message.usage.cache_creation_input_tokens ?? 0,
-              cacheReadInputTokens:
-                data.message.usage.cache_read_input_tokens ?? 0,
-            };
-          }
-          if (data.type === 'message_delta') {
-            const incremental = withIncrementalMessageDeltaUsage(
-              chunk,
-              streamUsage,
-              data.usage
-            );
-            chunk = incremental.chunk;
-            streamUsage = incremental.usage;
-          }
-
-          const [token = '', tokenType] = extractToken(chunk);
-          if (
-            !tokenType ||
-            tokenType === 'input' ||
-            (token === '' && (chunk.usage_metadata != null || chunk.id != null))
-          ) {
-            await enqueueChunk({ token, chunk, smooth: false });
-            continue;
-          }
-
-          await enqueueTextChunks(token, tokenType, chunk);
+          chunk = incremental.chunk;
+          streamUsage = incremental.usage;
         }
-      } catch (error) {
-        producerState.error = error;
-      } finally {
-        producerState.done = true;
-        notifyConsumerForChunk();
-      }
-    })();
 
-    let hasEmittedText = false;
-    let lastVisibleTextAt: number | undefined;
-    let keepStreaming = true;
-    try {
-      while (keepStreaming) {
-        if (isSignalAborted(options.signal)) {
-          stream.controller.abort();
-          throw new Error('AbortError: User aborted the request.');
+        const [token = '', tokenType] = extractToken(chunk);
+        if (
+          !tokenType ||
+          tokenType === 'input' ||
+          (token === '' && (chunk.usage_metadata != null || chunk.id != null))
+        ) {
+          yield {
+            text: '',
+            smooth: false,
+            emit: (): AnthropicEmittedChunk => toEmittedChunk(token, chunk),
+          };
+          continue;
         }
 
-        await waitForNextChunk();
-        const queuedChunk = dequeue();
-
-        if (!queuedChunk) {
-          if (producerState.error != null) {
-            throw producerState.error;
-          }
-          if (producerState.done) {
-            keepStreaming = false;
-          }
+        if (token === '') {
           continue;
         }
 
-        if (queuedChunk.smooth) {
-          bufferedTextLength = Math.max(
-            0,
-            bufferedTextLength - queuedChunk.textLength
-          );
-          notifyProducerForSpace();
-          await waitForStreamDelay(
-            getCadencedStreamDelay({
-              targetDelay: hasEmittedText ? this._lc_stream_delay : 0,
-              lastVisibleTextAt,
-              now: Date.now(),
-            }),
-            options.signal
-          );
-          if (isSignalAborted(options.signal)) {
-            stream.controller.abort();
-            throw new Error('AbortError: User aborted the request.');
-          }
-          hasEmittedText = true;
-          lastVisibleTextAt = Date.now();
-        } else {
-          notifyProducerForSpace();
-        }
+        yield {
+          text: token,
+          smooth: true,
+          emit: (piece): AnthropicEmittedChunk => {
+            if (piece.isFirst && piece.isLast) {
+              return toEmittedChunk(token, chunk);
+            }
+            const cloned = cloneChunk(piece.text, tokenType, chunk);
+            const chunkForPiece =
+              !piece.isFirst && cloned.usage_metadata != null
+                ? new AIMessageChunk(
+                  Object.assign({}, cloned, { usage_metadata: undefined })
+                )
+                : cloned;
+            return toEmittedChunk(piece.text, chunkForPiece);
+          },
+        };
+      }
+    })();
+
+    const smoothed = smoothStream({
+      source,
+      delayMs: this._lc_stream_delay,
+      signal: options.signal,
+      abortUpstream: () => stream.controller.abort(),
+    });
 
-        yield queuedChunk.chunk;
+    try {
+      for await (const emitted of smoothed) {
+        yield emitted.chunk;
         await runManager?.handleLLMNewToken(
-          queuedChunk.token,
+          emitted.token,
           undefined,
           undefined,
           undefined,
           undefined,
-          { chunk: queuedChunk.chunk }
+          { chunk: emitted.chunk }
         );
       }
     } finally {
-      consumerClosed = true;
-      if (!producerState.done) {
-        stream.controller.abort();
-        notifyProducerForSpace();
-      }
-      await producer;
       this.resetTokenEvents();
     }
   }