@librechat/agents 1.9.93 → 1.9.95

package/src/splitStream.ts

@@ -0,0 +1,193 @@
+import { nanoid } from 'nanoid';
+import type * as t from '@/types';
+import { ContentTypes, GraphEvents, StepTypes } from '@/common';
+
+export const SEPARATORS = ['.', '?', '!', '۔', '。', '‥', ';', '¡', '¿', '\n', '```'];
+
+export class SplitStreamHandler {
+  private inCodeBlock = false;
+  private inThinkBlock = false;
+  private accumulate: boolean;
+  tokens: string[] = [];
+  lastToken = '';
+  reasoningTokens: string[] = [];
+  currentStepId?: string;
+  currentMessageId?: string;
+  currentType?: ContentTypes.TEXT | ContentTypes.THINK;
+  currentLength = 0;
+  reasoningKey: 'reasoning_content' | 'reasoning' = 'reasoning_content';
+  currentIndex = -1;
+  blockThreshold = 4500;
+  /** The run ID AKA the Message ID associated with the complete generation */
+  runId: string;
+  handlers?: t.SplitStreamHandlers;
+  constructor({
+    runId,
+    handlers,
+    accumulate,
+    reasoningKey,
+    blockThreshold,
+  }: {
+      runId: string,
+      accumulate?: boolean,
+      handlers: t.SplitStreamHandlers
+      blockThreshold?: number,
+      reasoningKey?: 'reasoning_content' | 'reasoning',
+    }) {
+    this.runId = runId;
+    this.handlers = handlers;
+    if (reasoningKey) {
+      this.reasoningKey = reasoningKey;
+    }
+    if (blockThreshold != null) {
+      this.blockThreshold = blockThreshold;
+    }
+    this.accumulate = accumulate ?? false;
+  }
+  getMessageId = (): string | undefined => {
+    const messageId = this.currentMessageId;
+    if (messageId != null && messageId) {
+      return messageId;
+    }
+    return undefined;
+  };
+  createMessageStep = (type?: ContentTypes.TEXT | ContentTypes.THINK): [string, string] => {
+    if (type != null && this.currentType !== type) {
+      this.currentType = type;
+    }
+    this.currentLength = 0;
+    this.currentIndex += 1;
+    this.currentStepId = `step_${nanoid()}`;
+    this.currentMessageId = `msg_${nanoid()}`;
+    return [this.currentStepId, this.currentMessageId];
+  };
+  dispatchRunStep = (stepId: string, stepDetails: t.StepDetails): void => {
+    const runStep: t.RunStep = {
+      id: stepId,
+      runId: this.runId,
+      type: stepDetails.type,
+      index: this.currentIndex,
+      stepDetails,
+      // usage: null,
+    };
+    this.handlers?.[GraphEvents.ON_RUN_STEP]?.({ event: GraphEvents.ON_RUN_STEP, data: runStep });
+  };
+  dispatchMessageDelta = (stepId: string, delta: t.MessageDelta): void => {
+    const messageDelta: t.MessageDeltaEvent = {
+      id: stepId,
+      delta,
+    };
+    this.handlers?.[GraphEvents.ON_MESSAGE_DELTA]?.({ event: GraphEvents.ON_MESSAGE_DELTA, data: messageDelta });
+  };
+  dispatchReasoningDelta = (stepId: string, delta: t.ReasoningDelta): void => {
+    const reasoningDelta: t.ReasoningDeltaEvent = {
+      id: stepId,
+      delta,
+    };
+    this.handlers?.[GraphEvents.ON_REASONING_DELTA]?.({ event: GraphEvents.ON_REASONING_DELTA, data: reasoningDelta });
+  };
+  handleContent = (content: string, stepId: string, _type: ContentTypes.TEXT | ContentTypes.THINK): void => {
+    let type = _type;
+    if (this.inThinkBlock && type === ContentTypes.TEXT) {
+      type = ContentTypes.THINK;
+    }
+    if (this.accumulate) {
+      if (type === ContentTypes.THINK) {
+        this.reasoningTokens.push(content);
+      } else {
+        this.tokens.push(content);
+      }
+    }
+
+    if (this.currentType !== type) {
+      const [newStepId, newMessageId] = this.createMessageStep(type);
+      this.dispatchRunStep(newStepId, {
+        type: StepTypes.MESSAGE_CREATION,
+        message_creation: {
+          message_id: newMessageId,
+        },
+      });
+    }
+
+    if (content.includes('```')) {
+      this.inCodeBlock = !this.inCodeBlock;
+    }
+
+    this.currentLength += content.length;
+
+    if (type === ContentTypes.THINK) {
+      this.dispatchReasoningDelta(stepId, {
+        content: [{
+          type: ContentTypes.THINK,
+          think: content,
+        }],
+      });
+    } else {
+      this.dispatchMessageDelta(stepId, {
+        content: [{
+          type: ContentTypes.TEXT,
+          text: content,
+        }],
+      });
+    }
+
+    if (this.inCodeBlock) {
+      return;
+    }
+
+    if (this.currentLength > this.blockThreshold && SEPARATORS.some(sep => content.includes(sep))) {
+      const [newStepId, newMessageId] = this.createMessageStep(type);
+      this.dispatchRunStep(newStepId, {
+        type: StepTypes.MESSAGE_CREATION,
+        message_creation: {
+          message_id: newMessageId,
+        },
+      });
+    }
+  };
+  handle(chunk?: t.CustomChunk): void {
+    if (!chunk) {
+      return;
+    }
+
+    const content = chunk.choices?.[0]?.delta.content ?? '';
+    const reasoning_content = chunk.choices?.[0]?.delta[this.reasoningKey] ?? '';
+
+    if (!content.length && !reasoning_content.length) {
+      return;
+    }
+
+    if (content === '<think>' && !this.inCodeBlock) {
+      this.inThinkBlock = true;
+    } else if (this.lastToken === '</think>' && !this.inCodeBlock) {
+      this.inThinkBlock = false;
+    }
+
+    this.lastToken = content;
+
+    const message_id = this.getMessageId() ?? '';
+
+    if (!message_id) {
+      const initialContentType = this.inThinkBlock ? ContentTypes.THINK : ContentTypes.TEXT;
+      const initialType = reasoning_content ? ContentTypes.THINK : initialContentType;
+      const [stepId, message_id] = this.createMessageStep(initialType);
+      this.dispatchRunStep(stepId, {
+        type: StepTypes.MESSAGE_CREATION,
+        message_creation: {
+          message_id,
+        },
+      });
+    }
+
+    const stepId = this.currentStepId ?? '';
+    if (!stepId) {
+      return;
+    }
+
+    if (reasoning_content) {
+      this.handleContent(reasoning_content, stepId, ContentTypes.THINK);
+    } else {
+      this.handleContent(content, stepId, ContentTypes.TEXT);
+    }
+  }
+}

package/src/stream.ts

@@ -266,6 +266,17 @@ export function createContentAggregator(): ContentAggregatorResult {
         update.tool_call_ids = contentPart.tool_call_ids;
       }
       contentParts[index] = update;
+    } else if (
+      partType.startsWith(ContentTypes.THINK) &&
+      ContentTypes.THINK in contentPart &&
+      typeof contentPart.think === 'string'
+    ) {
+      const currentContent = contentParts[index] as t.ReasoningDeltaUpdate;
+      const update: t.ReasoningDeltaUpdate = {
+        type: ContentTypes.THINK,
+        think: (currentContent.think || '') + contentPart.think,
+      };
+      contentParts[index] = update;
     } else if (partType === ContentTypes.IMAGE_URL && 'image_url' in contentPart) {
       const currentContent = contentParts[index] as { type: 'image_url'; image_url: string };
       contentParts[index] = {
@@ -332,6 +343,21 @@ export function createContentAggregator(): ContentAggregatorResult {
           ? messageDelta.delta.content[0]
           : messageDelta.delta.content;
 
+        updateContent(runStep.index, contentPart);
+      }
+    } else if (event === GraphEvents.ON_REASONING_DELTA) {
+      const reasoningDelta = data as t.ReasoningDeltaEvent;
+      const runStep = stepMap.get(reasoningDelta.id);
+      if (!runStep) {
+        console.warn('No run step or runId found for message delta event');
+        return;
+      }
+
+      if (reasoningDelta.delta.content) {
+        const contentPart = Array.isArray(reasoningDelta.delta.content)
+          ? reasoningDelta.delta.content[0]
+          : reasoningDelta.delta.content;
+
         updateContent(runStep.index, contentPart);
       }
     } else if (event === GraphEvents.ON_RUN_STEP_DELTA) {

package/src/types/stream.ts

@@ -1,8 +1,9 @@
 // src/types/stream.ts
+import type OpenAITypes from 'openai';
 import type { MessageContentImageUrl, MessageContentText, ToolMessage, BaseMessage } from '@langchain/core/messages';
 import type { ToolCall, ToolCallChunk } from '@langchain/core/messages/tool';
 import type { LLMResult, Generation } from '@langchain/core/outputs';
-import { StepTypes, ContentTypes } from '@/common/enum';
+import { StepTypes, ContentTypes, GraphEvents } from '@/common/enum';
 
 export type HandleLLMEnd = (output: LLMResult, runId: string, parentRunId?: string, tags?: string[]) => void;
 
@@ -54,7 +55,7 @@ export type RunStep = {
   index: number; // #new
   stepIndex?: number; // #new
   stepDetails: StepDetails;
-  usage: null | {
+  usage?: null | {
     // Define usage structure if it's ever non-null
     // prompt_tokens: number; // #new
     // completion_tokens: number; // #new
@@ -178,13 +179,40 @@ export interface MessageDelta {
   tool_call_ids?: string[];
 }
 
+/**
+ * Represents a reasoning delta i.e. any changed fields on a message during
+ * streaming.
+ */
+export interface ReasoningDeltaEvent {
+  /**
+   * The identifier of the message, which can be referenced in API endpoints.
+   */
+  id: string;
+
+  /**
+   * The delta containing the fields that have changed.
+   */
+  delta: ReasoningDelta;
+}
+
+/**
+ * The reasoning delta containing the fields that have changed on the Message.
+ */
+export interface ReasoningDelta {
+  /**
+   * The content of the message in array of text and/or images.
+   */
+  content?: MessageContentComplex[];
+}
+
 export type MessageDeltaUpdate = { type: ContentTypes.TEXT; text: string; tool_call_ids?: string[] };
+export type ReasoningDeltaUpdate = { type: ContentTypes.THINK; think: string; };
 
-export type ContentType = 'text' | 'image_url' | 'tool_call' | string;
+export type ContentType = 'text' | 'image_url' | 'tool_call' | 'think' | string;
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export type MessageContentComplex = (MessageContentText | MessageContentImageUrl | (Record<string, any> & {
-  type?: 'text' | 'image_url' | string;
+  type?: 'text' | 'image_url' | 'think' | string;
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 }) | (Record<string, any> & {
   type?: never;
@@ -192,3 +220,18 @@ export type MessageContentComplex = (MessageContentText | MessageContentImageUrl
   tool_call_ids?: string[];
 };
 // #new
+
+export type CustomChunk = Partial<OpenAITypes.ChatCompletionChunk> & {
+  choices?: Partial<Array<Partial<OpenAITypes.Chat.Completions.ChatCompletionChunk.Choice> & {
+    delta?: Partial<OpenAITypes.Chat.Completions.ChatCompletionChunk.Choice.Delta> & {
+      reasoning?: string | null;
+      reasoning_content?: string | null;
+    };
+  }>>;
+}
+
+export type SplitStreamHandlers = Partial<{
+  [GraphEvents.ON_RUN_STEP]: ({ event, data}: { event: GraphEvents, data: RunStep }) => void;
+  [GraphEvents.ON_MESSAGE_DELTA]: ({ event, data}: { event: GraphEvents, data: MessageDeltaEvent }) => void;
+  [GraphEvents.ON_REASONING_DELTA]: ({ event, data}: { event: GraphEvents, data: ReasoningDeltaEvent }) => void;
+}>

package/src/utils/llmConfig.ts

@@ -9,6 +9,7 @@ export const llmConfigs: Record<string, t.LLMConfig | undefined> = {
     temperature: 0.7,
     streaming: true,
     streamUsage: true,
+    // disableStreaming: true,
   },
   [Providers.OLLAMA]: {
     provider: Providers.OLLAMA,
@@ -44,7 +45,8 @@ export const llmConfigs: Record<string, t.LLMConfig | undefined> = {
   },
   [Providers.BEDROCK]: {
     provider: Providers.BEDROCK,
-    model: 'anthropic.claude-3-sonnet-20240229-v1:0',
+    // model: 'anthropic.claude-3-sonnet-20240229-v1:0',
+    model: 'us.anthropic.claude-3-5-sonnet-20241022-v2:0',
     region: process.env.BEDROCK_AWS_REGION,
     credentials: {
       accessKeyId: process.env.BEDROCK_AWS_ACCESS_KEY_ID!,