@librechat/agents 1.4.4 → 1.4.6

package/src/messages.ts

@@ -9,7 +9,7 @@ export function getConverseOverrideMessage({
   lastMessageY
 }: {
   userMessage: string[];
-  lastMessageX: AIMessageChunk;
+  lastMessageX: AIMessageChunk | null;
   lastMessageY: ToolMessage;
 }): HumanMessage {
   const content = `
@@ -19,7 +19,7 @@ User: ${userMessage[1]}
 # YOU HAVE ALREADY RESPONDED TO THE LATEST USER MESSAGE:
 
 # Observations:
-- ${lastMessageX.content}
+- ${lastMessageX?.content}
 
 # Tool Calls:
 - ${lastMessageX?.tool_calls?.join('\n- ')}
@@ -149,8 +149,8 @@ export function formatAnthropicMessage(message: AIMessageChunk): AIMessage {
 export function convertMessagesToContent(messages: BaseMessage[]): t.MessageContentComplex[] {
   const processedContent: t.MessageContentComplex[] = [];
 
-  const addContentPart = (message: BaseMessage): void => {
-    const content = message?.content;
+  const addContentPart = (message: BaseMessage | null): void => {
+    const content = message?.lc_kwargs.content != null ? message.lc_kwargs.content : message?.content;
     if (content === undefined) {
       return;
     }
@@ -169,7 +169,7 @@ export function convertMessagesToContent(messages: BaseMessage[]): t.MessageCont
   const toolCallMap = new Map<string, t.CustomToolCall>();
 
   for (let i = 0; i < messages.length; i++) {
-    const message = messages[i];
+    const message = messages[i] as BaseMessage | null;
     const messageType = message?._getType();
 
     if (messageType === 'ai' && (message as AIMessage).tool_calls?.length) {

package/src/run.ts

@@ -16,6 +16,7 @@ export class Run<T extends t.BaseGraphState> {
   private Graph: StandardGraph | undefined;
   provider: Providers | undefined;
   run_id: string | undefined;
+  returnContent: boolean = false;
 
   private constructor(config: t.RunConfig) {
     const handlerRegistry = new HandlerRegistry();
@@ -35,6 +36,8 @@ export class Run<T extends t.BaseGraphState> {
         this.Graph.handlerRegistry = handlerRegistry;
       }
     }
+
+    this.returnContent = config.returnContent ?? false;
   }
 
   private createStandardGraph(config: t.StandardGraphConfig): t.CompiledWorkflow<t.IState, Partial<t.IState>, string> {
@@ -108,7 +111,9 @@ export class Run<T extends t.BaseGraphState> {
       }
     }
 
-    return this.Graph.getContentParts();
+    if (this.returnContent) {
+      return this.Graph.getContentParts();
+    }
   }
 
   private createSystemCallback<K extends keyof ClientCallbacks>(

package/src/scripts/content.ts

@@ -68,6 +68,7 @@ async function testStandardStreaming(): Promise<void> {
       instructions: 'You are a friendly AI assistant. Always address the user by their name.',
       additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
     },
+    returnContent: true,
     customHandlers,
   });
 
@@ -99,9 +100,9 @@ async function testStandardStreaming(): Promise<void> {
     conversationHistory.push(...finalMessages);
     console.dir(conversationHistory, { depth: null });
   }
-  console.dir(finalContentParts, { depth: null });
+  // console.dir(finalContentParts, { depth: null });
   console.log('\n\n====================\n\n');
-  console.dir(contentParts, { depth: null });
+  // console.dir(contentParts, { depth: null });
 }
 
 process.on('unhandledRejection', (reason, promise) => {

package/src/scripts/tools.ts

@@ -0,0 +1,120 @@
+/* eslint-disable no-console */
+// src/scripts/cli.ts
+import { config } from 'dotenv';
+config();
+import { HumanMessage, BaseMessage } from '@langchain/core/messages';
+import { TavilySearchResults } from '@langchain/community/tools/tavily_search';
+import type * as t from '@/types';
+import { ChatModelStreamHandler, createContentAggregator } from '@/stream';
+import { ToolEndHandler } from '@/events';
+
+
+import { getArgs } from '@/scripts/args';
+import { Run } from '@/run';
+import { GraphEvents, Callback } from '@/common';
+import { getLLMConfig } from '@/utils/llmConfig';
+
+const conversationHistory: BaseMessage[] = [];
+async function testStandardStreaming(): Promise<void> {
+  const { userName, location, provider, currentDate } = await getArgs();
+  const { contentParts, aggregateContent } = createContentAggregator();
+  const customHandlers = {
+    [GraphEvents.TOOL_END]: new ToolEndHandler(),
+    // [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(),
+    [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(),
+    [GraphEvents.ON_RUN_STEP_COMPLETED]: {
+      handle: (event: GraphEvents.ON_RUN_STEP_COMPLETED, data: t.StreamEventData): void => {
+        console.log('====== ON_RUN_STEP_COMPLETED ======');
+        // console.dir(data, { depth: null });
+        aggregateContent({ event, data: data as unknown as { result: t.ToolEndEvent } });
+      }
+    },
+    [GraphEvents.ON_RUN_STEP]: {
+      handle: (event: GraphEvents.ON_RUN_STEP, data: t.StreamEventData): void => {
+        console.log('====== ON_RUN_STEP ======');
+        console.dir(data, { depth: null });
+        aggregateContent({ event, data: data as t.RunStep });
+      }
+    },
+    [GraphEvents.ON_RUN_STEP_DELTA]: {
+      handle: (event: GraphEvents.ON_RUN_STEP_DELTA, data: t.StreamEventData): void => {
+        console.log('====== ON_RUN_STEP_DELTA ======');
+        console.dir(data, { depth: null });
+        aggregateContent({ event, data: data as t.RunStepDeltaEvent });
+      }
+    },
+    [GraphEvents.ON_MESSAGE_DELTA]: {
+      handle: (event: GraphEvents.ON_MESSAGE_DELTA, data: t.StreamEventData): void => {
+        console.log('====== ON_MESSAGE_DELTA ======');
+        console.dir(data, { depth: null });
+        aggregateContent({ event, data: data as t.MessageDeltaEvent });
+      }
+    },
+    [GraphEvents.TOOL_START]: {
+      handle: (_event: string, data: t.StreamEventData, metadata?: Record<string, unknown>): void => {
+        console.log('====== TOOL_START ======');
+        // console.dir(data, { depth: null });
+      }
+    },
+  };
+
+  const llmConfig = getLLMConfig(provider);
+
+  const run = await Run.create<t.IState>({
+    graphConfig: {
+      type: 'standard',
+      llmConfig,
+      tools: [new TavilySearchResults()],
+      instructions: 'You are a friendly AI assistant. Always address the user by their name.',
+      additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
+    },
+    returnContent: true,
+    customHandlers,
+  });
+
+  const config = {
+    configurable: {
+      provider,
+      thread_id: 'conversation-num-1',
+    },
+    streamMode: 'values',
+    version: 'v2' as const,
+  };
+
+  console.log('Test 1: Weather query (content parts test)');
+
+  const userMessage = `
+  Make a search for the weather in ${location} today, which is ${currentDate}.
+  Before making the search, please let me know what you're about to do, then immediately start searching without hesitation.
+  Make sure to always refer to me by name, which is ${userName}.
+  After giving me a thorough summary, tell me a joke about the weather forecast we went over.
+  `;
+
+  conversationHistory.push(new HumanMessage(userMessage));
+
+  const inputs = {
+    messages: conversationHistory,
+  };
+  const finalContentParts = await run.processStream(inputs, config);
+  const finalMessages = run.getRunMessages();
+  if (finalMessages) {
+    conversationHistory.push(...finalMessages);
+    console.dir(conversationHistory, { depth: null });
+  }
+  // console.dir(finalContentParts, { depth: null });
+  console.log('\n\n====================\n\n');
+  // console.dir(contentParts, { depth: null });
+}
+
+process.on('unhandledRejection', (reason, promise) => {
+  console.error('Unhandled Rejection at:', promise, 'reason:', reason);
+  console.log('Conversation history:');
+  process.exit(1);
+});
+
+testStandardStreaming().catch((err) => {
+  console.error(err);
+  console.log('Conversation history:');
+  console.dir(conversationHistory, { depth: null });
+  process.exit(1);
+});

package/src/stream.ts

@@ -56,15 +56,51 @@ export class ChatModelStreamHandler implements t.EventHandler {
 
     if (hasToolCalls && chunk.tool_calls?.every((tc) => tc.id)) {
       const tool_calls: ToolCall[] = [];
+      const tool_call_ids: string[] = [];
       for (const tool_call of chunk.tool_calls) {
-        if (!tool_call.id || graph.toolCallStepIds.has(tool_call.id)) {
+        const toolCallId = tool_call.id ?? '';
+        if (!toolCallId || graph.toolCallStepIds.has(toolCallId)) {
           continue;
         }
 
         tool_calls.push(tool_call);
+        tool_call_ids.push(toolCallId);
       }
 
       const stepKey = graph.getStepKey(metadata);
+
+      let prevStepId = '';
+      let prevRunStep: t.RunStep | undefined;
+      try {
+        prevStepId = graph.getStepIdByKey(stepKey, graph.contentData.length - 1);
+        prevRunStep = graph.getRunStep(prevStepId);
+      } catch (e) {
+        // no previous step
+      }
+
+      const dispatchToolCallIds = (lastMessageStepId: string): void => {
+        graph.dispatchMessageDelta(lastMessageStepId, {
+          content: [{
+            type: 'text',
+            text: '',
+            tool_call_ids,
+          }],
+        });
+      };
+      /* If the previous step exists and is a message creation */
+      if (prevStepId && prevRunStep && prevRunStep.type === StepTypes.MESSAGE_CREATION) {
+        dispatchToolCallIds(prevStepId);
+        /* If the previous step doesn't exist or is not a message creation */
+      } else if (!prevRunStep || prevRunStep.type !== StepTypes.MESSAGE_CREATION) {
+        const messageId = getMessageId(stepKey, graph) ?? '';
+        const stepId = graph.dispatchRunStep(stepKey, {
+          type: StepTypes.MESSAGE_CREATION,
+          message_creation: {
+            message_id: messageId,
+          },
+        });
+        dispatchToolCallIds(stepId);
+      }
       graph.dispatchRunStep(stepKey, {
         type: StepTypes.TOOL_CALLS,
         tool_calls,
@@ -90,7 +126,9 @@ export class ChatModelStreamHandler implements t.EventHandler {
     const stepKey = graph.getStepKey(metadata);
 
     if (hasToolCallChunks && chunk.tool_call_chunks?.length && typeof chunk.tool_call_chunks[0]?.index === 'number') {
-      const stepId = graph.getStepIdByKey(stepKey, chunk.tool_call_chunks[0].index);
+      const prevStepId = graph.getStepIdByKey(stepKey, graph.contentData.length - 1);
+      const prevRunStep = graph.getRunStep(prevStepId);
+      const stepId = graph.getStepIdByKey(stepKey, prevRunStep?.index);
       graph.dispatchRunStepDelta(stepId, {
         type: StepTypes.TOOL_CALLS,
         tool_calls: chunk.tool_call_chunks,
@@ -183,16 +221,27 @@ export function createContentAggregator(): ContentAggregatorResult {
       contentParts[index] = { type: partType };
     }
 
+    if (contentPart.type !== contentParts[index]?.type) {
+      console.warn('Content type mismatch');
+      return;
+    }
+
     if (
       partType.startsWith(ContentTypes.TEXT) &&
       ContentTypes.TEXT in contentPart &&
       typeof contentPart.text === 'string'
     ) {
-      const currentContent = contentParts[index] as { type: ContentTypes.TEXT; text: string };
-      contentParts[index] = {
+      // TODO: update this!!
+      const currentContent = contentParts[index] as t.MessageDeltaUpdate;
+      const update: t.MessageDeltaUpdate = {
         type: ContentTypes.TEXT,
         text: (currentContent.text || '') + contentPart.text,
       };
+
+      if (contentPart.tool_call_ids) {
+        update.tool_call_ids = contentPart.tool_call_ids;
+      }
+      contentParts[index] = update;
     } else if (partType === ContentTypes.IMAGE_URL && 'image_url' in contentPart) {
       const currentContent = contentParts[index] as { type: 'image_url'; image_url: string };
       contentParts[index] = {

package/src/types/run.ts

@@ -52,5 +52,6 @@ export type TaskManagerGraphConfig = {
 export type RunConfig = {
   graphConfig: StandardGraphConfig | CollaborativeGraphConfig | TaskManagerGraphConfig;
   customHandlers?: Record<string, g.EventHandler>;
+  returnContent?: boolean;
   runId?: string;
 };

package/src/types/stream.ts

@@ -1,7 +1,7 @@
 // src/types/stream.ts
 import type { MessageContentImageUrl, MessageContentText, ToolMessage } from '@langchain/core/messages';
 import type { ToolCall, ToolCallChunk } from '@langchain/core/messages/tool';
-import { StepTypes } from '@/common/enum';
+import { StepTypes, ContentTypes } from '@/common/enum';
 
 /** Event names are of the format: on_[runnable_type]_(start|stream|end).
 
@@ -158,8 +158,14 @@ export interface MessageDelta {
    * The content of the message in array of text and/or images.
    */
   content?: MessageContentComplex[];
+  /**
+   * The tool call ids associated with the message.
+   */
+  tool_call_ids?: string[];
 }
 
+export type MessageDeltaUpdate = { type: ContentTypes.TEXT; text: string; tool_call_ids?: string[] };
+
 export type ContentType = 'text' | 'image_url' | 'tool_call' | string;
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any