@librechat/agents 3.3.3 → 3.3.5

package/src/tools/ToolNode.ts

@@ -38,6 +38,15 @@ import type {
   PostToolBatchEntry,
 } from '@/hooks';
 import type * as t from '@/types';
+import {
+  cloneToolMessageWithContent,
+  compactToolContent,
+  hasComputerCallOutputMarker,
+  isComputerCallOutputContent,
+  isComputerCallOutputMessage,
+  serializeStructuredValueBounded,
+  serializeToolContentBounded,
+} from '@/utils/toolContent';
 import {
   buildToolExecutionRequestPlan,
   resolveRuntimeSessionHint,
@@ -61,7 +70,7 @@ import {
 } from '@/tools/local';
 import { stripCodeSessionFileSummary } from '@/tools/CodeSessionFileSummary';
 import { Constants, GraphEvents, CODE_EXECUTION_TOOLS } from '@/common';
-import { toLangChainContent } from '@/messages/langchain';
+import { convertInjectedMessages } from '@/messages/injected';
 import { safeDispatchCustomEvent } from '@/utils/events';
 import { RunnableCallable } from '@/utils';
 import { executeHooks } from '@/hooks';
@@ -119,6 +128,40 @@ type RunToolBatchContext<T = unknown> = {
   runInput?: T;
 };
 
+type BoundedToolOutput = {
+  content: string;
+  registryContent: string;
+};
+
+/**
+ * Produces the provider preview and, when requested, the exact registry prefix
+ * in one bounded traversal. String outputs already exist in memory when a tool
+ * returns them; structured outputs must never be fully JSON-materialized before
+ * either limit applies.
+ */
+function serializeToolOutputWithinLimits(
+  output: unknown,
+  maxToolResultChars: number,
+  registryPrefixChars = 0
+): BoundedToolOutput {
+  if (typeof output === 'string') {
+    return {
+      content: truncateToolResultContent(output, maxToolResultChars),
+      registryContent: registryPrefixChars > 0 ? output : '',
+    };
+  }
+
+  const serialized = serializeStructuredValueBounded(
+    output,
+    maxToolResultChars,
+    registryPrefixChars
+  );
+  return {
+    content: serialized.content,
+    registryContent: serialized.prefix,
+  };
+}
+
 const TOOL_NODE_RUN_NAME = 'tool_batch';
 const NANOID_URL_ALPHABET =
   '_-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
@@ -1107,6 +1150,17 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
       }
       if (isBaseMessage(output) && output._getType() === 'tool') {
         const toolMsg = output as ToolMessage;
+        if (isComputerCallOutputMessage(toolMsg)) {
+          return toolMsg;
+        }
+        const originalContent = toolMsg.content;
+        const compacted = compactToolContent(
+          originalContent,
+          this.maxToolResultChars
+        );
+        if (compacted.changed) {
+          toolMsg.content = compacted.content;
+        }
         const isError = toolMsg.status === 'error';
         if (isError) {
           /**
@@ -1125,14 +1179,9 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
           return toolMsg;
         }
         if (this.toolOutputRegistry != null || unresolvedRefs.length > 0) {
-          if (typeof toolMsg.content === 'string') {
-            const rawContent = toolMsg.content;
+          if (typeof originalContent === 'string') {
+            const rawContent = originalContent;
             const registryContent = stripCodeSessionFileSummary(rawContent);
-            const llmContent = truncateToolResultContent(
-              rawContent,
-              this.maxToolResultChars
-            );
-            toolMsg.content = llmContent;
             const refMeta = this.recordOutputReference(
               runId,
               registryContent,
@@ -1148,7 +1197,7 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
           } else {
             /**
              * Non-string content (multi-part content blocks — text +
-             * image). Known limitation: we cannot register under a
+             * image). It is now bounded for the LLM, but cannot register under a
              * reference key because there's no canonical serialized
              * form. Warn once per tool per run when the caller
              * intended to register. The unresolved-refs hint is still
@@ -1176,22 +1225,23 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
         }
         return toolMsg;
       }
-      const rawContent =
-        typeof output === 'string' ? output : JSON.stringify(output);
-      const truncated = truncateToolResultContent(
-        rawContent,
-        this.maxToolResultChars
+      const serialized = serializeToolOutputWithinLimits(
+        output,
+        this.maxToolResultChars,
+        this.toolOutputRegistry != null && refKey != null
+          ? this.toolOutputRegistry.perOutputLimit
+          : 0
       );
       const refMeta = this.recordOutputReference(
         runId,
-        stripCodeSessionFileSummary(rawContent),
+        stripCodeSessionFileSummary(serialized.registryContent),
         refKey,
         unresolvedRefs
       );
       return new ToolMessage({
         status: 'success',
         name: tool.name,
-        content: truncated,
+        content: serialized.content,
         tool_call_id: call.id!,
         ...(refMeta != null && {
           additional_kwargs: refMeta as Record<string, unknown>,
@@ -1258,7 +1308,10 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
           });
         }
       }
-      const errorContent = `Error: ${e.message}\n Please fix your mistakes.`;
+      const errorContent = truncateToolResultContent(
+        `Error: ${e.message}\n Please fix your mistakes.`,
+        this.maxToolResultChars
+      );
       const refMeta =
         unresolvedRefs.length > 0
           ? this.recordOutputReference(
@@ -1630,7 +1683,10 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
           error:
             typeof output.content === 'string'
               ? output.content
-              : JSON.stringify(output.content),
+              : serializeStructuredValueBounded(
+                output.content,
+                this.maxToolResultChars
+              ).content,
           stepId,
           turn,
         },
@@ -1682,10 +1738,14 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
       }
 
       if (postResult?.updatedOutput != null) {
-        const replaced =
-          typeof postResult.updatedOutput === 'string'
-            ? postResult.updatedOutput
-            : JSON.stringify(postResult.updatedOutput);
+        if (hasComputerCallOutputMarker(output)) {
+          if (!isComputerCallOutputContent(postResult.updatedOutput)) {
+            throw new Error(
+              'PostToolUse updatedOutput for a computer call must be a valid screenshot URL or screenshot content block.'
+            );
+          }
+          return cloneToolMessageWithContent(output, postResult.updatedOutput);
+        }
         // Keep the tool-output registry in sync with what the model
         // actually sees. Without this, `runTool` already registered
         // the PRE-hook content under `_refKey`, and a later
@@ -1700,17 +1760,21 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
           | undefined;
         const refKey = refMeta?._refKey;
         const refScope = refMeta?._refScope;
+        const replaced = serializeToolOutputWithinLimits(
+          postResult.updatedOutput,
+          this.maxToolResultChars,
+          this.toolOutputRegistry != null && refKey != null
+            ? this.toolOutputRegistry.perOutputLimit
+            : 0
+        );
         if (this.toolOutputRegistry != null && refKey != null) {
-          this.toolOutputRegistry.set(refScope, refKey, replaced);
+          this.toolOutputRegistry.set(
+            refScope,
+            refKey,
+            replaced.registryContent
+          );
         }
-        return new ToolMessage({
-          status: output.status,
-          name: output.name,
-          content: replaced,
-          artifact: output.artifact,
-          tool_call_id: output.tool_call_id,
-          additional_kwargs: output.additional_kwargs,
-        });
+        return cloneToolMessageWithContent(output, replaced.content);
       }
     }
 
@@ -1993,10 +2057,20 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
         continue;
       }
 
-      const contentString =
-        typeof toolMessage.content === 'string'
-          ? toolMessage.content
-          : JSON.stringify(toolMessage.content);
+      let contentString: string;
+      if (isComputerCallOutputMessage(toolMessage)) {
+        contentString = truncateToolResultContent(
+          '[Computer screenshot omitted from completion event]',
+          this.maxToolResultChars
+        );
+      } else if (typeof toolMessage.content === 'string') {
+        contentString = toolMessage.content;
+      } else {
+        contentString = serializeStructuredValueBounded(
+          toolMessage.content,
+          this.maxToolResultChars
+        ).content;
+      }
 
       /**
        * Prefer the post-substitution args when a `{{…}}` placeholder
@@ -2006,10 +2080,10 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
        */
       const effectiveArgs = resolvedArgsByCallId?.get(toolCallId) ?? call.args;
       const tool_call: t.ProcessedToolCall = {
-        args:
-          typeof effectiveArgs === 'string'
-            ? (effectiveArgs as string)
-            : JSON.stringify((effectiveArgs as unknown) ?? {}),
+        args: serializeToolContentBounded(
+          (effectiveArgs as unknown) ?? {},
+          this.maxToolResultChars
+        ),
         name: call.name,
         id: toolCallId,
         output: contentString,
@@ -2860,7 +2934,7 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
         if (result.injectedMessages && result.injectedMessages.length > 0) {
           try {
             injected.push(
-              ...this.convertInjectedMessages(result.injectedMessages)
+              ...convertInjectedMessages(result.injectedMessages)
             );
           } catch (e) {
             // eslint-disable-next-line no-console
@@ -2886,7 +2960,10 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
         let finalToolOutput: unknown = result.content;
 
         if (result.status === 'error') {
-          contentString = `Error: ${result.errorMessage ?? 'Unknown error'}\n Please fix your mistakes.`;
+          contentString = truncateToolResultContent(
+            `Error: ${result.errorMessage ?? 'Unknown error'}\n Please fix your mistakes.`,
+            this.maxToolResultChars
+          );
           /**
            * Error results bypass registration but stamp the
            * unresolved-refs hint into `additional_kwargs` so the lazy
@@ -2949,14 +3026,22 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
             }
           }
         } else {
-          let registryRaw =
-            typeof result.content === 'string'
-              ? result.content
-              : JSON.stringify(result.content);
-          contentString = truncateToolResultContent(
-            registryRaw,
-            this.maxToolResultChars
+          const batchIndex = batchIndexByCallId.get(result.toolCallId);
+          const refKey =
+            this.toolOutputRegistry != null &&
+            batchIndex != null &&
+            turn != null
+              ? buildReferenceKey(batchIndex, turn)
+              : undefined;
+          let serialized = serializeToolOutputWithinLimits(
+            result.content,
+            this.maxToolResultChars,
+            this.toolOutputRegistry != null && refKey != null
+              ? this.toolOutputRegistry.perOutputLimit
+              : 0
           );
+          let registryRaw = serialized.registryContent;
+          contentString = serialized.content;
 
           if (hasPostHook) {
             const hookResult = await executeHooks({
@@ -2983,15 +3068,15 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
               }
             }
             if (hookResult?.updatedOutput != null) {
-              const replaced =
-                typeof hookResult.updatedOutput === 'string'
-                  ? hookResult.updatedOutput
-                  : JSON.stringify(hookResult.updatedOutput);
-              registryRaw = replaced;
-              contentString = truncateToolResultContent(
-                replaced,
-                this.maxToolResultChars
+              serialized = serializeToolOutputWithinLimits(
+                hookResult.updatedOutput,
+                this.maxToolResultChars,
+                this.toolOutputRegistry != null && refKey != null
+                  ? this.toolOutputRegistry.perOutputLimit
+                  : 0
               );
+              registryRaw = serialized.registryContent;
+              contentString = serialized.content;
               finalToolOutput = hookResult.updatedOutput;
               /**
                * The hook ACTUALLY rewrote this output: any completion the
@@ -3005,14 +3090,7 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
             }
           }
 
-          const batchIndex = batchIndexByCallId.get(result.toolCallId);
           const unresolved = unresolvedByCallId.get(result.toolCallId) ?? [];
-          const refKey =
-            this.toolOutputRegistry != null &&
-            batchIndex != null &&
-            turn != null
-              ? buildReferenceKey(batchIndex, turn)
-              : undefined;
           const successRefMeta = this.recordOutputReference(
             registryRunId,
             stripCodeSessionFileSummary(registryRaw),
@@ -3266,7 +3344,7 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
      */
     if (hookInjectedMessages.length > 0) {
       try {
-        injected.push(...this.convertInjectedMessages(hookInjectedMessages));
+        injected.push(...convertInjectedMessages(hookInjectedMessages));
       } catch (e) {
         // eslint-disable-next-line no-console
         console.warn(
@@ -3303,7 +3381,7 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
           index: turn ?? this.toolUsageCount.get(toolName) ?? 0,
           type: 'tool_call' as const,
           tool_call: {
-            args: JSON.stringify(args),
+            args: serializeToolContentBounded(args, this.maxToolResultChars),
             name: toolName,
             id: toolCallId,
             output,
@@ -3330,13 +3408,14 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
   ): Promise<boolean> {
     const output =
       result.status === 'error'
-        ? `Error: ${result.errorMessage ?? 'Unknown error'}\n Please fix your mistakes.`
-        : truncateToolResultContent(
-          typeof result.content === 'string'
-            ? result.content
-            : JSON.stringify(result.content),
+        ? truncateToolResultContent(
+          `Error: ${result.errorMessage ?? 'Unknown error'}\n Please fix your mistakes.`,
           this.maxToolResultChars
-        );
+        )
+        : serializeToolOutputWithinLimits(
+          result.content,
+          this.maxToolResultChars
+        ).content;
     return this.dispatchStepCompleted(
       result.toolCallId,
       request.name,
@@ -3347,34 +3426,6 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
     );
   }
 
-  /**
-   * Converts InjectedMessage instances to LangChain HumanMessage objects.
-   * Both 'user' and 'system' roles become HumanMessage to avoid provider
-   * rejections (Anthropic/Google reject non-leading SystemMessages).
-   * The original role is preserved in additional_kwargs for downstream consumers.
-   */
-  private convertInjectedMessages(
-    messages: t.InjectedMessage[]
-  ): BaseMessage[] {
-    const converted: BaseMessage[] = [];
-    for (const msg of messages) {
-      const additional_kwargs: Record<string, unknown> = {
-        role: msg.role,
-      };
-      if (msg.isMeta != null) additional_kwargs.isMeta = msg.isMeta;
-      if (msg.source != null) additional_kwargs.source = msg.source;
-      if (msg.skillName != null) additional_kwargs.skillName = msg.skillName;
-
-      converted.push(
-        new HumanMessage({
-          content: toLangChainContent(msg.content),
-          additional_kwargs,
-        })
-      );
-    }
-    return converted;
-  }
-
   /**
    * Execute a group of direct (in-process) tool calls with interrupt-safe
    * ordering, returning outputs aligned 1:1 with `directCalls`.

package/src/tools/__tests__/ToolNode.onResultCompletion.test.ts

@@ -29,7 +29,7 @@ function createAIMessageWithToolCalls(
 type CompletionEvent = {
   result: {
     id: string;
-    tool_call: { id: string; output: string };
+    tool_call: { id: string; output: string; args?: string };
   };
 };
 
@@ -124,6 +124,153 @@ describe('ToolNode per-call onResult completion emission', () => {
     expect(result.messages.map((m) => m.content)).toEqual(['sunny', '42']);
   });
 
+  it('serializes bigint output before early and batch completion paths', async () => {
+    const completions: CompletionEvent[] = [];
+    const structuredOutput = [{ rowsRead: BigInt(42), status: 'complete' }];
+
+    jest
+      .spyOn(events, 'safeDispatchCustomEvent')
+      .mockImplementation(async (event, data): Promise<void> => {
+        if (event === GraphEvents.ON_RUN_STEP_COMPLETED) {
+          completions.push(data as CompletionEvent);
+          return;
+        }
+        if (event !== GraphEvents.ON_TOOL_EXECUTE) {
+          return;
+        }
+        const batch = data as t.ToolExecuteBatchRequest;
+        batch.onResult?.({
+          toolCallId: 'call_query',
+          status: 'success',
+          content: structuredOutput,
+        });
+        await flushAsync();
+        batch.resolve([
+          {
+            toolCallId: 'call_query',
+            status: 'success',
+            content: structuredOutput,
+          },
+        ]);
+      });
+
+    const toolNode = new ToolNode({
+      tools: [createDummyTool('query')],
+      eventDrivenMode: true,
+      toolCallStepIds: new Map([['call_query', 'step_query']]),
+    });
+    const result = (await toolNode.invoke({
+      messages: [
+        createAIMessageWithToolCalls([
+          { id: 'call_query', name: 'query', args: {} },
+        ]),
+      ],
+    })) as { messages: ToolMessage[] };
+
+    const serialized = '[{"rowsRead":"42","status":"complete"}]';
+    expect(completions).toHaveLength(1);
+    expect(completions[0].result.tool_call.output).toBe(serialized);
+    expect(result.messages[0].content).toBe(serialized);
+  });
+
+  it('omits native computer screenshots from completion events', async () => {
+    const completions: CompletionEvent[] = [];
+    const screenshot = `data:image/png;base64,${'A'.repeat(2_000)}`;
+    const computerOutput = new ToolMessage({
+      content: screenshot,
+      tool_call_id: 'call_computer',
+      additional_kwargs: { type: 'computer_call_output' },
+    });
+    const computer = createDummyTool('computer_use');
+    (
+      computer as unknown as {
+        invoke: () => Promise<ToolMessage>;
+      }
+    ).invoke = async () => computerOutput;
+    jest
+      .spyOn(events, 'safeDispatchCustomEvent')
+      .mockImplementation(async (event, data): Promise<void> => {
+        if (event === GraphEvents.ON_RUN_STEP_COMPLETED) {
+          completions.push(data as CompletionEvent);
+        }
+      });
+    const toolNode = new ToolNode({
+      tools: [computer],
+      eventDrivenMode: true,
+      directToolNames: new Set(['computer_use']),
+      toolCallStepIds: new Map([['call_computer', 'step_computer']]),
+      maxToolResultChars: 80,
+    });
+
+    const result = (await toolNode.invoke({
+      messages: [
+        createAIMessageWithToolCalls([
+          { id: 'call_computer', name: 'computer_use', args: {} },
+        ]),
+      ],
+    })) as { messages: ToolMessage[] };
+
+    expect(result.messages[0]).toBe(computerOutput);
+    expect(result.messages[0].content).toBe(screenshot);
+    expect(completions).toHaveLength(1);
+    expect(completions[0].result.tool_call.output).toContain(
+      'Computer screenshot omitted'
+    );
+    expect(completions[0].result.tool_call.output.length).toBeLessThanOrEqual(
+      80
+    );
+  });
+
+  it('bounds cyclic tool args without losing the completion event', async () => {
+    const completions: CompletionEvent[] = [];
+    const cyclicArgs: Record<string, unknown> = { city: 'NYC' };
+    cyclicArgs.self = cyclicArgs;
+
+    jest
+      .spyOn(events, 'safeDispatchCustomEvent')
+      .mockImplementation(async (event, data): Promise<void> => {
+        if (event === GraphEvents.ON_RUN_STEP_COMPLETED) {
+          completions.push(data as CompletionEvent);
+          return;
+        }
+        if (event !== GraphEvents.ON_TOOL_EXECUTE) {
+          return;
+        }
+        const batch = data as t.ToolExecuteBatchRequest;
+        batch.onResult?.({
+          toolCallId: 'call_weather',
+          status: 'success',
+          content: 'sunny',
+        });
+        await flushAsync();
+        batch.resolve([
+          {
+            toolCallId: 'call_weather',
+            status: 'success',
+            content: 'sunny',
+          },
+        ]);
+      });
+
+    const toolNode = new ToolNode({
+      tools: [createDummyTool('weather')],
+      eventDrivenMode: true,
+      toolCallStepIds: new Map([['call_weather', 'step_weather']]),
+    });
+
+    await toolNode.invoke({
+      messages: [
+        createAIMessageWithToolCalls([
+          { id: 'call_weather', name: 'weather', args: cyclicArgs },
+        ]),
+      ],
+    });
+
+    expect(completions).toHaveLength(1);
+    expect(completions[0].result.tool_call.args).toContain('[Circular]');
+    expect(completions[0].result.tool_call.output).toBe('sunny');
+  });
+
   it('ignores duplicate and unknown onResult reports', async () => {
     const completions: CompletionEvent[] = [];