@librechat/agents 3.3.3 → 3.3.5

package/src/messages/cache.ts

@@ -1,16 +1,10 @@
-import {
-  AIMessage,
-  BaseMessage,
-  ToolMessage,
-  HumanMessage,
-  SystemMessage,
-  MessageContentComplex,
-} from '@langchain/core/messages';
+import { isProxy } from 'node:util/types';
+import { BaseMessage, MessageContentComplex } from '@langchain/core/messages';
 import type Anthropic from '@anthropic-ai/sdk';
 import type { AnthropicMessage } from '@/types/messages';
+import { hasComputerCallOutputMarker } from '@/utils/toolContent';
 import { toLangChainContent } from './langchain';
 import { ContentTypes } from '@/common/enum';
-import { withMessageRole } from './format';
 
 type MessageWithContent = {
   content?: string | MessageContentComplex[];
@@ -134,13 +128,30 @@ export function resolveBedrockPromptCacheTtl(
  * Deep clones a message's content to prevent mutation of the original.
  */
 function deepCloneContent<T extends string | MessageContentComplex[]>(
-  content: T
+  content: T,
+  omittedProperty?: string
 ): T {
   if (typeof content === 'string') {
     return content;
   }
   if (Array.isArray(content)) {
-    return content.map((block) => ({ ...block })) as T;
+    return content.map((block) => {
+      if (typeof block !== 'object' || block == null || isProxy(block)) {
+        return block;
+      }
+      try {
+        const descriptors = Object.getOwnPropertyDescriptors(block);
+        if (omittedProperty != null) {
+          delete descriptors[omittedProperty];
+        }
+        return Object.create(
+          Object.getPrototypeOf(block),
+          descriptors
+        ) as MessageContentComplex;
+      } catch {
+        return block;
+      }
+    }) as T;
   }
   return content;
 }
@@ -156,45 +167,36 @@ export function cloneMessage<T extends MessageWithContent>(
   content: string | MessageContentComplex[]
 ): T {
   if (message instanceof BaseMessage) {
-    const baseParams = {
-      content: toLangChainContent(content),
-      additional_kwargs: { ...message.additional_kwargs },
-      response_metadata: { ...message.response_metadata },
-      id: message.id,
-      name: message.name,
+    const descriptors = Object.getOwnPropertyDescriptors(message) as Record<
+      PropertyKey,
+      PropertyDescriptor | undefined
+    >;
+    const contentDescriptor = descriptors.content;
+    descriptors.content = {
+      configurable: contentDescriptor?.configurable ?? true,
+      enumerable: contentDescriptor?.enumerable ?? true,
+      value: toLangChainContent(content),
+      writable: contentDescriptor?.writable ?? true,
     };
-
-    const msgType = message.getType();
-    switch (msgType) {
-    case 'ai':
-      return withMessageRole(
-        new AIMessage({
-          ...baseParams,
-          tool_calls: (message as unknown as AIMessage).tool_calls,
-        }),
-        'assistant'
-      ) as unknown as T;
-    case 'human':
-      return withMessageRole(
-        new HumanMessage(baseParams),
-        'user'
-      ) as unknown as T;
-    case 'system':
-      return withMessageRole(
-        new SystemMessage(baseParams),
-        'system'
-      ) as unknown as T;
-    case 'tool':
-      return withMessageRole(
-        new ToolMessage({
-          ...baseParams,
-          tool_call_id: (message as unknown as ToolMessage).tool_call_id,
-        }),
-        'tool'
-      ) as unknown as T;
-    default:
-      break;
+    const lcKwargs = descriptors.lc_kwargs;
+    if (
+      lcKwargs != null &&
+      'value' in lcKwargs &&
+      typeof lcKwargs.value === 'object' &&
+      lcKwargs.value != null
+    ) {
+      descriptors.lc_kwargs = {
+        ...lcKwargs,
+        value: {
+          ...(lcKwargs.value as Record<string, unknown>),
+          content: toLangChainContent(content),
+        },
+      };
     }
+    return Object.create(
+      Object.getPrototypeOf(message),
+      descriptors as PropertyDescriptorMap
+    ) as T;
   }
 
   const {
@@ -251,7 +253,11 @@ function sanitizeBedrockSystemMessage<T extends MessageWithContent>(
   let modified = false;
   for (const block of content) {
     if (isCachePoint(block)) {
-      const existing = (block as { cachePoint?: { ttl?: unknown } }).cachePoint;
+      const descriptor = Object.getOwnPropertyDescriptor(block, 'cachePoint');
+      const existing =
+        descriptor != null && 'value' in descriptor
+          ? (descriptor.value as { ttl?: unknown } | undefined)
+          : undefined;
       const desired = buildBedrockCachePoint(ttl);
       if (existing?.ttl !== desired.ttl) {
         modified = true;
@@ -378,8 +384,17 @@ export function addCacheControl<T extends AnthropicMessage | BaseMessage>(
 /**
  * Checks if a content block is a cache point
  */
-function isCachePoint(block: MessageContentComplex): boolean {
-  return 'cachePoint' in block && !('type' in block);
+function isCachePoint(block: unknown): boolean {
+  if (typeof block !== 'object' || block == null || isProxy(block)) {
+    return false;
+  }
+  try {
+    const cachePoint = Object.getOwnPropertyDescriptor(block, 'cachePoint');
+    const type = Object.getOwnPropertyDescriptor(block, 'type');
+    return cachePoint?.enumerable === true && type == null;
+  } catch {
+    return false;
+  }
 }
 
 /**
@@ -460,8 +475,13 @@ export function addTailCacheControl<T extends AnthropicMessage | BaseMessage>(
     const originalMessage = updatedMessages[i];
     const content = originalMessage.content;
     const hasArrayContent = Array.isArray(content);
+    const isComputerOutput =
+      originalMessage instanceof BaseMessage &&
+      hasComputerCallOutputMarker(originalMessage);
     const canPlaceMarker =
-      !markerPlaced && !isSyntheticMetaMessage(originalMessage);
+      !markerPlaced &&
+      !isSyntheticMetaMessage(originalMessage) &&
+      !isComputerOutput;
 
     // Earlier string-content messages carry no markers to strip.
     if (!canPlaceMarker && !hasArrayContent) {
@@ -706,7 +726,21 @@ export function addCacheControlToStablePrefixMessages<
  */
 function hasAnthropicCacheControl(content: MessageContentComplex[]): boolean {
   for (let i = 0; i < content.length; i++) {
-    if ('cache_control' in content[i]) return true;
+    const block = content[i];
+    if (typeof block !== 'object' || block == null || isProxy(block)) {
+      continue;
+    }
+    try {
+      const cacheControl = Object.getOwnPropertyDescriptor(
+        block,
+        'cache_control'
+      );
+      if (cacheControl?.enumerable === true) {
+        return true;
+      }
+    } catch {
+      continue;
+    }
   }
   return false;
 }
@@ -733,13 +767,7 @@ export function stripAnthropicCacheControl<T extends MessageWithContent>(
       continue;
     }
 
-    const clonedContent = deepCloneContent(content);
-    for (let j = 0; j < clonedContent.length; j++) {
-      const block = clonedContent[j] as Record<string, unknown>;
-      if ('cache_control' in block) {
-        delete block.cache_control;
-      }
-    }
+    const clonedContent = deepCloneContent(content, 'cache_control');
     updatedMessages[i] = cloneMessage(originalMessage, clonedContent);
   }
 
@@ -961,10 +989,14 @@ export function addBedrockTailCacheControl<
       'lc_namespace' in originalMessage;
     const hasArrayContent = Array.isArray(content);
     const isEmptyString = typeof content === 'string' && content === '';
+    const isComputerOutput =
+      originalMessage instanceof BaseMessage &&
+      hasComputerCallOutputMarker(originalMessage);
     const canPlaceCachePoint =
       !cachePointPlaced &&
       !isEmptyString &&
       !isSyntheticMetaMessage(originalMessage) &&
+      !isComputerOutput &&
       (typeof content === 'string' || hasArrayContent);
 
     if (!canPlaceCachePoint && !hasArrayContent && !hasSerializationProps) {

package/src/messages/contextPruning.test.ts

@@ -0,0 +1,184 @@
+import {
+  AIMessage,
+  HumanMessage,
+  ToolMessage,
+  type BaseMessage,
+} from '@langchain/core/messages';
+import { applyContextPruning } from './contextPruning';
+
+function charCounter(message: BaseMessage): number {
+  return (
+    typeof message.content === 'string'
+      ? message.content
+      : JSON.stringify(message.content)
+  ).length;
+}
+
+describe('applyContextPruning', () => {
+  it('soft-trims old structured tool results and preserves metadata', () => {
+    const toolCallId = 'tc-old';
+    const messages: BaseMessage[] = [
+      new HumanMessage('query the table'),
+      new AIMessage({
+        content: '',
+        tool_calls: [
+          {
+            id: toolCallId,
+            name: 'run_select_query',
+            args: {},
+            type: 'tool_call',
+          },
+        ],
+      }),
+      new ToolMessage({
+        content: [
+          {
+            type: 'json',
+            rows: Array.from({ length: 20 }, (_, index) => ({
+              id: index,
+              value: `${'x'.repeat(100)}-${index}`,
+            })),
+          },
+        ],
+        tool_call_id: toolCallId,
+        name: 'run_select_query',
+        status: 'success',
+        artifact: { source: 'clickhouse' },
+      }),
+      new AIMessage('The query returned 20 rows.'),
+      new HumanMessage('continue'),
+      new AIMessage('Ready.'),
+      new HumanMessage('next question'),
+    ];
+    const indexTokenCountMap: Record<string, number | undefined> = {};
+    for (let i = 0; i < messages.length; i++) {
+      indexTokenCountMap[i] = charCounter(messages[i]);
+    }
+
+    const result = applyContextPruning({
+      messages,
+      indexTokenCountMap,
+      tokenCounter: charCounter,
+      config: {
+        enabled: true,
+        keepLastAssistants: 1,
+        softTrimRatio: 0,
+        minPrunableToolChars: 1,
+        softTrim: { maxChars: 200, headChars: 80, tailChars: 40 },
+        hardClear: { enabled: false },
+      },
+    });
+
+    expect(result.softTrimmed).toBe(1);
+    expect(typeof messages[2].content).toBe('string');
+    expect(messages[2].content).toContain('soft-trimmed');
+    const trimmed = messages[2] as ToolMessage;
+    expect(trimmed.tool_call_id).toBe(toolCallId);
+    expect(trimmed.name).toBe('run_select_query');
+    expect(trimmed.status).toBe('success');
+    expect(trimmed.artifact).toEqual({ source: 'clickhouse' });
+    expect(indexTokenCountMap[2]).toBe(charCounter(trimmed));
+  });
+
+  it('preserves small non-image media blocks while trimming adjacent text', () => {
+    const document = {
+      type: 'document',
+      source: { type: 'url', url: 'https://example.com/report.pdf' },
+    };
+    const messages: BaseMessage[] = [
+      new HumanMessage('read the report'),
+      new AIMessage({
+        content: '',
+        tool_calls: [
+          {
+            id: 'tc-document',
+            name: 'read_document',
+            args: {},
+            type: 'tool_call',
+          },
+        ],
+      }),
+      new ToolMessage({
+        content: [{ type: 'text', text: 'x'.repeat(2_000) }, document],
+        tool_call_id: 'tc-document',
+      }),
+      new AIMessage('I read the report.'),
+      new HumanMessage('continue'),
+      new AIMessage('Ready.'),
+      new HumanMessage('next question'),
+    ];
+    const indexTokenCountMap: Record<string, number | undefined> = {};
+    for (let i = 0; i < messages.length; i++) {
+      indexTokenCountMap[i] = charCounter(messages[i]);
+    }
+
+    const result = applyContextPruning({
+      messages,
+      indexTokenCountMap,
+      tokenCounter: charCounter,
+      config: {
+        enabled: true,
+        keepLastAssistants: 1,
+        softTrimRatio: 0,
+        minPrunableToolChars: 1,
+        softTrim: { maxChars: 400, headChars: 80, tailChars: 40 },
+        hardClear: { enabled: false },
+      },
+    });
+
+    expect(result.softTrimmed).toBe(1);
+    expect(Array.isArray(messages[2].content)).toBe(true);
+    expect(messages[2].content).toContain(document);
+  });
+
+  it('does not hard-clear native computer screenshots', () => {
+    const screenshot = `data:image/png;base64,${'A'.repeat(2_000)}`;
+    const computerOutput = new ToolMessage({
+      content: screenshot,
+      tool_call_id: 'tc-computer',
+      additional_kwargs: { type: 'computer_call_output' },
+    });
+    const messages: BaseMessage[] = [
+      new HumanMessage('take a screenshot'),
+      new AIMessage({
+        content: '',
+        tool_calls: [
+          {
+            id: 'tc-computer',
+            name: 'computer',
+            args: {},
+            type: 'tool_call',
+          },
+        ],
+      }),
+      computerOutput,
+      new AIMessage('Done.'),
+      new HumanMessage('continue'),
+      new AIMessage('Ready.'),
+      new HumanMessage('next question'),
+    ];
+    const indexTokenCountMap: Record<string, number | undefined> = {};
+    for (let i = 0; i < messages.length; i++) {
+      indexTokenCountMap[i] = charCounter(messages[i]);
+    }
+
+    const result = applyContextPruning({
+      messages,
+      indexTokenCountMap,
+      tokenCounter: charCounter,
+      config: {
+        enabled: true,
+        keepLastAssistants: 1,
+        softTrimRatio: 0,
+        minPrunableToolChars: 1,
+        softTrim: { maxChars: 80, headChars: 20, tailChars: 20 },
+        hardClear: { enabled: true },
+      },
+    });
+
+    expect(result.softTrimmed).toBe(0);
+    expect(result.hardCleared).toBe(0);
+    expect(messages[2]).toBe(computerOutput);
+    expect(messages[2].content).toBe(screenshot);
+  });
+});

package/src/messages/contextPruning.ts

@@ -9,32 +9,24 @@
  * - Hard-clear: Replace entire content with a placeholder.
  *
  * Messages in the "protected zone" (recent assistant turns, system/pre-first-human
- * messages, and messages with image content) are never pruned.
+ * messages) are never pruned. Atomic media/resource blocks are preserved when
+ * they fit; oversized inline payloads are replaced with bounded placeholders.
  */
 
 import { ToolMessage, type BaseMessage } from '@langchain/core/messages';
 import type { ContextPruningSettings } from './contextPruningSettings';
 import type { ContextPruningConfig } from '@/types/graph';
 import type { TokenCounter } from '@/types/run';
+import {
+  cloneToolMessageWithContent,
+  compactToolContent,
+  getToolContentCharLength,
+  isAtomicToolContentBlock,
+  isComputerCallOutputMessage,
+  serializeToolContentBounded,
+} from '@/utils/toolContent';
 import { resolveContextPruningSettings } from './contextPruningSettings';
 
-/**
- * Checks if a message contains image content blocks.
- * Messages with images are skipped by position-based content degradation
- * because images cannot be meaningfully soft-trimmed or replaced with placeholders.
- */
-function hasImageContent(message: BaseMessage): boolean {
-  if (!Array.isArray(message.content)) {
-    return false;
-  }
-  return message.content.some(
-    (block) =>
-      typeof block === 'object' &&
-      'type' in block &&
-      (block.type === 'image_url' || block.type === 'image')
-  );
-}
-
 /**
  * Applies head+tail soft-trim to tool result content.
  */
@@ -138,48 +130,63 @@ export function applyContextPruning(params: {
     if (message.getType() !== 'tool') {
       continue;
     }
-    if (protectedIndices.has(i)) {
+    if (isComputerCallOutputMessage(message)) {
       continue;
     }
-    if (hasImageContent(message)) {
+    if (protectedIndices.has(i)) {
       continue;
     }
-
     const content = message.content;
-    if (typeof content !== 'string') {
-      continue;
-    }
-    if (content.length < settings.minPrunableToolChars) {
+    const contentLength = getToolContentCharLength(content);
+    if (contentLength < settings.minPrunableToolChars) {
       continue;
     }
 
     // Compute age ratio: how far back from the end (0 = latest, 1 = oldest)
     const ageRatio = (totalMessages - i) / totalMessages;
 
+    if (
+      Array.isArray(content) &&
+      content.some(isAtomicToolContentBlock) &&
+      ageRatio >= settings.softTrimRatio
+    ) {
+      const compacted = compactToolContent(content, settings.softTrim.maxChars);
+      if (compacted.changed) {
+        const cloned = cloneToolMessageWithContent(
+          message as ToolMessage,
+          compacted.content
+        );
+        messages[i] = cloned;
+        indexTokenCountMap[i] = tokenCounter(cloned);
+        softTrimmed++;
+      }
+      continue;
+    }
+
     if (ageRatio >= settings.hardClearRatio && settings.hardClear.enabled) {
       // Hard-clear: replace with placeholder
-      const cloned = new ToolMessage({
-        content: settings.hardClear.placeholder,
-        tool_call_id: (message as ToolMessage).tool_call_id,
-        name: message.name,
-        id: message.id,
-        additional_kwargs: message.additional_kwargs,
-        response_metadata: message.response_metadata,
-      });
+      const cloned = cloneToolMessageWithContent(
+        message as ToolMessage,
+        settings.hardClear.placeholder
+      );
       messages[i] = cloned;
       indexTokenCountMap[i] = tokenCounter(cloned);
       hardCleared++;
     } else if (ageRatio >= settings.softTrimRatio) {
       // Soft-trim: keep head + tail
-      if (content.length > settings.softTrim.maxChars) {
-        const cloned = new ToolMessage({
-          content: softTrimContent(content, settings.softTrim),
-          tool_call_id: (message as ToolMessage).tool_call_id,
-          name: message.name,
-          id: message.id,
-          additional_kwargs: message.additional_kwargs,
-          response_metadata: message.response_metadata,
-        });
+      if (contentLength > settings.softTrim.maxChars) {
+        const cloned = cloneToolMessageWithContent(
+          message as ToolMessage,
+          softTrimContent(
+            typeof content === 'string'
+              ? content
+              : serializeToolContentBounded(
+                content,
+                settings.softTrim.maxChars
+              ),
+            settings.softTrim
+          )
+        );
         messages[i] = cloned;
         indexTokenCountMap[i] = tokenCounter(cloned);
         softTrimmed++;