@librechat/agents 3.2.64 → 3.2.66

package/src/llm/anthropic/index.ts

@@ -6,6 +6,7 @@ import type {
   BaseMessage,
   MessageContentComplex,
 } from '@langchain/core/messages';
+import type { ChatModelStreamEvent } from '@langchain/core/language_models/event';
 import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
 import type { AnthropicInput } from '@langchain/anthropic';
 import type { Anthropic } from '@anthropic-ai/sdk';
@@ -18,12 +19,15 @@ import type {
   AnthropicMCPServerURLDefinition,
   AnthropicContextManagementConfigParam,
   AnthropicRequestOptions,
+  AnthropicMessageStreamEvent,
 } from '@/llm/anthropic/types';
+import type { AnthropicUsageData } from './utils/message_outputs';
 import {
   _convertMessagesToAnthropicPayload,
   stripUnsupportedAssistantPrefill,
 } from './utils/message_inputs';
 import { _makeMessageChunkFromAnthropicEvent } from './utils/message_outputs';
+import { convertAnthropicStream } from './utils/stream_events';
 import { handleToolChoice } from './utils/tools';
 
 const DEFAULT_STREAM_DELAY = 25;
@@ -34,6 +38,23 @@ const STREAM_BOUNDARIES = new Set([' ', '.', ',', '!', '?', ';', ':']);
 
 type StreamTokenType = 'string' | 'input' | 'content';
 
+interface AnthropicStreamUsage {
+  inputTokens: number;
+  cacheCreationInputTokens: number;
+  cacheReadInputTokens: number;
+  outputTokens: number;
+}
+
+interface CumulativeUsageValue {
+  cumulative: number;
+  increment: number;
+}
+
+interface AnthropicEventStream
+  extends AsyncIterable<AnthropicMessageStreamEvent> {
+  controller?: { abort: () => void };
+}
+
 const ANTHROPIC_TOOL_BETAS: Partial<Record<string, AnthropicBeta>> = {
   tool_search_tool_regex_20251119: 'advanced-tool-use-2025-11-20',
   tool_search_tool_bm25_20251119: 'advanced-tool-use-2025-11-20',
@@ -324,6 +345,19 @@ function isSignalAborted(signal?: AbortSignal): boolean {
   return signal?.aborted === true;
 }
 
+async function* abortableAnthropicStream(
+  source: AnthropicEventStream,
+  signal?: AbortSignal
+): AsyncGenerator<AnthropicMessageStreamEvent> {
+  for await (const data of source) {
+    if (isSignalAborted(signal)) {
+      source.controller?.abort();
+      return;
+    }
+    yield data;
+  }
+}
+
 function extractToken(
   chunk: AIMessageChunk
 ): [string, StreamTokenType] | [undefined] {
@@ -392,27 +426,77 @@ function cloneChunk(
   return chunk;
 }
 
+function resolveCumulativeUsageValue(
+  current: number | null | undefined,
+  previous: number
+): CumulativeUsageValue {
+  if (current == null) {
+    return { cumulative: previous, increment: 0 };
+  }
+  const cumulative = Math.max(previous, current);
+  return { cumulative, increment: cumulative - previous };
+}
+
 function withIncrementalMessageDeltaUsage(
   chunk: AIMessageChunk,
-  previousOutputTokens: number
-): { chunk: AIMessageChunk; outputTokens: number } {
+  previousUsage: AnthropicStreamUsage,
+  cumulativeUsage: AnthropicUsageData
+): { chunk: AIMessageChunk; usage: AnthropicStreamUsage } {
   const usage = chunk.usage_metadata;
   if (usage == null) {
-    return { chunk, outputTokens: previousOutputTokens };
+    return { chunk, usage: previousUsage };
   }
 
-  const outputTokens = Math.max(0, usage.output_tokens - previousOutputTokens);
+  const inputTokens = resolveCumulativeUsageValue(
+    cumulativeUsage.input_tokens,
+    previousUsage.inputTokens
+  );
+  const cacheCreationInputTokens = resolveCumulativeUsageValue(
+    cumulativeUsage.cache_creation_input_tokens,
+    previousUsage.cacheCreationInputTokens
+  );
+  const cacheReadInputTokens = resolveCumulativeUsageValue(
+    cumulativeUsage.cache_read_input_tokens,
+    previousUsage.cacheReadInputTokens
+  );
+  const outputTokens = resolveCumulativeUsageValue(
+    cumulativeUsage.output_tokens,
+    previousUsage.outputTokens
+  );
+  const incrementalInputTokens =
+    inputTokens.increment +
+    cacheCreationInputTokens.increment +
+    cacheReadInputTokens.increment;
+  const hasCacheUsage =
+    cumulativeUsage.cache_creation_input_tokens != null ||
+    cumulativeUsage.cache_read_input_tokens != null;
+  const inputTokenDetails = {
+    ...(cumulativeUsage.cache_creation_input_tokens != null && {
+      cache_creation: cacheCreationInputTokens.increment,
+    }),
+    ...(cumulativeUsage.cache_read_input_tokens != null && {
+      cache_read: cacheReadInputTokens.increment,
+    }),
+  };
+
   return {
     chunk: new AIMessageChunk(
       Object.assign({}, chunk, {
         usage_metadata: {
           ...usage,
-          output_tokens: outputTokens,
-          total_tokens: usage.input_tokens + outputTokens,
+          input_tokens: incrementalInputTokens,
+          output_tokens: outputTokens.increment,
+          total_tokens: incrementalInputTokens + outputTokens.increment,
+          input_token_details: hasCacheUsage ? inputTokenDetails : undefined,
         },
       })
     ),
-    outputTokens: usage.output_tokens,
+    usage: {
+      inputTokens: inputTokens.cumulative,
+      cacheCreationInputTokens: cacheCreationInputTokens.cumulative,
+      cacheReadInputTokens: cacheReadInputTokens.cumulative,
+      outputTokens: outputTokens.cumulative,
+    },
   };
 }
 
@@ -626,6 +710,29 @@ export class CustomAnthropic extends ChatAnthropicMessages {
     );
   }
 
+  async *_streamChatModelEvents(
+    messages: BaseMessage[],
+    options: this['ParsedCallOptions'],
+    _runManager?: CallbackManagerForLLMRun
+  ): AsyncGenerator<ChatModelStreamEvent> {
+    const params = this.invocationParams(options);
+    const formattedMessages = _convertMessagesToAnthropicPayload(messages);
+    const payload = stripUnsupportedAssistantPrefill({
+      ...params,
+      ...formattedMessages,
+      stream: true,
+    } as const);
+    const stream = await this.createStreamWithRetry(payload, {
+      headers: options.headers,
+      signal: options.signal,
+    });
+    const shouldStreamUsage = options.streamUsage ?? this.streamUsage;
+    yield* convertAnthropicStream(
+      abortableAnthropicStream(stream, options.signal),
+      { streamUsage: shouldStreamUsage }
+    );
+  }
+
   async *_streamResponseChunks(
     messages: BaseMessage[],
     options: this['ParsedCallOptions'],
@@ -651,7 +758,12 @@ export class CustomAnthropic extends ChatAnthropicMessages {
     });
 
     const shouldStreamUsage = options.streamUsage ?? this.streamUsage;
-    let messageDeltaOutputTokens = 0;
+    let streamUsage: AnthropicStreamUsage = {
+      inputTokens: 0,
+      cacheCreationInputTokens: 0,
+      cacheReadInputTokens: 0,
+      outputTokens: 0,
+    };
     const queuedChunks: QueuedGenerationChunk[] = [];
     const producerState: {
       done: boolean;
@@ -833,13 +945,25 @@ export class CustomAnthropic extends ChatAnthropicMessages {
           }
 
           let { chunk } = result;
+          if (data.type === 'message_start') {
+            streamUsage = {
+              ...streamUsage,
+              inputTokens: data.message.usage.input_tokens,
+              outputTokens: data.message.usage.output_tokens,
+              cacheCreationInputTokens:
+                data.message.usage.cache_creation_input_tokens ?? 0,
+              cacheReadInputTokens:
+                data.message.usage.cache_read_input_tokens ?? 0,
+            };
+          }
           if (data.type === 'message_delta') {
             const incremental = withIncrementalMessageDeltaUsage(
               chunk,
-              messageDeltaOutputTokens
+              streamUsage,
+              data.usage
             );
             chunk = incremental.chunk;
-            messageDeltaOutputTokens = incremental.outputTokens;
+            streamUsage = incremental.usage;
           }
 
           const [token = '', tokenType] = extractToken(chunk);

package/src/llm/anthropic/inherited-content-utils.spec.ts

@@ -185,7 +185,7 @@ describe('_makeMessageChunkFromAnthropicEvent (ported from message_outputs.test.
     expect(usage.input_token_details?.cache_read).toBe(1000);
   });
 
-  it('message_delta chunk has input_tokens=0 and no cache token details', () => {
+  it('message_delta chunk preserves cumulative input and cache usage', () => {
     const event = {
       type: 'message_delta' as const,
       delta: { stop_reason: 'end_turn' as const, stop_sequence: null },
@@ -202,10 +202,15 @@ describe('_makeMessageChunkFromAnthropicEvent (ported from message_outputs.test.
     expect(result).not.toBeNull();
 
     const usage = result!.chunk.usage_metadata!;
-    expect(usage.output_tokens).toBe(42);
-    expect(usage.input_tokens).toBe(0);
-    expect(usage.input_token_details?.cache_creation).toBeUndefined();
-    expect(usage.input_token_details?.cache_read).toBeUndefined();
+    expect(usage).toEqual({
+      input_tokens: 1600,
+      output_tokens: 42,
+      total_tokens: 1642,
+      input_token_details: {
+        cache_creation: 500,
+        cache_read: 1000,
+      },
+    });
   });
 });