@librechat/agents 3.1.78 → 3.1.79

package/src/llm/anthropic/llm.spec.ts

@@ -31,16 +31,39 @@ import type {
   MessageContentComplex,
 } from '@langchain/core/messages';
 import { toLangChainContent } from '@/messages/langchain';
+import { formatAgentMessages } from '@/messages/format';
+import { Constants, ContentTypes, GraphEvents, Providers } from '@/common';
 import { _documentsInParams, CustomAnthropic as ChatAnthropic } from './index';
+import { partitionAndMarkAnthropicToolCache } from '@/messages/anthropicToolCache';
+import { ChatModelStreamHandler, createContentAggregator } from '@/stream';
+import { ModelEndHandler, ToolEndHandler } from '@/events';
+import { Run } from '@/run';
 import type { CustomAnthropicCallOptions } from './index';
 import type {
   AnthropicContextManagementConfigParam,
   AnthropicMessageCreateParams,
+  AnthropicMessageStreamEvent,
   AnthropicMessageResponse,
   AnthropicOutputConfig,
+  AnthropicRequestOptions,
+  AnthropicStreamingMessageCreateParams,
   AnthropicThinkingConfigParam,
   ChatAnthropicContentBlock,
 } from './types';
+import type {
+  AnthropicClientOptions,
+  IState,
+  MessageContentComplex as LibreChatContentBlock,
+  MessageDeltaEvent,
+  ReasoningDeltaEvent,
+  RunConfig,
+  RunStep,
+  RunStepDeltaEvent,
+  SharedLLMConfig,
+  StreamEventData,
+  ToolEndEvent,
+  TPayload,
+} from '@/types';
 import { _convertMessagesToAnthropicPayload } from './utils/message_inputs';
 import {
   _makeMessageChunkFromAnthropicEvent,
@@ -86,6 +109,8 @@ const remoteImageUrl =
 
 // Use this model for all other tests
 const modelName = 'claude-haiku-4-5-20251001';
+const webSearchModelName =
+  process.env.ANTHROPIC_WEB_SEARCH_MODEL ?? 'claude-opus-4-7';
 
 type AnthropicThinkingResponseBlock = Anthropic.Messages.ThinkingBlock & {
   index?: number;
@@ -119,6 +144,12 @@ type CompactionContentBlock = ContentBlock & {
   content: string;
 };
 
+type AnthropicContentBlockWithId = ContentBlock & {
+  id?: unknown;
+  input?: unknown;
+  name?: unknown;
+};
+
 function getLangChainErrorCode(error: unknown): string | undefined {
   if (typeof error !== 'object' || error == null) {
     return undefined;
@@ -200,6 +231,184 @@ function isCompactionBlock(
   return typeof content === 'string';
 }
 
+function isServerToolUseBlock(
+  block: ContentBlock
+): block is AnthropicContentBlockWithId {
+  return (
+    block.type === 'server_tool_use' &&
+    typeof (block as AnthropicContentBlockWithId).id === 'string' &&
+    ((block as AnthropicContentBlockWithId).id as string).startsWith(
+      Constants.ANTHROPIC_SERVER_TOOL_PREFIX
+    )
+  );
+}
+
+function expectAnthropicPayloadContentIsNonEmpty(
+  payload: AnthropicMessageCreateParams
+): void {
+  for (const message of payload.messages) {
+    if (typeof message.content === 'string') {
+      expect(message.content.trim().length).toBeGreaterThan(0);
+      continue;
+    }
+
+    expect(message.content.length).toBeGreaterThan(0);
+    for (const block of message.content) {
+      if (block.type !== 'text') {
+        continue;
+      }
+      expect(block.text.trim().length).toBeGreaterThan(0);
+    }
+  }
+}
+
+function expectNoDanglingServerToolUses(
+  payload: AnthropicMessageCreateParams
+): void {
+  for (const message of payload.messages) {
+    if (typeof message.content === 'string') {
+      continue;
+    }
+
+    const serverToolResultIds = new Set(
+      message.content
+        .map((block) =>
+          'tool_use_id' in block &&
+          typeof block.tool_use_id === 'string' &&
+          block.tool_use_id.startsWith(Constants.ANTHROPIC_SERVER_TOOL_PREFIX)
+            ? block.tool_use_id
+            : undefined
+        )
+        .filter((id): id is string => id != null)
+    );
+
+    for (const block of message.content) {
+      if (block.type !== 'server_tool_use') {
+        continue;
+      }
+      expect(serverToolResultIds.has(block.id)).toBe(true);
+    }
+  }
+}
+
+function getPromptCachedWebSearchTools(): Parameters<
+  ChatAnthropic['bindTools']
+>[0] {
+  const tools = partitionAndMarkAnthropicToolCache(
+    [
+      {
+        type: 'web_search_20250305',
+        name: 'web_search',
+        max_uses: 3,
+      },
+    ] as never,
+    () => false
+  );
+  return tools as Parameters<ChatAnthropic['bindTools']>[0];
+}
+
+function getWebSearchTool(): {
+  type: 'web_search_20250305';
+  name: 'web_search';
+  max_uses: number;
+} {
+  return {
+    type: 'web_search_20250305',
+    name: 'web_search',
+    max_uses: 3,
+  };
+}
+
+function getWebSearchLLMConfig(): AnthropicClientOptions & SharedLLMConfig {
+  return {
+    provider: Providers.ANTHROPIC,
+    model: webSearchModelName,
+    maxTokens: 1024,
+    promptCache: true,
+    streaming: true,
+    streamUsage: true,
+    thinking: { type: 'adaptive' },
+  } as AnthropicClientOptions & SharedLLMConfig;
+}
+
+async function createWebSearchRun({
+  runId,
+  customHandlers,
+}: {
+  runId: string;
+  customHandlers?: RunConfig['customHandlers'];
+}): Promise<Run<IState>> {
+  return await Run.create<IState>({
+    runId,
+    graphConfig: {
+      type: 'standard',
+      llmConfig: getWebSearchLLMConfig(),
+      tools: [getWebSearchTool()],
+      instructions:
+        'You are a concise assistant. Use web search when current facts are needed.',
+    },
+    returnContent: true,
+    skipCleanup: true,
+    customHandlers,
+  });
+}
+
+function createLibreChatContentHandlers(): {
+  aggregateContent: ReturnType<
+    typeof createContentAggregator
+  >['aggregateContent'];
+  contentParts: Array<LibreChatContentBlock | undefined>;
+  customHandlers: NonNullable<RunConfig['customHandlers']>;
+} {
+  const { contentParts, aggregateContent } = createContentAggregator();
+  const customHandlers = {
+    [GraphEvents.TOOL_END]: new ToolEndHandler(),
+    [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(),
+    [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(),
+    [GraphEvents.ON_RUN_STEP_COMPLETED]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_COMPLETED,
+        data: StreamEventData
+      ): void => {
+        aggregateContent({
+          event,
+          data: data as unknown as { result: ToolEndEvent },
+        });
+      },
+    },
+    [GraphEvents.ON_RUN_STEP]: {
+      handle: (event: GraphEvents.ON_RUN_STEP, data: StreamEventData): void => {
+        aggregateContent({ event, data: data as RunStep });
+      },
+    },
+    [GraphEvents.ON_RUN_STEP_DELTA]: {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_DELTA,
+        data: StreamEventData
+      ): void => {
+        aggregateContent({ event, data: data as RunStepDeltaEvent });
+      },
+    },
+    [GraphEvents.ON_MESSAGE_DELTA]: {
+      handle: (
+        event: GraphEvents.ON_MESSAGE_DELTA,
+        data: StreamEventData
+      ): void => {
+        aggregateContent({ event, data: data as MessageDeltaEvent });
+      },
+    },
+    [GraphEvents.ON_REASONING_DELTA]: {
+      handle: (
+        event: GraphEvents.ON_REASONING_DELTA,
+        data: StreamEventData
+      ): void => {
+        aggregateContent({ event, data: data as ReasoningDeltaEvent });
+      },
+    },
+  };
+  return { aggregateContent, contentParts, customHandlers };
+}
+
 test('Test ChatAnthropic', async () => {
   const chat = new ChatAnthropic({
     modelName,
@@ -691,6 +900,287 @@ test('Anthropic usage metadata includes cache input token buckets', () => {
   });
 });
 
+type AnthropicStreamEvent = Anthropic.Beta.Messages.BetaRawMessageStreamEvent;
+
+function createMockAnthropicStream(events: AnthropicStreamEvent[]) {
+  return {
+    controller: { abort: jest.fn() },
+    async *[Symbol.asyncIterator]() {
+      for (const event of events) {
+        yield event;
+      }
+    },
+  };
+}
+
+class MockStreamingAnthropic extends ChatAnthropic {
+  constructor(private readonly mockEvents: AnthropicStreamEvent[]) {
+    super({
+      modelName,
+      apiKey: 'test-key',
+      maxTokens: 10,
+      streamUsage: true,
+    });
+  }
+
+  protected override async createStreamWithRetry() {
+    return createMockAnthropicStream(this.mockEvents) as never;
+  }
+}
+
+class RecordingStreamingAnthropic extends ChatAnthropic {
+  messageStartOutputTokens = 0;
+  readonly messageDeltaOutputTokens: number[] = [];
+
+  protected override async createStreamWithRetry(
+    request: AnthropicStreamingMessageCreateParams,
+    options?: AnthropicRequestOptions
+  ) {
+    const stream = await super.createStreamWithRetry(request, options);
+    const recorder = this;
+
+    return {
+      controller: stream.controller,
+      async *[Symbol.asyncIterator](): AsyncGenerator<AnthropicMessageStreamEvent> {
+        for await (const event of stream) {
+          if (event.type === 'message_start') {
+            recorder.messageStartOutputTokens =
+              event.message.usage.output_tokens ??
+              recorder.messageStartOutputTokens;
+          } else if (event.type === 'message_delta') {
+            recorder.messageDeltaOutputTokens.push(event.usage.output_tokens);
+          }
+          yield event;
+        }
+      },
+    } as unknown as typeof stream;
+  }
+}
+
+test('Anthropic message_delta usage emits only output token totals', () => {
+  const event: AnthropicStreamEvent = {
+    type: 'message_delta',
+    context_management: null,
+    delta: {
+      container: null,
+      stop_details: null,
+      stop_reason: 'end_turn',
+      stop_sequence: null,
+    },
+    usage: {
+      input_tokens: 243,
+      output_tokens: 375,
+      cache_creation_input_tokens: 11,
+      cache_read_input_tokens: 13,
+      server_tool_use: null,
+      iterations: null,
+    },
+  };
+
+  const result = _makeMessageChunkFromAnthropicEvent(event, {
+    streamUsage: true,
+    coerceContentToString: true,
+  });
+
+  expect(result?.chunk.usage_metadata).toEqual({
+    input_tokens: 0,
+    output_tokens: 375,
+    total_tokens: 375,
+  });
+});
+
+test('Anthropic stream usage does not double-count cumulative input tokens', async () => {
+  const events: AnthropicStreamEvent[] = [
+    {
+      type: 'message_start',
+      message: {
+        id: 'msg_token_accounting',
+        container: null,
+        context_management: null,
+        content: [],
+        model: modelName,
+        role: 'assistant',
+        stop_details: null,
+        stop_reason: null,
+        stop_sequence: null,
+        type: 'message',
+        usage: {
+          cache_creation: null,
+          cache_creation_input_tokens: 0,
+          cache_read_input_tokens: 0,
+          inference_geo: null,
+          input_tokens: 243,
+          iterations: null,
+          output_tokens: 0,
+          server_tool_use: null,
+          service_tier: null,
+          speed: null,
+        },
+      },
+    },
+    {
+      type: 'message_delta',
+      context_management: null,
+      delta: {
+        container: null,
+        stop_details: null,
+        stop_reason: 'end_turn',
+        stop_sequence: null,
+      },
+      usage: {
+        input_tokens: 243,
+        output_tokens: 375,
+        cache_creation_input_tokens: 0,
+        cache_read_input_tokens: 0,
+        server_tool_use: null,
+        iterations: null,
+      },
+    },
+    { type: 'message_stop' },
+  ];
+  const model = new MockStreamingAnthropic(events);
+
+  let full: AIMessageChunk | undefined;
+  for await (const chunk of await model.stream('hello')) {
+    full = !full ? chunk : concat(full, chunk);
+  }
+
+  expect(full?.usage_metadata).toEqual({
+    input_tokens: 243,
+    output_tokens: 375,
+    total_tokens: 618,
+    input_token_details: {
+      cache_creation: 0,
+      cache_read: 0,
+    },
+    output_token_details: {},
+  });
+});
+
+test('Anthropic stream usage handles multiple cumulative message_delta events', async () => {
+  const events: AnthropicStreamEvent[] = [
+    {
+      type: 'message_start',
+      message: {
+        id: 'msg_token_accounting_multi_delta',
+        container: null,
+        context_management: null,
+        content: [],
+        model: modelName,
+        role: 'assistant',
+        stop_details: null,
+        stop_reason: null,
+        stop_sequence: null,
+        type: 'message',
+        usage: {
+          cache_creation: null,
+          cache_creation_input_tokens: 0,
+          cache_read_input_tokens: 0,
+          inference_geo: null,
+          input_tokens: 243,
+          iterations: null,
+          output_tokens: 0,
+          server_tool_use: null,
+          service_tier: null,
+          speed: null,
+        },
+      },
+    },
+    {
+      type: 'message_delta',
+      context_management: null,
+      delta: {
+        container: null,
+        stop_details: null,
+        stop_reason: null,
+        stop_sequence: null,
+      },
+      usage: {
+        input_tokens: 243,
+        output_tokens: 100,
+        cache_creation_input_tokens: 0,
+        cache_read_input_tokens: 0,
+        server_tool_use: null,
+        iterations: null,
+      },
+    },
+    {
+      type: 'message_delta',
+      context_management: null,
+      delta: {
+        container: null,
+        stop_details: null,
+        stop_reason: 'end_turn',
+        stop_sequence: null,
+      },
+      usage: {
+        input_tokens: 243,
+        output_tokens: 375,
+        cache_creation_input_tokens: 0,
+        cache_read_input_tokens: 0,
+        server_tool_use: null,
+        iterations: null,
+      },
+    },
+    { type: 'message_stop' },
+  ];
+  const model = new MockStreamingAnthropic(events);
+
+  let full: AIMessageChunk | undefined;
+  for await (const chunk of await model.stream('hello')) {
+    full = !full ? chunk : concat(full, chunk);
+  }
+
+  expect(full?.usage_metadata).toEqual({
+    input_tokens: 243,
+    output_tokens: 375,
+    total_tokens: 618,
+    input_token_details: {
+      cache_creation: 0,
+      cache_read: 0,
+    },
+    output_token_details: {},
+  });
+});
+
+test('Anthropic live stream usage matches raw cumulative output snapshots', async () => {
+  const model = new RecordingStreamingAnthropic({
+    modelName,
+    temperature: 0,
+    maxTokens: 500,
+    _lc_stream_delay: 0,
+  });
+
+  let full: AIMessageChunk | undefined;
+  const stream = await model.stream(
+    'Write exactly 18 numbered lines about reliable software telemetry. Each line should contain exactly seven words. Do not add an intro or outro.'
+  );
+  for await (const chunk of stream) {
+    full = !full ? chunk : concat(full, chunk);
+  }
+
+  expect(model.messageDeltaOutputTokens.length).toBeGreaterThan(0);
+  const rawOutputTokens =
+    model.messageDeltaOutputTokens[model.messageDeltaOutputTokens.length - 1];
+  expect(full?.usage_metadata?.output_tokens).toBe(
+    model.messageStartOutputTokens + rawOutputTokens
+  );
+  expect(full?.usage_metadata?.total_tokens).toBe(
+    (full?.usage_metadata?.input_tokens ?? 0) +
+      (full?.usage_metadata?.output_tokens ?? 0)
+  );
+
+  if (model.messageDeltaOutputTokens.length > 1) {
+    const summedOutputTokens = model.messageDeltaOutputTokens.reduce(
+      (sum, tokens) => sum + tokens,
+      0
+    );
+    expect(full?.usage_metadata?.output_tokens).toBeLessThan(
+      model.messageStartOutputTokens + summedOutputTokens
+    );
+  }
+});
+
 test('document detection ignores null content placeholders', () => {
   const params: AnthropicMessageCreateParams = {
     model: modelName,
@@ -1166,6 +1656,101 @@ test('human message caching', async () => {
   );
 });
 
+describe('Anthropic web search live regressions', () => {
+  test('accepts prompt-cache markers on built-in web search tools', async () => {
+    const model = new ChatAnthropic({
+      model: webSearchModelName,
+      maxTokens: 1024,
+      thinking: { type: 'adaptive' },
+    });
+    const tools = getPromptCachedWebSearchTools();
+    const formattedTools = model.formatStructuredToolToAnthropic(tools);
+
+    expect(formattedTools?.[0]).toMatchObject({
+      type: 'web_search_20250305',
+      name: 'web_search',
+      cache_control: { type: 'ephemeral' },
+    });
+    expect(formattedTools?.[0]).not.toHaveProperty('extras');
+
+    const response = await model
+      .bindTools(tools)
+      .invoke([
+        new HumanMessage(
+          'Use web search once and answer with only the word: ok'
+        ),
+      ]);
+
+    expect(response.content.length).toBeGreaterThan(0);
+  });
+
+  test('replays LibreChat-persisted web search content across runs', async () => {
+    const threadId = `web-search-e2e-${Date.now()}`;
+    const firstPrompt =
+      'Use web search. Who is the lowest seed survived in 2026 NBA playoffs? Answer with only the team name.';
+    const followUpPrompt = "Who are 76ers' opponents in current series?";
+    const { contentParts: firstContentParts, customHandlers: firstHandlers } =
+      createLibreChatContentHandlers();
+    const firstRun = await createWebSearchRun({
+      runId: `${threadId}-turn-1`,
+      customHandlers: firstHandlers,
+    });
+    const runConfig = {
+      configurable: { provider: Providers.ANTHROPIC, thread_id: threadId },
+      streamMode: 'values',
+      version: 'v2' as const,
+    };
+
+    const firstRunContent = await firstRun.processStream(
+      { messages: [new HumanMessage(firstPrompt)] },
+      runConfig
+    );
+    const persistedAssistantContent = firstContentParts.filter(
+      (part): part is LibreChatContentBlock => part != null
+    );
+    const hasPersistedServerToolCall = persistedAssistantContent.some(
+      (part) =>
+        part.type === ContentTypes.TOOL_CALL &&
+        typeof part.tool_call?.id === 'string' &&
+        part.tool_call.id.startsWith(Constants.ANTHROPIC_SERVER_TOOL_PREFIX)
+    );
+    const hasPersistedAnswerText = persistedAssistantContent.some(
+      (part) =>
+        part.type === ContentTypes.TEXT &&
+        typeof part.text === 'string' &&
+        part.text.trim().length > 0
+    );
+
+    expect(firstRunContent).toBeDefined();
+    expect(persistedAssistantContent.length).toBeGreaterThan(0);
+    expect(hasPersistedServerToolCall).toBe(true);
+    expect(hasPersistedAnswerText).toBe(true);
+
+    const persistedPayload: TPayload = [
+      { role: 'user', content: firstPrompt },
+      { role: 'assistant', content: persistedAssistantContent },
+      { role: 'user', content: followUpPrompt },
+    ];
+    const { messages } = formatAgentMessages(
+      persistedPayload,
+      undefined,
+      new Set(['web_search']),
+      undefined,
+      { provider: Providers.ANTHROPIC }
+    );
+    const anthropicPayload = _convertMessagesToAnthropicPayload(messages);
+    const secondRun = await createWebSearchRun({
+      runId: `${threadId}-turn-2`,
+    });
+
+    expectAnthropicPayloadContentIsNonEmpty(anthropicPayload);
+    expectNoDanglingServerToolUses(anthropicPayload);
+    await expect(
+      secondRun.processStream({ messages }, runConfig)
+    ).resolves.toBeDefined();
+  });
+});
+
 test('Can accept PDF documents', async () => {
   const model = new ChatAnthropic({
     modelName: pdfModelName,