@librechat/agents 2.1.3 → 2.1.4

package/src/llm/anthropic/llm.ts

@@ -1,17 +1,48 @@
 import { AIMessageChunk } from '@langchain/core/messages';
 import { ChatAnthropicMessages } from '@langchain/anthropic';
 import { ChatGenerationChunk } from '@langchain/core/outputs';
+import type { BaseChatModelParams } from '@langchain/core/language_models/chat_models';
 import type { BaseMessage, MessageContentComplex } from '@langchain/core/messages';
 import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
 import type { AnthropicInput } from '@langchain/anthropic';
-import type { AnthropicMessageCreateParams, AnthropicStreamUsage, AnthropicMessageStartEvent, AnthropicMessageDeltaEvent } from '@/llm/anthropic/types';
+import type { AnthropicMessageCreateParams, AnthropicStreamingMessageCreateParams, AnthropicStreamUsage, AnthropicMessageStartEvent, AnthropicMessageDeltaEvent } from '@/llm/anthropic/types';
 import { _makeMessageChunkFromAnthropicEvent } from './utils/message_outputs';
 import { _convertMessagesToAnthropicPayload } from './utils/message_inputs';
 import { TextStream } from '@/llm/text';
 
-function _toolsInParams(params: AnthropicMessageCreateParams): boolean {
+function _toolsInParams(
+  params: AnthropicMessageCreateParams | AnthropicStreamingMessageCreateParams
+): boolean {
   return !!(params.tools && params.tools.length > 0);
 }
+function _documentsInParams(
+  params: AnthropicMessageCreateParams | AnthropicStreamingMessageCreateParams
+): boolean {
+  for (const message of params.messages ?? []) {
+    if (typeof message.content === "string") {
+      continue;
+    }
+    for (const block of message.content ?? []) {
+      if (
+        typeof block === "object" &&
+        block != null &&
+        block.type === "document" &&
+        typeof block.citations === "object" &&
+        block.citations.enabled
+      ) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+function _thinkingInParams(
+  params: AnthropicMessageCreateParams | AnthropicStreamingMessageCreateParams
+): boolean {
+  return !!(params.thinking && params.thinking.type === "enabled");
+}
+
 
 function extractToken(chunk: AIMessageChunk): [string, 'string' | 'input' | 'content'] | [undefined] {
   if (typeof chunk.content === 'string') {
@@ -30,6 +61,12 @@ function extractToken(chunk: AIMessageChunk): [string, 'string' | 'input' | 'con
     'text' in chunk.content[0]
   ) {
     return [chunk.content[0].text, 'content'];
+  } else if (
+    Array.isArray(chunk.content) &&
+    chunk.content.length >= 1 &&
+    'thinking' in chunk.content[0]
+  ) {
+    return [chunk.content[0].thinking, 'content'];
   }
   return [undefined];
 }
@@ -45,12 +82,14 @@ function cloneChunk(text: string, tokenType: string, chunk: AIMessageChunk): AIM
     return new AIMessageChunk(Object.assign({}, chunk, { content: [Object.assign({}, content, { text })] }));
   } else if (tokenType === 'content' && content.type === 'text_delta') {
     return new AIMessageChunk(Object.assign({}, chunk, { content: [Object.assign({}, content, { text })] }));
+  } else if (tokenType === 'content' && content.type?.startsWith('thinking')) {
+    return new AIMessageChunk(Object.assign({}, chunk, { content: [Object.assign({}, content, { thinking: text })] }));
   }
 
   return chunk;
 }
 
-export type CustomAnthropicInput = AnthropicInput & { _lc_stream_delay?: number };
+export type CustomAnthropicInput = AnthropicInput & { _lc_stream_delay?: number } & BaseChatModelParams;
 
 export class CustomAnthropic extends ChatAnthropicMessages {
   _lc_stream_delay: number;
@@ -58,9 +97,9 @@ export class CustomAnthropic extends ChatAnthropicMessages {
   private message_delta: AnthropicMessageDeltaEvent | undefined;
   private tools_in_params?: boolean;
   private emitted_usage?: boolean;
-  constructor(fields: CustomAnthropicInput) {
+  constructor(fields?: CustomAnthropicInput) {
     super(fields);
-    this._lc_stream_delay = fields._lc_stream_delay ?? 25;
+    this._lc_stream_delay = fields?._lc_stream_delay ?? 25;
   }
 
   /**
@@ -76,19 +115,21 @@ export class CustomAnthropic extends ChatAnthropicMessages {
     if (!outputUsage) {
       return;
     }
-    const totalUsage = {
-      total_tokens: (inputUsage?.input_tokens ?? 0)
-      + (inputUsage?.output_tokens ?? 0)
-      + (inputUsage?.cache_creation_input_tokens ?? 0)
-      + (inputUsage?.cache_read_input_tokens ?? 0)
-      + (outputUsage.input_tokens ?? 0)
-      + (outputUsage.output_tokens ?? 0)
-      + (outputUsage.cache_creation_input_tokens ?? 0)
-      + (outputUsage.cache_read_input_tokens ?? 0),
+    const totalUsage: AnthropicStreamUsage = {
+      input_tokens: inputUsage?.input_tokens ?? 0,
+      output_tokens: outputUsage?.output_tokens ?? 0,
+      total_tokens: (inputUsage?.input_tokens ?? 0) + (outputUsage?.output_tokens ?? 0),
     };
 
+    if (inputUsage?.cache_creation_input_tokens != null || inputUsage?.cache_read_input_tokens != null) {
+      totalUsage.input_token_details = {
+        cache_creation: inputUsage.cache_creation_input_tokens ?? 0,
+        cache_read: inputUsage.cache_read_input_tokens ?? 0,
+      };
+    }
+
     this.emitted_usage = true;
-    return Object.assign(totalUsage, inputUsage, outputUsage);
+    return totalUsage;
   }
 
   resetTokenEvents(): void {
@@ -131,12 +172,15 @@ export class CustomAnthropic extends ChatAnthropicMessages {
   ): AsyncGenerator<ChatGenerationChunk> {
     const params = this.invocationParams(options);
     const formattedMessages = _convertMessagesToAnthropicPayload(messages);
-    this.tools_in_params = _toolsInParams({
+    const payload = {
       ...params,
       ...formattedMessages,
-      stream: false,
-    });
-    const coerceContentToString = !this.tools_in_params;
+      stream: true,
+    } as const;
+    const coerceContentToString =
+      !_toolsInParams(payload) &&
+      !_documentsInParams(payload) &&
+      !_thinkingInParams(payload);
 
     const stream = await this.createStreamWithRetry(
       {
@@ -157,10 +201,9 @@ export class CustomAnthropic extends ChatAnthropicMessages {
         throw new Error('AbortError: User aborted the request.');
       }
 
-      const type = data.type ?? '';
-      if (type === 'message_start') {
+      if (data.type === 'message_start') {
         this.message_start = data as AnthropicMessageStartEvent;
-      } else if (type === 'message_delta') {
+      } else if (data.type === 'message_delta') {
         this.message_delta = data as AnthropicMessageDeltaEvent;
       }
 

package/src/llm/anthropic/types.ts

@@ -57,12 +57,10 @@ export interface AnthropicStreamUsage {
    * The number of cache creation input tokens used (write operations)
    */
   cache_creation_input_tokens?: number;
-
   /**
    * The number of cache input tokens used (read operations)
    */
   cache_read_input_tokens?: number;
-
   /**
    * The number of output tokens generated in the response
    */
@@ -71,4 +69,11 @@ export interface AnthropicStreamUsage {
    * The total number of tokens generated in the response
    */
   total_tokens: number;
+  /**
+   * Details about input token usage
+   */
+  input_token_details?: {
+    cache_creation: number;
+    cache_read: number;
+  };
 }

package/src/llm/anthropic/utils/message_outputs.ts

@@ -47,16 +47,28 @@ export function _makeMessageChunkFromAnthropicEvent(
         filteredAdditionalKwargs[key] = value;
       }
     }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const { input_tokens, output_tokens, ...rest }: Record<string, any> =
+      usage ?? {};
     const usageMetadata: UsageMetadata = {
-      input_tokens: usage.input_tokens,
-      output_tokens: usage.output_tokens,
-      total_tokens: usage.input_tokens + usage.output_tokens,
+      input_tokens,
+      output_tokens,
+      total_tokens: input_tokens + output_tokens,
+      input_token_details: {
+        cache_creation: rest.cache_creation_input_tokens,
+        cache_read: rest.cache_read_input_tokens,
+      },
     };
     return {
       chunk: new AIMessageChunk({
         content: fields.coerceContentToString ? '' : [],
         additional_kwargs: filteredAdditionalKwargs,
         usage_metadata: fields.streamUsage ? usageMetadata : undefined,
+        response_metadata: {
+          usage: {
+            ...rest,
+          },
+        },
         id: data.message.id,
       }),
     };
@@ -65,6 +77,12 @@ export function _makeMessageChunkFromAnthropicEvent(
       input_tokens: 0,
       output_tokens: data.usage.output_tokens,
       total_tokens: data.usage.output_tokens,
+      input_token_details: {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        cache_creation: (data.usage as any).cache_creation_input_tokens,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        cache_read: (data.usage as any).cache_read_input_tokens,
+      },
     };
     return {
       chunk: new AIMessageChunk({
@@ -189,6 +207,29 @@ export function _makeMessageChunkFromAnthropicEvent(
         }),
       };
     }
+  } else if (
+    data.type === "content_block_start" &&
+    data.content_block.type === "redacted_thinking"
+  ) {
+    return {
+      chunk: new AIMessageChunk({
+        content: fields.coerceContentToString
+          ? ""
+          : [{ index: data.index, ...data.content_block }],
+      }),
+    };
+  } else if (
+    data.type === "content_block_start" &&
+    data.content_block.type === "thinking"
+  ) {
+    const content = data.content_block.thinking;
+    return {
+      chunk: new AIMessageChunk({
+        content: fields.coerceContentToString
+          ? content
+          : [{ index: data.index, ...data.content_block }],
+      }),
+    };
   }
 
   return null;
@@ -200,13 +241,17 @@ export function anthropicResponseToChatMessages(
 ): ChatGeneration[] {
   const usage: Record<string, number> | null | undefined =
     additionalKwargs.usage as Record<string, number> | null | undefined;
-  const usageMetadata =
+    const usageMetadata =
     usage != null
       ? {
-        input_tokens: usage.input_tokens ?? 0,
-        output_tokens: usage.output_tokens ?? 0,
-        total_tokens: (usage.input_tokens ?? 0) + (usage.output_tokens ?? 0),
-      }
+          input_tokens: usage.input_tokens ?? 0,
+          output_tokens: usage.output_tokens ?? 0,
+          total_tokens: (usage.input_tokens ?? 0) + (usage.output_tokens ?? 0),
+          input_token_details: {
+            cache_creation: usage.cache_creation_input_tokens,
+            cache_read: usage.cache_read_input_tokens,
+          },
+        }
       : undefined;
   if (messages.length === 1 && messages[0].type === 'text') {
     return [

package/src/scripts/thinking.ts

@@ -0,0 +1,152 @@
+// src/scripts/test-thinking.ts
+import { config } from 'dotenv';
+config();
+import { HumanMessage, SystemMessage, BaseMessage } from '@langchain/core/messages';
+import type { UsageMetadata } from '@langchain/core/messages';
+import * as t from '@/types';
+import { ChatModelStreamHandler, createContentAggregator } from '@/stream';
+import { ToolEndHandler, ModelEndHandler } from '@/events';
+import { GraphEvents, Providers } from '@/common';
+import { getLLMConfig } from '@/utils/llmConfig';
+import { getArgs } from '@/scripts/args';
+import { Run } from '@/run';
+
+const conversationHistory: BaseMessage[] = [];
+let _contentParts: t.MessageContentComplex[] = [];
+const collectedUsage: UsageMetadata[] = [];
+
+async function testThinking(): Promise<void> {
+  const { userName } = await getArgs();
+  const instructions = `You are a helpful AI assistant for ${userName}. When answering questions, be thorough in your reasoning.`;
+  const { contentParts, aggregateContent } = createContentAggregator();
+  _contentParts = contentParts as t.MessageContentComplex[];
+  
+  // Set up event handlers
+  const customHandlers = {
+    [GraphEvents.TOOL_END]: new ToolEndHandler(),
+    [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(collectedUsage),
+    [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(),
+    [GraphEvents.ON_RUN_STEP_COMPLETED]: {
+      handle: (event: GraphEvents.ON_RUN_STEP_COMPLETED, data: t.StreamEventData): void => {
+        console.log('====== ON_RUN_STEP_COMPLETED ======');
+        aggregateContent({ event, data: data as unknown as { result: t.ToolEndEvent } });
+      }
+    },
+    [GraphEvents.ON_RUN_STEP]: {
+      handle: (event: GraphEvents.ON_RUN_STEP, data: t.RunStep) => {
+        aggregateContent({ event, data });
+      },
+    },
+    [GraphEvents.ON_RUN_STEP_DELTA]: {
+      handle: (event: GraphEvents.ON_RUN_STEP_DELTA, data: t.RunStepDeltaEvent) => {
+        aggregateContent({ event, data });
+      },
+    },
+    [GraphEvents.ON_MESSAGE_DELTA]: {
+      handle: (event: GraphEvents.ON_MESSAGE_DELTA, data: t.MessageDeltaEvent) => {
+        aggregateContent({ event, data });
+      },
+    },
+    [GraphEvents.ON_REASONING_DELTA]: {
+      handle: (event: GraphEvents.ON_REASONING_DELTA, data: t.ReasoningDeltaEvent) => {
+        aggregateContent({ event, data });
+      },
+    },
+  };
+
+  const baseLlmConfig: t.LLMConfig = getLLMConfig(Providers.ANTHROPIC);
+  
+  if (baseLlmConfig.provider !== 'anthropic') {
+    console.error('This test requires Anthropic as the LLM provider. Please specify provider=anthropic');
+    process.exit(1);
+  }
+  
+  // Enable thinking with token budget
+  const llmConfig = {
+    ...baseLlmConfig,
+    model: 'claude-3-7-sonnet-latest',
+    thinking: { type: "enabled", budget_tokens: 2000 }
+  };
+
+  const run = await Run.create<t.IState>({
+    runId: 'test-thinking-id',
+    graphConfig: {
+      instructions,
+      type: 'standard',
+      llmConfig,
+    },
+    returnContent: true,
+    customHandlers: customHandlers as t.RunConfig['customHandlers'],
+  });
+
+  const config = {
+    configurable: {
+      thread_id: 'thinking-test-thread',
+    },
+    streamMode: 'values',
+    version: 'v2' as const,
+  };
+
+  // Test 1: Regular thinking mode
+  console.log('\n\nTest 1: Regular thinking mode');
+  const userMessage1 = `What would be the environmental and economic impacts if all cars globally were replaced by electric vehicles overnight?`;
+  conversationHistory.push(new HumanMessage(userMessage1));
+
+  console.log('Running first query with thinking enabled...');
+  const firstInputs = { messages: [...conversationHistory] };
+  await run.processStream(firstInputs, config);
+  
+  // Extract and display thinking blocks
+  const finalMessages = run.getRunMessages();
+
+  // Test 2: Try multi-turn conversation
+  console.log('\n\nTest 2: Multi-turn conversation with thinking enabled');
+  const userMessage2 = `Given your previous analysis, what would be the most significant technical challenges in making this transition?`;
+  conversationHistory.push(new HumanMessage(userMessage2));
+  
+  console.log('Running second query with thinking enabled...');
+  const secondInputs = { messages: [...conversationHistory] };
+  await run.processStream(secondInputs, config);
+  
+  // Display thinking blocks for second response
+  const finalMessages2 = run.getRunMessages();
+
+  // Test 3: Redacted thinking mode
+  console.log('\n\nTest 3: Redacted thinking mode');
+  const magicString = "ANTHROPIC_MAGIC_STRING_TRIGGER_REDACTED_THINKING_46C9A13E193C177646C7398A98432ECCCE4C1253D5E2D82641AC0E52CC2876CB";
+  const userMessage3 = `${magicString}\n\nExplain how quantum computing works in simple terms.`;
+  
+  // Reset conversation for clean test
+  conversationHistory.length = 0;
+  conversationHistory.push(new HumanMessage(userMessage3));
+  
+  console.log('Running query with redacted thinking...');
+  const thirdInputs = { messages: [...conversationHistory] };
+  await run.processStream(thirdInputs, config);
+  
+  // Display redacted thinking blocks
+  const finalMessages3 = run.getRunMessages();
+  console.log('\n\nThinking feature test completed!');
+}
+
+process.on('unhandledRejection', (reason, promise) => {
+  console.error('Unhandled Rejection at:', promise, 'reason:', reason);
+  console.log('Conversation history:');
+  console.dir(conversationHistory, { depth: null });
+  console.log('Content parts:');
+  console.dir(_contentParts, { depth: null });
+  process.exit(1);
+});
+
+process.on('uncaughtException', (err) => {
+  console.error('Uncaught Exception:', err);
+});
+
+testThinking().catch((err) => {
+  console.error(err);
+  console.log('Conversation history:');
+  console.dir(conversationHistory, { depth: null });
+  console.log('Content parts:');
+  console.dir(_contentParts, { depth: null });
+  process.exit(1);
+});

package/src/stream.ts

@@ -213,6 +213,12 @@ hasToolCallChunks: ${hasToolCallChunks}
       graph.dispatchMessageDelta(stepId, {
         content,
       });
+    } else if (content.every((c) => c.type?.startsWith(ContentTypes.THINKING))) {
+      graph.dispatchReasoningDelta(stepId, {
+        content: content.map((c) => ({
+          type: ContentTypes.THINK,
+          think: (c as t.ThinkingContentText).thinking,
+      }))});
     }
   }
   handleToolCallChunks = ({
@@ -271,8 +277,11 @@ hasToolCallChunks: ${hasToolCallChunks}
     });
   };
   handleReasoning(chunk: Partial<AIMessageChunk>, graph: Graph): void {
-    const reasoning_content = chunk.additional_kwargs?.[graph.reasoningKey] as string | undefined;
-    if (reasoning_content != null && reasoning_content && (chunk.content == null || chunk.content === '')) {
+    let reasoning_content = chunk.additional_kwargs?.[graph.reasoningKey] as string | undefined;
+    if (Array.isArray(chunk.content) && chunk.content[0]?.type === 'thinking') {
+      reasoning_content = 'valid';
+    }
+    if (reasoning_content != null && reasoning_content && (chunk.content == null || chunk.content === '' || reasoning_content === 'valid')) {
       graph.currentTokenType = ContentTypes.THINK;
       graph.tokenTypeSwitch = 'reasoning';
       return;

package/src/types/stream.ts

@@ -216,9 +216,16 @@ export type ReasoningContentText = {
   think: string;
 };
 
+/** Anthropic's Reasoning Content Block Format */
+export type ThinkingContentText = {
+  type: ContentTypes.THINKING;
+  index?: number;
+  thinking: string;
+};
+
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-export type MessageContentComplex = (ReasoningContentText | MessageContentText | MessageContentImageUrl | (Record<string, any> & {
-  type?: 'text' | 'image_url' | 'think' | string;
+export type MessageContentComplex = (ThinkingContentText | ReasoningContentText | MessageContentText | MessageContentImageUrl | (Record<string, any> & {
+  type?: 'text' | 'image_url' | 'think' | 'thinking' | string;
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 }) | (Record<string, any> & {
   type?: never;