@librechat/agents 2.4.322 → 3.0.0-rc10

package/src/llm/anthropic/index.ts

@@ -9,6 +9,7 @@ import type {
 } from '@langchain/core/messages';
 import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
 import type { AnthropicInput } from '@langchain/anthropic';
+import type { Anthropic } from '@anthropic-ai/sdk';
 import type {
   AnthropicMessageCreateParams,
   AnthropicStreamingMessageCreateParams,
@@ -18,6 +19,7 @@ import type {
 } from '@/llm/anthropic/types';
 import { _makeMessageChunkFromAnthropicEvent } from './utils/message_outputs';
 import { _convertMessagesToAnthropicPayload } from './utils/message_inputs';
+import { handleToolChoice } from './utils/tools';
 import { TextStream } from '@/llm/text';
 
 function _toolsInParams(
@@ -120,6 +122,13 @@ export type CustomAnthropicInput = AnthropicInput & {
   _lc_stream_delay?: number;
 } & BaseChatModelParams;
 
+/**
+ * A type representing additional parameters that can be passed to the
+ * Anthropic API.
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type Kwargs = Record<string, any>;
+
 export class CustomAnthropic extends ChatAnthropicMessages {
   _lc_stream_delay: number;
   private message_start: AnthropicMessageStartEvent | undefined;
@@ -128,9 +137,68 @@ export class CustomAnthropic extends ChatAnthropicMessages {
   private emitted_usage?: boolean;
   constructor(fields?: CustomAnthropicInput) {
     super(fields);
+    this.resetTokenEvents();
     this._lc_stream_delay = fields?._lc_stream_delay ?? 25;
   }
 
+  /**
+   * Get the parameters used to invoke the model
+   */
+  override invocationParams(
+    options?: this['ParsedCallOptions']
+  ): Omit<
+    AnthropicMessageCreateParams | AnthropicStreamingMessageCreateParams,
+    'messages'
+  > &
+    Kwargs {
+    const tool_choice:
+      | Anthropic.Messages.ToolChoiceAuto
+      | Anthropic.Messages.ToolChoiceAny
+      | Anthropic.Messages.ToolChoiceTool
+      | undefined = handleToolChoice(options?.tool_choice);
+
+    if (this.thinking.type === 'enabled') {
+      if (this.topK !== -1 && (this.topK as number | undefined) != null) {
+        throw new Error('topK is not supported when thinking is enabled');
+      }
+      if (this.topP !== -1 && (this.topP as number | undefined) != null) {
+        throw new Error('topP is not supported when thinking is enabled');
+      }
+      if (
+        this.temperature !== 1 &&
+        (this.temperature as number | undefined) != null
+      ) {
+        throw new Error(
+          'temperature is not supported when thinking is enabled'
+        );
+      }
+
+      return {
+        model: this.model,
+        stop_sequences: options?.stop ?? this.stopSequences,
+        stream: this.streaming,
+        max_tokens: this.maxTokens,
+        tools: this.formatStructuredToolToAnthropic(options?.tools),
+        tool_choice,
+        thinking: this.thinking,
+        ...this.invocationKwargs,
+      };
+    }
+    return {
+      model: this.model,
+      temperature: this.temperature,
+      top_k: this.topK,
+      top_p: this.topP,
+      stop_sequences: options?.stop ?? this.stopSequences,
+      stream: this.streaming,
+      max_tokens: this.maxTokens,
+      tools: this.formatStructuredToolToAnthropic(options?.tools),
+      tool_choice,
+      thinking: this.thinking,
+      ...this.invocationKwargs,
+    };
+  }
+
   /**
    * Get stream usage as returned by this client's API response.
    * @returns The stream usage object.
@@ -209,6 +277,7 @@ export class CustomAnthropic extends ChatAnthropicMessages {
     options: this['ParsedCallOptions'],
     runManager?: CallbackManagerForLLMRun
   ): AsyncGenerator<ChatGenerationChunk> {
+    this.resetTokenEvents();
     const params = this.invocationParams(options);
     const formattedMessages = _convertMessagesToAnthropicPayload(messages);
     const payload = {
@@ -221,16 +290,9 @@ export class CustomAnthropic extends ChatAnthropicMessages {
       !_documentsInParams(payload) &&
       !_thinkingInParams(payload);
 
-    const stream = await this.createStreamWithRetry(
-      {
-        ...params,
-        ...formattedMessages,
-        stream: true,
-      },
-      {
-        headers: options.headers,
-      }
-    );
+    const stream = await this.createStreamWithRetry(payload, {
+      headers: options.headers,
+    });
 
     const shouldStreamUsage = this.streamUsage ?? options.streamUsage;
 
@@ -263,7 +325,7 @@ export class CustomAnthropic extends ChatAnthropicMessages {
       if (
         !tokenType ||
         tokenType === 'input' ||
-        (token === '' && usageMetadata)
+        (token === '' && (usageMetadata != null || chunk.id != null))
       ) {
         const generationChunk = this.createGenerationChunk({
           token,
@@ -290,10 +352,13 @@ export class CustomAnthropic extends ChatAnthropicMessages {
         maxChunkSize: 8,
       });
 
-      const generator = textStream.generateText();
+      const generator = textStream.generateText(options.signal);
       try {
         let emittedUsage = false;
         for await (const currentToken of generator) {
+          if ((options.signal as AbortSignal | undefined)?.aborted === true) {
+            break;
+          }
           const newChunk = cloneChunk(currentToken, tokenType, chunk);
 
           const generationChunk = this.createGenerationChunk({
@@ -309,7 +374,7 @@ export class CustomAnthropic extends ChatAnthropicMessages {
           yield generationChunk;
 
           await runManager?.handleLLMNewToken(
-            token,
+            currentToken,
             undefined,
             undefined,
             undefined,