@librechat/agents 2.4.322 → 3.0.0-rc1

package/src/llm/anthropic/types.ts

@@ -1,6 +1,10 @@
 import Anthropic from '@anthropic-ai/sdk';
 import { BindToolsInput } from '@langchain/core/language_models/chat_models';
 
+export type AnthropicStreamUsage = Anthropic.Usage;
+export type AnthropicMessageDeltaEvent = Anthropic.MessageDeltaEvent;
+export type AnthropicMessageStartEvent = Anthropic.MessageStartEvent;
+
 export type AnthropicToolResponse = {
   type: 'tool_use';
   id: string;
@@ -9,10 +13,7 @@ export type AnthropicToolResponse = {
   input: Record<string, any>;
 };
 
-export type AnthropicStreamUsage = Anthropic.Usage;
 export type AnthropicMessageParam = Anthropic.MessageParam;
-export type AnthropicMessageDeltaEvent = Anthropic.MessageDeltaEvent;
-export type AnthropicMessageStartEvent = Anthropic.MessageStartEvent;
 export type AnthropicMessageResponse =
   | Anthropic.ContentBlock
   | AnthropicToolResponse;
@@ -42,6 +43,41 @@ export type AnthropicDocumentBlockParam = Anthropic.Messages.DocumentBlockParam;
 export type AnthropicThinkingBlockParam = Anthropic.Messages.ThinkingBlockParam;
 export type AnthropicRedactedThinkingBlockParam =
   Anthropic.Messages.RedactedThinkingBlockParam;
+export type AnthropicServerToolUseBlockParam =
+  Anthropic.Messages.ServerToolUseBlockParam;
+export type AnthropicWebSearchToolResultBlockParam =
+  Anthropic.Messages.WebSearchToolResultBlockParam;
+export type AnthropicWebSearchResultBlockParam =
+  Anthropic.Messages.WebSearchResultBlockParam;
+export type AnthropicSearchResultBlockParam =
+  Anthropic.Beta.BetaSearchResultBlockParam;
+
+// Union of all possible content block types including server tool use
+export type AnthropicContentBlock =
+  | AnthropicTextBlockParam
+  | AnthropicImageBlockParam
+  | AnthropicToolUseBlockParam
+  | AnthropicToolResultBlockParam
+  | AnthropicDocumentBlockParam
+  | AnthropicThinkingBlockParam
+  | AnthropicRedactedThinkingBlockParam
+  | AnthropicServerToolUseBlockParam
+  | AnthropicWebSearchToolResultBlockParam
+  | AnthropicWebSearchResultBlockParam;
+
+// Union of all possible content block types including server tool use
+export type ChatAnthropicContentBlock =
+  | AnthropicTextBlockParam
+  | AnthropicImageBlockParam
+  | AnthropicToolUseBlockParam
+  | AnthropicToolResultBlockParam
+  | AnthropicDocumentBlockParam
+  | AnthropicThinkingBlockParam
+  | AnthropicRedactedThinkingBlockParam
+  | AnthropicServerToolUseBlockParam
+  | AnthropicWebSearchToolResultBlockParam
+  | AnthropicWebSearchResultBlockParam
+  | AnthropicSearchResultBlockParam;
 
 export function isAnthropicImageBlockParam(
   block: unknown

package/src/llm/anthropic/utils/message_inputs.ts

@@ -9,7 +9,6 @@ import {
   HumanMessage,
   type AIMessage,
   type ToolMessage,
-  type MessageContent,
   isAIMessage,
   type StandardContentBlockConverter,
   type StandardTextBlock,
@@ -21,18 +20,21 @@ import {
   parseBase64DataUrl,
 } from '@langchain/core/messages';
 import { ToolCall } from '@langchain/core/messages/tool';
-import type {
+import {
   AnthropicImageBlockParam,
   AnthropicMessageCreateParams,
   AnthropicTextBlockParam,
-  AnthropicToolResponse,
   AnthropicToolResultBlockParam,
   AnthropicToolUseBlockParam,
   AnthropicDocumentBlockParam,
   AnthropicThinkingBlockParam,
   AnthropicRedactedThinkingBlockParam,
-} from '@/llm/anthropic/types';
-import { isAnthropicImageBlockParam } from '@/llm/anthropic/types';
+  AnthropicServerToolUseBlockParam,
+  AnthropicWebSearchToolResultBlockParam,
+  isAnthropicImageBlockParam,
+  AnthropicSearchResultBlockParam,
+  AnthropicToolResponse,
+} from '../types';
 
 function _formatImage(imageUrl: string) {
   const parsed = parseBase64DataUrl({ dataUrl: imageUrl });
@@ -118,7 +120,10 @@ function _ensureMessageContents(
             content: [
               {
                 type: 'tool_result',
-                content: _formatContent(message.content),
+                // rare case: message.content could be undefined
+                ...(message.content != null
+                  ? { content: _formatContent(message) }
+                  : {}),
                 tool_use_id: (message as ToolMessage).tool_call_id,
               },
             ],
@@ -342,9 +347,17 @@ const standardContentBlockConverter: StandardContentBlockConverter<{
   },
 };
 
-function _formatContent(content: MessageContent) {
-  const toolTypes = ['tool_use', 'tool_result', 'input_json_delta'];
+function _formatContent(message: BaseMessage) {
+  const toolTypes = [
+    'tool_use',
+    'tool_result',
+    'input_json_delta',
+    'server_tool_use',
+    'web_search_tool_result',
+    'web_search_result',
+  ];
   const textTypes = ['text', 'text_delta'];
+  const { content } = message;
 
   if (typeof content === 'string') {
     return content;
@@ -395,6 +408,20 @@ function _formatContent(content: MessageContent) {
           ...(cacheControl ? { cache_control: cacheControl } : {}),
         };
         return block;
+      } else if (contentPart.type === 'search_result') {
+        const block: AnthropicSearchResultBlockParam = {
+          type: 'search_result' as const, // Explicitly setting the type as "search_result"
+          title: contentPart.title,
+          source: contentPart.source,
+          ...('cache_control' in contentPart && contentPart.cache_control
+            ? { cache_control: contentPart.cache_control }
+            : {}),
+          ...('citations' in contentPart && contentPart.citations
+            ? { citations: contentPart.citations }
+            : {}),
+          content: contentPart.content,
+        };
+        return block;
       } else if (
         textTypes.find((t) => t === contentPart.type) &&
         'text' in contentPart
@@ -404,6 +431,9 @@ function _formatContent(content: MessageContent) {
           type: 'text' as const, // Explicitly setting the type as "text"
           text: contentPart.text,
           ...(cacheControl ? { cache_control: cacheControl } : {}),
+          ...('citations' in contentPart && contentPart.citations
+            ? { citations: contentPart.citations }
+            : {}),
         };
       } else if (toolTypes.find((t) => t === contentPart.type)) {
         const contentPartCopy = { ...contentPart };
@@ -435,6 +465,27 @@ function _formatContent(content: MessageContent) {
           ...(cacheControl ? { cache_control: cacheControl } : {}),
           // eslint-disable-next-line @typescript-eslint/no-explicit-any
         } as any;
+      } else if (
+        'functionCall' in contentPart &&
+        contentPart.functionCall &&
+        typeof contentPart.functionCall === 'object' &&
+        isAIMessage(message)
+      ) {
+        const correspondingToolCall = message.tool_calls?.find(
+          (toolCall) => toolCall.name === contentPart.functionCall.name
+        );
+        if (!correspondingToolCall) {
+          throw new Error(
+            `Could not find tool call for function call ${contentPart.functionCall.name}`
+          );
+        }
+        // Google GenAI models include a `functionCall` object inside content. We should ignore it as Anthropic will not support it.
+        return {
+          id: correspondingToolCall.id,
+          type: 'tool_use',
+          name: correspondingToolCall.name,
+          input: contentPart.functionCall.args,
+        };
       } else {
         throw new Error('Unsupported message content format');
       }
@@ -498,7 +549,8 @@ export function _convertMessagesToAnthropicPayload(
           content.find(
             (contentPart) =>
               (contentPart.type === 'tool_use' ||
-                contentPart.type === 'input_json_delta') &&
+                contentPart.type === 'input_json_delta' ||
+                contentPart.type === 'server_tool_use') &&
               contentPart.id === toolCall.id
           )
         );
@@ -509,13 +561,13 @@ export function _convertMessagesToAnthropicPayload(
         }
         return {
           role,
-          content: _formatContent(message.content),
+          content: _formatContent(message),
         };
       }
     } else {
       return {
         role,
-        content: _formatContent(message.content),
+        content: _formatContent(message),
       };
     }
   });
@@ -544,6 +596,8 @@ function mergeMessages(messages: AnthropicMessageCreateParams['messages']) {
           | AnthropicDocumentBlockParam
           | AnthropicThinkingBlockParam
           | AnthropicRedactedThinkingBlockParam
+          | AnthropicServerToolUseBlockParam
+          | AnthropicWebSearchToolResultBlockParam
         >
   ): Array<
     | AnthropicTextBlockParam
@@ -553,6 +607,8 @@ function mergeMessages(messages: AnthropicMessageCreateParams['messages']) {
     | AnthropicDocumentBlockParam
     | AnthropicThinkingBlockParam
     | AnthropicRedactedThinkingBlockParam
+    | AnthropicServerToolUseBlockParam
+    | AnthropicWebSearchToolResultBlockParam
   > => {
     if (typeof content === 'string') {
       return [

package/src/llm/anthropic/utils/message_outputs.ts

@@ -77,7 +77,12 @@ export function _makeMessageChunkFromAnthropicEvent(
     };
   } else if (
     data.type === 'content_block_start' &&
-    ['tool_use', 'document'].includes(data.content_block.type)
+    [
+      'tool_use',
+      'document',
+      'server_tool_use',
+      'web_search_tool_result',
+    ].includes(data.content_block.type)
   ) {
     const contentBlock = data.content_block;
     let toolCallChunks: ToolCallChunk[];
@@ -90,6 +95,16 @@ export function _makeMessageChunkFromAnthropicEvent(
           args: '',
         },
       ];
+    } else if (contentBlock.type === 'server_tool_use') {
+      // Handle anthropic built-in server tool use (like web search)
+      toolCallChunks = [
+        {
+          id: contentBlock.id,
+          index: data.index,
+          name: contentBlock.name,
+          args: '',
+        },
+      ];
     } else {
       toolCallChunks = [];
     }
@@ -101,7 +116,11 @@ export function _makeMessageChunkFromAnthropicEvent(
             {
               index: data.index,
               ...data.content_block,
-              input: '',
+              input:
+                  contentBlock.type === 'server_tool_use' ||
+                  contentBlock.type === 'tool_use'
+                    ? ''
+                    : undefined,
             },
           ],
         additional_kwargs: {},

package/src/llm/anthropic/utils/output_parsers.ts

@@ -1,17 +1,20 @@
 /* eslint-disable @typescript-eslint/explicit-function-return-type */
 /* eslint-disable @typescript-eslint/no-empty-object-type */
-import { z } from 'zod';
 import {
   BaseLLMOutputParser,
   OutputParserException,
 } from '@langchain/core/output_parsers';
-import { JsonOutputKeyToolsParserParams } from '@langchain/core/output_parsers/openai_tools';
+import { JsonOutputKeyToolsParserParamsInterop } from '@langchain/core/output_parsers/openai_tools';
 import { ChatGeneration } from '@langchain/core/outputs';
 import { ToolCall } from '@langchain/core/messages/tool';
+import {
+  interopSafeParseAsync,
+  InteropZodType,
+} from '@langchain/core/utils/types';
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 interface AnthropicToolsOutputParserParams<T extends Record<string, any>>
-  extends JsonOutputKeyToolsParserParams<T> {}
+  extends JsonOutputKeyToolsParserParamsInterop<T> {}
 
 export class AnthropicToolsOutputParser<
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -31,7 +34,7 @@ export class AnthropicToolsOutputParser<
   /** Whether to return only the first tool call. */
   returnSingle = false;
 
-  zodSchema?: z.ZodType<T>;
+  zodSchema?: InteropZodType<T>;
 
   constructor(params: AnthropicToolsOutputParserParams<T>) {
     super(params);
@@ -62,7 +65,10 @@ export class AnthropicToolsOutputParser<
     if (this.zodSchema === undefined) {
       return parsedResult as T;
     }
-    const zodParsedResult = await this.zodSchema.safeParseAsync(parsedResult);
+    const zodParsedResult = await interopSafeParseAsync(
+      this.zodSchema,
+      parsedResult
+    );
     if (zodParsedResult.success) {
       return zodParsedResult.data;
     } else {
@@ -71,7 +77,7 @@ export class AnthropicToolsOutputParser<
           result,
           null,
           2
-        )}". Error: ${JSON.stringify(zodParsedResult.error.errors)}`,
+        )}". Error: ${JSON.stringify(zodParsedResult.error.issues)}`,
         JSON.stringify(parsedResult, null, 2)
       );
     }
@@ -100,6 +106,7 @@ export class AnthropicToolsOutputParser<
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export function extractToolCalls(content: Record<string, any>[]) {
   const toolCalls: ToolCall[] = [];
+
   for (const block of content) {
     if (block.type === 'tool_use') {
       toolCalls.push({
@@ -108,7 +115,19 @@ export function extractToolCalls(content: Record<string, any>[]) {
         id: block.id,
         type: 'tool_call',
       });
+    } else if (
+      block.type === 'server_tool_use' &&
+      block.name === 'web_search'
+    ) {
+      // Handle Anthropic built-in web search tool
+      toolCalls.push({
+        name: block.name,
+        args: block.input,
+        id: block.id,
+        type: 'tool_call',
+      });
     }
   }
+
   return toolCalls;
 }

package/src/llm/anthropic/utils/tools.ts

@@ -0,0 +1,29 @@
+import type { Anthropic } from '@anthropic-ai/sdk';
+import { AnthropicToolChoice } from '../types.js';
+
+export function handleToolChoice(
+  toolChoice?: AnthropicToolChoice
+):
+  | Anthropic.Messages.ToolChoiceAuto
+  | Anthropic.Messages.ToolChoiceAny
+  | Anthropic.Messages.ToolChoiceTool
+  | undefined {
+  if (toolChoice == null) {
+    return undefined;
+  } else if (toolChoice === 'any') {
+    return {
+      type: 'any',
+    };
+  } else if (toolChoice === 'auto') {
+    return {
+      type: 'auto',
+    };
+  } else if (typeof toolChoice === 'string') {
+    return {
+      type: 'tool',
+      name: toolChoice,
+    };
+  } else {
+    return toolChoice;
+  }
+}

package/src/llm/google/index.ts

@@ -0,0 +1,218 @@
+/* eslint-disable @typescript-eslint/ban-ts-comment */
+import { AIMessageChunk } from '@langchain/core/messages';
+import { ChatGenerationChunk } from '@langchain/core/outputs';
+import { ChatGoogleGenerativeAI } from '@langchain/google-genai';
+import { getEnvironmentVariable } from '@langchain/core/utils/env';
+import { GoogleGenerativeAI as GenerativeAI } from '@google/generative-ai';
+import type {
+  GenerateContentRequest,
+  SafetySetting,
+} from '@google/generative-ai';
+import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
+import type { BaseMessage, UsageMetadata } from '@langchain/core/messages';
+import type { GeminiGenerationConfig } from '@langchain/google-common';
+import type { GeminiApiUsageMetadata } from './types';
+import type { GoogleClientOptions } from '@/types';
+import {
+  convertResponseContentToChatGenerationChunk,
+  convertBaseMessagesToContent,
+} from './utils/common';
+
+export class CustomChatGoogleGenerativeAI extends ChatGoogleGenerativeAI {
+  thinkingConfig?: GeminiGenerationConfig['thinkingConfig'];
+  constructor(fields: GoogleClientOptions) {
+    super(fields);
+
+    this.model = fields.model.replace(/^models\//, '');
+
+    this.maxOutputTokens = fields.maxOutputTokens ?? this.maxOutputTokens;
+
+    if (this.maxOutputTokens != null && this.maxOutputTokens < 0) {
+      throw new Error('`maxOutputTokens` must be a positive integer');
+    }
+
+    this.temperature = fields.temperature ?? this.temperature;
+    if (
+      this.temperature != null &&
+      (this.temperature < 0 || this.temperature > 2)
+    ) {
+      throw new Error('`temperature` must be in the range of [0.0,2.0]');
+    }
+
+    this.topP = fields.topP ?? this.topP;
+    if (this.topP != null && this.topP < 0) {
+      throw new Error('`topP` must be a positive integer');
+    }
+
+    if (this.topP != null && this.topP > 1) {
+      throw new Error('`topP` must be below 1.');
+    }
+
+    this.topK = fields.topK ?? this.topK;
+    if (this.topK != null && this.topK < 0) {
+      throw new Error('`topK` must be a positive integer');
+    }
+
+    this.stopSequences = fields.stopSequences ?? this.stopSequences;
+
+    this.apiKey = fields.apiKey ?? getEnvironmentVariable('GOOGLE_API_KEY');
+    if (this.apiKey == null || this.apiKey === '') {
+      throw new Error(
+        'Please set an API key for Google GenerativeAI ' +
+          'in the environment variable GOOGLE_API_KEY ' +
+          'or in the `apiKey` field of the ' +
+          'ChatGoogleGenerativeAI constructor'
+      );
+    }
+
+    this.safetySettings = fields.safetySettings ?? this.safetySettings;
+    if (this.safetySettings && this.safetySettings.length > 0) {
+      const safetySettingsSet = new Set(
+        this.safetySettings.map((s) => s.category)
+      );
+      if (safetySettingsSet.size !== this.safetySettings.length) {
+        throw new Error(
+          'The categories in `safetySettings` array must be unique'
+        );
+      }
+    }
+
+    this.thinkingConfig = fields.thinkingConfig ?? this.thinkingConfig;
+
+    this.streaming = fields.streaming ?? this.streaming;
+    this.json = fields.json;
+
+    // @ts-ignore - Accessing private property from parent class
+    this.client = new GenerativeAI(this.apiKey).getGenerativeModel(
+      {
+        model: this.model,
+        safetySettings: this.safetySettings as SafetySetting[],
+        generationConfig: {
+          stopSequences: this.stopSequences,
+          maxOutputTokens: this.maxOutputTokens,
+          temperature: this.temperature,
+          topP: this.topP,
+          topK: this.topK,
+          ...(this.json != null
+            ? { responseMimeType: 'application/json' }
+            : {}),
+        },
+      },
+      {
+        apiVersion: fields.apiVersion,
+        baseUrl: fields.baseUrl,
+        customHeaders: fields.customHeaders,
+      }
+    );
+    this.streamUsage = fields.streamUsage ?? this.streamUsage;
+  }
+
+  invocationParams(
+    options?: this['ParsedCallOptions']
+  ): Omit<GenerateContentRequest, 'contents'> {
+    const params = super.invocationParams(options);
+    if (this.thinkingConfig) {
+      /** @ts-ignore */
+      this.client.generationConfig = {
+        /** @ts-ignore */
+        ...this.client.generationConfig,
+        /** @ts-ignore */
+        thinkingConfig: this.thinkingConfig,
+      };
+    }
+    return params;
+  }
+
+  async *_streamResponseChunks(
+    messages: BaseMessage[],
+    options: this['ParsedCallOptions'],
+    runManager?: CallbackManagerForLLMRun
+  ): AsyncGenerator<ChatGenerationChunk> {
+    const prompt = convertBaseMessagesToContent(
+      messages,
+      this._isMultimodalModel,
+      this.useSystemInstruction
+    );
+    let actualPrompt = prompt;
+    if (prompt?.[0].role === 'system') {
+      const [systemInstruction] = prompt;
+      /** @ts-ignore */
+      this.client.systemInstruction = systemInstruction;
+      actualPrompt = prompt.slice(1);
+    }
+    const parameters = this.invocationParams(options);
+    const request = {
+      ...parameters,
+      contents: actualPrompt,
+    };
+    const stream = await this.caller.callWithOptions(
+      { signal: options.signal },
+      async () => {
+        /** @ts-ignore */
+        const { stream } = await this.client.generateContentStream(request);
+        return stream;
+      }
+    );
+
+    let index = 0;
+    let lastUsageMetadata: UsageMetadata | undefined;
+    for await (const response of stream) {
+      if (
+        'usageMetadata' in response &&
+        this.streamUsage !== false &&
+        options.streamUsage !== false
+      ) {
+        const genAIUsageMetadata = response.usageMetadata as
+          | GeminiApiUsageMetadata
+          | undefined;
+
+        const output_tokens =
+          (genAIUsageMetadata?.candidatesTokenCount ?? 0) +
+          (genAIUsageMetadata?.thoughtsTokenCount ?? 0);
+        lastUsageMetadata = {
+          input_tokens: genAIUsageMetadata?.promptTokenCount ?? 0,
+          output_tokens,
+          total_tokens: genAIUsageMetadata?.totalTokenCount ?? 0,
+        };
+      }
+
+      const chunk = convertResponseContentToChatGenerationChunk(response, {
+        usageMetadata: undefined,
+        index,
+      });
+      index += 1;
+      if (!chunk) {
+        continue;
+      }
+
+      yield chunk;
+      await runManager?.handleLLMNewToken(
+        chunk.text || '',
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        { chunk }
+      );
+    }
+
+    if (lastUsageMetadata) {
+      const finalChunk = new ChatGenerationChunk({
+        text: '',
+        message: new AIMessageChunk({
+          content: '',
+          usage_metadata: lastUsageMetadata,
+        }),
+      });
+      yield finalChunk;
+      await runManager?.handleLLMNewToken(
+        finalChunk.text || '',
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        { chunk: finalChunk }
+      );
+    }
+  }
+}

package/src/llm/google/types.ts

@@ -0,0 +1,43 @@
+import {
+  CodeExecutionTool,
+  FunctionDeclarationsTool as GoogleGenerativeAIFunctionDeclarationsTool,
+  GoogleSearchRetrievalTool,
+} from '@google/generative-ai';
+import { BindToolsInput } from '@langchain/core/language_models/chat_models';
+
+export type GoogleGenerativeAIToolType =
+  | BindToolsInput
+  | GoogleGenerativeAIFunctionDeclarationsTool
+  | CodeExecutionTool
+  | GoogleSearchRetrievalTool;
+
+/** Enum for content modality types */
+enum Modality {
+  MODALITY_UNSPECIFIED = 'MODALITY_UNSPECIFIED',
+  TEXT = 'TEXT',
+  IMAGE = 'IMAGE',
+  VIDEO = 'VIDEO',
+  AUDIO = 'AUDIO',
+  DOCUMENT = 'DOCUMENT',
+}
+
+/** Interface for modality token count */
+interface ModalityTokenCount {
+  modality: Modality;
+  tokenCount: number;
+}
+
+/** Main interface for Gemini API usage metadata */
+export interface GeminiApiUsageMetadata {
+  promptTokenCount?: number;
+  totalTokenCount?: number;
+  thoughtsTokenCount?: number;
+  candidatesTokenCount?: number;
+  toolUsePromptTokenCount?: number;
+  cachedContentTokenCount?: number;
+  promptTokensDetails: ModalityTokenCount[];
+  candidatesTokensDetails?: ModalityTokenCount[];
+  cacheTokensDetails?: ModalityTokenCount[];
+  toolUsePromptTokensDetails?: ModalityTokenCount[];
+  trafficType?: string;
+}