@librechat/agents 3.2.21 → 3.2.32

package/src/llm/google/llm.spec.ts

@@ -29,7 +29,12 @@ import type {
 } from '@google/generative-ai';
 import type { ContentBlock } from '@langchain/core/messages';
 import { CustomChatGoogleGenerativeAI as ChatGoogleGenerativeAI } from './index';
-import { _FUNCTION_CALL_THOUGHT_SIGNATURES_MAP_KEY } from './utils/common';
+import {
+  convertMessageContentToParts,
+  convertResponseContentToChatGenerationChunk,
+  mapGenerateContentResultToChatResult,
+  _FUNCTION_CALL_THOUGHT_SIGNATURES_MAP_KEY,
+} from './utils/common';
 
 // Save the original value of the 'LANGCHAIN_CALLBACKS_BACKGROUND' environment variable
 const originalBackground = process.env.LANGCHAIN_CALLBACKS_BACKGROUND;
@@ -731,6 +736,173 @@ test('Invoke with JSON mode', async () => {
   );
 });
 
+test('Includes server-side tool invocation config when enabled', () => {
+  const model = new ChatGoogleGenerativeAI({
+    apiKey: 'test-key',
+    model: 'gemini-3.5-flash',
+    includeServerSideToolInvocations: true,
+  });
+  const getWeather = {
+    name: 'get_weather',
+    description: 'Get the weather',
+  };
+
+  const request = model.invocationParams({
+    tools: [{ googleSearch: {} }, { functionDeclarations: [getWeather] }],
+  });
+
+  expect(request.toolConfig).toEqual(
+    expect.objectContaining({
+      includeServerSideToolInvocations: true,
+    })
+  );
+});
+
+test('Normalizes auto tool choice when server-side tool invocation config is enabled', () => {
+  const model = new ChatGoogleGenerativeAI({
+    apiKey: 'test-key',
+    model: 'gemini-3.5-flash',
+    includeServerSideToolInvocations: true,
+  });
+  const getWeather = {
+    name: 'get_weather',
+    description: 'Get the weather',
+  };
+
+  const request = model.invocationParams({
+    tools: [{ googleSearch: {} }, { functionDeclarations: [getWeather] }],
+    tool_choice: 'auto',
+  });
+  const toolConfig = request.toolConfig as
+    | {
+        includeServerSideToolInvocations?: boolean;
+        functionCallingConfig?: { mode?: string };
+      }
+    | undefined;
+
+  expect(toolConfig).toEqual(
+    expect.objectContaining({
+      includeServerSideToolInvocations: true,
+    })
+  );
+  expect(toolConfig?.functionCallingConfig?.mode).toBe('VALIDATED');
+});
+
+test('Preserves Gemini server-side tool context parts in history', () => {
+  const toolCallPart = {
+    type: 'toolCall',
+    agentId: 'agent_1',
+    groupId: 1,
+    thoughtSignature: 'signature-1',
+    toolCall: {
+      id: 'server-search-1',
+      name: 'google_search',
+      args: {},
+    },
+  } as const;
+  const toolResponsePart = {
+    type: 'toolResponse',
+    agentId: 'agent_1',
+    groupId: 1,
+    toolResponse: {
+      id: 'server-search-1',
+      name: 'google_search',
+      response: { results: [] },
+    },
+  } as const;
+  const message = new AIMessage({
+    content: [toolCallPart, toolResponsePart],
+  });
+
+  expect(convertMessageContentToParts(message, true, [])).toEqual([
+    {
+      toolCall: toolCallPart.toolCall,
+      thoughtSignature: 'signature-1',
+    },
+    {
+      toolResponse: toolResponsePart.toolResponse,
+    },
+  ]);
+});
+
+test('Preserves Gemini function-call ids in history', () => {
+  const aiMessage = new AIMessage({
+    content: '',
+    tool_calls: [
+      {
+        id: 'call_weather_1',
+        name: 'get_weather',
+        args: { city: 'NYC' },
+      },
+    ],
+  });
+  const toolMessage = new ToolMessage({
+    content: 'sunny',
+    name: 'get_weather',
+    tool_call_id: 'call_weather_1',
+  });
+
+  expect(convertMessageContentToParts(aiMessage, true, [])).toEqual([
+    {
+      functionCall: {
+        id: 'call_weather_1',
+        name: 'get_weather',
+        args: { city: 'NYC' },
+      },
+    },
+  ]);
+  expect(convertMessageContentToParts(toolMessage, true, [aiMessage])).toEqual([
+    {
+      functionResponse: {
+        id: 'call_weather_1',
+        name: 'get_weather',
+        response: { result: 'sunny' },
+      },
+    },
+  ]);
+});
+
+test('Preserves Gemini server-side tool context parts from responses', () => {
+  const toolCallPart = {
+    toolCall: {
+      id: 'server-search-1',
+      name: 'google_search',
+      args: {},
+    },
+  };
+  const toolResponsePart = {
+    toolResponse: {
+      id: 'server-search-1',
+      name: 'google_search',
+      response: { results: [] },
+    },
+  };
+  const response = {
+    candidates: [
+      {
+        content: {
+          role: 'model',
+          parts: [toolCallPart, toolResponsePart],
+        },
+      },
+    ],
+  } as unknown as Parameters<
+    typeof convertResponseContentToChatGenerationChunk
+  >[0];
+
+  const chunk = convertResponseContentToChatGenerationChunk(response, {
+    index: 0,
+  });
+  const result = mapGenerateContentResultToChatResult(response);
+
+  const expectedContent = [
+    { ...toolCallPart, type: 'toolCall' },
+    { ...toolResponsePart, type: 'toolResponse' },
+  ];
+  expect(chunk?.message.content).toEqual(expectedContent);
+  expect(result.generations[0].message.content).toEqual(expectedContent);
+});
+
 test('Supports tool_choice', async () => {
   const model = createGoogleModel({
     model: 'gemini-2.0-flash',

package/src/llm/google/types.ts

@@ -1,9 +1,9 @@
+import { BindToolsInput } from '@langchain/core/language_models/chat_models';
 import {
   CodeExecutionTool,
   FunctionDeclarationsTool as GoogleGenerativeAIFunctionDeclarationsTool,
   GoogleSearchRetrievalTool,
 } from '@google/generative-ai';
-import { BindToolsInput } from '@langchain/core/language_models/chat_models';
 
 /** New GoogleSearch tool for Gemini 2.0+ models */
 export interface GoogleSearchTool {

package/src/llm/google/utils/common.ts

@@ -1,16 +1,8 @@
-import {
-  POSSIBLE_ROLES,
-  type Part,
-  type Content,
-  type TextPart,
-  type FileDataPart,
-  type InlineDataPart,
-  type FunctionCallPart,
-  type GenerateContentCandidate,
-  type EnhancedGenerateContentResponse,
-  type FunctionDeclaration as GenerativeAIFunctionDeclaration,
-  type FunctionDeclarationsTool as GoogleGenerativeAIFunctionDeclarationsTool,
-} from '@google/generative-ai';
+import { v4 as uuidv4 } from 'uuid';
+import { ChatGenerationChunk } from '@langchain/core/outputs';
+import { ToolCallChunk } from '@langchain/core/messages/tool';
+import { isOpenAITool } from '@langchain/core/language_models/base';
+import { isLangChainTool } from '@langchain/core/utils/function_calling';
 import {
   AIMessage,
   AIMessageChunk,
@@ -29,18 +21,26 @@ import {
   convertToProviderContentBlock,
   isDataContentBlock,
 } from '@langchain/core/messages';
-import { ChatGenerationChunk } from '@langchain/core/outputs';
+import {
+  POSSIBLE_ROLES,
+  type Part,
+  type Content,
+  type TextPart,
+  type FileDataPart,
+  type InlineDataPart,
+  type FunctionCallPart,
+  type GenerateContentCandidate,
+  type EnhancedGenerateContentResponse,
+  type FunctionDeclaration as GenerativeAIFunctionDeclaration,
+  type FunctionDeclarationsTool as GoogleGenerativeAIFunctionDeclarationsTool,
+} from '@google/generative-ai';
 import type { ChatGeneration, ChatResult } from '@langchain/core/outputs';
-import { isLangChainTool } from '@langchain/core/utils/function_calling';
-import { isOpenAITool } from '@langchain/core/language_models/base';
-import { ToolCallChunk } from '@langchain/core/messages/tool';
-import { v4 as uuidv4 } from 'uuid';
 import {
   jsonSchemaToGeminiParameters,
   schemaToGenerativeAIParameters,
 } from './zod_to_genai_parameters';
-import { GoogleGenerativeAIToolType } from '../types';
 import { toLangChainContent } from '@/messages/langchain';
+import { GoogleGenerativeAIToolType } from '../types';
 
 export const _FUNCTION_CALL_THOUGHT_SIGNATURES_MAP_KEY =
   '__gemini_function_call_thought_signatures__';
@@ -48,6 +48,49 @@ export const _FUNCTION_CALL_THOUGHT_SIGNATURES_MAP_KEY =
 const DUMMY_SIGNATURE =
   'ErYCCrMCAdHtim9kOoOkrPiCNVsmlpMIKd7ZMxgiFbVQOkgp7nlLcDMzVsZwIzvuT7nQROivoXA72ccC2lSDvR0Gh7dkWaGuj7ctv6t7ZceHnecx0QYa+ix8tYpRfjhyWozQ49lWiws6+YGjCt10KRTyWsZ2h6O7iHTYJwKIRwGUHRKy/qK/6kFxJm5ML00gLq4D8s5Z6DBpp2ZlR+uF4G8jJgeWQgyHWVdx2wGYElaceVAc66tZdPQRdOHpWtgYSI1YdaXgVI8KHY3/EfNc2YqqMIulvkDBAnuMhkAjV9xmBa54Tq+ih3Im4+r3DzqhGqYdsSkhS0kZMwte4Hjs65dZzCw9lANxIqYi1DJ639WNPYihp/DCJCos7o+/EeSPJaio5sgWDyUnMGkY1atsJZ+m7pj7DD5tvQ==';
 
+type GoogleServerSideToolPart = Part & {
+  type?: 'toolCall' | 'toolResponse';
+  toolCall?: object;
+  toolResponse?: object;
+};
+
+type GoogleServerSideToolPartMetadata = {
+  thought?: boolean;
+  thoughtSignature?: string;
+};
+
+type GoogleFunctionCallWithId = FunctionCallPart['functionCall'] & {
+  id?: string;
+};
+
+type GoogleFunctionResponseWithId = {
+  name: string;
+  response: object;
+  id?: string;
+};
+
+function getGoogleFunctionId(id?: string): string | undefined {
+  return id != null && id !== '' ? id : undefined;
+}
+
+function createGoogleFunctionResponsePart({
+  name,
+  response,
+  id,
+}: {
+  name: string;
+  response: object;
+  id?: string;
+}): Part {
+  const functionId = getGoogleFunctionId(id);
+  const functionResponse: GoogleFunctionResponseWithId = {
+    name,
+    response,
+    ...(functionId != null ? { id: functionId } : {}),
+  };
+  return { functionResponse };
+}
+
 /**
  * Executes a function immediately and returns its result.
  * Functional utility similar to an Immediately Invoked Function Expression (IIFE).
@@ -118,6 +161,63 @@ function messageContentMedia(content: MessageContentComplex): Part {
   throw new Error('Invalid media content');
 }
 
+function isGoogleServerSideToolPart(
+  content: MessageContentComplex
+): content is MessageContentComplex & GoogleServerSideToolPart {
+  return (
+    'toolCall' in content ||
+    'toolResponse' in content ||
+    content.type === 'toolCall' ||
+    content.type === 'toolResponse'
+  );
+}
+
+function convertGoogleServerSideToolPart(
+  content: MessageContentComplex & GoogleServerSideToolPart
+): Part {
+  const metadata: GoogleServerSideToolPartMetadata = {};
+  if ('thought' in content && typeof content.thought === 'boolean') {
+    metadata.thought = content.thought;
+  }
+  if (
+    'thoughtSignature' in content &&
+    typeof content.thoughtSignature === 'string'
+  ) {
+    metadata.thoughtSignature = content.thoughtSignature;
+  }
+  if ('toolCall' in content && content.toolCall != null) {
+    return { toolCall: content.toolCall, ...metadata } as unknown as Part;
+  }
+  if ('toolResponse' in content && content.toolResponse != null) {
+    return {
+      toolResponse: content.toolResponse,
+      ...metadata,
+    } as unknown as Part;
+  }
+
+  return content as Part;
+}
+
+function convertGoogleServerSideToolResponsePart(
+  part: Part
+): GoogleServerSideToolPart | undefined {
+  if (
+    'toolCall' in part &&
+    typeof part.toolCall === 'object' &&
+    part.toolCall != null
+  ) {
+    return { ...part, type: 'toolCall', toolCall: part.toolCall };
+  }
+  if (
+    'toolResponse' in part &&
+    typeof part.toolResponse === 'object' &&
+    part.toolResponse != null
+  ) {
+    return { ...part, type: 'toolResponse', toolResponse: part.toolResponse };
+  }
+  return undefined;
+}
+
 function inferToolNameFromPreviousMessages(
   message: ToolMessage | ToolMessageChunk,
   previousMessages: BaseMessage[]
@@ -280,6 +380,10 @@ function _convertLangChainContentToPart(
     );
   }
 
+  if (isGoogleServerSideToolPart(content)) {
+    return convertGoogleServerSideToolPart(content);
+  }
+
   if (content.type === 'text') {
     return { text: content.text };
   } else if (content.type === 'executableCode') {
@@ -375,25 +479,23 @@ export function convertMessageContentToParts(
 
     if (message.status === 'error') {
       return [
-        {
-          functionResponse: {
-            name: messageName,
-            // The API expects an object with an `error` field if the function call fails.
-            // `error` must be a valid object (not a string or array), so we wrap `message.content` here
-            response: { error: { details: result } },
-          },
-        },
+        createGoogleFunctionResponsePart({
+          name: messageName,
+          // The API expects an object with an `error` field if the function call fails.
+          // `error` must be a valid object (not a string or array), so we wrap `message.content` here
+          response: { error: { details: result } },
+          id: message.tool_call_id,
+        }),
       ];
     }
 
     return [
-      {
-        functionResponse: {
-          name: messageName,
-          // again, can't have a string or array value for `response`, so we wrap it as an object here
-          response: { result },
-        },
-      },
+      createGoogleFunctionResponsePart({
+        name: messageName,
+        // again, can't have a string or array value for `response`, so we wrap it as an object here
+        response: { result },
+        id: message.tool_call_id,
+      }),
     ];
   }
 
@@ -432,12 +534,15 @@ export function convertMessageContentToParts(
         }
         return '';
       });
+      const functionId = getGoogleFunctionId(tc.id);
+      const functionCall: GoogleFunctionCallWithId = {
+        name: tc.name,
+        args: tc.args,
+        ...(functionId != null ? { id: functionId } : {}),
+      };
 
       return {
-        functionCall: {
-          name: tc.name,
-          args: tc.args,
-        },
+        functionCall,
         ...(thoughtSignature ? { thoughtSignature } : {}),
       };
     });
@@ -598,6 +703,10 @@ export function convertResponseContentToChatGenerationChunk(
               codeExecutionResult: p.codeExecutionResult,
             };
           }
+          const serverSideToolPart = convertGoogleServerSideToolResponsePart(p);
+          if (serverSideToolPart !== undefined) {
+            return serverSideToolPart;
+          }
           return p;
         })
         .filter((p) => p !== undefined)
@@ -685,11 +794,7 @@ export function mapGenerateContentResultToChatResult(
     usageMetadata: UsageMetadata | undefined;
   }
 ): ChatResult {
-  if (
-    !response.candidates ||
-    response.candidates.length === 0 ||
-    !response.candidates[0]
-  ) {
+  if (!response.candidates || response.candidates.length === 0) {
     return {
       generations: [],
       llmOutput: {
@@ -725,7 +830,7 @@ export function mapGenerateContentResultToChatResult(
   if (
     Array.isArray(candidateContent?.parts) &&
     candidateContent.parts.length === 1 &&
-    candidateContent.parts[0].text &&
+    (candidateContent.parts[0].text ?? '') !== '' &&
     !(
       'thought' in candidateContent.parts[0] &&
       candidateContent.parts[0].thought === true
@@ -758,6 +863,10 @@ export function mapGenerateContentResultToChatResult(
               codeExecutionResult: p.codeExecutionResult,
             };
           }
+          const serverSideToolPart = convertGoogleServerSideToolResponsePart(p);
+          if (serverSideToolPart !== undefined) {
+            return serverSideToolPart;
+          }
           return p;
         })
         .filter((p) => p !== undefined)
@@ -807,7 +916,7 @@ export function mapGenerateContentResultToChatResult(
   const generation: ChatGeneration = {
     text,
     message: new AIMessage({
-      content: content ?? '',
+      content,
       tool_calls,
       additional_kwargs,
       usage_metadata: extra?.usageMetadata,

package/src/llm/google/utils/tools.ts

@@ -1,3 +1,10 @@
+import { StructuredToolInterface } from '@langchain/core/tools';
+import { ToolChoice } from '@langchain/core/language_models/chat_models';
+import { isLangChainTool } from '@langchain/core/utils/function_calling';
+import {
+  isOpenAITool,
+  ToolDefinition,
+} from '@langchain/core/language_models/base';
 import {
   Tool as GenerativeAITool,
   ToolConfig,
@@ -6,16 +13,9 @@ import {
   FunctionDeclarationsTool,
   FunctionDeclarationSchema,
 } from '@google/generative-ai';
-import { ToolChoice } from '@langchain/core/language_models/chat_models';
-import { StructuredToolInterface } from '@langchain/core/tools';
-import { isLangChainTool } from '@langchain/core/utils/function_calling';
-import {
-  isOpenAITool,
-  ToolDefinition,
-} from '@langchain/core/language_models/base';
+import { removeAdditionalProperties } from './zod_to_genai_parameters';
 import { convertToGenerativeAITools } from './common';
 import { GoogleGenerativeAIToolType } from '../types';
-import { removeAdditionalProperties } from './zod_to_genai_parameters';
 
 export function convertToolsToGenAI(
   tools: GoogleGenerativeAIToolType[],

package/src/llm/google/utils/zod_to_genai_parameters.ts

@@ -1,7 +1,3 @@
-import {
-  type FunctionDeclarationSchema as GenerativeAIFunctionDeclarationSchema,
-  type SchemaType as FunctionDeclarationSchemaType,
-} from '@google/generative-ai';
 import {
   InteropZodType,
   isInteropZodSchema,
@@ -10,6 +6,10 @@ import {
   type JsonSchema7Type,
   toJsonSchema,
 } from '@langchain/core/utils/json_schema';
+import {
+  type FunctionDeclarationSchema as GenerativeAIFunctionDeclarationSchema,
+  type SchemaType as FunctionDeclarationSchemaType,
+} from '@google/generative-ai';
 
 export interface GenerativeAIJsonSchema extends Record<string, unknown> {
   properties?: Record<string, GenerativeAIJsonSchema>;

package/src/llm/invoke.test.ts

@@ -1,17 +1,17 @@
 import { z } from 'zod';
 import { tool } from '@langchain/core/tools';
+import { describe, it, expect, jest } from '@jest/globals';
 import {
   AIMessage,
   AIMessageChunk,
   HumanMessage,
   ToolMessage,
 } from '@langchain/core/messages';
-import { describe, it, expect, jest } from '@jest/globals';
-import type { BaseMessage } from '@langchain/core/messages';
 import type { StructuredToolInterface } from '@langchain/core/tools';
+import type { BaseMessage } from '@langchain/core/messages';
 import type * as t from '@/types';
-import { attemptInvoke, tryFallbackProviders } from '@/llm/invoke';
 import { ToolOutputReferenceRegistry } from '@/tools/toolOutputReferences';
+import { attemptInvoke, tryFallbackProviders } from '@/llm/invoke';
 import { ToolNode } from '@/tools/ToolNode';
 import { Providers } from '@/common';