@librechat/agents 2.4.42 → 2.4.44

package/src/llm/google/utils/tools.ts

@@ -0,0 +1,160 @@
+import {
+  Tool as GenerativeAITool,
+  ToolConfig,
+  FunctionCallingMode,
+  FunctionDeclaration,
+  FunctionDeclarationsTool,
+  FunctionDeclarationSchema,
+} from '@google/generative-ai';
+import { ToolChoice } from '@langchain/core/language_models/chat_models';
+import { StructuredToolInterface } from '@langchain/core/tools';
+import { isLangChainTool } from '@langchain/core/utils/function_calling';
+import {
+  isOpenAITool,
+  ToolDefinition,
+} from '@langchain/core/language_models/base';
+import { convertToGenerativeAITools } from './common';
+import { GoogleGenerativeAIToolType } from '../types';
+import { removeAdditionalProperties } from './zod_to_genai_parameters';
+
+export function convertToolsToGenAI(
+  tools: GoogleGenerativeAIToolType[],
+  extra?: {
+    toolChoice?: ToolChoice;
+    allowedFunctionNames?: string[];
+  }
+): {
+  tools: GenerativeAITool[];
+  toolConfig?: ToolConfig;
+} {
+  // Extract function declaration processing to a separate function
+  const genAITools = processTools(tools);
+
+  // Simplify tool config creation
+  const toolConfig = createToolConfig(genAITools, extra);
+
+  return { tools: genAITools, toolConfig };
+}
+
+function processTools(tools: GoogleGenerativeAIToolType[]): GenerativeAITool[] {
+  let functionDeclarationTools: FunctionDeclaration[] = [];
+  const genAITools: GenerativeAITool[] = [];
+
+  tools.forEach((tool) => {
+    if (isLangChainTool(tool)) {
+      const [convertedTool] = convertToGenerativeAITools([
+        tool as StructuredToolInterface,
+      ]);
+      if (convertedTool.functionDeclarations) {
+        functionDeclarationTools.push(...convertedTool.functionDeclarations);
+      }
+    } else if (isOpenAITool(tool)) {
+      const { functionDeclarations } = convertOpenAIToolToGenAI(tool);
+      if (functionDeclarations) {
+        functionDeclarationTools.push(...functionDeclarations);
+      } else {
+        throw new Error(
+          'Failed to convert OpenAI structured tool to GenerativeAI tool'
+        );
+      }
+    } else {
+      genAITools.push(tool as GenerativeAITool);
+    }
+  });
+
+  const genAIFunctionDeclaration = genAITools.find(
+    (t) => 'functionDeclarations' in t
+  );
+  if (genAIFunctionDeclaration) {
+    return genAITools.map((tool) => {
+      if (
+        functionDeclarationTools.length > 0 &&
+        'functionDeclarations' in tool
+      ) {
+        const newTool = {
+          functionDeclarations: [
+            ...(tool.functionDeclarations || []),
+            ...functionDeclarationTools,
+          ],
+        };
+        // Clear the functionDeclarationTools array so it is not passed again
+        functionDeclarationTools = [];
+        return newTool;
+      }
+      return tool;
+    });
+  }
+
+  return [
+    ...genAITools,
+    ...(functionDeclarationTools.length > 0
+      ? [
+        {
+          functionDeclarations: functionDeclarationTools,
+        },
+      ]
+      : []),
+  ];
+}
+
+function convertOpenAIToolToGenAI(
+  tool: ToolDefinition
+): FunctionDeclarationsTool {
+  return {
+    functionDeclarations: [
+      {
+        name: tool.function.name,
+        description: tool.function.description,
+        parameters: removeAdditionalProperties(
+          tool.function.parameters
+        ) as FunctionDeclarationSchema,
+      },
+    ],
+  };
+}
+
+function createToolConfig(
+  genAITools: GenerativeAITool[],
+  extra?: {
+    toolChoice?: ToolChoice;
+    allowedFunctionNames?: string[];
+  }
+): ToolConfig | undefined {
+  if (!genAITools.length || !extra) return undefined;
+
+  const { toolChoice, allowedFunctionNames } = extra;
+
+  const modeMap: Record<string, FunctionCallingMode> = {
+    any: FunctionCallingMode.ANY,
+    auto: FunctionCallingMode.AUTO,
+    none: FunctionCallingMode.NONE,
+  };
+
+  if (
+    toolChoice != null &&
+    ['any', 'auto', 'none'].includes(toolChoice as string)
+  ) {
+    return {
+      functionCallingConfig: {
+        mode: modeMap[toolChoice as keyof typeof modeMap] ?? 'MODE_UNSPECIFIED',
+        allowedFunctionNames,
+      },
+    };
+  }
+
+  if (typeof toolChoice === 'string' || allowedFunctionNames) {
+    return {
+      functionCallingConfig: {
+        mode: FunctionCallingMode.ANY,
+        allowedFunctionNames: [
+          ...(allowedFunctionNames ?? []),
+          ...(toolChoice != null && typeof toolChoice === 'string'
+            ? [toolChoice]
+            : []),
+        ],
+      },
+    };
+  }
+
+  return undefined;
+}

package/src/llm/google/utils/zod_to_genai_parameters.ts

@@ -0,0 +1,88 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
+
+import {
+  type FunctionDeclarationSchema as GenerativeAIFunctionDeclarationSchema,
+  type SchemaType as FunctionDeclarationSchemaType,
+} from '@google/generative-ai';
+import {
+  InteropZodType,
+  isInteropZodSchema,
+} from '@langchain/core/utils/types';
+import {
+  type JsonSchema7Type,
+  toJsonSchema,
+} from '@langchain/core/utils/json_schema';
+
+export interface GenerativeAIJsonSchema extends Record<string, unknown> {
+  properties?: Record<string, GenerativeAIJsonSchema>;
+  type: FunctionDeclarationSchemaType;
+}
+
+export interface GenerativeAIJsonSchemaDirty extends GenerativeAIJsonSchema {
+  properties?: Record<string, GenerativeAIJsonSchemaDirty>;
+  additionalProperties?: boolean;
+}
+
+export function removeAdditionalProperties(
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  obj: Record<string, any>
+): GenerativeAIJsonSchema {
+  if (typeof obj === 'object' && obj !== null) {
+    const newObj = { ...obj };
+
+    if ('additionalProperties' in newObj) {
+      delete newObj.additionalProperties;
+    }
+    if ('$schema' in newObj) {
+      delete newObj.$schema;
+    }
+    if ('strict' in newObj) {
+      delete newObj.strict;
+    }
+
+    for (const key in newObj) {
+      if (key in newObj) {
+        if (Array.isArray(newObj[key])) {
+          newObj[key] = newObj[key].map(removeAdditionalProperties);
+        } else if (typeof newObj[key] === 'object' && newObj[key] !== null) {
+          newObj[key] = removeAdditionalProperties(newObj[key]);
+        }
+      }
+    }
+
+    return newObj as GenerativeAIJsonSchema;
+  }
+
+  return obj as GenerativeAIJsonSchema;
+}
+
+export function schemaToGenerativeAIParameters<
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  RunOutput extends Record<string, any> = Record<string, any>,
+>(
+  schema: InteropZodType<RunOutput> | JsonSchema7Type
+): GenerativeAIFunctionDeclarationSchema {
+  // GenerativeAI doesn't accept either the $schema or additionalProperties
+  // attributes, so we need to explicitly remove them.
+  const jsonSchema = removeAdditionalProperties(
+    isInteropZodSchema(schema) ? toJsonSchema(schema) : schema
+  );
+  const { $schema, ...rest } = jsonSchema;
+
+  return rest as GenerativeAIFunctionDeclarationSchema;
+}
+
+export function jsonSchemaToGeminiParameters(
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  schema: Record<string, any>
+): GenerativeAIFunctionDeclarationSchema {
+  // Gemini doesn't accept either the $schema or additionalProperties
+  // attributes, so we need to explicitly remove them.
+
+  const jsonSchema = removeAdditionalProperties(
+    schema as GenerativeAIJsonSchemaDirty
+  );
+  const { $schema, ...rest } = jsonSchema;
+
+  return rest as GenerativeAIFunctionDeclarationSchema;
+}

package/src/llm/providers.ts

@@ -3,22 +3,23 @@ import { ChatOllama } from '@langchain/ollama';
 import { ChatMistralAI } from '@langchain/mistralai';
 import { ChatBedrockConverse } from '@langchain/aws';
 // import { ChatAnthropic } from '@langchain/anthropic';
-import { ChatVertexAI } from '@langchain/google-vertexai';
+// import { ChatVertexAI } from '@langchain/google-vertexai';
 import { BedrockChat } from '@langchain/community/chat_models/bedrock/web';
 import type {
   ChatModelConstructorMap,
   ProviderOptionsMap,
   ChatModelMap,
 } from '@/types';
-import { CustomChatGoogleGenerativeAI } from '@/llm/google';
-import { CustomAnthropic } from '@/llm/anthropic';
-import { ChatOpenRouter } from '@/llm/openrouter';
 import {
-  ChatXAI,
-  ChatOpenAI,
-  ChatDeepSeek,
   AzureChatOpenAI,
+  ChatDeepSeek,
+  ChatOpenAI,
+  ChatXAI,
 } from '@/llm/openai';
+import { CustomChatGoogleGenerativeAI } from '@/llm/google';
+import { CustomAnthropic } from '@/llm/anthropic';
+import { ChatOpenRouter } from '@/llm/openrouter';
+import { ChatVertexAI } from '@/llm/vertexai';
 import { Providers } from '@/common';
 
 export const llmProviders: Partial<ChatModelConstructorMap> = {

package/src/llm/vertexai/index.ts

@@ -0,0 +1,360 @@
+import { ChatGoogle } from '@langchain/google-gauth';
+import { ChatConnection } from '@langchain/google-common';
+import type {
+  GeminiRequest,
+  GoogleAIModelRequestParams,
+  GoogleAbstractedClient,
+} from '@langchain/google-common';
+import type { BaseMessage } from '@langchain/core/messages';
+import type { VertexAIClientOptions } from '@/types';
+
+class CustomChatConnection extends ChatConnection<VertexAIClientOptions> {
+  async formatData(
+    input: BaseMessage[],
+    parameters: GoogleAIModelRequestParams
+  ): Promise<unknown> {
+    const formattedData = (await super.formatData(
+      input,
+      parameters
+    )) as GeminiRequest;
+    if (
+      formattedData.generationConfig?.thinkingConfig?.thinkingBudget === -1 &&
+      formattedData.generationConfig.thinkingConfig.includeThoughts === false
+    ) {
+      formattedData.generationConfig.thinkingConfig.includeThoughts = true;
+    }
+    return formattedData;
+  }
+}
+
+/**
+ * Integration with Google Vertex AI chat models.
+ *
+ * Setup:
+ * Install `@langchain/google-vertexai` and set your stringified
+ * Vertex AI credentials as an environment variable named `GOOGLE_APPLICATION_CREDENTIALS`.
+ *
+ * ```bash
+ * npm install @langchain/google-vertexai
+ * export GOOGLE_APPLICATION_CREDENTIALS="path/to/credentials"
+ * ```
+ *
+ * ## [Constructor args](https://api.js.langchain.com/classes/_langchain_google_vertexai.index.ChatVertexAI.html#constructor.new_ChatVertexAI)
+ *
+ * ## [Runtime args](https://api.js.langchain.com/interfaces/langchain_google_common_types.GoogleAIBaseLanguageModelCallOptions.html)
+ *
+ * Runtime args can be passed as the second argument to any of the base runnable methods `.invoke`. `.stream`, `.batch`, etc.
+ * They can also be passed via `.withConfig`, or the second arg in `.bindTools`, like shown in the examples below:
+ *
+ * ```typescript
+ * // When calling `.withConfig`, call options should be passed via the first argument
+ * const llmWithArgsBound = llm.withConfig({
+ *   stop: ["\n"],
+ *   tools: [...],
+ * });
+ *
+ * // When calling `.bindTools`, call options should be passed via the second argument
+ * const llmWithTools = llm.bindTools(
+ *   [...],
+ *   {
+ *     tool_choice: "auto",
+ *   }
+ * );
+ * ```
+ *
+ * ## Examples
+ *
+ * <details open>
+ * <summary><strong>Instantiate</strong></summary>
+ *
+ * ```typescript
+ * import { ChatVertexAI } from '@langchain/google-vertexai';
+ *
+ * const llm = new ChatVertexAI({
+ *   model: "gemini-1.5-pro",
+ *   temperature: 0,
+ *   // other params...
+ * });
+ * ```
+ * </details>
+ *
+ * <br />
+ *
+ * <details>
+ * <summary><strong>Invoking</strong></summary>
+ *
+ * ```typescript
+ * const input = `Translate "I love programming" into French.`;
+ *
+ * // Models also accept a list of chat messages or a formatted prompt
+ * const result = await llm.invoke(input);
+ * console.log(result);
+ * ```
+ *
+ * ```txt
+ * AIMessageChunk {
+ *   "content": "\"J'adore programmer\" \n\nHere's why this is the best translation:\n\n* **J'adore** means \"I love\" and conveys a strong passion.\n* **Programmer** is the French verb for \"to program.\"\n\nThis translation is natural and idiomatic in French. \n",
+ *   "additional_kwargs": {},
+ *   "response_metadata": {},
+ *   "tool_calls": [],
+ *   "tool_call_chunks": [],
+ *   "invalid_tool_calls": [],
+ *   "usage_metadata": {
+ *     "input_tokens": 9,
+ *     "output_tokens": 63,
+ *     "total_tokens": 72
+ *   }
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
+ *
+ * <details>
+ * <summary><strong>Streaming Chunks</strong></summary>
+ *
+ * ```typescript
+ * for await (const chunk of await llm.stream(input)) {
+ *   console.log(chunk);
+ * }
+ * ```
+ *
+ * ```txt
+ * AIMessageChunk {
+ *   "content": "\"",
+ *   "additional_kwargs": {},
+ *   "response_metadata": {},
+ *   "tool_calls": [],
+ *   "tool_call_chunks": [],
+ *   "invalid_tool_calls": []
+ * }
+ * AIMessageChunk {
+ *   "content": "J'adore programmer\" \n",
+ *   "additional_kwargs": {},
+ *   "response_metadata": {},
+ *   "tool_calls": [],
+ *   "tool_call_chunks": [],
+ *   "invalid_tool_calls": []
+ * }
+ * AIMessageChunk {
+ *   "content": "",
+ *   "additional_kwargs": {},
+ *   "response_metadata": {},
+ *   "tool_calls": [],
+ *   "tool_call_chunks": [],
+ *   "invalid_tool_calls": []
+ * }
+ * AIMessageChunk {
+ *   "content": "",
+ *   "additional_kwargs": {},
+ *   "response_metadata": {
+ *     "finishReason": "stop"
+ *   },
+ *   "tool_calls": [],
+ *   "tool_call_chunks": [],
+ *   "invalid_tool_calls": [],
+ *   "usage_metadata": {
+ *     "input_tokens": 9,
+ *     "output_tokens": 8,
+ *     "total_tokens": 17
+ *   }
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
+ *
+ * <details>
+ * <summary><strong>Aggregate Streamed Chunks</strong></summary>
+ *
+ * ```typescript
+ * import { AIMessageChunk } from '@langchain/core/messages';
+ * import { concat } from '@langchain/core/utils/stream';
+ *
+ * const stream = await llm.stream(input);
+ * let full: AIMessageChunk | undefined;
+ * for await (const chunk of stream) {
+ *   full = !full ? chunk : concat(full, chunk);
+ * }
+ * console.log(full);
+ * ```
+ *
+ * ```txt
+ * AIMessageChunk {
+ *   "content": "\"J'adore programmer\" \n",
+ *   "additional_kwargs": {},
+ *   "response_metadata": {
+ *     "finishReason": "stop"
+ *   },
+ *   "tool_calls": [],
+ *   "tool_call_chunks": [],
+ *   "invalid_tool_calls": [],
+ *   "usage_metadata": {
+ *     "input_tokens": 9,
+ *     "output_tokens": 8,
+ *     "total_tokens": 17
+ *   }
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
+ *
+ * <details>
+ * <summary><strong>Bind tools</strong></summary>
+ *
+ * ```typescript
+ * import { z } from 'zod';
+ *
+ * const GetWeather = {
+ *   name: "GetWeather",
+ *   description: "Get the current weather in a given location",
+ *   schema: z.object({
+ *     location: z.string().describe("The city and state, e.g. San Francisco, CA")
+ *   }),
+ * }
+ *
+ * const GetPopulation = {
+ *   name: "GetPopulation",
+ *   description: "Get the current population in a given location",
+ *   schema: z.object({
+ *     location: z.string().describe("The city and state, e.g. San Francisco, CA")
+ *   }),
+ * }
+ *
+ * const llmWithTools = llm.bindTools([GetWeather, GetPopulation]);
+ * const aiMsg = await llmWithTools.invoke(
+ *   "Which city is hotter today and which is bigger: LA or NY?"
+ * );
+ * console.log(aiMsg.tool_calls);
+ * ```
+ *
+ * ```txt
+ * [
+ *   {
+ *     name: 'GetPopulation',
+ *     args: { location: 'New York City, NY' },
+ *     id: '33c1c1f47e2f492799c77d2800a43912',
+ *     type: 'tool_call'
+ *   }
+ * ]
+ * ```
+ * </details>
+ *
+ * <br />
+ *
+ * <details>
+ * <summary><strong>Structured Output</strong></summary>
+ *
+ * ```typescript
+ * import { z } from 'zod';
+ *
+ * const Joke = z.object({
+ *   setup: z.string().describe("The setup of the joke"),
+ *   punchline: z.string().describe("The punchline to the joke"),
+ *   rating: z.number().optional().describe("How funny the joke is, from 1 to 10")
+ * }).describe('Joke to tell user.');
+ *
+ * const structuredLlm = llm.withStructuredOutput(Joke, { name: "Joke" });
+ * const jokeResult = await structuredLlm.invoke("Tell me a joke about cats");
+ * console.log(jokeResult);
+ * ```
+ *
+ * ```txt
+ * {
+ *   setup: 'What do you call a cat that loves to bowl?',
+ *   punchline: 'An alley cat!'
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
+ *
+ * <details>
+ * <summary><strong>Usage Metadata</strong></summary>
+ *
+ * ```typescript
+ * const aiMsgForMetadata = await llm.invoke(input);
+ * console.log(aiMsgForMetadata.usage_metadata);
+ * ```
+ *
+ * ```txt
+ * { input_tokens: 9, output_tokens: 8, total_tokens: 17 }
+ * ```
+ * </details>
+ *
+ * <br />
+ *
+ * <details>
+ * <summary><strong>Stream Usage Metadata</strong></summary>
+ *
+ * ```typescript
+ * const streamForMetadata = await llm.stream(
+ *   input,
+ *   {
+ *     streamUsage: true
+ *   }
+ * );
+ * let fullForMetadata: AIMessageChunk | undefined;
+ * for await (const chunk of streamForMetadata) {
+ *   fullForMetadata = !fullForMetadata ? chunk : concat(fullForMetadata, chunk);
+ * }
+ * console.log(fullForMetadata?.usage_metadata);
+ * ```
+ *
+ * ```txt
+ * { input_tokens: 9, output_tokens: 8, total_tokens: 17 }
+ * ```
+ * </details>
+ *
+ * <br />
+ */
+export class ChatVertexAI extends ChatGoogle {
+  lc_namespace = ['langchain', 'chat_models', 'vertexai'];
+  dynamicThinkingBudget = false;
+
+  static lc_name(): 'ChatVertexAI' {
+    return 'ChatVertexAI';
+  }
+
+  constructor(fields?: VertexAIClientOptions) {
+    let dynamicThinkingBudget = false;
+    if (fields?.thinkingBudget === -1) {
+      dynamicThinkingBudget = true;
+      fields.thinkingBudget = 1;
+    }
+    super({
+      ...fields,
+      platformType: 'gcp',
+    });
+    this.dynamicThinkingBudget = dynamicThinkingBudget;
+  }
+  invocationParams(
+    options?: this['ParsedCallOptions'] | undefined
+  ): GoogleAIModelRequestParams {
+    const params = super.invocationParams(options);
+    if (this.dynamicThinkingBudget) {
+      params.maxReasoningTokens = -1;
+    }
+    return params;
+  }
+
+  buildConnection(
+    fields: VertexAIClientOptions,
+    client: GoogleAbstractedClient
+  ): void {
+    this.connection = new CustomChatConnection(
+      { ...fields, ...this },
+      this.caller,
+      client,
+      false
+    );
+
+    this.streamedConnection = new CustomChatConnection(
+      { ...fields, ...this },
+      this.caller,
+      client,
+      true
+    );
+  }
+}

package/src/stream.ts

@@ -244,6 +244,7 @@ hasToolCallChunks: ${hasToolCallChunks}
       content.every(
         (c) =>
           (c.type?.startsWith(ContentTypes.THINKING) ?? false) ||
+          (c.type?.startsWith(ContentTypes.REASONING) ?? false) ||
           (c.type?.startsWith(ContentTypes.REASONING_CONTENT) ?? false)
       )
     ) {
@@ -252,6 +253,7 @@ hasToolCallChunks: ${hasToolCallChunks}
           type: ContentTypes.THINK,
           think:
             (c as t.ThinkingContentText).thinking ??
+            (c as Partial<t.GoogleReasoningContentText>).reasoning ??
             (c as Partial<t.BedrockReasoningContentText>).reasoningText?.text ??
             '',
         })),
@@ -264,8 +266,9 @@ hasToolCallChunks: ${hasToolCallChunks}
       | undefined;
     if (
       Array.isArray(chunk.content) &&
-      (chunk.content[0]?.type === 'thinking' ||
-        chunk.content[0]?.type === 'reasoning_content')
+      (chunk.content[0]?.type === ContentTypes.THINKING ||
+        chunk.content[0]?.type === ContentTypes.REASONING ||
+        chunk.content[0]?.type === ContentTypes.REASONING_CONTENT)
     ) {
       reasoning_content = 'valid';
     }