@librechat/agents 2.4.42 → 2.4.44

package/dist/types/llm/vertexai/index.d.ts

@@ -0,0 +1,293 @@
+import { ChatGoogle } from '@langchain/google-gauth';
+import type { GoogleAIModelRequestParams, GoogleAbstractedClient } from '@langchain/google-common';
+import type { VertexAIClientOptions } from '@/types';
+/**
+ * Integration with Google Vertex AI chat models.
+ *
+ * Setup:
+ * Install `@langchain/google-vertexai` and set your stringified
+ * Vertex AI credentials as an environment variable named `GOOGLE_APPLICATION_CREDENTIALS`.
+ *
+ * ```bash
+ * npm install @langchain/google-vertexai
+ * export GOOGLE_APPLICATION_CREDENTIALS="path/to/credentials"
+ * ```
+ *
+ * ## [Constructor args](https://api.js.langchain.com/classes/_langchain_google_vertexai.index.ChatVertexAI.html#constructor.new_ChatVertexAI)
+ *
+ * ## [Runtime args](https://api.js.langchain.com/interfaces/langchain_google_common_types.GoogleAIBaseLanguageModelCallOptions.html)
+ *
+ * Runtime args can be passed as the second argument to any of the base runnable methods `.invoke`. `.stream`, `.batch`, etc.
+ * They can also be passed via `.withConfig`, or the second arg in `.bindTools`, like shown in the examples below:
+ *
+ * ```typescript
+ * // When calling `.withConfig`, call options should be passed via the first argument
+ * const llmWithArgsBound = llm.withConfig({
+ *   stop: ["\n"],
+ *   tools: [...],
+ * });
+ *
+ * // When calling `.bindTools`, call options should be passed via the second argument
+ * const llmWithTools = llm.bindTools(
+ *   [...],
+ *   {
+ *     tool_choice: "auto",
+ *   }
+ * );
+ * ```
+ *
+ * ## Examples
+ *
+ * <details open>
+ * <summary><strong>Instantiate</strong></summary>
+ *
+ * ```typescript
+ * import { ChatVertexAI } from '@langchain/google-vertexai';
+ *
+ * const llm = new ChatVertexAI({
+ *   model: "gemini-1.5-pro",
+ *   temperature: 0,
+ *   // other params...
+ * });
+ * ```
+ * </details>
+ *
+ * <br />
+ *
+ * <details>
+ * <summary><strong>Invoking</strong></summary>
+ *
+ * ```typescript
+ * const input = `Translate "I love programming" into French.`;
+ *
+ * // Models also accept a list of chat messages or a formatted prompt
+ * const result = await llm.invoke(input);
+ * console.log(result);
+ * ```
+ *
+ * ```txt
+ * AIMessageChunk {
+ *   "content": "\"J'adore programmer\" \n\nHere's why this is the best translation:\n\n* **J'adore** means \"I love\" and conveys a strong passion.\n* **Programmer** is the French verb for \"to program.\"\n\nThis translation is natural and idiomatic in French. \n",
+ *   "additional_kwargs": {},
+ *   "response_metadata": {},
+ *   "tool_calls": [],
+ *   "tool_call_chunks": [],
+ *   "invalid_tool_calls": [],
+ *   "usage_metadata": {
+ *     "input_tokens": 9,
+ *     "output_tokens": 63,
+ *     "total_tokens": 72
+ *   }
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
+ *
+ * <details>
+ * <summary><strong>Streaming Chunks</strong></summary>
+ *
+ * ```typescript
+ * for await (const chunk of await llm.stream(input)) {
+ *   console.log(chunk);
+ * }
+ * ```
+ *
+ * ```txt
+ * AIMessageChunk {
+ *   "content": "\"",
+ *   "additional_kwargs": {},
+ *   "response_metadata": {},
+ *   "tool_calls": [],
+ *   "tool_call_chunks": [],
+ *   "invalid_tool_calls": []
+ * }
+ * AIMessageChunk {
+ *   "content": "J'adore programmer\" \n",
+ *   "additional_kwargs": {},
+ *   "response_metadata": {},
+ *   "tool_calls": [],
+ *   "tool_call_chunks": [],
+ *   "invalid_tool_calls": []
+ * }
+ * AIMessageChunk {
+ *   "content": "",
+ *   "additional_kwargs": {},
+ *   "response_metadata": {},
+ *   "tool_calls": [],
+ *   "tool_call_chunks": [],
+ *   "invalid_tool_calls": []
+ * }
+ * AIMessageChunk {
+ *   "content": "",
+ *   "additional_kwargs": {},
+ *   "response_metadata": {
+ *     "finishReason": "stop"
+ *   },
+ *   "tool_calls": [],
+ *   "tool_call_chunks": [],
+ *   "invalid_tool_calls": [],
+ *   "usage_metadata": {
+ *     "input_tokens": 9,
+ *     "output_tokens": 8,
+ *     "total_tokens": 17
+ *   }
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
+ *
+ * <details>
+ * <summary><strong>Aggregate Streamed Chunks</strong></summary>
+ *
+ * ```typescript
+ * import { AIMessageChunk } from '@langchain/core/messages';
+ * import { concat } from '@langchain/core/utils/stream';
+ *
+ * const stream = await llm.stream(input);
+ * let full: AIMessageChunk | undefined;
+ * for await (const chunk of stream) {
+ *   full = !full ? chunk : concat(full, chunk);
+ * }
+ * console.log(full);
+ * ```
+ *
+ * ```txt
+ * AIMessageChunk {
+ *   "content": "\"J'adore programmer\" \n",
+ *   "additional_kwargs": {},
+ *   "response_metadata": {
+ *     "finishReason": "stop"
+ *   },
+ *   "tool_calls": [],
+ *   "tool_call_chunks": [],
+ *   "invalid_tool_calls": [],
+ *   "usage_metadata": {
+ *     "input_tokens": 9,
+ *     "output_tokens": 8,
+ *     "total_tokens": 17
+ *   }
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
+ *
+ * <details>
+ * <summary><strong>Bind tools</strong></summary>
+ *
+ * ```typescript
+ * import { z } from 'zod';
+ *
+ * const GetWeather = {
+ *   name: "GetWeather",
+ *   description: "Get the current weather in a given location",
+ *   schema: z.object({
+ *     location: z.string().describe("The city and state, e.g. San Francisco, CA")
+ *   }),
+ * }
+ *
+ * const GetPopulation = {
+ *   name: "GetPopulation",
+ *   description: "Get the current population in a given location",
+ *   schema: z.object({
+ *     location: z.string().describe("The city and state, e.g. San Francisco, CA")
+ *   }),
+ * }
+ *
+ * const llmWithTools = llm.bindTools([GetWeather, GetPopulation]);
+ * const aiMsg = await llmWithTools.invoke(
+ *   "Which city is hotter today and which is bigger: LA or NY?"
+ * );
+ * console.log(aiMsg.tool_calls);
+ * ```
+ *
+ * ```txt
+ * [
+ *   {
+ *     name: 'GetPopulation',
+ *     args: { location: 'New York City, NY' },
+ *     id: '33c1c1f47e2f492799c77d2800a43912',
+ *     type: 'tool_call'
+ *   }
+ * ]
+ * ```
+ * </details>
+ *
+ * <br />
+ *
+ * <details>
+ * <summary><strong>Structured Output</strong></summary>
+ *
+ * ```typescript
+ * import { z } from 'zod';
+ *
+ * const Joke = z.object({
+ *   setup: z.string().describe("The setup of the joke"),
+ *   punchline: z.string().describe("The punchline to the joke"),
+ *   rating: z.number().optional().describe("How funny the joke is, from 1 to 10")
+ * }).describe('Joke to tell user.');
+ *
+ * const structuredLlm = llm.withStructuredOutput(Joke, { name: "Joke" });
+ * const jokeResult = await structuredLlm.invoke("Tell me a joke about cats");
+ * console.log(jokeResult);
+ * ```
+ *
+ * ```txt
+ * {
+ *   setup: 'What do you call a cat that loves to bowl?',
+ *   punchline: 'An alley cat!'
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
+ *
+ * <details>
+ * <summary><strong>Usage Metadata</strong></summary>
+ *
+ * ```typescript
+ * const aiMsgForMetadata = await llm.invoke(input);
+ * console.log(aiMsgForMetadata.usage_metadata);
+ * ```
+ *
+ * ```txt
+ * { input_tokens: 9, output_tokens: 8, total_tokens: 17 }
+ * ```
+ * </details>
+ *
+ * <br />
+ *
+ * <details>
+ * <summary><strong>Stream Usage Metadata</strong></summary>
+ *
+ * ```typescript
+ * const streamForMetadata = await llm.stream(
+ *   input,
+ *   {
+ *     streamUsage: true
+ *   }
+ * );
+ * let fullForMetadata: AIMessageChunk | undefined;
+ * for await (const chunk of streamForMetadata) {
+ *   fullForMetadata = !fullForMetadata ? chunk : concat(fullForMetadata, chunk);
+ * }
+ * console.log(fullForMetadata?.usage_metadata);
+ * ```
+ *
+ * ```txt
+ * { input_tokens: 9, output_tokens: 8, total_tokens: 17 }
+ * ```
+ * </details>
+ *
+ * <br />
+ */
+export declare class ChatVertexAI extends ChatGoogle {
+    lc_namespace: string[];
+    dynamicThinkingBudget: boolean;
+    static lc_name(): 'ChatVertexAI';
+    constructor(fields?: VertexAIClientOptions);
+    invocationParams(options?: this['ParsedCallOptions'] | undefined): GoogleAIModelRequestParams;
+    buildConnection(fields: VertexAIClientOptions, client: GoogleAbstractedClient): void;
+}

package/dist/types/types/llm.d.ts

@@ -2,12 +2,12 @@ import { ChatOllama } from '@langchain/ollama';
 import { ChatAnthropic } from '@langchain/anthropic';
 import { ChatMistralAI } from '@langchain/mistralai';
 import { ChatBedrockConverse } from '@langchain/aws';
-import { ChatVertexAI } from '@langchain/google-vertexai';
 import { BedrockChat } from '@langchain/community/chat_models/bedrock/web';
 import type { BindToolsInput, BaseChatModelParams } from '@langchain/core/language_models/chat_models';
 import type { OpenAIChatInput, ChatOpenAIFields, AzureOpenAIInput, ClientOptions as OAIClientOptions } from '@langchain/openai';
 import type { BedrockChatFields } from '@langchain/community/chat_models/bedrock/web';
 import type { GoogleGenerativeAIChatInput } from '@langchain/google-genai';
+import type { GeminiGenerationConfig } from '@langchain/google-common';
 import type { ChatVertexAIInput } from '@langchain/google-vertexai';
 import type { ChatDeepSeekCallOptions } from '@langchain/deepseek';
 import type { ChatOpenRouterCallOptions } from '@/llm/openrouter';
@@ -20,9 +20,10 @@ import type { Runnable } from '@langchain/core/runnables';
 import type { ChatOllamaInput } from '@langchain/ollama';
 import type { OpenAI as OpenAIClient } from 'openai';
 import type { ChatXAIInput } from '@langchain/xai';
-import { ChatXAI, ChatOpenAI, ChatDeepSeek, AzureChatOpenAI } from '@/llm/openai';
+import { AzureChatOpenAI, ChatDeepSeek, ChatOpenAI, ChatXAI } from '@/llm/openai';
 import { CustomChatGoogleGenerativeAI } from '@/llm/google';
 import { ChatOpenRouter } from '@/llm/openrouter';
+import { ChatVertexAI } from '@/llm/vertexai';
 import { Providers } from '@/common';
 export type AzureClientOptions = Partial<OpenAIChatInput> & Partial<AzureOpenAIInput> & {
     openAIApiKey?: string;
@@ -43,7 +44,9 @@ export type OpenAIClientOptions = ChatOpenAIFields;
 export type OllamaClientOptions = ChatOllamaInput;
 export type AnthropicClientOptions = AnthropicInput;
 export type MistralAIClientOptions = ChatMistralAIInput;
-export type VertexAIClientOptions = ChatVertexAIInput;
+export type VertexAIClientOptions = ChatVertexAIInput & {
+    includeThoughts?: boolean;
+};
 export type BedrockClientOptions = BedrockChatFields;
 export type BedrockAnthropicInput = ChatBedrockConverseInput & {
     additionalModelRequestFields?: ChatBedrockConverseInput['additionalModelRequestFields'] & AnthropicReasoning;
@@ -51,6 +54,7 @@ export type BedrockAnthropicInput = ChatBedrockConverseInput & {
 export type BedrockConverseClientOptions = ChatBedrockConverseInput;
 export type GoogleClientOptions = GoogleGenerativeAIChatInput & {
     customHeaders?: RequestOptions['customHeaders'];
+    thinkingConfig?: GeminiGenerationConfig['thinkingConfig'];
 };
 export type DeepSeekClientOptions = ChatDeepSeekCallOptions;
 export type XAIClientOptions = ChatXAIInput;

package/dist/types/types/stream.d.ts

@@ -191,6 +191,11 @@ export type ReasoningContentText = {
     type: ContentTypes.THINK;
     think: string;
 };
+/** Vertex AI / Google Common - Reasoning Content Block Format */
+export type GoogleReasoningContentText = {
+    type: ContentTypes.REASONING;
+    reasoning: string;
+};
 /** Anthropic's Reasoning Content Block Format */
 export type ThinkingContentText = {
     type: ContentTypes.THINKING;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@librechat/agents",
-  "version": "2.4.42",
+  "version": "2.4.44",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/esm/main.mjs",
   "types": "./dist/types/index.d.ts",

package/src/common/enum.ts

@@ -117,10 +117,12 @@ export enum ContentTypes {
   IMAGE_FILE = 'image_file',
   /** Anthropic */
   THINKING = 'thinking',
-  /** Bedrock */
-  REASONING_CONTENT = 'reasoning_content',
+  /** Vertex AI / Google Common */
+  REASONING = 'reasoning',
   /** Multi-Agent Switch */
   AGENT_UPDATE = 'agent_update',
+  /** Bedrock */
+  REASONING_CONTENT = 'reasoning_content',
 }
 
 export enum ToolCallTypes {

package/src/graphs/Graph.ts

@@ -426,13 +426,17 @@ export class StandardGraph extends Graph<t.BaseGraphState, GraphNode> {
     omitOptions?: Set<string>;
   }): t.ChatModelInstance {
     const ChatModelClass = getChatModelClass(provider);
-    const options = omitOptions
-      ? Object.fromEntries(
-        Object.entries(clientOptions ?? this.clientOptions).filter(
-          ([key]) => !omitOptions.has(key)
+    const options =
+      omitOptions && clientOptions == null
+        ? Object.assign(
+          Object.fromEntries(
+            Object.entries(this.clientOptions).filter(
+              ([key]) => !omitOptions.has(key)
+            )
+          ),
+          clientOptions
         )
-      )
-      : (clientOptions ?? this.clientOptions);
+        : (clientOptions ?? this.clientOptions);
     return new ChatModelClass(options);
   }
 

package/src/llm/google/index.ts

@@ -1,15 +1,25 @@
+/* eslint-disable @typescript-eslint/ban-ts-comment */
 import { ChatGoogleGenerativeAI } from '@langchain/google-genai';
 import { getEnvironmentVariable } from '@langchain/core/utils/env';
 import { GoogleGenerativeAI as GenerativeAI } from '@google/generative-ai';
-import type { GoogleGenerativeAIChatInput } from '@langchain/google-genai';
-import type { RequestOptions, SafetySetting } from '@google/generative-ai';
+import type {
+  GenerateContentRequest,
+  SafetySetting,
+} from '@google/generative-ai';
+import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
+import type { BaseMessage, UsageMetadata } from '@langchain/core/messages';
+import type { GeminiGenerationConfig } from '@langchain/google-common';
+import type { ChatGenerationChunk } from '@langchain/core/outputs';
+import type { GeminiApiUsageMetadata } from './types';
+import type { GoogleClientOptions } from '@/types';
+import {
+  convertResponseContentToChatGenerationChunk,
+  convertBaseMessagesToContent,
+} from './utils/common';
 
 export class CustomChatGoogleGenerativeAI extends ChatGoogleGenerativeAI {
-  constructor(
-    fields: GoogleGenerativeAIChatInput & {
-      customHeaders?: RequestOptions['customHeaders'];
-    }
-  ) {
+  thinkingConfig?: GeminiGenerationConfig['thinkingConfig'];
+  constructor(fields: GoogleClientOptions) {
     super(fields);
 
     this.model = fields.model.replace(/^models\//, '');
@@ -66,10 +76,11 @@ export class CustomChatGoogleGenerativeAI extends ChatGoogleGenerativeAI {
       }
     }
 
+    this.thinkingConfig = fields.thinkingConfig ?? this.thinkingConfig;
+
     this.streaming = fields.streaming ?? this.streaming;
     this.json = fields.json;
 
-    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
     // @ts-ignore - Accessing private property from parent class
     this.client = new GenerativeAI(this.apiKey).getGenerativeModel(
       {
@@ -94,4 +105,103 @@ export class CustomChatGoogleGenerativeAI extends ChatGoogleGenerativeAI {
     );
     this.streamUsage = fields.streamUsage ?? this.streamUsage;
   }
+
+  invocationParams(
+    options?: this['ParsedCallOptions']
+  ): Omit<GenerateContentRequest, 'contents'> {
+    const params = super.invocationParams(options);
+    return {
+      ...params,
+      generationConfig: {
+        ...params.generationConfig,
+
+        /** @ts-ignore */
+        thinkingConfig: this.thinkingConfig,
+      },
+    };
+  }
+
+  async *_streamResponseChunks(
+    messages: BaseMessage[],
+    options: this['ParsedCallOptions'],
+    runManager?: CallbackManagerForLLMRun
+  ): AsyncGenerator<ChatGenerationChunk> {
+    const prompt = convertBaseMessagesToContent(
+      messages,
+      this._isMultimodalModel,
+      this.useSystemInstruction
+    );
+    let actualPrompt = prompt;
+    if (prompt[0].role === 'system') {
+      const [systemInstruction] = prompt;
+      /** @ts-ignore */
+      this.client.systemInstruction = systemInstruction;
+      actualPrompt = prompt.slice(1);
+    }
+    const parameters = this.invocationParams(options);
+    const request = {
+      ...parameters,
+      contents: actualPrompt,
+    };
+    const stream = await this.caller.callWithOptions(
+      { signal: options.signal },
+      async () => {
+        /** @ts-ignore */
+        const { stream } = await this.client.generateContentStream(request);
+        return stream;
+      }
+    );
+
+    let usageMetadata: UsageMetadata | undefined;
+    let index = 0;
+    for await (const response of stream) {
+      if (
+        'usageMetadata' in response &&
+        this.streamUsage !== false &&
+        options.streamUsage !== false
+      ) {
+        const genAIUsageMetadata = response.usageMetadata as
+          | GeminiApiUsageMetadata
+          | undefined;
+        const output_tokens =
+          (genAIUsageMetadata?.candidatesTokenCount ?? 0) +
+          (genAIUsageMetadata?.thoughtsTokenCount ?? 0);
+        if (!usageMetadata) {
+          usageMetadata = {
+            input_tokens: genAIUsageMetadata?.promptTokenCount ?? 0,
+            output_tokens,
+            total_tokens: genAIUsageMetadata?.totalTokenCount ?? 0,
+          };
+        } else {
+          // Under the hood, LangChain combines the prompt tokens. Google returns the updated
+          // total each time, so we need to find the difference between the tokens.
+          const outputTokenDiff = output_tokens - usageMetadata.output_tokens;
+          usageMetadata = {
+            input_tokens: 0,
+            output_tokens: outputTokenDiff,
+            total_tokens: outputTokenDiff,
+          };
+        }
+      }
+
+      const chunk = convertResponseContentToChatGenerationChunk(response, {
+        usageMetadata,
+        index,
+      });
+      index += 1;
+      if (!chunk) {
+        continue;
+      }
+
+      yield chunk;
+      await runManager?.handleLLMNewToken(
+        chunk.text || '',
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        { chunk }
+      );
+    }
+  }
 }

package/src/llm/google/types.ts

@@ -0,0 +1,43 @@
+import {
+  CodeExecutionTool,
+  FunctionDeclarationsTool as GoogleGenerativeAIFunctionDeclarationsTool,
+  GoogleSearchRetrievalTool,
+} from '@google/generative-ai';
+import { BindToolsInput } from '@langchain/core/language_models/chat_models';
+
+export type GoogleGenerativeAIToolType =
+  | BindToolsInput
+  | GoogleGenerativeAIFunctionDeclarationsTool
+  | CodeExecutionTool
+  | GoogleSearchRetrievalTool;
+
+/** Enum for content modality types */
+enum Modality {
+  MODALITY_UNSPECIFIED = 'MODALITY_UNSPECIFIED',
+  TEXT = 'TEXT',
+  IMAGE = 'IMAGE',
+  VIDEO = 'VIDEO',
+  AUDIO = 'AUDIO',
+  DOCUMENT = 'DOCUMENT',
+}
+
+/** Interface for modality token count */
+interface ModalityTokenCount {
+  modality: Modality;
+  tokenCount: number;
+}
+
+/** Main interface for Gemini API usage metadata */
+export interface GeminiApiUsageMetadata {
+  promptTokenCount?: number;
+  totalTokenCount?: number;
+  thoughtsTokenCount?: number;
+  candidatesTokenCount?: number;
+  toolUsePromptTokenCount?: number;
+  cachedContentTokenCount?: number;
+  promptTokensDetails: ModalityTokenCount[];
+  candidatesTokensDetails?: ModalityTokenCount[];
+  cacheTokensDetails?: ModalityTokenCount[];
+  toolUsePromptTokensDetails?: ModalityTokenCount[];
+  trafficType?: string;
+}