@librechat/agents 2.4.49 → 2.4.51

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@librechat/agents",
-  "version": "2.4.49",
+  "version": "2.4.51",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/esm/main.mjs",
   "types": "./dist/types/index.d.ts",
@@ -75,14 +75,14 @@
     "@langchain/anthropic": "^0.3.23",
     "@langchain/aws": "^0.1.11",
     "@langchain/community": "^0.3.47",
-    "@langchain/core": "^0.3.60",
+    "@langchain/core": "^0.3.62",
     "@langchain/deepseek": "^0.0.2",
     "@langchain/google-genai": "^0.2.13",
     "@langchain/google-vertexai": "^0.2.13",
     "@langchain/langgraph": "^0.3.4",
     "@langchain/mistralai": "^0.2.1",
     "@langchain/ollama": "^0.2.3",
-    "@langchain/openai": "^0.5.14",
+    "@langchain/openai": "^0.5.16",
     "@langchain/xai": "^0.0.3",
     "cheerio": "^1.0.0",
     "dotenv": "^16.4.7",

package/src/graphs/Graph.ts

@@ -135,7 +135,7 @@ export class StandardGraph extends Graph<t.BaseGraphState, GraphNode> {
   systemMessage: SystemMessage | undefined;
   messages: BaseMessage[] = [];
   runId: string | undefined;
-  tools?: t.GenericTool[];
+  tools?: t.GraphTools;
   toolMap?: t.ToolMap;
   startIndex: number = 0;
   provider: Providers;
@@ -354,7 +354,7 @@ export class StandardGraph extends Graph<t.BaseGraphState, GraphNode> {
     | ToolNode<t.BaseGraphState> {
     // return new ToolNode<t.BaseGraphState>(this.tools);
     return new CustomToolNode<t.BaseGraphState>({
-      tools: this.tools || [],
+      tools: (this.tools as t.GenericTool[] | undefined) || [],
       toolMap: this.toolMap,
       toolCallStepIds: this.toolCallStepIds,
       errorHandler: (data, metadata) =>

package/src/llm/google/index.ts

@@ -133,7 +133,7 @@ export class CustomChatGoogleGenerativeAI extends ChatGoogleGenerativeAI {
       this.useSystemInstruction
     );
     let actualPrompt = prompt;
-    if (prompt[0].role === 'system') {
+    if (prompt?.[0].role === 'system') {
       const [systemInstruction] = prompt;
       /** @ts-ignore */
       this.client.systemInstruction = systemInstruction;

package/src/llm/google/utils/common.ts

@@ -1,14 +1,15 @@
 import {
-  EnhancedGenerateContentResponse,
-  Content,
-  Part,
-  type FunctionDeclarationsTool as GoogleGenerativeAIFunctionDeclarationsTool,
-  type FunctionDeclaration as GenerativeAIFunctionDeclaration,
   POSSIBLE_ROLES,
-  FunctionCallPart,
-  TextPart,
-  FileDataPart,
-  InlineDataPart,
+  type Part,
+  type Content,
+  type TextPart,
+  type FileDataPart,
+  type InlineDataPart,
+  type FunctionCallPart,
+  type GenerateContentCandidate,
+  type EnhancedGenerateContentResponse,
+  type FunctionDeclaration as GenerativeAIFunctionDeclaration,
+  type FunctionDeclarationsTool as GoogleGenerativeAIFunctionDeclarationsTool,
 } from '@google/generative-ai';
 import {
   AIMessageChunk,
@@ -412,9 +413,9 @@ export function convertBaseMessagesToContent(
   messages: BaseMessage[],
   isMultimodalModel: boolean,
   convertSystemMessageToHumanContent: boolean = false
-): Content[] {
+): Content[] | undefined {
   return messages.reduce<{
-    content: Content[];
+    content: Content[] | undefined;
     mergeWithPreviousContent: boolean;
   }>(
     (acc, message, index) => {
@@ -427,7 +428,7 @@ export function convertBaseMessagesToContent(
       }
       const role = convertAuthorToRole(author);
 
-      const prevContent = acc.content[acc.content.length];
+      const prevContent = acc.content?.[acc.content.length];
       if (
         !acc.mergeWithPreviousContent &&
         prevContent &&
@@ -445,7 +446,7 @@ export function convertBaseMessagesToContent(
       );
 
       if (acc.mergeWithPreviousContent) {
-        const prevContent = acc.content[acc.content.length - 1];
+        const prevContent = acc.content?.[acc.content.length - 1];
         if (!prevContent) {
           throw new Error(
             'There was a problem parsing your system message. Please try a prompt without one.'
@@ -473,7 +474,7 @@ export function convertBaseMessagesToContent(
       return {
         mergeWithPreviousContent:
           author === 'system' && !convertSystemMessageToHumanContent,
-        content: [...acc.content, content],
+        content: [...(acc.content ?? []), content],
       };
     },
     { content: [], mergeWithPreviousContent: false }
@@ -491,12 +492,15 @@ export function convertResponseContentToChatGenerationChunk(
     return null;
   }
   const functionCalls = response.functionCalls();
-  const [candidate] = response.candidates;
-  const { content: candidateContent, ...generationInfo } = candidate;
+  const [candidate] = response.candidates as [
+    Partial<GenerateContentCandidate> | undefined,
+  ];
+  const { content: candidateContent, ...generationInfo } = candidate ?? {};
   let content: MessageContent | undefined;
   // Checks if some parts do not have text. If false, it means that the content is a string.
   const reasoningParts: string[] = [];
   if (
+    candidateContent != null &&
     Array.isArray(candidateContent.parts) &&
     candidateContent.parts.every((p) => 'text' in p)
   ) {
@@ -510,7 +514,7 @@ export function convertResponseContentToChatGenerationChunk(
       textParts.push(part.text ?? '');
     }
     content = textParts.join('');
-  } else if (Array.isArray(candidateContent.parts)) {
+  } else if (candidateContent && Array.isArray(candidateContent.parts)) {
     content = candidateContent.parts.map((p) => {
       if ('text' in p && 'thought' in p && p.thought === true) {
         reasoningParts.push(p.text ?? '');
@@ -565,10 +569,14 @@ export function convertResponseContentToChatGenerationChunk(
     additional_kwargs.reasoning = reasoningParts.join('');
   }
 
+  if (candidate?.groundingMetadata) {
+    additional_kwargs.groundingMetadata = candidate.groundingMetadata;
+  }
+
   return new ChatGenerationChunk({
     text,
     message: new AIMessageChunk({
-      content: content || '',
+      content: content,
       name: !candidateContent ? undefined : candidateContent.role,
       tool_call_chunks: toolCallChunks,
       // Each chunk can have unique "generationInfo", and merging strategy is unclear,

package/src/llm/openai/index.ts

@@ -1,11 +1,7 @@
 import { AzureOpenAI as AzureOpenAIClient } from 'openai';
 import { ChatXAI as OriginalChatXAI } from '@langchain/xai';
 import { ChatDeepSeek as OriginalChatDeepSeek } from '@langchain/deepseek';
-import {
-  isOpenAITool,
-  ToolDefinition,
-  BaseLanguageModelInput,
-} from '@langchain/core/language_models/base';
+import { ToolDefinition } from '@langchain/core/language_models/base';
 import {
   getEndpoint,
   OpenAIClient,
@@ -18,8 +14,7 @@ import { isLangChainTool } from '@langchain/core/utils/function_calling';
 import { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
 import type { BindToolsInput } from '@langchain/core/language_models/chat_models';
 import type { OpenAIEndpointConfig } from '@langchain/openai/dist/utils/azure';
-import type { AIMessageChunk, BaseMessage } from '@langchain/core/messages';
-import type { Runnable } from '@langchain/core/runnables';
+import type { BaseMessage } from '@langchain/core/messages';
 import type * as t from '@langchain/openai';
 import {
   _convertMessagesToOpenAIResponsesParams,
@@ -27,14 +22,6 @@ import {
   type ResponseReturnStreamEvents,
 } from './utils';
 
-type ResponsesCreateParams = Parameters<OpenAIClient.Responses['create']>[0];
-type ResponsesTool = Exclude<ResponsesCreateParams['tools'], undefined>[number];
-
-type ChatOpenAIToolType =
-  | BindToolsInput
-  | OpenAIClient.ChatCompletionTool
-  | ResponsesTool;
-
 type HeaderValue = string | undefined | null;
 export type HeadersLike =
   | Headers
@@ -133,29 +120,6 @@ export function _convertToOpenAITool(
 
   return toolDef;
 }
-
-function _convertChatOpenAIToolTypeToOpenAITool(
-  tool: ChatOpenAIToolType,
-  fields?: {
-    strict?: boolean;
-  }
-): OpenAIClient.ChatCompletionTool {
-  if (isOpenAITool(tool)) {
-    if (fields?.strict !== undefined) {
-      return {
-        ...tool,
-        function: {
-          ...tool.function,
-          strict: fields.strict,
-        },
-      };
-    }
-
-    return tool;
-  }
-  return _convertToOpenAITool(tool, fields);
-}
-
 export class CustomOpenAIClient extends OpenAIClient {
   abortHandler?: () => void;
   async fetchWithTimeout(
@@ -227,33 +191,10 @@ export class CustomAzureOpenAIClient extends AzureOpenAIClient {
   }
 }
 
-function isBuiltInTool(tool: ChatOpenAIToolType): tool is ResponsesTool {
-  return 'type' in tool && tool.type !== 'function';
-}
-
 export class ChatOpenAI extends OriginalChatOpenAI<t.ChatOpenAICallOptions> {
   public get exposedClient(): CustomOpenAIClient {
     return this.client;
   }
-  override bindTools(
-    tools: ChatOpenAIToolType[],
-    kwargs?: Partial<t.ChatOpenAICallOptions>
-  ): Runnable<BaseLanguageModelInput, AIMessageChunk, t.ChatOpenAICallOptions> {
-    let strict: boolean | undefined;
-    if (kwargs?.strict !== undefined) {
-      strict = kwargs.strict;
-    } else if (this.supportsStrictToolCalling !== undefined) {
-      strict = this.supportsStrictToolCalling;
-    }
-    return this.withConfig({
-      tools: tools.map((tool) =>
-        isBuiltInTool(tool)
-          ? tool
-          : _convertChatOpenAIToolTypeToOpenAITool(tool, { strict })
-      ),
-      ...kwargs,
-    } as Partial<t.ChatOpenAICallOptions>);
-  }
   protected _getClientOptions(
     options?: OpenAICoreRequestOptions
   ): OpenAICoreRequestOptions {
@@ -324,25 +265,6 @@ export class ChatOpenAI extends OriginalChatOpenAI<t.ChatOpenAICallOptions> {
 }
 
 export class AzureChatOpenAI extends OriginalAzureChatOpenAI {
-  override bindTools(
-    tools: ChatOpenAIToolType[],
-    kwargs?: Partial<t.ChatOpenAICallOptions>
-  ): Runnable<BaseLanguageModelInput, AIMessageChunk, t.ChatOpenAICallOptions> {
-    let strict: boolean | undefined;
-    if (kwargs?.strict !== undefined) {
-      strict = kwargs.strict;
-    } else if (this.supportsStrictToolCalling !== undefined) {
-      strict = this.supportsStrictToolCalling;
-    }
-    return this.withConfig({
-      tools: tools.map((tool) =>
-        isBuiltInTool(tool)
-          ? tool
-          : _convertChatOpenAIToolTypeToOpenAITool(tool, { strict })
-      ),
-      ...kwargs,
-    } as Partial<t.ChatOpenAICallOptions>);
-  }
   public get exposedClient(): CustomOpenAIClient {
     return this.client;
   }

package/src/run.ts

@@ -1,8 +1,8 @@
 // src/run.ts
 import { zodToJsonSchema } from 'zod-to-json-schema';
 import { PromptTemplate } from '@langchain/core/prompts';
-import { AzureChatOpenAI, ChatOpenAI } from '@langchain/openai';
 import { SystemMessage } from '@langchain/core/messages';
+import { AzureChatOpenAI, ChatOpenAI } from '@langchain/openai';
 import type {
   BaseMessage,
   MessageContentComplex,
@@ -147,15 +147,18 @@ export class Run<T extends t.BaseGraphState> {
       (streamOptions?.indexTokenCountMap
         ? await createTokenCounter()
         : undefined);
+    const tools = this.Graph.tools as
+      | Array<t.GenericTool | undefined>
+      | undefined;
     const toolTokens = tokenCounter
-      ? (this.Graph.tools?.reduce((acc, tool) => {
+      ? (tools?.reduce((acc, tool) => {
         if (!(tool as Partial<t.GenericTool>).schema) {
           return acc;
         }
 
         const jsonSchema = zodToJsonSchema(
-          (tool.schema as t.ZodObjectAny).describe(tool.description ?? ''),
-          tool.name
+          (tool?.schema as t.ZodObjectAny).describe(tool?.description ?? ''),
+          tool?.name ?? ''
         );
         return (
           acc + tokenCounter(new SystemMessage(JSON.stringify(jsonSchema)))

package/src/specs/anthropic.simple.test.ts

@@ -25,7 +25,7 @@ import { Run } from '@/run';
 
 const provider = Providers.ANTHROPIC;
 describe(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
-  jest.setTimeout(30000);
+  jest.setTimeout(60000);
   let run: Run<t.IState>;
   let runningHistory: BaseMessage[];
   let collectedUsage: UsageMetadata[];

package/src/types/graph.ts

@@ -1,7 +1,13 @@
 // src/types/graph.ts
-import type { StateGraphArgs, StateGraph, CompiledStateGraph } from '@langchain/langgraph';
+import type {
+  StateGraphArgs,
+  StateGraph,
+  CompiledStateGraph,
+} from '@langchain/langgraph';
+import type { BindToolsInput } from '@langchain/core/language_models/chat_models';
 import type { BaseMessage, AIMessageChunk } from '@langchain/core/messages';
 import type { ChatGenerationChunk } from '@langchain/core/outputs';
+import type { GoogleAIToolType } from '@langchain/google-common';
 import type { RunnableConfig } from '@langchain/core/runnables';
 import type { ToolMap, GenericTool } from '@/types/tools';
 import type { ClientOptions } from '@/types/llm';
@@ -22,56 +28,78 @@ export type IState = BaseGraphState;
 // }
 
 export interface EventHandler {
-  handle(event: string, data: StreamEventData | ModelEndData, metadata?: Record<string, unknown>, graph?: Graph): void;
+  handle(
+    event: string,
+    data: StreamEventData | ModelEndData,
+    metadata?: Record<string, unknown>,
+    graph?: Graph
+  ): void;
 }
 
-export type GraphStateChannels<T extends BaseGraphState> = StateGraphArgs<T>['channels'];
+export type GraphStateChannels<T extends BaseGraphState> =
+  StateGraphArgs<T>['channels'];
 
-export type Workflow<T extends BaseGraphState = BaseGraphState, U extends Partial<T> = Partial<T>, N extends string = string> = StateGraph<T, U, N>;
+export type Workflow<
+  T extends BaseGraphState = BaseGraphState,
+  U extends Partial<T> = Partial<T>,
+  N extends string = string,
+> = StateGraph<T, U, N>;
 
-export type CompiledWorkflow<T extends BaseGraphState = BaseGraphState, U extends Partial<T> = Partial<T>, N extends string = string> = CompiledStateGraph<T, U, N>;
+export type CompiledWorkflow<
+  T extends BaseGraphState = BaseGraphState,
+  U extends Partial<T> = Partial<T>,
+  N extends string = string,
+> = CompiledStateGraph<T, U, N>;
 
-export type EventStreamCallbackHandlerInput = Parameters<CompiledWorkflow['streamEvents']>[2] extends Omit<infer T, 'autoClose'> ? T : never;
+export type EventStreamCallbackHandlerInput =
+  Parameters<CompiledWorkflow['streamEvents']>[2] extends Omit<
+    infer T,
+    'autoClose'
+  >
+    ? T
+    : never;
 
-export type StreamChunk = ChatGenerationChunk & {
-  message: AIMessageChunk;
-} | AIMessageChunk;
+export type StreamChunk =
+  | (ChatGenerationChunk & {
+      message: AIMessageChunk;
+    })
+  | AIMessageChunk;
 
 /**
  * Data associated with a StreamEvent.
  */
 export type StreamEventData = {
-    /**
-     * The input passed to the runnable that generated the event.
-     * Inputs will sometimes be available at the *START* of the runnable, and
-     * sometimes at the *END* of the runnable.
-     * If a runnable is able to stream its inputs, then its input by definition
-     * won't be known until the *END* of the runnable when it has finished streaming
-     * its inputs.
-     */
-    input?: unknown;
-    /**
-     * The output of the runnable that generated the event.
-     * Outputs will only be available at the *END* of the runnable.
-     * For most runnables, this field can be inferred from the `chunk` field,
-     * though there might be some exceptions for special cased runnables (e.g., like
-     * chat models), which may return more information.
-     */
-    output?: unknown;
-    /**
-     * A streaming chunk from the output that generated the event.
-     * chunks support addition in general, and adding them up should result
-     * in the output of the runnable that generated the event.
-     */
-    chunk?: StreamChunk;
-    /**
-     * Runnable config for invoking other runnables within handlers.
-     */
-    config?: RunnableConfig;
-    /**
-     * Custom result from the runnable that generated the event.
-     */
-    result?: unknown;
+  /**
+   * The input passed to the runnable that generated the event.
+   * Inputs will sometimes be available at the *START* of the runnable, and
+   * sometimes at the *END* of the runnable.
+   * If a runnable is able to stream its inputs, then its input by definition
+   * won't be known until the *END* of the runnable when it has finished streaming
+   * its inputs.
+   */
+  input?: unknown;
+  /**
+   * The output of the runnable that generated the event.
+   * Outputs will only be available at the *END* of the runnable.
+   * For most runnables, this field can be inferred from the `chunk` field,
+   * though there might be some exceptions for special cased runnables (e.g., like
+   * chat models), which may return more information.
+   */
+  output?: unknown;
+  /**
+   * A streaming chunk from the output that generated the event.
+   * chunks support addition in general, and adding them up should result
+   * in the output of the runnable that generated the event.
+   */
+  chunk?: StreamChunk;
+  /**
+   * Runnable config for invoking other runnables within handlers.
+   */
+  config?: RunnableConfig;
+  /**
+   * Custom result from the runnable that generated the event.
+   */
+  result?: unknown;
 };
 
 /**
@@ -80,54 +108,54 @@ export type StreamEventData = {
  * Schema of a streaming event which is produced from the streamEvents method.
  */
 export type StreamEvent = {
-    /**
-     * Event names are of the format: on_[runnable_type]_(start|stream|end).
-     *
-     * Runnable types are one of:
-     * - llm - used by non chat models
-     * - chat_model - used by chat models
-     * - prompt --  e.g., ChatPromptTemplate
-     * - tool -- LangChain tools
-     * - chain - most Runnables are of this type
-     *
-     * Further, the events are categorized as one of:
-     * - start - when the runnable starts
-     * - stream - when the runnable is streaming
-     * - end - when the runnable ends
-     *
-     * start, stream and end are associated with slightly different `data` payload.
-     *
-     * Please see the documentation for `EventData` for more details.
-     */
-    event: string;
-    /** The name of the runnable that generated the event. */
-    name: string;
-    /**
-     * An randomly generated ID to keep track of the execution of the given runnable.
-     *
-     * Each child runnable that gets invoked as part of the execution of a parent runnable
-     * is assigned its own unique ID.
-     */
-    run_id: string;
-    /**
-     * Tags associated with the runnable that generated this event.
-     * Tags are always inherited from parent runnables.
-     */
-    tags?: string[];
-    /** Metadata associated with the runnable that generated this event. */
-    metadata: Record<string, unknown>;
-    /**
-     * Event data.
-     *
-     * The contents of the event data depend on the event type.
-     */
-    data: StreamEventData;
+  /**
+   * Event names are of the format: on_[runnable_type]_(start|stream|end).
+   *
+   * Runnable types are one of:
+   * - llm - used by non chat models
+   * - chat_model - used by chat models
+   * - prompt --  e.g., ChatPromptTemplate
+   * - tool -- LangChain tools
+   * - chain - most Runnables are of this type
+   *
+   * Further, the events are categorized as one of:
+   * - start - when the runnable starts
+   * - stream - when the runnable is streaming
+   * - end - when the runnable ends
+   *
+   * start, stream and end are associated with slightly different `data` payload.
+   *
+   * Please see the documentation for `EventData` for more details.
+   */
+  event: string;
+  /** The name of the runnable that generated the event. */
+  name: string;
+  /**
+   * An randomly generated ID to keep track of the execution of the given runnable.
+   *
+   * Each child runnable that gets invoked as part of the execution of a parent runnable
+   * is assigned its own unique ID.
+   */
+  run_id: string;
+  /**
+   * Tags associated with the runnable that generated this event.
+   * Tags are always inherited from parent runnables.
+   */
+  tags?: string[];
+  /** Metadata associated with the runnable that generated this event. */
+  metadata: Record<string, unknown>;
+  /**
+   * Event data.
+   *
+   * The contents of the event data depend on the event type.
+   */
+  data: StreamEventData;
 };
 
 export type GraphConfig = {
-    provider: string;
-    thread_id?: string;
-    run_id?: string;
+  provider: string;
+  thread_id?: string;
+  run_id?: string;
 };
 
 export type PartMetadata = {
@@ -138,8 +166,10 @@ export type PartMetadata = {
   output?: string;
 };
 
-export type ModelEndData = StreamEventData & { output: AIMessageChunk | undefined } | undefined;
-
+export type ModelEndData =
+  | (StreamEventData & { output: AIMessageChunk | undefined })
+  | undefined;
+export type GraphTools = GenericTool[] | BindToolsInput[] | GoogleAIToolType[];
 export type StandardGraphInput = {
   runId?: string;
   toolEnd?: boolean;
@@ -148,8 +178,8 @@ export type StandardGraphInput = {
   signal?: AbortSignal;
   instructions?: string;
   streamBuffer?: number;
-  tools?: GenericTool[];
   clientOptions: ClientOptions;
   additional_instructions?: string;
   reasoningKey?: 'reasoning_content' | 'reasoning';
-};
+  tools?: GraphTools;
+};

package/src/types/tools.ts

@@ -1,6 +1,6 @@
 // src/types/tools.ts
-import type { RunnableToolLike } from '@langchain/core/runnables';
 import type { StructuredToolInterface } from '@langchain/core/tools';
+import type { RunnableToolLike } from '@langchain/core/runnables';
 import type { ToolCall } from '@langchain/core/messages/tool';
 import type { ToolErrorData } from './stream';
 import { EnvVar } from '@/common';
@@ -13,9 +13,10 @@ export type CustomToolCall = {
   id?: string;
   type?: 'tool_call';
   output?: string;
-}
+};
 
 export type GenericTool = StructuredToolInterface | RunnableToolLike;
+
 export type ToolMap = Map<string, GenericTool>;
 export type ToolRefs = {
   tools: GenericTool[];
@@ -30,7 +31,10 @@ export type ToolNodeOptions = {
   handleToolErrors?: boolean;
   loadRuntimeTools?: ToolRefGenerator;
   toolCallStepIds?: Map<string, string>;
-  errorHandler?: (data: ToolErrorData, metadata?: Record<string, unknown>) => void
+  errorHandler?: (
+    data: ToolErrorData,
+    metadata?: Record<string, unknown>
+  ) => void;
 };
 
 export type ToolNodeConstructorParams = ToolRefs & ToolNodeOptions;
@@ -50,13 +54,15 @@ export type CodeEnvFile = {
   session_id: string;
 };
 
-export type CodeExecutionToolParams = undefined | {
-  session_id?: string;
-  user_id?: string;
-  apiKey?: string;
-  files?: CodeEnvFile[];
-  [EnvVar.CODE_API_KEY]?: string;
-}
+export type CodeExecutionToolParams =
+  | undefined
+  | {
+      session_id?: string;
+      user_id?: string;
+      apiKey?: string;
+      files?: CodeEnvFile[];
+      [EnvVar.CODE_API_KEY]?: string;
+    };
 
 export type FileRef = {
   id: string;