@librechat/agents 2.4.41 → 2.4.43

package/src/llm/google/utils/tools.ts

@@ -0,0 +1,160 @@
+import {
+  Tool as GenerativeAITool,
+  ToolConfig,
+  FunctionCallingMode,
+  FunctionDeclaration,
+  FunctionDeclarationsTool,
+  FunctionDeclarationSchema,
+} from '@google/generative-ai';
+import { ToolChoice } from '@langchain/core/language_models/chat_models';
+import { StructuredToolInterface } from '@langchain/core/tools';
+import { isLangChainTool } from '@langchain/core/utils/function_calling';
+import {
+  isOpenAITool,
+  ToolDefinition,
+} from '@langchain/core/language_models/base';
+import { convertToGenerativeAITools } from './common';
+import { GoogleGenerativeAIToolType } from '../types';
+import { removeAdditionalProperties } from './zod_to_genai_parameters';
+
+export function convertToolsToGenAI(
+  tools: GoogleGenerativeAIToolType[],
+  extra?: {
+    toolChoice?: ToolChoice;
+    allowedFunctionNames?: string[];
+  }
+): {
+  tools: GenerativeAITool[];
+  toolConfig?: ToolConfig;
+} {
+  // Extract function declaration processing to a separate function
+  const genAITools = processTools(tools);
+
+  // Simplify tool config creation
+  const toolConfig = createToolConfig(genAITools, extra);
+
+  return { tools: genAITools, toolConfig };
+}
+
+function processTools(tools: GoogleGenerativeAIToolType[]): GenerativeAITool[] {
+  let functionDeclarationTools: FunctionDeclaration[] = [];
+  const genAITools: GenerativeAITool[] = [];
+
+  tools.forEach((tool) => {
+    if (isLangChainTool(tool)) {
+      const [convertedTool] = convertToGenerativeAITools([
+        tool as StructuredToolInterface,
+      ]);
+      if (convertedTool.functionDeclarations) {
+        functionDeclarationTools.push(...convertedTool.functionDeclarations);
+      }
+    } else if (isOpenAITool(tool)) {
+      const { functionDeclarations } = convertOpenAIToolToGenAI(tool);
+      if (functionDeclarations) {
+        functionDeclarationTools.push(...functionDeclarations);
+      } else {
+        throw new Error(
+          'Failed to convert OpenAI structured tool to GenerativeAI tool'
+        );
+      }
+    } else {
+      genAITools.push(tool as GenerativeAITool);
+    }
+  });
+
+  const genAIFunctionDeclaration = genAITools.find(
+    (t) => 'functionDeclarations' in t
+  );
+  if (genAIFunctionDeclaration) {
+    return genAITools.map((tool) => {
+      if (
+        functionDeclarationTools.length > 0 &&
+        'functionDeclarations' in tool
+      ) {
+        const newTool = {
+          functionDeclarations: [
+            ...(tool.functionDeclarations || []),
+            ...functionDeclarationTools,
+          ],
+        };
+        // Clear the functionDeclarationTools array so it is not passed again
+        functionDeclarationTools = [];
+        return newTool;
+      }
+      return tool;
+    });
+  }
+
+  return [
+    ...genAITools,
+    ...(functionDeclarationTools.length > 0
+      ? [
+        {
+          functionDeclarations: functionDeclarationTools,
+        },
+      ]
+      : []),
+  ];
+}
+
+function convertOpenAIToolToGenAI(
+  tool: ToolDefinition
+): FunctionDeclarationsTool {
+  return {
+    functionDeclarations: [
+      {
+        name: tool.function.name,
+        description: tool.function.description,
+        parameters: removeAdditionalProperties(
+          tool.function.parameters
+        ) as FunctionDeclarationSchema,
+      },
+    ],
+  };
+}
+
+function createToolConfig(
+  genAITools: GenerativeAITool[],
+  extra?: {
+    toolChoice?: ToolChoice;
+    allowedFunctionNames?: string[];
+  }
+): ToolConfig | undefined {
+  if (!genAITools.length || !extra) return undefined;
+
+  const { toolChoice, allowedFunctionNames } = extra;
+
+  const modeMap: Record<string, FunctionCallingMode> = {
+    any: FunctionCallingMode.ANY,
+    auto: FunctionCallingMode.AUTO,
+    none: FunctionCallingMode.NONE,
+  };
+
+  if (
+    toolChoice != null &&
+    ['any', 'auto', 'none'].includes(toolChoice as string)
+  ) {
+    return {
+      functionCallingConfig: {
+        mode: modeMap[toolChoice as keyof typeof modeMap] ?? 'MODE_UNSPECIFIED',
+        allowedFunctionNames,
+      },
+    };
+  }
+
+  if (typeof toolChoice === 'string' || allowedFunctionNames) {
+    return {
+      functionCallingConfig: {
+        mode: FunctionCallingMode.ANY,
+        allowedFunctionNames: [
+          ...(allowedFunctionNames ?? []),
+          ...(toolChoice != null && typeof toolChoice === 'string'
+            ? [toolChoice]
+            : []),
+        ],
+      },
+    };
+  }
+
+  return undefined;
+}

package/src/llm/google/utils/zod_to_genai_parameters.ts

@@ -0,0 +1,88 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
+
+import {
+  type FunctionDeclarationSchema as GenerativeAIFunctionDeclarationSchema,
+  type SchemaType as FunctionDeclarationSchemaType,
+} from '@google/generative-ai';
+import {
+  InteropZodType,
+  isInteropZodSchema,
+} from '@langchain/core/utils/types';
+import {
+  type JsonSchema7Type,
+  toJsonSchema,
+} from '@langchain/core/utils/json_schema';
+
+export interface GenerativeAIJsonSchema extends Record<string, unknown> {
+  properties?: Record<string, GenerativeAIJsonSchema>;
+  type: FunctionDeclarationSchemaType;
+}
+
+export interface GenerativeAIJsonSchemaDirty extends GenerativeAIJsonSchema {
+  properties?: Record<string, GenerativeAIJsonSchemaDirty>;
+  additionalProperties?: boolean;
+}
+
+export function removeAdditionalProperties(
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  obj: Record<string, any>
+): GenerativeAIJsonSchema {
+  if (typeof obj === 'object' && obj !== null) {
+    const newObj = { ...obj };
+
+    if ('additionalProperties' in newObj) {
+      delete newObj.additionalProperties;
+    }
+    if ('$schema' in newObj) {
+      delete newObj.$schema;
+    }
+    if ('strict' in newObj) {
+      delete newObj.strict;
+    }
+
+    for (const key in newObj) {
+      if (key in newObj) {
+        if (Array.isArray(newObj[key])) {
+          newObj[key] = newObj[key].map(removeAdditionalProperties);
+        } else if (typeof newObj[key] === 'object' && newObj[key] !== null) {
+          newObj[key] = removeAdditionalProperties(newObj[key]);
+        }
+      }
+    }
+
+    return newObj as GenerativeAIJsonSchema;
+  }
+
+  return obj as GenerativeAIJsonSchema;
+}
+
+export function schemaToGenerativeAIParameters<
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  RunOutput extends Record<string, any> = Record<string, any>,
+>(
+  schema: InteropZodType<RunOutput> | JsonSchema7Type
+): GenerativeAIFunctionDeclarationSchema {
+  // GenerativeAI doesn't accept either the $schema or additionalProperties
+  // attributes, so we need to explicitly remove them.
+  const jsonSchema = removeAdditionalProperties(
+    isInteropZodSchema(schema) ? toJsonSchema(schema) : schema
+  );
+  const { $schema, ...rest } = jsonSchema;
+
+  return rest as GenerativeAIFunctionDeclarationSchema;
+}
+
+export function jsonSchemaToGeminiParameters(
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  schema: Record<string, any>
+): GenerativeAIFunctionDeclarationSchema {
+  // Gemini doesn't accept either the $schema or additionalProperties
+  // attributes, so we need to explicitly remove them.
+
+  const jsonSchema = removeAdditionalProperties(
+    schema as GenerativeAIJsonSchemaDirty
+  );
+  const { $schema, ...rest } = jsonSchema;
+
+  return rest as GenerativeAIFunctionDeclarationSchema;
+}

package/src/run.ts

@@ -235,6 +235,7 @@ export class Run<T extends t.BaseGraphState> {
   }
 
   async generateTitle({
+    provider,
     inputText,
     contentParts,
     titlePrompt,
@@ -255,8 +256,9 @@ export class Run<T extends t.BaseGraphState> {
       await convoTemplate.invoke({ input: inputText, output: response })
     ).value;
     const model = this.Graph?.getNewModel({
+      provider,
       clientOptions,
-      omitOriginalOptions: new Set([
+      omitOptions: new Set([
         'clientOptions',
         'streaming',
         'stream',
@@ -270,7 +272,7 @@ export class Run<T extends t.BaseGraphState> {
       return { language: '', title: '' };
     }
     if (
-      isOpenAILike(this.provider) &&
+      isOpenAILike(provider) &&
       (model instanceof ChatOpenAI || model instanceof AzureChatOpenAI)
     ) {
       model.temperature = (clientOptions as t.OpenAIClientOptions | undefined)

package/src/scripts/args.ts

@@ -4,41 +4,45 @@ import { hideBin } from 'yargs/helpers';
 import { llmConfigs } from '@/utils/llmConfig';
 import { Providers } from '@/common';
 
-export async function getArgs(): Promise<{ userName: string; location: string; provider: string; currentDate: string; }> {
+export async function getArgs(): Promise<{
+  userName: string;
+  location: string;
+  provider: Providers;
+  currentDate: string;
+}> {
   const argv = yargs(hideBin(process.argv))
     .option('name', {
       alias: 'n',
       type: 'string',
       description: 'User name',
-      default: 'Jo'
+      default: 'Jo',
     })
     .option('location', {
       alias: 'l',
       type: 'string',
       description: 'User location',
-      default: 'New York'
+      default: 'New York',
     })
     .option('provider', {
       alias: 'p',
       type: 'string',
       description: 'LLM provider',
       choices: Object.keys(llmConfigs),
-      default: Providers.OPENAI
+      default: Providers.OPENAI,
     })
     .help()
-    .alias('help', 'h')
-    .argv;
+    .alias('help', 'h').argv;
 
   const args = await argv;
   const userName = args.name as string;
   const location = args.location as string;
-  const provider = args.provider as string;
+  const provider = args.provider as Providers;
   const currentDate = new Date().toLocaleString();
 
   return {
     userName,
     location,
     provider,
-    currentDate
+    currentDate,
   };
 }

package/src/scripts/code_exec.ts

@@ -6,7 +6,11 @@ import { TavilySearchResults } from '@langchain/community/tools/tavily_search';
 import type { RunnableConfig } from '@langchain/core/runnables';
 import type * as t from '@/types';
 import { ChatModelStreamHandler, createContentAggregator } from '@/stream';
-import { ToolEndHandler, ModelEndHandler, createMetadataAggregator } from '@/events';
+import {
+  ToolEndHandler,
+  ModelEndHandler,
+  createMetadataAggregator,
+} from '@/events';
 import { getLLMConfig } from '@/utils/llmConfig';
 import { getArgs } from '@/scripts/args';
 import { GraphEvents } from '@/common';
@@ -23,38 +27,57 @@ async function testCodeExecution(): Promise<void> {
     [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(),
     [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(),
     [GraphEvents.ON_RUN_STEP_COMPLETED]: {
-      handle: (event: GraphEvents.ON_RUN_STEP_COMPLETED, data: t.StreamEventData): void => {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_COMPLETED,
+        data: t.StreamEventData
+      ): void => {
         console.log('====== ON_RUN_STEP_COMPLETED ======');
         console.dir(data, { depth: null });
-        aggregateContent({ event, data: data as unknown as { result: t.ToolEndEvent } });
-      }
+        aggregateContent({
+          event,
+          data: data as unknown as { result: t.ToolEndEvent },
+        });
+      },
     },
     [GraphEvents.ON_RUN_STEP]: {
-      handle: (event: GraphEvents.ON_RUN_STEP, data: t.StreamEventData): void => {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP,
+        data: t.StreamEventData
+      ): void => {
         console.log('====== ON_RUN_STEP ======');
         console.dir(data, { depth: null });
         aggregateContent({ event, data: data as t.RunStep });
-      }
+      },
     },
     [GraphEvents.ON_RUN_STEP_DELTA]: {
-      handle: (event: GraphEvents.ON_RUN_STEP_DELTA, data: t.StreamEventData): void => {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_DELTA,
+        data: t.StreamEventData
+      ): void => {
         console.log('====== ON_RUN_STEP_DELTA ======');
         console.dir(data, { depth: null });
         aggregateContent({ event, data: data as t.RunStepDeltaEvent });
-      }
+      },
     },
     [GraphEvents.ON_MESSAGE_DELTA]: {
-      handle: (event: GraphEvents.ON_MESSAGE_DELTA, data: t.StreamEventData): void => {
+      handle: (
+        event: GraphEvents.ON_MESSAGE_DELTA,
+        data: t.StreamEventData
+      ): void => {
         console.log('====== ON_MESSAGE_DELTA ======');
         console.dir(data, { depth: null });
         aggregateContent({ event, data: data as t.MessageDeltaEvent });
-      }
+      },
     },
     [GraphEvents.TOOL_START]: {
-      handle: (_event: string, data: t.StreamEventData, metadata?: Record<string, unknown>): void => {
+      handle: (
+        _event: string,
+        data: t.StreamEventData,
+        metadata?: Record<string, unknown>
+      ): void => {
         console.log('====== TOOL_START ======');
         console.dir(data, { depth: null });
-      }
+      },
     },
   };
 
@@ -66,14 +89,19 @@ async function testCodeExecution(): Promise<void> {
       type: 'standard',
       llmConfig,
       tools: [new TavilySearchResults(), createCodeExecutionTool()],
-      instructions: 'You are a friendly AI assistant with coding capabilities. Always address the user by their name.',
+      instructions:
+        'You are a friendly AI assistant with coding capabilities. Always address the user by their name.',
       additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
     },
     returnContent: true,
     customHandlers,
   });
 
-  const config: Partial<RunnableConfig> & { version: 'v1' | 'v2'; run_id?: string; streamMode: string } = {
+  const config: Partial<RunnableConfig> & {
+    version: 'v1' | 'v2';
+    run_id?: string;
+    streamMode: string;
+  } = {
     configurable: {
       provider,
       thread_id: 'conversation-num-1',
@@ -152,12 +180,15 @@ async function testCodeExecution(): Promise<void> {
 
   const { handleLLMEnd, collected } = createMetadataAggregator();
   const titleResult = await run.generateTitle({
+    provider,
     inputText: userMessage2,
     contentParts,
     chainOptions: {
-      callbacks: [{
-        handleLLMEnd,
-      }],
+      callbacks: [
+        {
+          handleLLMEnd,
+        },
+      ],
     },
   });
   console.log('Generated Title:', titleResult);
@@ -180,4 +211,4 @@ testCodeExecution().catch((err) => {
   console.log('Conversation history:');
   console.dir(conversationHistory, { depth: null });
   process.exit(1);
-});
+});

package/src/scripts/code_exec_files.ts

@@ -5,7 +5,11 @@ import { HumanMessage, BaseMessage } from '@langchain/core/messages';
 import type { RunnableConfig } from '@langchain/core/runnables';
 import type * as t from '@/types';
 import { ChatModelStreamHandler, createContentAggregator } from '@/stream';
-import { ToolEndHandler, ModelEndHandler, createMetadataAggregator } from '@/events';
+import {
+  ToolEndHandler,
+  ModelEndHandler,
+  createMetadataAggregator,
+} from '@/events';
 import { getLLMConfig } from '@/utils/llmConfig';
 import { getArgs } from '@/scripts/args';
 import { GraphEvents } from '@/common';
@@ -22,38 +26,57 @@ async function testCodeExecution(): Promise<void> {
     [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(),
     [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(),
     [GraphEvents.ON_RUN_STEP_COMPLETED]: {
-      handle: (event: GraphEvents.ON_RUN_STEP_COMPLETED, data: t.StreamEventData): void => {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_COMPLETED,
+        data: t.StreamEventData
+      ): void => {
         console.log('====== ON_RUN_STEP_COMPLETED ======');
         console.dir(data, { depth: null });
-        aggregateContent({ event, data: data as unknown as { result: t.ToolEndEvent } });
-      }
+        aggregateContent({
+          event,
+          data: data as unknown as { result: t.ToolEndEvent },
+        });
+      },
     },
     [GraphEvents.ON_RUN_STEP]: {
-      handle: (event: GraphEvents.ON_RUN_STEP, data: t.StreamEventData): void => {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP,
+        data: t.StreamEventData
+      ): void => {
         console.log('====== ON_RUN_STEP ======');
         console.dir(data, { depth: null });
         aggregateContent({ event, data: data as t.RunStep });
-      }
+      },
     },
     [GraphEvents.ON_RUN_STEP_DELTA]: {
-      handle: (event: GraphEvents.ON_RUN_STEP_DELTA, data: t.StreamEventData): void => {
+      handle: (
+        event: GraphEvents.ON_RUN_STEP_DELTA,
+        data: t.StreamEventData
+      ): void => {
         console.log('====== ON_RUN_STEP_DELTA ======');
         console.dir(data, { depth: null });
         aggregateContent({ event, data: data as t.RunStepDeltaEvent });
-      }
+      },
     },
     [GraphEvents.ON_MESSAGE_DELTA]: {
-      handle: (event: GraphEvents.ON_MESSAGE_DELTA, data: t.StreamEventData): void => {
+      handle: (
+        event: GraphEvents.ON_MESSAGE_DELTA,
+        data: t.StreamEventData
+      ): void => {
         console.log('====== ON_MESSAGE_DELTA ======');
         console.dir(data, { depth: null });
         aggregateContent({ event, data: data as t.MessageDeltaEvent });
-      }
+      },
     },
     [GraphEvents.TOOL_START]: {
-      handle: (_event: string, data: t.StreamEventData, metadata?: Record<string, unknown>): void => {
+      handle: (
+        _event: string,
+        data: t.StreamEventData,
+        metadata?: Record<string, unknown>
+      ): void => {
         console.log('====== TOOL_START ======');
         console.dir(data, { depth: null });
-      }
+      },
     },
   };
 
@@ -65,14 +88,19 @@ async function testCodeExecution(): Promise<void> {
       type: 'standard',
       llmConfig,
       tools: [createCodeExecutionTool()],
-      instructions: 'You are a friendly AI assistant with coding capabilities. Always address the user by their name.',
+      instructions:
+        'You are a friendly AI assistant with coding capabilities. Always address the user by their name.',
       additional_instructions: `The user's name is ${userName} and they are located in ${location}. The current date is ${currentDate}.`,
     },
     returnContent: true,
     customHandlers,
   });
 
-  const config: Partial<RunnableConfig> & { version: 'v1' | 'v2'; run_id?: string; streamMode: string } = {
+  const config: Partial<RunnableConfig> & {
+    version: 'v1' | 'v2';
+    run_id?: string;
+    streamMode: string;
+  } = {
     configurable: {
       provider,
       thread_id: 'conversation-num-1',
@@ -131,12 +159,15 @@ async function testCodeExecution(): Promise<void> {
 
   const { handleLLMEnd, collected } = createMetadataAggregator();
   const titleResult = await run.generateTitle({
+    provider,
     inputText: userMessage2,
     contentParts,
     chainOptions: {
-      callbacks: [{
-        handleLLMEnd,
-      }],
+      callbacks: [
+        {
+          handleLLMEnd,
+        },
+      ],
     },
   });
   console.log('Generated Title:', titleResult);