@langchain/google-genai 0.2.9 → 0.2.11

package/README.md

@@ -50,6 +50,7 @@ Then initialize
 
 ```typescript
 import { ChatGoogleGenerativeAI } from "@langchain/google-genai";
+import { HumanMessage } from "@langchain/core/messages";
 
 const model = new ChatGoogleGenerativeAI({
   modelName: "gemini-pro",

package/dist/chat_models.cjs

@@ -27,13 +27,12 @@ const tools_js_1 = require("./utils/tools.cjs");
  * ## [Runtime args](https://api.js.langchain.com/interfaces/langchain_google_genai.GoogleGenerativeAIChatCallOptions.html)
  *
  * Runtime args can be passed as the second argument to any of the base runnable methods `.invoke`. `.stream`, `.batch`, etc.
- * They can also be passed via `.bind`, or the second arg in `.bindTools`, like shown in the examples below:
+ * They can also be passed via `.withConfig`, or the second arg in `.bindTools`, like shown in the examples below:
  *
  * ```typescript
- * // When calling `.bind`, call options should be passed via the first argument
- * const llmWithArgsBound = llm.bind({
+ * // When calling `.withConfig`, call options should be passed via the first argument
+ * const llmWithArgsBound = llm.withConfig({
  *   stop: ["\n"],
- *   tools: [...],
  * });
  *
  * // When calling `.bindTools`, call options should be passed via the second argument
@@ -598,7 +597,10 @@ class ChatGoogleGenerativeAI extends chat_models_1.BaseChatModel {
         return "googlegenerativeai";
     }
     bindTools(tools, kwargs) {
-        return this.bind({ tools: (0, tools_js_1.convertToolsToGenAI)(tools)?.tools, ...kwargs });
+        return this.withConfig({
+            tools: (0, tools_js_1.convertToolsToGenAI)(tools)?.tools,
+            ...kwargs,
+        });
     }
     invocationParams(options) {
         const toolsAndConfig = options?.tools?.length
@@ -668,7 +670,10 @@ class ChatGoogleGenerativeAI extends chat_models_1.BaseChatModel {
         const generationResult = (0, common_js_1.mapGenerateContentResultToChatResult)(res.response, {
             usageMetadata,
         });
-        await runManager?.handleLLMNewToken(generationResult.generations[0].text ?? "");
+        // may not have generations in output if there was a refusal for safety reasons, malformed function call, etc.
+        if (generationResult.generations?.length > 0) {
+            await runManager?.handleLLMNewToken(generationResult.generations[0]?.text ?? "");
+        }
         return generationResult;
     }
     async *_streamResponseChunks(messages, options, runManager) {
@@ -755,7 +760,7 @@ class ChatGoogleGenerativeAI extends chat_models_1.BaseChatModel {
         if (method === "functionCalling") {
             let functionName = name ?? "extract";
             let tools;
-            if ((0, types_1.isZodSchema)(schema)) {
+            if ((0, types_1.isInteropZodSchema)(schema)) {
                 const jsonSchema = (0, zod_to_genai_parameters_js_1.schemaToGenerativeAIParameters)(schema);
                 tools = [
                     {
@@ -800,14 +805,13 @@ class ChatGoogleGenerativeAI extends chat_models_1.BaseChatModel {
                     keyName: functionName,
                 });
             }
-            llm = this.bind({
-                tools,
-                tool_choice: functionName,
+            llm = this.bindTools(tools).withConfig({
+                allowedFunctionNames: [functionName],
             });
         }
         else {
             const jsonSchema = (0, zod_to_genai_parameters_js_1.schemaToGenerativeAIParameters)(schema);
-            llm = this.bind({
+            llm = this.withConfig({
                 responseSchema: jsonSchema,
             });
             outputParser = new output_parsers_1.JsonOutputParser();

package/dist/chat_models.d.ts

@@ -5,7 +5,7 @@ import { ChatGenerationChunk, ChatResult } from "@langchain/core/outputs";
 import { BaseChatModel, type BaseChatModelCallOptions, type LangSmithParams, type BaseChatModelParams } from "@langchain/core/language_models/chat_models";
 import { BaseLanguageModelInput, StructuredOutputMethodOptions } from "@langchain/core/language_models/base";
 import { Runnable } from "@langchain/core/runnables";
-import type { z } from "zod";
+import { InteropZodType } from "@langchain/core/utils/types";
 import { GoogleGenerativeAIToolType } from "./types.js";
 export type BaseMessageExamplePair = {
     input: BaseMessage;
@@ -71,7 +71,7 @@ export interface GoogleGenerativeAIChatInput extends BaseChatModelParams, Pick<G
      * Top-k changes how the model selects tokens for output.
      *
      * A top-k of 1 means the selected token is the most probable among
-     * all tokens in the model’s vocabulary (also called greedy decoding),
+     * all tokens in the model's vocabulary (also called greedy decoding),
      * while a top-k of 3 means that the next token is selected from
      * among the 3 most probable tokens (using temperature).
      *
@@ -139,13 +139,12 @@ export interface GoogleGenerativeAIChatInput extends BaseChatModelParams, Pick<G
  * ## [Runtime args](https://api.js.langchain.com/interfaces/langchain_google_genai.GoogleGenerativeAIChatCallOptions.html)
  *
  * Runtime args can be passed as the second argument to any of the base runnable methods `.invoke`. `.stream`, `.batch`, etc.
- * They can also be passed via `.bind`, or the second arg in `.bindTools`, like shown in the examples below:
+ * They can also be passed via `.withConfig`, or the second arg in `.bindTools`, like shown in the examples below:
  *
  * ```typescript
- * // When calling `.bind`, call options should be passed via the first argument
- * const llmWithArgsBound = llm.bind({
+ * // When calling `.withConfig`, call options should be passed via the first argument
+ * const llmWithArgsBound = llm.withConfig({
  *   stop: ["\n"],
- *   tools: [...],
  * });
  *
  * // When calling `.bindTools`, call options should be passed via the second argument
@@ -535,8 +534,8 @@ export declare class ChatGoogleGenerativeAI extends BaseChatModel<GoogleGenerati
     _generate(messages: BaseMessage[], options: this["ParsedCallOptions"], runManager?: CallbackManagerForLLMRun): Promise<ChatResult>;
     _streamResponseChunks(messages: BaseMessage[], options: this["ParsedCallOptions"], runManager?: CallbackManagerForLLMRun): AsyncGenerator<ChatGenerationChunk>;
     completionWithRetry(request: string | GenerateContentRequest | (string | GenerativeAIPart)[], options?: this["ParsedCallOptions"]): Promise<import("@google/generative-ai").GenerateContentResult>;
-    withStructuredOutput<RunOutput extends Record<string, any> = Record<string, any>>(outputSchema: z.ZodType<RunOutput> | Record<string, any>, config?: StructuredOutputMethodOptions<false>): Runnable<BaseLanguageModelInput, RunOutput>;
-    withStructuredOutput<RunOutput extends Record<string, any> = Record<string, any>>(outputSchema: z.ZodType<RunOutput> | Record<string, any>, config?: StructuredOutputMethodOptions<true>): Runnable<BaseLanguageModelInput, {
+    withStructuredOutput<RunOutput extends Record<string, any> = Record<string, any>>(outputSchema: InteropZodType<RunOutput> | Record<string, any>, config?: StructuredOutputMethodOptions<false>): Runnable<BaseLanguageModelInput, RunOutput>;
+    withStructuredOutput<RunOutput extends Record<string, any> = Record<string, any>>(outputSchema: InteropZodType<RunOutput> | Record<string, any>, config?: StructuredOutputMethodOptions<true>): Runnable<BaseLanguageModelInput, {
         raw: BaseMessage;
         parsed: RunOutput;
     }>;

package/dist/chat_models.js

@@ -2,7 +2,7 @@ import { GoogleGenerativeAI as GenerativeAI, } from "@google/generative-ai";
 import { getEnvironmentVariable } from "@langchain/core/utils/env";
 import { BaseChatModel, } from "@langchain/core/language_models/chat_models";
 import { RunnablePassthrough, RunnableSequence, } from "@langchain/core/runnables";
-import { isZodSchema } from "@langchain/core/utils/types";
+import { isInteropZodSchema, } from "@langchain/core/utils/types";
 import { JsonOutputParser, } from "@langchain/core/output_parsers";
 import { schemaToGenerativeAIParameters, removeAdditionalProperties, } from "./utils/zod_to_genai_parameters.js";
 import { convertBaseMessagesToContent, convertResponseContentToChatGenerationChunk, mapGenerateContentResultToChatResult, } from "./utils/common.js";
@@ -24,13 +24,12 @@ import { convertToolsToGenAI } from "./utils/tools.js";
  * ## [Runtime args](https://api.js.langchain.com/interfaces/langchain_google_genai.GoogleGenerativeAIChatCallOptions.html)
  *
  * Runtime args can be passed as the second argument to any of the base runnable methods `.invoke`. `.stream`, `.batch`, etc.
- * They can also be passed via `.bind`, or the second arg in `.bindTools`, like shown in the examples below:
+ * They can also be passed via `.withConfig`, or the second arg in `.bindTools`, like shown in the examples below:
  *
  * ```typescript
- * // When calling `.bind`, call options should be passed via the first argument
- * const llmWithArgsBound = llm.bind({
+ * // When calling `.withConfig`, call options should be passed via the first argument
+ * const llmWithArgsBound = llm.withConfig({
  *   stop: ["\n"],
- *   tools: [...],
  * });
  *
  * // When calling `.bindTools`, call options should be passed via the second argument
@@ -595,7 +594,10 @@ export class ChatGoogleGenerativeAI extends BaseChatModel {
         return "googlegenerativeai";
     }
     bindTools(tools, kwargs) {
-        return this.bind({ tools: convertToolsToGenAI(tools)?.tools, ...kwargs });
+        return this.withConfig({
+            tools: convertToolsToGenAI(tools)?.tools,
+            ...kwargs,
+        });
     }
     invocationParams(options) {
         const toolsAndConfig = options?.tools?.length
@@ -665,7 +667,10 @@ export class ChatGoogleGenerativeAI extends BaseChatModel {
         const generationResult = mapGenerateContentResultToChatResult(res.response, {
             usageMetadata,
         });
-        await runManager?.handleLLMNewToken(generationResult.generations[0].text ?? "");
+        // may not have generations in output if there was a refusal for safety reasons, malformed function call, etc.
+        if (generationResult.generations?.length > 0) {
+            await runManager?.handleLLMNewToken(generationResult.generations[0]?.text ?? "");
+        }
         return generationResult;
     }
     async *_streamResponseChunks(messages, options, runManager) {
@@ -752,7 +757,7 @@ export class ChatGoogleGenerativeAI extends BaseChatModel {
         if (method === "functionCalling") {
             let functionName = name ?? "extract";
             let tools;
-            if (isZodSchema(schema)) {
+            if (isInteropZodSchema(schema)) {
                 const jsonSchema = schemaToGenerativeAIParameters(schema);
                 tools = [
                     {
@@ -797,14 +802,13 @@ export class ChatGoogleGenerativeAI extends BaseChatModel {
                     keyName: functionName,
                 });
             }
-            llm = this.bind({
-                tools,
-                tool_choice: functionName,
+            llm = this.bindTools(tools).withConfig({
+                allowedFunctionNames: [functionName],
             });
         }
         else {
             const jsonSchema = schemaToGenerativeAIParameters(schema);
-            llm = this.bind({
+            llm = this.withConfig({
                 responseSchema: jsonSchema,
             });
             outputParser = new JsonOutputParser();

package/dist/output_parsers.cjs

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.GoogleGenerativeAIToolsOutputParser = void 0;
 const output_parsers_1 = require("@langchain/core/output_parsers");
+const types_1 = require("@langchain/core/utils/types");
 class GoogleGenerativeAIToolsOutputParser extends output_parsers_1.BaseLLMOutputParser {
     static lc_name() {
         return "GoogleGenerativeAIToolsOutputParser";
@@ -48,12 +49,12 @@ class GoogleGenerativeAIToolsOutputParser extends output_parsers_1.BaseLLMOutput
         if (this.zodSchema === undefined) {
             return result;
         }
-        const zodParsedResult = await this.zodSchema.safeParseAsync(result);
+        const zodParsedResult = await (0, types_1.interopSafeParseAsync)(this.zodSchema, result);
         if (zodParsedResult.success) {
             return zodParsedResult.data;
         }
         else {
-            throw new output_parsers_1.OutputParserException(`Failed to parse. Text: "${JSON.stringify(result, null, 2)}". Error: ${JSON.stringify(zodParsedResult.error.errors)}`, JSON.stringify(result, null, 2));
+            throw new output_parsers_1.OutputParserException(`Failed to parse. Text: "${JSON.stringify(result, null, 2)}". Error: ${JSON.stringify(zodParsedResult.error.issues)}`, JSON.stringify(result, null, 2));
         }
     }
     async parseResult(generations) {

package/dist/output_parsers.d.ts

@@ -1,8 +1,8 @@
-import type { z } from "zod";
 import { BaseLLMOutputParser } from "@langchain/core/output_parsers";
-import { JsonOutputKeyToolsParserParams } from "@langchain/core/output_parsers/openai_tools";
 import { ChatGeneration } from "@langchain/core/outputs";
-interface GoogleGenerativeAIToolsOutputParserParams<T extends Record<string, any>> extends JsonOutputKeyToolsParserParams<T> {
+import { InteropZodType } from "@langchain/core/utils/types";
+import { JsonOutputKeyToolsParserParamsInterop } from "@langchain/core/output_parsers/openai_tools";
+interface GoogleGenerativeAIToolsOutputParserParams<T extends Record<string, any>> extends JsonOutputKeyToolsParserParamsInterop<T> {
 }
 export declare class GoogleGenerativeAIToolsOutputParser<T extends Record<string, any> = Record<string, any>> extends BaseLLMOutputParser<T> {
     static lc_name(): string;
@@ -12,7 +12,7 @@ export declare class GoogleGenerativeAIToolsOutputParser<T extends Record<string
     keyName: string;
     /** Whether to return only the first tool call. */
     returnSingle: boolean;
-    zodSchema?: z.ZodType<T>;
+    zodSchema?: InteropZodType<T>;
     constructor(params: GoogleGenerativeAIToolsOutputParserParams<T>);
     protected _validateResult(result: unknown): Promise<T>;
     parseResult(generations: ChatGeneration[]): Promise<T>;

package/dist/output_parsers.js

@@ -1,4 +1,5 @@
 import { BaseLLMOutputParser, OutputParserException, } from "@langchain/core/output_parsers";
+import { interopSafeParseAsync, } from "@langchain/core/utils/types";
 export class GoogleGenerativeAIToolsOutputParser extends BaseLLMOutputParser {
     static lc_name() {
         return "GoogleGenerativeAIToolsOutputParser";
@@ -45,12 +46,12 @@ export class GoogleGenerativeAIToolsOutputParser extends BaseLLMOutputParser {
         if (this.zodSchema === undefined) {
             return result;
         }
-        const zodParsedResult = await this.zodSchema.safeParseAsync(result);
+        const zodParsedResult = await interopSafeParseAsync(this.zodSchema, result);
         if (zodParsedResult.success) {
             return zodParsedResult.data;
         }
         else {
-            throw new OutputParserException(`Failed to parse. Text: "${JSON.stringify(result, null, 2)}". Error: ${JSON.stringify(zodParsedResult.error.errors)}`, JSON.stringify(result, null, 2));
+            throw new OutputParserException(`Failed to parse. Text: "${JSON.stringify(result, null, 2)}". Error: ${JSON.stringify(zodParsedResult.error.issues)}`, JSON.stringify(result, null, 2));
         }
     }
     async parseResult(generations) {

package/dist/utils/common.cjs

@@ -78,6 +78,200 @@ function inferToolNameFromPreviousMessages(message, previousMessages) {
         return toolCall.id === message.tool_call_id;
     })?.name;
 }
+function _getStandardContentBlockConverter(isMultimodalModel) {
+    const standardContentBlockConverter = {
+        providerName: "Google Gemini",
+        fromStandardTextBlock(block) {
+            return {
+                text: block.text,
+            };
+        },
+        fromStandardImageBlock(block) {
+            if (!isMultimodalModel) {
+                throw new Error("This model does not support images");
+            }
+            if (block.source_type === "url") {
+                const data = (0, messages_1.parseBase64DataUrl)({ dataUrl: block.url });
+                if (data) {
+                    return {
+                        inlineData: {
+                            mimeType: data.mime_type,
+                            data: data.data,
+                        },
+                    };
+                }
+                else {
+                    return {
+                        fileData: {
+                            mimeType: block.mime_type ?? "",
+                            fileUri: block.url,
+                        },
+                    };
+                }
+            }
+            if (block.source_type === "base64") {
+                return {
+                    inlineData: {
+                        mimeType: block.mime_type ?? "",
+                        data: block.data,
+                    },
+                };
+            }
+            throw new Error(`Unsupported source type: ${block.source_type}`);
+        },
+        fromStandardAudioBlock(block) {
+            if (!isMultimodalModel) {
+                throw new Error("This model does not support audio");
+            }
+            if (block.source_type === "url") {
+                const data = (0, messages_1.parseBase64DataUrl)({ dataUrl: block.url });
+                if (data) {
+                    return {
+                        inlineData: {
+                            mimeType: data.mime_type,
+                            data: data.data,
+                        },
+                    };
+                }
+                else {
+                    return {
+                        fileData: {
+                            mimeType: block.mime_type ?? "",
+                            fileUri: block.url,
+                        },
+                    };
+                }
+            }
+            if (block.source_type === "base64") {
+                return {
+                    inlineData: {
+                        mimeType: block.mime_type ?? "",
+                        data: block.data,
+                    },
+                };
+            }
+            throw new Error(`Unsupported source type: ${block.source_type}`);
+        },
+        fromStandardFileBlock(block) {
+            if (!isMultimodalModel) {
+                throw new Error("This model does not support files");
+            }
+            if (block.source_type === "text") {
+                return {
+                    text: block.text,
+                };
+            }
+            if (block.source_type === "url") {
+                const data = (0, messages_1.parseBase64DataUrl)({ dataUrl: block.url });
+                if (data) {
+                    return {
+                        inlineData: {
+                            mimeType: data.mime_type,
+                            data: data.data,
+                        },
+                    };
+                }
+                else {
+                    return {
+                        fileData: {
+                            mimeType: block.mime_type ?? "",
+                            fileUri: block.url,
+                        },
+                    };
+                }
+            }
+            if (block.source_type === "base64") {
+                return {
+                    inlineData: {
+                        mimeType: block.mime_type ?? "",
+                        data: block.data,
+                    },
+                };
+            }
+            throw new Error(`Unsupported source type: ${block.source_type}`);
+        },
+    };
+    return standardContentBlockConverter;
+}
+function _convertLangChainContentToPart(content, isMultimodalModel) {
+    if ((0, messages_1.isDataContentBlock)(content)) {
+        return (0, messages_1.convertToProviderContentBlock)(content, _getStandardContentBlockConverter(isMultimodalModel));
+    }
+    if (content.type === "text") {
+        return { text: content.text };
+    }
+    else if (content.type === "executableCode") {
+        return { executableCode: content.executableCode };
+    }
+    else if (content.type === "codeExecutionResult") {
+        return { codeExecutionResult: content.codeExecutionResult };
+    }
+    else if (content.type === "image_url") {
+        if (!isMultimodalModel) {
+            throw new Error(`This model does not support images`);
+        }
+        let source;
+        if (typeof content.image_url === "string") {
+            source = content.image_url;
+        }
+        else if (typeof content.image_url === "object" &&
+            "url" in content.image_url) {
+            source = content.image_url.url;
+        }
+        else {
+            throw new Error("Please provide image as base64 encoded data URL");
+        }
+        const [dm, data] = source.split(",");
+        if (!dm.startsWith("data:")) {
+            throw new Error("Please provide image as base64 encoded data URL");
+        }
+        const [mimeType, encoding] = dm.replace(/^data:/, "").split(";");
+        if (encoding !== "base64") {
+            throw new Error("Please provide image as base64 encoded data URL");
+        }
+        return {
+            inlineData: {
+                data,
+                mimeType,
+            },
+        };
+    }
+    else if (content.type === "media") {
+        return messageContentMedia(content);
+    }
+    else if (content.type === "tool_use") {
+        return {
+            functionCall: {
+                name: content.name,
+                args: content.input,
+            },
+        };
+    }
+    else if (content.type?.includes("/") &&
+        // Ensure it's a single slash.
+        content.type.split("/").length === 2 &&
+        "data" in content &&
+        typeof content.data === "string") {
+        return {
+            inlineData: {
+                mimeType: content.type,
+                data: content.data,
+            },
+        };
+    }
+    else if ("functionCall" in content) {
+        // No action needed here — function calls will be added later from message.tool_calls
+        return undefined;
+    }
+    else {
+        if ("type" in content) {
+            throw new Error(`Unknown content type ${content.type}`);
+        }
+        else {
+            throw new Error(`Unknown content ${JSON.stringify(content)}`);
+        }
+    }
+}
 function convertMessageContentToParts(message, isMultimodalModel, previousMessages) {
     if ((0, messages_1.isToolMessage)(message)) {
         const messageName = message.name ??
@@ -85,13 +279,29 @@ function convertMessageContentToParts(message, isMultimodalModel, previousMessag
         if (messageName === undefined) {
             throw new Error(`Google requires a tool name for each tool call response, and we could not infer a called tool name for ToolMessage "${message.id}" from your passed messages. Please populate a "name" field on that ToolMessage explicitly.`);
         }
+        const result = Array.isArray(message.content)
+            ? message.content
+                .map((c) => _convertLangChainContentToPart(c, isMultimodalModel))
+                .filter((p) => p !== undefined)
+            : message.content;
+        if (message.status === "error") {
+            return [
+                {
+                    functionResponse: {
+                        name: messageName,
+                        // The API expects an object with an `error` field if the function call fails.
+                        // `error` must be a valid object (not a string or array), so we wrap `message.content` here
+                        response: { error: { details: result } },
+                    },
+                },
+            ];
+        }
         return [
             {
                 functionResponse: {
                     name: messageName,
-                    response: typeof message.content === "string"
-                        ? { result: message.content }
-                        : message.content,
+                    // again, can't have a string or array value for `response`, so we wrap it as an object here
+                    response: { result },
                 },
             },
         ];
@@ -102,80 +312,9 @@ function convertMessageContentToParts(message, isMultimodalModel, previousMessag
         messageParts.push({ text: message.content });
     }
     if (Array.isArray(message.content)) {
-        message.content.forEach((c) => {
-            if (c.type === "text") {
-                messageParts.push({ text: c.text });
-            }
-            else if (c.type === "executableCode") {
-                messageParts.push({ executableCode: c.executableCode });
-            }
-            else if (c.type === "codeExecutionResult") {
-                messageParts.push({ codeExecutionResult: c.codeExecutionResult });
-            }
-            else if (c.type === "image_url") {
-                if (!isMultimodalModel) {
-                    throw new Error(`This model does not support images`);
-                }
-                let source;
-                if (typeof c.image_url === "string") {
-                    source = c.image_url;
-                }
-                else if (typeof c.image_url === "object" && "url" in c.image_url) {
-                    source = c.image_url.url;
-                }
-                else {
-                    throw new Error("Please provide image as base64 encoded data URL");
-                }
-                const [dm, data] = source.split(",");
-                if (!dm.startsWith("data:")) {
-                    throw new Error("Please provide image as base64 encoded data URL");
-                }
-                const [mimeType, encoding] = dm.replace(/^data:/, "").split(";");
-                if (encoding !== "base64") {
-                    throw new Error("Please provide image as base64 encoded data URL");
-                }
-                messageParts.push({
-                    inlineData: {
-                        data,
-                        mimeType,
-                    },
-                });
-            }
-            else if (c.type === "media") {
-                messageParts.push(messageContentMedia(c));
-            }
-            else if (c.type === "tool_use") {
-                functionCalls.push({
-                    functionCall: {
-                        name: c.name,
-                        args: c.input,
-                    },
-                });
-            }
-            else if (c.type?.includes("/") &&
-                // Ensure it's a single slash.
-                c.type.split("/").length === 2 &&
-                "data" in c &&
-                typeof c.data === "string") {
-                messageParts.push({
-                    inlineData: {
-                        mimeType: c.type,
-                        data: c.data,
-                    },
-                });
-            }
-            else if ("functionCall" in c) {
-                // No action needed here — function calls will be added later from message.tool_calls
-            }
-            else {
-                if ("type" in c) {
-                    throw new Error(`Unknown content type ${c.type}`);
-                }
-                else {
-                    throw new Error(`Unknown content ${JSON.stringify(c)}`);
-                }
-            }
-        });
+        messageParts.push(...message.content
+            .map((c) => _convertLangChainContentToPart(c, isMultimodalModel))
+            .filter((p) => p !== undefined));
     }
     if ((0, messages_1.isAIMessage)(message) && message.tool_calls?.length) {
         functionCalls = message.tool_calls.map((tc) => {
@@ -251,10 +390,13 @@ function mapGenerateContentResultToChatResult(response, extra) {
     const [candidate] = response.candidates;
     const { content: candidateContent, ...generationInfo } = candidate;
     let content;
-    if (candidateContent?.parts.length === 1 && candidateContent.parts[0].text) {
+    if (Array.isArray(candidateContent?.parts) &&
+        candidateContent.parts.length === 1 &&
+        candidateContent.parts[0].text) {
         content = candidateContent.parts[0].text;
     }
-    else {
+    else if (Array.isArray(candidateContent?.parts) &&
+        candidateContent.parts.length > 0) {
         content = candidateContent.parts.map((p) => {
             if ("text" in p) {
                 return {
@@ -277,17 +419,22 @@ function mapGenerateContentResultToChatResult(response, extra) {
             return p;
         });
     }
+    else {
+        // no content returned - likely due to abnormal stop reason, e.g. malformed function call
+        content = [];
+    }
     let text = "";
     if (typeof content === "string") {
         text = content;
     }
-    else if ("text" in content[0]) {
-        text = content[0].text;
+    else if (Array.isArray(content) && content.length > 0) {
+        const block = content.find((b) => "text" in b);
+        text = block?.text ?? text;
     }
     const generation = {
         text,
         message: new messages_1.AIMessage({
-            content,
+            content: content ?? "",
             tool_calls: functionCalls?.map((fc) => {
                 return {
                     ...fc,
@@ -323,11 +470,11 @@ function convertResponseContentToChatGenerationChunk(response, extra) {
     const { content: candidateContent, ...generationInfo } = candidate;
     let content;
     // Checks if some parts do not have text. If false, it means that the content is a string.
-    if (candidateContent?.parts &&
+    if (Array.isArray(candidateContent?.parts) &&
         candidateContent.parts.every((p) => "text" in p)) {
         content = candidateContent.parts.map((p) => p.text).join("");
     }
-    else if (candidateContent.parts) {
+    else if (Array.isArray(candidateContent?.parts)) {
         content = candidateContent.parts.map((p) => {
             if ("text" in p) {
                 return {
@@ -350,12 +497,17 @@ function convertResponseContentToChatGenerationChunk(response, extra) {
             return p;
         });
     }
+    else {
+        // no content returned - likely due to abnormal stop reason, e.g. malformed function call
+        content = [];
+    }
     let text = "";
     if (content && typeof content === "string") {
         text = content;
     }
-    else if (content && typeof content === "object" && "text" in content[0]) {
-        text = content[0].text;
+    else if (Array.isArray(content)) {
+        const block = content.find((b) => "text" in b);
+        text = block?.text ?? "";
     }
     const toolCallChunks = [];
     if (functionCalls) {

package/dist/utils/common.js

@@ -1,4 +1,4 @@
-import { AIMessage, AIMessageChunk, ChatMessage, isAIMessage, isBaseMessage, isToolMessage, } from "@langchain/core/messages";
+import { AIMessage, AIMessageChunk, ChatMessage, isAIMessage, isBaseMessage, isToolMessage, parseBase64DataUrl, convertToProviderContentBlock, isDataContentBlock, } from "@langchain/core/messages";
 import { ChatGenerationChunk, } from "@langchain/core/outputs";
 import { isLangChainTool } from "@langchain/core/utils/function_calling";
 import { isOpenAITool } from "@langchain/core/language_models/base";
@@ -73,6 +73,200 @@ function inferToolNameFromPreviousMessages(message, previousMessages) {
         return toolCall.id === message.tool_call_id;
     })?.name;
 }
+function _getStandardContentBlockConverter(isMultimodalModel) {
+    const standardContentBlockConverter = {
+        providerName: "Google Gemini",
+        fromStandardTextBlock(block) {
+            return {
+                text: block.text,
+            };
+        },
+        fromStandardImageBlock(block) {
+            if (!isMultimodalModel) {
+                throw new Error("This model does not support images");
+            }
+            if (block.source_type === "url") {
+                const data = parseBase64DataUrl({ dataUrl: block.url });
+                if (data) {
+                    return {
+                        inlineData: {
+                            mimeType: data.mime_type,
+                            data: data.data,
+                        },
+                    };
+                }
+                else {
+                    return {
+                        fileData: {
+                            mimeType: block.mime_type ?? "",
+                            fileUri: block.url,
+                        },
+                    };
+                }
+            }
+            if (block.source_type === "base64") {
+                return {
+                    inlineData: {
+                        mimeType: block.mime_type ?? "",
+                        data: block.data,
+                    },
+                };
+            }
+            throw new Error(`Unsupported source type: ${block.source_type}`);
+        },
+        fromStandardAudioBlock(block) {
+            if (!isMultimodalModel) {
+                throw new Error("This model does not support audio");
+            }
+            if (block.source_type === "url") {
+                const data = parseBase64DataUrl({ dataUrl: block.url });
+                if (data) {
+                    return {
+                        inlineData: {
+                            mimeType: data.mime_type,
+                            data: data.data,
+                        },
+                    };
+                }
+                else {
+                    return {
+                        fileData: {
+                            mimeType: block.mime_type ?? "",
+                            fileUri: block.url,
+                        },
+                    };
+                }
+            }
+            if (block.source_type === "base64") {
+                return {
+                    inlineData: {
+                        mimeType: block.mime_type ?? "",
+                        data: block.data,
+                    },
+                };
+            }
+            throw new Error(`Unsupported source type: ${block.source_type}`);
+        },
+        fromStandardFileBlock(block) {
+            if (!isMultimodalModel) {
+                throw new Error("This model does not support files");
+            }
+            if (block.source_type === "text") {
+                return {
+                    text: block.text,
+                };
+            }
+            if (block.source_type === "url") {
+                const data = parseBase64DataUrl({ dataUrl: block.url });
+                if (data) {
+                    return {
+                        inlineData: {
+                            mimeType: data.mime_type,
+                            data: data.data,
+                        },
+                    };
+                }
+                else {
+                    return {
+                        fileData: {
+                            mimeType: block.mime_type ?? "",
+                            fileUri: block.url,
+                        },
+                    };
+                }
+            }
+            if (block.source_type === "base64") {
+                return {
+                    inlineData: {
+                        mimeType: block.mime_type ?? "",
+                        data: block.data,
+                    },
+                };
+            }
+            throw new Error(`Unsupported source type: ${block.source_type}`);
+        },
+    };
+    return standardContentBlockConverter;
+}
+function _convertLangChainContentToPart(content, isMultimodalModel) {
+    if (isDataContentBlock(content)) {
+        return convertToProviderContentBlock(content, _getStandardContentBlockConverter(isMultimodalModel));
+    }
+    if (content.type === "text") {
+        return { text: content.text };
+    }
+    else if (content.type === "executableCode") {
+        return { executableCode: content.executableCode };
+    }
+    else if (content.type === "codeExecutionResult") {
+        return { codeExecutionResult: content.codeExecutionResult };
+    }
+    else if (content.type === "image_url") {
+        if (!isMultimodalModel) {
+            throw new Error(`This model does not support images`);
+        }
+        let source;
+        if (typeof content.image_url === "string") {
+            source = content.image_url;
+        }
+        else if (typeof content.image_url === "object" &&
+            "url" in content.image_url) {
+            source = content.image_url.url;
+        }
+        else {
+            throw new Error("Please provide image as base64 encoded data URL");
+        }
+        const [dm, data] = source.split(",");
+        if (!dm.startsWith("data:")) {
+            throw new Error("Please provide image as base64 encoded data URL");
+        }
+        const [mimeType, encoding] = dm.replace(/^data:/, "").split(";");
+        if (encoding !== "base64") {
+            throw new Error("Please provide image as base64 encoded data URL");
+        }
+        return {
+            inlineData: {
+                data,
+                mimeType,
+            },
+        };
+    }
+    else if (content.type === "media") {
+        return messageContentMedia(content);
+    }
+    else if (content.type === "tool_use") {
+        return {
+            functionCall: {
+                name: content.name,
+                args: content.input,
+            },
+        };
+    }
+    else if (content.type?.includes("/") &&
+        // Ensure it's a single slash.
+        content.type.split("/").length === 2 &&
+        "data" in content &&
+        typeof content.data === "string") {
+        return {
+            inlineData: {
+                mimeType: content.type,
+                data: content.data,
+            },
+        };
+    }
+    else if ("functionCall" in content) {
+        // No action needed here — function calls will be added later from message.tool_calls
+        return undefined;
+    }
+    else {
+        if ("type" in content) {
+            throw new Error(`Unknown content type ${content.type}`);
+        }
+        else {
+            throw new Error(`Unknown content ${JSON.stringify(content)}`);
+        }
+    }
+}
 export function convertMessageContentToParts(message, isMultimodalModel, previousMessages) {
     if (isToolMessage(message)) {
         const messageName = message.name ??
@@ -80,13 +274,29 @@ export function convertMessageContentToParts(message, isMultimodalModel, previou
         if (messageName === undefined) {
             throw new Error(`Google requires a tool name for each tool call response, and we could not infer a called tool name for ToolMessage "${message.id}" from your passed messages. Please populate a "name" field on that ToolMessage explicitly.`);
         }
+        const result = Array.isArray(message.content)
+            ? message.content
+                .map((c) => _convertLangChainContentToPart(c, isMultimodalModel))
+                .filter((p) => p !== undefined)
+            : message.content;
+        if (message.status === "error") {
+            return [
+                {
+                    functionResponse: {
+                        name: messageName,
+                        // The API expects an object with an `error` field if the function call fails.
+                        // `error` must be a valid object (not a string or array), so we wrap `message.content` here
+                        response: { error: { details: result } },
+                    },
+                },
+            ];
+        }
         return [
             {
                 functionResponse: {
                     name: messageName,
-                    response: typeof message.content === "string"
-                        ? { result: message.content }
-                        : message.content,
+                    // again, can't have a string or array value for `response`, so we wrap it as an object here
+                    response: { result },
                 },
             },
         ];
@@ -97,80 +307,9 @@ export function convertMessageContentToParts(message, isMultimodalModel, previou
         messageParts.push({ text: message.content });
     }
     if (Array.isArray(message.content)) {
-        message.content.forEach((c) => {
-            if (c.type === "text") {
-                messageParts.push({ text: c.text });
-            }
-            else if (c.type === "executableCode") {
-                messageParts.push({ executableCode: c.executableCode });
-            }
-            else if (c.type === "codeExecutionResult") {
-                messageParts.push({ codeExecutionResult: c.codeExecutionResult });
-            }
-            else if (c.type === "image_url") {
-                if (!isMultimodalModel) {
-                    throw new Error(`This model does not support images`);
-                }
-                let source;
-                if (typeof c.image_url === "string") {
-                    source = c.image_url;
-                }
-                else if (typeof c.image_url === "object" && "url" in c.image_url) {
-                    source = c.image_url.url;
-                }
-                else {
-                    throw new Error("Please provide image as base64 encoded data URL");
-                }
-                const [dm, data] = source.split(",");
-                if (!dm.startsWith("data:")) {
-                    throw new Error("Please provide image as base64 encoded data URL");
-                }
-                const [mimeType, encoding] = dm.replace(/^data:/, "").split(";");
-                if (encoding !== "base64") {
-                    throw new Error("Please provide image as base64 encoded data URL");
-                }
-                messageParts.push({
-                    inlineData: {
-                        data,
-                        mimeType,
-                    },
-                });
-            }
-            else if (c.type === "media") {
-                messageParts.push(messageContentMedia(c));
-            }
-            else if (c.type === "tool_use") {
-                functionCalls.push({
-                    functionCall: {
-                        name: c.name,
-                        args: c.input,
-                    },
-                });
-            }
-            else if (c.type?.includes("/") &&
-                // Ensure it's a single slash.
-                c.type.split("/").length === 2 &&
-                "data" in c &&
-                typeof c.data === "string") {
-                messageParts.push({
-                    inlineData: {
-                        mimeType: c.type,
-                        data: c.data,
-                    },
-                });
-            }
-            else if ("functionCall" in c) {
-                // No action needed here — function calls will be added later from message.tool_calls
-            }
-            else {
-                if ("type" in c) {
-                    throw new Error(`Unknown content type ${c.type}`);
-                }
-                else {
-                    throw new Error(`Unknown content ${JSON.stringify(c)}`);
-                }
-            }
-        });
+        messageParts.push(...message.content
+            .map((c) => _convertLangChainContentToPart(c, isMultimodalModel))
+            .filter((p) => p !== undefined));
     }
     if (isAIMessage(message) && message.tool_calls?.length) {
         functionCalls = message.tool_calls.map((tc) => {
@@ -244,10 +383,13 @@ export function mapGenerateContentResultToChatResult(response, extra) {
     const [candidate] = response.candidates;
     const { content: candidateContent, ...generationInfo } = candidate;
     let content;
-    if (candidateContent?.parts.length === 1 && candidateContent.parts[0].text) {
+    if (Array.isArray(candidateContent?.parts) &&
+        candidateContent.parts.length === 1 &&
+        candidateContent.parts[0].text) {
         content = candidateContent.parts[0].text;
     }
-    else {
+    else if (Array.isArray(candidateContent?.parts) &&
+        candidateContent.parts.length > 0) {
         content = candidateContent.parts.map((p) => {
             if ("text" in p) {
                 return {
@@ -270,17 +412,22 @@ export function mapGenerateContentResultToChatResult(response, extra) {
             return p;
         });
     }
+    else {
+        // no content returned - likely due to abnormal stop reason, e.g. malformed function call
+        content = [];
+    }
     let text = "";
     if (typeof content === "string") {
         text = content;
     }
-    else if ("text" in content[0]) {
-        text = content[0].text;
+    else if (Array.isArray(content) && content.length > 0) {
+        const block = content.find((b) => "text" in b);
+        text = block?.text ?? text;
     }
     const generation = {
         text,
         message: new AIMessage({
-            content,
+            content: content ?? "",
             tool_calls: functionCalls?.map((fc) => {
                 return {
                     ...fc,
@@ -315,11 +462,11 @@ export function convertResponseContentToChatGenerationChunk(response, extra) {
     const { content: candidateContent, ...generationInfo } = candidate;
     let content;
     // Checks if some parts do not have text. If false, it means that the content is a string.
-    if (candidateContent?.parts &&
+    if (Array.isArray(candidateContent?.parts) &&
         candidateContent.parts.every((p) => "text" in p)) {
         content = candidateContent.parts.map((p) => p.text).join("");
     }
-    else if (candidateContent.parts) {
+    else if (Array.isArray(candidateContent?.parts)) {
         content = candidateContent.parts.map((p) => {
             if ("text" in p) {
                 return {
@@ -342,12 +489,17 @@ export function convertResponseContentToChatGenerationChunk(response, extra) {
             return p;
         });
     }
+    else {
+        // no content returned - likely due to abnormal stop reason, e.g. malformed function call
+        content = [];
+    }
     let text = "";
     if (content && typeof content === "string") {
         text = content;
     }
-    else if (content && typeof content === "object" && "text" in content[0]) {
-        text = content[0].text;
+    else if (Array.isArray(content)) {
+        const block = content.find((b) => "text" in b);
+        text = block?.text ?? "";
     }
     const toolCallChunks = [];
     if (functionCalls) {

package/dist/utils/zod_to_genai_parameters.cjs

@@ -3,7 +3,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.jsonSchemaToGeminiParameters = exports.schemaToGenerativeAIParameters = exports.removeAdditionalProperties = void 0;
 const types_1 = require("@langchain/core/utils/types");
-const zod_to_json_schema_1 = require("zod-to-json-schema");
+const json_schema_1 = require("@langchain/core/utils/json_schema");
 function removeAdditionalProperties(
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 obj) {
@@ -36,7 +36,7 @@ exports.removeAdditionalProperties = removeAdditionalProperties;
 function schemaToGenerativeAIParameters(schema) {
     // GenerativeAI doesn't accept either the $schema or additionalProperties
     // attributes, so we need to explicitly remove them.
-    const jsonSchema = removeAdditionalProperties((0, types_1.isZodSchema)(schema) ? (0, zod_to_json_schema_1.zodToJsonSchema)(schema) : schema);
+    const jsonSchema = removeAdditionalProperties((0, types_1.isInteropZodSchema)(schema) ? (0, json_schema_1.toJsonSchema)(schema) : schema);
     const { $schema, ...rest } = jsonSchema;
     return rest;
 }

package/dist/utils/zod_to_genai_parameters.d.ts

@@ -1,6 +1,6 @@
-import type { z } from "zod";
 import { type FunctionDeclarationSchema as GenerativeAIFunctionDeclarationSchema, type SchemaType as FunctionDeclarationSchemaType } from "@google/generative-ai";
-import { type JsonSchema7Type } from "zod-to-json-schema";
+import { InteropZodType } from "@langchain/core/utils/types";
+import { type JsonSchema7Type } from "@langchain/core/utils/json_schema";
 export interface GenerativeAIJsonSchema extends Record<string, unknown> {
     properties?: Record<string, GenerativeAIJsonSchema>;
     type: FunctionDeclarationSchemaType;
@@ -10,5 +10,5 @@ export interface GenerativeAIJsonSchemaDirty extends GenerativeAIJsonSchema {
     additionalProperties?: boolean;
 }
 export declare function removeAdditionalProperties(obj: Record<string, any>): GenerativeAIJsonSchema;
-export declare function schemaToGenerativeAIParameters<RunOutput extends Record<string, any> = Record<string, any>>(schema: z.ZodType<RunOutput> | z.ZodEffects<z.ZodType<RunOutput>> | JsonSchema7Type): GenerativeAIFunctionDeclarationSchema;
+export declare function schemaToGenerativeAIParameters<RunOutput extends Record<string, any> = Record<string, any>>(schema: InteropZodType<RunOutput> | JsonSchema7Type): GenerativeAIFunctionDeclarationSchema;
 export declare function jsonSchemaToGeminiParameters(schema: Record<string, any>): GenerativeAIFunctionDeclarationSchema;

package/dist/utils/zod_to_genai_parameters.js

@@ -1,6 +1,6 @@
 /* eslint-disable @typescript-eslint/no-unused-vars */
-import { isZodSchema } from "@langchain/core/utils/types";
-import { zodToJsonSchema } from "zod-to-json-schema";
+import { isInteropZodSchema, } from "@langchain/core/utils/types";
+import { toJsonSchema, } from "@langchain/core/utils/json_schema";
 export function removeAdditionalProperties(
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 obj) {
@@ -32,7 +32,7 @@ obj) {
 export function schemaToGenerativeAIParameters(schema) {
     // GenerativeAI doesn't accept either the $schema or additionalProperties
     // attributes, so we need to explicitly remove them.
-    const jsonSchema = removeAdditionalProperties(isZodSchema(schema) ? zodToJsonSchema(schema) : schema);
+    const jsonSchema = removeAdditionalProperties(isInteropZodSchema(schema) ? toJsonSchema(schema) : schema);
     const { $schema, ...rest } = jsonSchema;
     return rest;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/google-genai",
-  "version": "0.2.9",
+  "version": "0.2.11",
   "description": "Google Generative AI integration for LangChain.js",
   "type": "module",
   "engines": {
@@ -36,15 +36,14 @@
   "license": "MIT",
   "dependencies": {
     "@google/generative-ai": "^0.24.0",
-    "uuid": "^11.1.0",
-    "zod-to-json-schema": "^3.22.4"
+    "uuid": "^11.1.0"
   },
   "peerDependencies": {
-    "@langchain/core": ">=0.3.55 <0.4.0"
+    "@langchain/core": ">=0.3.58 <0.4.0"
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@langchain/core": "0.3.55",
+    "@langchain/core": "workspace:*",
     "@langchain/scripts": ">=0.1.0 <0.2.0",
     "@langchain/standard-tests": "0.0.0",
     "@swc/core": "^1.3.90",
@@ -68,7 +67,7 @@
     "rollup": "^4.5.2",
     "ts-jest": "^29.1.0",
     "typescript": "<5.2.0",
-    "zod": "^3.22.4"
+    "zod": "^3.25.32"
   },
   "publishConfig": {
     "access": "public"
@@ -92,4 +91,4 @@
     "index.d.ts",
     "index.d.cts"
   ]
-}
+}