@langchain/google-common 0.0.4 → 0.0.6

package/dist/auth.cjs

@@ -19,7 +19,8 @@ class GoogleAbstractedFetchClient {
         }
         const res = await fetch(url, fetchOptions);
         if (!res.ok) {
-            const error = new Error(`Could not get access token for Google with status code: ${res.status}`);
+            const resText = await res.text();
+            const error = new Error(`Google request failed with status code ${res.status}: ${resText}`);
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             error.response = res;
             throw error;

package/dist/auth.js

@@ -16,7 +16,8 @@ export class GoogleAbstractedFetchClient {
         }
         const res = await fetch(url, fetchOptions);
         if (!res.ok) {
-            const error = new Error(`Could not get access token for Google with status code: ${res.status}`);
+            const resText = await res.text();
+            const error = new Error(`Google request failed with status code ${res.status}: ${resText}`);
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             error.response = res;
             throw error;

package/dist/chat_models.cjs

@@ -7,6 +7,7 @@ const outputs_1 = require("@langchain/core/outputs");
 const messages_1 = require("@langchain/core/messages");
 const runnables_1 = require("@langchain/core/runnables");
 const openai_tools_1 = require("@langchain/core/output_parsers/openai_tools");
+const function_calling_1 = require("@langchain/core/utils/function_calling");
 const common_js_1 = require("./utils/common.cjs");
 const connection_js_1 = require("./connection.cjs");
 const gemini_js_1 = require("./utils/gemini.cjs");
@@ -16,10 +17,27 @@ const zod_to_gemini_parameters_js_1 = require("./utils/zod_to_gemini_parameters.
 class ChatConnection extends connection_js_1.AbstractGoogleLLMConnection {
     formatContents(input, _parameters) {
         return input
-            .map((msg) => (0, gemini_js_1.baseMessageToContent)(msg))
+            .map((msg, i) => (0, gemini_js_1.baseMessageToContent)(msg, input[i - 1]))
             .reduce((acc, cur) => [...acc, ...cur]);
     }
 }
+function convertToGeminiTools(structuredTools) {
+    return [
+        {
+            functionDeclarations: structuredTools.map((structuredTool) => {
+                if ((0, function_calling_1.isStructuredTool)(structuredTool)) {
+                    const jsonSchema = (0, zod_to_gemini_parameters_js_1.zodToGeminiParameters)(structuredTool.schema);
+                    return {
+                        name: structuredTool.name,
+                        description: structuredTool.description,
+                        parameters: jsonSchema,
+                    };
+                }
+                return structuredTool;
+            }),
+        },
+    ];
+}
 /**
  * Integration with a chat model.
  */
@@ -36,7 +54,6 @@ class ChatGoogleBase extends chat_models_1.BaseChatModel {
             writable: true,
             value: true
         });
-        /** @deprecated Prefer `modelName` */
         Object.defineProperty(this, "model", {
             enumerable: true,
             configurable: true,
@@ -131,19 +148,28 @@ class ChatGoogleBase extends chat_models_1.BaseChatModel {
     get platform() {
         return this.connection.platform;
     }
+    bindTools(tools, kwargs) {
+        return this.bind({ tools: convertToGeminiTools(tools), ...kwargs });
+    }
     // Replace
     _llmType() {
         return "chat_integration";
     }
+    /**
+     * Get the parameters used to invoke the model
+     */
+    invocationParams(options) {
+        return (0, common_js_1.copyAIModelParams)(this, options);
+    }
     async _generate(messages, options, _runManager) {
-        const parameters = (0, common_js_1.copyAIModelParams)(this, options);
+        const parameters = this.invocationParams(options);
         const response = await this.connection.request(messages, parameters, options);
         const ret = (0, gemini_js_1.safeResponseToChatResult)(response, this.safetyHandler);
         return ret;
     }
     async *_streamResponseChunks(_messages, options, _runManager) {
         // Make the call as a streaming request
-        const parameters = (0, common_js_1.copyAIModelParams)(this, options);
+        const parameters = this.invocationParams(options);
         const response = await this.streamedConnection.request(_messages, parameters, options);
         // Get the streaming parser of the response
         const stream = response.data;

package/dist/chat_models.d.ts

@@ -2,9 +2,11 @@ import { type BaseMessage } from "@langchain/core/messages";
 import { CallbackManagerForLLMRun } from "@langchain/core/callbacks/manager";
 import { BaseChatModel, type BaseChatModelParams } from "@langchain/core/language_models/chat_models";
 import { ChatGenerationChunk, ChatResult } from "@langchain/core/outputs";
+import { AIMessageChunk } from "@langchain/core/messages";
 import { BaseLanguageModelInput, StructuredOutputMethodOptions } from "@langchain/core/language_models/base";
 import type { z } from "zod";
-import { Runnable } from "@langchain/core/runnables";
+import { Runnable, RunnableInterface } from "@langchain/core/runnables";
+import { StructuredToolInterface } from "@langchain/core/tools";
 import { GoogleAIBaseLLMInput, GoogleAIModelParams, GoogleAISafetySetting, GoogleConnectionParams, GooglePlatformType, GeminiContent, GoogleAIBaseLanguageModelCallOptions } from "./types.js";
 import { AbstractGoogleLLMConnection } from "./connection.js";
 import { GoogleAbstractedClient } from "./auth.js";
@@ -20,10 +22,9 @@ export interface ChatGoogleBaseInput<AuthOptions> extends BaseChatModelParams, G
 /**
  * Integration with a chat model.
  */
-export declare abstract class ChatGoogleBase<AuthOptions> extends BaseChatModel<GoogleAIBaseLanguageModelCallOptions> implements ChatGoogleBaseInput<AuthOptions> {
+export declare abstract class ChatGoogleBase<AuthOptions> extends BaseChatModel<GoogleAIBaseLanguageModelCallOptions, AIMessageChunk> implements ChatGoogleBaseInput<AuthOptions> {
     static lc_name(): string;
     lc_serializable: boolean;
-    /** @deprecated Prefer `modelName` */
     model: string;
     modelName: string;
     temperature: number;
@@ -42,7 +43,12 @@ export declare abstract class ChatGoogleBase<AuthOptions> extends BaseChatModel<
     buildClient(fields?: GoogleAIBaseLLMInput<AuthOptions>): GoogleAbstractedClient;
     buildConnection(fields: GoogleBaseLLMInput<AuthOptions>, client: GoogleAbstractedClient): void;
     get platform(): GooglePlatformType;
+    bindTools(tools: (StructuredToolInterface | Record<string, unknown>)[], kwargs?: Partial<GoogleAIBaseLanguageModelCallOptions>): RunnableInterface<BaseLanguageModelInput, AIMessageChunk, GoogleAIBaseLanguageModelCallOptions>;
     _llmType(): string;
+    /**
+     * Get the parameters used to invoke the model
+     */
+    invocationParams(options?: this["ParsedCallOptions"]): import("./types.js").GoogleAIModelRequestParams;
     _generate(messages: BaseMessage[], options: this["ParsedCallOptions"], _runManager: CallbackManagerForLLMRun | undefined): Promise<ChatResult>;
     _streamResponseChunks(_messages: BaseMessage[], options: this["ParsedCallOptions"], _runManager?: CallbackManagerForLLMRun): AsyncGenerator<ChatGenerationChunk>;
     /** @ignore */

package/dist/chat_models.js

@@ -4,6 +4,7 @@ import { ChatGenerationChunk } from "@langchain/core/outputs";
 import { AIMessageChunk } from "@langchain/core/messages";
 import { RunnablePassthrough, RunnableSequence, } from "@langchain/core/runnables";
 import { JsonOutputKeyToolsParser } from "@langchain/core/output_parsers/openai_tools";
+import { isStructuredTool } from "@langchain/core/utils/function_calling";
 import { copyAIModelParams, copyAndValidateModelParamsInto, } from "./utils/common.js";
 import { AbstractGoogleLLMConnection } from "./connection.js";
 import { baseMessageToContent, safeResponseToChatGeneration, safeResponseToChatResult, DefaultGeminiSafetyHandler, } from "./utils/gemini.js";
@@ -13,10 +14,27 @@ import { zodToGeminiParameters } from "./utils/zod_to_gemini_parameters.js";
 class ChatConnection extends AbstractGoogleLLMConnection {
     formatContents(input, _parameters) {
         return input
-            .map((msg) => baseMessageToContent(msg))
+            .map((msg, i) => baseMessageToContent(msg, input[i - 1]))
             .reduce((acc, cur) => [...acc, ...cur]);
     }
 }
+function convertToGeminiTools(structuredTools) {
+    return [
+        {
+            functionDeclarations: structuredTools.map((structuredTool) => {
+                if (isStructuredTool(structuredTool)) {
+                    const jsonSchema = zodToGeminiParameters(structuredTool.schema);
+                    return {
+                        name: structuredTool.name,
+                        description: structuredTool.description,
+                        parameters: jsonSchema,
+                    };
+                }
+                return structuredTool;
+            }),
+        },
+    ];
+}
 /**
  * Integration with a chat model.
  */
@@ -33,7 +51,6 @@ export class ChatGoogleBase extends BaseChatModel {
             writable: true,
             value: true
         });
-        /** @deprecated Prefer `modelName` */
         Object.defineProperty(this, "model", {
             enumerable: true,
             configurable: true,
@@ -128,19 +145,28 @@ export class ChatGoogleBase extends BaseChatModel {
     get platform() {
         return this.connection.platform;
     }
+    bindTools(tools, kwargs) {
+        return this.bind({ tools: convertToGeminiTools(tools), ...kwargs });
+    }
     // Replace
     _llmType() {
         return "chat_integration";
     }
+    /**
+     * Get the parameters used to invoke the model
+     */
+    invocationParams(options) {
+        return copyAIModelParams(this, options);
+    }
     async _generate(messages, options, _runManager) {
-        const parameters = copyAIModelParams(this, options);
+        const parameters = this.invocationParams(options);
         const response = await this.connection.request(messages, parameters, options);
         const ret = safeResponseToChatResult(response, this.safetyHandler);
         return ret;
     }
     async *_streamResponseChunks(_messages, options, _runManager) {
         // Make the call as a streaming request
-        const parameters = copyAIModelParams(this, options);
+        const parameters = this.invocationParams(options);
         const response = await this.streamedConnection.request(_messages, parameters, options);
         // Get the streaming parser of the response
         const stream = response.data;

package/dist/connection.cjs

@@ -124,7 +124,6 @@ exports.GoogleHostConnection = GoogleHostConnection;
 class GoogleAIConnection extends GoogleHostConnection {
     constructor(fields, caller, client, streaming) {
         super(fields, caller, client, streaming);
-        /** @deprecated Prefer `modelName` */
         Object.defineProperty(this, "model", {
             enumerable: true,
             configurable: true,
@@ -144,10 +143,11 @@ class GoogleAIConnection extends GoogleHostConnection {
             value: void 0
         });
         this.client = client;
-        this.modelName = fields?.modelName ?? fields?.model ?? this.modelName;
+        this.modelName = fields?.model ?? fields?.modelName ?? this.model;
+        this.model = this.modelName;
     }
     get modelFamily() {
-        if (this.modelName.startsWith("gemini")) {
+        if (this.model.startsWith("gemini")) {
             return "gemini";
         }
         else {
@@ -164,13 +164,13 @@ class GoogleAIConnection extends GoogleHostConnection {
     }
     async buildUrlGenerativeLanguage() {
         const method = await this.buildUrlMethod();
-        const url = `https://generativelanguage.googleapis.com/${this.apiVersion}/models/${this.modelName}:${method}`;
+        const url = `https://generativelanguage.googleapis.com/${this.apiVersion}/models/${this.model}:${method}`;
         return url;
     }
     async buildUrlVertex() {
         const projectId = await this.client.getProjectId();
         const method = await this.buildUrlMethod();
-        const url = `https://${this.endpoint}/${this.apiVersion}/projects/${projectId}/locations/${this.location}/publishers/google/models/${this.modelName}:${method}`;
+        const url = `https://${this.endpoint}/${this.apiVersion}/projects/${projectId}/locations/${this.location}/publishers/google/models/${this.model}:${method}`;
         return url;
     }
     async buildUrl() {

package/dist/connection.d.ts

@@ -26,7 +26,6 @@ export declare abstract class GoogleHostConnection<CallOptions extends AsyncCall
     buildMethod(): GoogleAbstractedClientOpsMethod;
 }
 export declare abstract class GoogleAIConnection<CallOptions extends BaseLanguageModelCallOptions, MessageType, AuthOptions> extends GoogleHostConnection<CallOptions, GoogleLLMResponse, AuthOptions> implements GoogleAIBaseLLMInput<AuthOptions> {
-    /** @deprecated Prefer `modelName` */
     model: string;
     modelName: string;
     client: GoogleAbstractedClient;

package/dist/connection.js

@@ -119,7 +119,6 @@ export class GoogleHostConnection extends GoogleConnection {
 export class GoogleAIConnection extends GoogleHostConnection {
     constructor(fields, caller, client, streaming) {
         super(fields, caller, client, streaming);
-        /** @deprecated Prefer `modelName` */
         Object.defineProperty(this, "model", {
             enumerable: true,
             configurable: true,
@@ -139,10 +138,11 @@ export class GoogleAIConnection extends GoogleHostConnection {
             value: void 0
         });
         this.client = client;
-        this.modelName = fields?.modelName ?? fields?.model ?? this.modelName;
+        this.modelName = fields?.model ?? fields?.modelName ?? this.model;
+        this.model = this.modelName;
     }
     get modelFamily() {
-        if (this.modelName.startsWith("gemini")) {
+        if (this.model.startsWith("gemini")) {
             return "gemini";
         }
         else {
@@ -159,13 +159,13 @@ export class GoogleAIConnection extends GoogleHostConnection {
     }
     async buildUrlGenerativeLanguage() {
         const method = await this.buildUrlMethod();
-        const url = `https://generativelanguage.googleapis.com/${this.apiVersion}/models/${this.modelName}:${method}`;
+        const url = `https://generativelanguage.googleapis.com/${this.apiVersion}/models/${this.model}:${method}`;
         return url;
     }
     async buildUrlVertex() {
         const projectId = await this.client.getProjectId();
         const method = await this.buildUrlMethod();
-        const url = `https://${this.endpoint}/${this.apiVersion}/projects/${projectId}/locations/${this.location}/publishers/google/models/${this.modelName}:${method}`;
+        const url = `https://${this.endpoint}/${this.apiVersion}/projects/${projectId}/locations/${this.location}/publishers/google/models/${this.model}:${method}`;
         return url;
     }
     async buildUrl() {

package/dist/llms.cjs

@@ -59,6 +59,12 @@ class GoogleBaseLLM extends llms_1.LLM {
             writable: true,
             value: "gemini-pro"
         });
+        Object.defineProperty(this, "model", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "gemini-pro"
+        });
         Object.defineProperty(this, "temperature", {
             enumerable: true,
             configurable: true,

package/dist/llms.d.ts

@@ -19,6 +19,7 @@ export declare abstract class GoogleBaseLLM<AuthOptions> extends LLM<BaseLanguag
     originalFields?: GoogleBaseLLMInput<AuthOptions>;
     lc_serializable: boolean;
     modelName: string;
+    model: string;
     temperature: number;
     maxOutputTokens: number;
     topP: number;

package/dist/llms.js

@@ -56,6 +56,12 @@ export class GoogleBaseLLM extends LLM {
             writable: true,
             value: "gemini-pro"
         });
+        Object.defineProperty(this, "model", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "gemini-pro"
+        });
         Object.defineProperty(this, "temperature", {
             enumerable: true,
             configurable: true,

package/dist/types.d.ts

@@ -38,9 +38,12 @@ export interface GoogleAISafetySetting {
     threshold: string;
 }
 export interface GoogleAIModelParams {
-    /** @deprecated Prefer `modelName` */
-    model?: string;
     /** Model to use */
+    model?: string;
+    /**
+     * Model to use
+     * Alias for `model`
+     */
     modelName?: string;
     /** Sampling temperature to use */
     temperature?: number;

package/dist/utils/common.cjs

@@ -8,7 +8,10 @@ function copyAIModelParams(params, options) {
 exports.copyAIModelParams = copyAIModelParams;
 function copyAIModelParamsInto(params, options, target) {
     const ret = target || {};
-    ret.modelName = options?.modelName ?? params?.modelName ?? target.modelName;
+    const model = options?.model ?? params?.model ?? target.model;
+    ret.modelName =
+        model ?? options?.modelName ?? params?.modelName ?? target.modelName;
+    ret.model = model;
     ret.temperature =
         options?.temperature ?? params?.temperature ?? target.temperature;
     ret.maxOutputTokens =
@@ -74,6 +77,7 @@ function copyAIModelParamsInto(params, options, target) {
         throw new Error(`Cannot mix structured tools with Gemini tools.\nReceived ${structuredOutputTools.length} structured tools and ${geminiTools.length} Gemini tools.`);
     }
     ret.tools = geminiTools ?? structuredOutputTools;
+    console.log(ret);
     return ret;
 }
 exports.copyAIModelParamsInto = copyAIModelParamsInto;
@@ -91,7 +95,8 @@ function modelToFamily(modelName) {
 exports.modelToFamily = modelToFamily;
 function validateModelParams(params) {
     const testParams = params ?? {};
-    switch (modelToFamily(testParams.modelName)) {
+    const model = testParams.model ?? testParams.modelName;
+    switch (modelToFamily(model)) {
         case "gemini":
             return (0, gemini_js_1.validateGeminiParams)(testParams);
         default:

package/dist/utils/common.js

@@ -4,7 +4,10 @@ export function copyAIModelParams(params, options) {
 }
 export function copyAIModelParamsInto(params, options, target) {
     const ret = target || {};
-    ret.modelName = options?.modelName ?? params?.modelName ?? target.modelName;
+    const model = options?.model ?? params?.model ?? target.model;
+    ret.modelName =
+        model ?? options?.modelName ?? params?.modelName ?? target.modelName;
+    ret.model = model;
     ret.temperature =
         options?.temperature ?? params?.temperature ?? target.temperature;
     ret.maxOutputTokens =
@@ -70,6 +73,7 @@ export function copyAIModelParamsInto(params, options, target) {
         throw new Error(`Cannot mix structured tools with Gemini tools.\nReceived ${structuredOutputTools.length} structured tools and ${geminiTools.length} Gemini tools.`);
     }
     ret.tools = geminiTools ?? structuredOutputTools;
+    console.log(ret);
     return ret;
 }
 export function modelToFamily(modelName) {
@@ -85,7 +89,8 @@ export function modelToFamily(modelName) {
 }
 export function validateModelParams(params) {
     const testParams = params ?? {};
-    switch (modelToFamily(testParams.modelName)) {
+    const model = testParams.model ?? testParams.modelName;
+    switch (modelToFamily(model)) {
         case "gemini":
             return validateGeminiParams(testParams);
         default:

package/dist/utils/gemini.cjs

@@ -1,9 +1,19 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.MessageGeminiSafetyHandler = exports.DefaultGeminiSafetyHandler = exports.isModelGemini = exports.validateGeminiParams = exports.safeResponseToChatResult = exports.responseToChatResult = exports.safeResponseToBaseMessage = exports.responseToBaseMessage = exports.partsToBaseMessageFields = exports.responseToBaseMessageFields = exports.responseToChatGenerations = exports.partToChatGeneration = exports.partToMessage = exports.chunkToString = exports.safeResponseToChatGeneration = exports.responseToChatGeneration = exports.safeResponseToGeneration = exports.responseToGeneration = exports.safeResponseToString = exports.responseToString = exports.partToText = exports.responseToParts = exports.responseToGenerateContentResponseData = exports.toolsRawToTools = exports.partsToToolsRaw = exports.partsToMessageContent = exports.baseMessageToContent = exports.messageContentToParts = void 0;
+exports.MessageGeminiSafetyHandler = exports.DefaultGeminiSafetyHandler = exports.isModelGemini = exports.validateGeminiParams = exports.safeResponseToChatResult = exports.responseToChatResult = exports.safeResponseToBaseMessage = exports.responseToBaseMessage = exports.partsToBaseMessageFields = exports.responseToBaseMessageFields = exports.responseToChatGenerations = exports.partToChatGeneration = exports.partToMessageChunk = exports.chunkToString = exports.safeResponseToChatGeneration = exports.responseToChatGeneration = exports.safeResponseToGeneration = exports.responseToGeneration = exports.safeResponseToString = exports.responseToString = exports.partToText = exports.responseToParts = exports.responseToGenerateContentResponseData = exports.toolsRawToTools = exports.partsToToolsRaw = exports.partsToMessageContent = exports.baseMessageToContent = exports.messageContentToParts = void 0;
+const uuid_1 = require("uuid");
 const messages_1 = require("@langchain/core/messages");
 const outputs_1 = require("@langchain/core/outputs");
 const safety_js_1 = require("./safety.cjs");
+const extractMimeType = (str) => {
+    if (str.startsWith("data:")) {
+        return {
+            mimeType: str.split(":")[1].split(";")[0],
+            data: str.split(",")[1],
+        };
+    }
+    return null;
+};
 function messageContentText(content) {
     if (content?.text && content?.text.length > 0) {
         return {
@@ -21,12 +31,10 @@ function messageContentImageUrl(content) {
     if (!url) {
         throw new Error("Missing Image URL");
     }
-    if (url.startsWith("data:")) {
+    const mineTypeAndData = extractMimeType(url);
+    if (mineTypeAndData) {
         return {
-            inlineData: {
-                mimeType: url.split(":")[1].split(";")[0],
-                data: url.split(",")[1],
-            },
+            inlineData: mineTypeAndData,
         };
     }
     else {
@@ -39,6 +47,27 @@ function messageContentImageUrl(content) {
         };
     }
 }
+function messageContentMedia(
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+content) {
+    if ("mimeType" in content && "data" in content) {
+        return {
+            inlineData: {
+                mimeType: content.mimeType,
+                data: content.data,
+            },
+        };
+    }
+    else if ("mimeType" in content && "fileUri" in content) {
+        return {
+            fileData: {
+                mimeType: content.mimeType,
+                fileUri: content.fileUri,
+            },
+        };
+    }
+    throw new Error("Invalid media content");
+}
 function messageContentToParts(content) {
     // Convert a string to a text type MessageContent if needed
     const messageContent = typeof content === "string"
@@ -64,6 +93,8 @@ function messageContentToParts(content) {
                     return messageContentImageUrl(content);
                 }
                 break;
+            case "media":
+                return messageContentMedia(content);
             default:
                 throw new Error(`Unsupported type received while converting message to message parts`);
         }
@@ -107,7 +138,18 @@ function messageKwargsToParts(kwargs) {
 }
 function roleMessageToContent(role, message) {
     const contentParts = messageContentToParts(message.content);
-    const toolParts = messageKwargsToParts(message.additional_kwargs);
+    let toolParts;
+    if ((0, messages_1.isAIMessage)(message) && !!message.tool_calls?.length) {
+        toolParts = message.tool_calls.map((toolCall) => ({
+            functionCall: {
+                name: toolCall.name,
+                args: toolCall.args,
+            },
+        }));
+    }
+    else {
+        toolParts = messageKwargsToParts(message.additional_kwargs);
+    }
     const parts = [...contentParts, ...toolParts];
     return [
         {
@@ -122,7 +164,7 @@ function systemMessageToContent(message) {
         ...roleMessageToContent("model", new messages_1.AIMessage("Ok")),
     ];
 }
-function toolMessageToContent(message) {
+function toolMessageToContent(message, prevMessage) {
     const contentStr = typeof message.content === "string"
         ? message.content
         : message.content.reduce((acc, content) => {
@@ -133,6 +175,11 @@ function toolMessageToContent(message) {
                 return acc;
             }
         }, "");
+    // Hacky :(
+    const responseName = ((0, messages_1.isAIMessage)(prevMessage) && !!prevMessage.tool_calls?.length
+        ? prevMessage.tool_calls[0].name
+        : prevMessage.name) ?? message.tool_call_id;
+    console.log(contentStr);
     try {
         const content = JSON.parse(contentStr);
         return [
@@ -141,8 +188,8 @@ function toolMessageToContent(message) {
                 parts: [
                     {
                         functionResponse: {
-                            name: message.tool_call_id,
-                            response: content,
+                            name: responseName,
+                            response: { content },
                         },
                     },
                 ],
@@ -156,10 +203,8 @@ function toolMessageToContent(message) {
                 parts: [
                     {
                         functionResponse: {
-                            name: message.tool_call_id,
-                            response: {
-                                response: contentStr,
-                            },
+                            name: responseName,
+                            response: { content: contentStr },
                         },
                     },
                 ],
@@ -167,7 +212,7 @@ function toolMessageToContent(message) {
         ];
     }
 }
-function baseMessageToContent(message) {
+function baseMessageToContent(message, prevMessage) {
     const type = message._getType();
     switch (type) {
         case "system":
@@ -177,7 +222,10 @@ function baseMessageToContent(message) {
         case "ai":
             return roleMessageToContent("model", message);
         case "tool":
-            return toolMessageToContent(message);
+            if (!prevMessage) {
+                throw new Error("Tool messages cannot be the first message passed to the model.");
+            }
+            return toolMessageToContent(message, prevMessage);
         default:
             console.log(`Unsupported message type: ${type}`);
             return [];
@@ -241,7 +289,7 @@ function toolRawToTool(raw) {
 }
 function functionCallPartToToolRaw(part) {
     return {
-        id: part?.functionCall?.name ?? "",
+        id: (0, uuid_1.v4)().replace(/-/g, ""),
         type: "function",
         function: {
             name: part.functionCall.name,
@@ -346,7 +394,7 @@ exports.safeResponseToGeneration = safeResponseToGeneration;
 function responseToChatGeneration(response) {
     return new outputs_1.ChatGenerationChunk({
         text: responseToString(response),
-        message: partToMessage(responseToParts(response)[0]),
+        message: partToMessageChunk(responseToParts(response)[0]),
         generationInfo: response,
     });
 }
@@ -373,7 +421,7 @@ function chunkToString(chunk) {
     }
 }
 exports.chunkToString = chunkToString;
-function partToMessage(part) {
+function partToMessageChunk(part) {
     const fields = partsToBaseMessageFields([part]);
     if (typeof fields.content === "string") {
         return new messages_1.AIMessageChunk(fields);
@@ -389,9 +437,9 @@ function partToMessage(part) {
     }
     return new messages_1.AIMessageChunk(fields);
 }
-exports.partToMessage = partToMessage;
+exports.partToMessageChunk = partToMessageChunk;
 function partToChatGeneration(part) {
-    const message = partToMessage(part);
+    const message = partToMessageChunk(part);
     const text = partToText(part);
     return new outputs_1.ChatGenerationChunk({
         text,
@@ -405,11 +453,18 @@ function responseToChatGenerations(response) {
     if (ret.every((item) => typeof item.message.content === "string")) {
         const combinedContent = ret.map((item) => item.message.content).join("");
         const combinedText = ret.map((item) => item.text).join("");
+        const toolCallChunks = ret[ret.length - 1].message.additional_kwargs?.tool_calls?.map((toolCall, i) => ({
+            name: toolCall.function.name,
+            args: toolCall.function.arguments,
+            id: toolCall.id,
+            index: i,
+        }));
         ret = [
             new outputs_1.ChatGenerationChunk({
                 message: new messages_1.AIMessageChunk({
                     content: combinedContent,
                     additional_kwargs: ret[ret.length - 1].message.additional_kwargs,
+                    tool_call_chunks: toolCallChunks,
                 }),
                 text: combinedText,
                 generationInfo: ret[ret.length - 1].generationInfo,
@@ -427,10 +482,30 @@ exports.responseToBaseMessageFields = responseToBaseMessageFields;
 function partsToBaseMessageFields(parts) {
     const fields = {
         content: partsToMessageContent(parts),
+        tool_calls: [],
+        invalid_tool_calls: [],
     };
     const rawTools = partsToToolsRaw(parts);
     if (rawTools.length > 0) {
         const tools = toolsRawToTools(rawTools);
+        for (const tool of tools) {
+            try {
+                fields.tool_calls?.push({
+                    name: tool.function.name,
+                    args: JSON.parse(tool.function.arguments),
+                    id: tool.id,
+                });
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            }
+            catch (e) {
+                fields.invalid_tool_calls?.push({
+                    name: tool.function.name,
+                    args: JSON.parse(tool.function.arguments),
+                    id: tool.id,
+                    error: e.message,
+                });
+            }
+        }
         fields.additional_kwargs = {
             tool_calls: tools,
         };

package/dist/utils/gemini.d.ts

@@ -1,8 +1,8 @@
-import { BaseMessage, BaseMessageChunk, BaseMessageFields, MessageContent } from "@langchain/core/messages";
+import { AIMessageFields, BaseMessage, BaseMessageChunk, BaseMessageFields, MessageContent } from "@langchain/core/messages";
 import { ChatGeneration, ChatGenerationChunk, ChatResult, Generation } from "@langchain/core/outputs";
 import type { GoogleLLMResponse, GoogleAIModelParams, GeminiPart, GeminiContent, GenerateContentResponseData, GoogleAISafetyHandler } from "../types.js";
 export declare function messageContentToParts(content: MessageContent): GeminiPart[];
-export declare function baseMessageToContent(message: BaseMessage): GeminiContent[];
+export declare function baseMessageToContent(message: BaseMessage, prevMessage?: BaseMessage): GeminiContent[];
 export declare function partsToMessageContent(parts: GeminiPart[]): MessageContent;
 interface FunctionCall {
     name: string;
@@ -34,11 +34,11 @@ export declare function safeResponseToGeneration(response: GoogleLLMResponse, sa
 export declare function responseToChatGeneration(response: GoogleLLMResponse): ChatGenerationChunk;
 export declare function safeResponseToChatGeneration(response: GoogleLLMResponse, safetyHandler: GoogleAISafetyHandler): ChatGenerationChunk;
 export declare function chunkToString(chunk: BaseMessageChunk): string;
-export declare function partToMessage(part: GeminiPart): BaseMessageChunk;
+export declare function partToMessageChunk(part: GeminiPart): BaseMessageChunk;
 export declare function partToChatGeneration(part: GeminiPart): ChatGeneration;
 export declare function responseToChatGenerations(response: GoogleLLMResponse): ChatGeneration[];
 export declare function responseToBaseMessageFields(response: GoogleLLMResponse): BaseMessageFields;
-export declare function partsToBaseMessageFields(parts: GeminiPart[]): BaseMessageFields;
+export declare function partsToBaseMessageFields(parts: GeminiPart[]): AIMessageFields;
 export declare function responseToBaseMessage(response: GoogleLLMResponse): BaseMessage;
 export declare function safeResponseToBaseMessage(response: GoogleLLMResponse, safetyHandler: GoogleAISafetyHandler): BaseMessage;
 export declare function responseToChatResult(response: GoogleLLMResponse): ChatResult;

package/dist/utils/gemini.js

@@ -1,6 +1,16 @@
-import { AIMessage, AIMessageChunk, } from "@langchain/core/messages";
+import { v4 as uuidv4 } from "uuid";
+import { AIMessage, AIMessageChunk, isAIMessage, } from "@langchain/core/messages";
 import { ChatGenerationChunk, } from "@langchain/core/outputs";
 import { GoogleAISafetyError } from "./safety.js";
+const extractMimeType = (str) => {
+    if (str.startsWith("data:")) {
+        return {
+            mimeType: str.split(":")[1].split(";")[0],
+            data: str.split(",")[1],
+        };
+    }
+    return null;
+};
 function messageContentText(content) {
     if (content?.text && content?.text.length > 0) {
         return {
@@ -18,12 +28,10 @@ function messageContentImageUrl(content) {
     if (!url) {
         throw new Error("Missing Image URL");
     }
-    if (url.startsWith("data:")) {
+    const mineTypeAndData = extractMimeType(url);
+    if (mineTypeAndData) {
         return {
-            inlineData: {
-                mimeType: url.split(":")[1].split(";")[0],
-                data: url.split(",")[1],
-            },
+            inlineData: mineTypeAndData,
         };
     }
     else {
@@ -36,6 +44,27 @@ function messageContentImageUrl(content) {
         };
     }
 }
+function messageContentMedia(
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+content) {
+    if ("mimeType" in content && "data" in content) {
+        return {
+            inlineData: {
+                mimeType: content.mimeType,
+                data: content.data,
+            },
+        };
+    }
+    else if ("mimeType" in content && "fileUri" in content) {
+        return {
+            fileData: {
+                mimeType: content.mimeType,
+                fileUri: content.fileUri,
+            },
+        };
+    }
+    throw new Error("Invalid media content");
+}
 export function messageContentToParts(content) {
     // Convert a string to a text type MessageContent if needed
     const messageContent = typeof content === "string"
@@ -61,6 +90,8 @@ export function messageContentToParts(content) {
                     return messageContentImageUrl(content);
                 }
                 break;
+            case "media":
+                return messageContentMedia(content);
             default:
                 throw new Error(`Unsupported type received while converting message to message parts`);
         }
@@ -103,7 +134,18 @@ function messageKwargsToParts(kwargs) {
 }
 function roleMessageToContent(role, message) {
     const contentParts = messageContentToParts(message.content);
-    const toolParts = messageKwargsToParts(message.additional_kwargs);
+    let toolParts;
+    if (isAIMessage(message) && !!message.tool_calls?.length) {
+        toolParts = message.tool_calls.map((toolCall) => ({
+            functionCall: {
+                name: toolCall.name,
+                args: toolCall.args,
+            },
+        }));
+    }
+    else {
+        toolParts = messageKwargsToParts(message.additional_kwargs);
+    }
     const parts = [...contentParts, ...toolParts];
     return [
         {
@@ -118,7 +160,7 @@ function systemMessageToContent(message) {
         ...roleMessageToContent("model", new AIMessage("Ok")),
     ];
 }
-function toolMessageToContent(message) {
+function toolMessageToContent(message, prevMessage) {
     const contentStr = typeof message.content === "string"
         ? message.content
         : message.content.reduce((acc, content) => {
@@ -129,6 +171,11 @@ function toolMessageToContent(message) {
                 return acc;
             }
         }, "");
+    // Hacky :(
+    const responseName = (isAIMessage(prevMessage) && !!prevMessage.tool_calls?.length
+        ? prevMessage.tool_calls[0].name
+        : prevMessage.name) ?? message.tool_call_id;
+    console.log(contentStr);
     try {
         const content = JSON.parse(contentStr);
         return [
@@ -137,8 +184,8 @@ function toolMessageToContent(message) {
                 parts: [
                     {
                         functionResponse: {
-                            name: message.tool_call_id,
-                            response: content,
+                            name: responseName,
+                            response: { content },
                         },
                     },
                 ],
@@ -152,10 +199,8 @@ function toolMessageToContent(message) {
                 parts: [
                     {
                         functionResponse: {
-                            name: message.tool_call_id,
-                            response: {
-                                response: contentStr,
-                            },
+                            name: responseName,
+                            response: { content: contentStr },
                         },
                     },
                 ],
@@ -163,7 +208,7 @@ function toolMessageToContent(message) {
         ];
     }
 }
-export function baseMessageToContent(message) {
+export function baseMessageToContent(message, prevMessage) {
     const type = message._getType();
     switch (type) {
         case "system":
@@ -173,7 +218,10 @@ export function baseMessageToContent(message) {
         case "ai":
             return roleMessageToContent("model", message);
         case "tool":
-            return toolMessageToContent(message);
+            if (!prevMessage) {
+                throw new Error("Tool messages cannot be the first message passed to the model.");
+            }
+            return toolMessageToContent(message, prevMessage);
         default:
             console.log(`Unsupported message type: ${type}`);
             return [];
@@ -235,7 +283,7 @@ function toolRawToTool(raw) {
 }
 function functionCallPartToToolRaw(part) {
     return {
-        id: part?.functionCall?.name ?? "",
+        id: uuidv4().replace(/-/g, ""),
         type: "function",
         function: {
             name: part.functionCall.name,
@@ -331,7 +379,7 @@ export function safeResponseToGeneration(response, safetyHandler) {
 export function responseToChatGeneration(response) {
     return new ChatGenerationChunk({
         text: responseToString(response),
-        message: partToMessage(responseToParts(response)[0]),
+        message: partToMessageChunk(responseToParts(response)[0]),
         generationInfo: response,
     });
 }
@@ -355,7 +403,7 @@ export function chunkToString(chunk) {
         throw new Error(`Unexpected chunk: ${chunk}`);
     }
 }
-export function partToMessage(part) {
+export function partToMessageChunk(part) {
     const fields = partsToBaseMessageFields([part]);
     if (typeof fields.content === "string") {
         return new AIMessageChunk(fields);
@@ -372,7 +420,7 @@ export function partToMessage(part) {
     return new AIMessageChunk(fields);
 }
 export function partToChatGeneration(part) {
-    const message = partToMessage(part);
+    const message = partToMessageChunk(part);
     const text = partToText(part);
     return new ChatGenerationChunk({
         text,
@@ -385,11 +433,18 @@ export function responseToChatGenerations(response) {
     if (ret.every((item) => typeof item.message.content === "string")) {
         const combinedContent = ret.map((item) => item.message.content).join("");
         const combinedText = ret.map((item) => item.text).join("");
+        const toolCallChunks = ret[ret.length - 1].message.additional_kwargs?.tool_calls?.map((toolCall, i) => ({
+            name: toolCall.function.name,
+            args: toolCall.function.arguments,
+            id: toolCall.id,
+            index: i,
+        }));
         ret = [
             new ChatGenerationChunk({
                 message: new AIMessageChunk({
                     content: combinedContent,
                     additional_kwargs: ret[ret.length - 1].message.additional_kwargs,
+                    tool_call_chunks: toolCallChunks,
                 }),
                 text: combinedText,
                 generationInfo: ret[ret.length - 1].generationInfo,
@@ -405,10 +460,30 @@ export function responseToBaseMessageFields(response) {
 export function partsToBaseMessageFields(parts) {
     const fields = {
         content: partsToMessageContent(parts),
+        tool_calls: [],
+        invalid_tool_calls: [],
     };
     const rawTools = partsToToolsRaw(parts);
     if (rawTools.length > 0) {
         const tools = toolsRawToTools(rawTools);
+        for (const tool of tools) {
+            try {
+                fields.tool_calls?.push({
+                    name: tool.function.name,
+                    args: JSON.parse(tool.function.arguments),
+                    id: tool.id,
+                });
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            }
+            catch (e) {
+                fields.invalid_tool_calls?.push({
+                    name: tool.function.name,
+                    args: JSON.parse(tool.function.arguments),
+                    id: tool.id,
+                    error: e.message,
+                });
+            }
+        }
         fields.additional_kwargs = {
             tool_calls: tools,
         };

package/dist/utils/zod_to_gemini_parameters.cjs

@@ -1,4 +1,5 @@
 "use strict";
+/* eslint-disable @typescript-eslint/no-unused-vars */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.zodToGeminiParameters = void 0;
 const zod_to_json_schema_1 = require("zod-to-json-schema");

package/dist/utils/zod_to_gemini_parameters.js

@@ -1,3 +1,4 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
 import { zodToJsonSchema } from "zod-to-json-schema";
 export function zodToGeminiParameters(
 // eslint-disable-next-line @typescript-eslint/no-explicit-any

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/google-common",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "description": "Core types and classes for Google services.",
   "type": "module",
   "engines": {
@@ -39,7 +39,8 @@
   "author": "LangChain",
   "license": "MIT",
   "dependencies": {
-    "@langchain/core": "~0.1.1",
+    "@langchain/core": "~0.1.56",
+    "uuid": "^9.0.0",
     "zod-to-json-schema": "^3.22.4"
   },
   "devDependencies": {