@langchain/google-genai 0.1.3 → 0.1.5

package/dist/chat_models.cjs

@@ -442,6 +442,12 @@ class ChatGoogleGenerativeAI extends chat_models_1.BaseChatModel {
             writable: true,
             value: true
         });
+        Object.defineProperty(this, "convertSystemMessageToHumanContent", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "client", {
             enumerable: true,
             configurable: true,
@@ -506,6 +512,35 @@ class ChatGoogleGenerativeAI extends chat_models_1.BaseChatModel {
         });
         this.streamUsage = fields?.streamUsage ?? this.streamUsage;
     }
+    useCachedContent(cachedContent, modelParams, requestOptions) {
+        if (!this.apiKey)
+            return;
+        this.client = new generative_ai_1.GoogleGenerativeAI(this.apiKey).getGenerativeModelFromCachedContent(cachedContent, modelParams, requestOptions);
+    }
+    get useSystemInstruction() {
+        return typeof this.convertSystemMessageToHumanContent === "boolean"
+            ? !this.convertSystemMessageToHumanContent
+            : this.computeUseSystemInstruction;
+    }
+    get computeUseSystemInstruction() {
+        // This works on models from April 2024 and later
+        //   Vertex AI: gemini-1.5-pro and gemini-1.0-002 and later
+        //   AI Studio: gemini-1.5-pro-latest
+        if (this.modelName === "gemini-1.0-pro-001") {
+            return false;
+        }
+        else if (this.modelName.startsWith("gemini-pro-vision")) {
+            return false;
+        }
+        else if (this.modelName.startsWith("gemini-1.0-pro-vision")) {
+            return false;
+        }
+        else if (this.modelName === "gemini-pro") {
+            // on AI Studio gemini-pro is still pointing at gemini-1.0-pro-001
+            return false;
+        }
+        return true;
+    }
     getLsParams(options) {
         return {
             ls_provider: "google_genai",
@@ -540,7 +575,13 @@ class ChatGoogleGenerativeAI extends chat_models_1.BaseChatModel {
         };
     }
     async _generate(messages, options, runManager) {
-        const prompt = (0, common_js_1.convertBaseMessagesToContent)(messages, this._isMultimodalModel);
+        const prompt = (0, common_js_1.convertBaseMessagesToContent)(messages, this._isMultimodalModel, this.useSystemInstruction);
+        let actualPrompt = prompt;
+        if (prompt[0].role === "system") {
+            const [systemInstruction] = prompt;
+            this.client.systemInstruction = systemInstruction;
+            actualPrompt = prompt.slice(1);
+        }
         const parameters = this.invocationParams(options);
         // Handle streaming
         if (this.streaming) {
@@ -563,7 +604,7 @@ class ChatGoogleGenerativeAI extends chat_models_1.BaseChatModel {
         }
         const res = await this.completionWithRetry({
             ...parameters,
-            contents: prompt,
+            contents: actualPrompt,
         });
         let usageMetadata;
         if ("usageMetadata" in res.response) {
@@ -581,11 +622,17 @@ class ChatGoogleGenerativeAI extends chat_models_1.BaseChatModel {
         return generationResult;
     }
     async *_streamResponseChunks(messages, options, runManager) {
-        const prompt = (0, common_js_1.convertBaseMessagesToContent)(messages, this._isMultimodalModel);
+        const prompt = (0, common_js_1.convertBaseMessagesToContent)(messages, this._isMultimodalModel, this.useSystemInstruction);
+        let actualPrompt = prompt;
+        if (prompt[0].role === "system") {
+            const [systemInstruction] = prompt;
+            this.client.systemInstruction = systemInstruction;
+            actualPrompt = prompt.slice(1);
+        }
         const parameters = this.invocationParams(options);
         const request = {
             ...parameters,
-            contents: prompt,
+            contents: actualPrompt,
         };
         const stream = await this.caller.callWithOptions({ signal: options?.signal }, async () => {
             const { stream } = await this.client.generateContentStream(request);
@@ -702,6 +749,7 @@ class ChatGoogleGenerativeAI extends chat_models_1.BaseChatModel {
         }
         const llm = this.bind({
             tools,
+            tool_choice: functionName,
         });
         if (!includeRaw) {
             return llm.pipe(outputParser).withConfig({

package/dist/chat_models.d.ts

@@ -1,4 +1,4 @@
-import { GenerateContentRequest, SafetySetting, Part as GenerativeAIPart } from "@google/generative-ai";
+import { GenerateContentRequest, SafetySetting, Part as GenerativeAIPart, ModelParams, RequestOptions, type CachedContent } from "@google/generative-ai";
 import { CallbackManagerForLLMRun } from "@langchain/core/callbacks/manager";
 import { AIMessageChunk, BaseMessage } from "@langchain/core/messages";
 import { ChatGenerationChunk, ChatResult } from "@langchain/core/outputs";
@@ -120,6 +120,14 @@ export interface GoogleGenerativeAIChatInput extends BaseChatModelParams, Pick<G
      * @default false
      */
     json?: boolean;
+    /**
+     * Whether or not model supports system instructions.
+     * The following models support system instructions:
+     * - All Gemini 1.5 Pro model versions
+     * - All Gemini 1.5 Flash model versions
+     * - Gemini 1.0 Pro version gemini-1.0-pro-002
+     */
+    convertSystemMessageToHumanContent?: boolean | undefined;
 }
 /**
  * Google Generative AI chat model integration.
@@ -477,9 +485,13 @@ export declare class ChatGoogleGenerativeAI extends BaseChatModel<GoogleGenerati
     apiKey?: string;
     streaming: boolean;
     streamUsage: boolean;
+    convertSystemMessageToHumanContent: boolean | undefined;
     private client;
     get _isMultimodalModel(): boolean;
     constructor(fields?: GoogleGenerativeAIChatInput);
+    useCachedContent(cachedContent: CachedContent, modelParams?: ModelParams, requestOptions?: RequestOptions): void;
+    get useSystemInstruction(): boolean;
+    get computeUseSystemInstruction(): boolean;
     getLsParams(options: this["ParsedCallOptions"]): LangSmithParams;
     _combineLLMOutput(): never[];
     _llmType(): string;

package/dist/chat_models.js

@@ -439,6 +439,12 @@ export class ChatGoogleGenerativeAI extends BaseChatModel {
             writable: true,
             value: true
         });
+        Object.defineProperty(this, "convertSystemMessageToHumanContent", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "client", {
             enumerable: true,
             configurable: true,
@@ -503,6 +509,35 @@ export class ChatGoogleGenerativeAI extends BaseChatModel {
         });
         this.streamUsage = fields?.streamUsage ?? this.streamUsage;
     }
+    useCachedContent(cachedContent, modelParams, requestOptions) {
+        if (!this.apiKey)
+            return;
+        this.client = new GenerativeAI(this.apiKey).getGenerativeModelFromCachedContent(cachedContent, modelParams, requestOptions);
+    }
+    get useSystemInstruction() {
+        return typeof this.convertSystemMessageToHumanContent === "boolean"
+            ? !this.convertSystemMessageToHumanContent
+            : this.computeUseSystemInstruction;
+    }
+    get computeUseSystemInstruction() {
+        // This works on models from April 2024 and later
+        //   Vertex AI: gemini-1.5-pro and gemini-1.0-002 and later
+        //   AI Studio: gemini-1.5-pro-latest
+        if (this.modelName === "gemini-1.0-pro-001") {
+            return false;
+        }
+        else if (this.modelName.startsWith("gemini-pro-vision")) {
+            return false;
+        }
+        else if (this.modelName.startsWith("gemini-1.0-pro-vision")) {
+            return false;
+        }
+        else if (this.modelName === "gemini-pro") {
+            // on AI Studio gemini-pro is still pointing at gemini-1.0-pro-001
+            return false;
+        }
+        return true;
+    }
     getLsParams(options) {
         return {
             ls_provider: "google_genai",
@@ -537,7 +572,13 @@ export class ChatGoogleGenerativeAI extends BaseChatModel {
         };
     }
     async _generate(messages, options, runManager) {
-        const prompt = convertBaseMessagesToContent(messages, this._isMultimodalModel);
+        const prompt = convertBaseMessagesToContent(messages, this._isMultimodalModel, this.useSystemInstruction);
+        let actualPrompt = prompt;
+        if (prompt[0].role === "system") {
+            const [systemInstruction] = prompt;
+            this.client.systemInstruction = systemInstruction;
+            actualPrompt = prompt.slice(1);
+        }
         const parameters = this.invocationParams(options);
         // Handle streaming
         if (this.streaming) {
@@ -560,7 +601,7 @@ export class ChatGoogleGenerativeAI extends BaseChatModel {
         }
         const res = await this.completionWithRetry({
             ...parameters,
-            contents: prompt,
+            contents: actualPrompt,
         });
         let usageMetadata;
         if ("usageMetadata" in res.response) {
@@ -578,11 +619,17 @@ export class ChatGoogleGenerativeAI extends BaseChatModel {
         return generationResult;
     }
     async *_streamResponseChunks(messages, options, runManager) {
-        const prompt = convertBaseMessagesToContent(messages, this._isMultimodalModel);
+        const prompt = convertBaseMessagesToContent(messages, this._isMultimodalModel, this.useSystemInstruction);
+        let actualPrompt = prompt;
+        if (prompt[0].role === "system") {
+            const [systemInstruction] = prompt;
+            this.client.systemInstruction = systemInstruction;
+            actualPrompt = prompt.slice(1);
+        }
         const parameters = this.invocationParams(options);
         const request = {
             ...parameters,
-            contents: prompt,
+            contents: actualPrompt,
         };
         const stream = await this.caller.callWithOptions({ signal: options?.signal }, async () => {
             const { stream } = await this.client.generateContentStream(request);
@@ -699,6 +746,7 @@ export class ChatGoogleGenerativeAI extends BaseChatModel {
         }
         const llm = this.bind({
             tools,
+            tool_choice: functionName,
         });
         if (!includeRaw) {
             return llm.pipe(outputParser).withConfig({

package/dist/utils/common.cjs

@@ -33,6 +33,7 @@ function convertAuthorToRole(author) {
         case "model": // getMessageAuthor returns message.name. code ex.: return message.name ?? type;
             return "model";
         case "system":
+            return "system";
         case "human":
             return "user";
         case "tool":
@@ -144,7 +145,7 @@ function convertMessageContentToParts(message, isMultimodalModel) {
     return [...messageParts, ...functionCalls, ...functionResponses];
 }
 exports.convertMessageContentToParts = convertMessageContentToParts;
-function convertBaseMessagesToContent(messages, isMultimodalModel) {
+function convertBaseMessagesToContent(messages, isMultimodalModel, convertSystemMessageToHumanContent = false) {
     return messages.reduce((acc, message, index) => {
         if (!(0, messages_1.isBaseMessage)(message)) {
             throw new Error("Unsupported message input");
@@ -173,7 +174,8 @@ function convertBaseMessagesToContent(messages, isMultimodalModel) {
             };
         }
         let actualRole = role;
-        if (actualRole === "function") {
+        if (actualRole === "function" ||
+            (actualRole === "system" && !convertSystemMessageToHumanContent)) {
             // GenerativeAI API will throw an error if the role is not "user" or "model."
             actualRole = "user";
         }
@@ -182,7 +184,7 @@ function convertBaseMessagesToContent(messages, isMultimodalModel) {
             parts,
         };
         return {
-            mergeWithPreviousContent: author === "system",
+            mergeWithPreviousContent: author === "system" && !convertSystemMessageToHumanContent,
             content: [...acc.content, content],
         };
     }, { content: [], mergeWithPreviousContent: false }).content;

package/dist/utils/common.d.ts

@@ -11,7 +11,7 @@ export declare function getMessageAuthor(message: BaseMessage): string;
  */
 export declare function convertAuthorToRole(author: string): (typeof POSSIBLE_ROLES)[number];
 export declare function convertMessageContentToParts(message: BaseMessage, isMultimodalModel: boolean): Part[];
-export declare function convertBaseMessagesToContent(messages: BaseMessage[], isMultimodalModel: boolean): Content[];
+export declare function convertBaseMessagesToContent(messages: BaseMessage[], isMultimodalModel: boolean, convertSystemMessageToHumanContent?: boolean): Content[];
 export declare function mapGenerateContentResultToChatResult(response: EnhancedGenerateContentResponse, extra?: {
     usageMetadata: UsageMetadata | undefined;
 }): ChatResult;

package/dist/utils/common.js

@@ -29,6 +29,7 @@ export function convertAuthorToRole(author) {
         case "model": // getMessageAuthor returns message.name. code ex.: return message.name ?? type;
             return "model";
         case "system":
+            return "system";
         case "human":
             return "user";
         case "tool":
@@ -138,7 +139,7 @@ export function convertMessageContentToParts(message, isMultimodalModel) {
     }
     return [...messageParts, ...functionCalls, ...functionResponses];
 }
-export function convertBaseMessagesToContent(messages, isMultimodalModel) {
+export function convertBaseMessagesToContent(messages, isMultimodalModel, convertSystemMessageToHumanContent = false) {
     return messages.reduce((acc, message, index) => {
         if (!isBaseMessage(message)) {
             throw new Error("Unsupported message input");
@@ -167,7 +168,8 @@ export function convertBaseMessagesToContent(messages, isMultimodalModel) {
             };
         }
         let actualRole = role;
-        if (actualRole === "function") {
+        if (actualRole === "function" ||
+            (actualRole === "system" && !convertSystemMessageToHumanContent)) {
             // GenerativeAI API will throw an error if the role is not "user" or "model."
             actualRole = "user";
         }
@@ -176,7 +178,7 @@ export function convertBaseMessagesToContent(messages, isMultimodalModel) {
             parts,
         };
         return {
-            mergeWithPreviousContent: author === "system",
+            mergeWithPreviousContent: author === "system" && !convertSystemMessageToHumanContent,
             content: [...acc.content, content],
         };
     }, { content: [], mergeWithPreviousContent: false }).content;

package/dist/utils/tools.cjs

@@ -3,7 +3,9 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.convertToolsToGenAI = void 0;
 const generative_ai_1 = require("@google/generative-ai");
 const function_calling_1 = require("@langchain/core/utils/function_calling");
+const base_1 = require("@langchain/core/language_models/base");
 const common_js_1 = require("./common.cjs");
+const zod_to_genai_parameters_js_1 = require("./zod_to_genai_parameters.cjs");
 function convertToolsToGenAI(tools, extra) {
     // Extract function declaration processing to a separate function
     const genAITools = processTools(tools);
@@ -24,6 +26,15 @@ function processTools(tools) {
                 functionDeclarationTools.push(...convertedTool.functionDeclarations);
             }
         }
+        else if ((0, base_1.isOpenAITool)(tool)) {
+            const { functionDeclarations } = convertOpenAIToolToGenAI(tool);
+            if (functionDeclarations) {
+                functionDeclarationTools.push(...functionDeclarations);
+            }
+            else {
+                throw new Error("Failed to convert OpenAI structured tool to GenerativeAI tool");
+            }
+        }
         else {
             genAITools.push(tool);
         }
@@ -57,6 +68,17 @@ function processTools(tools) {
             : []),
     ];
 }
+function convertOpenAIToolToGenAI(tool) {
+    return {
+        functionDeclarations: [
+            {
+                name: tool.function.name,
+                description: tool.function.description,
+                parameters: (0, zod_to_genai_parameters_js_1.removeAdditionalProperties)(tool.function.parameters),
+            },
+        ],
+    };
+}
 function createToolConfig(genAITools, extra) {
     if (!genAITools.length || !extra)
         return undefined;

package/dist/utils/tools.js

@@ -1,6 +1,8 @@
 import { FunctionCallingMode, } from "@google/generative-ai";
 import { isLangChainTool } from "@langchain/core/utils/function_calling";
+import { isOpenAITool, } from "@langchain/core/language_models/base";
 import { convertToGenerativeAITools } from "./common.js";
+import { removeAdditionalProperties } from "./zod_to_genai_parameters.js";
 export function convertToolsToGenAI(tools, extra) {
     // Extract function declaration processing to a separate function
     const genAITools = processTools(tools);
@@ -20,6 +22,15 @@ function processTools(tools) {
                 functionDeclarationTools.push(...convertedTool.functionDeclarations);
             }
         }
+        else if (isOpenAITool(tool)) {
+            const { functionDeclarations } = convertOpenAIToolToGenAI(tool);
+            if (functionDeclarations) {
+                functionDeclarationTools.push(...functionDeclarations);
+            }
+            else {
+                throw new Error("Failed to convert OpenAI structured tool to GenerativeAI tool");
+            }
+        }
         else {
             genAITools.push(tool);
         }
@@ -53,6 +64,17 @@ function processTools(tools) {
             : []),
     ];
 }
+function convertOpenAIToolToGenAI(tool) {
+    return {
+        functionDeclarations: [
+            {
+                name: tool.function.name,
+                description: tool.function.description,
+                parameters: removeAdditionalProperties(tool.function.parameters),
+            },
+        ],
+    };
+}
 function createToolConfig(genAITools, extra) {
     if (!genAITools.length || !extra)
         return undefined;

package/dist/utils/zod_to_genai_parameters.cjs

@@ -11,6 +11,9 @@ obj) {
         if ("additionalProperties" in newObj) {
             delete newObj.additionalProperties;
         }
+        if ("$schema" in newObj) {
+            delete newObj.$schema;
+        }
         for (const key in newObj) {
             if (key in newObj) {
                 if (Array.isArray(newObj[key])) {

package/dist/utils/zod_to_genai_parameters.js

@@ -8,6 +8,9 @@ obj) {
         if ("additionalProperties" in newObj) {
             delete newObj.additionalProperties;
         }
+        if ("$schema" in newObj) {
+            delete newObj.$schema;
+        }
         for (const key in newObj) {
             if (key in newObj) {
                 if (Array.isArray(newObj[key])) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/google-genai",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "Google Generative AI integration for LangChain.js",
   "type": "module",
   "engines": {