@langchain/google-common 0.0.0 → 0.0.2

package/dist/connection.js

@@ -1,4 +1,5 @@
 import { getRuntimeEnvironment } from "@langchain/core/utils/env";
+import { zodToGeminiParameters } from "./utils/zod_to_gemini_parameters.js";
 export class GoogleConnection {
     constructor(caller, client, streaming) {
         Object.defineProperty(this, "caller", {
@@ -118,12 +119,19 @@ export class GoogleHostConnection extends GoogleConnection {
 export class GoogleAIConnection extends GoogleHostConnection {
     constructor(fields, caller, client, streaming) {
         super(fields, caller, client, streaming);
+        /** @deprecated Prefer `modelName` */
         Object.defineProperty(this, "model", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "modelName", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "client", {
             enumerable: true,
             configurable: true,
@@ -131,10 +139,10 @@ export class GoogleAIConnection extends GoogleHostConnection {
             value: void 0
         });
         this.client = client;
-        this.model = fields?.model ?? this.model;
+        this.modelName = fields?.modelName ?? fields?.model ?? this.modelName;
     }
     get modelFamily() {
-        if (this.model.startsWith("gemini")) {
+        if (this.modelName.startsWith("gemini")) {
             return "gemini";
         }
         else {
@@ -151,13 +159,13 @@ export class GoogleAIConnection extends GoogleHostConnection {
     }
     async buildUrlGenerativeLanguage() {
         const method = await this.buildUrlMethod();
-        const url = `https://generativelanguage.googleapis.com/${this.apiVersion}/models/${this.model}:${method}`;
+        const url = `https://generativelanguage.googleapis.com/${this.apiVersion}/models/${this.modelName}:${method}`;
         return url;
     }
     async buildUrlVertex() {
         const projectId = await this.client.getProjectId();
         const method = await this.buildUrlMethod();
-        const url = `https://${this.endpoint}/${this.apiVersion}/projects/${projectId}/locations/${this.location}/publishers/google/models/${this.model}:${method}`;
+        const url = `https://${this.endpoint}/${this.apiVersion}/projects/${projectId}/locations/${this.location}/publishers/google/models/${this.modelName}:${method}`;
         return url;
     }
     async buildUrl() {
@@ -199,6 +207,38 @@ export class AbstractGoogleLLMConnection extends GoogleAIConnection {
     formatSafetySettings(_input, parameters) {
         return parameters.safetySettings ?? [];
     }
+    // Borrowed from the OpenAI invocation params test
+    isStructuredToolArray(tools) {
+        return (tools !== undefined &&
+            tools.every((tool) => Array.isArray(tool.lc_namespace)));
+    }
+    structuredToolToFunctionDeclaration(tool) {
+        const jsonSchema = zodToGeminiParameters(tool.schema);
+        return {
+            name: tool.name,
+            description: tool.description,
+            parameters: jsonSchema,
+        };
+    }
+    structuredToolsToGeminiTools(tools) {
+        return [
+            {
+                functionDeclarations: tools.map(this.structuredToolToFunctionDeclaration),
+            },
+        ];
+    }
+    formatTools(_input, parameters) {
+        const tools = parameters?.tools;
+        if (!tools || tools.length === 0) {
+            return [];
+        }
+        if (this.isStructuredToolArray(tools)) {
+            return this.structuredToolsToGeminiTools(tools);
+        }
+        else {
+            return tools;
+        }
+    }
     formatData(input, parameters) {
         /*
         const parts = messageContentToParts(input);
@@ -211,11 +251,15 @@ export class AbstractGoogleLLMConnection extends GoogleAIConnection {
         */
         const contents = this.formatContents(input, parameters);
         const generationConfig = this.formatGenerationConfig(input, parameters);
+        const tools = this.formatTools(input, parameters);
         const safetySettings = this.formatSafetySettings(input, parameters);
         const ret = {
             contents,
             generationConfig,
         };
+        if (tools && tools.length) {
+            ret.tools = tools;
+        }
         if (safetySettings && safetySettings.length) {
             ret.safetySettings = safetySettings;
         }

package/dist/index.cjs

@@ -21,3 +21,4 @@ __exportStar(require("./connection.cjs"), exports);
 __exportStar(require("./types.cjs"), exports);
 __exportStar(require("./utils/stream.cjs"), exports);
 __exportStar(require("./utils/common.cjs"), exports);
+__exportStar(require("./utils/zod_to_gemini_parameters.cjs"), exports);

package/dist/index.d.ts

@@ -5,3 +5,4 @@ export * from "./connection.js";
 export * from "./types.js";
 export * from "./utils/stream.js";
 export * from "./utils/common.js";
+export * from "./utils/zod_to_gemini_parameters.js";

package/dist/index.js

@@ -5,3 +5,4 @@ export * from "./connection.js";
 export * from "./types.js";
 export * from "./utils/stream.js";
 export * from "./utils/common.js";
+export * from "./utils/zod_to_gemini_parameters.js";

package/dist/llms.cjs

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.GoogleBaseLLM = void 0;
+const manager_1 = require("@langchain/core/callbacks/manager");
 const llms_1 = require("@langchain/core/language_models/llms");
 const outputs_1 = require("@langchain/core/outputs");
 const env_1 = require("@langchain/core/utils/env");
@@ -8,6 +9,8 @@ const connection_js_1 = require("./connection.cjs");
 const common_js_1 = require("./utils/common.cjs");
 const gemini_js_1 = require("./utils/gemini.cjs");
 const auth_js_1 = require("./auth.cjs");
+const failed_handler_js_1 = require("./utils/failed_handler.cjs");
+const chat_models_js_1 = require("./chat_models.cjs");
 class GoogleLLMConnection extends connection_js_1.AbstractGoogleLLMConnection {
     formatContents(input, _parameters) {
         const parts = (0, gemini_js_1.messageContentToParts)(input);
@@ -20,6 +23,14 @@ class GoogleLLMConnection extends connection_js_1.AbstractGoogleLLMConnection {
         return contents;
     }
 }
+class ProxyChatGoogle extends chat_models_js_1.ChatGoogleBase {
+    constructor(fields) {
+        super(fields);
+    }
+    buildAbstractedClient(fields) {
+        return fields.connection.client;
+    }
+}
 /**
  * Integration with an LLM.
  */
@@ -29,14 +40,20 @@ class GoogleBaseLLM extends llms_1.LLM {
         return "GoogleLLM";
     }
     constructor(fields) {
-        super(fields ?? {});
+        super((0, failed_handler_js_1.ensureParams)(fields));
+        Object.defineProperty(this, "originalFields", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "lc_serializable", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: true
         });
-        Object.defineProperty(this, "model", {
+        Object.defineProperty(this, "modelName", {
             enumerable: true,
             configurable: true,
             writable: true,
@@ -78,6 +95,12 @@ class GoogleBaseLLM extends llms_1.LLM {
             writable: true,
             value: []
         });
+        Object.defineProperty(this, "safetyHandler", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "connection", {
             enumerable: true,
             configurable: true,
@@ -90,7 +113,10 @@ class GoogleBaseLLM extends llms_1.LLM {
             writable: true,
             value: void 0
         });
+        this.originalFields = fields;
         (0, common_js_1.copyAndValidateModelParamsInto)(fields, this);
+        this.safetyHandler =
+            fields?.safetyHandler ?? new gemini_js_1.DefaultGeminiSafetyHandler();
         const client = this.buildClient(fields);
         this.buildConnection(fields ?? {}, client);
     }
@@ -125,38 +151,79 @@ class GoogleBaseLLM extends llms_1.LLM {
     }
     /**
      * For some given input string and options, return a string output.
+     *
+     * Despite the fact that `invoke` is overridden below, we still need this
+     * in order to handle public APi calls to `generate()`.
      */
-    async _call(_prompt, _options, _runManager) {
-        const parameters = (0, common_js_1.copyAIModelParams)(this);
-        const result = await this.connection.request(_prompt, parameters, _options);
-        const ret = (0, gemini_js_1.responseToString)(result);
+    async _call(prompt, options) {
+        const parameters = (0, common_js_1.copyAIModelParams)(this, options);
+        const result = await this.connection.request(prompt, parameters, options);
+        const ret = (0, gemini_js_1.safeResponseToString)(result, this.safetyHandler);
         return ret;
     }
-    async *_streamResponseChunks(_prompt, _options, _runManager) {
-        // Make the call as a streaming request
-        const parameters = (0, common_js_1.copyAIModelParams)(this);
-        const result = await this.streamedConnection.request(_prompt, parameters, _options);
-        // Get the streaming parser of the response
-        const stream = result.data;
-        // Loop until the end of the stream
-        // During the loop, yield each time we get a chunk from the streaming parser
-        // that is either available or added to the queue
-        while (!stream.streamDone) {
-            const output = await stream.nextChunk();
-            const chunk = output !== null
-                ? new outputs_1.GenerationChunk((0, gemini_js_1.responseToGeneration)({ data: output }))
-                : new outputs_1.GenerationChunk({
-                    text: "",
-                    generationInfo: { finishReason: "stop" },
+    // Normally, you should not override this method and instead should override
+    // _streamResponseChunks. We are doing so here to allow for multimodal inputs into
+    // the LLM.
+    async *_streamIterator(input, options) {
+        // TODO: Refactor callback setup and teardown code into core
+        const prompt = llms_1.BaseLLM._convertInputToPromptValue(input);
+        const [runnableConfig, callOptions] = this._separateRunnableConfigFromCallOptions(options);
+        const callbackManager_ = await manager_1.CallbackManager.configure(runnableConfig.callbacks, this.callbacks, runnableConfig.tags, this.tags, runnableConfig.metadata, this.metadata, { verbose: this.verbose });
+        const extra = {
+            options: callOptions,
+            invocation_params: this?.invocationParams(callOptions),
+            batch_size: 1,
+        };
+        const runManagers = await callbackManager_?.handleLLMStart(this.toJSON(), [prompt.toString()], undefined, undefined, extra, undefined, undefined, runnableConfig.runName);
+        let generation = new outputs_1.GenerationChunk({
+            text: "",
+        });
+        const proxyChat = this.createProxyChat();
+        try {
+            for await (const chunk of proxyChat._streamIterator(input, options)) {
+                const stringValue = (0, gemini_js_1.chunkToString)(chunk);
+                const generationChunk = new outputs_1.GenerationChunk({
+                    text: stringValue,
                 });
-            yield chunk;
+                generation = generation.concat(generationChunk);
+                yield stringValue;
+            }
+        }
+        catch (err) {
+            await Promise.all((runManagers ?? []).map((runManager) => runManager?.handleLLMError(err)));
+            throw err;
         }
+        await Promise.all((runManagers ?? []).map((runManager) => runManager?.handleLLMEnd({
+            generations: [[generation]],
+        })));
     }
     async predictMessages(messages, options, _callbacks) {
         const { content } = messages[0];
         const result = await this.connection.request(content, {}, options);
-        const ret = (0, gemini_js_1.responseToBaseMessage)(result);
+        const ret = (0, gemini_js_1.safeResponseToBaseMessage)(result, this.safetyHandler);
         return ret;
     }
+    /**
+     * Internal implementation detail to allow Google LLMs to support
+     * multimodal input by delegating to the chat model implementation.
+     *
+     * TODO: Replace with something less hacky.
+     */
+    createProxyChat() {
+        return new ProxyChatGoogle({
+            ...this.originalFields,
+            connection: this.connection,
+        });
+    }
+    // TODO: Remove the need to override this - we are doing it to
+    // allow the LLM to handle multimodal types of input.
+    async invoke(input, options) {
+        const stream = await this._streamIterator(input, options);
+        let generatedOutput = "";
+        for await (const chunk of stream) {
+            generatedOutput += chunk;
+        }
+        return generatedOutput;
+    }
 }
 exports.GoogleBaseLLM = GoogleBaseLLM;

package/dist/llms.d.ts

@@ -1,32 +1,31 @@
-import { CallbackManagerForLLMRun, Callbacks } from "@langchain/core/callbacks/manager";
+import { Callbacks } from "@langchain/core/callbacks/manager";
 import { LLM } from "@langchain/core/language_models/llms";
-import { type BaseLanguageModelCallOptions } from "@langchain/core/language_models/base";
+import { type BaseLanguageModelCallOptions, BaseLanguageModelInput } from "@langchain/core/language_models/base";
 import { BaseMessage, MessageContent } from "@langchain/core/messages";
-import { GenerationChunk } from "@langchain/core/outputs";
 import { AbstractGoogleLLMConnection } from "./connection.js";
 import { GoogleAIBaseLLMInput, GoogleAIModelParams, GoogleAISafetySetting, GooglePlatformType, GeminiContent } from "./types.js";
 import { GoogleAbstractedClient } from "./auth.js";
+import { ChatGoogleBase } from "./chat_models.js";
+import type { GoogleBaseLLMInput, GoogleAISafetyHandler } from "./types.js";
+export { GoogleBaseLLMInput };
 declare class GoogleLLMConnection<AuthOptions> extends AbstractGoogleLLMConnection<MessageContent, AuthOptions> {
     formatContents(input: MessageContent, _parameters: GoogleAIModelParams): GeminiContent[];
 }
-/**
- * Input to LLM class.
- */
-export interface GoogleBaseLLMInput<AuthOptions> extends GoogleAIBaseLLMInput<AuthOptions> {
-}
 /**
  * Integration with an LLM.
  */
 export declare abstract class GoogleBaseLLM<AuthOptions> extends LLM<BaseLanguageModelCallOptions> implements GoogleBaseLLMInput<AuthOptions> {
     static lc_name(): string;
+    originalFields?: GoogleBaseLLMInput<AuthOptions>;
     lc_serializable: boolean;
-    model: string;
+    modelName: string;
     temperature: number;
     maxOutputTokens: number;
     topP: number;
     topK: number;
     stopSequences: string[];
     safetySettings: GoogleAISafetySetting[];
+    safetyHandler: GoogleAISafetyHandler;
     protected connection: GoogleLLMConnection<AuthOptions>;
     protected streamedConnection: GoogleLLMConnection<AuthOptions>;
     constructor(fields?: GoogleBaseLLMInput<AuthOptions>);
@@ -40,9 +39,19 @@ export declare abstract class GoogleBaseLLM<AuthOptions> extends LLM<BaseLanguag
     formatPrompt(prompt: string): MessageContent;
     /**
      * For some given input string and options, return a string output.
+     *
+     * Despite the fact that `invoke` is overridden below, we still need this
+     * in order to handle public APi calls to `generate()`.
      */
-    _call(_prompt: string, _options: this["ParsedCallOptions"], _runManager?: CallbackManagerForLLMRun): Promise<string>;
-    _streamResponseChunks(_prompt: string, _options: this["ParsedCallOptions"], _runManager?: CallbackManagerForLLMRun): AsyncGenerator<GenerationChunk>;
+    _call(prompt: string, options: this["ParsedCallOptions"]): Promise<string>;
+    _streamIterator(input: BaseLanguageModelInput, options?: BaseLanguageModelCallOptions): AsyncGenerator<string>;
     predictMessages(messages: BaseMessage[], options?: string[] | BaseLanguageModelCallOptions, _callbacks?: Callbacks): Promise<BaseMessage>;
+    /**
+     * Internal implementation detail to allow Google LLMs to support
+     * multimodal input by delegating to the chat model implementation.
+     *
+     * TODO: Replace with something less hacky.
+     */
+    protected createProxyChat(): ChatGoogleBase<AuthOptions>;
+    invoke(input: BaseLanguageModelInput, options?: BaseLanguageModelCallOptions): Promise<string>;
 }
-export {};

package/dist/llms.js

@@ -1,10 +1,13 @@
-import { LLM } from "@langchain/core/language_models/llms";
+import { CallbackManager } from "@langchain/core/callbacks/manager";
+import { BaseLLM, LLM } from "@langchain/core/language_models/llms";
 import { GenerationChunk } from "@langchain/core/outputs";
 import { getEnvironmentVariable } from "@langchain/core/utils/env";
 import { AbstractGoogleLLMConnection } from "./connection.js";
 import { copyAIModelParams, copyAndValidateModelParamsInto, } from "./utils/common.js";
-import { messageContentToParts, responseToBaseMessage, responseToGeneration, responseToString, } from "./utils/gemini.js";
+import { chunkToString, messageContentToParts, safeResponseToBaseMessage, safeResponseToString, DefaultGeminiSafetyHandler, } from "./utils/gemini.js";
 import { ApiKeyGoogleAuth } from "./auth.js";
+import { ensureParams } from "./utils/failed_handler.js";
+import { ChatGoogleBase } from "./chat_models.js";
 class GoogleLLMConnection extends AbstractGoogleLLMConnection {
     formatContents(input, _parameters) {
         const parts = messageContentToParts(input);
@@ -17,6 +20,14 @@ class GoogleLLMConnection extends AbstractGoogleLLMConnection {
         return contents;
     }
 }
+class ProxyChatGoogle extends ChatGoogleBase {
+    constructor(fields) {
+        super(fields);
+    }
+    buildAbstractedClient(fields) {
+        return fields.connection.client;
+    }
+}
 /**
  * Integration with an LLM.
  */
@@ -26,14 +37,20 @@ export class GoogleBaseLLM extends LLM {
         return "GoogleLLM";
     }
     constructor(fields) {
-        super(fields ?? {});
+        super(ensureParams(fields));
+        Object.defineProperty(this, "originalFields", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "lc_serializable", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: true
         });
-        Object.defineProperty(this, "model", {
+        Object.defineProperty(this, "modelName", {
             enumerable: true,
             configurable: true,
             writable: true,
@@ -75,6 +92,12 @@ export class GoogleBaseLLM extends LLM {
             writable: true,
             value: []
         });
+        Object.defineProperty(this, "safetyHandler", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "connection", {
             enumerable: true,
             configurable: true,
@@ -87,7 +110,10 @@ export class GoogleBaseLLM extends LLM {
             writable: true,
             value: void 0
         });
+        this.originalFields = fields;
         copyAndValidateModelParamsInto(fields, this);
+        this.safetyHandler =
+            fields?.safetyHandler ?? new DefaultGeminiSafetyHandler();
         const client = this.buildClient(fields);
         this.buildConnection(fields ?? {}, client);
     }
@@ -122,37 +148,78 @@ export class GoogleBaseLLM extends LLM {
     }
     /**
      * For some given input string and options, return a string output.
+     *
+     * Despite the fact that `invoke` is overridden below, we still need this
+     * in order to handle public APi calls to `generate()`.
      */
-    async _call(_prompt, _options, _runManager) {
-        const parameters = copyAIModelParams(this);
-        const result = await this.connection.request(_prompt, parameters, _options);
-        const ret = responseToString(result);
+    async _call(prompt, options) {
+        const parameters = copyAIModelParams(this, options);
+        const result = await this.connection.request(prompt, parameters, options);
+        const ret = safeResponseToString(result, this.safetyHandler);
         return ret;
     }
-    async *_streamResponseChunks(_prompt, _options, _runManager) {
-        // Make the call as a streaming request
-        const parameters = copyAIModelParams(this);
-        const result = await this.streamedConnection.request(_prompt, parameters, _options);
-        // Get the streaming parser of the response
-        const stream = result.data;
-        // Loop until the end of the stream
-        // During the loop, yield each time we get a chunk from the streaming parser
-        // that is either available or added to the queue
-        while (!stream.streamDone) {
-            const output = await stream.nextChunk();
-            const chunk = output !== null
-                ? new GenerationChunk(responseToGeneration({ data: output }))
-                : new GenerationChunk({
-                    text: "",
-                    generationInfo: { finishReason: "stop" },
+    // Normally, you should not override this method and instead should override
+    // _streamResponseChunks. We are doing so here to allow for multimodal inputs into
+    // the LLM.
+    async *_streamIterator(input, options) {
+        // TODO: Refactor callback setup and teardown code into core
+        const prompt = BaseLLM._convertInputToPromptValue(input);
+        const [runnableConfig, callOptions] = this._separateRunnableConfigFromCallOptions(options);
+        const callbackManager_ = await CallbackManager.configure(runnableConfig.callbacks, this.callbacks, runnableConfig.tags, this.tags, runnableConfig.metadata, this.metadata, { verbose: this.verbose });
+        const extra = {
+            options: callOptions,
+            invocation_params: this?.invocationParams(callOptions),
+            batch_size: 1,
+        };
+        const runManagers = await callbackManager_?.handleLLMStart(this.toJSON(), [prompt.toString()], undefined, undefined, extra, undefined, undefined, runnableConfig.runName);
+        let generation = new GenerationChunk({
+            text: "",
+        });
+        const proxyChat = this.createProxyChat();
+        try {
+            for await (const chunk of proxyChat._streamIterator(input, options)) {
+                const stringValue = chunkToString(chunk);
+                const generationChunk = new GenerationChunk({
+                    text: stringValue,
                 });
-            yield chunk;
+                generation = generation.concat(generationChunk);
+                yield stringValue;
+            }
+        }
+        catch (err) {
+            await Promise.all((runManagers ?? []).map((runManager) => runManager?.handleLLMError(err)));
+            throw err;
         }
+        await Promise.all((runManagers ?? []).map((runManager) => runManager?.handleLLMEnd({
+            generations: [[generation]],
+        })));
     }
     async predictMessages(messages, options, _callbacks) {
         const { content } = messages[0];
         const result = await this.connection.request(content, {}, options);
-        const ret = responseToBaseMessage(result);
+        const ret = safeResponseToBaseMessage(result, this.safetyHandler);
         return ret;
     }
+    /**
+     * Internal implementation detail to allow Google LLMs to support
+     * multimodal input by delegating to the chat model implementation.
+     *
+     * TODO: Replace with something less hacky.
+     */
+    createProxyChat() {
+        return new ProxyChatGoogle({
+            ...this.originalFields,
+            connection: this.connection,
+        });
+    }
+    // TODO: Remove the need to override this - we are doing it to
+    // allow the LLM to handle multimodal types of input.
+    async invoke(input, options) {
+        const stream = await this._streamIterator(input, options);
+        let generatedOutput = "";
+        for await (const chunk of stream) {
+            generatedOutput += chunk;
+        }
+        return generatedOutput;
+    }
 }