@langchain/google-common 0.0.0 → 0.0.2

package/dist/utils/gemini.cjs

@@ -1,12 +1,18 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isModelGemini = exports.validateGeminiParams = exports.responseToChatResult = exports.responseToBaseMessage = exports.responseToMessageContent = exports.responseToChatGenerations = exports.partToChatGeneration = exports.partToMessage = exports.responseToChatGeneration = exports.responseToGeneration = exports.responseToString = exports.partToText = exports.responseToParts = exports.responseToGenerateContentResponseData = exports.partsToMessageContent = exports.baseMessageToContent = exports.messageContentToParts = void 0;
+exports.MessageGeminiSafetyHandler = exports.DefaultGeminiSafetyHandler = exports.isModelGemini = exports.validateGeminiParams = exports.safeResponseToChatResult = exports.responseToChatResult = exports.safeResponseToBaseMessage = exports.responseToBaseMessage = exports.partsToBaseMessageFields = exports.responseToBaseMessageFields = exports.responseToChatGenerations = exports.partToChatGeneration = exports.partToMessage = exports.chunkToString = exports.safeResponseToChatGeneration = exports.responseToChatGeneration = exports.safeResponseToGeneration = exports.responseToGeneration = exports.safeResponseToString = exports.responseToString = exports.partToText = exports.responseToParts = exports.responseToGenerateContentResponseData = exports.toolsRawToTools = exports.partsToToolsRaw = exports.partsToMessageContent = exports.baseMessageToContent = exports.messageContentToParts = void 0;
 const messages_1 = require("@langchain/core/messages");
 const outputs_1 = require("@langchain/core/outputs");
+const safety_js_1 = require("./safety.cjs");
 function messageContentText(content) {
-    return {
-        text: content.text,
-    };
+    if (content?.text && content?.text.length > 0) {
+        return {
+            text: content.text,
+        };
+    }
+    else {
+        return null;
+    }
 }
 function messageContentImageUrl(content) {
     const url = typeof content.image_url === "string"
@@ -17,15 +23,19 @@ function messageContentImageUrl(content) {
     }
     if (url.startsWith("data:")) {
         return {
-            mimeType: url.split(":")[1].split(";")[0],
-            data: url.split(",")[1],
+            inlineData: {
+                mimeType: url.split(":")[1].split(";")[0],
+                data: url.split(",")[1],
+            },
         };
     }
     else {
         // FIXME - need some way to get mime type
         return {
-            mimeType: "image/png",
-            fileUri: url,
+            fileData: {
+                mimeType: "image/png",
+                fileUri: url,
+            },
         };
     }
 }
@@ -40,23 +50,61 @@ function messageContentToParts(content) {
         ]
         : content;
     // eslint-disable-next-line array-callback-return
-    const parts = messageContent.map((content) => {
-        // eslint-disable-next-line default-case
+    const parts = messageContent
+        .map((content) => {
         switch (content.type) {
             case "text":
                 return messageContentText(content);
             case "image_url":
                 return messageContentImageUrl(content);
+            default:
+                throw new Error(`Unsupported type received while converting message to message parts`);
         }
-    });
+    })
+        .reduce((acc, val) => {
+        if (val) {
+            return [...acc, val];
+        }
+        else {
+            return acc;
+        }
+    }, []);
     return parts;
 }
 exports.messageContentToParts = messageContentToParts;
+function messageToolCallsToParts(toolCalls) {
+    if (!toolCalls || toolCalls.length === 0) {
+        return [];
+    }
+    return toolCalls.map((tool) => {
+        let args = {};
+        if (tool?.function?.arguments) {
+            const argStr = tool.function.arguments;
+            args = JSON.parse(argStr);
+        }
+        return {
+            functionCall: {
+                name: tool.function.name,
+                args,
+            },
+        };
+    });
+}
+function messageKwargsToParts(kwargs) {
+    const ret = [];
+    if (kwargs?.tool_calls) {
+        ret.push(...messageToolCallsToParts(kwargs.tool_calls));
+    }
+    return ret;
+}
 function roleMessageToContent(role, message) {
+    const contentParts = messageContentToParts(message.content);
+    const toolParts = messageKwargsToParts(message.additional_kwargs);
+    const parts = [...contentParts, ...toolParts];
     return [
         {
             role,
-            parts: messageContentToParts(message.content),
+            parts,
         },
     ];
 }
@@ -66,6 +114,32 @@ function systemMessageToContent(message) {
         ...roleMessageToContent("model", new messages_1.AIMessage("Ok")),
     ];
 }
+function toolMessageToContent(message) {
+    const contentStr = typeof message.content === "string"
+        ? message.content
+        : message.content.reduce((acc, content) => {
+            if (content.type === "text") {
+                return acc + content.text;
+            }
+            else {
+                return acc;
+            }
+        }, "");
+    const content = JSON.parse(contentStr);
+    return [
+        {
+            role: "function",
+            parts: [
+                {
+                    functionResponse: {
+                        name: message.tool_call_id,
+                        response: content,
+                    },
+                },
+            ],
+        },
+    ];
+}
 function baseMessageToContent(message) {
     const type = message._getType();
     switch (type) {
@@ -75,6 +149,8 @@ function baseMessageToContent(message) {
             return roleMessageToContent("user", message);
         case "ai":
             return roleMessageToContent("model", message);
+        case "tool":
+            return toolMessageToContent(message);
         default:
             console.log(`Unsupported message type: ${type}`);
             return [];
@@ -90,25 +166,28 @@ function textPartToMessageContent(part) {
 function inlineDataPartToMessageContent(part) {
     return {
         type: "image_url",
-        image_url: `data:${part.mimeType};base64,${part.data}`,
+        image_url: `data:${part.inlineData.mimeType};base64,${part.inlineData.data}`,
     };
 }
 function fileDataPartToMessageContent(part) {
     return {
         type: "image_url",
-        image_url: part.fileUri,
+        image_url: part.fileData.fileUri,
     };
 }
 function partsToMessageContent(parts) {
     return parts
         .map((part) => {
-        if ("text" in part) {
+        if (part === undefined || part === null) {
+            return null;
+        }
+        else if ("text" in part) {
             return textPartToMessageContent(part);
         }
-        else if ("mimeType" in part && "data" in part) {
+        else if ("inlineData" in part) {
             return inlineDataPartToMessageContent(part);
         }
-        else if ("mimeType" in part && "fileUri" in part) {
+        else if ("fileData" in part) {
             return fileDataPartToMessageContent(part);
         }
         else {
@@ -123,6 +202,51 @@ function partsToMessageContent(parts) {
     }, []);
 }
 exports.partsToMessageContent = partsToMessageContent;
+function toolRawToTool(raw) {
+    return {
+        id: raw.id,
+        type: raw.type,
+        function: {
+            name: raw.function.name,
+            arguments: JSON.stringify(raw.function.arguments),
+        },
+    };
+}
+function functionCallPartToToolRaw(part) {
+    return {
+        id: part?.functionCall?.name ?? "",
+        type: "function",
+        function: {
+            name: part.functionCall.name,
+            arguments: part.functionCall.args ?? {},
+        },
+    };
+}
+function partsToToolsRaw(parts) {
+    return parts
+        .map((part) => {
+        if (part === undefined || part === null) {
+            return null;
+        }
+        else if ("functionCall" in part) {
+            return functionCallPartToToolRaw(part);
+        }
+        else {
+            return null;
+        }
+    })
+        .reduce((acc, content) => {
+        if (content) {
+            acc.push(content);
+        }
+        return acc;
+    }, []);
+}
+exports.partsToToolsRaw = partsToToolsRaw;
+function toolsRawToTools(raws) {
+    return raws.map((raw) => toolRawToTool(raw));
+}
+exports.toolsRawToTools = toolsRawToTools;
 function responseToGenerateContentResponseData(response) {
     if ("nextChunk" in response.data) {
         throw new Error("Cannot convert Stream to GenerateContentResponseData");
@@ -163,6 +287,24 @@ function responseToString(response) {
     return ret;
 }
 exports.responseToString = responseToString;
+function safeResponseTo(response, safetyHandler, responseTo) {
+    try {
+        const safeResponse = safetyHandler.handle(response);
+        return responseTo(safeResponse);
+    }
+    catch (xx) {
+        // eslint-disable-next-line no-instanceof/no-instanceof
+        if (xx instanceof safety_js_1.GoogleAISafetyError) {
+            const ret = responseTo(xx.response);
+            xx.reply = ret;
+        }
+        throw xx;
+    }
+}
+function safeResponseToString(response, safetyHandler) {
+    return safeResponseTo(response, safetyHandler, responseToString);
+}
+exports.safeResponseToString = safeResponseToString;
 function responseToGeneration(response) {
     return {
         text: responseToString(response),
@@ -170,6 +312,10 @@ function responseToGeneration(response) {
     };
 }
 exports.responseToGeneration = responseToGeneration;
+function safeResponseToGeneration(response, safetyHandler) {
+    return safeResponseTo(response, safetyHandler, responseToGeneration);
+}
+exports.safeResponseToGeneration = safeResponseToGeneration;
 function responseToChatGeneration(response) {
     return new outputs_1.ChatGenerationChunk({
         text: responseToString(response),
@@ -178,9 +324,31 @@ function responseToChatGeneration(response) {
     });
 }
 exports.responseToChatGeneration = responseToChatGeneration;
+function safeResponseToChatGeneration(response, safetyHandler) {
+    return safeResponseTo(response, safetyHandler, responseToChatGeneration);
+}
+exports.safeResponseToChatGeneration = safeResponseToChatGeneration;
+function chunkToString(chunk) {
+    if (chunk === null) {
+        return "";
+    }
+    else if (typeof chunk.content === "string") {
+        return chunk.content;
+    }
+    else if (chunk.content.length === 0) {
+        return "";
+    }
+    else if (chunk.content[0].type === "text") {
+        return chunk.content[0].text;
+    }
+    else {
+        throw new Error(`Unexpected chunk: ${chunk}`);
+    }
+}
+exports.chunkToString = chunkToString;
 function partToMessage(part) {
-    const content = partsToMessageContent([part]);
-    return new messages_1.AIMessageChunk({ content });
+    const fields = partsToBaseMessageFields([part]);
+    return new messages_1.AIMessageChunk(fields);
 }
 exports.partToMessage = partToMessage;
 function partToChatGeneration(part) {
@@ -198,17 +366,34 @@ function responseToChatGenerations(response) {
     return ret;
 }
 exports.responseToChatGenerations = responseToChatGenerations;
-function responseToMessageContent(response) {
+function responseToBaseMessageFields(response) {
     const parts = responseToParts(response);
-    return partsToMessageContent(parts);
+    return partsToBaseMessageFields(parts);
 }
-exports.responseToMessageContent = responseToMessageContent;
+exports.responseToBaseMessageFields = responseToBaseMessageFields;
+function partsToBaseMessageFields(parts) {
+    const fields = {
+        content: partsToMessageContent(parts),
+    };
+    const rawTools = partsToToolsRaw(parts);
+    if (rawTools.length > 0) {
+        const tools = toolsRawToTools(rawTools);
+        fields.additional_kwargs = {
+            tool_calls: tools,
+        };
+    }
+    return fields;
+}
+exports.partsToBaseMessageFields = partsToBaseMessageFields;
 function responseToBaseMessage(response) {
-    return new messages_1.AIMessage({
-        content: responseToMessageContent(response),
-    });
+    const fields = responseToBaseMessageFields(response);
+    return new messages_1.AIMessage(fields);
 }
 exports.responseToBaseMessage = responseToBaseMessage;
+function safeResponseToBaseMessage(response, safetyHandler) {
+    return safeResponseTo(response, safetyHandler, responseToBaseMessage);
+}
+exports.safeResponseToBaseMessage = safeResponseToBaseMessage;
 function responseToChatResult(response) {
     const generations = responseToChatGenerations(response);
     return {
@@ -217,6 +402,10 @@ function responseToChatResult(response) {
     };
 }
 exports.responseToChatResult = responseToChatResult;
+function safeResponseToChatResult(response, safetyHandler) {
+    return safeResponseTo(response, safetyHandler, responseToChatResult);
+}
+exports.safeResponseToChatResult = safeResponseToChatResult;
 function validateGeminiParams(params) {
     if (params.maxOutputTokens && params.maxOutputTokens < 0) {
         throw new Error("`maxOutputTokens` must be a positive integer");
@@ -237,3 +426,110 @@ function isModelGemini(modelName) {
     return modelName.toLowerCase().startsWith("gemini");
 }
 exports.isModelGemini = isModelGemini;
+class DefaultGeminiSafetyHandler {
+    constructor(settings) {
+        Object.defineProperty(this, "errorFinish", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: ["SAFETY", "RECITATION", "OTHER"]
+        });
+        this.errorFinish = settings?.errorFinish ?? this.errorFinish;
+    }
+    handleDataPromptFeedback(response, data) {
+        // Check to see if our prompt was blocked in the first place
+        const promptFeedback = data?.promptFeedback;
+        const blockReason = promptFeedback?.blockReason;
+        if (blockReason) {
+            throw new safety_js_1.GoogleAISafetyError(response, `Prompt blocked: ${blockReason}`);
+        }
+        return data;
+    }
+    handleDataFinishReason(response, data) {
+        const firstCandidate = data?.candidates?.[0];
+        const finishReason = firstCandidate?.finishReason;
+        if (this.errorFinish.includes(finishReason)) {
+            throw new safety_js_1.GoogleAISafetyError(response, `Finish reason: ${finishReason}`);
+        }
+        return data;
+    }
+    handleData(response, data) {
+        let ret = data;
+        ret = this.handleDataPromptFeedback(response, ret);
+        ret = this.handleDataFinishReason(response, ret);
+        return ret;
+    }
+    handle(response) {
+        let newdata;
+        if ("nextChunk" in response.data) {
+            // TODO: This is a stream. How to handle?
+            newdata = response.data;
+        }
+        else if (Array.isArray(response.data)) {
+            // If it is an array, try to handle every item in the array
+            try {
+                newdata = response.data.map((item) => this.handleData(response, item));
+            }
+            catch (xx) {
+                // eslint-disable-next-line no-instanceof/no-instanceof
+                if (xx instanceof safety_js_1.GoogleAISafetyError) {
+                    throw new safety_js_1.GoogleAISafetyError(response, xx.message);
+                }
+                else {
+                    throw xx;
+                }
+            }
+        }
+        else {
+            const data = response.data;
+            newdata = this.handleData(response, data);
+        }
+        return {
+            ...response,
+            data: newdata,
+        };
+    }
+}
+exports.DefaultGeminiSafetyHandler = DefaultGeminiSafetyHandler;
+class MessageGeminiSafetyHandler extends DefaultGeminiSafetyHandler {
+    constructor(settings) {
+        super(settings);
+        Object.defineProperty(this, "msg", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: ""
+        });
+        Object.defineProperty(this, "forceNewMessage", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        this.msg = settings?.msg ?? this.msg;
+        this.forceNewMessage = settings?.forceNewMessage ?? this.forceNewMessage;
+    }
+    setMessage(data) {
+        const ret = data;
+        if (this.forceNewMessage ||
+            !data?.candidates?.[0]?.content?.parts?.length) {
+            ret.candidates = data.candidates ?? [];
+            ret.candidates[0] = data.candidates[0] ?? {};
+            ret.candidates[0].content = data.candidates[0].content ?? {};
+            ret.candidates[0].content = {
+                role: "model",
+                parts: [{ text: this.msg }],
+            };
+        }
+        return ret;
+    }
+    handleData(response, data) {
+        try {
+            return super.handleData(response, data);
+        }
+        catch (xx) {
+            return this.setMessage(data);
+        }
+    }
+}
+exports.MessageGeminiSafetyHandler = MessageGeminiSafetyHandler;

package/dist/utils/gemini.d.ts

@@ -1,20 +1,70 @@
-import { BaseMessage, BaseMessageChunk, MessageContent } from "@langchain/core/messages";
+import { BaseMessage, BaseMessageChunk, BaseMessageFields, MessageContent } from "@langchain/core/messages";
 import { ChatGeneration, ChatGenerationChunk, ChatResult, Generation } from "@langchain/core/outputs";
-import type { GoogleLLMResponse, GoogleAIModelParams, GeminiPart, GeminiContent, GenerateContentResponseData } from "../types.js";
+import type { GoogleLLMResponse, GoogleAIModelParams, GeminiPart, GeminiContent, GenerateContentResponseData, GoogleAISafetyHandler } from "../types.js";
 export declare function messageContentToParts(content: MessageContent): GeminiPart[];
 export declare function baseMessageToContent(message: BaseMessage): GeminiContent[];
 export declare function partsToMessageContent(parts: GeminiPart[]): MessageContent;
+interface FunctionCall {
+    name: string;
+    arguments: string;
+}
+interface ToolCall {
+    id: string;
+    type: "function";
+    function: FunctionCall;
+}
+interface FunctionCallRaw {
+    name: string;
+    arguments: object;
+}
+interface ToolCallRaw {
+    id: string;
+    type: "function";
+    function: FunctionCallRaw;
+}
+export declare function partsToToolsRaw(parts: GeminiPart[]): ToolCallRaw[];
+export declare function toolsRawToTools(raws: ToolCallRaw[]): ToolCall[];
 export declare function responseToGenerateContentResponseData(response: GoogleLLMResponse): GenerateContentResponseData;
 export declare function responseToParts(response: GoogleLLMResponse): GeminiPart[];
 export declare function partToText(part: GeminiPart): string;
 export declare function responseToString(response: GoogleLLMResponse): string;
+export declare function safeResponseToString(response: GoogleLLMResponse, safetyHandler: GoogleAISafetyHandler): string;
 export declare function responseToGeneration(response: GoogleLLMResponse): Generation;
+export declare function safeResponseToGeneration(response: GoogleLLMResponse, safetyHandler: GoogleAISafetyHandler): Generation;
 export declare function responseToChatGeneration(response: GoogleLLMResponse): ChatGenerationChunk;
+export declare function safeResponseToChatGeneration(response: GoogleLLMResponse, safetyHandler: GoogleAISafetyHandler): ChatGenerationChunk;
+export declare function chunkToString(chunk: BaseMessageChunk): string;
 export declare function partToMessage(part: GeminiPart): BaseMessageChunk;
 export declare function partToChatGeneration(part: GeminiPart): ChatGeneration;
 export declare function responseToChatGenerations(response: GoogleLLMResponse): ChatGeneration[];
-export declare function responseToMessageContent(response: GoogleLLMResponse): MessageContent;
+export declare function responseToBaseMessageFields(response: GoogleLLMResponse): BaseMessageFields;
+export declare function partsToBaseMessageFields(parts: GeminiPart[]): BaseMessageFields;
 export declare function responseToBaseMessage(response: GoogleLLMResponse): BaseMessage;
+export declare function safeResponseToBaseMessage(response: GoogleLLMResponse, safetyHandler: GoogleAISafetyHandler): BaseMessage;
 export declare function responseToChatResult(response: GoogleLLMResponse): ChatResult;
+export declare function safeResponseToChatResult(response: GoogleLLMResponse, safetyHandler: GoogleAISafetyHandler): ChatResult;
 export declare function validateGeminiParams(params: GoogleAIModelParams): void;
 export declare function isModelGemini(modelName: string): boolean;
+export interface DefaultGeminiSafetySettings {
+    errorFinish?: string[];
+}
+export declare class DefaultGeminiSafetyHandler implements GoogleAISafetyHandler {
+    errorFinish: string[];
+    constructor(settings?: DefaultGeminiSafetySettings);
+    handleDataPromptFeedback(response: GoogleLLMResponse, data: GenerateContentResponseData): GenerateContentResponseData;
+    handleDataFinishReason(response: GoogleLLMResponse, data: GenerateContentResponseData): GenerateContentResponseData;
+    handleData(response: GoogleLLMResponse, data: GenerateContentResponseData): GenerateContentResponseData;
+    handle(response: GoogleLLMResponse): GoogleLLMResponse;
+}
+export interface MessageGeminiSafetySettings extends DefaultGeminiSafetySettings {
+    msg?: string;
+    forceNewMessage?: boolean;
+}
+export declare class MessageGeminiSafetyHandler extends DefaultGeminiSafetyHandler {
+    msg: string;
+    forceNewMessage: boolean;
+    constructor(settings?: MessageGeminiSafetySettings);
+    setMessage(data: GenerateContentResponseData): GenerateContentResponseData;
+    handleData(response: GoogleLLMResponse, data: GenerateContentResponseData): GenerateContentResponseData;
+}
+export {};