@langchain/google-common 0.2.16 → 0.2.17

package/dist/utils/gemini.cjs

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MessageGeminiSafetyHandler = exports.DefaultGeminiSafetyHandler = void 0;
 exports.normalizeSpeechConfig = normalizeSpeechConfig;
+exports.normalizeMessageContentComplex = normalizeMessageContentComplex;
 exports.getGeminiAPI = getGeminiAPI;
 exports.validateGeminiParams = validateGeminiParams;
 exports.isModelGemini = isModelGemini;
@@ -197,6 +198,49 @@ function normalizeSpeechConfig(config) {
     }
     return ret;
 }
+// Compatibility layer for other well known content block types
+function normalizeMessageContentComplex(content) {
+    return content.map((c) => {
+        // OpenAI completions `input_audio`
+        if (c.type === "input_audio" &&
+            "input_audio" in c &&
+            typeof c.input_audio === "object") {
+            const { format, data } = c.input_audio;
+            if (format === "wav") {
+                return {
+                    type: "audio",
+                    source_type: "base64",
+                    mime_type: "audio/wav",
+                    data,
+                };
+            }
+        }
+        // OpenAI completions `image_url`
+        if (c.type === "image_url" &&
+            "image_url" in c &&
+            typeof c.image_url === "object") {
+            const { url } = c.image_url;
+            return {
+                type: "image",
+                source_type: "url",
+                url,
+            };
+        }
+        // OpenAI completions `file`
+        if (c.type === "file" &&
+            "file" in c &&
+            typeof c.file === "object" &&
+            "file_data" in c.file) {
+            const { file_data } = c.file;
+            return {
+                type: "file",
+                source_type: "base64",
+                data: file_data,
+            };
+        }
+        return c;
+    });
+}
 function getGeminiAPI(config) {
     function messageContentText(content) {
         if (content?.text && content?.text.length > 0) {
@@ -444,8 +488,10 @@ function getGeminiAPI(config) {
                 },
             ]
             : content;
+        // Normalize the content to use standard format
+        const normalizedContent = normalizeMessageContentComplex(messageContent);
         // Get all of the parts, even those that don't correctly resolve
-        const allParts = await messageContentComplexToParts(messageContent);
+        const allParts = await messageContentComplexToParts(normalizedContent);
         // Remove any invalid parts
         const parts = allParts.reduce((acc, val) => {
             if (val) {

package/dist/utils/gemini.d.ts

@@ -1,3 +1,4 @@
+import { MessageContentComplex } from "@langchain/core/messages";
 import { GoogleLLMResponse, GoogleAIModelParams, GenerateContentResponseData, GoogleAISafetyHandler, GoogleAIAPI, GeminiAPIConfig, GoogleSpeechConfig, GoogleSpeechConfigSimplified } from "../types.js";
 export interface FunctionCall {
     name: string;
@@ -40,6 +41,7 @@ export declare class MessageGeminiSafetyHandler extends DefaultGeminiSafetyHandl
     handleData(response: GoogleLLMResponse, data: GenerateContentResponseData): GenerateContentResponseData;
 }
 export declare function normalizeSpeechConfig(config: GoogleSpeechConfig | GoogleSpeechConfigSimplified | undefined): GoogleSpeechConfig | undefined;
+export declare function normalizeMessageContentComplex(content: MessageContentComplex[]): MessageContentComplex[];
 export declare function getGeminiAPI(config?: GeminiAPIConfig): GoogleAIAPI;
 export declare function validateGeminiParams(params: GoogleAIModelParams): void;
 export declare function isModelGemini(modelName: string): boolean;

package/dist/utils/gemini.js

@@ -187,6 +187,49 @@ export function normalizeSpeechConfig(config) {
     }
     return ret;
 }
+// Compatibility layer for other well known content block types
+export function normalizeMessageContentComplex(content) {
+    return content.map((c) => {
+        // OpenAI completions `input_audio`
+        if (c.type === "input_audio" &&
+            "input_audio" in c &&
+            typeof c.input_audio === "object") {
+            const { format, data } = c.input_audio;
+            if (format === "wav") {
+                return {
+                    type: "audio",
+                    source_type: "base64",
+                    mime_type: "audio/wav",
+                    data,
+                };
+            }
+        }
+        // OpenAI completions `image_url`
+        if (c.type === "image_url" &&
+            "image_url" in c &&
+            typeof c.image_url === "object") {
+            const { url } = c.image_url;
+            return {
+                type: "image",
+                source_type: "url",
+                url,
+            };
+        }
+        // OpenAI completions `file`
+        if (c.type === "file" &&
+            "file" in c &&
+            typeof c.file === "object" &&
+            "file_data" in c.file) {
+            const { file_data } = c.file;
+            return {
+                type: "file",
+                source_type: "base64",
+                data: file_data,
+            };
+        }
+        return c;
+    });
+}
 export function getGeminiAPI(config) {
     function messageContentText(content) {
         if (content?.text && content?.text.length > 0) {
@@ -434,8 +477,10 @@ export function getGeminiAPI(config) {
                 },
             ]
             : content;
+        // Normalize the content to use standard format
+        const normalizedContent = normalizeMessageContentComplex(messageContent);
         // Get all of the parts, even those that don't correctly resolve
-        const allParts = await messageContentComplexToParts(messageContent);
+        const allParts = await messageContentComplexToParts(normalizedContent);
         // Remove any invalid parts
         const parts = allParts.reduce((acc, val) => {
             if (val) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/google-common",
-  "version": "0.2.16",
+  "version": "0.2.17",
   "description": "Core types and classes for Google services.",
   "type": "module",
   "engines": {