@langchain/core 0.3.46 → 0.3.48

package/dist/language_models/chat_models.cjs

@@ -27,6 +27,33 @@ function createChatMessageChunkEncoderStream() {
     });
 }
 exports.createChatMessageChunkEncoderStream = createChatMessageChunkEncoderStream;
+function _formatForTracing(messages) {
+    const messagesToTrace = [];
+    for (const message of messages) {
+        let messageToTrace = message;
+        if (Array.isArray(message.content)) {
+            for (let idx = 0; idx < message.content.length; idx++) {
+                const block = message.content[idx];
+                if ((0, index_js_1.isURLContentBlock)(block) || (0, index_js_1.isBase64ContentBlock)(block)) {
+                    if (messageToTrace === message) {
+                        // Also shallow-copy content
+                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                        messageToTrace = new message.constructor({
+                            ...messageToTrace,
+                            content: [
+                                ...message.content.slice(0, idx),
+                                (0, index_js_1.convertToOpenAIImageBlock)(block),
+                                ...message.content.slice(idx + 1),
+                            ],
+                        });
+                    }
+                }
+            }
+        }
+        messagesToTrace.push(messageToTrace);
+    }
+    return messagesToTrace;
+}
 /**
  * Base class for chat models. It extends the BaseLanguageModel class and
  * provides methods for generating chat based on input messages.
@@ -92,7 +119,7 @@ class BaseChatModel extends base_js_1.BaseLanguageModel {
                 invocation_params: this?.invocationParams(callOptions),
                 batch_size: 1,
             };
-            const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), [messages], runnableConfig.runId, undefined, extra, undefined, undefined, runnableConfig.runName);
+            const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), [_formatForTracing(messages)], runnableConfig.runId, undefined, extra, undefined, undefined, runnableConfig.runName);
             let generationChunk;
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             let llmOutput;
@@ -167,7 +194,7 @@ class BaseChatModel extends base_js_1.BaseLanguageModel {
                 invocation_params: this?.invocationParams(parsedOptions),
                 batch_size: 1,
             };
-            runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages, handledOptions.runId, undefined, extra, undefined, undefined, handledOptions.runName);
+            runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages.map(_formatForTracing), handledOptions.runId, undefined, extra, undefined, undefined, handledOptions.runName);
         }
         const generations = [];
         const llmOutputs = [];
@@ -288,7 +315,7 @@ class BaseChatModel extends base_js_1.BaseLanguageModel {
             invocation_params: this?.invocationParams(parsedOptions),
             batch_size: 1,
         };
-        const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages, handledOptions.runId, undefined, extra, undefined, undefined, handledOptions.runName);
+        const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages.map(_formatForTracing), handledOptions.runId, undefined, extra, undefined, undefined, handledOptions.runName);
         // generate results
         const missingPromptIndices = [];
         const results = await Promise.allSettled(baseMessages.map(async (baseMessage, index) => {

package/dist/language_models/chat_models.js

@@ -1,5 +1,5 @@
 import { zodToJsonSchema } from "zod-to-json-schema";
-import { AIMessage, HumanMessage, coerceMessageLikeToMessage, isAIMessageChunk, isBaseMessage, isAIMessage, } from "../messages/index.js";
+import { AIMessage, HumanMessage, coerceMessageLikeToMessage, isAIMessageChunk, isBaseMessage, isAIMessage, convertToOpenAIImageBlock, isURLContentBlock, isBase64ContentBlock, } from "../messages/index.js";
 import { RUN_KEY, } from "../outputs.js";
 import { BaseLanguageModel, } from "./base.js";
 import { CallbackManager, } from "../callbacks/manager.js";
@@ -23,6 +23,33 @@ export function createChatMessageChunkEncoderStream() {
         },
     });
 }
+function _formatForTracing(messages) {
+    const messagesToTrace = [];
+    for (const message of messages) {
+        let messageToTrace = message;
+        if (Array.isArray(message.content)) {
+            for (let idx = 0; idx < message.content.length; idx++) {
+                const block = message.content[idx];
+                if (isURLContentBlock(block) || isBase64ContentBlock(block)) {
+                    if (messageToTrace === message) {
+                        // Also shallow-copy content
+                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                        messageToTrace = new message.constructor({
+                            ...messageToTrace,
+                            content: [
+                                ...message.content.slice(0, idx),
+                                convertToOpenAIImageBlock(block),
+                                ...message.content.slice(idx + 1),
+                            ],
+                        });
+                    }
+                }
+            }
+        }
+        messagesToTrace.push(messageToTrace);
+    }
+    return messagesToTrace;
+}
 /**
  * Base class for chat models. It extends the BaseLanguageModel class and
  * provides methods for generating chat based on input messages.
@@ -88,7 +115,7 @@ export class BaseChatModel extends BaseLanguageModel {
                 invocation_params: this?.invocationParams(callOptions),
                 batch_size: 1,
             };
-            const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), [messages], runnableConfig.runId, undefined, extra, undefined, undefined, runnableConfig.runName);
+            const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), [_formatForTracing(messages)], runnableConfig.runId, undefined, extra, undefined, undefined, runnableConfig.runName);
             let generationChunk;
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             let llmOutput;
@@ -163,7 +190,7 @@ export class BaseChatModel extends BaseLanguageModel {
                 invocation_params: this?.invocationParams(parsedOptions),
                 batch_size: 1,
             };
-            runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages, handledOptions.runId, undefined, extra, undefined, undefined, handledOptions.runName);
+            runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages.map(_formatForTracing), handledOptions.runId, undefined, extra, undefined, undefined, handledOptions.runName);
         }
         const generations = [];
         const llmOutputs = [];
@@ -284,7 +311,7 @@ export class BaseChatModel extends BaseLanguageModel {
             invocation_params: this?.invocationParams(parsedOptions),
             batch_size: 1,
         };
-        const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages, handledOptions.runId, undefined, extra, undefined, undefined, handledOptions.runName);
+        const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages.map(_formatForTracing), handledOptions.runId, undefined, extra, undefined, undefined, handledOptions.runName);
         // generate results
         const missingPromptIndices = [];
         const results = await Promise.allSettled(baseMessages.map(async (baseMessage, index) => {

package/dist/messages/base.cjs

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.isBaseMessageChunk = exports.isBaseMessage = exports._isMessageFieldWithRole = exports.BaseMessageChunk = exports._mergeObj = exports._mergeLists = exports._mergeDicts = exports.isOpenAIToolCallArray = exports.BaseMessage = exports._mergeStatus = exports.mergeContent = void 0;
 const serializable_js_1 = require("../load/serializable.cjs");
+const content_blocks_js_1 = require("./content_blocks.cjs");
 function mergeContent(firstContent, secondContent) {
     // If first content is a string
     if (typeof firstContent === "string") {
@@ -11,6 +12,17 @@ function mergeContent(firstContent, secondContent) {
         if (typeof secondContent === "string") {
             return firstContent + secondContent;
         }
+        else if (Array.isArray(secondContent) &&
+            secondContent.some((c) => (0, content_blocks_js_1.isDataContentBlock)(c))) {
+            return [
+                {
+                    type: "text",
+                    source_type: "text",
+                    text: firstContent,
+                },
+                ...secondContent,
+            ];
+        }
         else {
             return [{ type: "text", text: firstContent }, ...secondContent];
         }
@@ -26,8 +38,20 @@ function mergeContent(firstContent, secondContent) {
         if (secondContent === "") {
             return firstContent;
         }
-        // Otherwise, add the second content as a new element of the list
-        return [...firstContent, { type: "text", text: secondContent }];
+        else if (Array.isArray(firstContent) &&
+            firstContent.some((c) => (0, content_blocks_js_1.isDataContentBlock)(c))) {
+            return [
+                ...firstContent,
+                {
+                    type: "file",
+                    source_type: "text",
+                    text: secondContent,
+                },
+            ];
+        }
+        else {
+            return [...firstContent, { type: "text", text: secondContent }];
+        }
     }
 }
 exports.mergeContent = mergeContent;

package/dist/messages/base.js

@@ -1,4 +1,5 @@
 import { Serializable } from "../load/serializable.js";
+import { isDataContentBlock, } from "./content_blocks.js";
 export function mergeContent(firstContent, secondContent) {
     // If first content is a string
     if (typeof firstContent === "string") {
@@ -8,6 +9,17 @@ export function mergeContent(firstContent, secondContent) {
         if (typeof secondContent === "string") {
             return firstContent + secondContent;
         }
+        else if (Array.isArray(secondContent) &&
+            secondContent.some((c) => isDataContentBlock(c))) {
+            return [
+                {
+                    type: "text",
+                    source_type: "text",
+                    text: firstContent,
+                },
+                ...secondContent,
+            ];
+        }
         else {
             return [{ type: "text", text: firstContent }, ...secondContent];
         }
@@ -23,8 +35,20 @@ export function mergeContent(firstContent, secondContent) {
         if (secondContent === "") {
             return firstContent;
         }
-        // Otherwise, add the second content as a new element of the list
-        return [...firstContent, { type: "text", text: secondContent }];
+        else if (Array.isArray(firstContent) &&
+            firstContent.some((c) => isDataContentBlock(c))) {
+            return [
+                ...firstContent,
+                {
+                    type: "file",
+                    source_type: "text",
+                    text: secondContent,
+                },
+            ];
+        }
+        else {
+            return [...firstContent, { type: "text", text: secondContent }];
+        }
     }
 }
 /**

package/dist/messages/content_blocks.cjs

@@ -0,0 +1,166 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.convertToProviderContentBlock = exports.parseBase64DataUrl = exports.parseMimeType = exports.convertToOpenAIImageBlock = exports.isIDContentBlock = exports.isPlainTextContentBlock = exports.isBase64ContentBlock = exports.isURLContentBlock = exports.isDataContentBlock = void 0;
+function isDataContentBlock(content_block) {
+    return (typeof content_block === "object" &&
+        content_block !== null &&
+        "type" in content_block &&
+        typeof content_block.type === "string" &&
+        "source_type" in content_block &&
+        (content_block.source_type === "url" ||
+            content_block.source_type === "base64" ||
+            content_block.source_type === "text" ||
+            content_block.source_type === "id"));
+}
+exports.isDataContentBlock = isDataContentBlock;
+function isURLContentBlock(content_block) {
+    return (isDataContentBlock(content_block) &&
+        content_block.source_type === "url" &&
+        "url" in content_block &&
+        typeof content_block.url === "string");
+}
+exports.isURLContentBlock = isURLContentBlock;
+function isBase64ContentBlock(content_block) {
+    return (isDataContentBlock(content_block) &&
+        content_block.source_type === "base64" &&
+        "data" in content_block &&
+        typeof content_block.data === "string");
+}
+exports.isBase64ContentBlock = isBase64ContentBlock;
+function isPlainTextContentBlock(content_block) {
+    return (isDataContentBlock(content_block) &&
+        content_block.source_type === "text" &&
+        "text" in content_block &&
+        typeof content_block.text === "string");
+}
+exports.isPlainTextContentBlock = isPlainTextContentBlock;
+function isIDContentBlock(content_block) {
+    return (isDataContentBlock(content_block) &&
+        content_block.source_type === "id" &&
+        "id" in content_block &&
+        typeof content_block.id === "string");
+}
+exports.isIDContentBlock = isIDContentBlock;
+function convertToOpenAIImageBlock(content_block) {
+    if (isDataContentBlock(content_block)) {
+        if (content_block.source_type === "url") {
+            return {
+                type: "image_url",
+                image_url: {
+                    url: content_block.url,
+                },
+            };
+        }
+        if (content_block.source_type === "base64") {
+            if (!content_block.mime_type) {
+                throw new Error("mime_type key is required for base64 data.");
+            }
+            const mime_type = content_block.mime_type;
+            return {
+                type: "image_url",
+                image_url: {
+                    url: `data:${mime_type};base64,${content_block.data}`,
+                },
+            };
+        }
+    }
+    throw new Error("Unsupported source type. Only 'url' and 'base64' are supported.");
+}
+exports.convertToOpenAIImageBlock = convertToOpenAIImageBlock;
+/**
+ * Utility function for ChatModelProviders. Parses a mime type into a type, subtype, and parameters.
+ *
+ * @param mime_type - The mime type to parse.
+ * @returns An object containing the type, subtype, and parameters.
+ */
+function parseMimeType(mime_type) {
+    const parts = mime_type.split(";")[0].split("/");
+    if (parts.length !== 2) {
+        throw new Error(`Invalid mime type: "${mime_type}" - does not match type/subtype format.`);
+    }
+    const type = parts[0].trim();
+    const subtype = parts[1].trim();
+    if (type === "" || subtype === "") {
+        throw new Error(`Invalid mime type: "${mime_type}" - type or subtype is empty.`);
+    }
+    const parameters = {};
+    for (const parameterKvp of mime_type.split(";").slice(1)) {
+        const parameterParts = parameterKvp.split("=");
+        if (parameterParts.length !== 2) {
+            throw new Error(`Invalid parameter syntax in mime type: "${mime_type}".`);
+        }
+        const key = parameterParts[0].trim();
+        const value = parameterParts[1].trim();
+        if (key === "") {
+            throw new Error(`Invalid parameter syntax in mime type: "${mime_type}".`);
+        }
+        parameters[key] = value;
+    }
+    return {
+        type,
+        subtype,
+        parameters,
+    };
+}
+exports.parseMimeType = parseMimeType;
+/**
+ * Utility function for ChatModelProviders. Parses a base64 data URL into a typed array or string.
+ *
+ * @param dataUrl - The base64 data URL to parse.
+ * @param asTypedArray - Whether to return the data as a typed array.
+ * @returns The parsed data and mime type, or undefined if the data URL is invalid.
+ */
+function parseBase64DataUrl({ dataUrl: data_url, asTypedArray = false, }) {
+    const formatMatch = data_url.match(/^data:(\w+\/\w+);base64,([A-Za-z0-9+/]+=*)$/);
+    let mime_type;
+    if (formatMatch) {
+        mime_type = formatMatch[1].toLowerCase();
+        const data = asTypedArray
+            ? Uint8Array.from(atob(formatMatch[2]), (c) => c.charCodeAt(0))
+            : formatMatch[2];
+        return {
+            mime_type,
+            data,
+        };
+    }
+    return undefined;
+}
+exports.parseBase64DataUrl = parseBase64DataUrl;
+/**
+ * Convert from a standard data content block to a provider's proprietary data content block format.
+ *
+ * Don't override this method. Instead, override the more specific conversion methods and use this
+ * method unmodified.
+ *
+ * @param block - The standard data content block to convert.
+ * @returns The provider data content block.
+ * @throws An error if the standard data content block type is not supported.
+ */
+function convertToProviderContentBlock(block, converter) {
+    if (block.type === "text") {
+        if (!converter.fromStandardTextBlock) {
+            throw new Error(`Converter for ${converter.providerName} does not implement \`fromStandardTextBlock\` method.`);
+        }
+        return converter.fromStandardTextBlock(block);
+    }
+    if (block.type === "image") {
+        if (!converter.fromStandardImageBlock) {
+            throw new Error(`Converter for ${converter.providerName} does not implement \`fromStandardImageBlock\` method.`);
+        }
+        return converter.fromStandardImageBlock(block);
+    }
+    if (block.type === "audio") {
+        if (!converter.fromStandardAudioBlock) {
+            throw new Error(`Converter for ${converter.providerName} does not implement \`fromStandardAudioBlock\` method.`);
+        }
+        return converter.fromStandardAudioBlock(block);
+    }
+    if (block.type === "file") {
+        if (!converter.fromStandardFileBlock) {
+            throw new Error(`Converter for ${converter.providerName} does not implement \`fromStandardFileBlock\` method.`);
+        }
+        return converter.fromStandardFileBlock(block);
+    }
+    throw new Error(`Unable to convert content block type '${block.type}' to provider-specific format: not recognized.`);
+}
+exports.convertToProviderContentBlock = convertToProviderContentBlock;

package/dist/messages/content_blocks.d.ts

@@ -0,0 +1,151 @@
+export interface BaseDataContentBlock {
+    mime_type?: string;
+    metadata?: Record<string, unknown>;
+}
+export interface URLContentBlock extends BaseDataContentBlock {
+    type: "image" | "audio" | "file";
+    source_type: "url";
+    url: string;
+}
+export interface Base64ContentBlock extends BaseDataContentBlock {
+    type: "image" | "audio" | "file";
+    source_type: "base64";
+    data: string;
+}
+export interface PlainTextContentBlock extends BaseDataContentBlock {
+    type: "file" | "text";
+    source_type: "text";
+    text: string;
+}
+export interface IDContentBlock extends BaseDataContentBlock {
+    type: "image" | "audio" | "file";
+    source_type: "id";
+    id: string;
+}
+export type DataContentBlock = URLContentBlock | Base64ContentBlock | PlainTextContentBlock | IDContentBlock;
+export type StandardImageBlock = (URLContentBlock | Base64ContentBlock | IDContentBlock) & {
+    type: "image";
+};
+export type StandardAudioBlock = (URLContentBlock | Base64ContentBlock | IDContentBlock) & {
+    type: "audio";
+};
+export type StandardFileBlock = (URLContentBlock | Base64ContentBlock | IDContentBlock | PlainTextContentBlock) & {
+    type: "file";
+};
+export type StandardTextBlock = PlainTextContentBlock & {
+    type: "text";
+};
+export type DataContentBlockType = DataContentBlock["type"];
+export declare function isDataContentBlock(content_block: object): content_block is DataContentBlock;
+export declare function isURLContentBlock(content_block: object): content_block is URLContentBlock;
+export declare function isBase64ContentBlock(content_block: object): content_block is Base64ContentBlock;
+export declare function isPlainTextContentBlock(content_block: object): content_block is PlainTextContentBlock;
+export declare function isIDContentBlock(content_block: object): content_block is IDContentBlock;
+export declare function convertToOpenAIImageBlock(content_block: URLContentBlock | Base64ContentBlock): {
+    type: string;
+    image_url: {
+        url: string;
+    };
+};
+/**
+ * Utility function for ChatModelProviders. Parses a mime type into a type, subtype, and parameters.
+ *
+ * @param mime_type - The mime type to parse.
+ * @returns An object containing the type, subtype, and parameters.
+ */
+export declare function parseMimeType(mime_type: string): {
+    type: string;
+    subtype: string;
+    parameters: Record<string, string>;
+};
+/**
+ * Utility function for ChatModelProviders. Parses a base64 data URL into a typed array or string.
+ *
+ * @param dataUrl - The base64 data URL to parse.
+ * @param asTypedArray - Whether to return the data as a typed array.
+ * @returns An object containing the parsed data and mime type, or undefined if the data URL is invalid.
+ */
+export declare function parseBase64DataUrl({ dataUrl, asTypedArray, }: {
+    dataUrl: string;
+    asTypedArray: true;
+}): {
+    data: Uint8Array;
+    mime_type: string;
+} | undefined;
+/**
+ * Utility function for ChatModelProviders. Parses a base64 data URL into a typed array or string.
+ *
+ * @param dataUrl - The base64 data URL to parse.
+ * @param asTypedArray - Whether to return the data as a typed array.
+ * @returns The parsed data and mime type, or undefined if the data URL is invalid.
+ */
+export declare function parseBase64DataUrl({ dataUrl, asTypedArray, }: {
+    dataUrl: string;
+    asTypedArray?: false;
+}): {
+    data: string;
+    mime_type: string;
+} | undefined;
+/**
+ * A bag of provider-specific content block types.
+ *
+ * Allows implementations of {@link StandardContentBlockConverter} and related to be defined only in
+ * terms of the types they support. Also allows for forward compatibility as the set of known
+ * standard types grows, as the set of types can be extended without breaking existing
+ * implementations of the aforementioned interfaces.
+ */
+export type ProviderFormatTypes<TextFormat = unknown, ImageFormat = unknown, AudioFormat = unknown, FileFormat = unknown> = {
+    text: TextFormat;
+    image: ImageFormat;
+    audio: AudioFormat;
+    file: FileFormat;
+};
+/**
+ * Utility interface for converting between standard and provider-specific data content blocks, to be
+ * used when implementing chat model providers.
+ *
+ * Meant to be used with {@link convertToProviderContentBlock} and
+ * {@link convertToStandardContentBlock} rather than being consumed directly.
+ */
+export interface StandardContentBlockConverter<Formats extends Partial<ProviderFormatTypes>> {
+    /**
+     * The name of the provider type that corresponds to the provider-specific content block types
+     * that this converter supports.
+     */
+    providerName: string;
+    /**
+     * Convert from a standard image block to a provider's proprietary image block format.
+     * @param block - The standard image block to convert.
+     * @returns The provider image block.
+     */
+    fromStandardImageBlock?(block: StandardImageBlock): Formats["image"];
+    /**
+     * Convert from a standard audio block to a provider's proprietary audio block format.
+     * @param block - The standard audio block to convert.
+     * @returns The provider audio block.
+     */
+    fromStandardAudioBlock?(block: StandardAudioBlock): Formats["audio"];
+    /**
+     * Convert from a standard file block to a provider's proprietary file block format.
+     * @param block - The standard file block to convert.
+     * @returns The provider file block.
+     */
+    fromStandardFileBlock?(block: StandardFileBlock): Formats["file"];
+    /**
+     * Convert from a standard text block to a provider's proprietary text block format.
+     * @param block - The standard text block to convert.
+     * @returns The provider text block.
+     */
+    fromStandardTextBlock?(block: StandardTextBlock): Formats["text"];
+}
+/**
+ * Convert from a standard data content block to a provider's proprietary data content block format.
+ *
+ * Don't override this method. Instead, override the more specific conversion methods and use this
+ * method unmodified.
+ *
+ * @param block - The standard data content block to convert.
+ * @returns The provider data content block.
+ * @throws An error if the standard data content block type is not supported.
+ */
+export declare function convertToProviderContentBlock<Formats extends Partial<ProviderFormatTypes>>(block: DataContentBlock, converter: StandardContentBlockConverter<Formats>): Formats[keyof Formats];

package/dist/messages/content_blocks.js

@@ -0,0 +1,154 @@
+export function isDataContentBlock(content_block) {
+    return (typeof content_block === "object" &&
+        content_block !== null &&
+        "type" in content_block &&
+        typeof content_block.type === "string" &&
+        "source_type" in content_block &&
+        (content_block.source_type === "url" ||
+            content_block.source_type === "base64" ||
+            content_block.source_type === "text" ||
+            content_block.source_type === "id"));
+}
+export function isURLContentBlock(content_block) {
+    return (isDataContentBlock(content_block) &&
+        content_block.source_type === "url" &&
+        "url" in content_block &&
+        typeof content_block.url === "string");
+}
+export function isBase64ContentBlock(content_block) {
+    return (isDataContentBlock(content_block) &&
+        content_block.source_type === "base64" &&
+        "data" in content_block &&
+        typeof content_block.data === "string");
+}
+export function isPlainTextContentBlock(content_block) {
+    return (isDataContentBlock(content_block) &&
+        content_block.source_type === "text" &&
+        "text" in content_block &&
+        typeof content_block.text === "string");
+}
+export function isIDContentBlock(content_block) {
+    return (isDataContentBlock(content_block) &&
+        content_block.source_type === "id" &&
+        "id" in content_block &&
+        typeof content_block.id === "string");
+}
+export function convertToOpenAIImageBlock(content_block) {
+    if (isDataContentBlock(content_block)) {
+        if (content_block.source_type === "url") {
+            return {
+                type: "image_url",
+                image_url: {
+                    url: content_block.url,
+                },
+            };
+        }
+        if (content_block.source_type === "base64") {
+            if (!content_block.mime_type) {
+                throw new Error("mime_type key is required for base64 data.");
+            }
+            const mime_type = content_block.mime_type;
+            return {
+                type: "image_url",
+                image_url: {
+                    url: `data:${mime_type};base64,${content_block.data}`,
+                },
+            };
+        }
+    }
+    throw new Error("Unsupported source type. Only 'url' and 'base64' are supported.");
+}
+/**
+ * Utility function for ChatModelProviders. Parses a mime type into a type, subtype, and parameters.
+ *
+ * @param mime_type - The mime type to parse.
+ * @returns An object containing the type, subtype, and parameters.
+ */
+export function parseMimeType(mime_type) {
+    const parts = mime_type.split(";")[0].split("/");
+    if (parts.length !== 2) {
+        throw new Error(`Invalid mime type: "${mime_type}" - does not match type/subtype format.`);
+    }
+    const type = parts[0].trim();
+    const subtype = parts[1].trim();
+    if (type === "" || subtype === "") {
+        throw new Error(`Invalid mime type: "${mime_type}" - type or subtype is empty.`);
+    }
+    const parameters = {};
+    for (const parameterKvp of mime_type.split(";").slice(1)) {
+        const parameterParts = parameterKvp.split("=");
+        if (parameterParts.length !== 2) {
+            throw new Error(`Invalid parameter syntax in mime type: "${mime_type}".`);
+        }
+        const key = parameterParts[0].trim();
+        const value = parameterParts[1].trim();
+        if (key === "") {
+            throw new Error(`Invalid parameter syntax in mime type: "${mime_type}".`);
+        }
+        parameters[key] = value;
+    }
+    return {
+        type,
+        subtype,
+        parameters,
+    };
+}
+/**
+ * Utility function for ChatModelProviders. Parses a base64 data URL into a typed array or string.
+ *
+ * @param dataUrl - The base64 data URL to parse.
+ * @param asTypedArray - Whether to return the data as a typed array.
+ * @returns The parsed data and mime type, or undefined if the data URL is invalid.
+ */
+export function parseBase64DataUrl({ dataUrl: data_url, asTypedArray = false, }) {
+    const formatMatch = data_url.match(/^data:(\w+\/\w+);base64,([A-Za-z0-9+/]+=*)$/);
+    let mime_type;
+    if (formatMatch) {
+        mime_type = formatMatch[1].toLowerCase();
+        const data = asTypedArray
+            ? Uint8Array.from(atob(formatMatch[2]), (c) => c.charCodeAt(0))
+            : formatMatch[2];
+        return {
+            mime_type,
+            data,
+        };
+    }
+    return undefined;
+}
+/**
+ * Convert from a standard data content block to a provider's proprietary data content block format.
+ *
+ * Don't override this method. Instead, override the more specific conversion methods and use this
+ * method unmodified.
+ *
+ * @param block - The standard data content block to convert.
+ * @returns The provider data content block.
+ * @throws An error if the standard data content block type is not supported.
+ */
+export function convertToProviderContentBlock(block, converter) {
+    if (block.type === "text") {
+        if (!converter.fromStandardTextBlock) {
+            throw new Error(`Converter for ${converter.providerName} does not implement \`fromStandardTextBlock\` method.`);
+        }
+        return converter.fromStandardTextBlock(block);
+    }
+    if (block.type === "image") {
+        if (!converter.fromStandardImageBlock) {
+            throw new Error(`Converter for ${converter.providerName} does not implement \`fromStandardImageBlock\` method.`);
+        }
+        return converter.fromStandardImageBlock(block);
+    }
+    if (block.type === "audio") {
+        if (!converter.fromStandardAudioBlock) {
+            throw new Error(`Converter for ${converter.providerName} does not implement \`fromStandardAudioBlock\` method.`);
+        }
+        return converter.fromStandardAudioBlock(block);
+    }
+    if (block.type === "file") {
+        if (!converter.fromStandardFileBlock) {
+            throw new Error(`Converter for ${converter.providerName} does not implement \`fromStandardFileBlock\` method.`);
+        }
+        return converter.fromStandardFileBlock(block);
+    }
+    throw new Error(`Unable to convert content block type '${block.type}' to provider-specific format: not recognized.`);
+}

package/dist/messages/human.cjs

@@ -12,6 +12,11 @@ class HumanMessage extends base_js_1.BaseMessage {
     _getType() {
         return "human";
     }
+    constructor(fields, 
+    /** @deprecated */
+    kwargs) {
+        super(fields, kwargs);
+    }
 }
 exports.HumanMessage = HumanMessage;
 /**
@@ -25,6 +30,11 @@ class HumanMessageChunk extends base_js_1.BaseMessageChunk {
     _getType() {
         return "human";
     }
+    constructor(fields, 
+    /** @deprecated */
+    kwargs) {
+        super(fields, kwargs);
+    }
     concat(chunk) {
         return new HumanMessageChunk({
             content: (0, base_js_1.mergeContent)(this.content, chunk.content),

package/dist/messages/human.d.ts

@@ -1,18 +1,30 @@
-import { BaseMessage, BaseMessageChunk, type MessageType } from "./base.js";
+import { BaseMessage, BaseMessageChunk, type MessageType, type BaseMessageFields, type MessageContentComplex } from "./base.js";
+import type { DataContentBlock } from "./content_blocks.js";
+export type HumanMessageFields = BaseMessageFields & {
+    content: string | (MessageContentComplex | DataContentBlock)[];
+};
 /**
  * Represents a human message in a conversation.
  */
 export declare class HumanMessage extends BaseMessage {
+    content: string | (MessageContentComplex | DataContentBlock)[];
     static lc_name(): string;
     _getType(): MessageType;
+    constructor(fields: string | HumanMessageFields, 
+    /** @deprecated */
+    kwargs?: Record<string, unknown>);
 }
 /**
  * Represents a chunk of a human message, which can be concatenated with
  * other human message chunks.
  */
 export declare class HumanMessageChunk extends BaseMessageChunk {
+    content: string | (MessageContentComplex | DataContentBlock)[];
     static lc_name(): string;
     _getType(): MessageType;
+    constructor(fields: string | HumanMessageFields, 
+    /** @deprecated */
+    kwargs?: Record<string, unknown>);
     concat(chunk: HumanMessageChunk): HumanMessageChunk;
 }
 export declare function isHumanMessage(x: BaseMessage): x is HumanMessage;

package/dist/messages/human.js

@@ -9,6 +9,11 @@ export class HumanMessage extends BaseMessage {
     _getType() {
         return "human";
     }
+    constructor(fields, 
+    /** @deprecated */
+    kwargs) {
+        super(fields, kwargs);
+    }
 }
 /**
  * Represents a chunk of a human message, which can be concatenated with
@@ -21,6 +26,11 @@ export class HumanMessageChunk extends BaseMessageChunk {
     _getType() {
         return "human";
     }
+    constructor(fields, 
+    /** @deprecated */
+    kwargs) {
+        super(fields, kwargs);
+    }
     concat(chunk) {
         return new HumanMessageChunk({
             content: mergeContent(this.content, chunk.content),

package/dist/messages/index.cjs

@@ -24,6 +24,7 @@ __exportStar(require("./system.cjs"), exports);
 __exportStar(require("./utils.cjs"), exports);
 __exportStar(require("./transformers.cjs"), exports);
 __exportStar(require("./modifier.cjs"), exports);
+__exportStar(require("./content_blocks.cjs"), exports);
 // TODO: Use a star export when we deprecate the
 // existing "ToolCall" type in "base.js".
 var tool_js_1 = require("./tool.cjs");

package/dist/messages/index.d.ts

@@ -7,4 +7,5 @@ export * from "./system.js";
 export * from "./utils.js";
 export * from "./transformers.js";
 export * from "./modifier.js";
+export * from "./content_blocks.js";
 export { type ToolMessageFieldsWithToolCallId, ToolMessage, ToolMessageChunk, type InvalidToolCall, isToolMessage, isToolMessageChunk, } from "./tool.js";

package/dist/messages/index.js

@@ -7,6 +7,7 @@ export * from "./system.js";
 export * from "./utils.js";
 export * from "./transformers.js";
 export * from "./modifier.js";
+export * from "./content_blocks.js";
 // TODO: Use a star export when we deprecate the
 // existing "ToolCall" type in "base.js".
 export { ToolMessage, ToolMessageChunk, isToolMessage, isToolMessageChunk, } from "./tool.js";

package/dist/messages/system.cjs

@@ -12,6 +12,11 @@ class SystemMessage extends base_js_1.BaseMessage {
     _getType() {
         return "system";
     }
+    constructor(fields, 
+    /** @deprecated */
+    kwargs) {
+        super(fields, kwargs);
+    }
 }
 exports.SystemMessage = SystemMessage;
 /**
@@ -25,6 +30,11 @@ class SystemMessageChunk extends base_js_1.BaseMessageChunk {
     _getType() {
         return "system";
     }
+    constructor(fields, 
+    /** @deprecated */
+    kwargs) {
+        super(fields, kwargs);
+    }
     concat(chunk) {
         return new SystemMessageChunk({
             content: (0, base_js_1.mergeContent)(this.content, chunk.content),

package/dist/messages/system.d.ts

@@ -1,18 +1,30 @@
-import { BaseMessage, BaseMessageChunk, type MessageType } from "./base.js";
+import { BaseMessage, BaseMessageChunk, type MessageType, type BaseMessageFields, type MessageContentComplex } from "./base.js";
+import type { DataContentBlock } from "./content_blocks.js";
+export type SystemMessageFields = BaseMessageFields & {
+    content: string | (MessageContentComplex | DataContentBlock)[];
+};
 /**
  * Represents a system message in a conversation.
  */
 export declare class SystemMessage extends BaseMessage {
+    content: string | (MessageContentComplex | DataContentBlock)[];
     static lc_name(): string;
     _getType(): MessageType;
+    constructor(fields: string | SystemMessageFields, 
+    /** @deprecated */
+    kwargs?: Record<string, unknown>);
 }
 /**
  * Represents a chunk of a system message, which can be concatenated with
  * other system message chunks.
  */
 export declare class SystemMessageChunk extends BaseMessageChunk {
+    content: string | (MessageContentComplex | DataContentBlock)[];
     static lc_name(): string;
     _getType(): MessageType;
+    constructor(fields: string | SystemMessageFields, 
+    /** @deprecated */
+    kwargs?: Record<string, unknown>);
     concat(chunk: SystemMessageChunk): SystemMessageChunk;
 }
 export declare function isSystemMessage(x: BaseMessage): x is SystemMessage;

package/dist/messages/system.js

@@ -9,6 +9,11 @@ export class SystemMessage extends BaseMessage {
     _getType() {
         return "system";
     }
+    constructor(fields, 
+    /** @deprecated */
+    kwargs) {
+        super(fields, kwargs);
+    }
 }
 /**
  * Represents a chunk of a system message, which can be concatenated with
@@ -21,6 +26,11 @@ export class SystemMessageChunk extends BaseMessageChunk {
     _getType() {
         return "system";
     }
+    constructor(fields, 
+    /** @deprecated */
+    kwargs) {
+        super(fields, kwargs);
+    }
     concat(chunk) {
         return new SystemMessageChunk({
             content: mergeContent(this.content, chunk.content),

package/dist/messages/tool.d.ts

@@ -1,5 +1,9 @@
-import { BaseMessage, BaseMessageChunk, type BaseMessageFields, type MessageType } from "./base.js";
-export interface ToolMessageFieldsWithToolCallId extends BaseMessageFields {
+import { BaseMessage, BaseMessageChunk, type BaseMessageFields, type MessageType, type MessageContentComplex } from "./base.js";
+import type { DataContentBlock } from "./content_blocks.js";
+export interface ToolMessageFields extends BaseMessageFields {
+    content: string | (MessageContentComplex | DataContentBlock)[];
+}
+export interface ToolMessageFieldsWithToolCallId extends ToolMessageFields {
     /**
      * Artifact of the Tool execution which is not meant to be sent to the model.
      *
@@ -30,6 +34,7 @@ export declare function isDirectToolOutput(x: unknown): x is DirectToolOutput;
  * Represents a tool message in a conversation.
  */
 export declare class ToolMessage extends BaseMessage implements DirectToolOutput {
+    content: string | (MessageContentComplex | DataContentBlock)[];
     static lc_name(): string;
     get lc_aliases(): Record<string, string>;
     lc_direct_tool_output: boolean;
@@ -48,7 +53,7 @@ export declare class ToolMessage extends BaseMessage implements DirectToolOutput
      */
     artifact?: any;
     constructor(fields: ToolMessageFieldsWithToolCallId);
-    constructor(fields: string | BaseMessageFields, tool_call_id: string, name?: string);
+    constructor(fields: string | ToolMessageFields, tool_call_id: string, name?: string);
     _getType(): MessageType;
     static isInstance(message: BaseMessage): message is ToolMessage;
     get _printableFields(): Record<string, unknown>;
@@ -58,6 +63,7 @@ export declare class ToolMessage extends BaseMessage implements DirectToolOutput
  * with other tool message chunks.
  */
 export declare class ToolMessageChunk extends BaseMessageChunk {
+    content: string | (MessageContentComplex | DataContentBlock)[];
     tool_call_id: string;
     /**
      * Status of the tool invocation.

package/dist/prompts/dict.cjs

@@ -5,7 +5,9 @@ const base_js_1 = require("../runnables/base.cjs");
 const template_js_1 = require("./template.cjs");
 class DictPromptTemplate extends base_js_1.Runnable {
     constructor(fields) {
-        super(fields);
+        const templateFormat = fields.templateFormat ?? "f-string";
+        const inputVariables = _getInputVariables(fields.template, templateFormat);
+        super({ inputVariables, ...fields });
         Object.defineProperty(this, "lc_namespace", {
             enumerable: true,
             configurable: true,
@@ -28,13 +30,17 @@ class DictPromptTemplate extends base_js_1.Runnable {
             enumerable: true,
             configurable: true,
             writable: true,
-            value: "f-string"
+            value: void 0
+        });
+        Object.defineProperty(this, "inputVariables", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
         });
         this.template = fields.template;
-        this.templateFormat = fields.templateFormat ?? this.templateFormat;
-    }
-    get inputVariables() {
-        return _getInputVariables(this.template, this.templateFormat);
+        this.templateFormat = templateFormat;
+        this.inputVariables = inputVariables;
     }
     async format(values) {
         return _insertInputVariables(this.template, values, this.templateFormat);

package/dist/prompts/dict.d.ts

@@ -7,11 +7,11 @@ export declare class DictPromptTemplate<RunInput extends InputValues = InputValu
     lc_serializable: boolean;
     template: Record<string, unknown>;
     templateFormat: TemplateFormat;
+    inputVariables: Array<Extract<keyof RunInput, string>>;
     constructor(fields: {
         template: Record<string, unknown>;
         templateFormat?: TemplateFormat;
     });
-    get inputVariables(): Array<Extract<keyof RunInput, string>>;
     format(values: TypedPromptInputValues<RunInput>): Promise<RunOutput>;
     invoke(values: TypedPromptInputValues<InputValues>): Promise<RunOutput>;
 }

package/dist/prompts/dict.js

@@ -2,7 +2,9 @@ import { Runnable } from "../runnables/base.js";
 import { parseTemplate, renderTemplate } from "./template.js";
 export class DictPromptTemplate extends Runnable {
     constructor(fields) {
-        super(fields);
+        const templateFormat = fields.templateFormat ?? "f-string";
+        const inputVariables = _getInputVariables(fields.template, templateFormat);
+        super({ inputVariables, ...fields });
         Object.defineProperty(this, "lc_namespace", {
             enumerable: true,
             configurable: true,
@@ -25,13 +27,17 @@ export class DictPromptTemplate extends Runnable {
             enumerable: true,
             configurable: true,
             writable: true,
-            value: "f-string"
+            value: void 0
+        });
+        Object.defineProperty(this, "inputVariables", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
         });
         this.template = fields.template;
-        this.templateFormat = fields.templateFormat ?? this.templateFormat;
-    }
-    get inputVariables() {
-        return _getInputVariables(this.template, this.templateFormat);
+        this.templateFormat = templateFormat;
+        this.inputVariables = inputVariables;
     }
     async format(values) {
         return _insertInputVariables(this.template, values, this.templateFormat);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/core",
-  "version": "0.3.46",
+  "version": "0.3.48",
   "description": "Core LangChain.js abstractions and schemas",
   "type": "module",
   "engines": {