@langchain/core 0.1.20 → 0.1.21

package/dist/prompts/chat.js

@@ -3,8 +3,11 @@
 import { AIMessage, HumanMessage, SystemMessage, BaseMessage, ChatMessage, coerceMessageLikeToMessage, isBaseMessage, } from "../messages/index.js";
 import { ChatPromptValue, } from "../prompt_values.js";
 import { Runnable } from "../runnables/base.js";
+import { BaseStringPromptTemplate } from "./string.js";
 import { BasePromptTemplate, } from "./base.js";
 import { PromptTemplate } from "./prompt.js";
+import { ImagePromptTemplate } from "./image.js";
+import { parseFString } from "./template.js";
 /**
  * Abstract class that serves as a base for creating message prompt
  * templates. It defines how to format messages for different roles in a
@@ -168,6 +171,210 @@ export class ChatMessagePromptTemplate extends BaseMessageStringPromptTemplate {
         return new this(PromptTemplate.fromTemplate(template), role);
     }
 }
+class _StringImageMessagePromptTemplate extends BaseMessagePromptTemplate {
+    static _messageClass() {
+        throw new Error("Can not invoke _messageClass from inside _StringImageMessagePromptTemplate");
+    }
+    constructor(
+    /** @TODO When we come up with a better way to type prompt templates, fix this */
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    fields, additionalOptions) {
+        if (!("prompt" in fields)) {
+            // eslint-disable-next-line no-param-reassign
+            fields = { prompt: fields };
+        }
+        super(fields);
+        Object.defineProperty(this, "lc_namespace", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: ["langchain_core", "prompts", "chat"]
+        });
+        Object.defineProperty(this, "lc_serializable", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: true
+        });
+        Object.defineProperty(this, "inputVariables", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+        Object.defineProperty(this, "additionalOptions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {}
+        });
+        Object.defineProperty(this, "prompt", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "messageClass", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // ChatMessage contains role field, others don't.
+        // Because of this, we have a separate class property for ChatMessage.
+        Object.defineProperty(this, "chatMessageClass", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.prompt = fields.prompt;
+        if (Array.isArray(this.prompt)) {
+            let inputVariables = [];
+            this.prompt.forEach((prompt) => {
+                if ("inputVariables" in prompt) {
+                    inputVariables = inputVariables.concat(prompt.inputVariables);
+                }
+            });
+            this.inputVariables = inputVariables;
+        }
+        else {
+            this.inputVariables = this.prompt.inputVariables;
+        }
+        this.additionalOptions = additionalOptions ?? this.additionalOptions;
+    }
+    createMessage(content) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const constructor = this.constructor;
+        if (constructor._messageClass()) {
+            const MsgClass = constructor._messageClass();
+            return new MsgClass({ content });
+        }
+        else if (constructor.chatMessageClass) {
+            const MsgClass = constructor.chatMessageClass();
+            // Assuming ChatMessage constructor also takes a content argument
+            return new MsgClass({
+                content,
+                role: this.getRoleFromMessageClass(MsgClass.lc_name()),
+            });
+        }
+        else {
+            throw new Error("No message class defined");
+        }
+    }
+    getRoleFromMessageClass(name) {
+        switch (name) {
+            case "HumanMessage":
+                return "human";
+            case "AIMessage":
+                return "ai";
+            case "SystemMessage":
+                return "system";
+            case "ChatMessage":
+                return "chat";
+            default:
+                throw new Error("Invalid message class name");
+        }
+    }
+    static fromTemplate(template, additionalOptions) {
+        if (typeof template === "string") {
+            return new this(PromptTemplate.fromTemplate(template));
+        }
+        const prompt = [];
+        for (const item of template) {
+            if (typeof item === "string" ||
+                (typeof item === "object" && "text" in item)) {
+                let text = "";
+                if (typeof item === "string") {
+                    text = item;
+                }
+                else if (typeof item.text === "string") {
+                    text = item.text ?? "";
+                }
+                prompt.push(PromptTemplate.fromTemplate(text));
+            }
+            else if (typeof item === "object" && "image_url" in item) {
+                let imgTemplate = item.image_url ?? "";
+                let imgTemplateObject;
+                let inputVariables = [];
+                if (typeof imgTemplate === "string") {
+                    const parsedTemplate = parseFString(imgTemplate);
+                    const variables = parsedTemplate.flatMap((item) => item.type === "variable" ? [item.name] : []);
+                    if (variables) {
+                        if (variables.length > 1) {
+                            throw new Error(`Only one format variable allowed per image template.\nGot: ${variables}\nFrom: ${imgTemplate}`);
+                        }
+                        inputVariables = [variables[0]];
+                    }
+                    else {
+                        inputVariables = [];
+                    }
+                    imgTemplate = { url: imgTemplate };
+                    imgTemplateObject = new ImagePromptTemplate({
+                        template: imgTemplate,
+                        inputVariables,
+                    });
+                }
+                else if (typeof imgTemplate === "object") {
+                    if ("url" in imgTemplate) {
+                        const parsedTemplate = parseFString(imgTemplate.url);
+                        inputVariables = parsedTemplate.flatMap((item) => item.type === "variable" ? [item.name] : []);
+                    }
+                    else {
+                        inputVariables = [];
+                    }
+                    imgTemplateObject = new ImagePromptTemplate({
+                        template: imgTemplate,
+                        inputVariables,
+                    });
+                }
+                else {
+                    throw new Error("Invalid image template");
+                }
+                prompt.push(imgTemplateObject);
+            }
+        }
+        return new this({ prompt, additionalOptions });
+    }
+    async format(input) {
+        // eslint-disable-next-line no-instanceof/no-instanceof
+        if (this.prompt instanceof BaseStringPromptTemplate) {
+            const text = await this.prompt.format(input);
+            return this.createMessage(text);
+        }
+        else {
+            const content = [];
+            for (const prompt of this.prompt) {
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                let inputs = {};
+                if (!("inputVariables" in prompt)) {
+                    throw new Error(`Prompt ${prompt} does not have inputVariables defined.`);
+                }
+                for (const item of prompt.inputVariables) {
+                    if (!inputs) {
+                        inputs = { [item]: input[item] };
+                    }
+                    inputs = { ...inputs, [item]: input[item] };
+                }
+                // eslint-disable-next-line no-instanceof/no-instanceof
+                if (prompt instanceof BaseStringPromptTemplate) {
+                    const formatted = await prompt.format(inputs);
+                    content.push({ type: "text", text: formatted });
+                    /** @TODO replace this */
+                    // eslint-disable-next-line no-instanceof/no-instanceof
+                }
+                else if (prompt instanceof ImagePromptTemplate) {
+                    const formatted = await prompt.format(inputs);
+                    content.push({ type: "image_url", image_url: formatted });
+                }
+            }
+            return this.createMessage(content);
+        }
+    }
+    async formatMessages(values) {
+        return [await this.format(values)];
+    }
+}
 /**
  * Class that represents a human message prompt template. It extends the
  * BaseMessageStringPromptTemplate.
@@ -182,31 +389,25 @@ export class ChatMessagePromptTemplate extends BaseMessageStringPromptTemplate {
  * });
  * ```
  */
-export class HumanMessagePromptTemplate extends BaseMessageStringPromptTemplate {
+export class HumanMessagePromptTemplate extends _StringImageMessagePromptTemplate {
+    static _messageClass() {
+        return HumanMessage;
+    }
     static lc_name() {
         return "HumanMessagePromptTemplate";
     }
-    async format(values) {
-        return new HumanMessage(await this.prompt.format(values));
-    }
-    static fromTemplate(template) {
-        return new this(PromptTemplate.fromTemplate(template));
-    }
 }
 /**
  * Class that represents an AI message prompt template. It extends the
  * BaseMessageStringPromptTemplate.
  */
-export class AIMessagePromptTemplate extends BaseMessageStringPromptTemplate {
+export class AIMessagePromptTemplate extends _StringImageMessagePromptTemplate {
+    static _messageClass() {
+        return AIMessage;
+    }
     static lc_name() {
         return "AIMessagePromptTemplate";
     }
-    async format(values) {
-        return new AIMessage(await this.prompt.format(values));
-    }
-    static fromTemplate(template) {
-        return new this(PromptTemplate.fromTemplate(template));
-    }
 }
 /**
  * Class that represents a system message prompt template. It extends the
@@ -222,16 +423,13 @@ export class AIMessagePromptTemplate extends BaseMessageStringPromptTemplate {
  * });
  * ```
  */
-export class SystemMessagePromptTemplate extends BaseMessageStringPromptTemplate {
+export class SystemMessagePromptTemplate extends _StringImageMessagePromptTemplate {
+    static _messageClass() {
+        return SystemMessage;
+    }
     static lc_name() {
         return "SystemMessagePromptTemplate";
     }
-    async format(values) {
-        return new SystemMessage(await this.prompt.format(values));
-    }
-    static fromTemplate(template) {
-        return new this(PromptTemplate.fromTemplate(template));
-    }
 }
 function _isBaseMessagePromptTemplate(baseMessagePromptTemplateLike) {
     return (typeof baseMessagePromptTemplateLike
@@ -339,16 +537,26 @@ export class ChatPromptTemplate extends BaseChatPromptTemplate {
             return message;
         }
         const formattedMessageContent = await Promise.all(message.content.map(async (item) => {
-            if (item.type !== "image_url" ||
-                typeof item.image_url === "string" ||
-                !item.image_url?.url) {
+            if (item.type !== "image_url") {
                 return item;
             }
-            const imageUrl = item.image_url.url;
+            let imageUrl = "";
+            if (typeof item.image_url === "string") {
+                imageUrl = item.image_url;
+            }
+            else {
+                imageUrl = item.image_url.url;
+            }
             const promptTemplatePlaceholder = PromptTemplate.fromTemplate(imageUrl);
             const formattedUrl = await promptTemplatePlaceholder.format(inputValues);
-            // eslint-disable-next-line no-param-reassign
-            item.image_url.url = formattedUrl;
+            if (typeof item.image_url !== "string" && "url" in item.image_url) {
+                // eslint-disable-next-line no-param-reassign
+                item.image_url.url = formattedUrl;
+            }
+            else {
+                // eslint-disable-next-line no-param-reassign
+                item.image_url = formattedUrl;
+            }
             return item;
         }));
         // eslint-disable-next-line no-param-reassign

package/dist/prompts/image.cjs

@@ -0,0 +1,115 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ImagePromptTemplate = void 0;
+const prompt_values_js_1 = require("../prompt_values.cjs");
+const base_js_1 = require("./base.cjs");
+const template_js_1 = require("./template.cjs");
+/**
+ * An image prompt template for a multimodal model.
+ */
+class ImagePromptTemplate extends base_js_1.BasePromptTemplate {
+    static lc_name() {
+        return "ImagePromptTemplate";
+    }
+    constructor(input) {
+        super(input);
+        Object.defineProperty(this, "template", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "templateFormat", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "f-string"
+        });
+        Object.defineProperty(this, "validateTemplate", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: true
+        });
+        this.template = input.template;
+        this.templateFormat = input.templateFormat ?? this.templateFormat;
+        this.validateTemplate = input.validateTemplate ?? this.validateTemplate;
+        if (this.validateTemplate) {
+            let totalInputVariables = this.inputVariables;
+            if (this.partialVariables) {
+                totalInputVariables = totalInputVariables.concat(Object.keys(this.partialVariables));
+            }
+            (0, template_js_1.checkValidTemplate)([
+                { type: "image_url", image_url: this.template },
+            ], this.templateFormat, totalInputVariables);
+        }
+    }
+    _getPromptType() {
+        return "prompt";
+    }
+    /**
+     * Partially applies values to the prompt template.
+     * @param values The values to be partially applied to the prompt template.
+     * @returns A new instance of ImagePromptTemplate with the partially applied values.
+     */
+    async partial(values) {
+        const newInputVariables = this.inputVariables.filter((iv) => !(iv in values));
+        const newPartialVariables = {
+            ...(this.partialVariables ?? {}),
+            ...values,
+        };
+        const promptDict = {
+            ...this,
+            inputVariables: newInputVariables,
+            partialVariables: newPartialVariables,
+        };
+        return new ImagePromptTemplate(promptDict);
+    }
+    /**
+     * Formats the prompt template with the provided values.
+     * @param values The values to be used to format the prompt template.
+     * @returns A promise that resolves to a string which is the formatted prompt.
+     */
+    async format(values) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const formatted = {};
+        for (const [key, value] of Object.entries(this.template)) {
+            if (typeof value === "string") {
+                formatted[key] = value.replace(/{([^{}]*)}/g, (match, group) => {
+                    const replacement = values[group];
+                    return typeof replacement === "string" ||
+                        typeof replacement === "number"
+                        ? String(replacement)
+                        : match;
+                });
+            }
+            else {
+                formatted[key] = value;
+            }
+        }
+        const url = values.url || formatted.url;
+        const detail = values.detail || formatted.detail;
+        if (!url) {
+            throw new Error("Must provide either an image URL.");
+        }
+        if (typeof url !== "string") {
+            throw new Error("url must be a string.");
+        }
+        const output = { url };
+        if (detail) {
+            output.detail = detail;
+        }
+        return output;
+    }
+    /**
+     * Formats the prompt given the input values and returns a formatted
+     * prompt value.
+     * @param values The input values to format the prompt.
+     * @returns A Promise that resolves to a formatted prompt value.
+     */
+    async formatPromptValue(values) {
+        const formattedPrompt = await this.format(values);
+        return new prompt_values_js_1.ImagePromptValue(formattedPrompt);
+    }
+}
+exports.ImagePromptTemplate = ImagePromptTemplate;

package/dist/prompts/image.d.ts

@@ -0,0 +1,56 @@
+import { ImagePromptValue, ImageContent } from "../prompt_values.js";
+import type { InputValues, PartialValues } from "../utils/types.js";
+import { BasePromptTemplate, BasePromptTemplateInput, TypedPromptInputValues } from "./base.js";
+import { TemplateFormat } from "./template.js";
+/**
+ * Inputs to create a {@link ImagePromptTemplate}
+ * @augments BasePromptTemplateInput
+ */
+export interface ImagePromptTemplateInput<RunInput extends InputValues = any, PartialVariableName extends string = any> extends BasePromptTemplateInput<RunInput, PartialVariableName> {
+    /**
+     * The prompt template
+     */
+    template: Record<string, unknown>;
+    /**
+     * The format of the prompt template. Options are 'f-string'
+     *
+     * @defaultValue 'f-string'
+     */
+    templateFormat?: TemplateFormat;
+    /**
+     * Whether or not to try validating the template on initialization
+     *
+     * @defaultValue `true`
+     */
+    validateTemplate?: boolean;
+}
+/**
+ * An image prompt template for a multimodal model.
+ */
+export declare class ImagePromptTemplate<RunInput extends InputValues = any, PartialVariableName extends string = any> extends BasePromptTemplate<RunInput, ImagePromptValue, PartialVariableName> {
+    static lc_name(): string;
+    template: Record<string, unknown>;
+    templateFormat: TemplateFormat;
+    validateTemplate: boolean;
+    constructor(input: ImagePromptTemplateInput<RunInput, PartialVariableName>);
+    _getPromptType(): "prompt";
+    /**
+     * Partially applies values to the prompt template.
+     * @param values The values to be partially applied to the prompt template.
+     * @returns A new instance of ImagePromptTemplate with the partially applied values.
+     */
+    partial<NewPartialVariableName extends string>(values: PartialValues<NewPartialVariableName>): Promise<ImagePromptTemplate<InputValues<Exclude<Extract<keyof RunInput, string>, NewPartialVariableName>>, any>>;
+    /**
+     * Formats the prompt template with the provided values.
+     * @param values The values to be used to format the prompt template.
+     * @returns A promise that resolves to a string which is the formatted prompt.
+     */
+    format<FormatOutput = ImageContent>(values: TypedPromptInputValues<RunInput>): Promise<FormatOutput>;
+    /**
+     * Formats the prompt given the input values and returns a formatted
+     * prompt value.
+     * @param values The input values to format the prompt.
+     * @returns A Promise that resolves to a formatted prompt value.
+     */
+    formatPromptValue(values: TypedPromptInputValues<RunInput>): Promise<ImagePromptValue>;
+}

package/dist/prompts/image.js

@@ -0,0 +1,111 @@
+import { ImagePromptValue } from "../prompt_values.js";
+import { BasePromptTemplate, } from "./base.js";
+import { checkValidTemplate } from "./template.js";
+/**
+ * An image prompt template for a multimodal model.
+ */
+export class ImagePromptTemplate extends BasePromptTemplate {
+    static lc_name() {
+        return "ImagePromptTemplate";
+    }
+    constructor(input) {
+        super(input);
+        Object.defineProperty(this, "template", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "templateFormat", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "f-string"
+        });
+        Object.defineProperty(this, "validateTemplate", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: true
+        });
+        this.template = input.template;
+        this.templateFormat = input.templateFormat ?? this.templateFormat;
+        this.validateTemplate = input.validateTemplate ?? this.validateTemplate;
+        if (this.validateTemplate) {
+            let totalInputVariables = this.inputVariables;
+            if (this.partialVariables) {
+                totalInputVariables = totalInputVariables.concat(Object.keys(this.partialVariables));
+            }
+            checkValidTemplate([
+                { type: "image_url", image_url: this.template },
+            ], this.templateFormat, totalInputVariables);
+        }
+    }
+    _getPromptType() {
+        return "prompt";
+    }
+    /**
+     * Partially applies values to the prompt template.
+     * @param values The values to be partially applied to the prompt template.
+     * @returns A new instance of ImagePromptTemplate with the partially applied values.
+     */
+    async partial(values) {
+        const newInputVariables = this.inputVariables.filter((iv) => !(iv in values));
+        const newPartialVariables = {
+            ...(this.partialVariables ?? {}),
+            ...values,
+        };
+        const promptDict = {
+            ...this,
+            inputVariables: newInputVariables,
+            partialVariables: newPartialVariables,
+        };
+        return new ImagePromptTemplate(promptDict);
+    }
+    /**
+     * Formats the prompt template with the provided values.
+     * @param values The values to be used to format the prompt template.
+     * @returns A promise that resolves to a string which is the formatted prompt.
+     */
+    async format(values) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const formatted = {};
+        for (const [key, value] of Object.entries(this.template)) {
+            if (typeof value === "string") {
+                formatted[key] = value.replace(/{([^{}]*)}/g, (match, group) => {
+                    const replacement = values[group];
+                    return typeof replacement === "string" ||
+                        typeof replacement === "number"
+                        ? String(replacement)
+                        : match;
+                });
+            }
+            else {
+                formatted[key] = value;
+            }
+        }
+        const url = values.url || formatted.url;
+        const detail = values.detail || formatted.detail;
+        if (!url) {
+            throw new Error("Must provide either an image URL.");
+        }
+        if (typeof url !== "string") {
+            throw new Error("url must be a string.");
+        }
+        const output = { url };
+        if (detail) {
+            output.detail = detail;
+        }
+        return output;
+    }
+    /**
+     * Formats the prompt given the input values and returns a formatted
+     * prompt value.
+     * @param values The input values to format the prompt.
+     * @returns A Promise that resolves to a formatted prompt value.
+     */
+    async formatPromptValue(values) {
+        const formattedPrompt = await this.format(values);
+        return new ImagePromptValue(formattedPrompt);
+    }
+}

package/dist/prompts/prompt.d.ts

@@ -3,6 +3,7 @@ import type { BasePromptTemplateInput, TypedPromptInputValues } from "./base.js"
 import { type TemplateFormat } from "./template.js";
 import type { SerializedPromptTemplate } from "./serde.js";
 import type { InputValues, PartialValues } from "../utils/types.js";
+import { MessageContent } from "../messages/index.js";
 /**
  * Inputs to create a {@link PromptTemplate}
  * @augments BasePromptTemplateInput
@@ -11,7 +12,7 @@ export interface PromptTemplateInput<RunInput extends InputValues = any, Partial
     /**
      * The prompt template
      */
-    template: string;
+    template: MessageContent;
     /**
      * The format of the prompt template. Options are 'f-string'
      *
@@ -52,7 +53,7 @@ export type ParamsFromFString<T extends string> = {
  */
 export declare class PromptTemplate<RunInput extends InputValues = any, PartialVariableName extends string = any> extends BaseStringPromptTemplate<RunInput, PartialVariableName> implements PromptTemplateInput<RunInput, PartialVariableName> {
     static lc_name(): string;
-    template: string;
+    template: MessageContent;
     templateFormat: TemplateFormat;
     validateTemplate: boolean;
     constructor(input: PromptTemplateInput<RunInput, PartialVariableName>);

package/dist/prompts/serde.d.ts

@@ -1,3 +1,4 @@
+import { MessageContent } from "../messages/index.js";
 import type { TemplateFormat } from "./template.js";
 /**
  * Represents a serialized version of a prompt template. This type is used
@@ -12,7 +13,7 @@ export type SerializedPromptTemplate = {
     _type?: "prompt";
     input_variables: string[];
     template_format?: TemplateFormat;
-    template?: string;
+    template?: MessageContent;
 };
 /**
  * Represents a serialized version of a few-shot template. This type

package/dist/prompts/template.cjs

@@ -81,7 +81,28 @@ const checkValidTemplate = (template, templateFormat, inputVariables) => {
             acc[v] = "foo";
             return acc;
         }, {});
-        (0, exports.renderTemplate)(template, templateFormat, dummyInputs);
+        if (Array.isArray(template)) {
+            template.forEach((message) => {
+                if (message.type === "text") {
+                    (0, exports.renderTemplate)(message.text, templateFormat, dummyInputs);
+                }
+                else if (message.type === "image_url") {
+                    if (typeof message.image_url === "string") {
+                        (0, exports.renderTemplate)(message.image_url, templateFormat, dummyInputs);
+                    }
+                    else {
+                        const imageUrl = message.image_url.url;
+                        (0, exports.renderTemplate)(imageUrl, templateFormat, dummyInputs);
+                    }
+                }
+                else {
+                    throw new Error(`Invalid message template received. ${JSON.stringify(message, null, 2)}`);
+                }
+            });
+        }
+        else {
+            (0, exports.renderTemplate)(template, templateFormat, dummyInputs);
+        }
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
     }
     catch (e) {

package/dist/prompts/template.d.ts

@@ -1,3 +1,4 @@
+import { MessageContent } from "../messages/index.js";
 import type { InputValues } from "../utils/types.js";
 /**
  * Type that specifies the format of a template. Only
@@ -32,5 +33,5 @@ export declare const DEFAULT_FORMATTER_MAPPING: Record<TemplateFormat, Interpola
 export declare const DEFAULT_PARSER_MAPPING: Record<TemplateFormat, Parser>;
 export declare const renderTemplate: (template: string, templateFormat: TemplateFormat, inputValues: InputValues) => string;
 export declare const parseTemplate: (template: string, templateFormat: TemplateFormat) => ParsedFStringNode[];
-export declare const checkValidTemplate: (template: string, templateFormat: TemplateFormat, inputVariables: string[]) => void;
+export declare const checkValidTemplate: (template: MessageContent, templateFormat: TemplateFormat, inputVariables: string[]) => void;
 export {};