modelfusion 0.103.0 → 0.105.0

package/model-function/generate-text/prompt-template/AlpacaPromptTemplate.d.ts

@@ -1,5 +1,5 @@
 import { TextGenerationPromptTemplate } from "../TextGenerationPromptTemplate.js";
-import { TextInstructionPrompt } from "./InstructionPrompt.js";
+import { InstructionPrompt } from "./InstructionPrompt.js";
 /**
  * Formats a text prompt as an Alpaca prompt.
  */
@@ -40,7 +40,7 @@ export declare function text(): TextGenerationPromptTemplate<string, string>;
  *
  * @see https://github.com/tatsu-lab/stanford_alpaca#data-release
  */
-export declare function instruction(): TextGenerationPromptTemplate<TextInstructionPrompt & {
+export declare function instruction(): TextGenerationPromptTemplate<InstructionPrompt & {
     input?: string;
 }, // optional input supported by Alpaca
 string>;

package/model-function/generate-text/prompt-template/AlpacaPromptTemplate.js

@@ -1,3 +1,4 @@
+import { validateContentIsString } from "./ContentPart.js";
 const DEFAULT_SYSTEM_PROMPT_INPUT = "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.";
 const DEFAULT_SYSTEM_PROMPT_NO_INPUT = "Below is an instruction that describes a task. Write a response that appropriately completes the request.";
 /**
@@ -63,7 +64,7 @@ export function instruction() {
             if (prompt.system != null) {
                 text += `${prompt.system}\n`;
             }
-            text += prompt.instruction;
+            text += validateContentIsString(prompt.instruction, prompt);
             if (prompt.input != null) {
                 text += `\n\n### Input:\n${prompt.input}`;
             }

package/model-function/generate-text/prompt-template/ChatMLPromptTemplate.cjs

@@ -1,7 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.chat = exports.instruction = exports.text = void 0;
-const ChatPrompt_js_1 = require("./ChatPrompt.cjs");
+const ContentPart_js_1 = require("./ContentPart.cjs");
+const InvalidPromptError_js_1 = require("./InvalidPromptError.cjs");
 const START_SEGMENT = "<|im_start|>";
 const END_SEGMENT = "<|im_end|>";
 function segmentStart(role) {
@@ -40,8 +41,9 @@ function instruction() {
     return {
         stopSequences: [END_SEGMENT],
         format(prompt) {
+            const instruction = (0, ContentPart_js_1.validateContentIsString)(prompt.instruction, prompt);
             return (segment("system", prompt.system) +
-                segment("user", prompt.instruction) +
+                segment("user", instruction) +
                 segmentStart("assistant") +
                 (prompt.responsePrefix ?? ""));
         },
@@ -64,18 +66,20 @@ exports.instruction = instruction;
 function chat() {
     return {
         format(prompt) {
-            (0, ChatPrompt_js_1.validateChatPrompt)(prompt);
             let text = prompt.system != null ? segment("system", prompt.system) : "";
             for (const { role, content } of prompt.messages) {
                 switch (role) {
                     case "user": {
-                        text += segment("user", content);
+                        text += segment("user", (0, ContentPart_js_1.validateContentIsString)(content, prompt));
                         break;
                     }
                     case "assistant": {
-                        text += segment("assistant", content);
+                        text += segment("assistant", (0, ContentPart_js_1.validateContentIsString)(content, prompt));
                         break;
                     }
+                    case "tool": {
+                        throw new InvalidPromptError_js_1.InvalidPromptError("Tool messages are not supported.", prompt);
+                    }
                     default: {
                         const _exhaustiveCheck = role;
                         throw new Error(`Unsupported role: ${_exhaustiveCheck}`);

package/model-function/generate-text/prompt-template/ChatMLPromptTemplate.d.ts

@@ -1,6 +1,6 @@
 import { TextGenerationPromptTemplate } from "../TextGenerationPromptTemplate.js";
-import { TextChatPrompt } from "./ChatPrompt.js";
-import { TextInstructionPrompt } from "./InstructionPrompt.js";
+import { ChatPrompt } from "./ChatPrompt.js";
+import { InstructionPrompt } from "./InstructionPrompt.js";
 /**
  * Formats a text prompt using the ChatML format.
  */
@@ -18,7 +18,7 @@ export declare function text(): TextGenerationPromptTemplate<string, string>;
  * ${response prefix}
  * ```
  */
-export declare function instruction(): TextGenerationPromptTemplate<TextInstructionPrompt, string>;
+export declare function instruction(): TextGenerationPromptTemplate<InstructionPrompt, string>;
 /**
  * Formats a chat prompt using the ChatML format.
  *
@@ -32,4 +32,4 @@ export declare function instruction(): TextGenerationPromptTemplate<TextInstruct
  * Paris<|im_end|>
  * ```
  */
-export declare function chat(): TextGenerationPromptTemplate<TextChatPrompt, string>;
+export declare function chat(): TextGenerationPromptTemplate<ChatPrompt, string>;

package/model-function/generate-text/prompt-template/ChatMLPromptTemplate.js

@@ -1,4 +1,5 @@
-import { validateChatPrompt } from "./ChatPrompt.js";
+import { validateContentIsString } from "./ContentPart.js";
+import { InvalidPromptError } from "./InvalidPromptError.js";
 const START_SEGMENT = "<|im_start|>";
 const END_SEGMENT = "<|im_end|>";
 function segmentStart(role) {
@@ -36,8 +37,9 @@ export function instruction() {
     return {
         stopSequences: [END_SEGMENT],
         format(prompt) {
+            const instruction = validateContentIsString(prompt.instruction, prompt);
             return (segment("system", prompt.system) +
-                segment("user", prompt.instruction) +
+                segment("user", instruction) +
                 segmentStart("assistant") +
                 (prompt.responsePrefix ?? ""));
         },
@@ -59,18 +61,20 @@ export function instruction() {
 export function chat() {
     return {
         format(prompt) {
-            validateChatPrompt(prompt);
             let text = prompt.system != null ? segment("system", prompt.system) : "";
             for (const { role, content } of prompt.messages) {
                 switch (role) {
                     case "user": {
-                        text += segment("user", content);
+                        text += segment("user", validateContentIsString(content, prompt));
                         break;
                     }
                     case "assistant": {
-                        text += segment("assistant", content);
+                        text += segment("assistant", validateContentIsString(content, prompt));
                         break;
                     }
+                    case "tool": {
+                        throw new InvalidPromptError("Tool messages are not supported.", prompt);
+                    }
                     default: {
                         const _exhaustiveCheck = role;
                         throw new Error(`Unsupported role: ${_exhaustiveCheck}`);

package/model-function/generate-text/prompt-template/ChatPrompt.cjs

@@ -1,26 +1,44 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.validateChatPrompt = void 0;
-const InvalidPromptError_js_1 = require("./InvalidPromptError.cjs");
-/**
- * Checks if a chat prompt is valid. Throws a {@link ChatPromptValidationError} if it's not.
- *
- * @throws {@link ChatPromptValidationError}
- */
-function validateChatPrompt(chatPrompt) {
-    const messages = chatPrompt.messages;
-    if (messages.length < 1) {
-        throw new InvalidPromptError_js_1.InvalidPromptError("ChatPrompt should have at least one message.", chatPrompt);
+exports.ChatMessage = void 0;
+exports.ChatMessage = {
+    user({ text }) {
+        return {
+            role: "user",
+            content: text,
+        };
+    },
+    tool({ toolResults, }) {
+        return {
+            role: "tool",
+            content: createToolContent({ toolResults }),
+        };
+    },
+    assistant({ text, toolResults, }) {
+        return {
+            role: "assistant",
+            content: createAssistantContent({ text, toolResults }),
+        };
+    },
+};
+function createToolContent({ toolResults, }) {
+    const toolContent = [];
+    for (const { result, toolCall } of toolResults ?? []) {
+        toolContent.push({
+            type: "tool-response",
+            id: toolCall.id,
+            response: result,
+        });
     }
-    for (let i = 0; i < messages.length; i++) {
-        const expectedRole = i % 2 === 0 ? "user" : "assistant";
-        const role = messages[i].role;
-        if (role !== expectedRole) {
-            throw new InvalidPromptError_js_1.InvalidPromptError(`Message at index ${i} should have role '${expectedRole}', but has role '${role}'.`, chatPrompt);
-        }
+    return toolContent;
+}
+function createAssistantContent({ text, toolResults, }) {
+    const content = [];
+    if (text != null) {
+        content.push({ type: "text", text });
     }
-    if (messages.length % 2 === 0) {
-        throw new InvalidPromptError_js_1.InvalidPromptError("The last message must be a user message.", chatPrompt);
+    for (const { toolCall } of toolResults ?? []) {
+        content.push({ type: "tool-call", ...toolCall });
     }
+    return content;
 }
-exports.validateChatPrompt = validateChatPrompt;

package/model-function/generate-text/prompt-template/ChatPrompt.d.ts

@@ -1,15 +1,14 @@
-import { MultiModalInput } from "./Content.js";
+import { ToolCallResult } from "../../../tool/ToolCallResult.js";
+import { ImagePart, TextPart, ToolCallPart, ToolResponsePart } from "./ContentPart.js";
 /**
- * A textual chat prompt is a combination of a system message and a list of messages with the following constraints:
+ * A chat prompt is a combination of a system message and a list
+ * of user, assistant, and tool messages.
  *
- * - A chat prompt can optionally have a system message.
- * - The first message of the chat must be a user message.
- * - Then it must be alternating between an assistant message and a user message.
- * - The last message must always be a user message (when submitting to a model).
+ * The user messages can contain multi-modal content.
+ * The assistant messages can contain tool calls.
  *
- * You can use a ChatPrompt without an final user message when you e.g. want to display the current state of a conversation.
- *
- * The type checking is done at runtime when you submit a chat prompt to a model with a prompt template.
+ * Note: Not all models and prompt formats support multi-modal inputs and tool calls.
+ * The validation happens at runtime.
  *
  * @example
  * ```ts
@@ -22,38 +21,38 @@ import { MultiModalInput } from "./Content.js";
  *  ],
  * };
  * ```
- *
- * @see validateChatPrompt
  */
-export interface TextChatPrompt {
+export interface ChatPrompt {
     system?: string;
-    messages: Array<TextChatMessage>;
+    messages: Array<ChatMessage>;
 }
+export type UserContent = string | Array<TextPart | ImagePart>;
+export type AssistantContent = string | Array<TextPart | ToolCallPart>;
+export type ToolContent = Array<ToolResponsePart>;
 /**
- * A text message in a chat prompt.
- * @see TextChatPrompt
+ * A message in a chat prompt.
+ *
+ * @see ChatPrompt
  */
-export type TextChatMessage = {
+export type ChatMessage = {
     role: "user";
-    content: string;
+    content: UserContent;
 } | {
     role: "assistant";
-    content: string;
-};
-export interface MultiModalChatPrompt {
-    system?: string;
-    messages: Array<MultiModalChatMessage>;
-}
-export type MultiModalChatMessage = {
-    role: "user";
-    content: MultiModalInput;
+    content: AssistantContent;
 } | {
-    role: "assistant";
-    content: string;
+    role: "tool";
+    content: ToolContent;
+};
+export declare const ChatMessage: {
+    user({ text }: {
+        text: string;
+    }): ChatMessage;
+    tool({ toolResults, }: {
+        toolResults: ToolCallResult<string, unknown, unknown>[] | null;
+    }): ChatMessage;
+    assistant({ text, toolResults, }: {
+        text: string | null;
+        toolResults: ToolCallResult<string, unknown, unknown>[] | null;
+    }): ChatMessage;
 };
-/**
- * Checks if a chat prompt is valid. Throws a {@link ChatPromptValidationError} if it's not.
- *
- * @throws {@link ChatPromptValidationError}
- */
-export declare function validateChatPrompt(chatPrompt: TextChatPrompt | MultiModalChatPrompt): void;

package/model-function/generate-text/prompt-template/ChatPrompt.js

@@ -1,22 +1,41 @@
-import { InvalidPromptError } from "./InvalidPromptError.js";
-/**
- * Checks if a chat prompt is valid. Throws a {@link ChatPromptValidationError} if it's not.
- *
- * @throws {@link ChatPromptValidationError}
- */
-export function validateChatPrompt(chatPrompt) {
-    const messages = chatPrompt.messages;
-    if (messages.length < 1) {
-        throw new InvalidPromptError("ChatPrompt should have at least one message.", chatPrompt);
+export const ChatMessage = {
+    user({ text }) {
+        return {
+            role: "user",
+            content: text,
+        };
+    },
+    tool({ toolResults, }) {
+        return {
+            role: "tool",
+            content: createToolContent({ toolResults }),
+        };
+    },
+    assistant({ text, toolResults, }) {
+        return {
+            role: "assistant",
+            content: createAssistantContent({ text, toolResults }),
+        };
+    },
+};
+function createToolContent({ toolResults, }) {
+    const toolContent = [];
+    for (const { result, toolCall } of toolResults ?? []) {
+        toolContent.push({
+            type: "tool-response",
+            id: toolCall.id,
+            response: result,
+        });
     }
-    for (let i = 0; i < messages.length; i++) {
-        const expectedRole = i % 2 === 0 ? "user" : "assistant";
-        const role = messages[i].role;
-        if (role !== expectedRole) {
-            throw new InvalidPromptError(`Message at index ${i} should have role '${expectedRole}', but has role '${role}'.`, chatPrompt);
-        }
+    return toolContent;
+}
+function createAssistantContent({ text, toolResults, }) {
+    const content = [];
+    if (text != null) {
+        content.push({ type: "text", text });
     }
-    if (messages.length % 2 === 0) {
-        throw new InvalidPromptError("The last message must be a user message.", chatPrompt);
+    for (const { toolCall } of toolResults ?? []) {
+        content.push({ type: "tool-call", ...toolCall });
     }
+    return content;
 }

package/model-function/generate-text/prompt-template/ContentPart.cjs

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.validateContentIsString = void 0;
+const InvalidPromptError_js_1 = require("./InvalidPromptError.cjs");
+function validateContentIsString(content, prompt) {
+    if (typeof content !== "string") {
+        throw new InvalidPromptError_js_1.InvalidPromptError("Only text prompts are are supported by this prompt template.", prompt);
+    }
+    return content;
+}
+exports.validateContentIsString = validateContentIsString;

package/model-function/generate-text/prompt-template/ContentPart.d.ts

@@ -0,0 +1,30 @@
+export interface TextPart {
+    type: "text";
+    /**
+     * The text content.
+     */
+    text: string;
+}
+export interface ImagePart {
+    type: "image";
+    /**
+     * Base-64 encoded image.
+     */
+    base64Image: string;
+    /**
+     * Optional mime type of the image.
+     */
+    mimeType?: string;
+}
+export interface ToolCallPart {
+    type: "tool-call";
+    id: string;
+    name: string;
+    args: unknown;
+}
+export interface ToolResponsePart {
+    type: "tool-response";
+    id: string;
+    response: unknown;
+}
+export declare function validateContentIsString(content: string | unknown, prompt: unknown): string;

package/model-function/generate-text/prompt-template/ContentPart.js

@@ -0,0 +1,7 @@
+import { InvalidPromptError } from "./InvalidPromptError.js";
+export function validateContentIsString(content, prompt) {
+    if (typeof content !== "string") {
+        throw new InvalidPromptError("Only text prompts are are supported by this prompt template.", prompt);
+    }
+    return content;
+}

package/model-function/generate-text/prompt-template/InstructionPrompt.d.ts

@@ -1,24 +1,10 @@
-import { MultiModalInput } from "./Content.js";
-/**
- * A single multi-modal instruction prompt. It can contain an optional system message to define
- * the role and behavior of the language model.
- * The instruction is a multi-model input (`array` of content).
- */
-export interface MultiModalInstructionPrompt {
-    /**
-     * Optional system message to provide context for the language model. Note that for some models,
-     * changing the system message can impact the results, because the model may be trained on the default system message.
-     */
-    system?: string;
-    /**
-     * The multi-modal instruction for the model.
-     */
-    instruction: MultiModalInput;
-}
+import { ImagePart, TextPart } from "./ContentPart";
 /**
  * A single text instruction prompt. It can contain an optional system message to define
  * the role and behavior of the language model.
  *
+ * The instruction can be a text instruction or a multi-modal instruction.
+ *
  * @example
  * ```ts
  * {
@@ -27,21 +13,20 @@ export interface MultiModalInstructionPrompt {
  * }
  * ```
  */
-export interface TextInstructionPrompt {
+export interface InstructionPrompt {
     /**
      * Optional system message to provide context for the language model. Note that for some models,
      * changing the system message can impact the results, because the model may be trained on the default system message.
      */
     system?: string;
     /**
-     * The text instruction for the model.
+     * The instruction for the model.
      */
-    instruction: string;
+    instruction: InstructionContent;
     /**
      * Response prefix that will be injected in the prompt at the beginning of the response.
      * This is useful for guiding the model by starting its response with a specific text.
-     *
-     * Note: Not all models support this feature. E.g. it is not supported by OpenAI chat models.
      */
     responsePrefix?: string;
 }
+export type InstructionContent = string | Array<TextPart | ImagePart>;

package/model-function/generate-text/prompt-template/Llama2PromptTemplate.cjs

@@ -1,7 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.chat = exports.instruction = exports.text = void 0;
-const ChatPrompt_js_1 = require("./ChatPrompt.cjs");
+exports.validateLlama2Prompt = exports.chat = exports.instruction = exports.text = void 0;
+const ContentPart_js_1 = require("./ContentPart.cjs");
+const InvalidPromptError_js_1 = require("./InvalidPromptError.cjs");
 // see https://github.com/facebookresearch/llama/blob/6c7fe276574e78057f917549435a2554000a876d/llama/generation.py#L44
 const BEGIN_SEGMENT = "<s>";
 const END_SEGMENT = " </s>";
@@ -47,9 +48,10 @@ function instruction() {
     return {
         stopSequences: [END_SEGMENT],
         format(prompt) {
+            const instruction = (0, ContentPart_js_1.validateContentIsString)(prompt.instruction, prompt);
             return `${BEGIN_SEGMENT}${BEGIN_INSTRUCTION}${prompt.system != null
                 ? `${BEGIN_SYSTEM}${prompt.system}${END_SYSTEM}`
-                : ""}${prompt.instruction}${END_INSTRUCTION}${prompt.responsePrefix ?? ""}`;
+                : ""}${instruction}${END_INSTRUCTION}${prompt.responsePrefix ?? ""}`;
         },
     };
 }
@@ -69,7 +71,7 @@ exports.instruction = instruction;
 function chat() {
     return {
         format(prompt) {
-            (0, ChatPrompt_js_1.validateChatPrompt)(prompt);
+            validateLlama2Prompt(prompt);
             let text = prompt.system != null
                 ? // Separate section for system message to simplify implementation
                     // (this is slightly different from the original instructions):
@@ -78,13 +80,17 @@ function chat() {
             for (const { role, content } of prompt.messages) {
                 switch (role) {
                     case "user": {
-                        text += `${BEGIN_SEGMENT}${BEGIN_INSTRUCTION}${content}${END_INSTRUCTION}`;
+                        const textContent = (0, ContentPart_js_1.validateContentIsString)(content, prompt);
+                        text += `${BEGIN_SEGMENT}${BEGIN_INSTRUCTION}${textContent}${END_INSTRUCTION}`;
                         break;
                     }
                     case "assistant": {
-                        text += `${content}${END_SEGMENT}`;
+                        text += `${(0, ContentPart_js_1.validateContentIsString)(content, prompt)}${END_SEGMENT}`;
                         break;
                     }
+                    case "tool": {
+                        throw new InvalidPromptError_js_1.InvalidPromptError("Tool messages are not supported.", prompt);
+                    }
                     default: {
                         const _exhaustiveCheck = role;
                         throw new Error(`Unsupported role: ${_exhaustiveCheck}`);
@@ -97,3 +103,31 @@ function chat() {
     };
 }
 exports.chat = chat;
+/**
+ * Checks if a Llama2 chat prompt is valid. Throws a {@link ChatPromptValidationError} if it's not.
+ *
+ * - The first message of the chat must be a user message.
+ * - Then it must be alternating between an assistant message and a user message.
+ * - The last message must always be a user message (when submitting to a model).
+ *
+ * The type checking is done at runtime when you submit a chat prompt to a model with a prompt template.
+ *
+ * @throws {@link ChatPromptValidationError}
+ */
+function validateLlama2Prompt(chatPrompt) {
+    const messages = chatPrompt.messages;
+    if (messages.length < 1) {
+        throw new InvalidPromptError_js_1.InvalidPromptError("ChatPrompt should have at least one message.", chatPrompt);
+    }
+    for (let i = 0; i < messages.length; i++) {
+        const expectedRole = i % 2 === 0 ? "user" : "assistant";
+        const role = messages[i].role;
+        if (role !== expectedRole) {
+            throw new InvalidPromptError_js_1.InvalidPromptError(`Message at index ${i} should have role '${expectedRole}', but has role '${role}'.`, chatPrompt);
+        }
+    }
+    if (messages.length % 2 === 0) {
+        throw new InvalidPromptError_js_1.InvalidPromptError("The last message must be a user message.", chatPrompt);
+    }
+}
+exports.validateLlama2Prompt = validateLlama2Prompt;

package/model-function/generate-text/prompt-template/Llama2PromptTemplate.d.ts

@@ -1,6 +1,6 @@
 import { TextGenerationPromptTemplate } from "../TextGenerationPromptTemplate.js";
-import { TextChatPrompt } from "./ChatPrompt.js";
-import { TextInstructionPrompt } from "./InstructionPrompt.js";
+import { ChatPrompt } from "./ChatPrompt.js";
+import { InstructionPrompt } from "./InstructionPrompt.js";
 /**
  * Formats a text prompt as a Llama 2 prompt.
  *
@@ -27,7 +27,7 @@ export declare function text(): TextGenerationPromptTemplate<string, string>;
  *
  * @see https://www.philschmid.de/llama-2#how-to-prompt-llama-2-chat
  */
-export declare function instruction(): TextGenerationPromptTemplate<TextInstructionPrompt, string>;
+export declare function instruction(): TextGenerationPromptTemplate<InstructionPrompt, string>;
 /**
  * Formats a chat prompt as a Llama 2 prompt.
  *
@@ -40,4 +40,16 @@ export declare function instruction(): TextGenerationPromptTemplate<TextInstruct
  * ${ user msg 1 } [/INST] ${ model response 1 } </s><s>[INST] ${ user msg 2 } [/INST] ${ model response 2 } </s><s>[INST] ${ user msg 3 } [/INST]
  * ```
  */
-export declare function chat(): TextGenerationPromptTemplate<TextChatPrompt, string>;
+export declare function chat(): TextGenerationPromptTemplate<ChatPrompt, string>;
+/**
+ * Checks if a Llama2 chat prompt is valid. Throws a {@link ChatPromptValidationError} if it's not.
+ *
+ * - The first message of the chat must be a user message.
+ * - Then it must be alternating between an assistant message and a user message.
+ * - The last message must always be a user message (when submitting to a model).
+ *
+ * The type checking is done at runtime when you submit a chat prompt to a model with a prompt template.
+ *
+ * @throws {@link ChatPromptValidationError}
+ */
+export declare function validateLlama2Prompt(chatPrompt: ChatPrompt): void;