modelfusion 0.87.1 → 0.88.0

package/model-function/generate-text/PromptFormatTextGenerationModel.d.ts

@@ -1,6 +1,6 @@
 import { FunctionOptions } from "../../core/FunctionOptions.js";
 import { StructureFromTextGenerationModel } from "../../model-function/generate-structure/StructureFromTextGenerationModel.js";
-import { StructureFromTextPromptFormat } from "model-function/generate-structure/StructureFromTextPromptFormat.js";
+import { StructureFromTextPromptFormat } from "../../model-function/generate-structure/StructureFromTextPromptFormat.js";
 import { TextGenerationToolCallModel, ToolCallPromptFormat } from "../../tool/generate-tool-call/TextGenerationToolCallModel.js";
 import { TextGenerationToolCallsOrGenerateTextModel, ToolCallsOrGenerateTextPromptFormat } from "../../tool/generate-tool-calls-or-text/TextGenerationToolCallsOrGenerateTextModel.js";
 import { TextGenerationModel, TextGenerationModelSettings } from "./TextGenerationModel.js";

package/model-function/generate-text/prompt-format/ChatMLPromptFormat.d.ts

@@ -1,5 +1,5 @@
 import { TextGenerationPromptFormat } from "../TextGenerationPromptFormat.js";
-import { ChatPrompt } from "./ChatPrompt.js";
+import { TextChatPrompt } from "./ChatPrompt.js";
 import { TextInstructionPrompt } from "./InstructionPrompt.js";
 /**
  * Formats a text prompt using the ChatML format.
@@ -32,4 +32,4 @@ export declare function instruction(): TextGenerationPromptFormat<TextInstructio
  * Paris<|im_end|>
  * ```
  */
-export declare function chat(): TextGenerationPromptFormat<ChatPrompt, string>;
+export declare function chat(): TextGenerationPromptFormat<TextChatPrompt, string>;

package/model-function/generate-text/prompt-format/ChatPrompt.d.ts

@@ -1,5 +1,6 @@
+import { MultiModalInput } from "./Content.js";
 /**
- * A chat prompt is a combination of a system message and a list of messages with the following constraints:
+ * A textual chat prompt is a combination of a system message and a list of messages with the following constraints:
  *
  * - A chat prompt can optionally have a system message.
  * - The first message of the chat must be a user message.
@@ -24,21 +25,35 @@
  *
  * @see validateChatPrompt
  */
-export interface ChatPrompt {
+export interface TextChatPrompt {
     system?: string;
-    messages: Array<ChatMessage>;
+    messages: Array<TextChatMessage>;
 }
 /**
- * A message in a chat prompt.
- * @see ChatPrompt
+ * A text message in a chat prompt.
+ * @see TextChatPrompt
  */
-export interface ChatMessage {
-    role: "user" | "assistant";
+export type TextChatMessage = {
+    role: "user";
     content: string;
+} | {
+    role: "assistant";
+    content: string;
+};
+export interface MultiModalChatPrompt {
+    system?: string;
+    messages: Array<MultiModalChatMessage>;
 }
+export type MultiModalChatMessage = {
+    role: "user";
+    content: MultiModalInput;
+} | {
+    role: "assistant";
+    content: string;
+};
 /**
  * Checks if a chat prompt is valid. Throws a {@link ChatPromptValidationError} if it's not.
  *
  * @throws {@link ChatPromptValidationError}
  */
-export declare function validateChatPrompt(chatPrompt: ChatPrompt): void;
+export declare function validateChatPrompt(chatPrompt: TextChatPrompt | MultiModalChatPrompt): void;

package/model-function/generate-text/prompt-format/Llama2PromptFormat.d.ts

@@ -1,5 +1,5 @@
 import { TextGenerationPromptFormat } from "../TextGenerationPromptFormat.js";
-import { ChatPrompt } from "./ChatPrompt.js";
+import { TextChatPrompt } from "./ChatPrompt.js";
 import { TextInstructionPrompt } from "./InstructionPrompt.js";
 /**
  * Formats a text prompt as a Llama 2 prompt.
@@ -39,4 +39,4 @@ export declare function instruction(): TextGenerationPromptFormat<TextInstructio
  * ${ user msg 1 } [/INST] ${ model response 1 } </s><s>[INST] ${ user msg 2 } [/INST] ${ model response 2 } </s><s>[INST] ${ user msg 3 } [/INST]
  * ```
  */
-export declare function chat(): TextGenerationPromptFormat<ChatPrompt, string>;
+export declare function chat(): TextGenerationPromptFormat<TextChatPrompt, string>;

package/model-function/generate-text/prompt-format/TextPromptFormat.d.ts

@@ -1,5 +1,5 @@
 import { TextGenerationPromptFormat } from "../TextGenerationPromptFormat.js";
-import { ChatPrompt } from "./ChatPrompt.js";
+import { TextChatPrompt } from "./ChatPrompt.js";
 import { TextInstructionPrompt } from "./InstructionPrompt.js";
 /**
  * Formats a text prompt as a basic text prompt. Does not change the text prompt in any way.
@@ -20,4 +20,4 @@ export declare const chat: (options?: {
     user?: string;
     assistant?: string;
     system?: string;
-}) => TextGenerationPromptFormat<ChatPrompt, string>;
+}) => TextGenerationPromptFormat<TextChatPrompt, string>;

package/model-function/generate-text/prompt-format/VicunaPromptFormat.d.ts

@@ -1,5 +1,5 @@
 import { TextGenerationPromptFormat } from "../TextGenerationPromptFormat.js";
-import { ChatPrompt } from "./ChatPrompt.js";
+import { TextChatPrompt } from "./ChatPrompt.js";
 /**
  * Formats a chat prompt as a Vicuna prompt.
  *
@@ -13,4 +13,4 @@ import { ChatPrompt } from "./ChatPrompt.js";
  * ASSISTANT:
  * ```
  */
-export declare function chat(): TextGenerationPromptFormat<ChatPrompt, string>;
+export declare function chat(): TextGenerationPromptFormat<TextChatPrompt, string>;

package/model-function/generate-text/prompt-format/trimChatPrompt.d.ts

@@ -1,5 +1,5 @@
 import { HasContextWindowSize, HasTokenizer, TextGenerationModel, TextGenerationModelSettings } from "../TextGenerationModel.js";
-import { ChatPrompt } from "./ChatPrompt.js";
+import { TextChatPrompt } from "./ChatPrompt.js";
 /**
  * Keeps only the most recent messages in the prompt, while leaving enough space for the completion.
  *
@@ -11,7 +11,7 @@ import { ChatPrompt } from "./ChatPrompt.js";
  * @see https://modelfusion.dev/guide/function/generate-text#limiting-the-chat-length
  */
 export declare function trimChatPrompt({ prompt, model, tokenLimit, }: {
-    prompt: ChatPrompt;
-    model: TextGenerationModel<ChatPrompt, TextGenerationModelSettings> & HasTokenizer<ChatPrompt> & HasContextWindowSize;
+    prompt: TextChatPrompt;
+    model: TextGenerationModel<TextChatPrompt, TextGenerationModelSettings> & HasTokenizer<TextChatPrompt> & HasContextWindowSize;
     tokenLimit?: number;
-}): Promise<ChatPrompt>;
+}): Promise<TextChatPrompt>;

package/model-provider/anthropic/AnthropicPromptFormat.d.ts

@@ -1,5 +1,5 @@
 import { TextGenerationPromptFormat } from "../../model-function/generate-text/TextGenerationPromptFormat.js";
-import { ChatPrompt } from "../../model-function/generate-text/prompt-format/ChatPrompt.js";
+import { TextChatPrompt } from "../../model-function/generate-text/prompt-format/ChatPrompt.js";
 import { TextInstructionPrompt } from "../../model-function/generate-text/prompt-format/InstructionPrompt.js";
 /**
  * Formats a text prompt as an Anthropic prompt.
@@ -14,4 +14,4 @@ export declare function instruction(): TextGenerationPromptFormat<TextInstructio
  *
  * @see https://docs.anthropic.com/claude/docs/constructing-a-prompt
  */
-export declare function chat(): TextGenerationPromptFormat<ChatPrompt, string>;
+export declare function chat(): TextGenerationPromptFormat<TextChatPrompt, string>;

package/model-provider/anthropic/AnthropicTextGenerationModel.d.ts

@@ -69,7 +69,7 @@ export declare class AnthropicTextGenerationModel extends AbstractModel<Anthropi
     /**
      * Returns this model with a chat prompt format.
      */
-    withChatPrompt(): PromptFormatTextStreamingModel<import("../../index.js").ChatPrompt, string, AnthropicTextGenerationModelSettings, this>;
+    withChatPrompt(): PromptFormatTextStreamingModel<import("../../index.js").TextChatPrompt, string, AnthropicTextGenerationModelSettings, this>;
     withPromptFormat<INPUT_PROMPT>(promptFormat: TextGenerationPromptFormat<INPUT_PROMPT, string>): PromptFormatTextStreamingModel<INPUT_PROMPT, string, AnthropicTextGenerationModelSettings, this>;
     withSettings(additionalSettings: Partial<AnthropicTextGenerationModelSettings>): this;
 }

package/model-provider/cohere/CohereTextGenerationModel.d.ts

@@ -94,7 +94,7 @@ export declare class CohereTextGenerationModel extends AbstractModel<CohereTextG
     withChatPrompt(options?: {
         user?: string;
         assistant?: string;
-    }): PromptFormatTextStreamingModel<import("../../index.js").ChatPrompt, string, CohereTextGenerationModelSettings, this>;
+    }): PromptFormatTextStreamingModel<import("../../index.js").TextChatPrompt, string, CohereTextGenerationModelSettings, this>;
     withPromptFormat<INPUT_PROMPT>(promptFormat: TextGenerationPromptFormat<INPUT_PROMPT, string>): PromptFormatTextStreamingModel<INPUT_PROMPT, string, CohereTextGenerationModelSettings, this>;
     withSettings(additionalSettings: Partial<CohereTextGenerationModelSettings>): this;
 }

package/model-provider/llamacpp/LlamaCppBakLLaVA1Format.cjs

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.instruction = void 0;
+exports.chat = exports.instruction = void 0;
+const ChatPrompt_js_1 = require("../../model-function/generate-text/prompt-format/ChatPrompt.cjs");
 // default Vicuna 1 system message
 const DEFAULT_SYSTEM_MESSAGE = "A chat between a curious user and an artificial intelligence assistant. " +
     "The assistant gives helpful, detailed, and polite answers to the user's questions.";
@@ -33,10 +34,57 @@ function instruction() {
                 }
                 text += `${content}\n`;
             }
-            text += `ASSISTANT: `;
+            text += `\nASSISTANT: `;
             return { text, images };
         },
         stopSequences: [`\nUSER:`],
     };
 }
 exports.instruction = instruction;
+function chat() {
+    return {
+        format(prompt) {
+            (0, ChatPrompt_js_1.validateChatPrompt)(prompt);
+            let text = "";
+            text += `${prompt.system ?? DEFAULT_SYSTEM_MESSAGE}\n\n`;
+            // construct text and image mapping:
+            let imageCounter = 1;
+            const images = {};
+            for (const { role, content } of prompt.messages) {
+                switch (role) {
+                    case "user": {
+                        text += `USER: `;
+                        for (const part of content) {
+                            switch (part.type) {
+                                case "text": {
+                                    text += part.text;
+                                    break;
+                                }
+                                case "image": {
+                                    text += `[img-${imageCounter}]`;
+                                    images[imageCounter.toString()] = part.base64Image;
+                                    imageCounter++;
+                                    break;
+                                }
+                            }
+                        }
+                        break;
+                    }
+                    case "assistant": {
+                        text += `ASSISTANT: ${content}`;
+                        break;
+                    }
+                    default: {
+                        const _exhaustiveCheck = role;
+                        throw new Error(`Unsupported role: ${_exhaustiveCheck}`);
+                    }
+                }
+                text += `\n\n`;
+            }
+            text += `ASSISTANT: `;
+            return { text, images };
+        },
+        stopSequences: [`\nUSER:`],
+    };
+}
+exports.chat = chat;

package/model-provider/llamacpp/LlamaCppBakLLaVA1Format.d.ts

@@ -1,5 +1,6 @@
-import { MultiModalInstructionPrompt } from "../../model-function/generate-text/prompt-format/InstructionPrompt.js";
 import { TextGenerationPromptFormat } from "../../model-function/generate-text/TextGenerationPromptFormat.js";
+import { MultiModalChatPrompt } from "../../model-function/generate-text/prompt-format/ChatPrompt.js";
+import { MultiModalInstructionPrompt } from "../../model-function/generate-text/prompt-format/InstructionPrompt.js";
 import { LlamaCppTextGenerationPrompt } from "./LlamaCppTextGenerationModel.js";
 /**
  * BakLLaVA 1 uses a Vicuna 1 prompt. This mapping combines it with the LlamaCpp prompt structure.
@@ -7,3 +8,4 @@ import { LlamaCppTextGenerationPrompt } from "./LlamaCppTextGenerationModel.js";
  * @see https://github.com/SkunkworksAI/BakLLaVA
  */
 export declare function instruction(): TextGenerationPromptFormat<MultiModalInstructionPrompt, LlamaCppTextGenerationPrompt>;
+export declare function chat(): TextGenerationPromptFormat<MultiModalChatPrompt, LlamaCppTextGenerationPrompt>;

package/model-provider/llamacpp/LlamaCppBakLLaVA1Format.js

@@ -1,3 +1,4 @@
+import { validateChatPrompt, } from "../../model-function/generate-text/prompt-format/ChatPrompt.js";
 // default Vicuna 1 system message
 const DEFAULT_SYSTEM_MESSAGE = "A chat between a curious user and an artificial intelligence assistant. " +
     "The assistant gives helpful, detailed, and polite answers to the user's questions.";
@@ -30,6 +31,52 @@ export function instruction() {
                 }
                 text += `${content}\n`;
             }
+            text += `\nASSISTANT: `;
+            return { text, images };
+        },
+        stopSequences: [`\nUSER:`],
+    };
+}
+export function chat() {
+    return {
+        format(prompt) {
+            validateChatPrompt(prompt);
+            let text = "";
+            text += `${prompt.system ?? DEFAULT_SYSTEM_MESSAGE}\n\n`;
+            // construct text and image mapping:
+            let imageCounter = 1;
+            const images = {};
+            for (const { role, content } of prompt.messages) {
+                switch (role) {
+                    case "user": {
+                        text += `USER: `;
+                        for (const part of content) {
+                            switch (part.type) {
+                                case "text": {
+                                    text += part.text;
+                                    break;
+                                }
+                                case "image": {
+                                    text += `[img-${imageCounter}]`;
+                                    images[imageCounter.toString()] = part.base64Image;
+                                    imageCounter++;
+                                    break;
+                                }
+                            }
+                        }
+                        break;
+                    }
+                    case "assistant": {
+                        text += `ASSISTANT: ${content}`;
+                        break;
+                    }
+                    default: {
+                        const _exhaustiveCheck = role;
+                        throw new Error(`Unsupported role: ${_exhaustiveCheck}`);
+                    }
+                }
+                text += `\n\n`;
+            }
             text += `ASSISTANT: `;
             return { text, images };
         },

package/model-provider/openai/OpenAICompletionModel.d.ts

@@ -183,7 +183,7 @@ export declare class OpenAICompletionModel extends AbstractModel<OpenAICompletio
     withChatPrompt(options?: {
         user?: string;
         assistant?: string;
-    }): PromptFormatTextStreamingModel<import("../../index.js").ChatPrompt, string, OpenAICompletionModelSettings, this>;
+    }): PromptFormatTextStreamingModel<import("../../index.js").TextChatPrompt, string, OpenAICompletionModelSettings, this>;
     withPromptFormat<INPUT_PROMPT>(promptFormat: TextGenerationPromptFormat<INPUT_PROMPT, string>): PromptFormatTextStreamingModel<INPUT_PROMPT, string, OpenAICompletionModelSettings, this>;
     withSettings(additionalSettings: Partial<OpenAICompletionModelSettings>): this;
 }

package/model-provider/openai/chat/OpenAIChatFunctionCallStructureGenerationModel.d.ts

@@ -31,7 +31,7 @@ OpenAIChatSettings> {
     /**
      * Returns this model with a chat prompt format.
      */
-    withChatPrompt(): OpenAIChatFunctionCallStructureGenerationModel<TextGenerationPromptFormat<import("../../../index.js").ChatPrompt, import("./OpenAIChatMessage.js").OpenAIChatMessage[]>>;
+    withChatPrompt(): OpenAIChatFunctionCallStructureGenerationModel<TextGenerationPromptFormat<import("../../../index.js").TextChatPrompt | import("../../../index.js").MultiModalChatPrompt, import("./OpenAIChatMessage.js").OpenAIChatMessage[]>>;
     withPromptFormat<TARGET_PROMPT_FORMAT extends TextGenerationPromptFormat<unknown, OpenAIChatPrompt>>(promptFormat: TARGET_PROMPT_FORMAT): OpenAIChatFunctionCallStructureGenerationModel<TARGET_PROMPT_FORMAT>;
     withSettings(additionalSettings: Partial<OpenAIChatSettings>): this;
     /**

package/model-provider/openai/chat/OpenAIChatModel.d.ts

@@ -155,7 +155,7 @@ export declare class OpenAIChatModel extends AbstractOpenAIChatModel<OpenAIChatS
     /**
      * Returns this model with a chat prompt format.
      */
-    withChatPrompt(): PromptFormatTextStreamingModel<import("../../../index.js").ChatPrompt, OpenAIChatPrompt, OpenAIChatSettings, this>;
+    withChatPrompt(): PromptFormatTextStreamingModel<import("../../../index.js").TextChatPrompt | import("../../../index.js").MultiModalChatPrompt, OpenAIChatPrompt, OpenAIChatSettings, this>;
     withPromptFormat<INPUT_PROMPT>(promptFormat: TextGenerationPromptFormat<INPUT_PROMPT, OpenAIChatPrompt>): PromptFormatTextStreamingModel<INPUT_PROMPT, OpenAIChatPrompt, OpenAIChatSettings, this>;
     withSettings(additionalSettings: Partial<OpenAIChatSettings>): this;
 }

package/model-provider/openai/chat/OpenAIChatPromptFormat.d.ts

@@ -1,5 +1,5 @@
 import { TextGenerationPromptFormat } from "../../../model-function/generate-text/TextGenerationPromptFormat.js";
-import { ChatPrompt } from "../../../model-function/generate-text/prompt-format/ChatPrompt.js";
+import { MultiModalChatPrompt, TextChatPrompt } from "../../../model-function/generate-text/prompt-format/ChatPrompt.js";
 import { MultiModalInstructionPrompt, TextInstructionPrompt } from "../../../model-function/generate-text/prompt-format/InstructionPrompt.js";
 import { OpenAIChatMessage } from "./OpenAIChatMessage.js";
 /**
@@ -17,4 +17,4 @@ export declare function instruction(): TextGenerationPromptFormat<MultiModalInst
 /**
  * Formats a chat prompt as an OpenAI chat prompt.
  */
-export declare function chat(): TextGenerationPromptFormat<ChatPrompt, Array<OpenAIChatMessage>>;
+export declare function chat(): TextGenerationPromptFormat<MultiModalChatPrompt | TextChatPrompt, Array<OpenAIChatMessage>>;

package/model-provider/openai-compatible/OpenAICompatibleChatModel.d.ts

@@ -39,7 +39,7 @@ export declare class OpenAICompatibleChatModel extends AbstractOpenAIChatModel<O
     /**
      * Returns this model with a chat prompt format.
      */
-    withChatPrompt(): PromptFormatTextStreamingModel<import("../../index.js").ChatPrompt, OpenAIChatPrompt, OpenAICompatibleChatSettings, this>;
+    withChatPrompt(): PromptFormatTextStreamingModel<import("../../index.js").TextChatPrompt | import("../../index.js").MultiModalChatPrompt, OpenAIChatPrompt, OpenAICompatibleChatSettings, this>;
     withPromptFormat<INPUT_PROMPT>(promptFormat: TextGenerationPromptFormat<INPUT_PROMPT, OpenAIChatPrompt>): PromptFormatTextStreamingModel<INPUT_PROMPT, OpenAIChatPrompt, OpenAICompatibleChatSettings, this>;
     withSettings(additionalSettings: Partial<OpenAICompatibleChatSettings>): this;
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "The TypeScript library for building multi-modal AI applications.",
-  "version": "0.87.1",
+  "version": "0.88.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [