modelfusion 0.63.0 → 0.65.0

package/README.md

@@ -14,8 +14,8 @@
 
 **ModelFusion** is a TypeScript library for building AI applications, chatbots, and agents.
 
-- **Vendor-neutral**: ModelFusion is a non-commercial open source project that is community-driven. You can use it with any supported vendor.
-- **Multimodal**: ModelFusion supports a wide range of models including text generation, image generation, text-to-speech, speech-to-text, and embedding models.
+- **Vendor-neutral**: ModelFusion is a non-commercial open source project that is community-driven. You can use it with any supported provider.
+- **Multimodal**: ModelFusion supports a wide range of models including text generation, image generation, vision, text-to-speech, speech-to-text, and embedding models.
 - **Streaming**: ModelFusion supports streaming for many generation models, e.g. text streaming, structure streaming, and full duplex speech streaming.
 - **Utility functions**: ModelFusion provides functionality for tools and tool usage, vector indices, and guards functions.
 - **Type inference and validation**: ModelFusion infers TypeScript types wherever possible and to validates model responses.
@@ -47,9 +47,7 @@ You can use [prompt formats](https://modelfusion.dev/guide/function/generate-tex
 
 ```ts
 const text = await generateText(
-  new OpenAICompletionModel({
-    model: "gpt-3.5-turbo-instruct",
-  }),
+  new OpenAICompletionModel({ model: "gpt-3.5-turbo-instruct" }),
   "Write a short story about a robot learning to love:\n\n"
 );
 ```
@@ -60,9 +58,7 @@ Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai),
 
 ```ts
 const textStream = await streamText(
-  new OpenAICompletionModel({
-    model: "gpt-3.5-turbo-instruct",
-  }),
+  new OpenAICompletionModel({ model: "gpt-3.5-turbo-instruct" }),
   "Write a short story about a robot learning to love:\n\n"
 );
 
@@ -73,16 +69,30 @@ for await (const textPart of textStream) {
 
 Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Anthropic](https://modelfusion.dev/integration/model-provider/anthropic), [Cohere](https://modelfusion.dev/integration/model-provider/cohere), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp), [Ollama](https://modelfusion.dev/integration/model-provider/ollama)
 
+#### streamText with multi-modal prompt
+
+Multi-modal vision models such as GPT 4 Vision can process images as part of the prompt.
+
+```ts
+const textStream = await streamText(
+  new OpenAIChatModel({ model: "gpt-4-vision-preview" }),
+  [
+    OpenAIChatMessage.user("Describe the image in detail:", {
+      image: { base64Content: image, mimeType: "image/png" },
+    }),
+  ]
+);
+```
+
+Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp)
+
 ### [Generate Image](https://modelfusion.dev/guide/function/generate-image)
 
 Generate an image from a prompt.
 
 ```ts
 const image = await generateImage(
-  new OpenAIImageGenerationModel({
-    model: "dall-e-3",
-    size: "1024x1024",
-  }),
+  new OpenAIImageGenerationModel({ model: "dall-e-3", size: "1024x1024" }),
   "the wicked witch of the west in the style of early 19th century painting"
 );
 ```
@@ -499,6 +509,8 @@ const textStream = await streamText(
 | Vicuna        | ❌                 | ✅          |
 | Generic Text  | ✅                 | ✅          |
 
+#### [Vision Prompts]
+
 #### [Image Generation Prompt Formats](https://modelfusion.dev/guide/function/generate-image/prompt-format)
 
 You an use prompt formats with image models as well, e.g. to use a basic text prompt. It is available as a shorthand method:

package/model-function/generate-text/index.cjs

@@ -14,18 +14,18 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./AlpacaPromptFormat.cjs"), exports);
-__exportStar(require("./ChatPrompt.cjs"), exports);
-__exportStar(require("./InstructionPrompt.cjs"), exports);
-__exportStar(require("./Llama2PromptFormat.cjs"), exports);
 __exportStar(require("./PromptFormatTextGenerationModel.cjs"), exports);
 __exportStar(require("./PromptFormatTextStreamingModel.cjs"), exports);
 __exportStar(require("./TextGenerationEvent.cjs"), exports);
 __exportStar(require("./TextGenerationModel.cjs"), exports);
 __exportStar(require("./TextGenerationPromptFormat.cjs"), exports);
-__exportStar(require("./TextPromptFormat.cjs"), exports);
-__exportStar(require("./VicunaPromptFormat.cjs"), exports);
 __exportStar(require("./generateText.cjs"), exports);
+__exportStar(require("./prompt-format/AlpacaPromptFormat.cjs"), exports);
+__exportStar(require("./prompt-format/ChatPrompt.cjs"), exports);
+__exportStar(require("./prompt-format/InstructionPrompt.cjs"), exports);
+__exportStar(require("./prompt-format/Llama2PromptFormat.cjs"), exports);
+__exportStar(require("./prompt-format/TextPromptFormat.cjs"), exports);
+__exportStar(require("./prompt-format/VicunaPromptFormat.cjs"), exports);
+__exportStar(require("./prompt-format/trimChatPrompt.cjs"), exports);
+__exportStar(require("./prompt-format/validateChatPrompt.cjs"), exports);
 __exportStar(require("./streamText.cjs"), exports);
-__exportStar(require("./trimChatPrompt.cjs"), exports);
-__exportStar(require("./validateChatPrompt.cjs"), exports);

package/model-function/generate-text/index.d.ts

@@ -1,15 +1,15 @@
-export * from "./AlpacaPromptFormat.js";
-export * from "./ChatPrompt.js";
-export * from "./InstructionPrompt.js";
-export * from "./Llama2PromptFormat.js";
 export * from "./PromptFormatTextGenerationModel.js";
 export * from "./PromptFormatTextStreamingModel.js";
 export * from "./TextGenerationEvent.js";
 export * from "./TextGenerationModel.js";
 export * from "./TextGenerationPromptFormat.js";
-export * from "./TextPromptFormat.js";
-export * from "./VicunaPromptFormat.js";
 export * from "./generateText.js";
+export * from "./prompt-format/AlpacaPromptFormat.js";
+export * from "./prompt-format/ChatPrompt.js";
+export * from "./prompt-format/InstructionPrompt.js";
+export * from "./prompt-format/Llama2PromptFormat.js";
+export * from "./prompt-format/TextPromptFormat.js";
+export * from "./prompt-format/VicunaPromptFormat.js";
+export * from "./prompt-format/trimChatPrompt.js";
+export * from "./prompt-format/validateChatPrompt.js";
 export * from "./streamText.js";
-export * from "./trimChatPrompt.js";
-export * from "./validateChatPrompt.js";

package/model-function/generate-text/index.js

@@ -1,15 +1,15 @@
-export * from "./AlpacaPromptFormat.js";
-export * from "./ChatPrompt.js";
-export * from "./InstructionPrompt.js";
-export * from "./Llama2PromptFormat.js";
 export * from "./PromptFormatTextGenerationModel.js";
 export * from "./PromptFormatTextStreamingModel.js";
 export * from "./TextGenerationEvent.js";
 export * from "./TextGenerationModel.js";
 export * from "./TextGenerationPromptFormat.js";
-export * from "./TextPromptFormat.js";
-export * from "./VicunaPromptFormat.js";
 export * from "./generateText.js";
+export * from "./prompt-format/AlpacaPromptFormat.js";
+export * from "./prompt-format/ChatPrompt.js";
+export * from "./prompt-format/InstructionPrompt.js";
+export * from "./prompt-format/Llama2PromptFormat.js";
+export * from "./prompt-format/TextPromptFormat.js";
+export * from "./prompt-format/VicunaPromptFormat.js";
+export * from "./prompt-format/trimChatPrompt.js";
+export * from "./prompt-format/validateChatPrompt.js";
 export * from "./streamText.js";
-export * from "./trimChatPrompt.js";
-export * from "./validateChatPrompt.js";

package/model-function/generate-text/{AlpacaPromptFormat.d.ts → prompt-format/AlpacaPromptFormat.d.ts}

@@ -1,5 +1,5 @@
 import { InstructionPrompt } from "./InstructionPrompt.js";
-import { TextGenerationPromptFormat } from "./TextGenerationPromptFormat.js";
+import { TextGenerationPromptFormat } from "../TextGenerationPromptFormat.js";
 /**
  * Formats an instruction prompt as an Alpaca prompt.
  *

package/model-function/generate-text/{InstructionPrompt.d.ts → prompt-format/InstructionPrompt.d.ts}

@@ -25,4 +25,17 @@ export type InstructionPrompt = {
      * Optional additional input or context, e.g. a the content from which information should be extracted.
      */
     input?: string;
+    /**
+     * Optional image to provide context for the language model. Only supported by some models.
+     */
+    image?: {
+        /**
+         * Base-64 encoded image.
+         */
+        base64Content: string;
+        /**
+         * Optional mime type of the image.
+         */
+        mimeType?: string;
+    };
 };

package/model-function/generate-text/{Llama2PromptFormat.d.ts → prompt-format/Llama2PromptFormat.d.ts}

@@ -1,6 +1,6 @@
 import { ChatPrompt } from "./ChatPrompt.js";
 import { InstructionPrompt } from "./InstructionPrompt.js";
-import { TextGenerationPromptFormat } from "./TextGenerationPromptFormat.js";
+import { TextGenerationPromptFormat } from "../TextGenerationPromptFormat.js";
 /**
  * Formats an instruction prompt as a Llama 2 prompt.
  *

package/model-function/generate-text/{TextPromptFormat.d.ts → prompt-format/TextPromptFormat.d.ts}

@@ -1,6 +1,6 @@
 import { ChatPrompt } from "./ChatPrompt.js";
 import { InstructionPrompt } from "./InstructionPrompt.js";
-import { TextGenerationPromptFormat } from "./TextGenerationPromptFormat.js";
+import { TextGenerationPromptFormat } from "../TextGenerationPromptFormat.js";
 /**
  * Formats an instruction prompt as a basic text prompt.
  */

package/model-function/generate-text/{VicunaPromptFormat.cjs → prompt-format/VicunaPromptFormat.cjs}

@@ -2,7 +2,9 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.mapChatPromptToVicunaFormat = void 0;
 const validateChatPrompt_js_1 = require("./validateChatPrompt.cjs");
-const DEFAULT_SYSTEM_PROMPT = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.";
+// default Vicuna 1 system message
+const DEFAULT_SYSTEM_MESSAGE = "A chat between a curious user and an artificial intelligence assistant. " +
+    "The assistant gives helpful, detailed, and polite answers to the user's questions.";
 /**
  * Formats a chat prompt as a Vicuna prompt.
  *
@@ -32,7 +34,7 @@ function mapChatPromptToVicunaFormat() {
                 }
                 // first message was not a system message:
                 if (i === 0) {
-                    text += `${DEFAULT_SYSTEM_PROMPT}\n\n`;
+                    text += `${DEFAULT_SYSTEM_MESSAGE}\n\n`;
                 }
                 // user message
                 if ("user" in message) {

package/model-function/generate-text/{VicunaPromptFormat.d.ts → prompt-format/VicunaPromptFormat.d.ts}

@@ -1,5 +1,5 @@
 import { ChatPrompt } from "./ChatPrompt.js";
-import { TextGenerationPromptFormat } from "./TextGenerationPromptFormat.js";
+import { TextGenerationPromptFormat } from "../TextGenerationPromptFormat.js";
 /**
  * Formats a chat prompt as a Vicuna prompt.
  *

package/model-function/generate-text/{VicunaPromptFormat.js → prompt-format/VicunaPromptFormat.js}

@@ -1,5 +1,7 @@
 import { validateChatPrompt } from "./validateChatPrompt.js";
-const DEFAULT_SYSTEM_PROMPT = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.";
+// default Vicuna 1 system message
+const DEFAULT_SYSTEM_MESSAGE = "A chat between a curious user and an artificial intelligence assistant. " +
+    "The assistant gives helpful, detailed, and polite answers to the user's questions.";
 /**
  * Formats a chat prompt as a Vicuna prompt.
  *
@@ -29,7 +31,7 @@ export function mapChatPromptToVicunaFormat() {
                 }
                 // first message was not a system message:
                 if (i === 0) {
-                    text += `${DEFAULT_SYSTEM_PROMPT}\n\n`;
+                    text += `${DEFAULT_SYSTEM_MESSAGE}\n\n`;
                 }
                 // user message
                 if ("user" in message) {

package/model-function/generate-text/{trimChatPrompt.d.ts → prompt-format/trimChatPrompt.d.ts}

@@ -1,5 +1,5 @@
 import { ChatPrompt } from "./ChatPrompt.js";
-import { HasContextWindowSize, HasTokenizer, TextGenerationModel, TextGenerationModelSettings } from "./TextGenerationModel.js";
+import { HasContextWindowSize, HasTokenizer, TextGenerationModel, TextGenerationModelSettings } from "../TextGenerationModel.js";
 /**
  * Keeps only the most recent messages in the prompt, while leaving enough space for the completion.
  *

package/model-provider/anthropic/AnthropicPromptFormat.cjs

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.mapChatPromptToAnthropicFormat = exports.mapInstructionPromptToAnthropicFormat = void 0;
-const validateChatPrompt_js_1 = require("../../model-function/generate-text/validateChatPrompt.cjs");
+const validateChatPrompt_js_1 = require("../../model-function/generate-text/prompt-format/validateChatPrompt.cjs");
 /**
  * Formats an instruction prompt as an Anthropic prompt.
  */

package/model-provider/anthropic/AnthropicPromptFormat.d.ts

@@ -1,5 +1,5 @@
-import { ChatPrompt } from "../../model-function/generate-text/ChatPrompt.js";
-import { InstructionPrompt } from "../../model-function/generate-text/InstructionPrompt.js";
+import { ChatPrompt } from "../../model-function/generate-text/prompt-format/ChatPrompt.js";
+import { InstructionPrompt } from "../../model-function/generate-text/prompt-format/InstructionPrompt.js";
 import { TextGenerationPromptFormat } from "../../model-function/generate-text/TextGenerationPromptFormat.js";
 /**
  * Formats an instruction prompt as an Anthropic prompt.

package/model-provider/anthropic/AnthropicPromptFormat.js

@@ -1,4 +1,4 @@
-import { validateChatPrompt } from "../../model-function/generate-text/validateChatPrompt.js";
+import { validateChatPrompt } from "../../model-function/generate-text/prompt-format/validateChatPrompt.js";
 /**
  * Formats an instruction prompt as an Anthropic prompt.
  */

package/model-provider/cohere/CohereTextGenerationModel.cjs

@@ -6,7 +6,7 @@ const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndTh
 const postToApi_js_1 = require("../../core/api/postToApi.cjs");
 const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
 const PromptFormatTextStreamingModel_js_1 = require("../../model-function/generate-text/PromptFormatTextStreamingModel.cjs");
-const TextPromptFormat_js_1 = require("../../model-function/generate-text/TextPromptFormat.cjs");
+const TextPromptFormat_js_1 = require("../../model-function/generate-text/prompt-format/TextPromptFormat.cjs");
 const countTokens_js_1 = require("../../model-function/tokenize-text/countTokens.cjs");
 const AsyncQueue_js_1 = require("../../util/AsyncQueue.cjs");
 const parseJsonStream_js_1 = require("../../util/streaming/parseJsonStream.cjs");

package/model-provider/cohere/CohereTextGenerationModel.js

@@ -3,7 +3,7 @@ import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottl
 import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
 import { AbstractModel } from "../../model-function/AbstractModel.js";
 import { PromptFormatTextStreamingModel } from "../../model-function/generate-text/PromptFormatTextStreamingModel.js";
-import { mapChatPromptToTextFormat, mapInstructionPromptToTextFormat, } from "../../model-function/generate-text/TextPromptFormat.js";
+import { mapChatPromptToTextFormat, mapInstructionPromptToTextFormat, } from "../../model-function/generate-text/prompt-format/TextPromptFormat.js";
 import { countTokens } from "../../model-function/tokenize-text/countTokens.js";
 import { AsyncQueue } from "../../util/AsyncQueue.js";
 import { parseJsonStream } from "../../util/streaming/parseJsonStream.js";

package/model-provider/llamacpp/index.cjs

@@ -21,3 +21,4 @@ Object.defineProperty(exports, "LlamaCppError", { enumerable: true, get: functio
 __exportStar(require("./LlamaCppTextEmbeddingModel.cjs"), exports);
 __exportStar(require("./LlamaCppTextGenerationModel.cjs"), exports);
 __exportStar(require("./LlamaCppTokenizer.cjs"), exports);
+__exportStar(require("./mapInstructionPromptToBakLLaVA1ForLlamaCppFormat.cjs"), exports);

package/model-provider/llamacpp/index.d.ts

@@ -3,3 +3,4 @@ export { LlamaCppError, LlamaCppErrorData } from "./LlamaCppError.js";
 export * from "./LlamaCppTextEmbeddingModel.js";
 export * from "./LlamaCppTextGenerationModel.js";
 export * from "./LlamaCppTokenizer.js";
+export * from "./mapInstructionPromptToBakLLaVA1ForLlamaCppFormat.js";

package/model-provider/llamacpp/index.js

@@ -3,3 +3,4 @@ export { LlamaCppError } from "./LlamaCppError.js";
 export * from "./LlamaCppTextEmbeddingModel.js";
 export * from "./LlamaCppTextGenerationModel.js";
 export * from "./LlamaCppTokenizer.js";
+export * from "./mapInstructionPromptToBakLLaVA1ForLlamaCppFormat.js";

package/model-provider/llamacpp/mapInstructionPromptToBakLLaVA1ForLlamaCppFormat.cjs

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mapInstructionPromptToBakLLaVA1ForLlamaCppFormat = void 0;
+// default Vicuna 1 system message
+const DEFAULT_SYSTEM_MESSAGE = "A chat between a curious user and an artificial intelligence assistant. " +
+    "The assistant gives helpful, detailed, and polite answers to the user's questions.";
+/**
+ * BakLLaVA 1 uses a Vicuna 1 prompt. This mapping combines it with the LlamaCpp prompts.
+ *
+ * @see https://github.com/SkunkworksAI/BakLLaVA
+ */
+function mapInstructionPromptToBakLLaVA1ForLlamaCppFormat() {
+    return {
+        format: (instruction) => {
+            let text = "";
+            text += `${instruction.system ?? DEFAULT_SYSTEM_MESSAGE}\n\n`;
+            text += `USER: `;
+            if (instruction.image != null) {
+                text += `[img-1]\n`;
+            }
+            text += `${instruction.instruction}\n`;
+            if (instruction.input != null) {
+                text += `${instruction.input}\n`;
+            }
+            text += `ASSISTANT: `;
+            return {
+                text,
+                images: instruction.image != null
+                    ? { "1": instruction.image.base64Content }
+                    : undefined,
+            };
+        },
+        stopSequences: [`\nUSER:`],
+    };
+}
+exports.mapInstructionPromptToBakLLaVA1ForLlamaCppFormat = mapInstructionPromptToBakLLaVA1ForLlamaCppFormat;

package/model-provider/llamacpp/mapInstructionPromptToBakLLaVA1ForLlamaCppFormat.d.ts

@@ -0,0 +1,9 @@
+import { InstructionPrompt } from "../../model-function/generate-text/prompt-format/InstructionPrompt.js";
+import { TextGenerationPromptFormat } from "../../model-function/generate-text/TextGenerationPromptFormat.js";
+import { LlamaCppTextGenerationPrompt } from "./LlamaCppTextGenerationModel.js";
+/**
+ * BakLLaVA 1 uses a Vicuna 1 prompt. This mapping combines it with the LlamaCpp prompts.
+ *
+ * @see https://github.com/SkunkworksAI/BakLLaVA
+ */
+export declare function mapInstructionPromptToBakLLaVA1ForLlamaCppFormat(): TextGenerationPromptFormat<InstructionPrompt, LlamaCppTextGenerationPrompt>;

package/model-provider/llamacpp/mapInstructionPromptToBakLLaVA1ForLlamaCppFormat.js

@@ -0,0 +1,32 @@
+// default Vicuna 1 system message
+const DEFAULT_SYSTEM_MESSAGE = "A chat between a curious user and an artificial intelligence assistant. " +
+    "The assistant gives helpful, detailed, and polite answers to the user's questions.";
+/**
+ * BakLLaVA 1 uses a Vicuna 1 prompt. This mapping combines it with the LlamaCpp prompts.
+ *
+ * @see https://github.com/SkunkworksAI/BakLLaVA
+ */
+export function mapInstructionPromptToBakLLaVA1ForLlamaCppFormat() {
+    return {
+        format: (instruction) => {
+            let text = "";
+            text += `${instruction.system ?? DEFAULT_SYSTEM_MESSAGE}\n\n`;
+            text += `USER: `;
+            if (instruction.image != null) {
+                text += `[img-1]\n`;
+            }
+            text += `${instruction.instruction}\n`;
+            if (instruction.input != null) {
+                text += `${instruction.input}\n`;
+            }
+            text += `ASSISTANT: `;
+            return {
+                text,
+                images: instruction.image != null
+                    ? { "1": instruction.image.base64Content }
+                    : undefined,
+            };
+        },
+        stopSequences: [`\nUSER:`],
+    };
+}

package/model-provider/openai/OpenAICompletionModel.cjs

@@ -6,7 +6,7 @@ const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndTh
 const postToApi_js_1 = require("../../core/api/postToApi.cjs");
 const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
 const PromptFormatTextStreamingModel_js_1 = require("../../model-function/generate-text/PromptFormatTextStreamingModel.cjs");
-const TextPromptFormat_js_1 = require("../../model-function/generate-text/TextPromptFormat.cjs");
+const TextPromptFormat_js_1 = require("../../model-function/generate-text/prompt-format/TextPromptFormat.cjs");
 const countTokens_js_1 = require("../../model-function/tokenize-text/countTokens.cjs");
 const AsyncQueue_js_1 = require("../../util/AsyncQueue.cjs");
 const parseJSON_js_1 = require("../../util/parseJSON.cjs");

package/model-provider/openai/OpenAICompletionModel.js

@@ -3,7 +3,7 @@ import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottl
 import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
 import { AbstractModel } from "../../model-function/AbstractModel.js";
 import { PromptFormatTextStreamingModel } from "../../model-function/generate-text/PromptFormatTextStreamingModel.js";
-import { mapChatPromptToTextFormat, mapInstructionPromptToTextFormat, } from "../../model-function/generate-text/TextPromptFormat.js";
+import { mapChatPromptToTextFormat, mapInstructionPromptToTextFormat, } from "../../model-function/generate-text/prompt-format/TextPromptFormat.js";
 import { countTokens } from "../../model-function/tokenize-text/countTokens.js";
 import { AsyncQueue } from "../../util/AsyncQueue.js";
 import { parseJsonWithZod } from "../../util/parseJSON.js";

package/model-provider/openai/chat/OpenAIChatMessage.cjs

@@ -5,8 +5,21 @@ exports.OpenAIChatMessage = {
     system(content) {
         return { role: "system", content };
     },
-    user(content) {
-        return { role: "user", content };
+    user(content, options) {
+        if (options?.image != null) {
+            return {
+                role: "user",
+                content: [
+                    { type: "text", text: content },
+                    {
+                        type: "image_url",
+                        image_url: `data:${options.image.mimeType ?? "image/jpeg"};base64,${options.image.base64Content}`,
+                    },
+                ],
+                name: options.name,
+            };
+        }
+        return { role: "user", content, name: options?.name };
     },
     assistant(content) {
         return { role: "assistant", content };

package/model-provider/openai/chat/OpenAIChatMessage.d.ts

@@ -1,14 +1,37 @@
 export type OpenAIChatMessage = {
-    role: "user" | "assistant" | "system";
+    role: "system";
     content: string;
     name?: string;
+} | {
+    role: "user";
+    content: string | Array<{
+        type: "text";
+        text: string;
+    } | {
+        type: "image_url";
+        image_url: string;
+    }>;
+    name?: string;
 } | {
     role: "assistant";
     content: string | null;
-    function_call: {
+    name?: string;
+    tool_calls?: Array<{
+        id: string;
+        type: "function";
+        function: {
+            name: string;
+            arguments: string;
+        };
+    }>;
+    function_call?: {
         name: string;
         arguments: string;
     };
+} | {
+    role: "tool";
+    tool_call_id: string;
+    content: string | null;
 } | {
     role: "function";
     content: string;
@@ -16,7 +39,13 @@ export type OpenAIChatMessage = {
 };
 export declare const OpenAIChatMessage: {
     system(content: string): OpenAIChatMessage;
-    user(content: string): OpenAIChatMessage;
+    user(content: string, options?: {
+        name?: string;
+        image?: {
+            base64Content: string;
+            mimeType?: string;
+        };
+    }): OpenAIChatMessage;
     assistant(content: string): OpenAIChatMessage;
     functionCall(content: string | null, functionCall: {
         name: string;

package/model-provider/openai/chat/OpenAIChatMessage.js

@@ -2,8 +2,21 @@ export const OpenAIChatMessage = {
     system(content) {
         return { role: "system", content };
     },
-    user(content) {
-        return { role: "user", content };
+    user(content, options) {
+        if (options?.image != null) {
+            return {
+                role: "user",
+                content: [
+                    { type: "text", text: content },
+                    {
+                        type: "image_url",
+                        image_url: `data:${options.image.mimeType ?? "image/jpeg"};base64,${options.image.base64Content}`,
+                    },
+                ],
+                name: options.name,
+            };
+        }
+        return { role: "user", content, name: options?.name };
     },
     assistant(content) {
         return { role: "assistant", content };

package/model-provider/openai/chat/OpenAIChatPromptFormat.cjs

@@ -1,7 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.mapChatPromptToOpenAIChatFormat = exports.mapInstructionPromptToOpenAIChatFormat = void 0;
-const validateChatPrompt_js_1 = require("../../../model-function/generate-text/validateChatPrompt.cjs");
+const validateChatPrompt_js_1 = require("../../../model-function/generate-text/prompt-format/validateChatPrompt.cjs");
+const OpenAIChatMessage_js_1 = require("./OpenAIChatMessage.cjs");
 /**
  * Formats an instruction prompt as an OpenAI chat prompt.
  */
@@ -10,20 +11,13 @@ function mapInstructionPromptToOpenAIChatFormat() {
         format: (instruction) => {
             const messages = [];
             if (instruction.system != null) {
-                messages.push({
-                    role: "system",
-                    content: instruction.system,
-                });
+                messages.push(OpenAIChatMessage_js_1.OpenAIChatMessage.system(instruction.system));
             }
-            messages.push({
-                role: "user",
-                content: instruction.instruction,
-            });
+            messages.push(OpenAIChatMessage_js_1.OpenAIChatMessage.user(instruction.instruction, {
+                image: instruction.image,
+            }));
             if (instruction.input != null) {
-                messages.push({
-                    role: "user",
-                    content: instruction.input,
-                });
+                messages.push(OpenAIChatMessage_js_1.OpenAIChatMessage.user(instruction.input));
             }
             return messages;
         },

package/model-provider/openai/chat/OpenAIChatPromptFormat.d.ts

@@ -1,6 +1,6 @@
-import { ChatPrompt } from "../../../model-function/generate-text/ChatPrompt.js";
-import { InstructionPrompt } from "../../../model-function/generate-text/InstructionPrompt.js";
 import { TextGenerationPromptFormat } from "../../../model-function/generate-text/TextGenerationPromptFormat.js";
+import { ChatPrompt } from "../../../model-function/generate-text/prompt-format/ChatPrompt.js";
+import { InstructionPrompt } from "../../../model-function/generate-text/prompt-format/InstructionPrompt.js";
 import { OpenAIChatMessage } from "./OpenAIChatMessage.js";
 /**
  * Formats an instruction prompt as an OpenAI chat prompt.

package/model-provider/openai/chat/OpenAIChatPromptFormat.js

@@ -1,4 +1,5 @@
-import { validateChatPrompt } from "../../../model-function/generate-text/validateChatPrompt.js";
+import { validateChatPrompt } from "../../../model-function/generate-text/prompt-format/validateChatPrompt.js";
+import { OpenAIChatMessage } from "./OpenAIChatMessage.js";
 /**
  * Formats an instruction prompt as an OpenAI chat prompt.
  */
@@ -7,20 +8,13 @@ export function mapInstructionPromptToOpenAIChatFormat() {
         format: (instruction) => {
             const messages = [];
             if (instruction.system != null) {
-                messages.push({
-                    role: "system",
-                    content: instruction.system,
-                });
+                messages.push(OpenAIChatMessage.system(instruction.system));
             }
-            messages.push({
-                role: "user",
-                content: instruction.instruction,
-            });
+            messages.push(OpenAIChatMessage.user(instruction.instruction, {
+                image: instruction.image,
+            }));
             if (instruction.input != null) {
-                messages.push({
-                    role: "user",
-                    content: instruction.input,
-                });
+                messages.push(OpenAIChatMessage.user(instruction.input));
             }
             return messages;
         },

package/model-provider/openai/chat/countOpenAIChatMessageTokens.cjs

@@ -15,10 +15,26 @@ exports.OPENAI_CHAT_PROMPT_BASE_TOKEN_COUNT = 2;
  */
 exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT = 5;
 async function countOpenAIChatMessageTokens({ message, model, }) {
-    const contentTokenCount = await (0, countTokens_js_1.countTokens)(new TikTokenTokenizer_js_1.TikTokenTokenizer({
+    const tokenizer = new TikTokenTokenizer_js_1.TikTokenTokenizer({
         model: (0, OpenAIChatModel_js_1.getOpenAIChatModelInformation)(model).baseModel,
-    }), message.content ?? "");
-    return exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT + contentTokenCount;
+    });
+    // case: function call without content
+    if (message.content == null) {
+        return exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT;
+    }
+    // case: simple text content
+    if (typeof message.content === "string") {
+        return (exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT +
+            (await (0, countTokens_js_1.countTokens)(tokenizer, message.content)));
+    }
+    // case: array of content objects
+    let contentTokenCount = exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT;
+    for (const content of message.content) {
+        if (content.type === "text") {
+            contentTokenCount += await (0, countTokens_js_1.countTokens)(tokenizer, content.text);
+        }
+    }
+    return contentTokenCount;
 }
 exports.countOpenAIChatMessageTokens = countOpenAIChatMessageTokens;
 async function countOpenAIChatPromptTokens({ messages, model, }) {

package/model-provider/openai/chat/countOpenAIChatMessageTokens.js

@@ -12,10 +12,26 @@ export const OPENAI_CHAT_PROMPT_BASE_TOKEN_COUNT = 2;
  */
 export const OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT = 5;
 export async function countOpenAIChatMessageTokens({ message, model, }) {
-    const contentTokenCount = await countTokens(new TikTokenTokenizer({
+    const tokenizer = new TikTokenTokenizer({
         model: getOpenAIChatModelInformation(model).baseModel,
-    }), message.content ?? "");
-    return OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT + contentTokenCount;
+    });
+    // case: function call without content
+    if (message.content == null) {
+        return OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT;
+    }
+    // case: simple text content
+    if (typeof message.content === "string") {
+        return (OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT +
+            (await countTokens(tokenizer, message.content)));
+    }
+    // case: array of content objects
+    let contentTokenCount = OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT;
+    for (const content of message.content) {
+        if (content.type === "text") {
+            contentTokenCount += await countTokens(tokenizer, content.text);
+        }
+    }
+    return contentTokenCount;
 }
 export async function countOpenAIChatPromptTokens({ messages, model, }) {
     let tokens = OPENAI_CHAT_PROMPT_BASE_TOKEN_COUNT;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "Build multimodal applications, chatbots, and agents with JavaScript and TypeScript.",
-  "version": "0.63.0",
+  "version": "0.65.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [