modelfusion 0.64.0 → 0.65.1

package/README.md

@@ -14,8 +14,8 @@
 
 **ModelFusion** is a TypeScript library for building AI applications, chatbots, and agents.
 
-- **Vendor-neutral**: ModelFusion is a non-commercial open source project that is community-driven. You can use it with any supported vendor.
-- **Multimodal**: ModelFusion supports a wide range of models including text generation, image generation, text-to-speech, speech-to-text, and embedding models.
+- **Vendor-neutral**: ModelFusion is a non-commercial open source project that is community-driven. You can use it with any supported provider.
+- **Multimodal**: ModelFusion supports a wide range of models including text generation, image generation, vision, text-to-speech, speech-to-text, and embedding models.
 - **Streaming**: ModelFusion supports streaming for many generation models, e.g. text streaming, structure streaming, and full duplex speech streaming.
 - **Utility functions**: ModelFusion provides functionality for tools and tool usage, vector indices, and guards functions.
 - **Type inference and validation**: ModelFusion infers TypeScript types wherever possible and to validates model responses.
@@ -47,9 +47,7 @@ You can use [prompt formats](https://modelfusion.dev/guide/function/generate-tex
 
 ```ts
 const text = await generateText(
-  new OpenAICompletionModel({
-    model: "gpt-3.5-turbo-instruct",
-  }),
+  new OpenAICompletionModel({ model: "gpt-3.5-turbo-instruct" }),
   "Write a short story about a robot learning to love:\n\n"
 );
 ```
@@ -60,9 +58,7 @@ Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai),
 
 ```ts
 const textStream = await streamText(
-  new OpenAICompletionModel({
-    model: "gpt-3.5-turbo-instruct",
-  }),
+  new OpenAICompletionModel({ model: "gpt-3.5-turbo-instruct" }),
   "Write a short story about a robot learning to love:\n\n"
 );
 
@@ -73,16 +69,30 @@ for await (const textPart of textStream) {
 
 Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Anthropic](https://modelfusion.dev/integration/model-provider/anthropic), [Cohere](https://modelfusion.dev/integration/model-provider/cohere), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp), [Ollama](https://modelfusion.dev/integration/model-provider/ollama)
 
+#### streamText with multi-modal prompt
+
+Multi-modal vision models such as GPT 4 Vision can process images as part of the prompt.
+
+```ts
+const textStream = await streamText(
+  new OpenAIChatModel({ model: "gpt-4-vision-preview" }),
+  [
+    OpenAIChatMessage.user("Describe the image in detail:", {
+      image: { base64Content: image, mimeType: "image/png" },
+    }),
+  ]
+);
+```
+
+Providers: [OpenAI](https://modelfusion.dev/integration/model-provider/openai), [Llama.cpp](https://modelfusion.dev/integration/model-provider/llamacpp)
+
 ### [Generate Image](https://modelfusion.dev/guide/function/generate-image)
 
 Generate an image from a prompt.
 
 ```ts
 const image = await generateImage(
-  new OpenAIImageGenerationModel({
-    model: "dall-e-3",
-    size: "1024x1024",
-  }),
+  new OpenAIImageGenerationModel({ model: "dall-e-3", size: "1024x1024" }),
   "the wicked witch of the west in the style of early 19th century painting"
 );
 ```
@@ -499,6 +509,8 @@ const textStream = await streamText(
 | Vicuna        | ❌                 | ✅          |
 | Generic Text  | ✅                 | ✅          |
 
+#### [Vision Prompts]
+
 #### [Image Generation Prompt Formats](https://modelfusion.dev/guide/function/generate-image/prompt-format)
 
 You an use prompt formats with image models as well, e.g. to use a basic text prompt. It is available as a shorthand method:

package/model-function/generate-text/index.cjs

@@ -26,7 +26,6 @@ __exportStar(require("./prompt-format/InstructionPrompt.cjs"), exports);
 __exportStar(require("./prompt-format/Llama2PromptFormat.cjs"), exports);
 __exportStar(require("./prompt-format/TextPromptFormat.cjs"), exports);
 __exportStar(require("./prompt-format/VicunaPromptFormat.cjs"), exports);
-__exportStar(require("./prompt-format/VisionInstructionPrompt.cjs"), exports);
 __exportStar(require("./prompt-format/trimChatPrompt.cjs"), exports);
 __exportStar(require("./prompt-format/validateChatPrompt.cjs"), exports);
 __exportStar(require("./streamText.cjs"), exports);

package/model-function/generate-text/index.d.ts

@@ -10,7 +10,6 @@ export * from "./prompt-format/InstructionPrompt.js";
 export * from "./prompt-format/Llama2PromptFormat.js";
 export * from "./prompt-format/TextPromptFormat.js";
 export * from "./prompt-format/VicunaPromptFormat.js";
-export * from "./prompt-format/VisionInstructionPrompt.js";
 export * from "./prompt-format/trimChatPrompt.js";
 export * from "./prompt-format/validateChatPrompt.js";
 export * from "./streamText.js";

package/model-function/generate-text/index.js

@@ -10,7 +10,6 @@ export * from "./prompt-format/InstructionPrompt.js";
 export * from "./prompt-format/Llama2PromptFormat.js";
 export * from "./prompt-format/TextPromptFormat.js";
 export * from "./prompt-format/VicunaPromptFormat.js";
-export * from "./prompt-format/VisionInstructionPrompt.js";
 export * from "./prompt-format/trimChatPrompt.js";
 export * from "./prompt-format/validateChatPrompt.js";
 export * from "./streamText.js";

package/model-function/generate-text/prompt-format/InstructionPrompt.d.ts

@@ -25,4 +25,17 @@ export type InstructionPrompt = {
      * Optional additional input or context, e.g. a the content from which information should be extracted.
      */
     input?: string;
+    /**
+     * Optional image to provide context for the language model. Only supported by some models.
+     */
+    image?: {
+        /**
+         * Base-64 encoded image.
+         */
+        base64Content: string;
+        /**
+         * Optional mime type of the image.
+         */
+        mimeType?: string;
+    };
 };

package/model-function/generate-text/prompt-format/VicunaPromptFormat.cjs

@@ -2,7 +2,9 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.mapChatPromptToVicunaFormat = void 0;
 const validateChatPrompt_js_1 = require("./validateChatPrompt.cjs");
-const DEFAULT_SYSTEM_PROMPT = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.";
+// default Vicuna 1 system message
+const DEFAULT_SYSTEM_MESSAGE = "A chat between a curious user and an artificial intelligence assistant. " +
+    "The assistant gives helpful, detailed, and polite answers to the user's questions.";
 /**
  * Formats a chat prompt as a Vicuna prompt.
  *
@@ -32,7 +34,7 @@ function mapChatPromptToVicunaFormat() {
                 }
                 // first message was not a system message:
                 if (i === 0) {
-                    text += `${DEFAULT_SYSTEM_PROMPT}\n\n`;
+                    text += `${DEFAULT_SYSTEM_MESSAGE}\n\n`;
                 }
                 // user message
                 if ("user" in message) {

package/model-function/generate-text/prompt-format/VicunaPromptFormat.js

@@ -1,5 +1,7 @@
 import { validateChatPrompt } from "./validateChatPrompt.js";
-const DEFAULT_SYSTEM_PROMPT = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.";
+// default Vicuna 1 system message
+const DEFAULT_SYSTEM_MESSAGE = "A chat between a curious user and an artificial intelligence assistant. " +
+    "The assistant gives helpful, detailed, and polite answers to the user's questions.";
 /**
  * Formats a chat prompt as a Vicuna prompt.
  *
@@ -29,7 +31,7 @@ export function mapChatPromptToVicunaFormat() {
                 }
                 // first message was not a system message:
                 if (i === 0) {
-                    text += `${DEFAULT_SYSTEM_PROMPT}\n\n`;
+                    text += `${DEFAULT_SYSTEM_MESSAGE}\n\n`;
                 }
                 // user message
                 if ("user" in message) {

package/model-provider/llamacpp/LlamaCppTextGenerationModel.cjs

@@ -12,7 +12,6 @@ const parseJSON_js_1 = require("../../util/parseJSON.cjs");
 const LlamaCppApiConfiguration_js_1 = require("./LlamaCppApiConfiguration.cjs");
 const LlamaCppError_js_1 = require("./LlamaCppError.cjs");
 const LlamaCppTokenizer_js_1 = require("./LlamaCppTokenizer.cjs");
-const mapVisionInstructionPromptToLlamaCppFormat_js_1 = require("./mapVisionInstructionPromptToLlamaCppFormat.cjs");
 class LlamaCppTextGenerationModel extends AbstractModel_js_1.AbstractModel {
     constructor(settings = {}) {
         super({ settings });
@@ -108,9 +107,6 @@ class LlamaCppTextGenerationModel extends AbstractModel_js_1.AbstractModel {
             stopSequences: [],
         });
     }
-    withVisionInstructionPrompt() {
-        return this.withPromptFormat((0, mapVisionInstructionPromptToLlamaCppFormat_js_1.mapVisionInstructionPromptToLlamaCppFormat)());
-    }
     withPromptFormat(promptFormat) {
         return new PromptFormatTextStreamingModel_js_1.PromptFormatTextStreamingModel({
             model: this.withSettings({

package/model-provider/llamacpp/LlamaCppTextGenerationModel.d.ts

@@ -111,7 +111,6 @@ export declare class LlamaCppTextGenerationModel<CONTEXT_WINDOW_SIZE extends num
     }>;
     doStreamText(prompt: LlamaCppTextGenerationPrompt, options?: FunctionOptions): Promise<AsyncIterable<Delta<string>>>;
     withTextPrompt(): PromptFormatTextStreamingModel<string, LlamaCppTextGenerationPrompt, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
-    withVisionInstructionPrompt(): PromptFormatTextStreamingModel<import("../../index.js").VisionInstructionPrompt, LlamaCppTextGenerationPrompt, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
     withPromptFormat<INPUT_PROMPT>(promptFormat: TextGenerationPromptFormat<INPUT_PROMPT, LlamaCppTextGenerationPrompt>): PromptFormatTextStreamingModel<INPUT_PROMPT, LlamaCppTextGenerationPrompt, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
     withSettings(additionalSettings: Partial<LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>>): this;
 }

package/model-provider/llamacpp/LlamaCppTextGenerationModel.js

@@ -9,7 +9,6 @@ import { parseJsonWithZod } from "../../util/parseJSON.js";
 import { LlamaCppApiConfiguration } from "./LlamaCppApiConfiguration.js";
 import { failedLlamaCppCallResponseHandler } from "./LlamaCppError.js";
 import { LlamaCppTokenizer } from "./LlamaCppTokenizer.js";
-import { mapVisionInstructionPromptToLlamaCppFormat } from "./mapVisionInstructionPromptToLlamaCppFormat.js";
 export class LlamaCppTextGenerationModel extends AbstractModel {
     constructor(settings = {}) {
         super({ settings });
@@ -105,9 +104,6 @@ export class LlamaCppTextGenerationModel extends AbstractModel {
             stopSequences: [],
         });
     }
-    withVisionInstructionPrompt() {
-        return this.withPromptFormat(mapVisionInstructionPromptToLlamaCppFormat());
-    }
     withPromptFormat(promptFormat) {
         return new PromptFormatTextStreamingModel({
             model: this.withSettings({

package/model-provider/llamacpp/index.cjs

@@ -21,3 +21,4 @@ Object.defineProperty(exports, "LlamaCppError", { enumerable: true, get: functio
 __exportStar(require("./LlamaCppTextEmbeddingModel.cjs"), exports);
 __exportStar(require("./LlamaCppTextGenerationModel.cjs"), exports);
 __exportStar(require("./LlamaCppTokenizer.cjs"), exports);
+__exportStar(require("./mapInstructionPromptToBakLLaVA1ForLlamaCppFormat.cjs"), exports);

package/model-provider/llamacpp/index.d.ts

@@ -3,3 +3,4 @@ export { LlamaCppError, LlamaCppErrorData } from "./LlamaCppError.js";
 export * from "./LlamaCppTextEmbeddingModel.js";
 export * from "./LlamaCppTextGenerationModel.js";
 export * from "./LlamaCppTokenizer.js";
+export * from "./mapInstructionPromptToBakLLaVA1ForLlamaCppFormat.js";

package/model-provider/llamacpp/index.js

@@ -3,3 +3,4 @@ export { LlamaCppError } from "./LlamaCppError.js";
 export * from "./LlamaCppTextEmbeddingModel.js";
 export * from "./LlamaCppTextGenerationModel.js";
 export * from "./LlamaCppTokenizer.js";
+export * from "./mapInstructionPromptToBakLLaVA1ForLlamaCppFormat.js";

package/model-provider/llamacpp/mapInstructionPromptToBakLLaVA1ForLlamaCppFormat.cjs

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mapInstructionPromptToBakLLaVA1ForLlamaCppFormat = void 0;
+// default Vicuna 1 system message
+const DEFAULT_SYSTEM_MESSAGE = "A chat between a curious user and an artificial intelligence assistant. " +
+    "The assistant gives helpful, detailed, and polite answers to the user's questions.";
+/**
+ * BakLLaVA 1 uses a Vicuna 1 prompt. This mapping combines it with the LlamaCpp prompts.
+ *
+ * @see https://github.com/SkunkworksAI/BakLLaVA
+ */
+function mapInstructionPromptToBakLLaVA1ForLlamaCppFormat() {
+    return {
+        format: (instruction) => {
+            let text = "";
+            text += `${instruction.system ?? DEFAULT_SYSTEM_MESSAGE}\n\n`;
+            text += `USER: `;
+            if (instruction.image != null) {
+                text += `[img-1]\n`;
+            }
+            text += `${instruction.instruction}\n`;
+            if (instruction.input != null) {
+                text += `${instruction.input}\n`;
+            }
+            text += `ASSISTANT: `;
+            return {
+                text,
+                images: instruction.image != null
+                    ? { "1": instruction.image.base64Content }
+                    : undefined,
+            };
+        },
+        stopSequences: [`\nUSER:`],
+    };
+}
+exports.mapInstructionPromptToBakLLaVA1ForLlamaCppFormat = mapInstructionPromptToBakLLaVA1ForLlamaCppFormat;

package/model-provider/llamacpp/mapInstructionPromptToBakLLaVA1ForLlamaCppFormat.d.ts

@@ -0,0 +1,9 @@
+import { InstructionPrompt } from "../../model-function/generate-text/prompt-format/InstructionPrompt.js";
+import { TextGenerationPromptFormat } from "../../model-function/generate-text/TextGenerationPromptFormat.js";
+import { LlamaCppTextGenerationPrompt } from "./LlamaCppTextGenerationModel.js";
+/**
+ * BakLLaVA 1 uses a Vicuna 1 prompt. This mapping combines it with the LlamaCpp prompts.
+ *
+ * @see https://github.com/SkunkworksAI/BakLLaVA
+ */
+export declare function mapInstructionPromptToBakLLaVA1ForLlamaCppFormat(): TextGenerationPromptFormat<InstructionPrompt, LlamaCppTextGenerationPrompt>;

package/model-provider/llamacpp/mapInstructionPromptToBakLLaVA1ForLlamaCppFormat.js

@@ -0,0 +1,32 @@
+// default Vicuna 1 system message
+const DEFAULT_SYSTEM_MESSAGE = "A chat between a curious user and an artificial intelligence assistant. " +
+    "The assistant gives helpful, detailed, and polite answers to the user's questions.";
+/**
+ * BakLLaVA 1 uses a Vicuna 1 prompt. This mapping combines it with the LlamaCpp prompts.
+ *
+ * @see https://github.com/SkunkworksAI/BakLLaVA
+ */
+export function mapInstructionPromptToBakLLaVA1ForLlamaCppFormat() {
+    return {
+        format: (instruction) => {
+            let text = "";
+            text += `${instruction.system ?? DEFAULT_SYSTEM_MESSAGE}\n\n`;
+            text += `USER: `;
+            if (instruction.image != null) {
+                text += `[img-1]\n`;
+            }
+            text += `${instruction.instruction}\n`;
+            if (instruction.input != null) {
+                text += `${instruction.input}\n`;
+            }
+            text += `ASSISTANT: `;
+            return {
+                text,
+                images: instruction.image != null
+                    ? { "1": instruction.image.base64Content }
+                    : undefined,
+            };
+        },
+        stopSequences: [`\nUSER:`],
+    };
+}

package/model-provider/openai/chat/OpenAIChatMessage.cjs

@@ -5,8 +5,21 @@ exports.OpenAIChatMessage = {
     system(content) {
         return { role: "system", content };
     },
-    user(content) {
-        return { role: "user", content };
+    user(content, options) {
+        if (options?.image != null) {
+            return {
+                role: "user",
+                content: [
+                    { type: "text", text: content },
+                    {
+                        type: "image_url",
+                        image_url: `data:${options.image.mimeType ?? "image/jpeg"};base64,${options.image.base64Content}`,
+                    },
+                ],
+                name: options.name,
+            };
+        }
+        return { role: "user", content, name: options?.name };
     },
     assistant(content) {
         return { role: "assistant", content };

package/model-provider/openai/chat/OpenAIChatMessage.d.ts

@@ -39,7 +39,13 @@ export type OpenAIChatMessage = {
 };
 export declare const OpenAIChatMessage: {
     system(content: string): OpenAIChatMessage;
-    user(content: string): OpenAIChatMessage;
+    user(content: string, options?: {
+        name?: string;
+        image?: {
+            base64Content: string;
+            mimeType?: string;
+        };
+    }): OpenAIChatMessage;
     assistant(content: string): OpenAIChatMessage;
     functionCall(content: string | null, functionCall: {
         name: string;

package/model-provider/openai/chat/OpenAIChatMessage.js

@@ -2,8 +2,21 @@ export const OpenAIChatMessage = {
     system(content) {
         return { role: "system", content };
     },
-    user(content) {
-        return { role: "user", content };
+    user(content, options) {
+        if (options?.image != null) {
+            return {
+                role: "user",
+                content: [
+                    { type: "text", text: content },
+                    {
+                        type: "image_url",
+                        image_url: `data:${options.image.mimeType ?? "image/jpeg"};base64,${options.image.base64Content}`,
+                    },
+                ],
+                name: options.name,
+            };
+        }
+        return { role: "user", content, name: options?.name };
     },
     assistant(content) {
         return { role: "assistant", content };

package/model-provider/openai/chat/OpenAIChatModel.cjs

@@ -364,9 +364,6 @@ class OpenAIChatModel extends AbstractModel_js_1.AbstractModel {
     withInstructionPrompt() {
         return this.withPromptFormat((0, OpenAIChatPromptFormat_js_1.mapInstructionPromptToOpenAIChatFormat)());
     }
-    withVisionInstructionPrompt() {
-        return this.withPromptFormat((0, OpenAIChatPromptFormat_js_1.mapVisionInstructionPromptToOpenAIChatFormat)());
-    }
     /**
      * Returns this model with a chat prompt format.
      */
@@ -391,10 +388,6 @@ class OpenAIChatModel extends AbstractModel_js_1.AbstractModel {
 exports.OpenAIChatModel = OpenAIChatModel;
 const openAIChatResponseSchema = zod_1.z.object({
     id: zod_1.z.string(),
-    object: zod_1.z.literal("chat.completion"),
-    created: zod_1.z.number(),
-    model: zod_1.z.string(),
-    system_fingerprint: zod_1.z.string(),
     choices: zod_1.z.array(zod_1.z.object({
         message: zod_1.z.object({
             role: zod_1.z.literal("assistant"),
@@ -416,6 +409,10 @@ const openAIChatResponseSchema = zod_1.z.object({
             "function_call",
         ]),
     })),
+    created: zod_1.z.number(),
+    model: zod_1.z.string(),
+    system_fingerprint: zod_1.z.string().optional(),
+    object: zod_1.z.literal("chat.completion"),
     usage: zod_1.z.object({
         prompt_tokens: zod_1.z.number(),
         completion_tokens: zod_1.z.number(),

package/model-provider/openai/chat/OpenAIChatModel.d.ts

@@ -198,7 +198,7 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                 logprobs?: any;
             }[];
             created: number;
-            system_fingerprint: string;
+            system_fingerprint?: string | undefined;
         };
         text: string;
         usage: {
@@ -239,7 +239,7 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                 logprobs?: any;
             }[];
             created: number;
-            system_fingerprint: string;
+            system_fingerprint?: string | undefined;
         };
         valueText: string;
         value: any;
@@ -274,7 +274,7 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                 logprobs?: any;
             }[];
             created: number;
-            system_fingerprint: string;
+            system_fingerprint?: string | undefined;
         };
         structureAndText: {
             structure: null;
@@ -311,7 +311,7 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                 logprobs?: any;
             }[];
             created: number;
-            system_fingerprint: string;
+            system_fingerprint?: string | undefined;
         };
         structureAndText: {
             structure: string;
@@ -334,7 +334,6 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
      * Returns this model with an instruction prompt format.
      */
     withInstructionPrompt(): PromptFormatTextStreamingModel<import("../../../index.js").InstructionPrompt, OpenAIChatMessage[], OpenAIChatSettings, this>;
-    withVisionInstructionPrompt(): PromptFormatTextStreamingModel<import("../../../index.js").VisionInstructionPrompt, OpenAIChatMessage[], OpenAIChatSettings, this>;
     /**
      * Returns this model with a chat prompt format.
      */
@@ -344,10 +343,6 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
 }
 declare const openAIChatResponseSchema: z.ZodObject<{
     id: z.ZodString;
-    object: z.ZodLiteral<"chat.completion">;
-    created: z.ZodNumber;
-    model: z.ZodString;
-    system_fingerprint: z.ZodString;
     choices: z.ZodArray<z.ZodObject<{
         message: z.ZodObject<{
             role: z.ZodLiteral<"assistant">;
@@ -405,6 +400,10 @@ declare const openAIChatResponseSchema: z.ZodObject<{
         index: number;
         logprobs?: any;
     }>, "many">;
+    created: z.ZodNumber;
+    model: z.ZodString;
+    system_fingerprint: z.ZodOptional<z.ZodString>;
+    object: z.ZodLiteral<"chat.completion">;
     usage: z.ZodObject<{
         prompt_tokens: z.ZodNumber;
         completion_tokens: z.ZodNumber;
@@ -441,7 +440,7 @@ declare const openAIChatResponseSchema: z.ZodObject<{
         logprobs?: any;
     }[];
     created: number;
-    system_fingerprint: string;
+    system_fingerprint?: string | undefined;
 }, {
     object: "chat.completion";
     usage: {
@@ -465,7 +464,7 @@ declare const openAIChatResponseSchema: z.ZodObject<{
         logprobs?: any;
     }[];
     created: number;
-    system_fingerprint: string;
+    system_fingerprint?: string | undefined;
 }>;
 export type OpenAIChatResponse = z.infer<typeof openAIChatResponseSchema>;
 export type OpenAIChatResponseFormatType<T> = {
@@ -501,7 +500,7 @@ export declare const OpenAIChatResponseFormat: {
                 logprobs?: any;
             }[];
             created: number;
-            system_fingerprint: string;
+            system_fingerprint?: string | undefined;
         }>;
     };
     /**

package/model-provider/openai/chat/OpenAIChatModel.js

@@ -9,7 +9,7 @@ import { PromptFormatTextStreamingModel } from "../../../model-function/generate
 import { OpenAIApiConfiguration } from "../OpenAIApiConfiguration.js";
 import { failedOpenAICallResponseHandler } from "../OpenAIError.js";
 import { TikTokenTokenizer } from "../TikTokenTokenizer.js";
-import { mapChatPromptToOpenAIChatFormat, mapInstructionPromptToOpenAIChatFormat, mapVisionInstructionPromptToOpenAIChatFormat, } from "./OpenAIChatPromptFormat.js";
+import { mapChatPromptToOpenAIChatFormat, mapInstructionPromptToOpenAIChatFormat, } from "./OpenAIChatPromptFormat.js";
 import { createOpenAIChatDeltaIterableQueue } from "./OpenAIChatStreamIterable.js";
 import { countOpenAIChatPromptTokens } from "./countOpenAIChatMessageTokens.js";
 /*
@@ -355,9 +355,6 @@ export class OpenAIChatModel extends AbstractModel {
     withInstructionPrompt() {
         return this.withPromptFormat(mapInstructionPromptToOpenAIChatFormat());
     }
-    withVisionInstructionPrompt() {
-        return this.withPromptFormat(mapVisionInstructionPromptToOpenAIChatFormat());
-    }
     /**
      * Returns this model with a chat prompt format.
      */
@@ -381,10 +378,6 @@ export class OpenAIChatModel extends AbstractModel {
 }
 const openAIChatResponseSchema = z.object({
     id: z.string(),
-    object: z.literal("chat.completion"),
-    created: z.number(),
-    model: z.string(),
-    system_fingerprint: z.string(),
     choices: z.array(z.object({
         message: z.object({
             role: z.literal("assistant"),
@@ -406,6 +399,10 @@ const openAIChatResponseSchema = z.object({
             "function_call",
         ]),
     })),
+    created: z.number(),
+    model: z.string(),
+    system_fingerprint: z.string().optional(),
+    object: z.literal("chat.completion"),
     usage: z.object({
         prompt_tokens: z.number(),
         completion_tokens: z.number(),

package/model-provider/openai/chat/OpenAIChatPromptFormat.cjs

@@ -1,7 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.mapChatPromptToOpenAIChatFormat = exports.mapVisionInstructionPromptToOpenAIChatFormat = exports.mapInstructionPromptToOpenAIChatFormat = void 0;
+exports.mapChatPromptToOpenAIChatFormat = exports.mapInstructionPromptToOpenAIChatFormat = void 0;
 const validateChatPrompt_js_1 = require("../../../model-function/generate-text/prompt-format/validateChatPrompt.cjs");
+const OpenAIChatMessage_js_1 = require("./OpenAIChatMessage.cjs");
 /**
  * Formats an instruction prompt as an OpenAI chat prompt.
  */
@@ -10,20 +11,13 @@ function mapInstructionPromptToOpenAIChatFormat() {
         format: (instruction) => {
             const messages = [];
             if (instruction.system != null) {
-                messages.push({
-                    role: "system",
-                    content: instruction.system,
-                });
+                messages.push(OpenAIChatMessage_js_1.OpenAIChatMessage.system(instruction.system));
             }
-            messages.push({
-                role: "user",
-                content: instruction.instruction,
-            });
+            messages.push(OpenAIChatMessage_js_1.OpenAIChatMessage.user(instruction.instruction, {
+                image: instruction.image,
+            }));
             if (instruction.input != null) {
-                messages.push({
-                    role: "user",
-                    content: instruction.input,
-                });
+                messages.push(OpenAIChatMessage_js_1.OpenAIChatMessage.user(instruction.input));
             }
             return messages;
         },
@@ -31,29 +25,6 @@ function mapInstructionPromptToOpenAIChatFormat() {
     };
 }
 exports.mapInstructionPromptToOpenAIChatFormat = mapInstructionPromptToOpenAIChatFormat;
-/**
- * Formats a version prompt as an OpenAI chat prompt.
- */
-function mapVisionInstructionPromptToOpenAIChatFormat() {
-    return {
-        format: ({ instruction, image, mimeType }) => {
-            return [
-                {
-                    role: "user",
-                    content: [
-                        { type: "text", text: instruction },
-                        {
-                            type: "image_url",
-                            image_url: `data:${mimeType ?? "image/jpeg"};base64,${image}`,
-                        },
-                    ],
-                },
-            ];
-        },
-        stopSequences: [],
-    };
-}
-exports.mapVisionInstructionPromptToOpenAIChatFormat = mapVisionInstructionPromptToOpenAIChatFormat;
 /**
  * Formats a chat prompt as an OpenAI chat prompt.
  */

package/model-provider/openai/chat/OpenAIChatPromptFormat.d.ts

@@ -1,16 +1,11 @@
+import { TextGenerationPromptFormat } from "../../../model-function/generate-text/TextGenerationPromptFormat.js";
 import { ChatPrompt } from "../../../model-function/generate-text/prompt-format/ChatPrompt.js";
 import { InstructionPrompt } from "../../../model-function/generate-text/prompt-format/InstructionPrompt.js";
-import { TextGenerationPromptFormat } from "../../../model-function/generate-text/TextGenerationPromptFormat.js";
 import { OpenAIChatMessage } from "./OpenAIChatMessage.js";
-import { VisionInstructionPrompt } from "../../../model-function/generate-text/prompt-format/VisionInstructionPrompt.js";
 /**
  * Formats an instruction prompt as an OpenAI chat prompt.
  */
 export declare function mapInstructionPromptToOpenAIChatFormat(): TextGenerationPromptFormat<InstructionPrompt, Array<OpenAIChatMessage>>;
-/**
- * Formats a version prompt as an OpenAI chat prompt.
- */
-export declare function mapVisionInstructionPromptToOpenAIChatFormat(): TextGenerationPromptFormat<VisionInstructionPrompt, Array<OpenAIChatMessage>>;
 /**
  * Formats a chat prompt as an OpenAI chat prompt.
  */

package/model-provider/openai/chat/OpenAIChatPromptFormat.js

@@ -1,4 +1,5 @@
 import { validateChatPrompt } from "../../../model-function/generate-text/prompt-format/validateChatPrompt.js";
+import { OpenAIChatMessage } from "./OpenAIChatMessage.js";
 /**
  * Formats an instruction prompt as an OpenAI chat prompt.
  */
@@ -7,48 +8,19 @@ export function mapInstructionPromptToOpenAIChatFormat() {
         format: (instruction) => {
             const messages = [];
             if (instruction.system != null) {
-                messages.push({
-                    role: "system",
-                    content: instruction.system,
-                });
+                messages.push(OpenAIChatMessage.system(instruction.system));
             }
-            messages.push({
-                role: "user",
-                content: instruction.instruction,
-            });
+            messages.push(OpenAIChatMessage.user(instruction.instruction, {
+                image: instruction.image,
+            }));
             if (instruction.input != null) {
-                messages.push({
-                    role: "user",
-                    content: instruction.input,
-                });
+                messages.push(OpenAIChatMessage.user(instruction.input));
             }
             return messages;
         },
         stopSequences: [],
     };
 }
-/**
- * Formats a version prompt as an OpenAI chat prompt.
- */
-export function mapVisionInstructionPromptToOpenAIChatFormat() {
-    return {
-        format: ({ instruction, image, mimeType }) => {
-            return [
-                {
-                    role: "user",
-                    content: [
-                        { type: "text", text: instruction },
-                        {
-                            type: "image_url",
-                            image_url: `data:${mimeType ?? "image/jpeg"};base64,${image}`,
-                        },
-                    ],
-                },
-            ];
-        },
-        stopSequences: [],
-    };
-}
 /**
  * Formats a chat prompt as an OpenAI chat prompt.
  */

package/model-provider/openai/chat/OpenAIChatStreamIterable.cjs

@@ -6,6 +6,7 @@ const AsyncQueue_js_1 = require("../../../util/AsyncQueue.cjs");
 const parseEventSourceStream_js_1 = require("../../../util/streaming/parseEventSourceStream.cjs");
 const parseJSON_js_1 = require("../../../util/parseJSON.cjs");
 const chatResponseStreamEventSchema = zod_1.z.object({
+    id: zod_1.z.string(),
     choices: zod_1.z.array(zod_1.z.object({
         delta: zod_1.z.object({
             role: zod_1.z.enum(["assistant", "user"]).optional(),
@@ -30,9 +31,9 @@ const chatResponseStreamEventSchema = zod_1.z.object({
         index: zod_1.z.number(),
     })),
     created: zod_1.z.number(),
-    id: zod_1.z.string(),
     model: zod_1.z.string(),
-    object: zod_1.z.string(),
+    system_fingerprint: zod_1.z.string().optional(),
+    object: zod_1.z.literal("chat.completion.chunk"),
 });
 async function createOpenAIChatDeltaIterableQueue(stream, extractDeltaValue) {
     const queue = new AsyncQueue_js_1.AsyncQueue();

package/model-provider/openai/chat/OpenAIChatStreamIterable.js

@@ -3,6 +3,7 @@ import { AsyncQueue } from "../../../util/AsyncQueue.js";
 import { parseEventSourceStream } from "../../../util/streaming/parseEventSourceStream.js";
 import { safeParseJsonWithZod } from "../../../util/parseJSON.js";
 const chatResponseStreamEventSchema = z.object({
+    id: z.string(),
     choices: z.array(z.object({
         delta: z.object({
             role: z.enum(["assistant", "user"]).optional(),
@@ -27,9 +28,9 @@ const chatResponseStreamEventSchema = z.object({
         index: z.number(),
     })),
     created: z.number(),
-    id: z.string(),
     model: z.string(),
-    object: z.string(),
+    system_fingerprint: z.string().optional(),
+    object: z.literal("chat.completion.chunk"),
 });
 export async function createOpenAIChatDeltaIterableQueue(stream, extractDeltaValue) {
     const queue = new AsyncQueue();

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "Build multimodal applications, chatbots, and agents with JavaScript and TypeScript.",
-  "version": "0.64.0",
+  "version": "0.65.1",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [

package/model-function/generate-text/prompt-format/VisionInstructionPrompt.cjs

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/model-function/generate-text/prompt-format/VisionInstructionPrompt.d.ts

@@ -1,31 +0,0 @@
-/**
- * A single instruction version prompt. It contains an instruction, a base64 encoded image
- * and an optional mime type of the image.
- *
- * If no mime type is provided, the mime type default to "image/jpeg".
- *
- * @example
- * ```ts
- * {
- *   instruction: "Describe the image in detail:",
- *   image: fs.readFileSync(path.join("data", "example-image.png"), {
- *     encoding: "base64",
- *   }),
- *   mimeType: "image/png"
- * }
- * ```
- */
-export type VisionInstructionPrompt = {
-    /**
-     * The instruction for the model.
-     */
-    instruction: string;
-    /**
-     * Base-64 encoded image.
-     */
-    image: string;
-    /**
-     * Optional mime type of the image.
-     */
-    mimeType?: string;
-};

package/model-function/generate-text/prompt-format/VisionInstructionPrompt.js

@@ -1 +0,0 @@
-export {};

package/model-provider/llamacpp/mapVisionInstructionPromptToLlamaCppFormat.cjs

@@ -1,15 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.mapVisionInstructionPromptToLlamaCppFormat = void 0;
-function mapVisionInstructionPromptToLlamaCppFormat() {
-    return {
-        format: ({ instruction, image }) => {
-            return {
-                text: `[img-1]\n\n${instruction}`,
-                images: { "1": image },
-            };
-        },
-        stopSequences: [],
-    };
-}
-exports.mapVisionInstructionPromptToLlamaCppFormat = mapVisionInstructionPromptToLlamaCppFormat;

package/model-provider/llamacpp/mapVisionInstructionPromptToLlamaCppFormat.d.ts

@@ -1,4 +0,0 @@
-import { TextGenerationPromptFormat } from "../../model-function/generate-text/TextGenerationPromptFormat.js";
-import { VisionInstructionPrompt } from "../../model-function/generate-text/prompt-format/VisionInstructionPrompt.js";
-import { LlamaCppTextGenerationPrompt } from "./LlamaCppTextGenerationModel.js";
-export declare function mapVisionInstructionPromptToLlamaCppFormat(): TextGenerationPromptFormat<VisionInstructionPrompt, LlamaCppTextGenerationPrompt>;

package/model-provider/llamacpp/mapVisionInstructionPromptToLlamaCppFormat.js

@@ -1,11 +0,0 @@
-export function mapVisionInstructionPromptToLlamaCppFormat() {
-    return {
-        format: ({ instruction, image }) => {
-            return {
-                text: `[img-1]\n\n${instruction}`,
-                images: { "1": image },
-            };
-        },
-        stopSequences: [],
-    };
-}