modelfusion 0.63.0 → 0.64.0

package/model-function/generate-text/index.cjs

@@ -14,18 +14,19 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./AlpacaPromptFormat.cjs"), exports);
-__exportStar(require("./ChatPrompt.cjs"), exports);
-__exportStar(require("./InstructionPrompt.cjs"), exports);
-__exportStar(require("./Llama2PromptFormat.cjs"), exports);
 __exportStar(require("./PromptFormatTextGenerationModel.cjs"), exports);
 __exportStar(require("./PromptFormatTextStreamingModel.cjs"), exports);
 __exportStar(require("./TextGenerationEvent.cjs"), exports);
 __exportStar(require("./TextGenerationModel.cjs"), exports);
 __exportStar(require("./TextGenerationPromptFormat.cjs"), exports);
-__exportStar(require("./TextPromptFormat.cjs"), exports);
-__exportStar(require("./VicunaPromptFormat.cjs"), exports);
 __exportStar(require("./generateText.cjs"), exports);
+__exportStar(require("./prompt-format/AlpacaPromptFormat.cjs"), exports);
+__exportStar(require("./prompt-format/ChatPrompt.cjs"), exports);
+__exportStar(require("./prompt-format/InstructionPrompt.cjs"), exports);
+__exportStar(require("./prompt-format/Llama2PromptFormat.cjs"), exports);
+__exportStar(require("./prompt-format/TextPromptFormat.cjs"), exports);
+__exportStar(require("./prompt-format/VicunaPromptFormat.cjs"), exports);
+__exportStar(require("./prompt-format/VisionInstructionPrompt.cjs"), exports);
+__exportStar(require("./prompt-format/trimChatPrompt.cjs"), exports);
+__exportStar(require("./prompt-format/validateChatPrompt.cjs"), exports);
 __exportStar(require("./streamText.cjs"), exports);
-__exportStar(require("./trimChatPrompt.cjs"), exports);
-__exportStar(require("./validateChatPrompt.cjs"), exports);

package/model-function/generate-text/index.d.ts

@@ -1,15 +1,16 @@
-export * from "./AlpacaPromptFormat.js";
-export * from "./ChatPrompt.js";
-export * from "./InstructionPrompt.js";
-export * from "./Llama2PromptFormat.js";
 export * from "./PromptFormatTextGenerationModel.js";
 export * from "./PromptFormatTextStreamingModel.js";
 export * from "./TextGenerationEvent.js";
 export * from "./TextGenerationModel.js";
 export * from "./TextGenerationPromptFormat.js";
-export * from "./TextPromptFormat.js";
-export * from "./VicunaPromptFormat.js";
 export * from "./generateText.js";
+export * from "./prompt-format/AlpacaPromptFormat.js";
+export * from "./prompt-format/ChatPrompt.js";
+export * from "./prompt-format/InstructionPrompt.js";
+export * from "./prompt-format/Llama2PromptFormat.js";
+export * from "./prompt-format/TextPromptFormat.js";
+export * from "./prompt-format/VicunaPromptFormat.js";
+export * from "./prompt-format/VisionInstructionPrompt.js";
+export * from "./prompt-format/trimChatPrompt.js";
+export * from "./prompt-format/validateChatPrompt.js";
 export * from "./streamText.js";
-export * from "./trimChatPrompt.js";
-export * from "./validateChatPrompt.js";

package/model-function/generate-text/index.js

@@ -1,15 +1,16 @@
-export * from "./AlpacaPromptFormat.js";
-export * from "./ChatPrompt.js";
-export * from "./InstructionPrompt.js";
-export * from "./Llama2PromptFormat.js";
 export * from "./PromptFormatTextGenerationModel.js";
 export * from "./PromptFormatTextStreamingModel.js";
 export * from "./TextGenerationEvent.js";
 export * from "./TextGenerationModel.js";
 export * from "./TextGenerationPromptFormat.js";
-export * from "./TextPromptFormat.js";
-export * from "./VicunaPromptFormat.js";
 export * from "./generateText.js";
+export * from "./prompt-format/AlpacaPromptFormat.js";
+export * from "./prompt-format/ChatPrompt.js";
+export * from "./prompt-format/InstructionPrompt.js";
+export * from "./prompt-format/Llama2PromptFormat.js";
+export * from "./prompt-format/TextPromptFormat.js";
+export * from "./prompt-format/VicunaPromptFormat.js";
+export * from "./prompt-format/VisionInstructionPrompt.js";
+export * from "./prompt-format/trimChatPrompt.js";
+export * from "./prompt-format/validateChatPrompt.js";
 export * from "./streamText.js";
-export * from "./trimChatPrompt.js";
-export * from "./validateChatPrompt.js";

package/model-function/generate-text/{AlpacaPromptFormat.d.ts → prompt-format/AlpacaPromptFormat.d.ts}

@@ -1,5 +1,5 @@
 import { InstructionPrompt } from "./InstructionPrompt.js";
-import { TextGenerationPromptFormat } from "./TextGenerationPromptFormat.js";
+import { TextGenerationPromptFormat } from "../TextGenerationPromptFormat.js";
 /**
  * Formats an instruction prompt as an Alpaca prompt.
  *

package/model-function/generate-text/{Llama2PromptFormat.d.ts → prompt-format/Llama2PromptFormat.d.ts}

@@ -1,6 +1,6 @@
 import { ChatPrompt } from "./ChatPrompt.js";
 import { InstructionPrompt } from "./InstructionPrompt.js";
-import { TextGenerationPromptFormat } from "./TextGenerationPromptFormat.js";
+import { TextGenerationPromptFormat } from "../TextGenerationPromptFormat.js";
 /**
  * Formats an instruction prompt as a Llama 2 prompt.
  *

package/model-function/generate-text/{TextPromptFormat.d.ts → prompt-format/TextPromptFormat.d.ts}

@@ -1,6 +1,6 @@
 import { ChatPrompt } from "./ChatPrompt.js";
 import { InstructionPrompt } from "./InstructionPrompt.js";
-import { TextGenerationPromptFormat } from "./TextGenerationPromptFormat.js";
+import { TextGenerationPromptFormat } from "../TextGenerationPromptFormat.js";
 /**
  * Formats an instruction prompt as a basic text prompt.
  */

package/model-function/generate-text/{VicunaPromptFormat.d.ts → prompt-format/VicunaPromptFormat.d.ts}

@@ -1,5 +1,5 @@
 import { ChatPrompt } from "./ChatPrompt.js";
-import { TextGenerationPromptFormat } from "./TextGenerationPromptFormat.js";
+import { TextGenerationPromptFormat } from "../TextGenerationPromptFormat.js";
 /**
  * Formats a chat prompt as a Vicuna prompt.
  *

package/model-function/generate-text/prompt-format/VisionInstructionPrompt.cjs

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/model-function/generate-text/prompt-format/VisionInstructionPrompt.d.ts

@@ -0,0 +1,31 @@
+/**
+ * A single instruction version prompt. It contains an instruction, a base64 encoded image
+ * and an optional mime type of the image.
+ *
+ * If no mime type is provided, the mime type default to "image/jpeg".
+ *
+ * @example
+ * ```ts
+ * {
+ *   instruction: "Describe the image in detail:",
+ *   image: fs.readFileSync(path.join("data", "example-image.png"), {
+ *     encoding: "base64",
+ *   }),
+ *   mimeType: "image/png"
+ * }
+ * ```
+ */
+export type VisionInstructionPrompt = {
+    /**
+     * The instruction for the model.
+     */
+    instruction: string;
+    /**
+     * Base-64 encoded image.
+     */
+    image: string;
+    /**
+     * Optional mime type of the image.
+     */
+    mimeType?: string;
+};

package/model-function/generate-text/prompt-format/VisionInstructionPrompt.js

@@ -0,0 +1 @@
+export {};

package/model-function/generate-text/{trimChatPrompt.d.ts → prompt-format/trimChatPrompt.d.ts}

@@ -1,5 +1,5 @@
 import { ChatPrompt } from "./ChatPrompt.js";
-import { HasContextWindowSize, HasTokenizer, TextGenerationModel, TextGenerationModelSettings } from "./TextGenerationModel.js";
+import { HasContextWindowSize, HasTokenizer, TextGenerationModel, TextGenerationModelSettings } from "../TextGenerationModel.js";
 /**
  * Keeps only the most recent messages in the prompt, while leaving enough space for the completion.
  *

package/model-provider/anthropic/AnthropicPromptFormat.cjs

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.mapChatPromptToAnthropicFormat = exports.mapInstructionPromptToAnthropicFormat = void 0;
-const validateChatPrompt_js_1 = require("../../model-function/generate-text/validateChatPrompt.cjs");
+const validateChatPrompt_js_1 = require("../../model-function/generate-text/prompt-format/validateChatPrompt.cjs");
 /**
  * Formats an instruction prompt as an Anthropic prompt.
  */

package/model-provider/anthropic/AnthropicPromptFormat.d.ts

@@ -1,5 +1,5 @@
-import { ChatPrompt } from "../../model-function/generate-text/ChatPrompt.js";
-import { InstructionPrompt } from "../../model-function/generate-text/InstructionPrompt.js";
+import { ChatPrompt } from "../../model-function/generate-text/prompt-format/ChatPrompt.js";
+import { InstructionPrompt } from "../../model-function/generate-text/prompt-format/InstructionPrompt.js";
 import { TextGenerationPromptFormat } from "../../model-function/generate-text/TextGenerationPromptFormat.js";
 /**
  * Formats an instruction prompt as an Anthropic prompt.

package/model-provider/anthropic/AnthropicPromptFormat.js

@@ -1,4 +1,4 @@
-import { validateChatPrompt } from "../../model-function/generate-text/validateChatPrompt.js";
+import { validateChatPrompt } from "../../model-function/generate-text/prompt-format/validateChatPrompt.js";
 /**
  * Formats an instruction prompt as an Anthropic prompt.
  */

package/model-provider/cohere/CohereTextGenerationModel.cjs

@@ -6,7 +6,7 @@ const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndTh
 const postToApi_js_1 = require("../../core/api/postToApi.cjs");
 const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
 const PromptFormatTextStreamingModel_js_1 = require("../../model-function/generate-text/PromptFormatTextStreamingModel.cjs");
-const TextPromptFormat_js_1 = require("../../model-function/generate-text/TextPromptFormat.cjs");
+const TextPromptFormat_js_1 = require("../../model-function/generate-text/prompt-format/TextPromptFormat.cjs");
 const countTokens_js_1 = require("../../model-function/tokenize-text/countTokens.cjs");
 const AsyncQueue_js_1 = require("../../util/AsyncQueue.cjs");
 const parseJsonStream_js_1 = require("../../util/streaming/parseJsonStream.cjs");

package/model-provider/cohere/CohereTextGenerationModel.js

@@ -3,7 +3,7 @@ import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottl
 import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
 import { AbstractModel } from "../../model-function/AbstractModel.js";
 import { PromptFormatTextStreamingModel } from "../../model-function/generate-text/PromptFormatTextStreamingModel.js";
-import { mapChatPromptToTextFormat, mapInstructionPromptToTextFormat, } from "../../model-function/generate-text/TextPromptFormat.js";
+import { mapChatPromptToTextFormat, mapInstructionPromptToTextFormat, } from "../../model-function/generate-text/prompt-format/TextPromptFormat.js";
 import { countTokens } from "../../model-function/tokenize-text/countTokens.js";
 import { AsyncQueue } from "../../util/AsyncQueue.js";
 import { parseJsonStream } from "../../util/streaming/parseJsonStream.js";

package/model-provider/llamacpp/LlamaCppTextGenerationModel.cjs

@@ -12,6 +12,7 @@ const parseJSON_js_1 = require("../../util/parseJSON.cjs");
 const LlamaCppApiConfiguration_js_1 = require("./LlamaCppApiConfiguration.cjs");
 const LlamaCppError_js_1 = require("./LlamaCppError.cjs");
 const LlamaCppTokenizer_js_1 = require("./LlamaCppTokenizer.cjs");
+const mapVisionInstructionPromptToLlamaCppFormat_js_1 = require("./mapVisionInstructionPromptToLlamaCppFormat.cjs");
 class LlamaCppTextGenerationModel extends AbstractModel_js_1.AbstractModel {
     constructor(settings = {}) {
         super({ settings });
@@ -107,6 +108,9 @@ class LlamaCppTextGenerationModel extends AbstractModel_js_1.AbstractModel {
             stopSequences: [],
         });
     }
+    withVisionInstructionPrompt() {
+        return this.withPromptFormat((0, mapVisionInstructionPromptToLlamaCppFormat_js_1.mapVisionInstructionPromptToLlamaCppFormat)());
+    }
     withPromptFormat(promptFormat) {
         return new PromptFormatTextStreamingModel_js_1.PromptFormatTextStreamingModel({
             model: this.withSettings({

package/model-provider/llamacpp/LlamaCppTextGenerationModel.d.ts

@@ -111,6 +111,7 @@ export declare class LlamaCppTextGenerationModel<CONTEXT_WINDOW_SIZE extends num
     }>;
     doStreamText(prompt: LlamaCppTextGenerationPrompt, options?: FunctionOptions): Promise<AsyncIterable<Delta<string>>>;
     withTextPrompt(): PromptFormatTextStreamingModel<string, LlamaCppTextGenerationPrompt, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
+    withVisionInstructionPrompt(): PromptFormatTextStreamingModel<import("../../index.js").VisionInstructionPrompt, LlamaCppTextGenerationPrompt, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
     withPromptFormat<INPUT_PROMPT>(promptFormat: TextGenerationPromptFormat<INPUT_PROMPT, LlamaCppTextGenerationPrompt>): PromptFormatTextStreamingModel<INPUT_PROMPT, LlamaCppTextGenerationPrompt, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
     withSettings(additionalSettings: Partial<LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>>): this;
 }

package/model-provider/llamacpp/LlamaCppTextGenerationModel.js

@@ -9,6 +9,7 @@ import { parseJsonWithZod } from "../../util/parseJSON.js";
 import { LlamaCppApiConfiguration } from "./LlamaCppApiConfiguration.js";
 import { failedLlamaCppCallResponseHandler } from "./LlamaCppError.js";
 import { LlamaCppTokenizer } from "./LlamaCppTokenizer.js";
+import { mapVisionInstructionPromptToLlamaCppFormat } from "./mapVisionInstructionPromptToLlamaCppFormat.js";
 export class LlamaCppTextGenerationModel extends AbstractModel {
     constructor(settings = {}) {
         super({ settings });
@@ -104,6 +105,9 @@ export class LlamaCppTextGenerationModel extends AbstractModel {
             stopSequences: [],
         });
     }
+    withVisionInstructionPrompt() {
+        return this.withPromptFormat(mapVisionInstructionPromptToLlamaCppFormat());
+    }
     withPromptFormat(promptFormat) {
         return new PromptFormatTextStreamingModel({
             model: this.withSettings({

package/model-provider/llamacpp/mapVisionInstructionPromptToLlamaCppFormat.cjs

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mapVisionInstructionPromptToLlamaCppFormat = void 0;
+function mapVisionInstructionPromptToLlamaCppFormat() {
+    return {
+        format: ({ instruction, image }) => {
+            return {
+                text: `[img-1]\n\n${instruction}`,
+                images: { "1": image },
+            };
+        },
+        stopSequences: [],
+    };
+}
+exports.mapVisionInstructionPromptToLlamaCppFormat = mapVisionInstructionPromptToLlamaCppFormat;

package/model-provider/llamacpp/mapVisionInstructionPromptToLlamaCppFormat.d.ts

@@ -0,0 +1,4 @@
+import { TextGenerationPromptFormat } from "../../model-function/generate-text/TextGenerationPromptFormat.js";
+import { VisionInstructionPrompt } from "../../model-function/generate-text/prompt-format/VisionInstructionPrompt.js";
+import { LlamaCppTextGenerationPrompt } from "./LlamaCppTextGenerationModel.js";
+export declare function mapVisionInstructionPromptToLlamaCppFormat(): TextGenerationPromptFormat<VisionInstructionPrompt, LlamaCppTextGenerationPrompt>;

package/model-provider/llamacpp/mapVisionInstructionPromptToLlamaCppFormat.js

@@ -0,0 +1,11 @@
+export function mapVisionInstructionPromptToLlamaCppFormat() {
+    return {
+        format: ({ instruction, image }) => {
+            return {
+                text: `[img-1]\n\n${instruction}`,
+                images: { "1": image },
+            };
+        },
+        stopSequences: [],
+    };
+}

package/model-provider/openai/OpenAICompletionModel.cjs

@@ -6,7 +6,7 @@ const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndTh
 const postToApi_js_1 = require("../../core/api/postToApi.cjs");
 const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
 const PromptFormatTextStreamingModel_js_1 = require("../../model-function/generate-text/PromptFormatTextStreamingModel.cjs");
-const TextPromptFormat_js_1 = require("../../model-function/generate-text/TextPromptFormat.cjs");
+const TextPromptFormat_js_1 = require("../../model-function/generate-text/prompt-format/TextPromptFormat.cjs");
 const countTokens_js_1 = require("../../model-function/tokenize-text/countTokens.cjs");
 const AsyncQueue_js_1 = require("../../util/AsyncQueue.cjs");
 const parseJSON_js_1 = require("../../util/parseJSON.cjs");

package/model-provider/openai/OpenAICompletionModel.js

@@ -3,7 +3,7 @@ import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottl
 import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
 import { AbstractModel } from "../../model-function/AbstractModel.js";
 import { PromptFormatTextStreamingModel } from "../../model-function/generate-text/PromptFormatTextStreamingModel.js";
-import { mapChatPromptToTextFormat, mapInstructionPromptToTextFormat, } from "../../model-function/generate-text/TextPromptFormat.js";
+import { mapChatPromptToTextFormat, mapInstructionPromptToTextFormat, } from "../../model-function/generate-text/prompt-format/TextPromptFormat.js";
 import { countTokens } from "../../model-function/tokenize-text/countTokens.js";
 import { AsyncQueue } from "../../util/AsyncQueue.js";
 import { parseJsonWithZod } from "../../util/parseJSON.js";

package/model-provider/openai/chat/OpenAIChatMessage.d.ts

@@ -1,14 +1,37 @@
 export type OpenAIChatMessage = {
-    role: "user" | "assistant" | "system";
+    role: "system";
     content: string;
     name?: string;
+} | {
+    role: "user";
+    content: string | Array<{
+        type: "text";
+        text: string;
+    } | {
+        type: "image_url";
+        image_url: string;
+    }>;
+    name?: string;
 } | {
     role: "assistant";
     content: string | null;
-    function_call: {
+    name?: string;
+    tool_calls?: Array<{
+        id: string;
+        type: "function";
+        function: {
+            name: string;
+            arguments: string;
+        };
+    }>;
+    function_call?: {
         name: string;
         arguments: string;
     };
+} | {
+    role: "tool";
+    tool_call_id: string;
+    content: string | null;
 } | {
     role: "function";
     content: string;

package/model-provider/openai/chat/OpenAIChatModel.cjs

@@ -364,6 +364,9 @@ class OpenAIChatModel extends AbstractModel_js_1.AbstractModel {
     withInstructionPrompt() {
         return this.withPromptFormat((0, OpenAIChatPromptFormat_js_1.mapInstructionPromptToOpenAIChatFormat)());
     }
+    withVisionInstructionPrompt() {
+        return this.withPromptFormat((0, OpenAIChatPromptFormat_js_1.mapVisionInstructionPromptToOpenAIChatFormat)());
+    }
     /**
      * Returns this model with a chat prompt format.
      */

package/model-provider/openai/chat/OpenAIChatModel.d.ts

@@ -334,6 +334,7 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
      * Returns this model with an instruction prompt format.
      */
     withInstructionPrompt(): PromptFormatTextStreamingModel<import("../../../index.js").InstructionPrompt, OpenAIChatMessage[], OpenAIChatSettings, this>;
+    withVisionInstructionPrompt(): PromptFormatTextStreamingModel<import("../../../index.js").VisionInstructionPrompt, OpenAIChatMessage[], OpenAIChatSettings, this>;
     /**
      * Returns this model with a chat prompt format.
      */

package/model-provider/openai/chat/OpenAIChatModel.js

@@ -9,7 +9,7 @@ import { PromptFormatTextStreamingModel } from "../../../model-function/generate
 import { OpenAIApiConfiguration } from "../OpenAIApiConfiguration.js";
 import { failedOpenAICallResponseHandler } from "../OpenAIError.js";
 import { TikTokenTokenizer } from "../TikTokenTokenizer.js";
-import { mapChatPromptToOpenAIChatFormat, mapInstructionPromptToOpenAIChatFormat, } from "./OpenAIChatPromptFormat.js";
+import { mapChatPromptToOpenAIChatFormat, mapInstructionPromptToOpenAIChatFormat, mapVisionInstructionPromptToOpenAIChatFormat, } from "./OpenAIChatPromptFormat.js";
 import { createOpenAIChatDeltaIterableQueue } from "./OpenAIChatStreamIterable.js";
 import { countOpenAIChatPromptTokens } from "./countOpenAIChatMessageTokens.js";
 /*
@@ -355,6 +355,9 @@ export class OpenAIChatModel extends AbstractModel {
     withInstructionPrompt() {
         return this.withPromptFormat(mapInstructionPromptToOpenAIChatFormat());
     }
+    withVisionInstructionPrompt() {
+        return this.withPromptFormat(mapVisionInstructionPromptToOpenAIChatFormat());
+    }
     /**
      * Returns this model with a chat prompt format.
      */

package/model-provider/openai/chat/OpenAIChatPromptFormat.cjs

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.mapChatPromptToOpenAIChatFormat = exports.mapInstructionPromptToOpenAIChatFormat = void 0;
-const validateChatPrompt_js_1 = require("../../../model-function/generate-text/validateChatPrompt.cjs");
+exports.mapChatPromptToOpenAIChatFormat = exports.mapVisionInstructionPromptToOpenAIChatFormat = exports.mapInstructionPromptToOpenAIChatFormat = void 0;
+const validateChatPrompt_js_1 = require("../../../model-function/generate-text/prompt-format/validateChatPrompt.cjs");
 /**
  * Formats an instruction prompt as an OpenAI chat prompt.
  */
@@ -31,6 +31,29 @@ function mapInstructionPromptToOpenAIChatFormat() {
     };
 }
 exports.mapInstructionPromptToOpenAIChatFormat = mapInstructionPromptToOpenAIChatFormat;
+/**
+ * Formats a version prompt as an OpenAI chat prompt.
+ */
+function mapVisionInstructionPromptToOpenAIChatFormat() {
+    return {
+        format: ({ instruction, image, mimeType }) => {
+            return [
+                {
+                    role: "user",
+                    content: [
+                        { type: "text", text: instruction },
+                        {
+                            type: "image_url",
+                            image_url: `data:${mimeType ?? "image/jpeg"};base64,${image}`,
+                        },
+                    ],
+                },
+            ];
+        },
+        stopSequences: [],
+    };
+}
+exports.mapVisionInstructionPromptToOpenAIChatFormat = mapVisionInstructionPromptToOpenAIChatFormat;
 /**
  * Formats a chat prompt as an OpenAI chat prompt.
  */

package/model-provider/openai/chat/OpenAIChatPromptFormat.d.ts

@@ -1,11 +1,16 @@
-import { ChatPrompt } from "../../../model-function/generate-text/ChatPrompt.js";
-import { InstructionPrompt } from "../../../model-function/generate-text/InstructionPrompt.js";
+import { ChatPrompt } from "../../../model-function/generate-text/prompt-format/ChatPrompt.js";
+import { InstructionPrompt } from "../../../model-function/generate-text/prompt-format/InstructionPrompt.js";
 import { TextGenerationPromptFormat } from "../../../model-function/generate-text/TextGenerationPromptFormat.js";
 import { OpenAIChatMessage } from "./OpenAIChatMessage.js";
+import { VisionInstructionPrompt } from "../../../model-function/generate-text/prompt-format/VisionInstructionPrompt.js";
 /**
  * Formats an instruction prompt as an OpenAI chat prompt.
  */
 export declare function mapInstructionPromptToOpenAIChatFormat(): TextGenerationPromptFormat<InstructionPrompt, Array<OpenAIChatMessage>>;
+/**
+ * Formats a version prompt as an OpenAI chat prompt.
+ */
+export declare function mapVisionInstructionPromptToOpenAIChatFormat(): TextGenerationPromptFormat<VisionInstructionPrompt, Array<OpenAIChatMessage>>;
 /**
  * Formats a chat prompt as an OpenAI chat prompt.
  */

package/model-provider/openai/chat/OpenAIChatPromptFormat.js

@@ -1,4 +1,4 @@
-import { validateChatPrompt } from "../../../model-function/generate-text/validateChatPrompt.js";
+import { validateChatPrompt } from "../../../model-function/generate-text/prompt-format/validateChatPrompt.js";
 /**
  * Formats an instruction prompt as an OpenAI chat prompt.
  */
@@ -27,6 +27,28 @@ export function mapInstructionPromptToOpenAIChatFormat() {
         stopSequences: [],
     };
 }
+/**
+ * Formats a version prompt as an OpenAI chat prompt.
+ */
+export function mapVisionInstructionPromptToOpenAIChatFormat() {
+    return {
+        format: ({ instruction, image, mimeType }) => {
+            return [
+                {
+                    role: "user",
+                    content: [
+                        { type: "text", text: instruction },
+                        {
+                            type: "image_url",
+                            image_url: `data:${mimeType ?? "image/jpeg"};base64,${image}`,
+                        },
+                    ],
+                },
+            ];
+        },
+        stopSequences: [],
+    };
+}
 /**
  * Formats a chat prompt as an OpenAI chat prompt.
  */

package/model-provider/openai/chat/countOpenAIChatMessageTokens.cjs

@@ -15,10 +15,26 @@ exports.OPENAI_CHAT_PROMPT_BASE_TOKEN_COUNT = 2;
  */
 exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT = 5;
 async function countOpenAIChatMessageTokens({ message, model, }) {
-    const contentTokenCount = await (0, countTokens_js_1.countTokens)(new TikTokenTokenizer_js_1.TikTokenTokenizer({
+    const tokenizer = new TikTokenTokenizer_js_1.TikTokenTokenizer({
         model: (0, OpenAIChatModel_js_1.getOpenAIChatModelInformation)(model).baseModel,
-    }), message.content ?? "");
-    return exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT + contentTokenCount;
+    });
+    // case: function call without content
+    if (message.content == null) {
+        return exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT;
+    }
+    // case: simple text content
+    if (typeof message.content === "string") {
+        return (exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT +
+            (await (0, countTokens_js_1.countTokens)(tokenizer, message.content)));
+    }
+    // case: array of content objects
+    let contentTokenCount = exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT;
+    for (const content of message.content) {
+        if (content.type === "text") {
+            contentTokenCount += await (0, countTokens_js_1.countTokens)(tokenizer, content.text);
+        }
+    }
+    return contentTokenCount;
 }
 exports.countOpenAIChatMessageTokens = countOpenAIChatMessageTokens;
 async function countOpenAIChatPromptTokens({ messages, model, }) {

package/model-provider/openai/chat/countOpenAIChatMessageTokens.js

@@ -12,10 +12,26 @@ export const OPENAI_CHAT_PROMPT_BASE_TOKEN_COUNT = 2;
  */
 export const OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT = 5;
 export async function countOpenAIChatMessageTokens({ message, model, }) {
-    const contentTokenCount = await countTokens(new TikTokenTokenizer({
+    const tokenizer = new TikTokenTokenizer({
         model: getOpenAIChatModelInformation(model).baseModel,
-    }), message.content ?? "");
-    return OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT + contentTokenCount;
+    });
+    // case: function call without content
+    if (message.content == null) {
+        return OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT;
+    }
+    // case: simple text content
+    if (typeof message.content === "string") {
+        return (OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT +
+            (await countTokens(tokenizer, message.content)));
+    }
+    // case: array of content objects
+    let contentTokenCount = OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT;
+    for (const content of message.content) {
+        if (content.type === "text") {
+            contentTokenCount += await countTokens(tokenizer, content.text);
+        }
+    }
+    return contentTokenCount;
 }
 export async function countOpenAIChatPromptTokens({ messages, model, }) {
     let tokens = OPENAI_CHAT_PROMPT_BASE_TOKEN_COUNT;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "Build multimodal applications, chatbots, and agents with JavaScript and TypeScript.",
-  "version": "0.63.0",
+  "version": "0.64.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [