modelfusion 0.62.0 → 0.64.0

package/model-function/generate-text/index.cjs

@@ -14,18 +14,19 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./AlpacaPromptFormat.cjs"), exports);
-__exportStar(require("./ChatPrompt.cjs"), exports);
-__exportStar(require("./InstructionPrompt.cjs"), exports);
-__exportStar(require("./Llama2PromptFormat.cjs"), exports);
 __exportStar(require("./PromptFormatTextGenerationModel.cjs"), exports);
 __exportStar(require("./PromptFormatTextStreamingModel.cjs"), exports);
 __exportStar(require("./TextGenerationEvent.cjs"), exports);
 __exportStar(require("./TextGenerationModel.cjs"), exports);
 __exportStar(require("./TextGenerationPromptFormat.cjs"), exports);
-__exportStar(require("./TextPromptFormat.cjs"), exports);
-__exportStar(require("./VicunaPromptFormat.cjs"), exports);
 __exportStar(require("./generateText.cjs"), exports);
+__exportStar(require("./prompt-format/AlpacaPromptFormat.cjs"), exports);
+__exportStar(require("./prompt-format/ChatPrompt.cjs"), exports);
+__exportStar(require("./prompt-format/InstructionPrompt.cjs"), exports);
+__exportStar(require("./prompt-format/Llama2PromptFormat.cjs"), exports);
+__exportStar(require("./prompt-format/TextPromptFormat.cjs"), exports);
+__exportStar(require("./prompt-format/VicunaPromptFormat.cjs"), exports);
+__exportStar(require("./prompt-format/VisionInstructionPrompt.cjs"), exports);
+__exportStar(require("./prompt-format/trimChatPrompt.cjs"), exports);
+__exportStar(require("./prompt-format/validateChatPrompt.cjs"), exports);
 __exportStar(require("./streamText.cjs"), exports);
-__exportStar(require("./trimChatPrompt.cjs"), exports);
-__exportStar(require("./validateChatPrompt.cjs"), exports);

package/model-function/generate-text/index.d.ts

@@ -1,15 +1,16 @@
-export * from "./AlpacaPromptFormat.js";
-export * from "./ChatPrompt.js";
-export * from "./InstructionPrompt.js";
-export * from "./Llama2PromptFormat.js";
 export * from "./PromptFormatTextGenerationModel.js";
 export * from "./PromptFormatTextStreamingModel.js";
 export * from "./TextGenerationEvent.js";
 export * from "./TextGenerationModel.js";
 export * from "./TextGenerationPromptFormat.js";
-export * from "./TextPromptFormat.js";
-export * from "./VicunaPromptFormat.js";
 export * from "./generateText.js";
+export * from "./prompt-format/AlpacaPromptFormat.js";
+export * from "./prompt-format/ChatPrompt.js";
+export * from "./prompt-format/InstructionPrompt.js";
+export * from "./prompt-format/Llama2PromptFormat.js";
+export * from "./prompt-format/TextPromptFormat.js";
+export * from "./prompt-format/VicunaPromptFormat.js";
+export * from "./prompt-format/VisionInstructionPrompt.js";
+export * from "./prompt-format/trimChatPrompt.js";
+export * from "./prompt-format/validateChatPrompt.js";
 export * from "./streamText.js";
-export * from "./trimChatPrompt.js";
-export * from "./validateChatPrompt.js";

package/model-function/generate-text/index.js

@@ -1,15 +1,16 @@
-export * from "./AlpacaPromptFormat.js";
-export * from "./ChatPrompt.js";
-export * from "./InstructionPrompt.js";
-export * from "./Llama2PromptFormat.js";
 export * from "./PromptFormatTextGenerationModel.js";
 export * from "./PromptFormatTextStreamingModel.js";
 export * from "./TextGenerationEvent.js";
 export * from "./TextGenerationModel.js";
 export * from "./TextGenerationPromptFormat.js";
-export * from "./TextPromptFormat.js";
-export * from "./VicunaPromptFormat.js";
 export * from "./generateText.js";
+export * from "./prompt-format/AlpacaPromptFormat.js";
+export * from "./prompt-format/ChatPrompt.js";
+export * from "./prompt-format/InstructionPrompt.js";
+export * from "./prompt-format/Llama2PromptFormat.js";
+export * from "./prompt-format/TextPromptFormat.js";
+export * from "./prompt-format/VicunaPromptFormat.js";
+export * from "./prompt-format/VisionInstructionPrompt.js";
+export * from "./prompt-format/trimChatPrompt.js";
+export * from "./prompt-format/validateChatPrompt.js";
 export * from "./streamText.js";
-export * from "./trimChatPrompt.js";
-export * from "./validateChatPrompt.js";

package/model-function/generate-text/{AlpacaPromptFormat.d.ts → prompt-format/AlpacaPromptFormat.d.ts}

@@ -1,5 +1,5 @@
 import { InstructionPrompt } from "./InstructionPrompt.js";
-import { TextGenerationPromptFormat } from "./TextGenerationPromptFormat.js";
+import { TextGenerationPromptFormat } from "../TextGenerationPromptFormat.js";
 /**
  * Formats an instruction prompt as an Alpaca prompt.
  *

package/model-function/generate-text/{Llama2PromptFormat.d.ts → prompt-format/Llama2PromptFormat.d.ts}

@@ -1,6 +1,6 @@
 import { ChatPrompt } from "./ChatPrompt.js";
 import { InstructionPrompt } from "./InstructionPrompt.js";
-import { TextGenerationPromptFormat } from "./TextGenerationPromptFormat.js";
+import { TextGenerationPromptFormat } from "../TextGenerationPromptFormat.js";
 /**
  * Formats an instruction prompt as a Llama 2 prompt.
  *

package/model-function/generate-text/{TextPromptFormat.d.ts → prompt-format/TextPromptFormat.d.ts}

@@ -1,6 +1,6 @@
 import { ChatPrompt } from "./ChatPrompt.js";
 import { InstructionPrompt } from "./InstructionPrompt.js";
-import { TextGenerationPromptFormat } from "./TextGenerationPromptFormat.js";
+import { TextGenerationPromptFormat } from "../TextGenerationPromptFormat.js";
 /**
  * Formats an instruction prompt as a basic text prompt.
  */

package/model-function/generate-text/{VicunaPromptFormat.d.ts → prompt-format/VicunaPromptFormat.d.ts}

@@ -1,5 +1,5 @@
 import { ChatPrompt } from "./ChatPrompt.js";
-import { TextGenerationPromptFormat } from "./TextGenerationPromptFormat.js";
+import { TextGenerationPromptFormat } from "../TextGenerationPromptFormat.js";
 /**
  * Formats a chat prompt as a Vicuna prompt.
  *

package/model-function/generate-text/prompt-format/VisionInstructionPrompt.cjs

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/model-function/generate-text/prompt-format/VisionInstructionPrompt.d.ts

@@ -0,0 +1,31 @@
+/**
+ * A single instruction version prompt. It contains an instruction, a base64 encoded image
+ * and an optional mime type of the image.
+ *
+ * If no mime type is provided, the mime type default to "image/jpeg".
+ *
+ * @example
+ * ```ts
+ * {
+ *   instruction: "Describe the image in detail:",
+ *   image: fs.readFileSync(path.join("data", "example-image.png"), {
+ *     encoding: "base64",
+ *   }),
+ *   mimeType: "image/png"
+ * }
+ * ```
+ */
+export type VisionInstructionPrompt = {
+    /**
+     * The instruction for the model.
+     */
+    instruction: string;
+    /**
+     * Base-64 encoded image.
+     */
+    image: string;
+    /**
+     * Optional mime type of the image.
+     */
+    mimeType?: string;
+};

package/model-function/generate-text/prompt-format/VisionInstructionPrompt.js

@@ -0,0 +1 @@
+export {};

package/model-function/generate-text/{trimChatPrompt.d.ts → prompt-format/trimChatPrompt.d.ts}

@@ -1,5 +1,5 @@
 import { ChatPrompt } from "./ChatPrompt.js";
-import { HasContextWindowSize, HasTokenizer, TextGenerationModel, TextGenerationModelSettings } from "./TextGenerationModel.js";
+import { HasContextWindowSize, HasTokenizer, TextGenerationModel, TextGenerationModelSettings } from "../TextGenerationModel.js";
 /**
  * Keeps only the most recent messages in the prompt, while leaving enough space for the completion.
  *

package/model-provider/anthropic/AnthropicPromptFormat.cjs

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.mapChatPromptToAnthropicFormat = exports.mapInstructionPromptToAnthropicFormat = void 0;
-const validateChatPrompt_js_1 = require("../../model-function/generate-text/validateChatPrompt.cjs");
+const validateChatPrompt_js_1 = require("../../model-function/generate-text/prompt-format/validateChatPrompt.cjs");
 /**
  * Formats an instruction prompt as an Anthropic prompt.
  */

package/model-provider/anthropic/AnthropicPromptFormat.d.ts

@@ -1,5 +1,5 @@
-import { ChatPrompt } from "../../model-function/generate-text/ChatPrompt.js";
-import { InstructionPrompt } from "../../model-function/generate-text/InstructionPrompt.js";
+import { ChatPrompt } from "../../model-function/generate-text/prompt-format/ChatPrompt.js";
+import { InstructionPrompt } from "../../model-function/generate-text/prompt-format/InstructionPrompt.js";
 import { TextGenerationPromptFormat } from "../../model-function/generate-text/TextGenerationPromptFormat.js";
 /**
  * Formats an instruction prompt as an Anthropic prompt.

package/model-provider/anthropic/AnthropicPromptFormat.js

@@ -1,4 +1,4 @@
-import { validateChatPrompt } from "../../model-function/generate-text/validateChatPrompt.js";
+import { validateChatPrompt } from "../../model-function/generate-text/prompt-format/validateChatPrompt.js";
 /**
  * Formats an instruction prompt as an Anthropic prompt.
  */

package/model-provider/cohere/CohereTextGenerationModel.cjs

@@ -6,7 +6,7 @@ const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndTh
 const postToApi_js_1 = require("../../core/api/postToApi.cjs");
 const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
 const PromptFormatTextStreamingModel_js_1 = require("../../model-function/generate-text/PromptFormatTextStreamingModel.cjs");
-const TextPromptFormat_js_1 = require("../../model-function/generate-text/TextPromptFormat.cjs");
+const TextPromptFormat_js_1 = require("../../model-function/generate-text/prompt-format/TextPromptFormat.cjs");
 const countTokens_js_1 = require("../../model-function/tokenize-text/countTokens.cjs");
 const AsyncQueue_js_1 = require("../../util/AsyncQueue.cjs");
 const parseJsonStream_js_1 = require("../../util/streaming/parseJsonStream.cjs");

package/model-provider/cohere/CohereTextGenerationModel.js

@@ -3,7 +3,7 @@ import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottl
 import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
 import { AbstractModel } from "../../model-function/AbstractModel.js";
 import { PromptFormatTextStreamingModel } from "../../model-function/generate-text/PromptFormatTextStreamingModel.js";
-import { mapChatPromptToTextFormat, mapInstructionPromptToTextFormat, } from "../../model-function/generate-text/TextPromptFormat.js";
+import { mapChatPromptToTextFormat, mapInstructionPromptToTextFormat, } from "../../model-function/generate-text/prompt-format/TextPromptFormat.js";
 import { countTokens } from "../../model-function/tokenize-text/countTokens.js";
 import { AsyncQueue } from "../../util/AsyncQueue.js";
 import { parseJsonStream } from "../../util/streaming/parseJsonStream.js";

package/model-provider/llamacpp/LlamaCppTextGenerationModel.cjs

@@ -12,6 +12,7 @@ const parseJSON_js_1 = require("../../util/parseJSON.cjs");
 const LlamaCppApiConfiguration_js_1 = require("./LlamaCppApiConfiguration.cjs");
 const LlamaCppError_js_1 = require("./LlamaCppError.cjs");
 const LlamaCppTokenizer_js_1 = require("./LlamaCppTokenizer.cjs");
+const mapVisionInstructionPromptToLlamaCppFormat_js_1 = require("./mapVisionInstructionPromptToLlamaCppFormat.cjs");
 class LlamaCppTextGenerationModel extends AbstractModel_js_1.AbstractModel {
     constructor(settings = {}) {
         super({ settings });
@@ -107,6 +108,9 @@ class LlamaCppTextGenerationModel extends AbstractModel_js_1.AbstractModel {
             stopSequences: [],
         });
     }
+    withVisionInstructionPrompt() {
+        return this.withPromptFormat((0, mapVisionInstructionPromptToLlamaCppFormat_js_1.mapVisionInstructionPromptToLlamaCppFormat)());
+    }
     withPromptFormat(promptFormat) {
         return new PromptFormatTextStreamingModel_js_1.PromptFormatTextStreamingModel({
             model: this.withSettings({

package/model-provider/llamacpp/LlamaCppTextGenerationModel.d.ts

@@ -111,6 +111,7 @@ export declare class LlamaCppTextGenerationModel<CONTEXT_WINDOW_SIZE extends num
     }>;
     doStreamText(prompt: LlamaCppTextGenerationPrompt, options?: FunctionOptions): Promise<AsyncIterable<Delta<string>>>;
     withTextPrompt(): PromptFormatTextStreamingModel<string, LlamaCppTextGenerationPrompt, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
+    withVisionInstructionPrompt(): PromptFormatTextStreamingModel<import("../../index.js").VisionInstructionPrompt, LlamaCppTextGenerationPrompt, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
     withPromptFormat<INPUT_PROMPT>(promptFormat: TextGenerationPromptFormat<INPUT_PROMPT, LlamaCppTextGenerationPrompt>): PromptFormatTextStreamingModel<INPUT_PROMPT, LlamaCppTextGenerationPrompt, LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
     withSettings(additionalSettings: Partial<LlamaCppTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>>): this;
 }

package/model-provider/llamacpp/LlamaCppTextGenerationModel.js

@@ -9,6 +9,7 @@ import { parseJsonWithZod } from "../../util/parseJSON.js";
 import { LlamaCppApiConfiguration } from "./LlamaCppApiConfiguration.js";
 import { failedLlamaCppCallResponseHandler } from "./LlamaCppError.js";
 import { LlamaCppTokenizer } from "./LlamaCppTokenizer.js";
+import { mapVisionInstructionPromptToLlamaCppFormat } from "./mapVisionInstructionPromptToLlamaCppFormat.js";
 export class LlamaCppTextGenerationModel extends AbstractModel {
     constructor(settings = {}) {
         super({ settings });
@@ -104,6 +105,9 @@ export class LlamaCppTextGenerationModel extends AbstractModel {
             stopSequences: [],
         });
     }
+    withVisionInstructionPrompt() {
+        return this.withPromptFormat(mapVisionInstructionPromptToLlamaCppFormat());
+    }
     withPromptFormat(promptFormat) {
         return new PromptFormatTextStreamingModel({
             model: this.withSettings({

package/model-provider/llamacpp/mapVisionInstructionPromptToLlamaCppFormat.cjs

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.mapVisionInstructionPromptToLlamaCppFormat = void 0;
+function mapVisionInstructionPromptToLlamaCppFormat() {
+    return {
+        format: ({ instruction, image }) => {
+            return {
+                text: `[img-1]\n\n${instruction}`,
+                images: { "1": image },
+            };
+        },
+        stopSequences: [],
+    };
+}
+exports.mapVisionInstructionPromptToLlamaCppFormat = mapVisionInstructionPromptToLlamaCppFormat;

package/model-provider/llamacpp/mapVisionInstructionPromptToLlamaCppFormat.d.ts

@@ -0,0 +1,4 @@
+import { TextGenerationPromptFormat } from "../../model-function/generate-text/TextGenerationPromptFormat.js";
+import { VisionInstructionPrompt } from "../../model-function/generate-text/prompt-format/VisionInstructionPrompt.js";
+import { LlamaCppTextGenerationPrompt } from "./LlamaCppTextGenerationModel.js";
+export declare function mapVisionInstructionPromptToLlamaCppFormat(): TextGenerationPromptFormat<VisionInstructionPrompt, LlamaCppTextGenerationPrompt>;

package/model-provider/llamacpp/mapVisionInstructionPromptToLlamaCppFormat.js

@@ -0,0 +1,11 @@
+export function mapVisionInstructionPromptToLlamaCppFormat() {
+    return {
+        format: ({ instruction, image }) => {
+            return {
+                text: `[img-1]\n\n${instruction}`,
+                images: { "1": image },
+            };
+        },
+        stopSequences: [],
+    };
+}

package/model-provider/openai/OpenAICompletionModel.cjs

@@ -6,7 +6,7 @@ const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndTh
 const postToApi_js_1 = require("../../core/api/postToApi.cjs");
 const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
 const PromptFormatTextStreamingModel_js_1 = require("../../model-function/generate-text/PromptFormatTextStreamingModel.cjs");
-const TextPromptFormat_js_1 = require("../../model-function/generate-text/TextPromptFormat.cjs");
+const TextPromptFormat_js_1 = require("../../model-function/generate-text/prompt-format/TextPromptFormat.cjs");
 const countTokens_js_1 = require("../../model-function/tokenize-text/countTokens.cjs");
 const AsyncQueue_js_1 = require("../../util/AsyncQueue.cjs");
 const parseJSON_js_1 = require("../../util/parseJSON.cjs");

package/model-provider/openai/OpenAICompletionModel.js

@@ -3,7 +3,7 @@ import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottl
 import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
 import { AbstractModel } from "../../model-function/AbstractModel.js";
 import { PromptFormatTextStreamingModel } from "../../model-function/generate-text/PromptFormatTextStreamingModel.js";
-import { mapChatPromptToTextFormat, mapInstructionPromptToTextFormat, } from "../../model-function/generate-text/TextPromptFormat.js";
+import { mapChatPromptToTextFormat, mapInstructionPromptToTextFormat, } from "../../model-function/generate-text/prompt-format/TextPromptFormat.js";
 import { countTokens } from "../../model-function/tokenize-text/countTokens.js";
 import { AsyncQueue } from "../../util/AsyncQueue.js";
 import { parseJsonWithZod } from "../../util/parseJSON.js";

package/model-provider/openai/OpenAICostCalculator.cjs

@@ -1,12 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.OpenAICostCalculator = void 0;
+const OpenAICompletionModel_js_1 = require("./OpenAICompletionModel.cjs");
 const OpenAIImageGenerationModel_js_1 = require("./OpenAIImageGenerationModel.cjs");
+const OpenAISpeechModel_js_1 = require("./OpenAISpeechModel.cjs");
 const OpenAITextEmbeddingModel_js_1 = require("./OpenAITextEmbeddingModel.cjs");
-const OpenAICompletionModel_js_1 = require("./OpenAICompletionModel.cjs");
 const OpenAITranscriptionModel_js_1 = require("./OpenAITranscriptionModel.cjs");
 const OpenAIChatModel_js_1 = require("./chat/OpenAIChatModel.cjs");
-const OpenAISpeechModel_js_1 = require("./OpenAISpeechModel.cjs");
 class OpenAICostCalculator {
     constructor() {
         Object.defineProperty(this, "provider", {
@@ -21,7 +21,11 @@ class OpenAICostCalculator {
         const model = call.model.modelName;
         switch (type) {
             case "generate-image": {
+                if (model == null) {
+                    return null;
+                }
                 return (0, OpenAIImageGenerationModel_js_1.calculateOpenAIImageGenerationCostInMillicents)({
+                    model: model,
                     settings: call.settings,
                 });
             }

package/model-provider/openai/OpenAICostCalculator.js

@@ -1,9 +1,9 @@
+import { calculateOpenAICompletionCostInMillicents, isOpenAICompletionModel, } from "./OpenAICompletionModel.js";
 import { calculateOpenAIImageGenerationCostInMillicents, } from "./OpenAIImageGenerationModel.js";
+import { calculateOpenAISpeechCostInMillicents, } from "./OpenAISpeechModel.js";
 import { calculateOpenAIEmbeddingCostInMillicents, isOpenAIEmbeddingModel, } from "./OpenAITextEmbeddingModel.js";
-import { calculateOpenAICompletionCostInMillicents, isOpenAICompletionModel, } from "./OpenAICompletionModel.js";
 import { calculateOpenAITranscriptionCostInMillicents, } from "./OpenAITranscriptionModel.js";
 import { calculateOpenAIChatCostInMillicents, isOpenAIChatModel, } from "./chat/OpenAIChatModel.js";
-import { calculateOpenAISpeechCostInMillicents, } from "./OpenAISpeechModel.js";
 export class OpenAICostCalculator {
     constructor() {
         Object.defineProperty(this, "provider", {
@@ -18,7 +18,11 @@ export class OpenAICostCalculator {
         const model = call.model.modelName;
         switch (type) {
             case "generate-image": {
+                if (model == null) {
+                    return null;
+                }
                 return calculateOpenAIImageGenerationCostInMillicents({
+                    model: model,
                     settings: call.settings,
                 });
             }

package/model-provider/openai/OpenAIImageGenerationModel.cjs

@@ -55,9 +55,8 @@ exports.OPENAI_IMAGE_MODELS = {
 /**
  * @see https://openai.com/pricing
  */
-const calculateOpenAIImageGenerationCostInMillicents = ({ settings, }) => {
-    console.log(settings);
-    const cost = exports.OPENAI_IMAGE_MODELS[settings.model]?.getCost(settings);
+const calculateOpenAIImageGenerationCostInMillicents = ({ model, settings, }) => {
+    const cost = exports.OPENAI_IMAGE_MODELS[model]?.getCost(settings);
     if (cost == null) {
         return null;
     }
@@ -106,7 +105,6 @@ class OpenAIImageGenerationModel extends AbstractModel_js_1.AbstractModel {
     }
     get settingsForEvent() {
         const eventSettingProperties = [
-            "model",
             "n",
             "size",
             "quality",

package/model-provider/openai/OpenAIImageGenerationModel.d.ts

@@ -17,7 +17,8 @@ export declare const OPENAI_IMAGE_MODELS: {
 /**
  * @see https://openai.com/pricing
  */
-export declare const calculateOpenAIImageGenerationCostInMillicents: ({ settings, }: {
+export declare const calculateOpenAIImageGenerationCostInMillicents: ({ model, settings, }: {
+    model: OpenAIImageModelType;
     settings: OpenAIImageGenerationSettings;
 }) => number | null;
 export type OpenAIImageModelType = keyof typeof OPENAI_IMAGE_MODELS;

package/model-provider/openai/OpenAIImageGenerationModel.js

@@ -52,9 +52,8 @@ export const OPENAI_IMAGE_MODELS = {
 /**
  * @see https://openai.com/pricing
  */
-export const calculateOpenAIImageGenerationCostInMillicents = ({ settings, }) => {
-    console.log(settings);
-    const cost = OPENAI_IMAGE_MODELS[settings.model]?.getCost(settings);
+export const calculateOpenAIImageGenerationCostInMillicents = ({ model, settings, }) => {
+    const cost = OPENAI_IMAGE_MODELS[model]?.getCost(settings);
     if (cost == null) {
         return null;
     }
@@ -102,7 +101,6 @@ export class OpenAIImageGenerationModel extends AbstractModel {
     }
     get settingsForEvent() {
         const eventSettingProperties = [
-            "model",
             "n",
             "size",
             "quality",

package/model-provider/openai/chat/OpenAIChatMessage.d.ts

@@ -1,14 +1,37 @@
 export type OpenAIChatMessage = {
-    role: "user" | "assistant" | "system";
+    role: "system";
     content: string;
     name?: string;
+} | {
+    role: "user";
+    content: string | Array<{
+        type: "text";
+        text: string;
+    } | {
+        type: "image_url";
+        image_url: string;
+    }>;
+    name?: string;
 } | {
     role: "assistant";
     content: string | null;
-    function_call: {
+    name?: string;
+    tool_calls?: Array<{
+        id: string;
+        type: "function";
+        function: {
+            name: string;
+            arguments: string;
+        };
+    }>;
+    function_call?: {
         name: string;
         arguments: string;
     };
+} | {
+    role: "tool";
+    tool_call_id: string;
+    content: string | null;
 } | {
     role: "function";
     content: string;

package/model-provider/openai/chat/OpenAIChatModel.cjs

@@ -211,6 +211,7 @@ class OpenAIChatModel extends AbstractModel_js_1.AbstractModel {
                 // map to OpenAI API names:
                 stop: this.settings.stopSequences,
                 maxTokens: this.settings.maxCompletionTokens,
+                openAIResponseFormat: this.settings.responseFormat,
                 // other settings:
                 user: this.settings.isUserIdForwardingEnabled
                     ? options.run?.userId
@@ -363,6 +364,9 @@ class OpenAIChatModel extends AbstractModel_js_1.AbstractModel {
     withInstructionPrompt() {
         return this.withPromptFormat((0, OpenAIChatPromptFormat_js_1.mapInstructionPromptToOpenAIChatFormat)());
     }
+    withVisionInstructionPrompt() {
+        return this.withPromptFormat((0, OpenAIChatPromptFormat_js_1.mapVisionInstructionPromptToOpenAIChatFormat)());
+    }
     /**
      * Returns this model with a chat prompt format.
      */
@@ -390,6 +394,7 @@ const openAIChatResponseSchema = zod_1.z.object({
     object: zod_1.z.literal("chat.completion"),
     created: zod_1.z.number(),
     model: zod_1.z.string(),
+    system_fingerprint: zod_1.z.string(),
     choices: zod_1.z.array(zod_1.z.object({
         message: zod_1.z.object({
             role: zod_1.z.literal("assistant"),
@@ -403,7 +408,13 @@ const openAIChatResponseSchema = zod_1.z.object({
         }),
         index: zod_1.z.number(),
         logprobs: zod_1.z.nullable(zod_1.z.any()),
-        finish_reason: zod_1.z.string(),
+        finish_reason: zod_1.z.enum([
+            "stop",
+            "length",
+            "tool_calls",
+            "content_filter",
+            "function_call",
+        ]),
     })),
     usage: zod_1.z.object({
         prompt_tokens: zod_1.z.number(),
@@ -411,7 +422,7 @@ const openAIChatResponseSchema = zod_1.z.object({
         total_tokens: zod_1.z.number(),
     }),
 });
-async function callOpenAIChatCompletionAPI({ api = new OpenAIApiConfiguration_js_1.OpenAIApiConfiguration(), abortSignal, responseFormat, model, messages, functions, functionCall, temperature, topP, n, stop, maxTokens, presencePenalty, frequencyPenalty, logitBias, user, }) {
+async function callOpenAIChatCompletionAPI({ api = new OpenAIApiConfiguration_js_1.OpenAIApiConfiguration(), abortSignal, responseFormat, model, messages, functions, functionCall, temperature, topP, n, stop, maxTokens, presencePenalty, frequencyPenalty, logitBias, user, openAIResponseFormat, seed, }) {
     // empty arrays are not allowed for stop:
     if (stop != null && Array.isArray(stop) && stop.length === 0) {
         stop = undefined;
@@ -433,6 +444,8 @@ async function callOpenAIChatCompletionAPI({ api = new OpenAIApiConfiguration_js
             presence_penalty: presencePenalty,
             frequency_penalty: frequencyPenalty,
             logit_bias: logitBias,
+            seed,
+            response_format: openAIResponseFormat,
             user,
         },
         failedResponseHandler: OpenAIError_js_1.failedOpenAICallResponseHandler,

package/model-provider/openai/chat/OpenAIChatModel.d.ts

@@ -119,6 +119,10 @@ export interface OpenAIChatCallSettings {
     maxTokens?: number;
     temperature?: number;
     topP?: number;
+    seed?: number | null;
+    responseFormat?: {
+        type?: "text" | "json_object";
+    };
     n?: number;
     presencePenalty?: number;
     frequencyPenalty?: number;
@@ -189,11 +193,12 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                         arguments: string;
                     } | undefined;
                 };
-                finish_reason: string;
+                finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
                 index: number;
                 logprobs?: any;
             }[];
             created: number;
+            system_fingerprint: string;
         };
         text: string;
         usage: {
@@ -229,11 +234,12 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                         arguments: string;
                     } | undefined;
                 };
-                finish_reason: string;
+                finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
                 index: number;
                 logprobs?: any;
             }[];
             created: number;
+            system_fingerprint: string;
         };
         valueText: string;
         value: any;
@@ -263,11 +269,12 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                         arguments: string;
                     } | undefined;
                 };
-                finish_reason: string;
+                finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
                 index: number;
                 logprobs?: any;
             }[];
             created: number;
+            system_fingerprint: string;
         };
         structureAndText: {
             structure: null;
@@ -299,11 +306,12 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                         arguments: string;
                     } | undefined;
                 };
-                finish_reason: string;
+                finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
                 index: number;
                 logprobs?: any;
             }[];
             created: number;
+            system_fingerprint: string;
         };
         structureAndText: {
             structure: string;
@@ -326,6 +334,7 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
      * Returns this model with an instruction prompt format.
      */
     withInstructionPrompt(): PromptFormatTextStreamingModel<import("../../../index.js").InstructionPrompt, OpenAIChatMessage[], OpenAIChatSettings, this>;
+    withVisionInstructionPrompt(): PromptFormatTextStreamingModel<import("../../../index.js").VisionInstructionPrompt, OpenAIChatMessage[], OpenAIChatSettings, this>;
     /**
      * Returns this model with a chat prompt format.
      */
@@ -338,6 +347,7 @@ declare const openAIChatResponseSchema: z.ZodObject<{
     object: z.ZodLiteral<"chat.completion">;
     created: z.ZodNumber;
     model: z.ZodString;
+    system_fingerprint: z.ZodString;
     choices: z.ZodArray<z.ZodObject<{
         message: z.ZodObject<{
             role: z.ZodLiteral<"assistant">;
@@ -369,7 +379,7 @@ declare const openAIChatResponseSchema: z.ZodObject<{
         }>;
         index: z.ZodNumber;
         logprobs: z.ZodNullable<z.ZodAny>;
-        finish_reason: z.ZodString;
+        finish_reason: z.ZodEnum<["stop", "length", "tool_calls", "content_filter", "function_call"]>;
     }, "strip", z.ZodTypeAny, {
         message: {
             content: string | null;
@@ -379,7 +389,7 @@ declare const openAIChatResponseSchema: z.ZodObject<{
                 arguments: string;
             } | undefined;
         };
-        finish_reason: string;
+        finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
         index: number;
         logprobs?: any;
     }, {
@@ -391,7 +401,7 @@ declare const openAIChatResponseSchema: z.ZodObject<{
                 arguments: string;
             } | undefined;
         };
-        finish_reason: string;
+        finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
         index: number;
         logprobs?: any;
     }>, "many">;
@@ -426,11 +436,12 @@ declare const openAIChatResponseSchema: z.ZodObject<{
                 arguments: string;
             } | undefined;
         };
-        finish_reason: string;
+        finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
         index: number;
         logprobs?: any;
     }[];
     created: number;
+    system_fingerprint: string;
 }, {
     object: "chat.completion";
     usage: {
@@ -449,11 +460,12 @@ declare const openAIChatResponseSchema: z.ZodObject<{
                 arguments: string;
             } | undefined;
         };
-        finish_reason: string;
+        finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
         index: number;
         logprobs?: any;
     }[];
     created: number;
+    system_fingerprint: string;
 }>;
 export type OpenAIChatResponse = z.infer<typeof openAIChatResponseSchema>;
 export type OpenAIChatResponseFormatType<T> = {
@@ -484,11 +496,12 @@ export declare const OpenAIChatResponseFormat: {
                         arguments: string;
                     } | undefined;
                 };
-                finish_reason: string;
+                finish_reason: "length" | "stop" | "function_call" | "tool_calls" | "content_filter";
                 index: number;
                 logprobs?: any;
             }[];
             created: number;
+            system_fingerprint: string;
         }>;
     };
     /**

package/model-provider/openai/chat/OpenAIChatModel.js

@@ -9,7 +9,7 @@ import { PromptFormatTextStreamingModel } from "../../../model-function/generate
 import { OpenAIApiConfiguration } from "../OpenAIApiConfiguration.js";
 import { failedOpenAICallResponseHandler } from "../OpenAIError.js";
 import { TikTokenTokenizer } from "../TikTokenTokenizer.js";
-import { mapChatPromptToOpenAIChatFormat, mapInstructionPromptToOpenAIChatFormat, } from "./OpenAIChatPromptFormat.js";
+import { mapChatPromptToOpenAIChatFormat, mapInstructionPromptToOpenAIChatFormat, mapVisionInstructionPromptToOpenAIChatFormat, } from "./OpenAIChatPromptFormat.js";
 import { createOpenAIChatDeltaIterableQueue } from "./OpenAIChatStreamIterable.js";
 import { countOpenAIChatPromptTokens } from "./countOpenAIChatMessageTokens.js";
 /*
@@ -202,6 +202,7 @@ export class OpenAIChatModel extends AbstractModel {
                 // map to OpenAI API names:
                 stop: this.settings.stopSequences,
                 maxTokens: this.settings.maxCompletionTokens,
+                openAIResponseFormat: this.settings.responseFormat,
                 // other settings:
                 user: this.settings.isUserIdForwardingEnabled
                     ? options.run?.userId
@@ -354,6 +355,9 @@ export class OpenAIChatModel extends AbstractModel {
     withInstructionPrompt() {
         return this.withPromptFormat(mapInstructionPromptToOpenAIChatFormat());
     }
+    withVisionInstructionPrompt() {
+        return this.withPromptFormat(mapVisionInstructionPromptToOpenAIChatFormat());
+    }
     /**
      * Returns this model with a chat prompt format.
      */
@@ -380,6 +384,7 @@ const openAIChatResponseSchema = z.object({
     object: z.literal("chat.completion"),
     created: z.number(),
     model: z.string(),
+    system_fingerprint: z.string(),
     choices: z.array(z.object({
         message: z.object({
             role: z.literal("assistant"),
@@ -393,7 +398,13 @@ const openAIChatResponseSchema = z.object({
         }),
         index: z.number(),
         logprobs: z.nullable(z.any()),
-        finish_reason: z.string(),
+        finish_reason: z.enum([
+            "stop",
+            "length",
+            "tool_calls",
+            "content_filter",
+            "function_call",
+        ]),
     })),
     usage: z.object({
         prompt_tokens: z.number(),
@@ -401,7 +412,7 @@ const openAIChatResponseSchema = z.object({
         total_tokens: z.number(),
     }),
 });
-async function callOpenAIChatCompletionAPI({ api = new OpenAIApiConfiguration(), abortSignal, responseFormat, model, messages, functions, functionCall, temperature, topP, n, stop, maxTokens, presencePenalty, frequencyPenalty, logitBias, user, }) {
+async function callOpenAIChatCompletionAPI({ api = new OpenAIApiConfiguration(), abortSignal, responseFormat, model, messages, functions, functionCall, temperature, topP, n, stop, maxTokens, presencePenalty, frequencyPenalty, logitBias, user, openAIResponseFormat, seed, }) {
     // empty arrays are not allowed for stop:
     if (stop != null && Array.isArray(stop) && stop.length === 0) {
         stop = undefined;
@@ -423,6 +434,8 @@ async function callOpenAIChatCompletionAPI({ api = new OpenAIApiConfiguration(),
             presence_penalty: presencePenalty,
             frequency_penalty: frequencyPenalty,
             logit_bias: logitBias,
+            seed,
+            response_format: openAIResponseFormat,
             user,
         },
         failedResponseHandler: failedOpenAICallResponseHandler,

package/model-provider/openai/chat/OpenAIChatPromptFormat.cjs

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.mapChatPromptToOpenAIChatFormat = exports.mapInstructionPromptToOpenAIChatFormat = void 0;
-const validateChatPrompt_js_1 = require("../../../model-function/generate-text/validateChatPrompt.cjs");
+exports.mapChatPromptToOpenAIChatFormat = exports.mapVisionInstructionPromptToOpenAIChatFormat = exports.mapInstructionPromptToOpenAIChatFormat = void 0;
+const validateChatPrompt_js_1 = require("../../../model-function/generate-text/prompt-format/validateChatPrompt.cjs");
 /**
  * Formats an instruction prompt as an OpenAI chat prompt.
  */
@@ -31,6 +31,29 @@ function mapInstructionPromptToOpenAIChatFormat() {
     };
 }
 exports.mapInstructionPromptToOpenAIChatFormat = mapInstructionPromptToOpenAIChatFormat;
+/**
+ * Formats a version prompt as an OpenAI chat prompt.
+ */
+function mapVisionInstructionPromptToOpenAIChatFormat() {
+    return {
+        format: ({ instruction, image, mimeType }) => {
+            return [
+                {
+                    role: "user",
+                    content: [
+                        { type: "text", text: instruction },
+                        {
+                            type: "image_url",
+                            image_url: `data:${mimeType ?? "image/jpeg"};base64,${image}`,
+                        },
+                    ],
+                },
+            ];
+        },
+        stopSequences: [],
+    };
+}
+exports.mapVisionInstructionPromptToOpenAIChatFormat = mapVisionInstructionPromptToOpenAIChatFormat;
 /**
  * Formats a chat prompt as an OpenAI chat prompt.
  */

package/model-provider/openai/chat/OpenAIChatPromptFormat.d.ts

@@ -1,11 +1,16 @@
-import { ChatPrompt } from "../../../model-function/generate-text/ChatPrompt.js";
-import { InstructionPrompt } from "../../../model-function/generate-text/InstructionPrompt.js";
+import { ChatPrompt } from "../../../model-function/generate-text/prompt-format/ChatPrompt.js";
+import { InstructionPrompt } from "../../../model-function/generate-text/prompt-format/InstructionPrompt.js";
 import { TextGenerationPromptFormat } from "../../../model-function/generate-text/TextGenerationPromptFormat.js";
 import { OpenAIChatMessage } from "./OpenAIChatMessage.js";
+import { VisionInstructionPrompt } from "../../../model-function/generate-text/prompt-format/VisionInstructionPrompt.js";
 /**
  * Formats an instruction prompt as an OpenAI chat prompt.
  */
 export declare function mapInstructionPromptToOpenAIChatFormat(): TextGenerationPromptFormat<InstructionPrompt, Array<OpenAIChatMessage>>;
+/**
+ * Formats a version prompt as an OpenAI chat prompt.
+ */
+export declare function mapVisionInstructionPromptToOpenAIChatFormat(): TextGenerationPromptFormat<VisionInstructionPrompt, Array<OpenAIChatMessage>>;
 /**
  * Formats a chat prompt as an OpenAI chat prompt.
  */

package/model-provider/openai/chat/OpenAIChatPromptFormat.js

@@ -1,4 +1,4 @@
-import { validateChatPrompt } from "../../../model-function/generate-text/validateChatPrompt.js";
+import { validateChatPrompt } from "../../../model-function/generate-text/prompt-format/validateChatPrompt.js";
 /**
  * Formats an instruction prompt as an OpenAI chat prompt.
  */
@@ -27,6 +27,28 @@ export function mapInstructionPromptToOpenAIChatFormat() {
         stopSequences: [],
     };
 }
+/**
+ * Formats a version prompt as an OpenAI chat prompt.
+ */
+export function mapVisionInstructionPromptToOpenAIChatFormat() {
+    return {
+        format: ({ instruction, image, mimeType }) => {
+            return [
+                {
+                    role: "user",
+                    content: [
+                        { type: "text", text: instruction },
+                        {
+                            type: "image_url",
+                            image_url: `data:${mimeType ?? "image/jpeg"};base64,${image}`,
+                        },
+                    ],
+                },
+            ];
+        },
+        stopSequences: [],
+    };
+}
 /**
  * Formats a chat prompt as an OpenAI chat prompt.
  */

package/model-provider/openai/chat/OpenAIChatStreamIterable.cjs

@@ -17,7 +17,16 @@ const chatResponseStreamEventSchema = zod_1.z.object({
             })
                 .optional(),
         }),
-        finish_reason: zod_1.z.enum(["stop", "length"]).nullable().optional(),
+        finish_reason: zod_1.z
+            .enum([
+            "stop",
+            "length",
+            "tool_calls",
+            "content_filter",
+            "function_call",
+        ])
+            .nullable()
+            .optional(),
         index: zod_1.z.number(),
     })),
     created: zod_1.z.number(),

package/model-provider/openai/chat/OpenAIChatStreamIterable.js

@@ -14,7 +14,16 @@ const chatResponseStreamEventSchema = z.object({
             })
                 .optional(),
         }),
-        finish_reason: z.enum(["stop", "length"]).nullable().optional(),
+        finish_reason: z
+            .enum([
+            "stop",
+            "length",
+            "tool_calls",
+            "content_filter",
+            "function_call",
+        ])
+            .nullable()
+            .optional(),
         index: z.number(),
     })),
     created: z.number(),

package/model-provider/openai/chat/countOpenAIChatMessageTokens.cjs

@@ -15,10 +15,26 @@ exports.OPENAI_CHAT_PROMPT_BASE_TOKEN_COUNT = 2;
  */
 exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT = 5;
 async function countOpenAIChatMessageTokens({ message, model, }) {
-    const contentTokenCount = await (0, countTokens_js_1.countTokens)(new TikTokenTokenizer_js_1.TikTokenTokenizer({
+    const tokenizer = new TikTokenTokenizer_js_1.TikTokenTokenizer({
         model: (0, OpenAIChatModel_js_1.getOpenAIChatModelInformation)(model).baseModel,
-    }), message.content ?? "");
-    return exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT + contentTokenCount;
+    });
+    // case: function call without content
+    if (message.content == null) {
+        return exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT;
+    }
+    // case: simple text content
+    if (typeof message.content === "string") {
+        return (exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT +
+            (await (0, countTokens_js_1.countTokens)(tokenizer, message.content)));
+    }
+    // case: array of content objects
+    let contentTokenCount = exports.OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT;
+    for (const content of message.content) {
+        if (content.type === "text") {
+            contentTokenCount += await (0, countTokens_js_1.countTokens)(tokenizer, content.text);
+        }
+    }
+    return contentTokenCount;
 }
 exports.countOpenAIChatMessageTokens = countOpenAIChatMessageTokens;
 async function countOpenAIChatPromptTokens({ messages, model, }) {

package/model-provider/openai/chat/countOpenAIChatMessageTokens.js

@@ -12,10 +12,26 @@ export const OPENAI_CHAT_PROMPT_BASE_TOKEN_COUNT = 2;
  */
 export const OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT = 5;
 export async function countOpenAIChatMessageTokens({ message, model, }) {
-    const contentTokenCount = await countTokens(new TikTokenTokenizer({
+    const tokenizer = new TikTokenTokenizer({
         model: getOpenAIChatModelInformation(model).baseModel,
-    }), message.content ?? "");
-    return OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT + contentTokenCount;
+    });
+    // case: function call without content
+    if (message.content == null) {
+        return OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT;
+    }
+    // case: simple text content
+    if (typeof message.content === "string") {
+        return (OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT +
+            (await countTokens(tokenizer, message.content)));
+    }
+    // case: array of content objects
+    let contentTokenCount = OPENAI_CHAT_MESSAGE_BASE_TOKEN_COUNT;
+    for (const content of message.content) {
+        if (content.type === "text") {
+            contentTokenCount += await countTokens(tokenizer, content.text);
+        }
+    }
+    return contentTokenCount;
 }
 export async function countOpenAIChatPromptTokens({ messages, model, }) {
     let tokens = OPENAI_CHAT_PROMPT_BASE_TOKEN_COUNT;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "Build multimodal applications, chatbots, and agents with JavaScript and TypeScript.",
-  "version": "0.62.0",
+  "version": "0.64.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [