modelfusion 0.60.0 → 0.62.0

package/README.md

@@ -14,6 +14,7 @@
 
 **ModelFusion** is a TypeScript library for building AI applications, chatbots, and agents.
 
+- **Vendor-neutral**: ModelFusion is a non-commercial open source project that is community-driven. You can use it with any supported vendor.
 - **Multimodal**: ModelFusion supports a wide range of models including text generation, image generation, text-to-speech, speech-to-text, and embedding models.
 - **Streaming**: ModelFusion supports streaming for many generation models, e.g. text streaming, structure streaming, and full duplex speech streaming.
 - **Utility functions**: ModelFusion provides functionality for tools and tool usage, vector indices, and guards functions.
@@ -78,7 +79,10 @@ Generate an image from a prompt.
 
 ```ts
 const image = await generateImage(
-  new OpenAIImageGenerationModel({ size: "512x512" }),
+  new OpenAIImageGenerationModel({
+    model: "dall-e-3",
+    size: "1024x1024",
+  }),
   "the wicked witch of the west in the style of early 19th century painting"
 );
 ```
@@ -106,7 +110,7 @@ const speech = await generateSpeech(
 );
 ```
 
-Providers: [Eleven Labs](https://modelfusion.dev/integration/model-provider/elevenlabs), [LMNT](https://modelfusion.dev/integration/model-provider/lmnt)
+Providers: [Eleven Labs](https://modelfusion.dev/integration/model-provider/elevenlabs), [LMNT](https://modelfusion.dev/integration/model-provider/lmnt), [OpenAI](https://modelfusion.dev/integration/model-provider/openai)
 
 #### streamSpeech
 
@@ -460,7 +464,9 @@ const text = await generateText(
   new LlamaCppTextGenerationModel({
     contextWindowSize: 4096, // Llama 2 context window size
     maxCompletionTokens: 1000,
-  }).withPromptFormat(mapInstructionPromptToLlama2Format()),
+  })
+    .withTextPrompt()
+    .withPromptFormat(mapInstructionPromptToLlama2Format()),
   {
     system: "You are a story writer.",
     instruction: "Write a short story about a robot learning to love.",

package/model-provider/openai/OpenAICompletionModel.d.ts

@@ -150,18 +150,18 @@ export declare class OpenAICompletionModel extends AbstractModel<OpenAICompletio
             object: "text_completion";
             usage: {
                 prompt_tokens: number;
-                completion_tokens: number;
                 total_tokens: number;
+                completion_tokens: number;
             };
             model: string;
             id: string;
-            created: number;
             choices: {
                 text: string;
                 finish_reason: string;
                 index: number;
                 logprobs?: any;
             }[];
+            created: number;
         };
         text: string;
         usage: {
@@ -212,45 +212,45 @@ declare const OpenAICompletionResponseSchema: z.ZodObject<{
         total_tokens: z.ZodNumber;
     }, "strip", z.ZodTypeAny, {
         prompt_tokens: number;
-        completion_tokens: number;
         total_tokens: number;
+        completion_tokens: number;
     }, {
         prompt_tokens: number;
-        completion_tokens: number;
         total_tokens: number;
+        completion_tokens: number;
     }>;
 }, "strip", z.ZodTypeAny, {
     object: "text_completion";
     usage: {
         prompt_tokens: number;
-        completion_tokens: number;
         total_tokens: number;
+        completion_tokens: number;
     };
     model: string;
     id: string;
-    created: number;
     choices: {
         text: string;
         finish_reason: string;
         index: number;
         logprobs?: any;
     }[];
+    created: number;
 }, {
     object: "text_completion";
     usage: {
         prompt_tokens: number;
-        completion_tokens: number;
         total_tokens: number;
+        completion_tokens: number;
     };
     model: string;
     id: string;
-    created: number;
     choices: {
         text: string;
         finish_reason: string;
         index: number;
         logprobs?: any;
     }[];
+    created: number;
 }>;
 export type OpenAICompletionResponse = z.infer<typeof OpenAICompletionResponseSchema>;
 export type OpenAITextResponseFormatType<T> = {
@@ -267,18 +267,18 @@ export declare const OpenAITextResponseFormat: {
             object: "text_completion";
             usage: {
                 prompt_tokens: number;
-                completion_tokens: number;
                 total_tokens: number;
+                completion_tokens: number;
             };
             model: string;
             id: string;
-            created: number;
             choices: {
                 text: string;
                 finish_reason: string;
                 index: number;
                 logprobs?: any;
             }[];
+            created: number;
         }>;
     };
     /**

package/model-provider/openai/OpenAICostCalculator.cjs

@@ -6,6 +6,7 @@ const OpenAITextEmbeddingModel_js_1 = require("./OpenAITextEmbeddingModel.cjs");
 const OpenAICompletionModel_js_1 = require("./OpenAICompletionModel.cjs");
 const OpenAITranscriptionModel_js_1 = require("./OpenAITranscriptionModel.cjs");
 const OpenAIChatModel_js_1 = require("./chat/OpenAIChatModel.cjs");
+const OpenAISpeechModel_js_1 = require("./OpenAISpeechModel.cjs");
 class OpenAICostCalculator {
     constructor() {
         Object.defineProperty(this, "provider", {
@@ -68,6 +69,15 @@ class OpenAICostCalculator {
                         .response,
                 });
             }
+            case "generate-speech": {
+                if (model == null) {
+                    return null;
+                }
+                return (0, OpenAISpeechModel_js_1.calculateOpenAISpeechCostInMillicents)({
+                    model: model,
+                    input: call.input,
+                });
+            }
         }
         return null;
     }

package/model-provider/openai/OpenAICostCalculator.js

@@ -3,6 +3,7 @@ import { calculateOpenAIEmbeddingCostInMillicents, isOpenAIEmbeddingModel, } fro
 import { calculateOpenAICompletionCostInMillicents, isOpenAICompletionModel, } from "./OpenAICompletionModel.js";
 import { calculateOpenAITranscriptionCostInMillicents, } from "./OpenAITranscriptionModel.js";
 import { calculateOpenAIChatCostInMillicents, isOpenAIChatModel, } from "./chat/OpenAIChatModel.js";
+import { calculateOpenAISpeechCostInMillicents, } from "./OpenAISpeechModel.js";
 export class OpenAICostCalculator {
     constructor() {
         Object.defineProperty(this, "provider", {
@@ -65,6 +66,15 @@ export class OpenAICostCalculator {
                         .response,
                 });
             }
+            case "generate-speech": {
+                if (model == null) {
+                    return null;
+                }
+                return calculateOpenAISpeechCostInMillicents({
+                    model: model,
+                    input: call.input,
+                });
+            }
         }
         return null;
     }

package/model-provider/openai/OpenAIImageGenerationModel.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.OpenAIImageGenerationResponseFormat = exports.OpenAIImageGenerationModel = exports.calculateOpenAIImageGenerationCostInMillicents = void 0;
+exports.OpenAIImageGenerationResponseFormat = exports.OpenAIImageGenerationModel = exports.calculateOpenAIImageGenerationCostInMillicents = exports.OPENAI_IMAGE_MODELS = void 0;
 const zod_1 = require("zod");
 const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndThrottle.cjs");
 const postToApi_js_1 = require("../../core/api/postToApi.cjs");
@@ -8,15 +8,61 @@ const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
 const PromptFormatImageGenerationModel_js_1 = require("../../model-function/generate-image/PromptFormatImageGenerationModel.cjs");
 const OpenAIApiConfiguration_js_1 = require("./OpenAIApiConfiguration.cjs");
 const OpenAIError_js_1 = require("./OpenAIError.cjs");
+exports.OPENAI_IMAGE_MODELS = {
+    "dall-e-2": {
+        getCost(settings) {
+            switch (settings.size ?? "1024x1024") {
+                case "1024x1024":
+                    return 2000;
+                case "512x512":
+                    return 1800;
+                case "256x256":
+                    return 1600;
+                default:
+                    return null;
+            }
+        },
+    },
+    "dall-e-3": {
+        getCost(settings) {
+            switch (settings.quality ?? "standard") {
+                case "standard": {
+                    switch (settings.size ?? "1024x1024") {
+                        case "1024x1024":
+                            return 4000;
+                        case "1024x1792":
+                        case "1792x1024":
+                            return 8000;
+                        default:
+                            return null;
+                    }
+                }
+                case "hd": {
+                    switch (settings.size ?? "1024x1024") {
+                        case "1024x1024":
+                            return 8000;
+                        case "1024x1792":
+                        case "1792x1024":
+                            return 12000;
+                        default:
+                            return null;
+                    }
+                }
+            }
+        },
+    },
+};
 /**
  * @see https://openai.com/pricing
  */
-const sizeToCostInMillicents = {
-    "1024x1024": 2000,
-    "512x512": 1800,
-    "256x256": 1600,
+const calculateOpenAIImageGenerationCostInMillicents = ({ settings, }) => {
+    console.log(settings);
+    const cost = exports.OPENAI_IMAGE_MODELS[settings.model]?.getCost(settings);
+    if (cost == null) {
+        return null;
+    }
+    return (settings.n ?? 1) * cost;
 };
-const calculateOpenAIImageGenerationCostInMillicents = ({ settings, }) => (settings.n ?? 1) * sizeToCostInMillicents[settings.size ?? "1024x1024"];
 exports.calculateOpenAIImageGenerationCostInMillicents = calculateOpenAIImageGenerationCostInMillicents;
 /**
  * Create an image generation model that calls the OpenAI AI image creation API.
@@ -38,12 +84,9 @@ class OpenAIImageGenerationModel extends AbstractModel_js_1.AbstractModel {
             writable: true,
             value: "openai"
         });
-        Object.defineProperty(this, "modelName", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: null
-        });
+    }
+    get modelName() {
+        return this.settings.model;
     }
     async callAPI(prompt, options) {
         const run = options?.run;
@@ -63,8 +106,11 @@ class OpenAIImageGenerationModel extends AbstractModel_js_1.AbstractModel {
     }
     get settingsForEvent() {
         const eventSettingProperties = [
+            "model",
             "n",
             "size",
+            "quality",
+            "style",
         ];
         return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
     }

package/model-provider/openai/OpenAIImageGenerationModel.d.ts

@@ -6,13 +6,28 @@ import { AbstractModel } from "../../model-function/AbstractModel.js";
 import { PromptFormat } from "../../model-function/PromptFormat.js";
 import { ImageGenerationModel, ImageGenerationModelSettings } from "../../model-function/generate-image/ImageGenerationModel.js";
 import { PromptFormatImageGenerationModel } from "../../model-function/generate-image/PromptFormatImageGenerationModel.js";
+export declare const OPENAI_IMAGE_MODELS: {
+    "dall-e-2": {
+        getCost(settings: OpenAIImageGenerationSettings): 2000 | 1800 | 1600 | null;
+    };
+    "dall-e-3": {
+        getCost(settings: OpenAIImageGenerationSettings): 8000 | 4000 | 12000 | null;
+    };
+};
+/**
+ * @see https://openai.com/pricing
+ */
+export declare const calculateOpenAIImageGenerationCostInMillicents: ({ settings, }: {
+    settings: OpenAIImageGenerationSettings;
+}) => number | null;
+export type OpenAIImageModelType = keyof typeof OPENAI_IMAGE_MODELS;
 export interface OpenAIImageGenerationCallSettings {
+    model: OpenAIImageModelType;
     n?: number;
-    size?: "256x256" | "512x512" | "1024x1024";
+    size?: "256x256" | "512x512" | "1024x1024" | "1792x1024" | "1024x1792";
+    quality?: "standard" | "hd";
+    style?: "vivid" | "natural";
 }
-export declare const calculateOpenAIImageGenerationCostInMillicents: ({ settings, }: {
-    settings: OpenAIImageGenerationSettings;
-}) => number;
 export interface OpenAIImageGenerationSettings extends ImageGenerationModelSettings, OpenAIImageGenerationCallSettings {
     api?: ApiConfiguration;
     isUserIdForwardingEnabled?: boolean;
@@ -31,7 +46,7 @@ export interface OpenAIImageGenerationSettings extends ImageGenerationModelSetti
 export declare class OpenAIImageGenerationModel extends AbstractModel<OpenAIImageGenerationSettings> implements ImageGenerationModel<string, OpenAIImageGenerationSettings> {
     constructor(settings: OpenAIImageGenerationSettings);
     readonly provider: "openai";
-    readonly modelName: null;
+    get modelName(): "dall-e-2" | "dall-e-3";
     callAPI<RESULT>(prompt: string, options: {
         responseFormat: OpenAIImageGenerationResponseFormatType<RESULT>;
     } & FunctionOptions): Promise<RESULT>;

package/model-provider/openai/OpenAIImageGenerationModel.js

@@ -5,15 +5,61 @@ import { AbstractModel } from "../../model-function/AbstractModel.js";
 import { PromptFormatImageGenerationModel } from "../../model-function/generate-image/PromptFormatImageGenerationModel.js";
 import { OpenAIApiConfiguration } from "./OpenAIApiConfiguration.js";
 import { failedOpenAICallResponseHandler } from "./OpenAIError.js";
+export const OPENAI_IMAGE_MODELS = {
+    "dall-e-2": {
+        getCost(settings) {
+            switch (settings.size ?? "1024x1024") {
+                case "1024x1024":
+                    return 2000;
+                case "512x512":
+                    return 1800;
+                case "256x256":
+                    return 1600;
+                default:
+                    return null;
+            }
+        },
+    },
+    "dall-e-3": {
+        getCost(settings) {
+            switch (settings.quality ?? "standard") {
+                case "standard": {
+                    switch (settings.size ?? "1024x1024") {
+                        case "1024x1024":
+                            return 4000;
+                        case "1024x1792":
+                        case "1792x1024":
+                            return 8000;
+                        default:
+                            return null;
+                    }
+                }
+                case "hd": {
+                    switch (settings.size ?? "1024x1024") {
+                        case "1024x1024":
+                            return 8000;
+                        case "1024x1792":
+                        case "1792x1024":
+                            return 12000;
+                        default:
+                            return null;
+                    }
+                }
+            }
+        },
+    },
+};
 /**
  * @see https://openai.com/pricing
  */
-const sizeToCostInMillicents = {
-    "1024x1024": 2000,
-    "512x512": 1800,
-    "256x256": 1600,
+export const calculateOpenAIImageGenerationCostInMillicents = ({ settings, }) => {
+    console.log(settings);
+    const cost = OPENAI_IMAGE_MODELS[settings.model]?.getCost(settings);
+    if (cost == null) {
+        return null;
+    }
+    return (settings.n ?? 1) * cost;
 };
-export const calculateOpenAIImageGenerationCostInMillicents = ({ settings, }) => (settings.n ?? 1) * sizeToCostInMillicents[settings.size ?? "1024x1024"];
 /**
  * Create an image generation model that calls the OpenAI AI image creation API.
  *
@@ -34,12 +80,9 @@ export class OpenAIImageGenerationModel extends AbstractModel {
             writable: true,
             value: "openai"
         });
-        Object.defineProperty(this, "modelName", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: null
-        });
+    }
+    get modelName() {
+        return this.settings.model;
     }
     async callAPI(prompt, options) {
         const run = options?.run;
@@ -59,8 +102,11 @@ export class OpenAIImageGenerationModel extends AbstractModel {
     }
     get settingsForEvent() {
         const eventSettingProperties = [
+            "model",
             "n",
             "size",
+            "quality",
+            "style",
         ];
         return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
     }

package/model-provider/openai/OpenAISpeechModel.cjs

@@ -0,0 +1,93 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAISpeechModel = exports.calculateOpenAISpeechCostInMillicents = exports.OPENAI_SPEECH_MODELS = void 0;
+const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
+const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndThrottle.cjs");
+const postToApi_js_1 = require("../../core/api/postToApi.cjs");
+const OpenAIApiConfiguration_js_1 = require("./OpenAIApiConfiguration.cjs");
+const OpenAIError_js_1 = require("./OpenAIError.cjs");
+/**
+ * @see https://openai.com/pricing
+ */
+exports.OPENAI_SPEECH_MODELS = {
+    "tts-1": {
+        costInMillicentsPerCharacter: 1.5, // = 1500 / 1000,
+    },
+    "tts-1-hd": {
+        costInMillicentsPerCharacter: 3, // = 3000 / 1000
+    },
+};
+const calculateOpenAISpeechCostInMillicents = ({ model, input, }) => {
+    if (!exports.OPENAI_SPEECH_MODELS[model]) {
+        return null;
+    }
+    return (input.length * exports.OPENAI_SPEECH_MODELS[model].costInMillicentsPerCharacter);
+};
+exports.calculateOpenAISpeechCostInMillicents = calculateOpenAISpeechCostInMillicents;
+/**
+ * Synthesize speech using the OpenAI API.
+ *
+ * @see https://platform.openai.com/docs/api-reference/audio/createSpeech
+ */
+class OpenAISpeechModel extends AbstractModel_js_1.AbstractModel {
+    constructor(settings) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "openai"
+        });
+    }
+    get voice() {
+        return this.settings.voice;
+    }
+    get modelName() {
+        return this.settings.model;
+    }
+    async callAPI(text, options) {
+        return (0, callWithRetryAndThrottle_js_1.callWithRetryAndThrottle)({
+            retry: this.settings.api?.retry,
+            throttle: this.settings.api?.throttle,
+            call: async () => callOpenAITextToSpeechAPI({
+                ...this.settings,
+                abortSignal: options?.run?.abortSignal,
+                text,
+            }),
+        });
+    }
+    get settingsForEvent() {
+        return {
+            voice: this.settings.voice,
+            speed: this.settings.speed,
+            model: this.settings.model,
+            responseFormat: this.settings.responseFormat,
+        };
+    }
+    doGenerateSpeechStandard(text, options) {
+        return this.callAPI(text, options);
+    }
+    withSettings(additionalSettings) {
+        return new OpenAISpeechModel({
+            ...this.settings,
+            ...additionalSettings,
+        });
+    }
+}
+exports.OpenAISpeechModel = OpenAISpeechModel;
+async function callOpenAITextToSpeechAPI({ api = new OpenAIApiConfiguration_js_1.OpenAIApiConfiguration(), abortSignal, text, voice, model, speed, responseFormat, }) {
+    return (0, postToApi_js_1.postJsonToApi)({
+        url: api.assembleUrl(`/audio/speech`),
+        headers: api.headers,
+        body: {
+            input: text,
+            voice,
+            speed,
+            model,
+            response_format: responseFormat,
+        },
+        failedResponseHandler: OpenAIError_js_1.failedOpenAICallResponseHandler,
+        successfulResponseHandler: (0, postToApi_js_1.createAudioMpegResponseHandler)(),
+        abortSignal,
+    });
+}

package/model-provider/openai/OpenAISpeechModel.d.ts

@@ -0,0 +1,52 @@
+/// <reference types="node" />
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { ApiConfiguration } from "../../core/api/ApiConfiguration.js";
+import { FunctionOptions } from "../../core/FunctionOptions.js";
+import { SpeechGenerationModel, SpeechGenerationModelSettings } from "../../model-function/generate-speech/SpeechGenerationModel.js";
+/**
+ * @see https://openai.com/pricing
+ */
+export declare const OPENAI_SPEECH_MODELS: {
+    "tts-1": {
+        costInMillicentsPerCharacter: number;
+    };
+    "tts-1-hd": {
+        costInMillicentsPerCharacter: number;
+    };
+};
+export type OpenAISpeechModelType = keyof typeof OPENAI_SPEECH_MODELS;
+export declare const calculateOpenAISpeechCostInMillicents: ({ model, input, }: {
+    model: OpenAISpeechModelType;
+    input: string;
+}) => number | null;
+export type OpenAISpeechVoice = "alloy" | "echo" | "fable" | "onyx" | "nova" | "shimmer";
+type OpenAISpeechModelResponseFormat = "mp3" | "opus" | "aac" | "flac";
+export interface OpenAISpeechModelSettings extends SpeechGenerationModelSettings {
+    api?: ApiConfiguration;
+    voice: OpenAISpeechVoice;
+    model: OpenAISpeechModelType;
+    /**
+     * The speed of the generated audio. Select a value from 0.25 to 4.0. 1.0 is the default.
+     */
+    speed?: number;
+    /**
+     * Defaults to mp3.
+     */
+    responseFormat?: OpenAISpeechModelResponseFormat;
+}
+/**
+ * Synthesize speech using the OpenAI API.
+ *
+ * @see https://platform.openai.com/docs/api-reference/audio/createSpeech
+ */
+export declare class OpenAISpeechModel extends AbstractModel<OpenAISpeechModelSettings> implements SpeechGenerationModel<OpenAISpeechModelSettings> {
+    constructor(settings: OpenAISpeechModelSettings);
+    readonly provider: "openai";
+    get voice(): OpenAISpeechVoice;
+    get modelName(): "tts-1" | "tts-1-hd";
+    private callAPI;
+    get settingsForEvent(): Partial<OpenAISpeechModelSettings>;
+    doGenerateSpeechStandard(text: string, options?: FunctionOptions): Promise<Buffer>;
+    withSettings(additionalSettings: Partial<OpenAISpeechModelSettings>): this;
+}
+export {};

package/model-provider/openai/OpenAISpeechModel.js

@@ -0,0 +1,88 @@
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottle.js";
+import { createAudioMpegResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
+import { OpenAIApiConfiguration } from "./OpenAIApiConfiguration.js";
+import { failedOpenAICallResponseHandler } from "./OpenAIError.js";
+/**
+ * @see https://openai.com/pricing
+ */
+export const OPENAI_SPEECH_MODELS = {
+    "tts-1": {
+        costInMillicentsPerCharacter: 1.5, // = 1500 / 1000,
+    },
+    "tts-1-hd": {
+        costInMillicentsPerCharacter: 3, // = 3000 / 1000
+    },
+};
+export const calculateOpenAISpeechCostInMillicents = ({ model, input, }) => {
+    if (!OPENAI_SPEECH_MODELS[model]) {
+        return null;
+    }
+    return (input.length * OPENAI_SPEECH_MODELS[model].costInMillicentsPerCharacter);
+};
+/**
+ * Synthesize speech using the OpenAI API.
+ *
+ * @see https://platform.openai.com/docs/api-reference/audio/createSpeech
+ */
+export class OpenAISpeechModel extends AbstractModel {
+    constructor(settings) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "openai"
+        });
+    }
+    get voice() {
+        return this.settings.voice;
+    }
+    get modelName() {
+        return this.settings.model;
+    }
+    async callAPI(text, options) {
+        return callWithRetryAndThrottle({
+            retry: this.settings.api?.retry,
+            throttle: this.settings.api?.throttle,
+            call: async () => callOpenAITextToSpeechAPI({
+                ...this.settings,
+                abortSignal: options?.run?.abortSignal,
+                text,
+            }),
+        });
+    }
+    get settingsForEvent() {
+        return {
+            voice: this.settings.voice,
+            speed: this.settings.speed,
+            model: this.settings.model,
+            responseFormat: this.settings.responseFormat,
+        };
+    }
+    doGenerateSpeechStandard(text, options) {
+        return this.callAPI(text, options);
+    }
+    withSettings(additionalSettings) {
+        return new OpenAISpeechModel({
+            ...this.settings,
+            ...additionalSettings,
+        });
+    }
+}
+async function callOpenAITextToSpeechAPI({ api = new OpenAIApiConfiguration(), abortSignal, text, voice, model, speed, responseFormat, }) {
+    return postJsonToApi({
+        url: api.assembleUrl(`/audio/speech`),
+        headers: api.headers,
+        body: {
+            input: text,
+            voice,
+            speed,
+            model,
+            response_format: responseFormat,
+        },
+        failedResponseHandler: failedOpenAICallResponseHandler,
+        successfulResponseHandler: createAudioMpegResponseHandler(),
+        abortSignal,
+    });
+}

package/model-provider/openai/TikTokenTokenizer.cjs

@@ -75,12 +75,15 @@ function getTiktokenBPE(model) {
         case "gpt-3.5-turbo":
         case "gpt-3.5-turbo-0301":
         case "gpt-3.5-turbo-0613":
+        case "gpt-3.5-turbo-1106":
         case "gpt-3.5-turbo-16k":
         case "gpt-3.5-turbo-16k-0613":
         case "gpt-3.5-turbo-instruct":
         case "gpt-4":
         case "gpt-4-0314":
         case "gpt-4-0613":
+        case "gpt-4-1106-preview":
+        case "gpt-4-vision-preview":
         case "gpt-4-32k":
         case "gpt-4-32k-0314":
         case "gpt-4-32k-0613":

package/model-provider/openai/TikTokenTokenizer.js

@@ -68,12 +68,15 @@ function getTiktokenBPE(model) {
         case "gpt-3.5-turbo":
         case "gpt-3.5-turbo-0301":
         case "gpt-3.5-turbo-0613":
+        case "gpt-3.5-turbo-1106":
         case "gpt-3.5-turbo-16k":
         case "gpt-3.5-turbo-16k-0613":
         case "gpt-3.5-turbo-instruct":
         case "gpt-4":
         case "gpt-4-0314":
         case "gpt-4-0613":
+        case "gpt-4-1106-preview":
+        case "gpt-4-vision-preview":
         case "gpt-4-32k":
         case "gpt-4-32k-0314":
         case "gpt-4-32k-0613":

package/model-provider/openai/chat/OpenAIChatModel.cjs

@@ -40,6 +40,16 @@ exports.OPENAI_CHAT_MODELS = {
         promptTokenCostInMillicents: 3,
         completionTokenCostInMillicents: 6,
     },
+    "gpt-4-1106-preview": {
+        contextWindowSize: 128000,
+        promptTokenCostInMillicents: 1,
+        completionTokenCostInMillicents: 3,
+    },
+    "gpt-4-vision-preview": {
+        contextWindowSize: 128000,
+        promptTokenCostInMillicents: 1,
+        completionTokenCostInMillicents: 3,
+    },
     "gpt-4-32k": {
         contextWindowSize: 32768,
         promptTokenCostInMillicents: 6,
@@ -59,8 +69,13 @@ exports.OPENAI_CHAT_MODELS = {
         contextWindowSize: 4096,
         promptTokenCostInMillicents: 0.15,
         completionTokenCostInMillicents: 0.2,
-        fineTunedPromptTokenCostInMillicents: 1.2,
-        fineTunedCompletionTokenCostInMillicents: 1.6,
+        fineTunedPromptTokenCostInMillicents: 0.3,
+        fineTunedCompletionTokenCostInMillicents: 0.6,
+    },
+    "gpt-3.5-turbo-1106": {
+        contextWindowSize: 16385,
+        promptTokenCostInMillicents: 0.1,
+        completionTokenCostInMillicents: 0.2,
     },
     "gpt-3.5-turbo-0301": {
         contextWindowSize: 4096,

package/model-provider/openai/chat/OpenAIChatModel.d.ts

@@ -28,6 +28,16 @@ export declare const OPENAI_CHAT_MODELS: {
         promptTokenCostInMillicents: number;
         completionTokenCostInMillicents: number;
     };
+    "gpt-4-1106-preview": {
+        contextWindowSize: number;
+        promptTokenCostInMillicents: number;
+        completionTokenCostInMillicents: number;
+    };
+    "gpt-4-vision-preview": {
+        contextWindowSize: number;
+        promptTokenCostInMillicents: number;
+        completionTokenCostInMillicents: number;
+    };
     "gpt-4-32k": {
         contextWindowSize: number;
         promptTokenCostInMillicents: number;
@@ -50,6 +60,11 @@ export declare const OPENAI_CHAT_MODELS: {
         fineTunedPromptTokenCostInMillicents: number;
         fineTunedCompletionTokenCostInMillicents: number;
     };
+    "gpt-3.5-turbo-1106": {
+        contextWindowSize: number;
+        promptTokenCostInMillicents: number;
+        completionTokenCostInMillicents: number;
+    };
     "gpt-3.5-turbo-0301": {
         contextWindowSize: number;
         promptTokenCostInMillicents: number;
@@ -160,12 +175,11 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
             object: "chat.completion";
             usage: {
                 prompt_tokens: number;
-                completion_tokens: number;
                 total_tokens: number;
+                completion_tokens: number;
             };
             model: string;
             id: string;
-            created: number;
             choices: {
                 message: {
                     content: string | null;
@@ -179,6 +193,7 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                 index: number;
                 logprobs?: any;
             }[];
+            created: number;
         };
         text: string;
         usage: {
@@ -200,12 +215,11 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
             object: "chat.completion";
             usage: {
                 prompt_tokens: number;
-                completion_tokens: number;
                 total_tokens: number;
+                completion_tokens: number;
             };
             model: string;
             id: string;
-            created: number;
             choices: {
                 message: {
                     content: string | null;
@@ -219,6 +233,7 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                 index: number;
                 logprobs?: any;
             }[];
+            created: number;
         };
         valueText: string;
         value: any;
@@ -234,12 +249,11 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
             object: "chat.completion";
             usage: {
                 prompt_tokens: number;
-                completion_tokens: number;
                 total_tokens: number;
+                completion_tokens: number;
             };
             model: string;
             id: string;
-            created: number;
             choices: {
                 message: {
                     content: string | null;
@@ -253,6 +267,7 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                 index: number;
                 logprobs?: any;
             }[];
+            created: number;
         };
         structureAndText: {
             structure: null;
@@ -270,12 +285,11 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
             object: "chat.completion";
             usage: {
                 prompt_tokens: number;
-                completion_tokens: number;
                 total_tokens: number;
+                completion_tokens: number;
             };
             model: string;
             id: string;
-            created: number;
             choices: {
                 message: {
                     content: string | null;
@@ -289,6 +303,7 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                 index: number;
                 logprobs?: any;
             }[];
+            created: number;
         };
         structureAndText: {
             structure: string;
@@ -386,23 +401,22 @@ declare const openAIChatResponseSchema: z.ZodObject<{
         total_tokens: z.ZodNumber;
     }, "strip", z.ZodTypeAny, {
         prompt_tokens: number;
-        completion_tokens: number;
         total_tokens: number;
+        completion_tokens: number;
     }, {
         prompt_tokens: number;
-        completion_tokens: number;
         total_tokens: number;
+        completion_tokens: number;
     }>;
 }, "strip", z.ZodTypeAny, {
     object: "chat.completion";
     usage: {
         prompt_tokens: number;
-        completion_tokens: number;
         total_tokens: number;
+        completion_tokens: number;
     };
     model: string;
     id: string;
-    created: number;
     choices: {
         message: {
             content: string | null;
@@ -416,16 +430,16 @@ declare const openAIChatResponseSchema: z.ZodObject<{
         index: number;
         logprobs?: any;
     }[];
+    created: number;
 }, {
     object: "chat.completion";
     usage: {
         prompt_tokens: number;
-        completion_tokens: number;
         total_tokens: number;
+        completion_tokens: number;
     };
     model: string;
     id: string;
-    created: number;
     choices: {
         message: {
             content: string | null;
@@ -439,6 +453,7 @@ declare const openAIChatResponseSchema: z.ZodObject<{
         index: number;
         logprobs?: any;
     }[];
+    created: number;
 }>;
 export type OpenAIChatResponse = z.infer<typeof openAIChatResponseSchema>;
 export type OpenAIChatResponseFormatType<T> = {
@@ -455,12 +470,11 @@ export declare const OpenAIChatResponseFormat: {
             object: "chat.completion";
             usage: {
                 prompt_tokens: number;
-                completion_tokens: number;
                 total_tokens: number;
+                completion_tokens: number;
             };
             model: string;
             id: string;
-            created: number;
             choices: {
                 message: {
                     content: string | null;
@@ -474,6 +488,7 @@ export declare const OpenAIChatResponseFormat: {
                 index: number;
                 logprobs?: any;
             }[];
+            created: number;
         }>;
     };
     /**

package/model-provider/openai/chat/OpenAIChatModel.js

@@ -34,6 +34,16 @@ export const OPENAI_CHAT_MODELS = {
         promptTokenCostInMillicents: 3,
         completionTokenCostInMillicents: 6,
     },
+    "gpt-4-1106-preview": {
+        contextWindowSize: 128000,
+        promptTokenCostInMillicents: 1,
+        completionTokenCostInMillicents: 3,
+    },
+    "gpt-4-vision-preview": {
+        contextWindowSize: 128000,
+        promptTokenCostInMillicents: 1,
+        completionTokenCostInMillicents: 3,
+    },
     "gpt-4-32k": {
         contextWindowSize: 32768,
         promptTokenCostInMillicents: 6,
@@ -53,8 +63,13 @@ export const OPENAI_CHAT_MODELS = {
         contextWindowSize: 4096,
         promptTokenCostInMillicents: 0.15,
         completionTokenCostInMillicents: 0.2,
-        fineTunedPromptTokenCostInMillicents: 1.2,
-        fineTunedCompletionTokenCostInMillicents: 1.6,
+        fineTunedPromptTokenCostInMillicents: 0.3,
+        fineTunedCompletionTokenCostInMillicents: 0.6,
+    },
+    "gpt-3.5-turbo-1106": {
+        contextWindowSize: 16385,
+        promptTokenCostInMillicents: 0.1,
+        completionTokenCostInMillicents: 0.2,
     },
     "gpt-3.5-turbo-0301": {
         contextWindowSize: 4096,

package/model-provider/openai/chat/OpenAIChatStreamIterable.cjs

@@ -17,7 +17,7 @@ const chatResponseStreamEventSchema = zod_1.z.object({
             })
                 .optional(),
         }),
-        finish_reason: zod_1.z.enum(["stop", "length"]).nullable(),
+        finish_reason: zod_1.z.enum(["stop", "length"]).nullable().optional(),
         index: zod_1.z.number(),
     })),
     created: zod_1.z.number(),

package/model-provider/openai/chat/OpenAIChatStreamIterable.js

@@ -14,7 +14,7 @@ const chatResponseStreamEventSchema = z.object({
             })
                 .optional(),
         }),
-        finish_reason: z.enum(["stop", "length"]).nullable(),
+        finish_reason: z.enum(["stop", "length"]).nullable().optional(),
         index: z.number(),
     })),
     created: z.number(),

package/model-provider/openai/index.cjs

@@ -17,12 +17,13 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.OpenAIError = void 0;
 __exportStar(require("./AzureOpenAIApiConfiguration.cjs"), exports);
 __exportStar(require("./OpenAIApiConfiguration.cjs"), exports);
+__exportStar(require("./OpenAICompletionModel.cjs"), exports);
 __exportStar(require("./OpenAICostCalculator.cjs"), exports);
 var OpenAIError_js_1 = require("./OpenAIError.cjs");
 Object.defineProperty(exports, "OpenAIError", { enumerable: true, get: function () { return OpenAIError_js_1.OpenAIError; } });
 __exportStar(require("./OpenAIImageGenerationModel.cjs"), exports);
+__exportStar(require("./OpenAISpeechModel.cjs"), exports);
 __exportStar(require("./OpenAITextEmbeddingModel.cjs"), exports);
-__exportStar(require("./OpenAICompletionModel.cjs"), exports);
 __exportStar(require("./OpenAITranscriptionModel.cjs"), exports);
 __exportStar(require("./TikTokenTokenizer.cjs"), exports);
 __exportStar(require("./chat/OpenAIChatMessage.cjs"), exports);

package/model-provider/openai/index.d.ts

@@ -1,10 +1,11 @@
 export * from "./AzureOpenAIApiConfiguration.js";
 export * from "./OpenAIApiConfiguration.js";
+export * from "./OpenAICompletionModel.js";
 export * from "./OpenAICostCalculator.js";
 export { OpenAIError, OpenAIErrorData } from "./OpenAIError.js";
 export * from "./OpenAIImageGenerationModel.js";
+export * from "./OpenAISpeechModel.js";
 export * from "./OpenAITextEmbeddingModel.js";
-export * from "./OpenAICompletionModel.js";
 export * from "./OpenAITranscriptionModel.js";
 export * from "./TikTokenTokenizer.js";
 export * from "./chat/OpenAIChatMessage.js";

package/model-provider/openai/index.js

@@ -1,10 +1,11 @@
 export * from "./AzureOpenAIApiConfiguration.js";
 export * from "./OpenAIApiConfiguration.js";
+export * from "./OpenAICompletionModel.js";
 export * from "./OpenAICostCalculator.js";
 export { OpenAIError } from "./OpenAIError.js";
 export * from "./OpenAIImageGenerationModel.js";
+export * from "./OpenAISpeechModel.js";
 export * from "./OpenAITextEmbeddingModel.js";
-export * from "./OpenAICompletionModel.js";
 export * from "./OpenAITranscriptionModel.js";
 export * from "./TikTokenTokenizer.js";
 export * from "./chat/OpenAIChatMessage.js";

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "Build multimodal applications, chatbots, and agents with JavaScript and TypeScript.",
-  "version": "0.60.0",
+  "version": "0.62.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [