modelfusion 0.61.0 → 0.62.0

package/README.md

@@ -14,6 +14,7 @@
 
 **ModelFusion** is a TypeScript library for building AI applications, chatbots, and agents.
 
+- **Vendor-neutral**: ModelFusion is a non-commercial open source project that is community-driven. You can use it with any supported vendor.
 - **Multimodal**: ModelFusion supports a wide range of models including text generation, image generation, text-to-speech, speech-to-text, and embedding models.
 - **Streaming**: ModelFusion supports streaming for many generation models, e.g. text streaming, structure streaming, and full duplex speech streaming.
 - **Utility functions**: ModelFusion provides functionality for tools and tool usage, vector indices, and guards functions.
@@ -109,7 +110,7 @@ const speech = await generateSpeech(
 );
 ```
 
-Providers: [Eleven Labs](https://modelfusion.dev/integration/model-provider/elevenlabs), [LMNT](https://modelfusion.dev/integration/model-provider/lmnt)
+Providers: [Eleven Labs](https://modelfusion.dev/integration/model-provider/elevenlabs), [LMNT](https://modelfusion.dev/integration/model-provider/lmnt), [OpenAI](https://modelfusion.dev/integration/model-provider/openai)
 
 #### streamSpeech
 

package/model-provider/openai/OpenAICompletionModel.d.ts

@@ -150,18 +150,18 @@ export declare class OpenAICompletionModel extends AbstractModel<OpenAICompletio
             object: "text_completion";
             usage: {
                 prompt_tokens: number;
-                completion_tokens: number;
                 total_tokens: number;
+                completion_tokens: number;
             };
             model: string;
             id: string;
-            created: number;
             choices: {
                 text: string;
                 finish_reason: string;
                 index: number;
                 logprobs?: any;
             }[];
+            created: number;
         };
         text: string;
         usage: {
@@ -212,45 +212,45 @@ declare const OpenAICompletionResponseSchema: z.ZodObject<{
         total_tokens: z.ZodNumber;
     }, "strip", z.ZodTypeAny, {
         prompt_tokens: number;
-        completion_tokens: number;
         total_tokens: number;
+        completion_tokens: number;
     }, {
         prompt_tokens: number;
-        completion_tokens: number;
         total_tokens: number;
+        completion_tokens: number;
     }>;
 }, "strip", z.ZodTypeAny, {
     object: "text_completion";
     usage: {
         prompt_tokens: number;
-        completion_tokens: number;
         total_tokens: number;
+        completion_tokens: number;
     };
     model: string;
     id: string;
-    created: number;
     choices: {
         text: string;
         finish_reason: string;
         index: number;
         logprobs?: any;
     }[];
+    created: number;
 }, {
     object: "text_completion";
     usage: {
         prompt_tokens: number;
-        completion_tokens: number;
         total_tokens: number;
+        completion_tokens: number;
     };
     model: string;
     id: string;
-    created: number;
     choices: {
         text: string;
         finish_reason: string;
         index: number;
         logprobs?: any;
     }[];
+    created: number;
 }>;
 export type OpenAICompletionResponse = z.infer<typeof OpenAICompletionResponseSchema>;
 export type OpenAITextResponseFormatType<T> = {
@@ -267,18 +267,18 @@ export declare const OpenAITextResponseFormat: {
             object: "text_completion";
             usage: {
                 prompt_tokens: number;
-                completion_tokens: number;
                 total_tokens: number;
+                completion_tokens: number;
             };
             model: string;
             id: string;
-            created: number;
             choices: {
                 text: string;
                 finish_reason: string;
                 index: number;
                 logprobs?: any;
             }[];
+            created: number;
         }>;
     };
     /**

package/model-provider/openai/OpenAICostCalculator.cjs

@@ -6,6 +6,7 @@ const OpenAITextEmbeddingModel_js_1 = require("./OpenAITextEmbeddingModel.cjs");
 const OpenAICompletionModel_js_1 = require("./OpenAICompletionModel.cjs");
 const OpenAITranscriptionModel_js_1 = require("./OpenAITranscriptionModel.cjs");
 const OpenAIChatModel_js_1 = require("./chat/OpenAIChatModel.cjs");
+const OpenAISpeechModel_js_1 = require("./OpenAISpeechModel.cjs");
 class OpenAICostCalculator {
     constructor() {
         Object.defineProperty(this, "provider", {
@@ -68,6 +69,15 @@ class OpenAICostCalculator {
                         .response,
                 });
             }
+            case "generate-speech": {
+                if (model == null) {
+                    return null;
+                }
+                return (0, OpenAISpeechModel_js_1.calculateOpenAISpeechCostInMillicents)({
+                    model: model,
+                    input: call.input,
+                });
+            }
         }
         return null;
     }

package/model-provider/openai/OpenAICostCalculator.js

@@ -3,6 +3,7 @@ import { calculateOpenAIEmbeddingCostInMillicents, isOpenAIEmbeddingModel, } fro
 import { calculateOpenAICompletionCostInMillicents, isOpenAICompletionModel, } from "./OpenAICompletionModel.js";
 import { calculateOpenAITranscriptionCostInMillicents, } from "./OpenAITranscriptionModel.js";
 import { calculateOpenAIChatCostInMillicents, isOpenAIChatModel, } from "./chat/OpenAIChatModel.js";
+import { calculateOpenAISpeechCostInMillicents, } from "./OpenAISpeechModel.js";
 export class OpenAICostCalculator {
     constructor() {
         Object.defineProperty(this, "provider", {
@@ -65,6 +66,15 @@ export class OpenAICostCalculator {
                         .response,
                 });
             }
+            case "generate-speech": {
+                if (model == null) {
+                    return null;
+                }
+                return calculateOpenAISpeechCostInMillicents({
+                    model: model,
+                    input: call.input,
+                });
+            }
         }
         return null;
     }

package/model-provider/openai/OpenAIImageGenerationModel.d.ts

@@ -11,7 +11,7 @@ export declare const OPENAI_IMAGE_MODELS: {
         getCost(settings: OpenAIImageGenerationSettings): 2000 | 1800 | 1600 | null;
     };
     "dall-e-3": {
-        getCost(settings: OpenAIImageGenerationSettings): 4000 | 8000 | 12000 | null;
+        getCost(settings: OpenAIImageGenerationSettings): 8000 | 4000 | 12000 | null;
     };
 };
 /**

package/model-provider/openai/OpenAISpeechModel.cjs

@@ -0,0 +1,93 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAISpeechModel = exports.calculateOpenAISpeechCostInMillicents = exports.OPENAI_SPEECH_MODELS = void 0;
+const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
+const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndThrottle.cjs");
+const postToApi_js_1 = require("../../core/api/postToApi.cjs");
+const OpenAIApiConfiguration_js_1 = require("./OpenAIApiConfiguration.cjs");
+const OpenAIError_js_1 = require("./OpenAIError.cjs");
+/**
+ * @see https://openai.com/pricing
+ */
+exports.OPENAI_SPEECH_MODELS = {
+    "tts-1": {
+        costInMillicentsPerCharacter: 1.5, // = 1500 / 1000,
+    },
+    "tts-1-hd": {
+        costInMillicentsPerCharacter: 3, // = 3000 / 1000
+    },
+};
+const calculateOpenAISpeechCostInMillicents = ({ model, input, }) => {
+    if (!exports.OPENAI_SPEECH_MODELS[model]) {
+        return null;
+    }
+    return (input.length * exports.OPENAI_SPEECH_MODELS[model].costInMillicentsPerCharacter);
+};
+exports.calculateOpenAISpeechCostInMillicents = calculateOpenAISpeechCostInMillicents;
+/**
+ * Synthesize speech using the OpenAI API.
+ *
+ * @see https://platform.openai.com/docs/api-reference/audio/createSpeech
+ */
+class OpenAISpeechModel extends AbstractModel_js_1.AbstractModel {
+    constructor(settings) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "openai"
+        });
+    }
+    get voice() {
+        return this.settings.voice;
+    }
+    get modelName() {
+        return this.settings.model;
+    }
+    async callAPI(text, options) {
+        return (0, callWithRetryAndThrottle_js_1.callWithRetryAndThrottle)({
+            retry: this.settings.api?.retry,
+            throttle: this.settings.api?.throttle,
+            call: async () => callOpenAITextToSpeechAPI({
+                ...this.settings,
+                abortSignal: options?.run?.abortSignal,
+                text,
+            }),
+        });
+    }
+    get settingsForEvent() {
+        return {
+            voice: this.settings.voice,
+            speed: this.settings.speed,
+            model: this.settings.model,
+            responseFormat: this.settings.responseFormat,
+        };
+    }
+    doGenerateSpeechStandard(text, options) {
+        return this.callAPI(text, options);
+    }
+    withSettings(additionalSettings) {
+        return new OpenAISpeechModel({
+            ...this.settings,
+            ...additionalSettings,
+        });
+    }
+}
+exports.OpenAISpeechModel = OpenAISpeechModel;
+async function callOpenAITextToSpeechAPI({ api = new OpenAIApiConfiguration_js_1.OpenAIApiConfiguration(), abortSignal, text, voice, model, speed, responseFormat, }) {
+    return (0, postToApi_js_1.postJsonToApi)({
+        url: api.assembleUrl(`/audio/speech`),
+        headers: api.headers,
+        body: {
+            input: text,
+            voice,
+            speed,
+            model,
+            response_format: responseFormat,
+        },
+        failedResponseHandler: OpenAIError_js_1.failedOpenAICallResponseHandler,
+        successfulResponseHandler: (0, postToApi_js_1.createAudioMpegResponseHandler)(),
+        abortSignal,
+    });
+}

package/model-provider/openai/OpenAISpeechModel.d.ts

@@ -0,0 +1,52 @@
+/// <reference types="node" />
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { ApiConfiguration } from "../../core/api/ApiConfiguration.js";
+import { FunctionOptions } from "../../core/FunctionOptions.js";
+import { SpeechGenerationModel, SpeechGenerationModelSettings } from "../../model-function/generate-speech/SpeechGenerationModel.js";
+/**
+ * @see https://openai.com/pricing
+ */
+export declare const OPENAI_SPEECH_MODELS: {
+    "tts-1": {
+        costInMillicentsPerCharacter: number;
+    };
+    "tts-1-hd": {
+        costInMillicentsPerCharacter: number;
+    };
+};
+export type OpenAISpeechModelType = keyof typeof OPENAI_SPEECH_MODELS;
+export declare const calculateOpenAISpeechCostInMillicents: ({ model, input, }: {
+    model: OpenAISpeechModelType;
+    input: string;
+}) => number | null;
+export type OpenAISpeechVoice = "alloy" | "echo" | "fable" | "onyx" | "nova" | "shimmer";
+type OpenAISpeechModelResponseFormat = "mp3" | "opus" | "aac" | "flac";
+export interface OpenAISpeechModelSettings extends SpeechGenerationModelSettings {
+    api?: ApiConfiguration;
+    voice: OpenAISpeechVoice;
+    model: OpenAISpeechModelType;
+    /**
+     * The speed of the generated audio. Select a value from 0.25 to 4.0. 1.0 is the default.
+     */
+    speed?: number;
+    /**
+     * Defaults to mp3.
+     */
+    responseFormat?: OpenAISpeechModelResponseFormat;
+}
+/**
+ * Synthesize speech using the OpenAI API.
+ *
+ * @see https://platform.openai.com/docs/api-reference/audio/createSpeech
+ */
+export declare class OpenAISpeechModel extends AbstractModel<OpenAISpeechModelSettings> implements SpeechGenerationModel<OpenAISpeechModelSettings> {
+    constructor(settings: OpenAISpeechModelSettings);
+    readonly provider: "openai";
+    get voice(): OpenAISpeechVoice;
+    get modelName(): "tts-1" | "tts-1-hd";
+    private callAPI;
+    get settingsForEvent(): Partial<OpenAISpeechModelSettings>;
+    doGenerateSpeechStandard(text: string, options?: FunctionOptions): Promise<Buffer>;
+    withSettings(additionalSettings: Partial<OpenAISpeechModelSettings>): this;
+}
+export {};

package/model-provider/openai/OpenAISpeechModel.js

@@ -0,0 +1,88 @@
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottle.js";
+import { createAudioMpegResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
+import { OpenAIApiConfiguration } from "./OpenAIApiConfiguration.js";
+import { failedOpenAICallResponseHandler } from "./OpenAIError.js";
+/**
+ * @see https://openai.com/pricing
+ */
+export const OPENAI_SPEECH_MODELS = {
+    "tts-1": {
+        costInMillicentsPerCharacter: 1.5, // = 1500 / 1000,
+    },
+    "tts-1-hd": {
+        costInMillicentsPerCharacter: 3, // = 3000 / 1000
+    },
+};
+export const calculateOpenAISpeechCostInMillicents = ({ model, input, }) => {
+    if (!OPENAI_SPEECH_MODELS[model]) {
+        return null;
+    }
+    return (input.length * OPENAI_SPEECH_MODELS[model].costInMillicentsPerCharacter);
+};
+/**
+ * Synthesize speech using the OpenAI API.
+ *
+ * @see https://platform.openai.com/docs/api-reference/audio/createSpeech
+ */
+export class OpenAISpeechModel extends AbstractModel {
+    constructor(settings) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "openai"
+        });
+    }
+    get voice() {
+        return this.settings.voice;
+    }
+    get modelName() {
+        return this.settings.model;
+    }
+    async callAPI(text, options) {
+        return callWithRetryAndThrottle({
+            retry: this.settings.api?.retry,
+            throttle: this.settings.api?.throttle,
+            call: async () => callOpenAITextToSpeechAPI({
+                ...this.settings,
+                abortSignal: options?.run?.abortSignal,
+                text,
+            }),
+        });
+    }
+    get settingsForEvent() {
+        return {
+            voice: this.settings.voice,
+            speed: this.settings.speed,
+            model: this.settings.model,
+            responseFormat: this.settings.responseFormat,
+        };
+    }
+    doGenerateSpeechStandard(text, options) {
+        return this.callAPI(text, options);
+    }
+    withSettings(additionalSettings) {
+        return new OpenAISpeechModel({
+            ...this.settings,
+            ...additionalSettings,
+        });
+    }
+}
+async function callOpenAITextToSpeechAPI({ api = new OpenAIApiConfiguration(), abortSignal, text, voice, model, speed, responseFormat, }) {
+    return postJsonToApi({
+        url: api.assembleUrl(`/audio/speech`),
+        headers: api.headers,
+        body: {
+            input: text,
+            voice,
+            speed,
+            model,
+            response_format: responseFormat,
+        },
+        failedResponseHandler: failedOpenAICallResponseHandler,
+        successfulResponseHandler: createAudioMpegResponseHandler(),
+        abortSignal,
+    });
+}

package/model-provider/openai/chat/OpenAIChatModel.d.ts

@@ -175,12 +175,11 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
             object: "chat.completion";
             usage: {
                 prompt_tokens: number;
-                completion_tokens: number;
                 total_tokens: number;
+                completion_tokens: number;
             };
             model: string;
             id: string;
-            created: number;
             choices: {
                 message: {
                     content: string | null;
@@ -194,6 +193,7 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                 index: number;
                 logprobs?: any;
             }[];
+            created: number;
         };
         text: string;
         usage: {
@@ -215,12 +215,11 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
             object: "chat.completion";
             usage: {
                 prompt_tokens: number;
-                completion_tokens: number;
                 total_tokens: number;
+                completion_tokens: number;
             };
             model: string;
             id: string;
-            created: number;
             choices: {
                 message: {
                     content: string | null;
@@ -234,6 +233,7 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                 index: number;
                 logprobs?: any;
             }[];
+            created: number;
         };
         valueText: string;
         value: any;
@@ -249,12 +249,11 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
             object: "chat.completion";
             usage: {
                 prompt_tokens: number;
-                completion_tokens: number;
                 total_tokens: number;
+                completion_tokens: number;
             };
             model: string;
             id: string;
-            created: number;
             choices: {
                 message: {
                     content: string | null;
@@ -268,6 +267,7 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                 index: number;
                 logprobs?: any;
             }[];
+            created: number;
         };
         structureAndText: {
             structure: null;
@@ -285,12 +285,11 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
             object: "chat.completion";
             usage: {
                 prompt_tokens: number;
-                completion_tokens: number;
                 total_tokens: number;
+                completion_tokens: number;
             };
             model: string;
             id: string;
-            created: number;
             choices: {
                 message: {
                     content: string | null;
@@ -304,6 +303,7 @@ export declare class OpenAIChatModel extends AbstractModel<OpenAIChatSettings> i
                 index: number;
                 logprobs?: any;
             }[];
+            created: number;
         };
         structureAndText: {
             structure: string;
@@ -401,23 +401,22 @@ declare const openAIChatResponseSchema: z.ZodObject<{
         total_tokens: z.ZodNumber;
     }, "strip", z.ZodTypeAny, {
         prompt_tokens: number;
-        completion_tokens: number;
         total_tokens: number;
+        completion_tokens: number;
     }, {
         prompt_tokens: number;
-        completion_tokens: number;
         total_tokens: number;
+        completion_tokens: number;
     }>;
 }, "strip", z.ZodTypeAny, {
     object: "chat.completion";
     usage: {
         prompt_tokens: number;
-        completion_tokens: number;
         total_tokens: number;
+        completion_tokens: number;
     };
     model: string;
     id: string;
-    created: number;
     choices: {
         message: {
             content: string | null;
@@ -431,16 +430,16 @@ declare const openAIChatResponseSchema: z.ZodObject<{
         index: number;
         logprobs?: any;
     }[];
+    created: number;
 }, {
     object: "chat.completion";
     usage: {
         prompt_tokens: number;
-        completion_tokens: number;
         total_tokens: number;
+        completion_tokens: number;
     };
     model: string;
     id: string;
-    created: number;
     choices: {
         message: {
             content: string | null;
@@ -454,6 +453,7 @@ declare const openAIChatResponseSchema: z.ZodObject<{
         index: number;
         logprobs?: any;
     }[];
+    created: number;
 }>;
 export type OpenAIChatResponse = z.infer<typeof openAIChatResponseSchema>;
 export type OpenAIChatResponseFormatType<T> = {
@@ -470,12 +470,11 @@ export declare const OpenAIChatResponseFormat: {
             object: "chat.completion";
             usage: {
                 prompt_tokens: number;
-                completion_tokens: number;
                 total_tokens: number;
+                completion_tokens: number;
             };
             model: string;
             id: string;
-            created: number;
             choices: {
                 message: {
                     content: string | null;
@@ -489,6 +488,7 @@ export declare const OpenAIChatResponseFormat: {
                 index: number;
                 logprobs?: any;
             }[];
+            created: number;
         }>;
     };
     /**

package/model-provider/openai/index.cjs

@@ -17,12 +17,13 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.OpenAIError = void 0;
 __exportStar(require("./AzureOpenAIApiConfiguration.cjs"), exports);
 __exportStar(require("./OpenAIApiConfiguration.cjs"), exports);
+__exportStar(require("./OpenAICompletionModel.cjs"), exports);
 __exportStar(require("./OpenAICostCalculator.cjs"), exports);
 var OpenAIError_js_1 = require("./OpenAIError.cjs");
 Object.defineProperty(exports, "OpenAIError", { enumerable: true, get: function () { return OpenAIError_js_1.OpenAIError; } });
 __exportStar(require("./OpenAIImageGenerationModel.cjs"), exports);
+__exportStar(require("./OpenAISpeechModel.cjs"), exports);
 __exportStar(require("./OpenAITextEmbeddingModel.cjs"), exports);
-__exportStar(require("./OpenAICompletionModel.cjs"), exports);
 __exportStar(require("./OpenAITranscriptionModel.cjs"), exports);
 __exportStar(require("./TikTokenTokenizer.cjs"), exports);
 __exportStar(require("./chat/OpenAIChatMessage.cjs"), exports);

package/model-provider/openai/index.d.ts

@@ -1,10 +1,11 @@
 export * from "./AzureOpenAIApiConfiguration.js";
 export * from "./OpenAIApiConfiguration.js";
+export * from "./OpenAICompletionModel.js";
 export * from "./OpenAICostCalculator.js";
 export { OpenAIError, OpenAIErrorData } from "./OpenAIError.js";
 export * from "./OpenAIImageGenerationModel.js";
+export * from "./OpenAISpeechModel.js";
 export * from "./OpenAITextEmbeddingModel.js";
-export * from "./OpenAICompletionModel.js";
 export * from "./OpenAITranscriptionModel.js";
 export * from "./TikTokenTokenizer.js";
 export * from "./chat/OpenAIChatMessage.js";

package/model-provider/openai/index.js

@@ -1,10 +1,11 @@
 export * from "./AzureOpenAIApiConfiguration.js";
 export * from "./OpenAIApiConfiguration.js";
+export * from "./OpenAICompletionModel.js";
 export * from "./OpenAICostCalculator.js";
 export { OpenAIError } from "./OpenAIError.js";
 export * from "./OpenAIImageGenerationModel.js";
+export * from "./OpenAISpeechModel.js";
 export * from "./OpenAITextEmbeddingModel.js";
-export * from "./OpenAICompletionModel.js";
 export * from "./OpenAITranscriptionModel.js";
 export * from "./TikTokenTokenizer.js";
 export * from "./chat/OpenAIChatMessage.js";

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "modelfusion",
   "description": "Build multimodal applications, chatbots, and agents with JavaScript and TypeScript.",
-  "version": "0.61.0",
+  "version": "0.62.0",
   "author": "Lars Grammel",
   "license": "MIT",
   "keywords": [