modelfusion 0.0.44

package/model-provider/openai/OpenAITranscriptionModel.js

@@ -0,0 +1,187 @@
+import z from "zod";
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { callWithRetryAndThrottle } from "../../util/api/callWithRetryAndThrottle.js";
+import { createJsonResponseHandler, createTextResponseHandler, postToApi, } from "../../util/api/postToApi.js";
+import { failedOpenAICallResponseHandler } from "./OpenAIError.js";
+/**
+ * @see https://openai.com/pricing
+ */
+export const OPENAI_TRANSCRIPTION_MODELS = {
+    "whisper-1": {
+        costInMillicentsPerSecond: 10, // = 600 / 60,
+    },
+};
+export const calculateOpenAITranscriptionCostInMillicents = ({ model, response, }) => {
+    if (model !== "whisper-1") {
+        return null;
+    }
+    const durationInSeconds = response.duration;
+    return (Math.ceil(durationInSeconds) *
+        OPENAI_TRANSCRIPTION_MODELS[model].costInMillicentsPerSecond);
+};
+/**
+ * Create a transcription model that calls the OpenAI transcription API.
+ *
+ * @see https://platform.openai.com/docs/api-reference/audio/create
+ *
+ * @example
+ * const data = await fs.promises.readFile("data/test.mp3");
+ *
+ * const { transcription } = await transcribe(
+ *   new OpenAITranscriptionModel({ model: "whisper-1" }),
+ *   {
+ *     type: "mp3",
+ *     data,
+ *   }
+ * );
+ */
+export class OpenAITranscriptionModel extends AbstractModel {
+    constructor(settings) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "openai"
+        });
+    }
+    get modelName() {
+        return this.settings.model;
+    }
+    generateTranscriptionResponse(data, options) {
+        return this.callAPI(data, {
+            responseFormat: OpenAITranscriptionResponseFormat.verboseJson,
+            functionId: options?.functionId,
+            settings: options?.settings,
+            run: options?.run,
+        });
+    }
+    extractTranscriptionText(response) {
+        return response.text;
+    }
+    get apiKey() {
+        const apiKey = this.settings.apiKey ?? process.env.OPENAI_API_KEY;
+        if (apiKey == null) {
+            throw new Error(`OpenAI API key is missing. Pass it as an argument to the constructor or set it as an environment variable named OPENAI_API_KEY.`);
+        }
+        return apiKey;
+    }
+    async callAPI(data, options) {
+        const run = options?.run;
+        const settings = options?.settings;
+        const responseFormat = options?.responseFormat;
+        const callSettings = Object.assign({
+            apiKey: this.apiKey,
+        }, this.settings, settings, {
+            abortSignal: run?.abortSignal,
+            file: {
+                name: `audio.${data.type}`,
+                data: data.data,
+            },
+            responseFormat,
+        });
+        return callWithRetryAndThrottle({
+            retry: this.settings.retry,
+            throttle: this.settings.throttle,
+            call: async () => callOpenAITranscriptionAPI(callSettings),
+        });
+    }
+    withSettings(additionalSettings) {
+        return new OpenAITranscriptionModel(Object.assign({}, this.settings, additionalSettings));
+    }
+}
+/**
+ * Call the OpenAI Transcription API to generate a transcription from an audio file.
+ *
+ * @see https://platform.openai.com/docs/api-reference/audio/create
+ *
+ * @example
+ * const transcriptionResponse = await callOpenAITranscriptionAPI({
+ *   apiKey: openAiApiKey,
+ *   model: "whisper-1",
+ *   file: {
+ *     name: "audio.mp3",
+ *     data: fileData, // Buffer
+ *   },
+ *   responseFormat: callOpenAITranscriptionAPI.responseFormat.json,
+ * });
+ */
+async function callOpenAITranscriptionAPI({ baseUrl = "https://api.openai.com/v1", abortSignal, apiKey, model, file, prompt, responseFormat, temperature, language, }) {
+    const formData = new FormData();
+    formData.append("file", new Blob([file.data]), file.name);
+    formData.append("model", model);
+    if (prompt) {
+        formData.append("prompt", prompt);
+    }
+    if (responseFormat) {
+        formData.append("response_format", responseFormat.type);
+    }
+    if (temperature) {
+        formData.append("temperature", temperature.toString());
+    }
+    if (language) {
+        formData.append("language", language);
+    }
+    return postToApi({
+        url: `${baseUrl}/audio/transcriptions`,
+        apiKey,
+        contentType: null,
+        body: {
+            content: formData,
+            values: {
+                model,
+                prompt,
+                response_format: responseFormat,
+                temperature,
+                language,
+            },
+        },
+        failedResponseHandler: failedOpenAICallResponseHandler,
+        successfulResponseHandler: responseFormat.handler,
+        abortSignal,
+    });
+}
+const openAITranscriptionJsonSchema = z.object({
+    text: z.string(),
+});
+const openAITranscriptionVerboseJsonSchema = z.object({
+    task: z.literal("transcribe"),
+    language: z.string(),
+    duration: z.number(),
+    segments: z.array(z.object({
+        id: z.number(),
+        seek: z.number(),
+        start: z.number(),
+        end: z.number(),
+        text: z.string(),
+        tokens: z.array(z.number()),
+        temperature: z.number(),
+        avg_logprob: z.number(),
+        compression_ratio: z.number(),
+        no_speech_prob: z.number(),
+        transient: z.boolean().optional(),
+    })),
+    text: z.string(),
+});
+export const OpenAITranscriptionResponseFormat = {
+    json: {
+        type: "json",
+        handler: createJsonResponseHandler(openAITranscriptionJsonSchema),
+    },
+    verboseJson: {
+        type: "verbose_json",
+        handler: createJsonResponseHandler(openAITranscriptionVerboseJsonSchema),
+    },
+    text: {
+        type: "text",
+        handler: createTextResponseHandler(),
+    },
+    srt: {
+        type: "srt",
+        handler: createTextResponseHandler(),
+    },
+    vtt: {
+        type: "vtt",
+        handler: createTextResponseHandler(),
+    },
+};

package/model-provider/openai/TikTokenTokenizer.cjs

@@ -0,0 +1,86 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TikTokenTokenizer = void 0;
+const js_tiktoken_1 = require("js-tiktoken");
+const never_js_1 = require("../../util/never.cjs");
+/**
+ * TikToken tokenizer for OpenAI language models.
+ *
+ * @see https://github.com/openai/tiktoken
+ *
+ * @example
+ * const tokenizer = new TikTokenTokenizer({ model: "gpt-4" });
+ *
+ * const text = "At first, Nox didn't know what to do with the pup.";
+ *
+ * const tokenCount = await countTokens(tokenizer, text);
+ * const tokens = await tokenizer.tokenize(text);
+ * const tokensAndTokenTexts = await tokenizer.tokenizeWithTexts(text);
+ * const reconstructedText = await tokenizer.detokenize(tokens);
+ */
+class TikTokenTokenizer {
+    /**
+     * Get a TikToken tokenizer for a specific model or encoding.
+     */
+    constructor(options) {
+        Object.defineProperty(this, "tiktoken", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.tiktoken = (0, js_tiktoken_1.getEncoding)("model" in options
+            ? getEncodingNameForModel(options.model)
+            : options.encoding);
+    }
+    async tokenize(text) {
+        return this.tiktoken.encode(text);
+    }
+    async tokenizeWithTexts(text) {
+        const tokens = this.tiktoken.encode(text);
+        return {
+            tokens,
+            tokenTexts: tokens.map((token) => this.tiktoken.decode([token])),
+        };
+    }
+    async detokenize(tokens) {
+        return this.tiktoken.decode(tokens);
+    }
+}
+exports.TikTokenTokenizer = TikTokenTokenizer;
+// implemented here (instead of using js-tiktoken) to be able to quickly updated it
+// when new models are released
+function getEncodingNameForModel(model) {
+    switch (model) {
+        case "code-davinci-002":
+        case "text-davinci-002":
+        case "text-davinci-003": {
+            return "p50k_base";
+        }
+        case "ada":
+        case "babbage":
+        case "curie":
+        case "davinci":
+        case "text-ada-001":
+        case "text-babbage-001":
+        case "text-curie-001":
+        case "gpt-3.5-turbo":
+        case "gpt-3.5-turbo-0301":
+        case "gpt-3.5-turbo-0613":
+        case "gpt-3.5-turbo-16k":
+        case "gpt-3.5-turbo-16k-0613":
+        case "gpt-4":
+        case "gpt-4-0314":
+        case "gpt-4-0613":
+        case "gpt-4-32k":
+        case "gpt-4-32k-0314":
+        case "gpt-4-32k-0613":
+        case "text-embedding-ada-002": {
+            return "cl100k_base";
+        }
+        default: {
+            (0, never_js_1.never)(model);
+            throw new Error(`Unknown model: ${model}`);
+        }
+    }
+}

package/model-provider/openai/TikTokenTokenizer.d.ts

@@ -0,0 +1,35 @@
+import { TiktokenEncoding } from "js-tiktoken";
+import { FullTokenizer } from "../../model-function/tokenize-text/Tokenizer.js";
+import { OpenAIChatModelType, OpenAITextEmbeddingModelType, OpenAITextGenerationModelType } from "./index.js";
+/**
+ * TikToken tokenizer for OpenAI language models.
+ *
+ * @see https://github.com/openai/tiktoken
+ *
+ * @example
+ * const tokenizer = new TikTokenTokenizer({ model: "gpt-4" });
+ *
+ * const text = "At first, Nox didn't know what to do with the pup.";
+ *
+ * const tokenCount = await countTokens(tokenizer, text);
+ * const tokens = await tokenizer.tokenize(text);
+ * const tokensAndTokenTexts = await tokenizer.tokenizeWithTexts(text);
+ * const reconstructedText = await tokenizer.detokenize(tokens);
+ */
+export declare class TikTokenTokenizer implements FullTokenizer {
+    /**
+     * Get a TikToken tokenizer for a specific model or encoding.
+     */
+    constructor(options: {
+        model: OpenAIChatModelType | OpenAITextEmbeddingModelType | OpenAITextGenerationModelType;
+    } | {
+        encoding: TiktokenEncoding;
+    });
+    private readonly tiktoken;
+    tokenize(text: string): Promise<number[]>;
+    tokenizeWithTexts(text: string): Promise<{
+        tokens: number[];
+        tokenTexts: string[];
+    }>;
+    detokenize(tokens: number[]): Promise<string>;
+}

package/model-provider/openai/TikTokenTokenizer.js

@@ -0,0 +1,82 @@
+import { getEncoding } from "js-tiktoken";
+import { never } from "../../util/never.js";
+/**
+ * TikToken tokenizer for OpenAI language models.
+ *
+ * @see https://github.com/openai/tiktoken
+ *
+ * @example
+ * const tokenizer = new TikTokenTokenizer({ model: "gpt-4" });
+ *
+ * const text = "At first, Nox didn't know what to do with the pup.";
+ *
+ * const tokenCount = await countTokens(tokenizer, text);
+ * const tokens = await tokenizer.tokenize(text);
+ * const tokensAndTokenTexts = await tokenizer.tokenizeWithTexts(text);
+ * const reconstructedText = await tokenizer.detokenize(tokens);
+ */
+export class TikTokenTokenizer {
+    /**
+     * Get a TikToken tokenizer for a specific model or encoding.
+     */
+    constructor(options) {
+        Object.defineProperty(this, "tiktoken", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.tiktoken = getEncoding("model" in options
+            ? getEncodingNameForModel(options.model)
+            : options.encoding);
+    }
+    async tokenize(text) {
+        return this.tiktoken.encode(text);
+    }
+    async tokenizeWithTexts(text) {
+        const tokens = this.tiktoken.encode(text);
+        return {
+            tokens,
+            tokenTexts: tokens.map((token) => this.tiktoken.decode([token])),
+        };
+    }
+    async detokenize(tokens) {
+        return this.tiktoken.decode(tokens);
+    }
+}
+// implemented here (instead of using js-tiktoken) to be able to quickly updated it
+// when new models are released
+function getEncodingNameForModel(model) {
+    switch (model) {
+        case "code-davinci-002":
+        case "text-davinci-002":
+        case "text-davinci-003": {
+            return "p50k_base";
+        }
+        case "ada":
+        case "babbage":
+        case "curie":
+        case "davinci":
+        case "text-ada-001":
+        case "text-babbage-001":
+        case "text-curie-001":
+        case "gpt-3.5-turbo":
+        case "gpt-3.5-turbo-0301":
+        case "gpt-3.5-turbo-0613":
+        case "gpt-3.5-turbo-16k":
+        case "gpt-3.5-turbo-16k-0613":
+        case "gpt-4":
+        case "gpt-4-0314":
+        case "gpt-4-0613":
+        case "gpt-4-32k":
+        case "gpt-4-32k-0314":
+        case "gpt-4-32k-0613":
+        case "text-embedding-ada-002": {
+            return "cl100k_base";
+        }
+        default: {
+            never(model);
+            throw new Error(`Unknown model: ${model}`);
+        }
+    }
+}

package/model-provider/openai/chat/OpenAIChatMessage.cjs

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OpenAIChatMessage = void 0;
+exports.OpenAIChatMessage = {
+    system(content) {
+        return { role: "system", content };
+    },
+    user(content) {
+        return { role: "user", content };
+    },
+    assistant(content) {
+        return { role: "assistant", content };
+    },
+    functionCall(content, functionCall) {
+        return {
+            role: "assistant",
+            content,
+            function_call: functionCall,
+        };
+    },
+    functionResult(name, content) {
+        return { role: "function", name, content };
+    },
+};

package/model-provider/openai/chat/OpenAIChatMessage.d.ts

@@ -0,0 +1,26 @@
+export type OpenAIChatMessage = {
+    role: "user" | "assistant" | "system";
+    content: string;
+    name?: string;
+} | {
+    role: "assistant";
+    content: string | null;
+    function_call: {
+        name: string;
+        arguments: string;
+    };
+} | {
+    role: "function";
+    content: string;
+    name: string;
+};
+export declare const OpenAIChatMessage: {
+    system(content: string): OpenAIChatMessage;
+    user(content: string): OpenAIChatMessage;
+    assistant(content: string): OpenAIChatMessage;
+    functionCall(content: string | null, functionCall: {
+        name: string;
+        arguments: string;
+    }): OpenAIChatMessage;
+    functionResult(name: string, content: string): OpenAIChatMessage;
+};

package/model-provider/openai/chat/OpenAIChatMessage.js

@@ -0,0 +1,21 @@
+export const OpenAIChatMessage = {
+    system(content) {
+        return { role: "system", content };
+    },
+    user(content) {
+        return { role: "user", content };
+    },
+    assistant(content) {
+        return { role: "assistant", content };
+    },
+    functionCall(content, functionCall) {
+        return {
+            role: "assistant",
+            content,
+            function_call: functionCall,
+        };
+    },
+    functionResult(name, content) {
+        return { role: "function", name, content };
+    },
+};