modelfusion 0.95.0 → 0.96.0

package/model-provider/mistral/MistralTextEmbeddingModel.js

@@ -0,0 +1,96 @@
+import { z } from "zod";
+import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottle.js";
+import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { MistralApiConfiguration } from "./MistralApiConfiguration.js";
+import { failedMistralCallResponseHandler } from "./MistralError.js";
+export class MistralTextEmbeddingModel extends AbstractModel {
+    constructor(settings) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "mistral"
+        });
+        Object.defineProperty(this, "maxValuesPerCall", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 32
+        });
+        /**
+         * Parallel calls are technically possible, but I have been hitting rate limits and disabled
+         * them for now.
+         */
+        Object.defineProperty(this, "isParallelizable", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
+        Object.defineProperty(this, "embeddingDimensions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 1024
+        });
+    }
+    get modelName() {
+        return this.settings.model;
+    }
+    async callAPI(texts, options) {
+        if (texts.length > this.maxValuesPerCall) {
+            throw new Error(`The Mistral embedding API only supports ${this.maxValuesPerCall} texts per API call.`);
+        }
+        const api = this.settings.api ?? new MistralApiConfiguration();
+        const abortSignal = options?.run?.abortSignal;
+        const model = this.settings.model;
+        const encodingFormat = this.settings.encodingFormat ?? "float";
+        return callWithRetryAndThrottle({
+            retry: this.settings.api?.retry,
+            throttle: this.settings.api?.throttle,
+            call: async () => postJsonToApi({
+                url: api.assembleUrl(`/embeddings`),
+                headers: api.headers,
+                body: {
+                    model,
+                    input: texts,
+                    encoding_format: encodingFormat,
+                },
+                failedResponseHandler: failedMistralCallResponseHandler,
+                successfulResponseHandler: createJsonResponseHandler(MistralTextEmbeddingResponseSchema),
+                abortSignal,
+            }),
+        });
+    }
+    get settingsForEvent() {
+        return {
+            encodingFormat: this.settings.encodingFormat,
+        };
+    }
+    async doEmbedValues(texts, options) {
+        const response = await this.callAPI(texts, options);
+        return {
+            response,
+            embeddings: response.data.map((entry) => entry.embedding),
+        };
+    }
+    withSettings(additionalSettings) {
+        return new MistralTextEmbeddingModel(Object.assign({}, this.settings, additionalSettings));
+    }
+}
+const MistralTextEmbeddingResponseSchema = z.object({
+    id: z.string(),
+    object: z.string(),
+    data: z.array(z.object({
+        object: z.string(),
+        embedding: z.array(z.number()),
+        index: z.number(),
+    })),
+    model: z.string(),
+    usage: z.object({
+        prompt_tokens: z.number(),
+        total_tokens: z.number(),
+    }),
+});

package/model-provider/mistral/MistralTextGenerationModel.cjs

@@ -0,0 +1,254 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MistralTextGenerationResponseFormat = exports.MistralTextGenerationModel = void 0;
+const zod_1 = require("zod");
+const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndThrottle.cjs");
+const postToApi_js_1 = require("../../core/api/postToApi.cjs");
+const ZodSchema_js_1 = require("../../core/schema/ZodSchema.cjs");
+const parseJSON_js_1 = require("../../core/schema/parseJSON.cjs");
+const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
+const PromptTemplateTextStreamingModel_js_1 = require("../../model-function/generate-text/PromptTemplateTextStreamingModel.cjs");
+const AsyncQueue_js_1 = require("../../util/AsyncQueue.cjs");
+const parseEventSourceStream_js_1 = require("../../util/streaming/parseEventSourceStream.cjs");
+const MistralApiConfiguration_js_1 = require("./MistralApiConfiguration.cjs");
+const MistralError_js_1 = require("./MistralError.cjs");
+const MistralPromptTemplate_js_1 = require("./MistralPromptTemplate.cjs");
+class MistralTextGenerationModel extends AbstractModel_js_1.AbstractModel {
+    constructor(settings) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "mistral"
+        });
+        Object.defineProperty(this, "contextWindowSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "tokenizer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "countPromptTokens", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+    }
+    get modelName() {
+        return this.settings.model;
+    }
+    async callAPI(prompt, options) {
+        const { model, temperature, topP, safeMode, randomSeed, maxCompletionTokens, } = this.settings;
+        const api = this.settings.api ?? new MistralApiConfiguration_js_1.MistralApiConfiguration();
+        const abortSignal = options.run?.abortSignal;
+        const stream = options.responseFormat.stream;
+        const successfulResponseHandler = options.responseFormat.handler;
+        return (0, callWithRetryAndThrottle_js_1.callWithRetryAndThrottle)({
+            retry: api.retry,
+            throttle: api.throttle,
+            call: async () => (0, postToApi_js_1.postJsonToApi)({
+                url: api.assembleUrl(`/chat/completions`),
+                headers: api.headers,
+                body: {
+                    stream,
+                    messages: prompt,
+                    model,
+                    temperature,
+                    top_p: topP,
+                    max_tokens: maxCompletionTokens,
+                    safe_mode: safeMode,
+                    random_seed: randomSeed,
+                },
+                failedResponseHandler: MistralError_js_1.failedMistralCallResponseHandler,
+                successfulResponseHandler,
+                abortSignal,
+            }),
+        });
+    }
+    get settingsForEvent() {
+        const eventSettingProperties = [
+            "maxCompletionTokens",
+            "temperature",
+            "topP",
+            "safeMode",
+            "randomSeed",
+        ];
+        return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
+    }
+    async doGenerateText(prompt, options) {
+        const response = await this.callAPI(prompt, {
+            ...options,
+            responseFormat: exports.MistralTextGenerationResponseFormat.json,
+        });
+        return {
+            response,
+            text: response.choices[0].message.content,
+        };
+    }
+    doStreamText(prompt, options) {
+        return this.callAPI(prompt, {
+            ...options,
+            responseFormat: exports.MistralTextGenerationResponseFormat.textDeltaIterable,
+        });
+    }
+    /**
+     * Returns this model with a text prompt template.
+     */
+    withTextPrompt() {
+        return this.withPromptTemplate((0, MistralPromptTemplate_js_1.text)());
+    }
+    /**
+     * Returns this model with an instruction prompt template.
+     */
+    withInstructionPrompt() {
+        return this.withPromptTemplate((0, MistralPromptTemplate_js_1.instruction)());
+    }
+    /**
+     * Returns this model with a chat prompt template.
+     */
+    withChatPrompt() {
+        return this.withPromptTemplate((0, MistralPromptTemplate_js_1.chat)());
+    }
+    withPromptTemplate(promptTemplate) {
+        return new PromptTemplateTextStreamingModel_js_1.PromptTemplateTextStreamingModel({
+            model: this,
+            promptTemplate,
+        });
+    }
+    withSettings(additionalSettings) {
+        return new MistralTextGenerationModel(Object.assign({}, this.settings, additionalSettings));
+    }
+}
+exports.MistralTextGenerationModel = MistralTextGenerationModel;
+const mistralTextGenerationResponseSchema = zod_1.z.object({
+    id: zod_1.z.string(),
+    object: zod_1.z.string(),
+    created: zod_1.z.number(),
+    model: zod_1.z.string(),
+    choices: zod_1.z.array(zod_1.z.object({
+        index: zod_1.z.number(),
+        message: zod_1.z.object({
+            role: zod_1.z.enum(["user", "assistant"]),
+            content: zod_1.z.string(),
+        }),
+        finish_reason: zod_1.z.enum(["stop", "length", "model_length"]),
+    })),
+    usage: zod_1.z.object({
+        prompt_tokens: zod_1.z.number(),
+        completion_tokens: zod_1.z.number(),
+        total_tokens: zod_1.z.number(),
+    }),
+});
+exports.MistralTextGenerationResponseFormat = {
+    /**
+     * Returns the response as a JSON object.
+     */
+    json: {
+        stream: false,
+        handler: (0, postToApi_js_1.createJsonResponseHandler)(mistralTextGenerationResponseSchema),
+    },
+    /**
+     * Returns an async iterable over the text deltas (only the tex different of the first choice).
+     */
+    textDeltaIterable: {
+        stream: true,
+        handler: async ({ response }) => createMistralTextGenerationDeltaIterableQueue(response.body, (delta) => delta[0]?.delta.content ?? ""),
+    },
+};
+const mistralTextGenerationChunkSchema = new ZodSchema_js_1.ZodSchema(zod_1.z.object({
+    id: zod_1.z.string(),
+    object: zod_1.z.string().optional(),
+    created: zod_1.z.number().optional(),
+    model: zod_1.z.string(),
+    choices: zod_1.z.array(zod_1.z.object({
+        index: zod_1.z.number(),
+        delta: zod_1.z.object({
+            role: zod_1.z.enum(["assistant", "user"]).optional().nullable(),
+            content: zod_1.z.string().nullable().optional(),
+        }),
+        finish_reason: zod_1.z
+            .enum(["stop", "length", "model_length"])
+            .nullable()
+            .optional(),
+    })),
+}));
+async function createMistralTextGenerationDeltaIterableQueue(stream, extractDeltaValue) {
+    const queue = new AsyncQueue_js_1.AsyncQueue();
+    const streamDelta = [];
+    // process the stream asynchonously (no 'await' on purpose):
+    (0, parseEventSourceStream_js_1.parseEventSourceStream)({ stream })
+        .then(async (events) => {
+        try {
+            for await (const event of events) {
+                const data = event.data;
+                if (data === "[DONE]") {
+                    queue.close();
+                    return;
+                }
+                const parseResult = (0, parseJSON_js_1.safeParseJSON)({
+                    text: data,
+                    schema: mistralTextGenerationChunkSchema,
+                });
+                if (!parseResult.success) {
+                    queue.push({
+                        type: "error",
+                        error: parseResult.error,
+                    });
+                    // Note: the queue is not closed on purpose. Some providers might add additional
+                    // chunks that are not parsable, and ModelFusion should be resilient to that.
+                    continue;
+                }
+                const completionChunk = parseResult.data;
+                for (let i = 0; i < completionChunk.choices.length; i++) {
+                    const eventChoice = completionChunk.choices[i];
+                    const delta = eventChoice.delta;
+                    if (streamDelta[i] == null) {
+                        streamDelta[i] = {
+                            role: undefined,
+                            content: "",
+                            isComplete: false,
+                            delta,
+                        };
+                    }
+                    const choice = streamDelta[i];
+                    choice.delta = delta;
+                    if (eventChoice.finish_reason != null) {
+                        choice.isComplete = true;
+                    }
+                    if (delta.content != undefined) {
+                        choice.content += delta.content;
+                    }
+                    if (delta.role != undefined) {
+                        choice.role = delta.role;
+                    }
+                }
+                // Since we're mutating the choices array in an async scenario,
+                // we need to make a deep copy:
+                const streamDeltaDeepCopy = JSON.parse(JSON.stringify(streamDelta));
+                queue.push({
+                    type: "delta",
+                    fullDelta: streamDeltaDeepCopy,
+                    valueDelta: extractDeltaValue(streamDeltaDeepCopy),
+                });
+            }
+        }
+        catch (error) {
+            queue.push({ type: "error", error });
+            queue.close();
+            return;
+        }
+    })
+        .catch((error) => {
+        queue.push({ type: "error", error });
+        queue.close();
+        return;
+    });
+    return queue;
+}

package/model-provider/mistral/MistralTextGenerationModel.d.ts

@@ -0,0 +1,231 @@
+import { z } from "zod";
+import { FunctionOptions } from "../../core/FunctionOptions.js";
+import { ApiConfiguration } from "../../core/api/ApiConfiguration.js";
+import { ResponseHandler } from "../../core/api/postToApi.js";
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { Delta } from "../../model-function/Delta.js";
+import { PromptTemplateTextStreamingModel } from "../../model-function/generate-text/PromptTemplateTextStreamingModel.js";
+import { TextGenerationModelSettings, TextStreamingModel } from "../../model-function/generate-text/TextGenerationModel.js";
+import { TextGenerationPromptTemplate } from "../../model-function/generate-text/TextGenerationPromptTemplate.js";
+export type MistralTextGenerationPrompt = Array<{
+    role: "system" | "user" | "assistant";
+    content: string;
+}>;
+export interface MistralTextGenerationModelSettings extends TextGenerationModelSettings {
+    api?: ApiConfiguration;
+    model: "mistral-tiny" | "mistral-small" | "mistral-medium";
+    /**
+     * What sampling temperature to use, between 0.0 and 2.0.
+     * Higher values like 0.8 will make the output more random,
+     * while lower values like 0.2 will make it more focused and deterministic.
+     *
+     * We generally recommend altering this or top_p but not both.
+     *
+     * Default: 0.7
+     */
+    temperature?: number | null;
+    /**
+     * Nucleus sampling, where the model considers the results of the tokens
+     * with top_p probability mass. So 0.1 means only the tokens comprising
+     * the top 10% probability mass are considered.
+     *
+     * We generally recommend altering this or temperature but not both.
+     *
+     * Default: 1
+     */
+    topP?: number;
+    /**
+     * Whether to inject a safety prompt before all conversations.
+     *
+     * Default: false
+     */
+    safeMode?: boolean;
+    /**
+     * The seed to use for random sampling. If set, different calls will
+     * generate deterministic results.
+     */
+    randomSeed?: number | null;
+}
+export declare class MistralTextGenerationModel extends AbstractModel<MistralTextGenerationModelSettings> implements TextStreamingModel<MistralTextGenerationPrompt, MistralTextGenerationModelSettings> {
+    constructor(settings: MistralTextGenerationModelSettings);
+    readonly provider = "mistral";
+    get modelName(): "mistral-tiny" | "mistral-small" | "mistral-medium";
+    readonly contextWindowSize: undefined;
+    readonly tokenizer: undefined;
+    readonly countPromptTokens: undefined;
+    callAPI<RESULT>(prompt: MistralTextGenerationPrompt, options: {
+        responseFormat: MistralTextGenerationResponseFormatType<RESULT>;
+    } & FunctionOptions): Promise<RESULT>;
+    get settingsForEvent(): Partial<MistralTextGenerationModelSettings>;
+    doGenerateText(prompt: MistralTextGenerationPrompt, options?: FunctionOptions): Promise<{
+        response: {
+            object: string;
+            usage: {
+                prompt_tokens: number;
+                total_tokens: number;
+                completion_tokens: number;
+            };
+            model: string;
+            id: string;
+            created: number;
+            choices: {
+                message: {
+                    role: "user" | "assistant";
+                    content: string;
+                };
+                finish_reason: "length" | "stop" | "model_length";
+                index: number;
+            }[];
+        };
+        text: string;
+    }>;
+    doStreamText(prompt: MistralTextGenerationPrompt, options?: FunctionOptions): Promise<AsyncIterable<Delta<string>>>;
+    /**
+     * Returns this model with a text prompt template.
+     */
+    withTextPrompt(): PromptTemplateTextStreamingModel<string, MistralTextGenerationPrompt, MistralTextGenerationModelSettings, this>;
+    /**
+     * Returns this model with an instruction prompt template.
+     */
+    withInstructionPrompt(): PromptTemplateTextStreamingModel<import("../../index.js").TextInstructionPrompt, MistralTextGenerationPrompt, MistralTextGenerationModelSettings, this>;
+    /**
+     * Returns this model with a chat prompt template.
+     */
+    withChatPrompt(): PromptTemplateTextStreamingModel<import("../../index.js").TextChatPrompt, MistralTextGenerationPrompt, MistralTextGenerationModelSettings, this>;
+    withPromptTemplate<INPUT_PROMPT>(promptTemplate: TextGenerationPromptTemplate<INPUT_PROMPT, MistralTextGenerationPrompt>): PromptTemplateTextStreamingModel<INPUT_PROMPT, MistralTextGenerationPrompt, MistralTextGenerationModelSettings, this>;
+    withSettings(additionalSettings: Partial<MistralTextGenerationModelSettings>): this;
+}
+declare const mistralTextGenerationResponseSchema: z.ZodObject<{
+    id: z.ZodString;
+    object: z.ZodString;
+    created: z.ZodNumber;
+    model: z.ZodString;
+    choices: z.ZodArray<z.ZodObject<{
+        index: z.ZodNumber;
+        message: z.ZodObject<{
+            role: z.ZodEnum<["user", "assistant"]>;
+            content: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            role: "user" | "assistant";
+            content: string;
+        }, {
+            role: "user" | "assistant";
+            content: string;
+        }>;
+        finish_reason: z.ZodEnum<["stop", "length", "model_length"]>;
+    }, "strip", z.ZodTypeAny, {
+        message: {
+            role: "user" | "assistant";
+            content: string;
+        };
+        finish_reason: "length" | "stop" | "model_length";
+        index: number;
+    }, {
+        message: {
+            role: "user" | "assistant";
+            content: string;
+        };
+        finish_reason: "length" | "stop" | "model_length";
+        index: number;
+    }>, "many">;
+    usage: z.ZodObject<{
+        prompt_tokens: z.ZodNumber;
+        completion_tokens: z.ZodNumber;
+        total_tokens: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        prompt_tokens: number;
+        total_tokens: number;
+        completion_tokens: number;
+    }, {
+        prompt_tokens: number;
+        total_tokens: number;
+        completion_tokens: number;
+    }>;
+}, "strip", z.ZodTypeAny, {
+    object: string;
+    usage: {
+        prompt_tokens: number;
+        total_tokens: number;
+        completion_tokens: number;
+    };
+    model: string;
+    id: string;
+    created: number;
+    choices: {
+        message: {
+            role: "user" | "assistant";
+            content: string;
+        };
+        finish_reason: "length" | "stop" | "model_length";
+        index: number;
+    }[];
+}, {
+    object: string;
+    usage: {
+        prompt_tokens: number;
+        total_tokens: number;
+        completion_tokens: number;
+    };
+    model: string;
+    id: string;
+    created: number;
+    choices: {
+        message: {
+            role: "user" | "assistant";
+            content: string;
+        };
+        finish_reason: "length" | "stop" | "model_length";
+        index: number;
+    }[];
+}>;
+export type MistralTextGenerationResponse = z.infer<typeof mistralTextGenerationResponseSchema>;
+export type MistralTextGenerationResponseFormatType<T> = {
+    stream: boolean;
+    handler: ResponseHandler<T>;
+};
+export declare const MistralTextGenerationResponseFormat: {
+    /**
+     * Returns the response as a JSON object.
+     */
+    json: {
+        stream: false;
+        handler: ResponseHandler<{
+            object: string;
+            usage: {
+                prompt_tokens: number;
+                total_tokens: number;
+                completion_tokens: number;
+            };
+            model: string;
+            id: string;
+            created: number;
+            choices: {
+                message: {
+                    role: "user" | "assistant";
+                    content: string;
+                };
+                finish_reason: "length" | "stop" | "model_length";
+                index: number;
+            }[];
+        }>;
+    };
+    /**
+     * Returns an async iterable over the text deltas (only the tex different of the first choice).
+     */
+    textDeltaIterable: {
+        stream: true;
+        handler: ({ response }: {
+            response: Response;
+        }) => Promise<AsyncIterable<Delta<string>>>;
+    };
+};
+export type MistralTextGenerationDelta = Array<{
+    role: "assistant" | "user" | undefined;
+    content: string;
+    isComplete: boolean;
+    delta: {
+        role?: "assistant" | "user" | null;
+        content?: string | null;
+    };
+}>;
+export {};