modelfusion 0.95.0 → 0.97.0

package/model-provider/mistral/MistralTextGenerationModel.js

@@ -0,0 +1,250 @@
+import { z } from "zod";
+import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottle.js";
+import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
+import { ZodSchema } from "../../core/schema/ZodSchema.js";
+import { safeParseJSON } from "../../core/schema/parseJSON.js";
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { PromptTemplateTextStreamingModel } from "../../model-function/generate-text/PromptTemplateTextStreamingModel.js";
+import { AsyncQueue } from "../../util/AsyncQueue.js";
+import { parseEventSourceStream } from "../../util/streaming/parseEventSourceStream.js";
+import { MistralApiConfiguration } from "./MistralApiConfiguration.js";
+import { failedMistralCallResponseHandler } from "./MistralError.js";
+import { chat, instruction, text } from "./MistralPromptTemplate.js";
+export class MistralTextGenerationModel extends AbstractModel {
+    constructor(settings) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "mistral"
+        });
+        Object.defineProperty(this, "contextWindowSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "tokenizer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "countPromptTokens", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+    }
+    get modelName() {
+        return this.settings.model;
+    }
+    async callAPI(prompt, options) {
+        const { model, temperature, topP, safeMode, randomSeed, maxCompletionTokens, } = this.settings;
+        const api = this.settings.api ?? new MistralApiConfiguration();
+        const abortSignal = options.run?.abortSignal;
+        const stream = options.responseFormat.stream;
+        const successfulResponseHandler = options.responseFormat.handler;
+        return callWithRetryAndThrottle({
+            retry: api.retry,
+            throttle: api.throttle,
+            call: async () => postJsonToApi({
+                url: api.assembleUrl(`/chat/completions`),
+                headers: api.headers,
+                body: {
+                    stream,
+                    messages: prompt,
+                    model,
+                    temperature,
+                    top_p: topP,
+                    max_tokens: maxCompletionTokens,
+                    safe_mode: safeMode,
+                    random_seed: randomSeed,
+                },
+                failedResponseHandler: failedMistralCallResponseHandler,
+                successfulResponseHandler,
+                abortSignal,
+            }),
+        });
+    }
+    get settingsForEvent() {
+        const eventSettingProperties = [
+            "maxCompletionTokens",
+            "temperature",
+            "topP",
+            "safeMode",
+            "randomSeed",
+        ];
+        return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
+    }
+    async doGenerateText(prompt, options) {
+        const response = await this.callAPI(prompt, {
+            ...options,
+            responseFormat: MistralTextGenerationResponseFormat.json,
+        });
+        return {
+            response,
+            text: response.choices[0].message.content,
+        };
+    }
+    doStreamText(prompt, options) {
+        return this.callAPI(prompt, {
+            ...options,
+            responseFormat: MistralTextGenerationResponseFormat.textDeltaIterable,
+        });
+    }
+    /**
+     * Returns this model with a text prompt template.
+     */
+    withTextPrompt() {
+        return this.withPromptTemplate(text());
+    }
+    /**
+     * Returns this model with an instruction prompt template.
+     */
+    withInstructionPrompt() {
+        return this.withPromptTemplate(instruction());
+    }
+    /**
+     * Returns this model with a chat prompt template.
+     */
+    withChatPrompt() {
+        return this.withPromptTemplate(chat());
+    }
+    withPromptTemplate(promptTemplate) {
+        return new PromptTemplateTextStreamingModel({
+            model: this,
+            promptTemplate,
+        });
+    }
+    withSettings(additionalSettings) {
+        return new MistralTextGenerationModel(Object.assign({}, this.settings, additionalSettings));
+    }
+}
+const mistralTextGenerationResponseSchema = z.object({
+    id: z.string(),
+    object: z.string(),
+    created: z.number(),
+    model: z.string(),
+    choices: z.array(z.object({
+        index: z.number(),
+        message: z.object({
+            role: z.enum(["user", "assistant"]),
+            content: z.string(),
+        }),
+        finish_reason: z.enum(["stop", "length", "model_length"]),
+    })),
+    usage: z.object({
+        prompt_tokens: z.number(),
+        completion_tokens: z.number(),
+        total_tokens: z.number(),
+    }),
+});
+export const MistralTextGenerationResponseFormat = {
+    /**
+     * Returns the response as a JSON object.
+     */
+    json: {
+        stream: false,
+        handler: createJsonResponseHandler(mistralTextGenerationResponseSchema),
+    },
+    /**
+     * Returns an async iterable over the text deltas (only the tex different of the first choice).
+     */
+    textDeltaIterable: {
+        stream: true,
+        handler: async ({ response }) => createMistralTextGenerationDeltaIterableQueue(response.body, (delta) => delta[0]?.delta.content ?? ""),
+    },
+};
+const mistralTextGenerationChunkSchema = new ZodSchema(z.object({
+    id: z.string(),
+    object: z.string().optional(),
+    created: z.number().optional(),
+    model: z.string(),
+    choices: z.array(z.object({
+        index: z.number(),
+        delta: z.object({
+            role: z.enum(["assistant", "user"]).optional().nullable(),
+            content: z.string().nullable().optional(),
+        }),
+        finish_reason: z
+            .enum(["stop", "length", "model_length"])
+            .nullable()
+            .optional(),
+    })),
+}));
+async function createMistralTextGenerationDeltaIterableQueue(stream, extractDeltaValue) {
+    const queue = new AsyncQueue();
+    const streamDelta = [];
+    // process the stream asynchonously (no 'await' on purpose):
+    parseEventSourceStream({ stream })
+        .then(async (events) => {
+        try {
+            for await (const event of events) {
+                const data = event.data;
+                if (data === "[DONE]") {
+                    queue.close();
+                    return;
+                }
+                const parseResult = safeParseJSON({
+                    text: data,
+                    schema: mistralTextGenerationChunkSchema,
+                });
+                if (!parseResult.success) {
+                    queue.push({
+                        type: "error",
+                        error: parseResult.error,
+                    });
+                    // Note: the queue is not closed on purpose. Some providers might add additional
+                    // chunks that are not parsable, and ModelFusion should be resilient to that.
+                    continue;
+                }
+                const completionChunk = parseResult.data;
+                for (let i = 0; i < completionChunk.choices.length; i++) {
+                    const eventChoice = completionChunk.choices[i];
+                    const delta = eventChoice.delta;
+                    if (streamDelta[i] == null) {
+                        streamDelta[i] = {
+                            role: undefined,
+                            content: "",
+                            isComplete: false,
+                            delta,
+                        };
+                    }
+                    const choice = streamDelta[i];
+                    choice.delta = delta;
+                    if (eventChoice.finish_reason != null) {
+                        choice.isComplete = true;
+                    }
+                    if (delta.content != undefined) {
+                        choice.content += delta.content;
+                    }
+                    if (delta.role != undefined) {
+                        choice.role = delta.role;
+                    }
+                }
+                // Since we're mutating the choices array in an async scenario,
+                // we need to make a deep copy:
+                const streamDeltaDeepCopy = JSON.parse(JSON.stringify(streamDelta));
+                queue.push({
+                    type: "delta",
+                    fullDelta: streamDeltaDeepCopy,
+                    valueDelta: extractDeltaValue(streamDeltaDeepCopy),
+                });
+            }
+        }
+        catch (error) {
+            queue.push({ type: "error", error });
+            queue.close();
+            return;
+        }
+    })
+        .catch((error) => {
+        queue.push({ type: "error", error });
+        queue.close();
+        return;
+    });
+    return queue;
+}

package/model-provider/mistral/index.cjs

@@ -0,0 +1,34 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MistralPrompt = exports.mistral = void 0;
+__exportStar(require("./MistralApiConfiguration.cjs"), exports);
+exports.mistral = __importStar(require("./MistralFacade.cjs"));
+exports.MistralPrompt = __importStar(require("./MistralPromptTemplate.cjs"));
+__exportStar(require("./MistralTextEmbeddingModel.cjs"), exports);
+__exportStar(require("./MistralTextGenerationModel.cjs"), exports);

package/model-provider/mistral/index.d.ts

@@ -0,0 +1,6 @@
+export * from "./MistralApiConfiguration.js";
+export { MistralErrorData } from "./MistralError.js";
+export * as mistral from "./MistralFacade.js";
+export * as MistralPrompt from "./MistralPromptTemplate.js";
+export * from "./MistralTextEmbeddingModel.js";
+export * from "./MistralTextGenerationModel.js";

package/model-provider/mistral/index.js

@@ -0,0 +1,5 @@
+export * from "./MistralApiConfiguration.js";
+export * as mistral from "./MistralFacade.js";
+export * as MistralPrompt from "./MistralPromptTemplate.js";
+export * from "./MistralTextEmbeddingModel.js";
+export * from "./MistralTextGenerationModel.js";

package/model-provider/ollama/OllamaError.cjs

@@ -2,37 +2,12 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.failedOllamaCallResponseHandler = void 0;
 const zod_1 = require("zod");
-const ApiCallError_js_1 = require("../../core/api/ApiCallError.cjs");
+const postToApi_js_1 = require("../../core/api/postToApi.cjs");
 const ZodSchema_js_1 = require("../../core/schema/ZodSchema.cjs");
-const parseJSON_js_1 = require("../../core/schema/parseJSON.cjs");
 const ollamaErrorDataSchema = new ZodSchema_js_1.ZodSchema(zod_1.z.object({
     error: zod_1.z.string(),
 }));
-const failedOllamaCallResponseHandler = async ({ response, url, requestBodyValues }) => {
-    const responseBody = await response.text();
-    // resilient parsing in case the response is not JSON or does not match the schema:
-    try {
-        const parsedError = (0, parseJSON_js_1.parseJSON)({
-            text: responseBody,
-            schema: ollamaErrorDataSchema,
-        });
-        return new ApiCallError_js_1.ApiCallError({
-            message: parsedError.error,
-            url,
-            requestBodyValues,
-            statusCode: response.status,
-            responseBody,
-            data: parsedError,
-        });
-    }
-    catch (parseError) {
-        return new ApiCallError_js_1.ApiCallError({
-            message: responseBody.trim() !== "" ? responseBody : response.statusText,
-            url,
-            requestBodyValues,
-            statusCode: response.status,
-            responseBody,
-        });
-    }
-};
-exports.failedOllamaCallResponseHandler = failedOllamaCallResponseHandler;
+exports.failedOllamaCallResponseHandler = (0, postToApi_js_1.createJsonErrorResponseHandler)({
+    errorSchema: ollamaErrorDataSchema,
+    errorToMessage: (error) => error.error,
+});

package/model-provider/ollama/OllamaError.js

@@ -1,34 +1,10 @@
 import { z } from "zod";
-import { ApiCallError } from "../../core/api/ApiCallError.js";
+import { createJsonErrorResponseHandler, } from "../../core/api/postToApi.js";
 import { ZodSchema } from "../../core/schema/ZodSchema.js";
-import { parseJSON } from "../../core/schema/parseJSON.js";
 const ollamaErrorDataSchema = new ZodSchema(z.object({
     error: z.string(),
 }));
-export const failedOllamaCallResponseHandler = async ({ response, url, requestBodyValues }) => {
-    const responseBody = await response.text();
-    // resilient parsing in case the response is not JSON or does not match the schema:
-    try {
-        const parsedError = parseJSON({
-            text: responseBody,
-            schema: ollamaErrorDataSchema,
-        });
-        return new ApiCallError({
-            message: parsedError.error,
-            url,
-            requestBodyValues,
-            statusCode: response.status,
-            responseBody,
-            data: parsedError,
-        });
-    }
-    catch (parseError) {
-        return new ApiCallError({
-            message: responseBody.trim() !== "" ? responseBody : response.statusText,
-            url,
-            requestBodyValues,
-            statusCode: response.status,
-            responseBody,
-        });
-    }
-};
+export const failedOllamaCallResponseHandler = createJsonErrorResponseHandler({
+    errorSchema: ollamaErrorDataSchema,
+    errorToMessage: (error) => error.error,
+});

package/model-provider/ollama/OllamaTextEmbeddingModel.cjs

@@ -22,12 +22,6 @@ class OllamaTextEmbeddingModel extends AbstractModel_js_1.AbstractModel {
             writable: true,
             value: 1
         });
-        Object.defineProperty(this, "contextWindowSize", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: undefined
-        });
     }
     get modelName() {
         return null;
@@ -40,7 +34,7 @@ class OllamaTextEmbeddingModel extends AbstractModel_js_1.AbstractModel {
     }
     async callAPI(texts, options) {
         if (texts.length > this.maxValuesPerCall) {
-            throw new Error(`The Llama.cpp embedding API only supports ${this.maxValuesPerCall} texts per API call.`);
+            throw new Error(`The Ollama embedding API only supports ${this.maxValuesPerCall} texts per API call.`);
         }
         return (0, callWithRetryAndThrottle_js_1.callWithRetryAndThrottle)({
             retry: this.settings.api?.retry,

package/model-provider/ollama/OllamaTextEmbeddingModel.d.ts

@@ -15,7 +15,6 @@ export declare class OllamaTextEmbeddingModel extends AbstractModel<OllamaTextEm
     get modelName(): null;
     readonly maxValuesPerCall = 1;
     get isParallelizable(): boolean;
-    readonly contextWindowSize: undefined;
     get embeddingDimensions(): number | undefined;
     callAPI(texts: Array<string>, options?: FunctionOptions): Promise<OllamaTextEmbeddingResponse>;
     get settingsForEvent(): Partial<OllamaTextEmbeddingModelSettings>;

package/model-provider/ollama/OllamaTextEmbeddingModel.js

@@ -19,12 +19,6 @@ export class OllamaTextEmbeddingModel extends AbstractModel {
             writable: true,
             value: 1
         });
-        Object.defineProperty(this, "contextWindowSize", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: undefined
-        });
     }
     get modelName() {
         return null;
@@ -37,7 +31,7 @@ export class OllamaTextEmbeddingModel extends AbstractModel {
     }
     async callAPI(texts, options) {
         if (texts.length > this.maxValuesPerCall) {
-            throw new Error(`The Llama.cpp embedding API only supports ${this.maxValuesPerCall} texts per API call.`);
+            throw new Error(`The Ollama embedding API only supports ${this.maxValuesPerCall} texts per API call.`);
         }
         return callWithRetryAndThrottle({
             retry: this.settings.api?.retry,

package/model-provider/ollama/OllamaTextGenerationModel.cjs

@@ -44,15 +44,47 @@ class OllamaTextGenerationModel extends AbstractModel_js_1.AbstractModel {
         return this.settings.contextWindowSize;
     }
     async callAPI(prompt, options) {
+        const { responseFormat } = options;
+        const api = this.settings.api ?? new OllamaApiConfiguration_js_1.OllamaApiConfiguration();
+        const abortSignal = options.run?.abortSignal;
         return (0, callWithRetryAndThrottle_js_1.callWithRetryAndThrottle)({
-            retry: this.settings.api?.retry,
-            throttle: this.settings.api?.throttle,
-            call: async () => callOllamaTextGenerationAPI({
-                ...this.settings,
-                // other
-                abortSignal: options.run?.abortSignal,
-                prompt,
-                responseFormat: options.responseFormat,
+            retry: api.retry,
+            throttle: api.throttle,
+            call: async () => (0, postToApi_js_1.postJsonToApi)({
+                url: api.assembleUrl(`/api/generate`),
+                headers: api.headers,
+                body: {
+                    stream: responseFormat.stream,
+                    model: this.settings.model,
+                    prompt: prompt.prompt,
+                    images: prompt.images,
+                    format: this.settings.format,
+                    options: {
+                        mirostat: this.settings.mirostat,
+                        mirostat_eta: this.settings.mirostatEta,
+                        mirostat_tau: this.settings.mirostatTau,
+                        num_ctx: this.settings.contextWindowSize,
+                        num_gpu: this.settings.numGpu,
+                        num_gqa: this.settings.numGqa,
+                        num_predict: this.settings.maxCompletionTokens,
+                        num_threads: this.settings.numThreads,
+                        repeat_last_n: this.settings.repeatLastN,
+                        repeat_penalty: this.settings.repeatPenalty,
+                        seed: this.settings.seed,
+                        stop: this.settings.stopSequences,
+                        temperature: this.settings.temperature,
+                        tfs_z: this.settings.tfsZ,
+                        top_k: this.settings.topK,
+                        top_p: this.settings.topP,
+                    },
+                    system: this.settings.system,
+                    template: this.settings.template,
+                    context: this.settings.context,
+                    raw: this.settings.raw,
+                },
+                failedResponseHandler: OllamaError_js_1.failedOllamaCallResponseHandler,
+                successfulResponseHandler: responseFormat.handler,
+                abortSignal,
             }),
         });
     }
@@ -63,17 +95,17 @@ class OllamaTextGenerationModel extends AbstractModel_js_1.AbstractModel {
             "contextWindowSize",
             "temperature",
             "mirostat",
-            "mirostat_eta",
-            "mirostat_tau",
-            "num_gqa",
-            "num_gpu",
-            "num_threads",
-            "repeat_last_n",
-            "repeat_penalty",
+            "mirostatEta",
+            "mirostatTau",
+            "numGqa",
+            "numGpu",
+            "numThreads",
+            "repeatLastN",
+            "repeatPenalty",
             "seed",
-            "tfs_z",
-            "top_k",
-            "top_p",
+            "tfsZ",
+            "topK",
+            "topP",
             "system",
             "template",
             "context",
@@ -110,6 +142,14 @@ class OllamaTextGenerationModel extends AbstractModel_js_1.AbstractModel {
             template: promptTemplate,
         });
     }
+    withTextPrompt() {
+        return this.withPromptTemplate({
+            format(prompt) {
+                return { prompt: prompt };
+            },
+            stopSequences: [],
+        });
+    }
     withPromptTemplate(promptTemplate) {
         return new PromptTemplateTextStreamingModel_js_1.PromptTemplateTextStreamingModel({
             model: this.withSettings({
@@ -131,7 +171,7 @@ const ollamaTextGenerationResponseSchema = zod_1.z.object({
     model: zod_1.z.string(),
     response: zod_1.z.string(),
     total_duration: zod_1.z.number(),
-    load_duration: zod_1.z.number(),
+    load_duration: zod_1.z.number().optional(),
     prompt_eval_count: zod_1.z.number(),
     eval_count: zod_1.z.number(),
     eval_duration: zod_1.z.number(),
@@ -149,7 +189,7 @@ const ollamaTextStreamingResponseSchema = new ZodSchema_js_1.ZodSchema(zod_1.z.d
         model: zod_1.z.string(),
         created_at: zod_1.z.string(),
         total_duration: zod_1.z.number(),
-        load_duration: zod_1.z.number(),
+        load_duration: zod_1.z.number().optional(),
         sample_count: zod_1.z.number().optional(),
         sample_duration: zod_1.z.number().optional(),
         prompt_eval_count: zod_1.z.number(),
@@ -159,43 +199,6 @@ const ollamaTextStreamingResponseSchema = new ZodSchema_js_1.ZodSchema(zod_1.z.d
         context: zod_1.z.array(zod_1.z.number()).optional(),
     }),
 ]));
-async function callOllamaTextGenerationAPI({ api = new OllamaApiConfiguration_js_1.OllamaApiConfiguration(), abortSignal, responseFormat, prompt, model, format, contextWindowSize, maxCompletionTokens, mirostat, mirostat_eta, mirostat_tau, num_gpu, num_gqa, num_threads, repeat_last_n, repeat_penalty, seed, stopSequences, temperature, tfs_z, top_k, top_p, system, template, context, raw, }) {
-    return (0, postToApi_js_1.postJsonToApi)({
-        url: api.assembleUrl(`/api/generate`),
-        headers: api.headers,
-        body: {
-            stream: responseFormat.stream,
-            model,
-            prompt,
-            format,
-            options: {
-                mirostat,
-                mirostat_eta,
-                mirostat_tau,
-                num_ctx: contextWindowSize,
-                num_gpu,
-                num_gqa,
-                num_predict: maxCompletionTokens,
-                num_threads,
-                repeat_last_n,
-                repeat_penalty,
-                seed,
-                stop: stopSequences,
-                temperature,
-                tfs_z,
-                top_k,
-                top_p,
-            },
-            system,
-            template,
-            context,
-            raw,
-        },
-        failedResponseHandler: OllamaError_js_1.failedOllamaCallResponseHandler,
-        successfulResponseHandler: responseFormat.handler,
-        abortSignal,
-    });
-}
 async function createOllamaFullDeltaIterableQueue(stream) {
     const queue = new AsyncQueue_js_1.AsyncQueue();
     let accumulatedText = "";