modelfusion 0.94.0 → 0.96.0

package/model-provider/mistral/MistralTextGenerationModel.js

@@ -0,0 +1,250 @@
+import { z } from "zod";
+import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottle.js";
+import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
+import { ZodSchema } from "../../core/schema/ZodSchema.js";
+import { safeParseJSON } from "../../core/schema/parseJSON.js";
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { PromptTemplateTextStreamingModel } from "../../model-function/generate-text/PromptTemplateTextStreamingModel.js";
+import { AsyncQueue } from "../../util/AsyncQueue.js";
+import { parseEventSourceStream } from "../../util/streaming/parseEventSourceStream.js";
+import { MistralApiConfiguration } from "./MistralApiConfiguration.js";
+import { failedMistralCallResponseHandler } from "./MistralError.js";
+import { chat, instruction, text } from "./MistralPromptTemplate.js";
+export class MistralTextGenerationModel extends AbstractModel {
+    constructor(settings) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "mistral"
+        });
+        Object.defineProperty(this, "contextWindowSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "tokenizer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "countPromptTokens", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+    }
+    get modelName() {
+        return this.settings.model;
+    }
+    async callAPI(prompt, options) {
+        const { model, temperature, topP, safeMode, randomSeed, maxCompletionTokens, } = this.settings;
+        const api = this.settings.api ?? new MistralApiConfiguration();
+        const abortSignal = options.run?.abortSignal;
+        const stream = options.responseFormat.stream;
+        const successfulResponseHandler = options.responseFormat.handler;
+        return callWithRetryAndThrottle({
+            retry: api.retry,
+            throttle: api.throttle,
+            call: async () => postJsonToApi({
+                url: api.assembleUrl(`/chat/completions`),
+                headers: api.headers,
+                body: {
+                    stream,
+                    messages: prompt,
+                    model,
+                    temperature,
+                    top_p: topP,
+                    max_tokens: maxCompletionTokens,
+                    safe_mode: safeMode,
+                    random_seed: randomSeed,
+                },
+                failedResponseHandler: failedMistralCallResponseHandler,
+                successfulResponseHandler,
+                abortSignal,
+            }),
+        });
+    }
+    get settingsForEvent() {
+        const eventSettingProperties = [
+            "maxCompletionTokens",
+            "temperature",
+            "topP",
+            "safeMode",
+            "randomSeed",
+        ];
+        return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
+    }
+    async doGenerateText(prompt, options) {
+        const response = await this.callAPI(prompt, {
+            ...options,
+            responseFormat: MistralTextGenerationResponseFormat.json,
+        });
+        return {
+            response,
+            text: response.choices[0].message.content,
+        };
+    }
+    doStreamText(prompt, options) {
+        return this.callAPI(prompt, {
+            ...options,
+            responseFormat: MistralTextGenerationResponseFormat.textDeltaIterable,
+        });
+    }
+    /**
+     * Returns this model with a text prompt template.
+     */
+    withTextPrompt() {
+        return this.withPromptTemplate(text());
+    }
+    /**
+     * Returns this model with an instruction prompt template.
+     */
+    withInstructionPrompt() {
+        return this.withPromptTemplate(instruction());
+    }
+    /**
+     * Returns this model with a chat prompt template.
+     */
+    withChatPrompt() {
+        return this.withPromptTemplate(chat());
+    }
+    withPromptTemplate(promptTemplate) {
+        return new PromptTemplateTextStreamingModel({
+            model: this,
+            promptTemplate,
+        });
+    }
+    withSettings(additionalSettings) {
+        return new MistralTextGenerationModel(Object.assign({}, this.settings, additionalSettings));
+    }
+}
+const mistralTextGenerationResponseSchema = z.object({
+    id: z.string(),
+    object: z.string(),
+    created: z.number(),
+    model: z.string(),
+    choices: z.array(z.object({
+        index: z.number(),
+        message: z.object({
+            role: z.enum(["user", "assistant"]),
+            content: z.string(),
+        }),
+        finish_reason: z.enum(["stop", "length", "model_length"]),
+    })),
+    usage: z.object({
+        prompt_tokens: z.number(),
+        completion_tokens: z.number(),
+        total_tokens: z.number(),
+    }),
+});
+export const MistralTextGenerationResponseFormat = {
+    /**
+     * Returns the response as a JSON object.
+     */
+    json: {
+        stream: false,
+        handler: createJsonResponseHandler(mistralTextGenerationResponseSchema),
+    },
+    /**
+     * Returns an async iterable over the text deltas (only the tex different of the first choice).
+     */
+    textDeltaIterable: {
+        stream: true,
+        handler: async ({ response }) => createMistralTextGenerationDeltaIterableQueue(response.body, (delta) => delta[0]?.delta.content ?? ""),
+    },
+};
+const mistralTextGenerationChunkSchema = new ZodSchema(z.object({
+    id: z.string(),
+    object: z.string().optional(),
+    created: z.number().optional(),
+    model: z.string(),
+    choices: z.array(z.object({
+        index: z.number(),
+        delta: z.object({
+            role: z.enum(["assistant", "user"]).optional().nullable(),
+            content: z.string().nullable().optional(),
+        }),
+        finish_reason: z
+            .enum(["stop", "length", "model_length"])
+            .nullable()
+            .optional(),
+    })),
+}));
+async function createMistralTextGenerationDeltaIterableQueue(stream, extractDeltaValue) {
+    const queue = new AsyncQueue();
+    const streamDelta = [];
+    // process the stream asynchonously (no 'await' on purpose):
+    parseEventSourceStream({ stream })
+        .then(async (events) => {
+        try {
+            for await (const event of events) {
+                const data = event.data;
+                if (data === "[DONE]") {
+                    queue.close();
+                    return;
+                }
+                const parseResult = safeParseJSON({
+                    text: data,
+                    schema: mistralTextGenerationChunkSchema,
+                });
+                if (!parseResult.success) {
+                    queue.push({
+                        type: "error",
+                        error: parseResult.error,
+                    });
+                    // Note: the queue is not closed on purpose. Some providers might add additional
+                    // chunks that are not parsable, and ModelFusion should be resilient to that.
+                    continue;
+                }
+                const completionChunk = parseResult.data;
+                for (let i = 0; i < completionChunk.choices.length; i++) {
+                    const eventChoice = completionChunk.choices[i];
+                    const delta = eventChoice.delta;
+                    if (streamDelta[i] == null) {
+                        streamDelta[i] = {
+                            role: undefined,
+                            content: "",
+                            isComplete: false,
+                            delta,
+                        };
+                    }
+                    const choice = streamDelta[i];
+                    choice.delta = delta;
+                    if (eventChoice.finish_reason != null) {
+                        choice.isComplete = true;
+                    }
+                    if (delta.content != undefined) {
+                        choice.content += delta.content;
+                    }
+                    if (delta.role != undefined) {
+                        choice.role = delta.role;
+                    }
+                }
+                // Since we're mutating the choices array in an async scenario,
+                // we need to make a deep copy:
+                const streamDeltaDeepCopy = JSON.parse(JSON.stringify(streamDelta));
+                queue.push({
+                    type: "delta",
+                    fullDelta: streamDeltaDeepCopy,
+                    valueDelta: extractDeltaValue(streamDeltaDeepCopy),
+                });
+            }
+        }
+        catch (error) {
+            queue.push({ type: "error", error });
+            queue.close();
+            return;
+        }
+    })
+        .catch((error) => {
+        queue.push({ type: "error", error });
+        queue.close();
+        return;
+    });
+    return queue;
+}

package/model-provider/mistral/index.cjs

@@ -0,0 +1,34 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MistralPrompt = exports.mistral = void 0;
+__exportStar(require("./MistralApiConfiguration.cjs"), exports);
+exports.mistral = __importStar(require("./MistralFacade.cjs"));
+exports.MistralPrompt = __importStar(require("./MistralPromptTemplate.cjs"));
+__exportStar(require("./MistralTextEmbeddingModel.cjs"), exports);
+__exportStar(require("./MistralTextGenerationModel.cjs"), exports);

package/model-provider/mistral/index.d.ts

@@ -0,0 +1,6 @@
+export * from "./MistralApiConfiguration.js";
+export { MistralErrorData } from "./MistralError.js";
+export * as mistral from "./MistralFacade.js";
+export * as MistralPrompt from "./MistralPromptTemplate.js";
+export * from "./MistralTextEmbeddingModel.js";
+export * from "./MistralTextGenerationModel.js";

package/model-provider/mistral/index.js

@@ -0,0 +1,5 @@
+export * from "./MistralApiConfiguration.js";
+export * as mistral from "./MistralFacade.js";
+export * as MistralPrompt from "./MistralPromptTemplate.js";
+export * from "./MistralTextEmbeddingModel.js";
+export * from "./MistralTextGenerationModel.js";

package/model-provider/ollama/OllamaError.cjs

@@ -2,37 +2,12 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.failedOllamaCallResponseHandler = void 0;
 const zod_1 = require("zod");
-const ApiCallError_js_1 = require("../../core/api/ApiCallError.cjs");
+const postToApi_js_1 = require("../../core/api/postToApi.cjs");
 const ZodSchema_js_1 = require("../../core/schema/ZodSchema.cjs");
-const parseJSON_js_1 = require("../../core/schema/parseJSON.cjs");
 const ollamaErrorDataSchema = new ZodSchema_js_1.ZodSchema(zod_1.z.object({
     error: zod_1.z.string(),
 }));
-const failedOllamaCallResponseHandler = async ({ response, url, requestBodyValues }) => {
-    const responseBody = await response.text();
-    // resilient parsing in case the response is not JSON or does not match the schema:
-    try {
-        const parsedError = (0, parseJSON_js_1.parseJSON)({
-            text: responseBody,
-            schema: ollamaErrorDataSchema,
-        });
-        return new ApiCallError_js_1.ApiCallError({
-            message: parsedError.error,
-            url,
-            requestBodyValues,
-            statusCode: response.status,
-            responseBody,
-            data: parsedError,
-        });
-    }
-    catch (parseError) {
-        return new ApiCallError_js_1.ApiCallError({
-            message: responseBody.trim() !== "" ? responseBody : response.statusText,
-            url,
-            requestBodyValues,
-            statusCode: response.status,
-            responseBody,
-        });
-    }
-};
-exports.failedOllamaCallResponseHandler = failedOllamaCallResponseHandler;
+exports.failedOllamaCallResponseHandler = (0, postToApi_js_1.createJsonErrorResponseHandler)({
+    errorSchema: ollamaErrorDataSchema,
+    errorToMessage: (error) => error.error,
+});

package/model-provider/ollama/OllamaError.js

@@ -1,34 +1,10 @@
 import { z } from "zod";
-import { ApiCallError } from "../../core/api/ApiCallError.js";
+import { createJsonErrorResponseHandler, } from "../../core/api/postToApi.js";
 import { ZodSchema } from "../../core/schema/ZodSchema.js";
-import { parseJSON } from "../../core/schema/parseJSON.js";
 const ollamaErrorDataSchema = new ZodSchema(z.object({
     error: z.string(),
 }));
-export const failedOllamaCallResponseHandler = async ({ response, url, requestBodyValues }) => {
-    const responseBody = await response.text();
-    // resilient parsing in case the response is not JSON or does not match the schema:
-    try {
-        const parsedError = parseJSON({
-            text: responseBody,
-            schema: ollamaErrorDataSchema,
-        });
-        return new ApiCallError({
-            message: parsedError.error,
-            url,
-            requestBodyValues,
-            statusCode: response.status,
-            responseBody,
-            data: parsedError,
-        });
-    }
-    catch (parseError) {
-        return new ApiCallError({
-            message: responseBody.trim() !== "" ? responseBody : response.statusText,
-            url,
-            requestBodyValues,
-            statusCode: response.status,
-            responseBody,
-        });
-    }
-};
+export const failedOllamaCallResponseHandler = createJsonErrorResponseHandler({
+    errorSchema: ollamaErrorDataSchema,
+    errorToMessage: (error) => error.error,
+});

package/model-provider/ollama/OllamaTextEmbeddingModel.cjs

@@ -22,12 +22,6 @@ class OllamaTextEmbeddingModel extends AbstractModel_js_1.AbstractModel {
             writable: true,
             value: 1
         });
-        Object.defineProperty(this, "contextWindowSize", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: undefined
-        });
     }
     get modelName() {
         return null;
@@ -40,7 +34,7 @@ class OllamaTextEmbeddingModel extends AbstractModel_js_1.AbstractModel {
     }
     async callAPI(texts, options) {
         if (texts.length > this.maxValuesPerCall) {
-            throw new Error(`The Llama.cpp embedding API only supports ${this.maxValuesPerCall} texts per API call.`);
+            throw new Error(`The Ollama embedding API only supports ${this.maxValuesPerCall} texts per API call.`);
         }
         return (0, callWithRetryAndThrottle_js_1.callWithRetryAndThrottle)({
             retry: this.settings.api?.retry,

package/model-provider/ollama/OllamaTextEmbeddingModel.d.ts

@@ -15,7 +15,6 @@ export declare class OllamaTextEmbeddingModel extends AbstractModel<OllamaTextEm
     get modelName(): null;
     readonly maxValuesPerCall = 1;
     get isParallelizable(): boolean;
-    readonly contextWindowSize: undefined;
     get embeddingDimensions(): number | undefined;
     callAPI(texts: Array<string>, options?: FunctionOptions): Promise<OllamaTextEmbeddingResponse>;
     get settingsForEvent(): Partial<OllamaTextEmbeddingModelSettings>;

package/model-provider/ollama/OllamaTextEmbeddingModel.js

@@ -19,12 +19,6 @@ export class OllamaTextEmbeddingModel extends AbstractModel {
             writable: true,
             value: 1
         });
-        Object.defineProperty(this, "contextWindowSize", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: undefined
-        });
     }
     get modelName() {
         return null;
@@ -37,7 +31,7 @@ export class OllamaTextEmbeddingModel extends AbstractModel {
     }
     async callAPI(texts, options) {
         if (texts.length > this.maxValuesPerCall) {
-            throw new Error(`The Llama.cpp embedding API only supports ${this.maxValuesPerCall} texts per API call.`);
+            throw new Error(`The Ollama embedding API only supports ${this.maxValuesPerCall} texts per API call.`);
         }
         return callWithRetryAndThrottle({
             retry: this.settings.api?.retry,

package/model-provider/openai/OpenAICompletionModel.d.ts

@@ -156,13 +156,13 @@ export declare class OpenAICompletionModel extends AbstractModel<OpenAICompletio
             };
             model: string;
             id: string;
+            created: number;
             choices: {
                 text: string;
                 index: number;
                 finish_reason?: "length" | "stop" | "content_filter" | null | undefined;
                 logprobs?: any;
             }[];
-            created: number;
             system_fingerprint?: string | undefined;
         };
         text: string;
@@ -231,13 +231,13 @@ declare const OpenAICompletionResponseSchema: z.ZodObject<{
     };
     model: string;
     id: string;
+    created: number;
     choices: {
         text: string;
         index: number;
         finish_reason?: "length" | "stop" | "content_filter" | null | undefined;
         logprobs?: any;
     }[];
-    created: number;
     system_fingerprint?: string | undefined;
 }, {
     object: "text_completion";
@@ -248,13 +248,13 @@ declare const OpenAICompletionResponseSchema: z.ZodObject<{
     };
     model: string;
     id: string;
+    created: number;
     choices: {
         text: string;
         index: number;
         finish_reason?: "length" | "stop" | "content_filter" | null | undefined;
         logprobs?: any;
     }[];
-    created: number;
     system_fingerprint?: string | undefined;
 }>;
 export type OpenAICompletionResponse = z.infer<typeof OpenAICompletionResponseSchema>;
@@ -277,13 +277,13 @@ export declare const OpenAITextResponseFormat: {
             };
             model: string;
             id: string;
+            created: number;
             choices: {
                 text: string;
                 index: number;
                 finish_reason?: "length" | "stop" | "content_filter" | null | undefined;
                 logprobs?: any;
             }[];
-            created: number;
             system_fingerprint?: string | undefined;
         }>;
     };

package/model-provider/openai/OpenAIError.cjs

@@ -2,9 +2,8 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.failedOpenAICallResponseHandler = void 0;
 const zod_1 = require("zod");
-const ApiCallError_js_1 = require("../../core/api/ApiCallError.cjs");
+const postToApi_js_1 = require("../../core/api/postToApi.cjs");
 const ZodSchema_js_1 = require("../../core/schema/ZodSchema.cjs");
-const parseJSON_js_1 = require("../../core/schema/parseJSON.cjs");
 const openAIErrorDataSchema = new ZodSchema_js_1.ZodSchema(zod_1.z.object({
     error: zod_1.z.object({
         message: zod_1.z.string(),
@@ -13,35 +12,11 @@ const openAIErrorDataSchema = new ZodSchema_js_1.ZodSchema(zod_1.z.object({
         code: zod_1.z.string().nullable(),
     }),
 }));
-const failedOpenAICallResponseHandler = async ({ response, url, requestBodyValues }) => {
-    const responseBody = await response.text();
-    // resilient parsing in case the response is not JSON or does not match the schema:
-    try {
-        const parsedError = (0, parseJSON_js_1.parseJSON)({
-            text: responseBody,
-            schema: openAIErrorDataSchema,
-        });
-        return new ApiCallError_js_1.ApiCallError({
-            message: parsedError.error.message,
-            url,
-            requestBodyValues,
-            statusCode: response.status,
-            responseBody,
-            data: parsedError,
-            isRetryable: (response.status === 429 &&
-                // insufficient_quota is also reported as a 429, but it's not retryable:
-                parsedError?.error.type !== "insufficient_quota") ||
-                response.status >= 500,
-        });
-    }
-    catch (parseError) {
-        return new ApiCallError_js_1.ApiCallError({
-            message: responseBody.trim() !== "" ? responseBody : response.statusText,
-            url,
-            requestBodyValues,
-            statusCode: response.status,
-            responseBody,
-        });
-    }
-};
-exports.failedOpenAICallResponseHandler = failedOpenAICallResponseHandler;
+exports.failedOpenAICallResponseHandler = (0, postToApi_js_1.createJsonErrorResponseHandler)({
+    errorSchema: openAIErrorDataSchema,
+    errorToMessage: (error) => error.error.message,
+    isRetryable: (error, response) => (response.status === 429 &&
+        // insufficient_quota is also reported as a 429, but it's not retryable:
+        error.error.type !== "insufficient_quota") ||
+        response.status >= 500,
+});

package/model-provider/openai/OpenAIError.d.ts

@@ -1,5 +1,3 @@
-import { ApiCallError } from "../../core/api/ApiCallError.js";
-import { ResponseHandler } from "../../core/api/postToApi.js";
 import { ZodSchema } from "../../core/schema/ZodSchema.js";
 declare const openAIErrorDataSchema: ZodSchema<{
     error: {
@@ -10,5 +8,5 @@ declare const openAIErrorDataSchema: ZodSchema<{
     };
 }>;
 export type OpenAIErrorData = (typeof openAIErrorDataSchema)["_type"];
-export declare const failedOpenAICallResponseHandler: ResponseHandler<ApiCallError>;
+export declare const failedOpenAICallResponseHandler: import("../../core/api/postToApi.js").ResponseHandler<import("../../index.js").ApiCallError>;
 export {};

package/model-provider/openai/OpenAIError.js

@@ -1,7 +1,6 @@
 import { z } from "zod";
-import { ApiCallError } from "../../core/api/ApiCallError.js";
+import { createJsonErrorResponseHandler } from "../../core/api/postToApi.js";
 import { ZodSchema } from "../../core/schema/ZodSchema.js";
-import { parseJSON } from "../../core/schema/parseJSON.js";
 const openAIErrorDataSchema = new ZodSchema(z.object({
     error: z.object({
         message: z.string(),
@@ -10,34 +9,11 @@ const openAIErrorDataSchema = new ZodSchema(z.object({
         code: z.string().nullable(),
     }),
 }));
-export const failedOpenAICallResponseHandler = async ({ response, url, requestBodyValues }) => {
-    const responseBody = await response.text();
-    // resilient parsing in case the response is not JSON or does not match the schema:
-    try {
-        const parsedError = parseJSON({
-            text: responseBody,
-            schema: openAIErrorDataSchema,
-        });
-        return new ApiCallError({
-            message: parsedError.error.message,
-            url,
-            requestBodyValues,
-            statusCode: response.status,
-            responseBody,
-            data: parsedError,
-            isRetryable: (response.status === 429 &&
-                // insufficient_quota is also reported as a 429, but it's not retryable:
-                parsedError?.error.type !== "insufficient_quota") ||
-                response.status >= 500,
-        });
-    }
-    catch (parseError) {
-        return new ApiCallError({
-            message: responseBody.trim() !== "" ? responseBody : response.statusText,
-            url,
-            requestBodyValues,
-            statusCode: response.status,
-            responseBody,
-        });
-    }
-};
+export const failedOpenAICallResponseHandler = createJsonErrorResponseHandler({
+    errorSchema: openAIErrorDataSchema,
+    errorToMessage: (error) => error.error.message,
+    isRetryable: (error, response) => (response.status === 429 &&
+        // insufficient_quota is also reported as a 429, but it's not retryable:
+        error.error.type !== "insufficient_quota") ||
+        response.status >= 500,
+});