modelfusion 0.99.0 → 0.100.0

package/model-provider/ollama/OllamaChatModel.js

@@ -0,0 +1,299 @@
+import { z } from "zod";
+import { ApiCallError } from "../../core/api/ApiCallError.js";
+import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottle.js";
+import { postJsonToApi } from "../../core/api/postToApi.js";
+import { ZodSchema } from "../../core/schema/ZodSchema.js";
+import { safeParseJSON } from "../../core/schema/parseJSON.js";
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { PromptTemplateTextStreamingModel } from "../../model-function/generate-text/PromptTemplateTextStreamingModel.js";
+import { TextGenerationToolCallModel, } from "../../tool/generate-tool-call/TextGenerationToolCallModel.js";
+import { TextGenerationToolCallsOrGenerateTextModel, } from "../../tool/generate-tool-calls-or-text/TextGenerationToolCallsOrGenerateTextModel.js";
+import { AsyncQueue } from "../../util/AsyncQueue.js";
+import { parseJsonStream } from "../../util/streaming/parseJsonStream.js";
+import { OllamaApiConfiguration } from "./OllamaApiConfiguration.js";
+import { chat, instruction, text } from "./OllamaChatPromptTemplate.js";
+import { failedOllamaCallResponseHandler } from "./OllamaError.js";
+/**
+ * Text generation model that uses the Ollama chat API.
+ */
+export class OllamaChatModel extends AbstractModel {
+    constructor(settings) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "ollama"
+        });
+        Object.defineProperty(this, "tokenizer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "countPromptTokens", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "contextWindowSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+    }
+    get modelName() {
+        return this.settings.model;
+    }
+    async callAPI(prompt, options) {
+        const { responseFormat } = options;
+        const api = this.settings.api ?? new OllamaApiConfiguration();
+        const abortSignal = options.run?.abortSignal;
+        return callWithRetryAndThrottle({
+            retry: api.retry,
+            throttle: api.throttle,
+            call: async () => postJsonToApi({
+                url: api.assembleUrl(`/api/chat`),
+                headers: api.headers,
+                body: {
+                    stream: responseFormat.stream,
+                    model: this.settings.model,
+                    messages: prompt,
+                    format: this.settings.format,
+                    options: {
+                        mirostat: this.settings.mirostat,
+                        mirostat_eta: this.settings.mirostatEta,
+                        mirostat_tau: this.settings.mirostatTau,
+                        num_gpu: this.settings.numGpu,
+                        num_gqa: this.settings.numGqa,
+                        num_predict: this.settings.maxGenerationTokens,
+                        num_threads: this.settings.numThreads,
+                        repeat_last_n: this.settings.repeatLastN,
+                        repeat_penalty: this.settings.repeatPenalty,
+                        seed: this.settings.seed,
+                        stop: this.settings.stopSequences,
+                        temperature: this.settings.temperature,
+                        tfs_z: this.settings.tfsZ,
+                        top_k: this.settings.topK,
+                        top_p: this.settings.topP,
+                    },
+                    template: this.settings.template,
+                },
+                failedResponseHandler: failedOllamaCallResponseHandler,
+                successfulResponseHandler: responseFormat.handler,
+                abortSignal,
+            }),
+        });
+    }
+    get settingsForEvent() {
+        const eventSettingProperties = [
+            "maxGenerationTokens",
+            "stopSequences",
+            "temperature",
+            "mirostat",
+            "mirostatEta",
+            "mirostatTau",
+            "numGqa",
+            "numGpu",
+            "numThreads",
+            "repeatLastN",
+            "repeatPenalty",
+            "seed",
+            "tfsZ",
+            "topK",
+            "topP",
+            "template",
+            "format",
+        ];
+        return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
+    }
+    async doGenerateTexts(prompt, options) {
+        const response = await this.callAPI(prompt, {
+            ...options,
+            responseFormat: OllamaChatResponseFormat.json,
+        });
+        return {
+            response,
+            texts: [response.message.content],
+        };
+    }
+    doStreamText(prompt, options) {
+        return this.callAPI(prompt, {
+            ...options,
+            responseFormat: OllamaChatResponseFormat.deltaIterable,
+        });
+    }
+    asToolCallGenerationModel(promptTemplate) {
+        return new TextGenerationToolCallModel({
+            model: this,
+            format: promptTemplate,
+        });
+    }
+    asToolCallsOrTextGenerationModel(promptTemplate) {
+        return new TextGenerationToolCallsOrGenerateTextModel({
+            model: this,
+            template: promptTemplate,
+        });
+    }
+    /**
+     * Returns this model with a text prompt template.
+     */
+    withTextPrompt() {
+        return this.withPromptTemplate(text());
+    }
+    /**
+     * Returns this model with an instruction prompt template.
+     */
+    withInstructionPrompt() {
+        return this.withPromptTemplate(instruction());
+    }
+    /**
+     * Returns this model with a chat prompt template.
+     */
+    withChatPrompt() {
+        return this.withPromptTemplate(chat());
+    }
+    withPromptTemplate(promptTemplate) {
+        return new PromptTemplateTextStreamingModel({
+            model: this.withSettings({
+                stopSequences: [
+                    ...(this.settings.stopSequences ?? []),
+                    ...promptTemplate.stopSequences,
+                ],
+            }),
+            promptTemplate,
+        });
+    }
+    withSettings(additionalSettings) {
+        return new OllamaChatModel(Object.assign({}, this.settings, additionalSettings));
+    }
+}
+const ollamaChatResponseSchema = z.object({
+    model: z.string(),
+    created_at: z.string(),
+    done: z.literal(true),
+    message: z.object({
+        role: z.string(),
+        content: z.string(),
+    }),
+    total_duration: z.number(),
+    load_duration: z.number().optional(),
+    prompt_eval_count: z.number(),
+    prompt_eval_duration: z.number().optional(),
+    eval_count: z.number(),
+    eval_duration: z.number(),
+});
+const ollamaChatStreamSchema = new ZodSchema(z.discriminatedUnion("done", [
+    z.object({
+        done: z.literal(false),
+        model: z.string(),
+        created_at: z.string(),
+        message: z.object({
+            role: z.string(),
+            content: z.string(),
+        }),
+    }),
+    z.object({
+        done: z.literal(true),
+        model: z.string(),
+        created_at: z.string(),
+        total_duration: z.number(),
+        load_duration: z.number().optional(),
+        prompt_eval_count: z.number(),
+        prompt_eval_duration: z.number().optional(),
+        eval_count: z.number(),
+        eval_duration: z.number(),
+    }),
+]));
+async function createOllamaFullDeltaIterableQueue(stream) {
+    const queue = new AsyncQueue();
+    let accumulatedText = "";
+    // process the stream asynchonously (no 'await' on purpose):
+    parseJsonStream({
+        stream,
+        schema: ollamaChatStreamSchema,
+        process(event) {
+            if (event.done === true) {
+                queue.push({
+                    type: "delta",
+                    fullDelta: {
+                        content: accumulatedText,
+                        isComplete: true,
+                        delta: "",
+                    },
+                    valueDelta: "",
+                });
+            }
+            else {
+                const deltaText = event.message.content;
+                accumulatedText += deltaText;
+                queue.push({
+                    type: "delta",
+                    fullDelta: {
+                        content: accumulatedText,
+                        isComplete: false,
+                        delta: deltaText,
+                    },
+                    valueDelta: deltaText,
+                });
+            }
+        },
+        onDone() {
+            queue.close();
+        },
+    });
+    return queue;
+}
+export const OllamaChatResponseFormat = {
+    /**
+     * Returns the response as a JSON object.
+     */
+    json: {
+        stream: false,
+        handler: (async ({ response, url, requestBodyValues }) => {
+            const responseBody = await response.text();
+            const parsedResult = safeParseJSON({
+                text: responseBody,
+                schema: new ZodSchema(z.union([
+                    ollamaChatResponseSchema,
+                    z.object({
+                        done: z.literal(false),
+                        model: z.string(),
+                        created_at: z.string(),
+                    }),
+                ])),
+            });
+            if (!parsedResult.success) {
+                throw new ApiCallError({
+                    message: "Invalid JSON response",
+                    cause: parsedResult.error,
+                    statusCode: response.status,
+                    responseBody,
+                    url,
+                    requestBodyValues,
+                });
+            }
+            if (parsedResult.data.done === false) {
+                throw new ApiCallError({
+                    message: "Incomplete Ollama response received",
+                    statusCode: response.status,
+                    responseBody,
+                    url,
+                    requestBodyValues,
+                    isRetryable: true,
+                });
+            }
+            return parsedResult.data;
+        }),
+    },
+    /**
+     * Returns an async iterable over the full deltas (all choices, including full current state at time of event)
+     * of the response stream.
+     */
+    deltaIterable: {
+        stream: true,
+        handler: async ({ response }) => createOllamaFullDeltaIterableQueue(response.body),
+    },
+};

package/model-provider/ollama/OllamaChatPromptTemplate.cjs

@@ -0,0 +1,76 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.chat = exports.instruction = exports.text = exports.identity = void 0;
+const ChatPrompt_js_1 = require("../../model-function/generate-text/prompt-template/ChatPrompt.cjs");
+/**
+ * OllamaChatPrompt identity chat format.
+ */
+function identity() {
+    return { format: (prompt) => prompt, stopSequences: [] };
+}
+exports.identity = identity;
+/**
+ * Formats a text prompt as an Ollama chat prompt.
+ */
+function text() {
+    return {
+        format: (prompt) => [{ role: "user", content: prompt }],
+        stopSequences: [],
+    };
+}
+exports.text = text;
+/**
+ * Formats an instruction prompt as an Ollama chat prompt.
+ */
+function instruction() {
+    return {
+        format(prompt) {
+            const messages = [];
+            if (prompt.system != null) {
+                messages.push({ role: "system", content: prompt.system });
+            }
+            messages.push({ role: "user", ...extractContent(prompt.instruction) });
+            return messages;
+        },
+        stopSequences: [],
+    };
+}
+exports.instruction = instruction;
+/**
+ * Formats a chat prompt as an Ollama chat prompt.
+ */
+function chat() {
+    return {
+        format(prompt) {
+            (0, ChatPrompt_js_1.validateChatPrompt)(prompt);
+            const messages = [];
+            if (prompt.system != null) {
+                messages.push({ role: "system", content: prompt.system });
+            }
+            for (const { role, content } of prompt.messages) {
+                messages.push({ role, ...extractContent(content) });
+            }
+            return messages;
+        },
+        stopSequences: [],
+    };
+}
+exports.chat = chat;
+function extractContent(input) {
+    if (typeof input === "string") {
+        return { content: input, images: undefined };
+    }
+    else {
+        const images = [];
+        let content = "";
+        for (const part of input) {
+            if (part.type === "text") {
+                content += part.text;
+            }
+            else {
+                images.push(part.base64Image);
+            }
+        }
+        return { content, images };
+    }
+}

package/model-provider/ollama/OllamaChatPromptTemplate.d.ts

@@ -0,0 +1,20 @@
+import { TextGenerationPromptTemplate } from "../../model-function/generate-text/TextGenerationPromptTemplate.js";
+import { MultiModalChatPrompt, TextChatPrompt } from "../../model-function/generate-text/prompt-template/ChatPrompt.js";
+import { MultiModalInstructionPrompt, TextInstructionPrompt } from "../../model-function/generate-text/prompt-template/InstructionPrompt.js";
+import { OllamaChatPrompt } from "./OllamaChatModel.js";
+/**
+ * OllamaChatPrompt identity chat format.
+ */
+export declare function identity(): TextGenerationPromptTemplate<OllamaChatPrompt, OllamaChatPrompt>;
+/**
+ * Formats a text prompt as an Ollama chat prompt.
+ */
+export declare function text(): TextGenerationPromptTemplate<string, OllamaChatPrompt>;
+/**
+ * Formats an instruction prompt as an Ollama chat prompt.
+ */
+export declare function instruction(): TextGenerationPromptTemplate<MultiModalInstructionPrompt | TextInstructionPrompt, OllamaChatPrompt>;
+/**
+ * Formats a chat prompt as an Ollama chat prompt.
+ */
+export declare function chat(): TextGenerationPromptTemplate<MultiModalChatPrompt | TextChatPrompt, OllamaChatPrompt>;

package/model-provider/ollama/OllamaChatPromptTemplate.js

@@ -0,0 +1,69 @@
+import { validateChatPrompt, } from "../../model-function/generate-text/prompt-template/ChatPrompt.js";
+/**
+ * OllamaChatPrompt identity chat format.
+ */
+export function identity() {
+    return { format: (prompt) => prompt, stopSequences: [] };
+}
+/**
+ * Formats a text prompt as an Ollama chat prompt.
+ */
+export function text() {
+    return {
+        format: (prompt) => [{ role: "user", content: prompt }],
+        stopSequences: [],
+    };
+}
+/**
+ * Formats an instruction prompt as an Ollama chat prompt.
+ */
+export function instruction() {
+    return {
+        format(prompt) {
+            const messages = [];
+            if (prompt.system != null) {
+                messages.push({ role: "system", content: prompt.system });
+            }
+            messages.push({ role: "user", ...extractContent(prompt.instruction) });
+            return messages;
+        },
+        stopSequences: [],
+    };
+}
+/**
+ * Formats a chat prompt as an Ollama chat prompt.
+ */
+export function chat() {
+    return {
+        format(prompt) {
+            validateChatPrompt(prompt);
+            const messages = [];
+            if (prompt.system != null) {
+                messages.push({ role: "system", content: prompt.system });
+            }
+            for (const { role, content } of prompt.messages) {
+                messages.push({ role, ...extractContent(content) });
+            }
+            return messages;
+        },
+        stopSequences: [],
+    };
+}
+function extractContent(input) {
+    if (typeof input === "string") {
+        return { content: input, images: undefined };
+    }
+    else {
+        const images = [];
+        let content = "";
+        for (const part of input) {
+            if (part.type === "text") {
+                content += part.text;
+            }
+            else {
+                images.push(part.base64Image);
+            }
+        }
+        return { content, images };
+    }
+}

package/model-provider/ollama/{OllamaTextGenerationModel.cjs → OllamaCompletionModel.cjs}

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.OllamaTextGenerationResponseFormat = exports.OllamaTextGenerationModel = void 0;
+exports.OllamaCompletionResponseFormat = exports.OllamaCompletionModel = void 0;
 const zod_1 = require("zod");
 const ApiCallError_js_1 = require("../../core/api/ApiCallError.cjs");
 const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndThrottle.cjs");
@@ -15,7 +15,7 @@ const AsyncQueue_js_1 = require("../../util/AsyncQueue.cjs");
 const parseJsonStream_js_1 = require("../../util/streaming/parseJsonStream.cjs");
 const OllamaApiConfiguration_js_1 = require("./OllamaApiConfiguration.cjs");
 const OllamaError_js_1 = require("./OllamaError.cjs");
-class OllamaTextGenerationModel extends AbstractModel_js_1.AbstractModel {
+class OllamaCompletionModel extends AbstractModel_js_1.AbstractModel {
     constructor(settings) {
         super({ settings });
         Object.defineProperty(this, "provider", {
@@ -117,7 +117,7 @@ class OllamaTextGenerationModel extends AbstractModel_js_1.AbstractModel {
     async doGenerateTexts(prompt, options) {
         const response = await this.callAPI(prompt, {
             ...options,
-            responseFormat: exports.OllamaTextGenerationResponseFormat.json,
+            responseFormat: exports.OllamaCompletionResponseFormat.json,
         });
         return {
             response,
@@ -127,7 +127,7 @@ class OllamaTextGenerationModel extends AbstractModel_js_1.AbstractModel {
     doStreamText(prompt, options) {
         return this.callAPI(prompt, {
             ...options,
-            responseFormat: exports.OllamaTextGenerationResponseFormat.deltaIterable,
+            responseFormat: exports.OllamaCompletionResponseFormat.deltaIterable,
         });
     }
     asToolCallGenerationModel(promptTemplate) {
@@ -162,22 +162,24 @@ class OllamaTextGenerationModel extends AbstractModel_js_1.AbstractModel {
         });
     }
     withSettings(additionalSettings) {
-        return new OllamaTextGenerationModel(Object.assign({}, this.settings, additionalSettings));
+        return new OllamaCompletionModel(Object.assign({}, this.settings, additionalSettings));
     }
 }
-exports.OllamaTextGenerationModel = OllamaTextGenerationModel;
-const ollamaTextGenerationResponseSchema = zod_1.z.object({
+exports.OllamaCompletionModel = OllamaCompletionModel;
+const ollamaCompletionResponseSchema = zod_1.z.object({
     done: zod_1.z.literal(true),
     model: zod_1.z.string(),
+    created_at: zod_1.z.string(),
     response: zod_1.z.string(),
     total_duration: zod_1.z.number(),
     load_duration: zod_1.z.number().optional(),
     prompt_eval_count: zod_1.z.number(),
+    prompt_eval_duration: zod_1.z.number().optional(),
     eval_count: zod_1.z.number(),
     eval_duration: zod_1.z.number(),
     context: zod_1.z.array(zod_1.z.number()).optional(),
 });
-const ollamaTextStreamingResponseSchema = new ZodSchema_js_1.ZodSchema(zod_1.z.discriminatedUnion("done", [
+const ollamaCompletionStreamSchema = new ZodSchema_js_1.ZodSchema(zod_1.z.discriminatedUnion("done", [
     zod_1.z.object({
         done: zod_1.z.literal(false),
         model: zod_1.z.string(),
@@ -205,7 +207,7 @@ async function createOllamaFullDeltaIterableQueue(stream) {
     // process the stream asynchonously (no 'await' on purpose):
     (0, parseJsonStream_js_1.parseJsonStream)({
         stream,
-        schema: ollamaTextStreamingResponseSchema,
+        schema: ollamaCompletionStreamSchema,
         process(event) {
             if (event.done === true) {
                 queue.push({
@@ -237,7 +239,7 @@ async function createOllamaFullDeltaIterableQueue(stream) {
     });
     return queue;
 }
-exports.OllamaTextGenerationResponseFormat = {
+exports.OllamaCompletionResponseFormat = {
     /**
      * Returns the response as a JSON object.
      */
@@ -248,7 +250,7 @@ exports.OllamaTextGenerationResponseFormat = {
             const parsedResult = (0, parseJSON_js_1.safeParseJSON)({
                 text: responseBody,
                 schema: new ZodSchema_js_1.ZodSchema(zod_1.z.union([
-                    ollamaTextGenerationResponseSchema,
+                    ollamaCompletionResponseSchema,
                     zod_1.z.object({
                         done: zod_1.z.literal(false),
                         model: zod_1.z.string(),