modelfusion 0.99.0 → 0.100.0

package/model-provider/ollama/OllamaChatModel.cjs

@@ -0,0 +1,303 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OllamaChatResponseFormat = exports.OllamaChatModel = void 0;
+const zod_1 = require("zod");
+const ApiCallError_js_1 = require("../../core/api/ApiCallError.cjs");
+const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndThrottle.cjs");
+const postToApi_js_1 = require("../../core/api/postToApi.cjs");
+const ZodSchema_js_1 = require("../../core/schema/ZodSchema.cjs");
+const parseJSON_js_1 = require("../../core/schema/parseJSON.cjs");
+const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
+const PromptTemplateTextStreamingModel_js_1 = require("../../model-function/generate-text/PromptTemplateTextStreamingModel.cjs");
+const TextGenerationToolCallModel_js_1 = require("../../tool/generate-tool-call/TextGenerationToolCallModel.cjs");
+const TextGenerationToolCallsOrGenerateTextModel_js_1 = require("../../tool/generate-tool-calls-or-text/TextGenerationToolCallsOrGenerateTextModel.cjs");
+const AsyncQueue_js_1 = require("../../util/AsyncQueue.cjs");
+const parseJsonStream_js_1 = require("../../util/streaming/parseJsonStream.cjs");
+const OllamaApiConfiguration_js_1 = require("./OllamaApiConfiguration.cjs");
+const OllamaChatPromptTemplate_js_1 = require("./OllamaChatPromptTemplate.cjs");
+const OllamaError_js_1 = require("./OllamaError.cjs");
+/**
+ * Text generation model that uses the Ollama chat API.
+ */
+class OllamaChatModel extends AbstractModel_js_1.AbstractModel {
+    constructor(settings) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "ollama"
+        });
+        Object.defineProperty(this, "tokenizer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "countPromptTokens", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "contextWindowSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+    }
+    get modelName() {
+        return this.settings.model;
+    }
+    async callAPI(prompt, options) {
+        const { responseFormat } = options;
+        const api = this.settings.api ?? new OllamaApiConfiguration_js_1.OllamaApiConfiguration();
+        const abortSignal = options.run?.abortSignal;
+        return (0, callWithRetryAndThrottle_js_1.callWithRetryAndThrottle)({
+            retry: api.retry,
+            throttle: api.throttle,
+            call: async () => (0, postToApi_js_1.postJsonToApi)({
+                url: api.assembleUrl(`/api/chat`),
+                headers: api.headers,
+                body: {
+                    stream: responseFormat.stream,
+                    model: this.settings.model,
+                    messages: prompt,
+                    format: this.settings.format,
+                    options: {
+                        mirostat: this.settings.mirostat,
+                        mirostat_eta: this.settings.mirostatEta,
+                        mirostat_tau: this.settings.mirostatTau,
+                        num_gpu: this.settings.numGpu,
+                        num_gqa: this.settings.numGqa,
+                        num_predict: this.settings.maxGenerationTokens,
+                        num_threads: this.settings.numThreads,
+                        repeat_last_n: this.settings.repeatLastN,
+                        repeat_penalty: this.settings.repeatPenalty,
+                        seed: this.settings.seed,
+                        stop: this.settings.stopSequences,
+                        temperature: this.settings.temperature,
+                        tfs_z: this.settings.tfsZ,
+                        top_k: this.settings.topK,
+                        top_p: this.settings.topP,
+                    },
+                    template: this.settings.template,
+                },
+                failedResponseHandler: OllamaError_js_1.failedOllamaCallResponseHandler,
+                successfulResponseHandler: responseFormat.handler,
+                abortSignal,
+            }),
+        });
+    }
+    get settingsForEvent() {
+        const eventSettingProperties = [
+            "maxGenerationTokens",
+            "stopSequences",
+            "temperature",
+            "mirostat",
+            "mirostatEta",
+            "mirostatTau",
+            "numGqa",
+            "numGpu",
+            "numThreads",
+            "repeatLastN",
+            "repeatPenalty",
+            "seed",
+            "tfsZ",
+            "topK",
+            "topP",
+            "template",
+            "format",
+        ];
+        return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
+    }
+    async doGenerateTexts(prompt, options) {
+        const response = await this.callAPI(prompt, {
+            ...options,
+            responseFormat: exports.OllamaChatResponseFormat.json,
+        });
+        return {
+            response,
+            texts: [response.message.content],
+        };
+    }
+    doStreamText(prompt, options) {
+        return this.callAPI(prompt, {
+            ...options,
+            responseFormat: exports.OllamaChatResponseFormat.deltaIterable,
+        });
+    }
+    asToolCallGenerationModel(promptTemplate) {
+        return new TextGenerationToolCallModel_js_1.TextGenerationToolCallModel({
+            model: this,
+            format: promptTemplate,
+        });
+    }
+    asToolCallsOrTextGenerationModel(promptTemplate) {
+        return new TextGenerationToolCallsOrGenerateTextModel_js_1.TextGenerationToolCallsOrGenerateTextModel({
+            model: this,
+            template: promptTemplate,
+        });
+    }
+    /**
+     * Returns this model with a text prompt template.
+     */
+    withTextPrompt() {
+        return this.withPromptTemplate((0, OllamaChatPromptTemplate_js_1.text)());
+    }
+    /**
+     * Returns this model with an instruction prompt template.
+     */
+    withInstructionPrompt() {
+        return this.withPromptTemplate((0, OllamaChatPromptTemplate_js_1.instruction)());
+    }
+    /**
+     * Returns this model with a chat prompt template.
+     */
+    withChatPrompt() {
+        return this.withPromptTemplate((0, OllamaChatPromptTemplate_js_1.chat)());
+    }
+    withPromptTemplate(promptTemplate) {
+        return new PromptTemplateTextStreamingModel_js_1.PromptTemplateTextStreamingModel({
+            model: this.withSettings({
+                stopSequences: [
+                    ...(this.settings.stopSequences ?? []),
+                    ...promptTemplate.stopSequences,
+                ],
+            }),
+            promptTemplate,
+        });
+    }
+    withSettings(additionalSettings) {
+        return new OllamaChatModel(Object.assign({}, this.settings, additionalSettings));
+    }
+}
+exports.OllamaChatModel = OllamaChatModel;
+const ollamaChatResponseSchema = zod_1.z.object({
+    model: zod_1.z.string(),
+    created_at: zod_1.z.string(),
+    done: zod_1.z.literal(true),
+    message: zod_1.z.object({
+        role: zod_1.z.string(),
+        content: zod_1.z.string(),
+    }),
+    total_duration: zod_1.z.number(),
+    load_duration: zod_1.z.number().optional(),
+    prompt_eval_count: zod_1.z.number(),
+    prompt_eval_duration: zod_1.z.number().optional(),
+    eval_count: zod_1.z.number(),
+    eval_duration: zod_1.z.number(),
+});
+const ollamaChatStreamSchema = new ZodSchema_js_1.ZodSchema(zod_1.z.discriminatedUnion("done", [
+    zod_1.z.object({
+        done: zod_1.z.literal(false),
+        model: zod_1.z.string(),
+        created_at: zod_1.z.string(),
+        message: zod_1.z.object({
+            role: zod_1.z.string(),
+            content: zod_1.z.string(),
+        }),
+    }),
+    zod_1.z.object({
+        done: zod_1.z.literal(true),
+        model: zod_1.z.string(),
+        created_at: zod_1.z.string(),
+        total_duration: zod_1.z.number(),
+        load_duration: zod_1.z.number().optional(),
+        prompt_eval_count: zod_1.z.number(),
+        prompt_eval_duration: zod_1.z.number().optional(),
+        eval_count: zod_1.z.number(),
+        eval_duration: zod_1.z.number(),
+    }),
+]));
+async function createOllamaFullDeltaIterableQueue(stream) {
+    const queue = new AsyncQueue_js_1.AsyncQueue();
+    let accumulatedText = "";
+    // process the stream asynchonously (no 'await' on purpose):
+    (0, parseJsonStream_js_1.parseJsonStream)({
+        stream,
+        schema: ollamaChatStreamSchema,
+        process(event) {
+            if (event.done === true) {
+                queue.push({
+                    type: "delta",
+                    fullDelta: {
+                        content: accumulatedText,
+                        isComplete: true,
+                        delta: "",
+                    },
+                    valueDelta: "",
+                });
+            }
+            else {
+                const deltaText = event.message.content;
+                accumulatedText += deltaText;
+                queue.push({
+                    type: "delta",
+                    fullDelta: {
+                        content: accumulatedText,
+                        isComplete: false,
+                        delta: deltaText,
+                    },
+                    valueDelta: deltaText,
+                });
+            }
+        },
+        onDone() {
+            queue.close();
+        },
+    });
+    return queue;
+}
+exports.OllamaChatResponseFormat = {
+    /**
+     * Returns the response as a JSON object.
+     */
+    json: {
+        stream: false,
+        handler: (async ({ response, url, requestBodyValues }) => {
+            const responseBody = await response.text();
+            const parsedResult = (0, parseJSON_js_1.safeParseJSON)({
+                text: responseBody,
+                schema: new ZodSchema_js_1.ZodSchema(zod_1.z.union([
+                    ollamaChatResponseSchema,
+                    zod_1.z.object({
+                        done: zod_1.z.literal(false),
+                        model: zod_1.z.string(),
+                        created_at: zod_1.z.string(),
+                    }),
+                ])),
+            });
+            if (!parsedResult.success) {
+                throw new ApiCallError_js_1.ApiCallError({
+                    message: "Invalid JSON response",
+                    cause: parsedResult.error,
+                    statusCode: response.status,
+                    responseBody,
+                    url,
+                    requestBodyValues,
+                });
+            }
+            if (parsedResult.data.done === false) {
+                throw new ApiCallError_js_1.ApiCallError({
+                    message: "Incomplete Ollama response received",
+                    statusCode: response.status,
+                    responseBody,
+                    url,
+                    requestBodyValues,
+                    isRetryable: true,
+                });
+            }
+            return parsedResult.data;
+        }),
+    },
+    /**
+     * Returns an async iterable over the full deltas (all choices, including full current state at time of event)
+     * of the response stream.
+     */
+    deltaIterable: {
+        stream: true,
+        handler: async ({ response }) => createOllamaFullDeltaIterableQueue(response.body),
+    },
+};

package/model-provider/ollama/OllamaChatModel.d.ts

@@ -0,0 +1,171 @@
+import { z } from "zod";
+import { FunctionOptions } from "../../core/FunctionOptions.js";
+import { ApiConfiguration } from "../../core/api/ApiConfiguration.js";
+import { ResponseHandler } from "../../core/api/postToApi.js";
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { Delta } from "../../model-function/Delta.js";
+import { PromptTemplateTextStreamingModel } from "../../model-function/generate-text/PromptTemplateTextStreamingModel.js";
+import { TextStreamingModel } from "../../model-function/generate-text/TextGenerationModel.js";
+import { TextGenerationPromptTemplate } from "../../model-function/generate-text/TextGenerationPromptTemplate.js";
+import { TextGenerationToolCallModel, ToolCallPromptTemplate } from "../../tool/generate-tool-call/TextGenerationToolCallModel.js";
+import { TextGenerationToolCallsOrGenerateTextModel, ToolCallsOrGenerateTextPromptTemplate } from "../../tool/generate-tool-calls-or-text/TextGenerationToolCallsOrGenerateTextModel.js";
+import { OllamaTextGenerationSettings } from "./OllamaTextGenerationSettings.js";
+export type OllamaChatMessage = {
+    role: "system" | "user" | "assistant";
+    content: string;
+    /**
+     Images. Supports base64-encoded `png` and `jpeg` images up to 100MB in size.
+     */
+    images?: Array<string>;
+};
+export type OllamaChatPrompt = Array<OllamaChatMessage>;
+export interface OllamaChatModelSettings extends OllamaTextGenerationSettings {
+    api?: ApiConfiguration;
+}
+/**
+ * Text generation model that uses the Ollama chat API.
+ */
+export declare class OllamaChatModel extends AbstractModel<OllamaChatModelSettings> implements TextStreamingModel<OllamaChatPrompt, OllamaChatModelSettings> {
+    constructor(settings: OllamaChatModelSettings);
+    readonly provider = "ollama";
+    get modelName(): string;
+    readonly tokenizer: undefined;
+    readonly countPromptTokens: undefined;
+    readonly contextWindowSize: undefined;
+    callAPI<RESPONSE>(prompt: OllamaChatPrompt, options: {
+        responseFormat: OllamaChatResponseFormatType<RESPONSE>;
+    } & FunctionOptions): Promise<RESPONSE>;
+    get settingsForEvent(): Partial<OllamaChatModelSettings>;
+    doGenerateTexts(prompt: OllamaChatPrompt, options?: FunctionOptions): Promise<{
+        response: {
+            model: string;
+            message: {
+                role: string;
+                content: string;
+            };
+            done: true;
+            created_at: string;
+            total_duration: number;
+            prompt_eval_count: number;
+            eval_count: number;
+            eval_duration: number;
+            load_duration?: number | undefined;
+            prompt_eval_duration?: number | undefined;
+        };
+        texts: string[];
+    }>;
+    doStreamText(prompt: OllamaChatPrompt, options?: FunctionOptions): Promise<AsyncIterable<Delta<string>>>;
+    asToolCallGenerationModel<INPUT_PROMPT>(promptTemplate: ToolCallPromptTemplate<INPUT_PROMPT, OllamaChatPrompt>): TextGenerationToolCallModel<INPUT_PROMPT, OllamaChatPrompt, this>;
+    asToolCallsOrTextGenerationModel<INPUT_PROMPT>(promptTemplate: ToolCallsOrGenerateTextPromptTemplate<INPUT_PROMPT, OllamaChatPrompt>): TextGenerationToolCallsOrGenerateTextModel<INPUT_PROMPT, OllamaChatPrompt, this>;
+    /**
+     * Returns this model with a text prompt template.
+     */
+    withTextPrompt(): PromptTemplateTextStreamingModel<string, OllamaChatPrompt, OllamaChatModelSettings, this>;
+    /**
+     * Returns this model with an instruction prompt template.
+     */
+    withInstructionPrompt(): PromptTemplateTextStreamingModel<import("../../index.js").MultiModalInstructionPrompt | import("../../index.js").TextInstructionPrompt, OllamaChatPrompt, OllamaChatModelSettings, this>;
+    /**
+     * Returns this model with a chat prompt template.
+     */
+    withChatPrompt(): PromptTemplateTextStreamingModel<import("../../index.js").TextChatPrompt | import("../../index.js").MultiModalChatPrompt, OllamaChatPrompt, OllamaChatModelSettings, this>;
+    withPromptTemplate<INPUT_PROMPT>(promptTemplate: TextGenerationPromptTemplate<INPUT_PROMPT, OllamaChatPrompt>): PromptTemplateTextStreamingModel<INPUT_PROMPT, OllamaChatPrompt, OllamaChatModelSettings, this>;
+    withSettings(additionalSettings: Partial<OllamaChatModelSettings>): this;
+}
+declare const ollamaChatResponseSchema: z.ZodObject<{
+    model: z.ZodString;
+    created_at: z.ZodString;
+    done: z.ZodLiteral<true>;
+    message: z.ZodObject<{
+        role: z.ZodString;
+        content: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        role: string;
+        content: string;
+    }, {
+        role: string;
+        content: string;
+    }>;
+    total_duration: z.ZodNumber;
+    load_duration: z.ZodOptional<z.ZodNumber>;
+    prompt_eval_count: z.ZodNumber;
+    prompt_eval_duration: z.ZodOptional<z.ZodNumber>;
+    eval_count: z.ZodNumber;
+    eval_duration: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    model: string;
+    message: {
+        role: string;
+        content: string;
+    };
+    done: true;
+    created_at: string;
+    total_duration: number;
+    prompt_eval_count: number;
+    eval_count: number;
+    eval_duration: number;
+    load_duration?: number | undefined;
+    prompt_eval_duration?: number | undefined;
+}, {
+    model: string;
+    message: {
+        role: string;
+        content: string;
+    };
+    done: true;
+    created_at: string;
+    total_duration: number;
+    prompt_eval_count: number;
+    eval_count: number;
+    eval_duration: number;
+    load_duration?: number | undefined;
+    prompt_eval_duration?: number | undefined;
+}>;
+export type OllamaChatResponse = z.infer<typeof ollamaChatResponseSchema>;
+export type OllamaChatDelta = {
+    content: string;
+    isComplete: boolean;
+    delta: string;
+};
+export type OllamaChatResponseFormatType<T> = {
+    stream: boolean;
+    handler: ResponseHandler<T>;
+};
+export declare const OllamaChatResponseFormat: {
+    /**
+     * Returns the response as a JSON object.
+     */
+    json: {
+        stream: false;
+        handler: ({ response, url, requestBodyValues }: {
+            url: string;
+            requestBodyValues: unknown;
+            response: Response;
+        }) => Promise<{
+            model: string;
+            message: {
+                role: string;
+                content: string;
+            };
+            done: true;
+            created_at: string;
+            total_duration: number;
+            prompt_eval_count: number;
+            eval_count: number;
+            eval_duration: number;
+            load_duration?: number | undefined;
+            prompt_eval_duration?: number | undefined;
+        }>;
+    };
+    /**
+     * Returns an async iterable over the full deltas (all choices, including full current state at time of event)
+     * of the response stream.
+     */
+    deltaIterable: {
+        stream: true;
+        handler: ({ response }: {
+            response: Response;
+        }) => Promise<AsyncIterable<Delta<string>>>;
+    };
+};
+export {};