modelfusion 0.53.2 → 0.55.0

package/model-provider/llamacpp/LlamaCppTextGenerationModel.cjs

@@ -5,7 +5,7 @@ const zod_1 = require("zod");
 const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndThrottle.cjs");
 const postToApi_js_1 = require("../../core/api/postToApi.cjs");
 const AsyncQueue_js_1 = require("../../util/AsyncQueue.cjs");
-const parseEventSourceStream_js_1 = require("../../event-source/parseEventSourceStream.cjs");
+const parseEventSourceStream_js_1 = require("../../util/streaming/parseEventSourceStream.cjs");
 const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
 const PromptFormatTextStreamingModel_js_1 = require("../../model-function/generate-text/PromptFormatTextStreamingModel.cjs");
 const parseJSON_js_1 = require("../../util/parseJSON.cjs");

package/model-provider/llamacpp/LlamaCppTextGenerationModel.js

@@ -2,7 +2,7 @@ import { z } from "zod";
 import { callWithRetryAndThrottle } from "../../core/api/callWithRetryAndThrottle.js";
 import { createJsonResponseHandler, postJsonToApi, } from "../../core/api/postToApi.js";
 import { AsyncQueue } from "../../util/AsyncQueue.js";
-import { parseEventSourceStream } from "../../event-source/parseEventSourceStream.js";
+import { parseEventSourceStream } from "../../util/streaming/parseEventSourceStream.js";
 import { AbstractModel } from "../../model-function/AbstractModel.js";
 import { PromptFormatTextStreamingModel } from "../../model-function/generate-text/PromptFormatTextStreamingModel.js";
 import { parseJsonWithZod } from "../../util/parseJSON.js";

package/model-provider/ollama/OllamaApiConfiguration.cjs

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OllamaApiConfiguration = void 0;
+const BaseUrlApiConfiguration_js_1 = require("../../core/api/BaseUrlApiConfiguration.cjs");
+class OllamaApiConfiguration extends BaseUrlApiConfiguration_js_1.BaseUrlApiConfiguration {
+    constructor({ baseUrl = "http://127.0.0.1:11434", retry, throttle, } = {}) {
+        super({
+            baseUrl,
+            headers: {},
+            retry,
+            throttle,
+        });
+    }
+}
+exports.OllamaApiConfiguration = OllamaApiConfiguration;

package/model-provider/ollama/OllamaApiConfiguration.d.ts

@@ -0,0 +1,10 @@
+import { BaseUrlApiConfiguration } from "../../core/api/BaseUrlApiConfiguration.js";
+import { RetryFunction } from "../../core/api/RetryFunction.js";
+import { ThrottleFunction } from "../../core/api/ThrottleFunction.js";
+export declare class OllamaApiConfiguration extends BaseUrlApiConfiguration {
+    constructor({ baseUrl, retry, throttle, }?: {
+        baseUrl?: string;
+        retry?: RetryFunction;
+        throttle?: ThrottleFunction;
+    });
+}

package/model-provider/ollama/OllamaApiConfiguration.js

@@ -0,0 +1,11 @@
+import { BaseUrlApiConfiguration } from "../../core/api/BaseUrlApiConfiguration.js";
+export class OllamaApiConfiguration extends BaseUrlApiConfiguration {
+    constructor({ baseUrl = "http://127.0.0.1:11434", retry, throttle, } = {}) {
+        super({
+            baseUrl,
+            headers: {},
+            retry,
+            throttle,
+        });
+    }
+}

package/model-provider/ollama/OllamaError.cjs

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.failedOllamaCallResponseHandler = exports.OllamaError = exports.ollamaErrorDataSchema = void 0;
+const zod_1 = require("zod");
+const ApiCallError_js_1 = require("../../core/api/ApiCallError.cjs");
+const parseJSON_js_1 = require("../../util/parseJSON.cjs");
+exports.ollamaErrorDataSchema = zod_1.z.object({
+    error: zod_1.z.string(),
+});
+class OllamaError extends ApiCallError_js_1.ApiCallError {
+    constructor({ data, statusCode, url, requestBodyValues, message = data.error, }) {
+        super({ message, statusCode, requestBodyValues, url });
+        Object.defineProperty(this, "data", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.data = data;
+    }
+}
+exports.OllamaError = OllamaError;
+const failedOllamaCallResponseHandler = async ({ response, url, requestBodyValues }) => new OllamaError({
+    url,
+    requestBodyValues,
+    statusCode: response.status,
+    data: (0, parseJSON_js_1.parseJsonWithZod)(await response.text(), exports.ollamaErrorDataSchema),
+});
+exports.failedOllamaCallResponseHandler = failedOllamaCallResponseHandler;

package/model-provider/ollama/OllamaError.d.ts

@@ -0,0 +1,22 @@
+import { z } from "zod";
+import { ApiCallError } from "../../core/api/ApiCallError.js";
+import { ResponseHandler } from "../../core/api/postToApi.js";
+export declare const ollamaErrorDataSchema: z.ZodObject<{
+    error: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    error: string;
+}, {
+    error: string;
+}>;
+export type OllamaErrorData = z.infer<typeof ollamaErrorDataSchema>;
+export declare class OllamaError extends ApiCallError {
+    readonly data: OllamaErrorData;
+    constructor({ data, statusCode, url, requestBodyValues, message, }: {
+        message?: string;
+        statusCode: number;
+        url: string;
+        requestBodyValues: unknown;
+        data: OllamaErrorData;
+    });
+}
+export declare const failedOllamaCallResponseHandler: ResponseHandler<ApiCallError>;

package/model-provider/ollama/OllamaError.js

@@ -0,0 +1,24 @@
+import { z } from "zod";
+import { ApiCallError } from "../../core/api/ApiCallError.js";
+import { parseJsonWithZod } from "../../util/parseJSON.js";
+export const ollamaErrorDataSchema = z.object({
+    error: z.string(),
+});
+export class OllamaError extends ApiCallError {
+    constructor({ data, statusCode, url, requestBodyValues, message = data.error, }) {
+        super({ message, statusCode, requestBodyValues, url });
+        Object.defineProperty(this, "data", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.data = data;
+    }
+}
+export const failedOllamaCallResponseHandler = async ({ response, url, requestBodyValues }) => new OllamaError({
+    url,
+    requestBodyValues,
+    statusCode: response.status,
+    data: parseJsonWithZod(await response.text(), ollamaErrorDataSchema),
+});

package/model-provider/ollama/OllamaTextGenerationModel.cjs

@@ -0,0 +1,216 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OllamaTextGenerationResponseFormat = exports.OllamaTextGenerationModel = void 0;
+const zod_1 = require("zod");
+const callWithRetryAndThrottle_js_1 = require("../../core/api/callWithRetryAndThrottle.cjs");
+const postToApi_js_1 = require("../../core/api/postToApi.cjs");
+const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
+const PromptFormatTextStreamingModel_js_1 = require("../../model-function/generate-text/PromptFormatTextStreamingModel.cjs");
+const AsyncQueue_js_1 = require("../../util/AsyncQueue.cjs");
+const parseJsonStream_js_1 = require("../../util/streaming/parseJsonStream.cjs");
+const OllamaApiConfiguration_js_1 = require("./OllamaApiConfiguration.cjs");
+const OllamaError_js_1 = require("./OllamaError.cjs");
+class OllamaTextGenerationModel extends AbstractModel_js_1.AbstractModel {
+    constructor(settings) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "ollama"
+        });
+        Object.defineProperty(this, "tokenizer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "countPromptTokens", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+    }
+    get modelName() {
+        return this.settings.model;
+    }
+    get contextWindowSize() {
+        return this.settings.contextWindowSize;
+    }
+    async callAPI(prompt, options) {
+        return (0, callWithRetryAndThrottle_js_1.callWithRetryAndThrottle)({
+            retry: this.settings.api?.retry,
+            throttle: this.settings.api?.throttle,
+            call: async () => callOllamaTextGenerationAPI({
+                ...this.settings,
+                // other
+                abortSignal: options.run?.abortSignal,
+                prompt,
+                responseFormat: options.responseFormat,
+            }),
+        });
+    }
+    get settingsForEvent() {
+        const eventSettingProperties = [
+            // "maxCompletionTokens",
+            // "stopSequences",
+            "contextWindowSize",
+        ];
+        return Object.fromEntries(Object.entries(this.settings).filter(([key]) => eventSettingProperties.includes(key)));
+    }
+    async doGenerateText(prompt, options) {
+        const response = await this.callAPI(prompt, {
+            ...options,
+            responseFormat: exports.OllamaTextGenerationResponseFormat.json,
+        });
+        return {
+            response,
+            text: response.response,
+        };
+    }
+    doStreamText(prompt, options) {
+        return this.callAPI(prompt, {
+            ...options,
+            responseFormat: exports.OllamaTextGenerationResponseFormat.deltaIterable,
+        });
+    }
+    withPromptFormat(promptFormat) {
+        return new PromptFormatTextStreamingModel_js_1.PromptFormatTextStreamingModel({
+            model: this.withSettings({
+                stopSequences: [
+                    ...(this.settings.stopSequences ?? []),
+                    ...promptFormat.stopSequences,
+                ],
+            }),
+            promptFormat,
+        });
+    }
+    withSettings(additionalSettings) {
+        return new OllamaTextGenerationModel(Object.assign({}, this.settings, additionalSettings));
+    }
+}
+exports.OllamaTextGenerationModel = OllamaTextGenerationModel;
+const ollamaTextGenerationResponseSchema = zod_1.z.object({
+    done: zod_1.z.literal(true),
+    model: zod_1.z.string(),
+    response: zod_1.z.string(),
+    total_duration: zod_1.z.number(),
+    load_duration: zod_1.z.number(),
+    prompt_eval_count: zod_1.z.number(),
+    eval_count: zod_1.z.number(),
+    eval_duration: zod_1.z.number(),
+    context: zod_1.z.array(zod_1.z.number()),
+});
+const ollamaTextStreamingResponseSchema = zod_1.z.discriminatedUnion("done", [
+    zod_1.z.object({
+        done: zod_1.z.literal(false),
+        model: zod_1.z.string(),
+        created_at: zod_1.z.string(),
+        response: zod_1.z.string(),
+    }),
+    zod_1.z.object({
+        done: zod_1.z.literal(true),
+        model: zod_1.z.string(),
+        created_at: zod_1.z.string(),
+        total_duration: zod_1.z.number(),
+        load_duration: zod_1.z.number(),
+        sample_count: zod_1.z.number().optional(),
+        sample_duration: zod_1.z.number().optional(),
+        prompt_eval_count: zod_1.z.number(),
+        prompt_eval_duration: zod_1.z.number().optional(),
+        eval_count: zod_1.z.number(),
+        eval_duration: zod_1.z.number(),
+        context: zod_1.z.array(zod_1.z.number()),
+    }),
+]);
+async function callOllamaTextGenerationAPI({ api = new OllamaApiConfiguration_js_1.OllamaApiConfiguration(), abortSignal, responseFormat, prompt, model, contextWindowSize, maxCompletionTokens, mirostat, mirostat_eta, mirostat_tau, num_gpu, num_gqa, num_threads, repeat_last_n, repeat_penalty, seed, stopSequences, temperature, tfs_z, top_k, top_p, system, template, context, }) {
+    return (0, postToApi_js_1.postJsonToApi)({
+        url: api.assembleUrl(`/api/generate`),
+        headers: api.headers,
+        body: {
+            stream: responseFormat.stream,
+            model,
+            prompt,
+            options: {
+                mirostat,
+                mirostat_eta,
+                mirostat_tau,
+                num_ctx: contextWindowSize,
+                num_gpu,
+                num_gqa,
+                num_predict: maxCompletionTokens,
+                num_threads,
+                repeat_last_n,
+                repeat_penalty,
+                seed,
+                stop: stopSequences,
+                temperature,
+                tfs_z,
+                top_k,
+                top_p,
+            },
+            system,
+            template,
+            context,
+        },
+        failedResponseHandler: OllamaError_js_1.failedOllamaCallResponseHandler,
+        successfulResponseHandler: responseFormat.handler,
+        abortSignal,
+    });
+}
+async function createOllamaFullDeltaIterableQueue(stream) {
+    const queue = new AsyncQueue_js_1.AsyncQueue();
+    let accumulatedText = "";
+    // process the stream asynchonously (no 'await' on purpose):
+    (0, parseJsonStream_js_1.parseJsonStream)({
+        stream,
+        schema: ollamaTextStreamingResponseSchema,
+        process(event) {
+            if (event.done === true) {
+                queue.push({
+                    type: "delta",
+                    fullDelta: {
+                        content: accumulatedText,
+                        isComplete: true,
+                        delta: "",
+                    },
+                    valueDelta: "",
+                });
+            }
+            else {
+                accumulatedText += event.response;
+                queue.push({
+                    type: "delta",
+                    fullDelta: {
+                        content: accumulatedText,
+                        isComplete: false,
+                        delta: event.response,
+                    },
+                    valueDelta: event.response,
+                });
+            }
+        },
+        onDone() {
+            queue.close();
+        },
+    });
+    return queue;
+}
+exports.OllamaTextGenerationResponseFormat = {
+    /**
+     * Returns the response as a JSON object.
+     */
+    json: {
+        stream: false,
+        handler: (0, postToApi_js_1.createJsonResponseHandler)(ollamaTextGenerationResponseSchema),
+    },
+    /**
+     * Returns an async iterable over the full deltas (all choices, including full current state at time of event)
+     * of the response stream.
+     */
+    deltaIterable: {
+        stream: true,
+        handler: async ({ response }) => createOllamaFullDeltaIterableQueue(response.body),
+    },
+};

package/model-provider/ollama/OllamaTextGenerationModel.d.ts

@@ -0,0 +1,134 @@
+import { z } from "zod";
+import { FunctionOptions } from "../../core/FunctionOptions.js";
+import { ApiConfiguration } from "../../core/api/ApiConfiguration.js";
+import { ResponseHandler } from "../../core/api/postToApi.js";
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { Delta } from "../../model-function/Delta.js";
+import { PromptFormatTextStreamingModel } from "../../model-function/generate-text/PromptFormatTextStreamingModel.js";
+import { TextGenerationModelSettings, TextStreamingModel } from "../../model-function/generate-text/TextGenerationModel.js";
+import { TextGenerationPromptFormat } from "../../model-function/generate-text/TextGenerationPromptFormat.js";
+export interface OllamaTextGenerationModelSettings<CONTEXT_WINDOW_SIZE extends number | undefined> extends TextGenerationModelSettings {
+    api?: ApiConfiguration;
+    model: string;
+    temperature?: number;
+    /**
+     * Specify the context window size of the model that you have loaded in your
+     * Ollama server.
+     */
+    contextWindowSize?: CONTEXT_WINDOW_SIZE;
+    mirostat?: number;
+    mirostat_eta?: number;
+    mirostat_tau?: number;
+    num_gqa?: number;
+    num_gpu?: number;
+    num_threads?: number;
+    repeat_last_n?: number;
+    repeat_penalty?: number;
+    seed?: number;
+    tfs_z?: number;
+    top_k?: number;
+    top_p?: number;
+    system?: string;
+    template?: string;
+    context?: number[];
+}
+export declare class OllamaTextGenerationModel<CONTEXT_WINDOW_SIZE extends number | undefined> extends AbstractModel<OllamaTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>> implements TextStreamingModel<string, OllamaTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>> {
+    constructor(settings: OllamaTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>);
+    readonly provider = "ollama";
+    get modelName(): string;
+    readonly tokenizer: undefined;
+    readonly countPromptTokens: undefined;
+    get contextWindowSize(): CONTEXT_WINDOW_SIZE;
+    callAPI<RESPONSE>(prompt: string, options: {
+        responseFormat: OllamaTextGenerationResponseFormatType<RESPONSE>;
+    } & FunctionOptions): Promise<RESPONSE>;
+    get settingsForEvent(): Partial<OllamaTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>>;
+    doGenerateText(prompt: string, options?: FunctionOptions): Promise<{
+        response: {
+            response: string;
+            model: string;
+            done: true;
+            context: number[];
+            total_duration: number;
+            load_duration: number;
+            prompt_eval_count: number;
+            eval_count: number;
+            eval_duration: number;
+        };
+        text: string;
+    }>;
+    doStreamText(prompt: string, options?: FunctionOptions): Promise<AsyncIterable<Delta<string>>>;
+    withPromptFormat<INPUT_PROMPT>(promptFormat: TextGenerationPromptFormat<INPUT_PROMPT, string>): PromptFormatTextStreamingModel<INPUT_PROMPT, string, OllamaTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>, this>;
+    withSettings(additionalSettings: Partial<OllamaTextGenerationModelSettings<CONTEXT_WINDOW_SIZE>>): this;
+}
+declare const ollamaTextGenerationResponseSchema: z.ZodObject<{
+    done: z.ZodLiteral<true>;
+    model: z.ZodString;
+    response: z.ZodString;
+    total_duration: z.ZodNumber;
+    load_duration: z.ZodNumber;
+    prompt_eval_count: z.ZodNumber;
+    eval_count: z.ZodNumber;
+    eval_duration: z.ZodNumber;
+    context: z.ZodArray<z.ZodNumber, "many">;
+}, "strip", z.ZodTypeAny, {
+    response: string;
+    model: string;
+    done: true;
+    context: number[];
+    total_duration: number;
+    load_duration: number;
+    prompt_eval_count: number;
+    eval_count: number;
+    eval_duration: number;
+}, {
+    response: string;
+    model: string;
+    done: true;
+    context: number[];
+    total_duration: number;
+    load_duration: number;
+    prompt_eval_count: number;
+    eval_count: number;
+    eval_duration: number;
+}>;
+export type OllamaTextGenerationResponse = z.infer<typeof ollamaTextGenerationResponseSchema>;
+export type OllamaTextGenerationDelta = {
+    content: string;
+    isComplete: boolean;
+    delta: string;
+};
+export type OllamaTextGenerationResponseFormatType<T> = {
+    stream: boolean;
+    handler: ResponseHandler<T>;
+};
+export declare const OllamaTextGenerationResponseFormat: {
+    /**
+     * Returns the response as a JSON object.
+     */
+    json: {
+        stream: false;
+        handler: ResponseHandler<{
+            response: string;
+            model: string;
+            done: true;
+            context: number[];
+            total_duration: number;
+            load_duration: number;
+            prompt_eval_count: number;
+            eval_count: number;
+            eval_duration: number;
+        }>;
+    };
+    /**
+     * Returns an async iterable over the full deltas (all choices, including full current state at time of event)
+     * of the response stream.
+     */
+    deltaIterable: {
+        stream: true;
+        handler: ({ response }: {
+            response: Response;
+        }) => Promise<AsyncIterable<Delta<string>>>;
+    };
+};
+export {};