modelfusion 0.0.44

package/model-provider/llamacpp/LlamaCppError.js

@@ -0,0 +1,44 @@
+import SecureJSON from "secure-json-parse";
+import { z } from "zod";
+import { ApiCallError } from "../../util/api/ApiCallError.js";
+export const llamaCppErrorDataSchema = z.object({
+    error: z.string(),
+});
+export class LlamaCppError extends ApiCallError {
+    constructor({ data, statusCode, url, requestBodyValues, message = data.error, }) {
+        super({ message, statusCode, requestBodyValues, url });
+        Object.defineProperty(this, "data", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.data = data;
+    }
+}
+export const failedLlamaCppCallResponseHandler = async ({ response, url, requestBodyValues }) => {
+    const responseBody = await response.text();
+    try {
+        const parsedError = llamaCppErrorDataSchema.parse(SecureJSON.parse(responseBody));
+        return new LlamaCppError({
+            url,
+            requestBodyValues,
+            statusCode: response.status,
+            data: parsedError,
+        });
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            if (error.name === "AbortError" || error instanceof ApiCallError) {
+                throw error;
+            }
+        }
+        throw new ApiCallError({
+            message: responseBody,
+            cause: error,
+            statusCode: response.status,
+            url,
+            requestBodyValues,
+        });
+    }
+};

package/model-provider/llamacpp/LlamaCppTextEmbeddingModel.cjs

@@ -0,0 +1,96 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LlamaCppTextEmbeddingModel = void 0;
+const zod_1 = __importDefault(require("zod"));
+const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
+const callWithRetryAndThrottle_js_1 = require("../../util/api/callWithRetryAndThrottle.cjs");
+const postToApi_js_1 = require("../../util/api/postToApi.cjs");
+const LlamaCppError_js_1 = require("./LlamaCppError.cjs");
+const LlamaCppTokenizer_js_1 = require("./LlamaCppTokenizer.cjs");
+class LlamaCppTextEmbeddingModel extends AbstractModel_js_1.AbstractModel {
+    constructor(settings = {}) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "llamacpp"
+        });
+        Object.defineProperty(this, "maxTextsPerCall", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 1
+        });
+        Object.defineProperty(this, "contextWindowSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "embeddingDimensions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "tokenizer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.tokenizer = new LlamaCppTokenizer_js_1.LlamaCppTokenizer({
+            baseUrl: this.settings.baseUrl,
+            retry: this.settings.tokenizerSettings?.retry,
+            throttle: this.settings.tokenizerSettings?.throttle,
+        });
+    }
+    get modelName() {
+        return null;
+    }
+    async tokenize(text) {
+        return this.tokenizer.tokenize(text);
+    }
+    async callAPI(texts, options) {
+        if (texts.length > this.maxTextsPerCall) {
+            throw new Error(`The Llama.cpp embedding API only supports ${this.maxTextsPerCall} texts per API call.`);
+        }
+        const run = options?.run;
+        const settings = options?.settings;
+        const callSettings = Object.assign({}, this.settings, settings, {
+            abortSignal: run?.abortSignal,
+            content: texts[0],
+        });
+        return (0, callWithRetryAndThrottle_js_1.callWithRetryAndThrottle)({
+            retry: this.settings.retry,
+            throttle: this.settings.throttle,
+            call: async () => callLlamaCppEmbeddingAPI(callSettings),
+        });
+    }
+    generateEmbeddingResponse(texts, options) {
+        return this.callAPI(texts, options);
+    }
+    extractEmbeddings(response) {
+        return [response.embedding];
+    }
+    withSettings(additionalSettings) {
+        return new LlamaCppTextEmbeddingModel(Object.assign({}, this.settings, additionalSettings));
+    }
+}
+exports.LlamaCppTextEmbeddingModel = LlamaCppTextEmbeddingModel;
+const llamaCppTextEmbeddingResponseSchema = zod_1.default.object({
+    embedding: zod_1.default.array(zod_1.default.number()),
+});
+async function callLlamaCppEmbeddingAPI({ baseUrl = "http://127.0.0.1:8080", abortSignal, content, }) {
+    return (0, postToApi_js_1.postJsonToApi)({
+        url: `${baseUrl}/embedding`,
+        body: { content },
+        failedResponseHandler: LlamaCppError_js_1.failedLlamaCppCallResponseHandler,
+        successfulResponseHandler: (0, postToApi_js_1.createJsonResponseHandler)(llamaCppTextEmbeddingResponseSchema),
+        abortSignal,
+    });
+}

package/model-provider/llamacpp/LlamaCppTextEmbeddingModel.d.ts

@@ -0,0 +1,40 @@
+import z from "zod";
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { FunctionOptions } from "../../model-function/FunctionOptions.js";
+import { TextEmbeddingModel, TextEmbeddingModelSettings } from "../../model-function/embed-text/TextEmbeddingModel.js";
+import { RetryFunction } from "../../util/api/RetryFunction.js";
+import { ThrottleFunction } from "../../util/api/ThrottleFunction.js";
+export interface LlamaCppEmbeddingModelSettings extends TextEmbeddingModelSettings {
+    baseUrl?: string;
+    retry?: RetryFunction;
+    throttle?: ThrottleFunction;
+    tokenizerSettings?: {
+        retry?: RetryFunction;
+        throttle?: ThrottleFunction;
+    };
+}
+export declare class LlamaCppTextEmbeddingModel extends AbstractModel<LlamaCppEmbeddingModelSettings> implements TextEmbeddingModel<LlamaCppTextEmbeddingResponse, LlamaCppEmbeddingModelSettings> {
+    constructor(settings?: LlamaCppEmbeddingModelSettings);
+    readonly provider: "llamacpp";
+    get modelName(): null;
+    readonly maxTextsPerCall = 1;
+    readonly contextWindowSize: undefined;
+    readonly embeddingDimensions: undefined;
+    private readonly tokenizer;
+    tokenize(text: string): Promise<number[]>;
+    callAPI(texts: Array<string>, options?: FunctionOptions<LlamaCppEmbeddingModelSettings>): Promise<LlamaCppTextEmbeddingResponse>;
+    generateEmbeddingResponse(texts: string[], options?: FunctionOptions<LlamaCppEmbeddingModelSettings>): Promise<{
+        embedding: number[];
+    }>;
+    extractEmbeddings(response: LlamaCppTextEmbeddingResponse): number[][];
+    withSettings(additionalSettings: Partial<LlamaCppEmbeddingModelSettings>): this;
+}
+declare const llamaCppTextEmbeddingResponseSchema: z.ZodObject<{
+    embedding: z.ZodArray<z.ZodNumber, "many">;
+}, "strip", z.ZodTypeAny, {
+    embedding: number[];
+}, {
+    embedding: number[];
+}>;
+export type LlamaCppTextEmbeddingResponse = z.infer<typeof llamaCppTextEmbeddingResponseSchema>;
+export {};

package/model-provider/llamacpp/LlamaCppTextEmbeddingModel.js

@@ -0,0 +1,89 @@
+import z from "zod";
+import { AbstractModel } from "../../model-function/AbstractModel.js";
+import { callWithRetryAndThrottle } from "../../util/api/callWithRetryAndThrottle.js";
+import { createJsonResponseHandler, postJsonToApi, } from "../../util/api/postToApi.js";
+import { failedLlamaCppCallResponseHandler } from "./LlamaCppError.js";
+import { LlamaCppTokenizer } from "./LlamaCppTokenizer.js";
+export class LlamaCppTextEmbeddingModel extends AbstractModel {
+    constructor(settings = {}) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "llamacpp"
+        });
+        Object.defineProperty(this, "maxTextsPerCall", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 1
+        });
+        Object.defineProperty(this, "contextWindowSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "embeddingDimensions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
+        Object.defineProperty(this, "tokenizer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.tokenizer = new LlamaCppTokenizer({
+            baseUrl: this.settings.baseUrl,
+            retry: this.settings.tokenizerSettings?.retry,
+            throttle: this.settings.tokenizerSettings?.throttle,
+        });
+    }
+    get modelName() {
+        return null;
+    }
+    async tokenize(text) {
+        return this.tokenizer.tokenize(text);
+    }
+    async callAPI(texts, options) {
+        if (texts.length > this.maxTextsPerCall) {
+            throw new Error(`The Llama.cpp embedding API only supports ${this.maxTextsPerCall} texts per API call.`);
+        }
+        const run = options?.run;
+        const settings = options?.settings;
+        const callSettings = Object.assign({}, this.settings, settings, {
+            abortSignal: run?.abortSignal,
+            content: texts[0],
+        });
+        return callWithRetryAndThrottle({
+            retry: this.settings.retry,
+            throttle: this.settings.throttle,
+            call: async () => callLlamaCppEmbeddingAPI(callSettings),
+        });
+    }
+    generateEmbeddingResponse(texts, options) {
+        return this.callAPI(texts, options);
+    }
+    extractEmbeddings(response) {
+        return [response.embedding];
+    }
+    withSettings(additionalSettings) {
+        return new LlamaCppTextEmbeddingModel(Object.assign({}, this.settings, additionalSettings));
+    }
+}
+const llamaCppTextEmbeddingResponseSchema = z.object({
+    embedding: z.array(z.number()),
+});
+async function callLlamaCppEmbeddingAPI({ baseUrl = "http://127.0.0.1:8080", abortSignal, content, }) {
+    return postJsonToApi({
+        url: `${baseUrl}/embedding`,
+        body: { content },
+        failedResponseHandler: failedLlamaCppCallResponseHandler,
+        successfulResponseHandler: createJsonResponseHandler(llamaCppTextEmbeddingResponseSchema),
+        abortSignal,
+    });
+}

package/model-provider/llamacpp/LlamaCppTextGenerationModel.cjs

@@ -0,0 +1,245 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LlamaCppTextGenerationResponseFormat = exports.LlamaCppTextGenerationModel = void 0;
+const secure_json_parse_1 = __importDefault(require("secure-json-parse"));
+const zod_1 = __importDefault(require("zod"));
+const AbstractModel_js_1 = require("../../model-function/AbstractModel.cjs");
+const AsyncQueue_js_1 = require("../../model-function/generate-text/AsyncQueue.cjs");
+const parseEventSourceReadableStream_js_1 = require("../../model-function/generate-text/parseEventSourceReadableStream.cjs");
+const PromptMappingTextGenerationModel_js_1 = require("../../prompt/PromptMappingTextGenerationModel.cjs");
+const callWithRetryAndThrottle_js_1 = require("../../util/api/callWithRetryAndThrottle.cjs");
+const postToApi_js_1 = require("../../util/api/postToApi.cjs");
+const LlamaCppError_js_1 = require("./LlamaCppError.cjs");
+const LlamaCppTokenizer_js_1 = require("./LlamaCppTokenizer.cjs");
+class LlamaCppTextGenerationModel extends AbstractModel_js_1.AbstractModel {
+    constructor(settings = {}) {
+        super({ settings });
+        Object.defineProperty(this, "provider", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "llamacpp"
+        });
+        Object.defineProperty(this, "tokenizer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.tokenizer = new LlamaCppTokenizer_js_1.LlamaCppTokenizer({
+            baseUrl: this.settings.baseUrl,
+            retry: this.settings.tokenizerSettings?.retry,
+            throttle: this.settings.tokenizerSettings?.throttle,
+        });
+    }
+    get modelName() {
+        return null;
+    }
+    get contextWindowSize() {
+        return this.settings.contextWindowSize;
+    }
+    async callAPI(prompt, options) {
+        const { run, settings, responseFormat } = options;
+        const callSettings = Object.assign(this.settings, settings, {
+            abortSignal: run?.abortSignal,
+            prompt,
+            responseFormat,
+        });
+        return (0, callWithRetryAndThrottle_js_1.callWithRetryAndThrottle)({
+            retry: this.settings.retry,
+            throttle: this.settings.throttle,
+            call: async () => callLlamaCppTextGenerationAPI(callSettings),
+        });
+    }
+    async countPromptTokens(prompt) {
+        const tokens = await this.tokenizer.tokenize(prompt);
+        return tokens.length;
+    }
+    generateTextResponse(prompt, options) {
+        return this.callAPI(prompt, {
+            ...options,
+            responseFormat: exports.LlamaCppTextGenerationResponseFormat.json,
+        });
+    }
+    extractText(response) {
+        return response.content;
+    }
+    generateDeltaStreamResponse(prompt, options) {
+        return this.callAPI(prompt, {
+            ...options,
+            responseFormat: exports.LlamaCppTextGenerationResponseFormat.deltaIterable,
+        });
+    }
+    extractTextDelta(fullDelta) {
+        return fullDelta.delta;
+    }
+    mapPrompt(promptMapping) {
+        return new PromptMappingTextGenerationModel_js_1.PromptMappingTextGenerationModel({
+            model: this.withStopTokens(promptMapping.stopTokens),
+            promptMapping,
+        });
+    }
+    withSettings(additionalSettings) {
+        return new LlamaCppTextGenerationModel(Object.assign({}, this.settings, additionalSettings));
+    }
+    get maxCompletionTokens() {
+        return this.settings.nPredict;
+    }
+    withMaxCompletionTokens(maxCompletionTokens) {
+        return this.withSettings({ nPredict: maxCompletionTokens });
+    }
+    withStopTokens(stopTokens) {
+        return this.withSettings({ stop: stopTokens });
+    }
+}
+exports.LlamaCppTextGenerationModel = LlamaCppTextGenerationModel;
+const llamaCppTextGenerationResponseSchema = zod_1.default.object({
+    content: zod_1.default.string(),
+    stop: zod_1.default.literal(true),
+    generation_settings: zod_1.default.object({
+        frequency_penalty: zod_1.default.number(),
+        ignore_eos: zod_1.default.boolean(),
+        logit_bias: zod_1.default.array(zod_1.default.number()),
+        mirostat: zod_1.default.number(),
+        mirostat_eta: zod_1.default.number(),
+        mirostat_tau: zod_1.default.number(),
+        model: zod_1.default.string(),
+        n_ctx: zod_1.default.number(),
+        n_keep: zod_1.default.number(),
+        n_predict: zod_1.default.number(),
+        n_probs: zod_1.default.number(),
+        penalize_nl: zod_1.default.boolean(),
+        presence_penalty: zod_1.default.number(),
+        repeat_last_n: zod_1.default.number(),
+        repeat_penalty: zod_1.default.number(),
+        seed: zod_1.default.number(),
+        stop: zod_1.default.array(zod_1.default.string()),
+        stream: zod_1.default.boolean(),
+        temp: zod_1.default.number(),
+        tfs_z: zod_1.default.number(),
+        top_k: zod_1.default.number(),
+        top_p: zod_1.default.number(),
+        typical_p: zod_1.default.number(),
+    }),
+    model: zod_1.default.string(),
+    prompt: zod_1.default.string(),
+    stopped_eos: zod_1.default.boolean(),
+    stopped_limit: zod_1.default.boolean(),
+    stopped_word: zod_1.default.boolean(),
+    stopping_word: zod_1.default.string(),
+    timings: zod_1.default.object({
+        predicted_ms: zod_1.default.number(),
+        predicted_n: zod_1.default.number(),
+        predicted_per_second: zod_1.default.number().nullable(),
+        predicted_per_token_ms: zod_1.default.number().nullable(),
+        prompt_ms: zod_1.default.number().nullable(),
+        prompt_n: zod_1.default.number(),
+        prompt_per_second: zod_1.default.number().nullable(),
+        prompt_per_token_ms: zod_1.default.number().nullable(),
+    }),
+    tokens_cached: zod_1.default.number(),
+    tokens_evaluated: zod_1.default.number(),
+    tokens_predicted: zod_1.default.number(),
+    truncated: zod_1.default.boolean(),
+});
+const llamaCppTextStreamingResponseSchema = zod_1.default.discriminatedUnion("stop", [
+    zod_1.default.object({
+        content: zod_1.default.string(),
+        stop: zod_1.default.literal(false),
+    }),
+    llamaCppTextGenerationResponseSchema,
+]);
+async function callLlamaCppTextGenerationAPI({ baseUrl = "http://127.0.0.1:8080", abortSignal, responseFormat, prompt, temperature, topK, topP, nPredict, nKeep, stop, tfsZ, typicalP, repeatPenalty, repeatLastN, penalizeNl, mirostat, mirostatTau, mirostatEta, seed, ignoreEos, logitBias, }) {
+    return (0, postToApi_js_1.postJsonToApi)({
+        url: `${baseUrl}/completion`,
+        body: {
+            stream: responseFormat.stream,
+            prompt,
+            temperature,
+            top_k: topK,
+            top_p: topP,
+            n_predict: nPredict,
+            n_keep: nKeep,
+            stop,
+            tfs_z: tfsZ,
+            typical_p: typicalP,
+            repeat_penalty: repeatPenalty,
+            repeat_last_n: repeatLastN,
+            penalize_nl: penalizeNl,
+            mirostat,
+            mirostat_tau: mirostatTau,
+            mirostat_eta: mirostatEta,
+            seed,
+            ignore_eos: ignoreEos,
+            logit_bias: logitBias,
+        },
+        failedResponseHandler: LlamaCppError_js_1.failedLlamaCppCallResponseHandler,
+        successfulResponseHandler: responseFormat.handler,
+        abortSignal,
+    });
+}
+async function createLlamaCppFullDeltaIterableQueue(stream) {
+    const queue = new AsyncQueue_js_1.AsyncQueue();
+    let content = "";
+    // process the stream asynchonously (no 'await' on purpose):
+    (0, parseEventSourceReadableStream_js_1.parseEventSourceReadableStream)({
+        stream,
+        callback: (event) => {
+            if (event.type !== "event") {
+                return;
+            }
+            const data = event.data;
+            try {
+                const json = secure_json_parse_1.default.parse(data);
+                const parseResult = llamaCppTextStreamingResponseSchema.safeParse(json);
+                if (!parseResult.success) {
+                    queue.push({
+                        type: "error",
+                        error: parseResult.error,
+                    });
+                    queue.close();
+                    return;
+                }
+                const event = parseResult.data;
+                content += event.content;
+                queue.push({
+                    type: "delta",
+                    fullDelta: {
+                        content,
+                        isComplete: event.stop,
+                        delta: event.content,
+                    },
+                });
+                if (event.stop) {
+                    queue.close();
+                }
+            }
+            catch (error) {
+                queue.push({ type: "error", error });
+                queue.close();
+                return;
+            }
+        },
+    });
+    return queue;
+}
+exports.LlamaCppTextGenerationResponseFormat = {
+    /**
+     * Returns the response as a JSON object.
+     */
+    json: {
+        stream: false,
+        handler: (0, postToApi_js_1.createJsonResponseHandler)(llamaCppTextGenerationResponseSchema),
+    },
+    /**
+     * Returns an async iterable over the full deltas (all choices, including full current state at time of event)
+     * of the response stream.
+     */
+    deltaIterable: {
+        stream: true,
+        handler: async ({ response }) => createLlamaCppFullDeltaIterableQueue(response.body),
+    },
+};