@qvac/sdk 0.8.3 → 0.9.1

package/dist/server/bare/plugins/index.js

@@ -5,3 +5,4 @@ export { parakeetPlugin } from "./parakeet-transcription/plugin.js";
 export { nmtPlugin } from "./nmtcpp-translation/plugin.js";
 export { ttsPlugin } from "./onnx-tts/plugin.js";
 export { ocrPlugin } from "./onnx-ocr/plugin.js";
+export { diffusionPlugin } from "./sdcpp-generation/plugin.js";

package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.d.ts

@@ -1,4 +1,5 @@
 import type { CompletionParams, CompletionStats, GenerationParams, Tool, ToolCall, ToolCallEvent } from "../../../../../schemas/index";
+export declare function clearCachedMessageCounts(cachePath?: string): void;
 export declare function completion(params: CompletionParams & {
     tools?: Tool[];
     generationParams?: GenerationParams;
@@ -6,7 +7,8 @@ export declare function completion(params: CompletionParams & {
     token: string;
     toolCallEvent?: ToolCallEvent;
 }, {
-    stats: CompletionStats;
+    modelExecutionMs: number;
+    stats?: CompletionStats;
     toolCalls: ToolCall[];
 }, unknown>;
 //# sourceMappingURL=completion-stream.d.ts.map

package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.js

@@ -4,7 +4,18 @@ import { getModel, getModelConfig, } from "../../../registry/model-registry.js";
 import { checkForToolEvents, insertToolsIntoHistory, setupToolGrammar, } from "../../../../utils/tool-integration.js";
 import { parseToolCalls } from "../../../../utils/tool-parser.js";
 import { AttachmentNotFoundError } from "../../../../../utils/errors-server.js";
+import { nowMs } from "../../../../../profiling/index.js";
+import { buildStreamResult, hasDefinedValues, } from "../../../../../profiling/model-execution.js";
 import fs from "bare-fs";
+const cachedMessageCounts = new Map();
+export function clearCachedMessageCounts(cachePath) {
+    if (cachePath) {
+        cachedMessageCounts.delete(cachePath);
+    }
+    else {
+        cachedMessageCounts.clear();
+    }
+}
 function transformMessage(message) {
     const transformed = [];
     // Check if it's a tool definition (has type: "function")
@@ -64,11 +75,18 @@ async function initSystemPromptCache(model, cachePathToUse, systemPromptToUse, c
 }
 function prepareMessagesForCache(cachePathToUse, cacheExists, history) {
     if (cacheExists && history.length > 0) {
-        const lastMessage = history[history.length - 1];
-        const lastTransformedMessages = transformMessage(lastMessage);
+        const savedCount = cachedMessageCounts.get(cachePathToUse) ?? 0;
+        const canSlice = savedCount > 0 && savedCount <= history.length;
+        const newMessages = canSlice
+            ? history.slice(savedCount)
+            : history.filter((msg) => msg.role !== "system");
+        if (!canSlice && savedCount > 0) {
+            cachedMessageCounts.delete(cachePathToUse);
+        }
+        const transformedNewMessages = transformMessages(newMessages);
         return [
             { role: "session", content: cachePathToUse },
-            ...lastTransformedMessages,
+            ...transformedNewMessages,
         ];
     }
     const historyWithoutSystem = history.filter((msg) => msg.role !== "system");
@@ -81,6 +99,7 @@ function prepareMessagesForCache(cachePathToUse, cacheExists, history) {
 async function* processModelResponse(model, messagesToSend, shouldSaveCache, tools, generationParams) {
     const runFn = model.run.bind(model);
     const runOptions = generationParams ? { generationParams } : undefined;
+    const modelStart = nowMs();
     const response = await runFn(messagesToSend, runOptions);
     let accumulatedText = "";
     const emittedToolCallPositions = new Set();
@@ -96,6 +115,7 @@ async function* processModelResponse(model, messagesToSend, shouldSaveCache, too
             }
         }
     }
+    const modelExecutionMs = nowMs() - modelStart;
     if (tools && tools.length > 0) {
         const { toolCalls } = parseToolCalls(accumulatedText, tools);
         toolCallsResult = toolCalls;
@@ -119,11 +139,23 @@ async function* processModelResponse(model, messagesToSend, shouldSaveCache, too
     }
     const responseWithStats = response;
     const stats = {
-        timeToFirstToken: responseWithStats.stats?.TTFT ?? 0,
-        tokensPerSecond: responseWithStats.stats?.TPS ?? 0,
-        cacheTokens: responseWithStats.stats?.CacheTokens ?? 0,
+        ...(responseWithStats.stats?.TTFT !== undefined && {
+            timeToFirstToken: responseWithStats.stats.TTFT,
+        }),
+        ...(responseWithStats.stats?.TPS !== undefined && {
+            tokensPerSecond: responseWithStats.stats.TPS,
+        }),
+        ...(responseWithStats.stats?.CacheTokens !== undefined && {
+            cacheTokens: responseWithStats.stats.CacheTokens,
+        }),
+        ...(responseWithStats.stats?.backendDevice !== undefined && {
+            backendDevice: responseWithStats.stats.backendDevice,
+        }),
+    };
+    return {
+        ...buildStreamResult(modelExecutionMs, hasDefinedValues(stats) ? stats : undefined),
+        toolCalls: toolCallsResult,
     };
-    return { stats, toolCalls: toolCallsResult };
 }
 export async function* completion(params) {
     const { history, modelId, kvCache, tools, generationParams } = params;
@@ -155,7 +187,9 @@ export async function* completion(params) {
             }
             const messagesToSend = prepareMessagesForCache(cachePathToUse, cacheExists, history);
             logMessagesToAddon(messagesToSend, "PROMPT_SEND");
-            return yield* processModelResponse(model, messagesToSend, true, tools, generationParams);
+            const result = yield* processModelResponse(model, messagesToSend, true, tools, generationParams);
+            cachedMessageCounts.set(cachePathToUse, history.length + 1);
+            return result;
         }
         else {
             // Auto-generate cache key based on conversation history
@@ -180,9 +214,11 @@ export async function* completion(params) {
             const messagesToSend = prepareMessagesForCache(cachePathToUse, cacheExists, history);
             logMessagesToAddon(messagesToSend, "PROMPT_SEND");
             const result = yield* processModelResponse(model, messagesToSend, true, tools, generationParams);
-            //If there was an existing cache, we rename it with new hash that includes the current message
+            cachedMessageCounts.set(cachePathToUse, history.length + 1);
             if (existingCache !== null &&
                 existingCache.cachePath !== currentCacheInfo.cachePath) {
+                cachedMessageCounts.delete(existingCache.cachePath);
+                cachedMessageCounts.set(currentCacheInfo.cachePath, history.length + 1);
                 await renameCacheFile(existingCache.cachePath, currentCacheInfo.cachePath);
             }
             return result;

package/dist/server/bare/plugins/llamacpp-completion/ops/finetune.d.ts

@@ -0,0 +1,11 @@
+import type { FinetuneRunParams, FinetuneRunRequest, FinetuneProgress, FinetuneRequest, FinetuneResult, FinetuneStatus, FinetuneGetStateRequest } from "../../../../../schemas/index";
+type FinetuneOptions = FinetuneRunParams["options"];
+export declare function clearFinetuneRuntimeState(modelId: string): void;
+export declare function getFinetuneStateFromCheckpoints(options: FinetuneOptions): FinetuneStatus;
+export declare function startFinetune(request: FinetuneRunRequest, onProgress?: (progress: FinetuneProgress) => void): Promise<FinetuneResult>;
+export declare function pauseFinetune(modelId: string): Promise<FinetuneResult>;
+export declare function cancelFinetune(modelId: string): Promise<FinetuneResult>;
+export declare function getFinetuneState(params: FinetuneGetStateRequest): FinetuneResult;
+export declare function finetune(request: FinetuneRequest, onProgress?: (progress: FinetuneProgress) => void): Promise<FinetuneResult>;
+export {};
+//# sourceMappingURL=finetune.d.ts.map

package/dist/server/bare/plugins/llamacpp-completion/ops/finetune.js

@@ -0,0 +1,112 @@
+import fs from "bare-fs";
+import { getModel, } from "../../../registry/model-registry.js";
+import { CompletionFailedError, } from "../../../../../utils/errors-server.js";
+const PAUSE_CHECKPOINT_PREFIX = "pause_checkpoint_step_";
+const finetuneRuntimeState = new Set();
+function getRunningFinetuneState(modelId) {
+    return finetuneRuntimeState.has(modelId);
+}
+function registerRunningFinetune(modelId) {
+    finetuneRuntimeState.add(modelId);
+}
+export function clearFinetuneRuntimeState(modelId) {
+    finetuneRuntimeState.delete(modelId);
+}
+export function getFinetuneStateFromCheckpoints(options) {
+    const checkpointDirectory = options.checkpointSaveDir ?? "./checkpoints";
+    if (!fs.existsSync(checkpointDirectory)) {
+        return "IDLE";
+    }
+    try {
+        const entries = fs.readdirSync(checkpointDirectory);
+        for (const entry of entries) {
+            if (typeof entry !== "string") {
+                continue;
+            }
+            if (entry.startsWith(PAUSE_CHECKPOINT_PREFIX)) {
+                return "PAUSED";
+            }
+        }
+    }
+    catch (error) {
+        throw new CompletionFailedError(`Failed to inspect finetune checkpoints in "${checkpointDirectory}"`, error);
+    }
+    return "IDLE";
+}
+function validateExplicitFinetuneOperation(request) {
+    if (!request.operation) {
+        return;
+    }
+    const state = getFinetuneStateFromCheckpoints(request.options);
+    if (request.operation === "start" && state === "PAUSED") {
+        throw new CompletionFailedError(`Model "${request.modelId}" has a paused finetune checkpoint; resume it or cancel it before starting from scratch`);
+    }
+    if (request.operation === "resume" && state === "IDLE") {
+        throw new CompletionFailedError(`Model "${request.modelId}" has no paused finetune checkpoint to resume`);
+    }
+}
+export async function startFinetune(request, onProgress) {
+    const model = getModel(request.modelId);
+    validateExplicitFinetuneOperation(request);
+    registerRunningFinetune(request.modelId);
+    try {
+        const handle = await model.finetune(request.options);
+        if (onProgress) {
+            handle.on("stats", onProgress);
+        }
+        try {
+            const result = await handle.await();
+            return {
+                type: "finetune",
+                status: result.status,
+                stats: result.stats,
+            };
+        }
+        finally {
+            if (onProgress) {
+                handle.removeListener("stats", onProgress);
+            }
+        }
+    }
+    finally {
+        clearFinetuneRuntimeState(request.modelId);
+    }
+}
+export async function pauseFinetune(modelId) {
+    const model = getModel(modelId);
+    await model.pause();
+    return {
+        type: "finetune",
+        status: "PAUSED",
+    };
+}
+export async function cancelFinetune(modelId) {
+    const model = getModel(modelId);
+    await model.cancel();
+    return {
+        type: "finetune",
+        status: "CANCELLED",
+    };
+}
+export function getFinetuneState(params) {
+    const runtimeState = getRunningFinetuneState(params.modelId);
+    return {
+        type: "finetune",
+        status: runtimeState ? "RUNNING" : getFinetuneStateFromCheckpoints(params.options),
+    };
+}
+export async function finetune(request, onProgress) {
+    if (request.operation === undefined ||
+        request.operation === "start" ||
+        request.operation === "resume") {
+        return startFinetune(request, onProgress);
+    }
+    switch (request.operation) {
+        case "getState":
+            return getFinetuneState(request);
+        case "pause":
+            return pauseFinetune(request.modelId);
+        case "cancel":
+            return cancelFinetune(request.modelId);
+    }
+}

package/dist/server/bare/plugins/llamacpp-completion/plugin.d.ts

@@ -22,6 +22,9 @@ export declare const llmPlugin: {
         stop_sequences: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodString>>;
         n_discarded: import("zod").ZodOptional<import("zod").ZodNumber>;
         tools: import("zod").ZodOptional<import("zod").ZodBoolean>;
+        "cache-type-k": import("zod").ZodOptional<import("zod").ZodString>;
+        "cache-type-v": import("zod").ZodOptional<import("zod").ZodString>;
+        openclCacheDir: import("zod").ZodOptional<import("zod").ZodString>;
         projectionModelSrc: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodObject<{
             src: import("zod").ZodString;
             name: import("zod").ZodOptional<import("zod").ZodString>;
@@ -41,6 +44,7 @@ export declare const llmPlugin: {
                 "onnx-tts": "onnx-tts";
                 "parakeet-transcription": "parakeet-transcription";
                 "onnx-ocr": "onnx-ocr";
+                "sdcpp-generation": "sdcpp-generation";
                 llm: "llm";
                 whisper: "whisper";
                 embeddings: "embeddings";
@@ -48,6 +52,7 @@ export declare const llmPlugin: {
                 parakeet: "parakeet";
                 tts: "tts";
                 ocr: "ocr";
+                diffusion: "diffusion";
             }>, import("zod").ZodLiteral<"vad">]>>;
         }, import("zod/v4/core").$strip>]>>;
     }, import("zod/v4/core").$strip>;
@@ -71,6 +76,9 @@ export declare const llmPlugin: {
             stop_sequences?: string[] | undefined;
             n_discarded?: number | undefined;
             tools?: boolean | undefined;
+            "cache-type-k"?: string | undefined;
+            "cache-type-v"?: string | undefined;
+            openclCacheDir?: string | undefined;
         };
         artifacts?: never;
     } | {
@@ -93,6 +101,9 @@ export declare const llmPlugin: {
             stop_sequences?: string[] | undefined;
             n_discarded?: number | undefined;
             tools?: boolean | undefined;
+            "cache-type-k"?: string | undefined;
+            "cache-type-v"?: string | undefined;
+            openclCacheDir?: string | undefined;
         };
         artifacts: {
             projectionModelPath: string;
@@ -148,9 +159,13 @@ export declare const llmPlugin: {
             token: import("zod").ZodString;
             done: import("zod").ZodOptional<import("zod").ZodBoolean>;
             stats: import("zod").ZodOptional<import("zod").ZodObject<{
-                timeToFirstToken: import("zod").ZodNumber;
-                tokensPerSecond: import("zod").ZodNumber;
-                cacheTokens: import("zod").ZodNumber;
+                timeToFirstToken: import("zod").ZodOptional<import("zod").ZodNumber>;
+                tokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
+                cacheTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
+                backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
+                    cpu: "cpu";
+                    gpu: "gpu";
+                }>>;
             }, import("zod/v4/core").$strip>>;
             toolCallEvent: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodObject<{
                 type: import("zod").ZodLiteral<"toolCall">;
@@ -180,6 +195,156 @@ export declare const llmPlugin: {
             }, import("zod/v4/core").$strip>>>;
             error: import("zod").ZodOptional<import("zod").ZodString>;
         }, import("zod/v4/core").$strip>>;
+        finetune: import("../../../../schemas/index").PluginHandlerDefinition<import("zod").ZodUnion<readonly [import("zod").ZodObject<{
+            modelId: import("zod").ZodString;
+            options: import("zod").ZodObject<{
+                trainDatasetDir: import("zod").ZodString;
+                validation: import("zod").ZodPipe<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
+                    type: import("zod").ZodLiteral<"none">;
+                }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+                    type: import("zod").ZodLiteral<"split">;
+                    fraction: import("zod").ZodOptional<import("zod").ZodNumber>;
+                }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+                    type: import("zod").ZodLiteral<"dataset">;
+                    path: import("zod").ZodString;
+                }, import("zod/v4/core").$strict>], "type">, import("zod").ZodTransform<{
+                    type: "none";
+                } | {
+                    type: "dataset";
+                    path: string;
+                } | {
+                    fraction: number;
+                    type: "split";
+                }, {
+                    type: "none";
+                } | {
+                    type: "split";
+                    fraction?: number | undefined;
+                } | {
+                    type: "dataset";
+                    path: string;
+                }>>;
+                outputParametersDir: import("zod").ZodString;
+                numberOfEpochs: import("zod").ZodOptional<import("zod").ZodNumber>;
+                learningRate: import("zod").ZodOptional<import("zod").ZodNumber>;
+                contextLength: import("zod").ZodOptional<import("zod").ZodNumber>;
+                batchSize: import("zod").ZodOptional<import("zod").ZodNumber>;
+                microBatchSize: import("zod").ZodOptional<import("zod").ZodNumber>;
+                assistantLossOnly: import("zod").ZodOptional<import("zod").ZodBoolean>;
+                loraRank: import("zod").ZodOptional<import("zod").ZodNumber>;
+                loraAlpha: import("zod").ZodOptional<import("zod").ZodNumber>;
+                loraInitStd: import("zod").ZodOptional<import("zod").ZodNumber>;
+                loraSeed: import("zod").ZodOptional<import("zod").ZodNumber>;
+                loraModules: import("zod").ZodOptional<import("zod").ZodString>;
+                checkpointSaveDir: import("zod").ZodOptional<import("zod").ZodString>;
+                checkpointSaveSteps: import("zod").ZodOptional<import("zod").ZodNumber>;
+                chatTemplatePath: import("zod").ZodOptional<import("zod").ZodString>;
+                lrScheduler: import("zod").ZodOptional<import("zod").ZodEnum<{
+                    constant: "constant";
+                    cosine: "cosine";
+                    linear: "linear";
+                }>>;
+                lrMin: import("zod").ZodOptional<import("zod").ZodNumber>;
+                warmupRatio: import("zod").ZodOptional<import("zod").ZodNumber>;
+                warmupRatioSet: import("zod").ZodOptional<import("zod").ZodBoolean>;
+                warmupSteps: import("zod").ZodOptional<import("zod").ZodNumber>;
+                warmupStepsSet: import("zod").ZodOptional<import("zod").ZodBoolean>;
+                weightDecay: import("zod").ZodOptional<import("zod").ZodNumber>;
+            }, import("zod/v4/core").$strict>;
+            type: import("zod").ZodLiteral<"finetune">;
+            operation: import("zod").ZodOptional<import("zod").ZodEnum<{
+                start: "start";
+                resume: "resume";
+            }>>;
+            withProgress: import("zod").ZodOptional<import("zod").ZodBoolean>;
+        }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+            modelId: import("zod").ZodString;
+            options: import("zod").ZodObject<{
+                trainDatasetDir: import("zod").ZodString;
+                validation: import("zod").ZodPipe<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
+                    type: import("zod").ZodLiteral<"none">;
+                }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+                    type: import("zod").ZodLiteral<"split">;
+                    fraction: import("zod").ZodOptional<import("zod").ZodNumber>;
+                }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+                    type: import("zod").ZodLiteral<"dataset">;
+                    path: import("zod").ZodString;
+                }, import("zod/v4/core").$strict>], "type">, import("zod").ZodTransform<{
+                    type: "none";
+                } | {
+                    type: "dataset";
+                    path: string;
+                } | {
+                    fraction: number;
+                    type: "split";
+                }, {
+                    type: "none";
+                } | {
+                    type: "split";
+                    fraction?: number | undefined;
+                } | {
+                    type: "dataset";
+                    path: string;
+                }>>;
+                outputParametersDir: import("zod").ZodString;
+                numberOfEpochs: import("zod").ZodOptional<import("zod").ZodNumber>;
+                learningRate: import("zod").ZodOptional<import("zod").ZodNumber>;
+                contextLength: import("zod").ZodOptional<import("zod").ZodNumber>;
+                batchSize: import("zod").ZodOptional<import("zod").ZodNumber>;
+                microBatchSize: import("zod").ZodOptional<import("zod").ZodNumber>;
+                assistantLossOnly: import("zod").ZodOptional<import("zod").ZodBoolean>;
+                loraRank: import("zod").ZodOptional<import("zod").ZodNumber>;
+                loraAlpha: import("zod").ZodOptional<import("zod").ZodNumber>;
+                loraInitStd: import("zod").ZodOptional<import("zod").ZodNumber>;
+                loraSeed: import("zod").ZodOptional<import("zod").ZodNumber>;
+                loraModules: import("zod").ZodOptional<import("zod").ZodString>;
+                checkpointSaveDir: import("zod").ZodOptional<import("zod").ZodString>;
+                checkpointSaveSteps: import("zod").ZodOptional<import("zod").ZodNumber>;
+                chatTemplatePath: import("zod").ZodOptional<import("zod").ZodString>;
+                lrScheduler: import("zod").ZodOptional<import("zod").ZodEnum<{
+                    constant: "constant";
+                    cosine: "cosine";
+                    linear: "linear";
+                }>>;
+                lrMin: import("zod").ZodOptional<import("zod").ZodNumber>;
+                warmupRatio: import("zod").ZodOptional<import("zod").ZodNumber>;
+                warmupRatioSet: import("zod").ZodOptional<import("zod").ZodBoolean>;
+                warmupSteps: import("zod").ZodOptional<import("zod").ZodNumber>;
+                warmupStepsSet: import("zod").ZodOptional<import("zod").ZodBoolean>;
+                weightDecay: import("zod").ZodOptional<import("zod").ZodNumber>;
+            }, import("zod/v4/core").$strict>;
+            type: import("zod").ZodLiteral<"finetune">;
+            operation: import("zod").ZodLiteral<"getState">;
+        }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
+            modelId: import("zod").ZodString;
+            type: import("zod").ZodLiteral<"finetune">;
+            operation: import("zod").ZodEnum<{
+                cancel: "cancel";
+                pause: "pause";
+            }>;
+        }, import("zod/v4/core").$strict>]>, import("zod").ZodObject<{
+            type: import("zod").ZodLiteral<"finetune">;
+            status: import("zod").ZodEnum<{
+                IDLE: "IDLE";
+                RUNNING: "RUNNING";
+                PAUSED: "PAUSED";
+                CANCELLED: "CANCELLED";
+                COMPLETED: "COMPLETED";
+            }>;
+            stats: import("zod").ZodOptional<import("zod").ZodObject<{
+                train_loss: import("zod").ZodOptional<import("zod").ZodNumber>;
+                train_loss_uncertainty: import("zod").ZodOptional<import("zod").ZodNullable<import("zod").ZodXor<readonly [import("zod").ZodNumber, import("zod").ZodNaN]>>>;
+                val_loss: import("zod").ZodOptional<import("zod").ZodNumber>;
+                val_loss_uncertainty: import("zod").ZodOptional<import("zod").ZodNullable<import("zod").ZodXor<readonly [import("zod").ZodNumber, import("zod").ZodNaN]>>>;
+                train_accuracy: import("zod").ZodOptional<import("zod").ZodNumber>;
+                train_accuracy_uncertainty: import("zod").ZodOptional<import("zod").ZodNullable<import("zod").ZodXor<readonly [import("zod").ZodNumber, import("zod").ZodNaN]>>>;
+                val_accuracy: import("zod").ZodOptional<import("zod").ZodNumber>;
+                val_accuracy_uncertainty: import("zod").ZodOptional<import("zod").ZodNullable<import("zod").ZodXor<readonly [import("zod").ZodNumber, import("zod").ZodNaN]>>>;
+                learning_rate: import("zod").ZodOptional<import("zod").ZodNumber>;
+                global_steps: import("zod").ZodNumber;
+                epochs_completed: import("zod").ZodNumber;
+            }, import("zod/v4/core").$strict>>;
+        }, import("zod/v4/core").$strict>>;
         translate: import("../../../../schemas/index").PluginHandlerDefinition<import("zod").ZodUnion<readonly [import("zod").ZodObject<{
             modelId: import("zod").ZodString;
             text: import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodArray<import("zod").ZodString>]>;
@@ -206,8 +371,13 @@ export declare const llmPlugin: {
             token: import("zod").ZodString;
             done: import("zod").ZodOptional<import("zod").ZodBoolean>;
             stats: import("zod").ZodOptional<import("zod").ZodObject<{
-                processedTokens: import("zod").ZodNumber;
-                processingTime: import("zod").ZodNumber;
+                totalTime: import("zod").ZodOptional<import("zod").ZodNumber>;
+                totalTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
+                tokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
+                timeToFirstToken: import("zod").ZodOptional<import("zod").ZodNumber>;
+                decodeTime: import("zod").ZodOptional<import("zod").ZodNumber>;
+                encodeTime: import("zod").ZodOptional<import("zod").ZodNumber>;
+                cacheTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
             }, import("zod/v4/core").$strip>>;
             error: import("zod").ZodOptional<import("zod").ZodString>;
         }, import("zod/v4/core").$strip>>;

package/dist/server/bare/plugins/llamacpp-completion/plugin.js

@@ -1,12 +1,14 @@
 import LlmLlamacpp, {} from "@qvac/llm-llamacpp";
 import llmAddonLogging from "@qvac/llm-llamacpp/addonLogging.js";
-import { definePlugin, defineHandler, completionStreamRequestSchema, completionStreamResponseSchema, translateRequestSchema, translateResponseSchema, ModelType, llmConfigBaseSchema, ADDON_LLM, } from "../../../../schemas/index.js";
+import { definePlugin, defineHandler, finetuneRequestSchema, completionStreamRequestSchema, completionStreamResponseSchema, finetuneResponseSchema, translateRequestSchema, translateResponseSchema, ModelType, llmConfigBaseSchema, ADDON_LLM, } from "../../../../schemas/index.js";
 import { createStreamLogger, registerAddonLogger } from "../../../../logging/index.js";
 import { parseModelPath } from "../../../utils/index.js";
 import FilesystemDL from "@qvac/dl-filesystem";
 import { asLoader } from "../../utils/loader-adapter.js";
 import { completion } from "./ops/completion-stream.js";
+import { finetune } from "./ops/finetune.js";
 import { translate } from "../../ops/translate.js";
+import { attachModelExecutionMs } from "../../../../profiling/model-execution.js";
 function transformLlmConfig(llmConfig) {
     const transformed = JSON.parse(JSON.stringify(llmConfig, (key, v) => key === "modelType"
         ? undefined
@@ -21,6 +23,10 @@ function transformLlmConfig(llmConfig) {
         transformed["reverse_prompt"] = transformed["stop_sequences"];
         delete transformed["stop_sequences"];
     }
+    if ("opencl_cache_dir" in transformed) {
+        transformed["openclCacheDir"] = transformed["opencl_cache_dir"];
+        delete transformed["opencl_cache_dir"];
+    }
     return transformed;
 }
 function createLlmModel(modelId, modelPath, llmConfig, projectionModelPath) {
@@ -83,18 +89,10 @@ export const llmPlugin = definePlugin({
                     ...(request.tools && { tools: request.tools }),
                     ...(request.generationParams && { generationParams: request.generationParams }),
                 });
-                let stats;
-                let toolCalls = [];
-                let done = false;
-                let buffer = "";
-                while (!done) {
-                    const result = await stream.next();
-                    if (result.done) {
-                        stats = result.value.stats;
-                        toolCalls = result.value.toolCalls;
-                        done = true;
-                    }
-                    else {
+                try {
+                    let buffer = "";
+                    let result = await stream.next();
+                    while (!result.done) {
                         if (request.stream) {
                             yield {
                                 type: "completionStream",
@@ -107,15 +105,28 @@ export const llmPlugin = definePlugin({
                         else {
                             buffer += result.value.token;
                         }
+                        result = await stream.next();
                     }
+                    const { modelExecutionMs, stats, toolCalls } = result.value;
+                    yield attachModelExecutionMs({
+                        type: "completionStream",
+                        token: request.stream ? "" : buffer,
+                        done: true,
+                        ...(stats && { stats }),
+                        ...(toolCalls.length > 0 && { toolCalls }),
+                    }, modelExecutionMs);
+                }
+                finally {
+                    await stream.return?.(undefined);
                 }
-                yield {
-                    type: "completionStream",
-                    token: request.stream ? "" : buffer,
-                    done: true,
-                    ...(stats && { stats }),
-                    ...(toolCalls.length > 0 && { toolCalls }),
-                };
+            },
+        }),
+        finetune: defineHandler({
+            requestSchema: finetuneRequestSchema,
+            responseSchema: finetuneResponseSchema,
+            streaming: false,
+            handler: function (request) {
+                return finetune(request);
             },
         }),
         translate: defineHandler({
@@ -124,27 +135,26 @@ export const llmPlugin = definePlugin({
             streaming: true,
             handler: async function* (request) {
                 const stream = translate(request);
-                let done = false;
-                let stats;
-                while (!done) {
-                    const result = await stream.next();
-                    if (result.done) {
-                        stats = result.value;
-                        done = true;
-                    }
-                    else {
+                try {
+                    let result = await stream.next();
+                    while (!result.done) {
                         yield {
                             type: "translate",
                             token: result.value,
                         };
+                        result = await stream.next();
                     }
+                    const { modelExecutionMs, stats } = result.value;
+                    yield attachModelExecutionMs({
+                        type: "translate",
+                        token: "",
+                        done: true,
+                        ...(stats && { stats }),
+                    }, modelExecutionMs);
+                }
+                finally {
+                    await stream.return?.(undefined);
                 }
-                yield {
-                    type: "translate",
-                    token: "",
-                    done: true,
-                    ...(stats && { stats }),
-                };
             },
         }),
     },

package/dist/server/bare/plugins/llamacpp-embedding/plugin.d.ts

@@ -6,8 +6,8 @@ export declare const embeddingsPlugin: {
     loadConfigSchema: import("zod").ZodObject<{
         gpuLayers: import("zod").ZodOptional<import("zod").ZodNumber>;
         device: import("zod").ZodOptional<import("zod").ZodEnum<{
-            gpu: "gpu";
             cpu: "cpu";
+            gpu: "gpu";
         }>>;
         batchSize: import("zod").ZodOptional<import("zod").ZodNumber>;
         pooling: import("zod").ZodOptional<import("zod").ZodEnum<{
@@ -32,6 +32,7 @@ export declare const embeddingsPlugin: {
             dedicated: "dedicated";
         }>]>>;
         verbosity: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodLiteral<0>, import("zod").ZodLiteral<1>, import("zod").ZodLiteral<2>, import("zod").ZodLiteral<3>]>>;
+        openclCacheDir: import("zod").ZodOptional<import("zod").ZodString>;
     }, import("zod/v4/core").$strip>;
     createModel(params: CreateModelParams): PluginModelResult;
     handlers: {
@@ -43,6 +44,15 @@ export declare const embeddingsPlugin: {
             type: import("zod").ZodLiteral<"embed">;
             success: import("zod").ZodBoolean;
             embedding: import("zod").ZodDefault<import("zod").ZodUnion<readonly [import("zod").ZodArray<import("zod").ZodNumber>, import("zod").ZodArray<import("zod").ZodArray<import("zod").ZodNumber>>]>>;
+            stats: import("zod").ZodOptional<import("zod").ZodObject<{
+                totalTime: import("zod").ZodOptional<import("zod").ZodNumber>;
+                tokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
+                totalTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
+                backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
+                    cpu: "cpu";
+                    gpu: "gpu";
+                }>>;
+            }, import("zod/v4/core").$strip>>;
             error: import("zod").ZodOptional<import("zod").ZodString>;
         }, import("zod/v4/core").$strip>>;
     };