@qvac/sdk 0.1.1 → 0.2.0

package/dist/schemas/rag.js

@@ -4,7 +4,9 @@ const llmChunkOptsSchema = z.object({
     chunkSize: z.number().optional(),
     chunkOverlap: z.number().optional(),
     chunkStrategy: z.enum(["character", "paragraph"]).optional(),
-    splitStrategy: z.enum(["character", "word", "sentence", "line"]).optional(),
+    splitStrategy: z
+        .enum(["character", "word", "token", "sentence", "line"])
+        .optional(),
 });
 const ragSaveEmbeddingsResultSchema = z.object({
     status: z.enum(["fulfilled", "rejected"]),

package/dist/server/bare/addons/llamacpp/handlers/completion-stream.js

@@ -1,65 +1,147 @@
 import { completionParamsSchema, } from "../../../../../schemas/index.js";
-import { getModel } from "../../../registry/model-registry.js";
-import { findMatchingCache, getCurrentCacheInfo, renameCacheFile, } from "../kv-cache-utils";
-export async function* completion(params) {
-    const { history, modelId, kvCache } = completionParamsSchema.parse(params);
-    const transformedHistory = [];
-    for (const message of history) {
-        if (message.attachments && message.attachments.length > 0) {
-            for (const attachment of message.attachments) {
-                transformedHistory.push({
-                    role: message.role,
-                    content: attachment.path,
-                    type: "media",
-                });
-            }
+import { getModel, getModelConfig, } from "../../../registry/model-registry.js";
+import { findMatchingCache, getCurrentCacheInfo, renameCacheFile, getCacheFilePath, extractSystemPrompt, generateConfigHash, customCacheExists, } from "../kv-cache-utils";
+function transformMessage(message) {
+    const transformed = [];
+    if (message.attachments && message.attachments.length > 0) {
+        for (const attachment of message.attachments) {
+            transformed.push({
+                role: message.role,
+                content: attachment.path,
+                type: "media",
+            });
         }
-        transformedHistory.push({
-            role: message.role,
-            content: message.content,
-        });
     }
-    const model = getModel(modelId);
-    if (kvCache) {
-        const cacheMessages = history.map((msg) => ({
-            role: msg.role,
-            content: msg.content,
-            attachments: msg.attachments ?? undefined,
-        }));
-        const existingCache = await findMatchingCache(cacheMessages);
-        const currentCacheInfo = getCurrentCacheInfo(cacheMessages);
-        const cachePathToUse = existingCache !== null
-            ? existingCache.cachePath
-            : currentCacheInfo.cachePath;
-        const messagesWithCache = [
+    transformed.push({
+        role: message.role,
+        content: message.content,
+    });
+    return transformed;
+}
+function transformMessages(messages) {
+    const transformed = [];
+    for (const message of messages) {
+        transformed.push(...transformMessage(message));
+    }
+    return transformed;
+}
+async function initSystemPromptCache(model, cachePathToUse, systemPromptToUse) {
+    const primeMessages = [
+        { role: "session", content: cachePathToUse },
+        { role: "system", content: systemPromptToUse },
+    ];
+    const primeResponse = await model.run(primeMessages);
+    primeResponse.once("output", () => {
+        void primeResponse.cancel();
+    });
+    await primeResponse.await();
+}
+function prepareMessagesForCache(cachePathToUse, cacheExists, history) {
+    if (cacheExists && history.length > 0) {
+        const lastMessage = history[history.length - 1];
+        const lastTransformedMessages = transformMessage(lastMessage);
+        return [
             { role: "session", content: cachePathToUse },
-            ...transformedHistory,
+            ...lastTransformedMessages,
         ];
-        const response = await model.run(messagesWithCache);
-        for await (const token of response.iterate()) {
-            yield token;
+    }
+    const historyWithoutSystem = history.filter((msg) => msg.role !== "system");
+    const transformedHistoryWithoutSystem = transformMessages(historyWithoutSystem);
+    return [
+        { role: "session", content: cachePathToUse },
+        ...transformedHistoryWithoutSystem,
+    ];
+}
+async function* processModelResponse(model, messagesToSend, shouldSaveCache) {
+    const response = await model.run(messagesToSend);
+    // Create a promise that rejects when onError is triggered
+    let rejectOnError = null;
+    const errorPromise = new Promise((_, reject) => {
+        rejectOnError = reject;
+    });
+    response.onError((error) => {
+        if (rejectOnError) {
+            rejectOnError(error);
+        }
+    });
+    const iterator = response.iterate();
+    try {
+        while (true) {
+            // Race between getting the next token and an error occurring
+            const result = await Promise.race([iterator.next(), errorPromise]);
+            if (result.done) {
+                break;
+            }
+            yield result.value;
+        }
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            throw error;
+        }
+        else {
+            throw new Error(String(error));
         }
+    }
+    if (shouldSaveCache) {
         await model.run([{ role: "session", content: "save" }]);
-        //If there was an existing cache, we rename it with new hash that includes the current message
-        if (existingCache !== null &&
-            existingCache.cachePath !== currentCacheInfo.cachePath) {
-            await renameCacheFile(existingCache.cachePath, currentCacheInfo.cachePath);
+    }
+    const responseWithStats = response;
+    return {
+        timeToFirstToken: responseWithStats.stats?.TTFT ?? 0,
+        tokensPerSecond: responseWithStats.stats?.TPS ?? 0,
+        cacheTokens: responseWithStats.stats?.CacheTokens ?? 0,
+    };
+}
+export async function* completion(params) {
+    const { history, modelId, kvCache } = completionParamsSchema.parse(params);
+    const transformedHistory = transformMessages(history);
+    const model = getModel(modelId);
+    if (kvCache) {
+        const modelConfig = getModelConfig(modelId);
+        const systemPromptFromHistory = extractSystemPrompt(history);
+        const configHash = generateConfigHash(modelConfig, systemPromptFromHistory);
+        const systemPromptToUse = systemPromptFromHistory ||
+            modelConfig.system_prompt ||
+            "You are a helpful assistant.";
+        let cachePathToUse;
+        if (typeof kvCache === "string") {
+            cachePathToUse = await getCacheFilePath(modelId, configHash, kvCache);
+            const cacheExists = await customCacheExists(modelId, configHash, kvCache);
+            if (!cacheExists) {
+                await initSystemPromptCache(model, cachePathToUse, systemPromptToUse);
+            }
+            const messagesToSend = prepareMessagesForCache(cachePathToUse, cacheExists, history);
+            return yield* processModelResponse(model, messagesToSend, true);
+        }
+        else {
+            // Auto-generate cache key based on conversation history
+            const cacheMessages = history.map((msg) => ({
+                role: msg.role,
+                content: msg.content,
+                attachments: msg.attachments ?? undefined,
+            }));
+            const existingCache = await findMatchingCache(modelId, configHash, cacheMessages);
+            const currentCacheInfo = await getCurrentCacheInfo(modelId, configHash, cacheMessages);
+            cachePathToUse =
+                existingCache !== null
+                    ? existingCache.cachePath
+                    : currentCacheInfo.cachePath;
+            const cacheExists = existingCache !== null;
+            if (!cacheExists) {
+                await initSystemPromptCache(model, cachePathToUse, systemPromptToUse);
+            }
+            const messagesToSend = prepareMessagesForCache(cachePathToUse, cacheExists, history);
+            const result = yield* processModelResponse(model, messagesToSend, true);
+            //If there was an existing cache, we rename it with new hash that includes the current message
+            if (existingCache !== null &&
+                existingCache.cachePath !== currentCacheInfo.cachePath) {
+                await renameCacheFile(existingCache.cachePath, currentCacheInfo.cachePath);
+            }
+            return result;
         }
-        const responseWithStats = response;
-        return {
-            timeToFirstToken: responseWithStats.stats?.TTFT ?? 0,
-            tokensPerSecond: responseWithStats.stats?.TPS ?? 0,
-        };
     }
     else {
-        const response = await model.run(transformedHistory);
-        for await (const token of response.iterate()) {
-            yield token;
-        }
-        const responseWithStats = response;
-        return {
-            timeToFirstToken: responseWithStats.stats?.TTFT ?? 0,
-            tokensPerSecond: responseWithStats.stats?.TPS ?? 0,
-        };
+        return yield* processModelResponse(model, transformedHistory, false);
     }
 }

package/dist/server/bare/addons/llamacpp/kv-cache-utils.d.ts

@@ -5,15 +5,24 @@ export interface CacheMessage {
         path: string;
     }[] | undefined;
 }
+export declare function extractSystemPrompt(messages: CacheMessage[]): string | null;
+export declare function generateConfigHash(modelConfig: unknown, systemPrompt: string | null): string;
 export declare function generateCacheKey(messages: CacheMessage[]): string;
-export declare function getCacheFilePath(cacheKey: string): string;
-export declare function findMatchingCache(currentHistory: CacheMessage[]): Promise<{
+export declare function getCacheFilePath(modelId: string, configHash: string, cacheKey: string): Promise<string>;
+export declare function findMatchingCache(modelId: string, configHash: string, currentHistory: CacheMessage[]): Promise<{
     cacheKey: string;
     cachePath: string;
 } | null>;
-export declare function getCurrentCacheInfo(currentHistory: CacheMessage[]): {
+export declare function getCurrentCacheInfo(modelId: string, configHash: string, currentHistory: CacheMessage[]): Promise<{
     cacheKey: string;
     cachePath: string;
-};
+}>;
 export declare function renameCacheFile(oldPath: string, newPath: string): Promise<void>;
+export declare function customCacheExists(modelId: string, configHash: string, cacheKey: string): Promise<boolean>;
+export declare function deleteCache(options: {
+    all: true;
+} | {
+    kvCacheKey: string;
+    modelId?: string;
+}): Promise<void>;
 //# sourceMappingURL=kv-cache-utils.d.ts.map

package/dist/server/bare/addons/llamacpp/kv-cache-utils.js

@@ -2,6 +2,18 @@ import crypto from "bare-crypto";
 import { promises as fsPromises } from "bare-fs";
 import path from "bare-path";
 import { getKVCacheDir } from "../../../utils/index.js";
+export function extractSystemPrompt(messages) {
+    const systemMessage = messages.find((msg) => msg.role === "system");
+    return systemMessage ? systemMessage.content : null;
+}
+export function generateConfigHash(modelConfig, systemPrompt) {
+    const hash = crypto.createHash("sha256");
+    const configStr = JSON.stringify({ config: modelConfig, systemPrompt });
+    const configBuffer = Buffer.from(configStr, "utf8");
+    hash.update(configBuffer);
+    const hashString = hash.digest("hex");
+    return hashString.substring(0, 16);
+}
 export function generateCacheKey(messages) {
     const hash = crypto.createHash("sha256");
     const historyString = JSON.stringify(messages);
@@ -10,18 +22,28 @@ export function generateCacheKey(messages) {
     const hashString = hash.digest("hex");
     return hashString.substring(0, 16);
 }
-export function getCacheFilePath(cacheKey) {
+export async function getCacheFilePath(modelId, configHash, cacheKey) {
     const cacheDir = getKVCacheDir();
-    return path.join(cacheDir, `kvcache${cacheKey}.bin`);
+    const sessionCacheDir = path.join(cacheDir, cacheKey);
+    const modelCacheDir = path.join(sessionCacheDir, modelId);
+    try {
+        await fsPromises.mkdir(sessionCacheDir, { recursive: true });
+        await fsPromises.mkdir(modelCacheDir, { recursive: true });
+    }
+    catch {
+        // Ignore if directories already exist
+    }
+    return path.join(modelCacheDir, `${configHash}.bin`);
 }
-export async function findMatchingCache(currentHistory) {
+// Used for auto-generated cache key
+export async function findMatchingCache(modelId, configHash, currentHistory) {
     if (currentHistory.length <= 1) {
         return null;
     }
     // Generate cache key for history minus the last message
     const previousHistory = currentHistory.slice(0, -1);
     const cacheKey = generateCacheKey(previousHistory);
-    const cachePath = getCacheFilePath(cacheKey);
+    const cachePath = await getCacheFilePath(modelId, configHash, cacheKey);
     try {
         await fsPromises.access(cachePath);
         return { cacheKey, cachePath };
@@ -30,10 +52,9 @@ export async function findMatchingCache(currentHistory) {
         return null;
     }
 }
-export function getCurrentCacheInfo(currentHistory) {
-    // Generate cache path for the current conversation state
+export async function getCurrentCacheInfo(modelId, configHash, currentHistory) {
     const cacheKey = generateCacheKey(currentHistory);
-    const cachePath = getCacheFilePath(cacheKey);
+    const cachePath = await getCacheFilePath(modelId, configHash, cacheKey);
     return { cacheKey, cachePath };
 }
 export async function renameCacheFile(oldPath, newPath) {
@@ -44,3 +65,45 @@ export async function renameCacheFile(oldPath, newPath) {
         console.error("Error renaming cache file:", error instanceof Error ? error.message : String(error));
     }
 }
+export async function customCacheExists(modelId, configHash, cacheKey) {
+    const cachePath = await getCacheFilePath(modelId, configHash, cacheKey);
+    try {
+        await fsPromises.access(cachePath);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export async function deleteCache(options) {
+    const cacheDir = getKVCacheDir();
+    if ("all" in options && options.all) {
+        try {
+            await fsPromises.rm(cacheDir, { recursive: true });
+            await fsPromises.mkdir(cacheDir, { recursive: true });
+        }
+        catch (error) {
+            console.error("Error deleting all caches:", error instanceof Error ? error.message : String(error));
+        }
+    }
+    else if ("kvCacheKey" in options) {
+        const kvCacheDir = path.join(cacheDir, options.kvCacheKey);
+        if (options.modelId) {
+            const modelCacheDir = path.join(kvCacheDir, options.modelId);
+            try {
+                await fsPromises.rm(modelCacheDir, { recursive: true });
+            }
+            catch (error) {
+                console.error(`Error deleting model cache ${options.kvCacheKey}/${options.modelId}:`, error instanceof Error ? error.message : String(error));
+            }
+        }
+        else {
+            try {
+                await fsPromises.rm(kvCacheDir, { recursive: true });
+            }
+            catch (error) {
+                console.error(`Error deleting cache ${options.kvCacheKey}:`, error instanceof Error ? error.message : String(error));
+            }
+        }
+    }
+}

package/dist/server/bare/addons/load-model.js

@@ -51,5 +51,5 @@ export async function loadModel(params) {
     console.log(`${modelType}: Loading model ${modelId}...`);
     await model.load(false);
     console.log(`${modelType} model ${modelId} loaded`);
-    registerModel(modelId, { model, path: modelPath });
+    registerModel(modelId, { model, path: modelPath, config: modelConfig });
 }

package/dist/server/bare/registry/model-registry.d.ts

@@ -1,17 +1,20 @@
+import type EventEmitter from "events";
 interface AddonInterface {
     cancel(jobId?: string): Promise<void>;
 }
 export interface AnyModel {
     load(...args: unknown[]): Promise<void>;
-    run(input: unknown): Promise<{
+    run(input: unknown): Promise<EventEmitter & {
         onUpdate(callback: (data: string) => void): {
             await: () => Promise<void>;
         };
+        onError(callback: (error: Error) => void): void;
         onUpdate(callback: (data: Array<{
             text: string;
         }>) => void): {
             await: () => Promise<void>;
         };
+        cancel(): Promise<void>;
         iterate(): AsyncIterableIterator<string | Array<{
             text: string;
         }>>;
@@ -29,6 +32,7 @@ interface LocalOptions {
     model: AnyModel;
     path: string;
     loadedAt: Date;
+    config: unknown;
 }
 export interface ModelEntry {
     id: string;
@@ -39,6 +43,7 @@ export interface ModelEntry {
 export declare function registerModel(id: string, options: {
     model: AnyModel;
     path: string;
+    config: unknown;
 } | {
     topic: string;
     providerPublicKey: string;
@@ -53,7 +58,9 @@ export declare function getModelInfo(id: string): {
     id: string;
     path: string;
     loadedAt: Date;
+    config: unknown;
 } | null;
+export declare function getModelConfig(id: string): unknown;
 export declare function clearRegistry(): void;
 export declare function getRegistryStats(): {
     totalModels: number;

package/dist/server/bare/registry/model-registry.js

@@ -26,6 +26,7 @@ export function registerModel(id, options) {
                 model: options.model,
                 path: options.path,
                 loadedAt: new Date(),
+                config: options.config,
             },
         });
         console.log(`Local model registered: ${id} -> ${options.path}`);
@@ -69,8 +70,16 @@ export function getModelInfo(id) {
         id: entry.id,
         path: entry.local.path,
         loadedAt: entry.local.loadedAt,
+        config: entry.local.config,
     };
 }
+export function getModelConfig(id) {
+    const entry = modelRegistry.get(id);
+    if (!entry || !entry.local) {
+        throw new Error(`Model with ID "${id}" not found or is delegated`);
+    }
+    return entry.local.config;
+}
 export function clearRegistry() {
     modelRegistry.clear();
     console.log("Model registry cleared");

package/dist/server/rpc/handle-request.js

@@ -13,6 +13,7 @@ import { handleLoggingStream } from "./handlers/logging-stream.js";
 import { cancelHandler } from "./handlers/cancelHandler.js";
 import { provideHandler } from "./handlers/provideHandler.js";
 import { handleRag } from "./handlers/rag.js";
+import { handleDeleteCache } from "./handlers/delete-cache.js";
 import { sendErrorResponse, sendStreamErrorResponse } from "../error-handlers";
 export async function handleRequest(req) {
     try {
@@ -209,6 +210,16 @@ export async function handleRequest(req) {
                 }
                 break;
             }
+            case "deleteCache": {
+                try {
+                    const response = await handleDeleteCache(request);
+                    req.reply(JSON.stringify(responseSchema.parse(response)), "utf-8");
+                }
+                catch (error) {
+                    sendErrorResponse(req, error);
+                }
+                break;
+            }
             default: {
                 throw new Error(`Unknown request type`);
             }

package/dist/server/rpc/handlers/completion-stream.js

@@ -1,52 +1,38 @@
 import { completion } from "../../bare/addons/llamacpp/index.js";
 export async function* handleCompletionStream(request) {
-    try {
-        // Log tools if provided
-        if (request.tools) {
-            console.log("Tools passed to completionStream:", request.tools);
+    const filteredHistory = request.history.map(({ role, content, attachments }) => ({
+        role,
+        content,
+        attachments: attachments ?? [],
+    }));
+    const stream = completion({
+        history: filteredHistory,
+        modelId: request.modelId,
+        kvCache: request.kvCache,
+    });
+    let stats;
+    let done = false;
+    let buffer = "";
+    while (!done) {
+        const result = await stream.next();
+        if (result.done) {
+            stats = result.value;
+            done = true;
         }
-        const filteredHistory = request.history.map(({ role, content, attachments }) => ({
-            role,
-            content,
-            attachments: attachments ?? [],
-        }));
-        const stream = completion({
-            history: filteredHistory,
-            modelId: request.modelId,
-            kvCache: request.kvCache,
-        });
-        let stats;
-        let done = false;
-        let buffer = "";
-        while (!done) {
-            const result = await stream.next();
-            if (result.done) {
-                stats = result.value;
-                done = true;
-            }
-            else {
-                buffer += result.value;
-                if (request.stream) {
-                    yield {
-                        type: "completionStream",
-                        token: result.value,
-                    };
-                }
+        else {
+            buffer += result.value;
+            if (request.stream) {
+                yield {
+                    type: "completionStream",
+                    token: result.value,
+                };
             }
         }
-        yield {
-            type: "completionStream",
-            token: buffer,
-            done: true,
-            stats,
-        };
-    }
-    catch (error) {
-        console.error("Error during completion stream:", error);
-        yield {
-            type: "completionStream",
-            token: "",
-            error: error instanceof Error ? error.message : "Unknown error",
-        };
     }
+    yield {
+        type: "completionStream",
+        token: buffer,
+        done: true,
+        stats,
+    };
 }

package/dist/server/rpc/handlers/delete-cache.d.ts

@@ -0,0 +1,3 @@
+import type { DeleteCacheRequest, DeleteCacheResponse } from "../../../schemas/index";
+export declare function handleDeleteCache(request: DeleteCacheRequest): Promise<DeleteCacheResponse>;
+//# sourceMappingURL=delete-cache.d.ts.map

package/dist/server/rpc/handlers/delete-cache.js

@@ -0,0 +1,29 @@
+import { deleteCache as deleteCacheUtil } from "../../bare/addons/llamacpp/kv-cache-utils.js";
+export async function handleDeleteCache(request) {
+    try {
+        if ("all" in request && request.all) {
+            await deleteCacheUtil({ all: true });
+        }
+        else if ("kvCacheKey" in request) {
+            const params = {
+                kvCacheKey: request.kvCacheKey,
+            };
+            if (request.modelId !== undefined) {
+                params.modelId = request.modelId;
+            }
+            await deleteCacheUtil(params);
+        }
+        return {
+            type: "deleteCache",
+            success: true,
+        };
+    }
+    catch (error) {
+        console.error("Error deleting cache:", error);
+        return {
+            type: "deleteCache",
+            success: false,
+            error: error instanceof Error ? error.message : "Unknown error",
+        };
+    }
+}

package/dist/server/rpc/handlers/index.d.ts

@@ -8,9 +8,12 @@ import { provideHandler } from "./provideHandler";
 import { handleTranslate } from "./translate";
 import { handleLoggingStream } from "./logging-stream";
 import { handleRag } from "./rag";
+import { cancelHandler } from "./cancelHandler";
+import { handleDeleteCache } from "./delete-cache";
 export declare const handlers: {
     completionStream: typeof handleCompletionStream;
     downloadAsset: typeof handleDownloadAsset;
+    deleteCache: typeof handleDeleteCache;
     loadModel: typeof handleLoadModel;
     unloadModel: typeof handleUnloadModel;
     embed: typeof handleEmbed;
@@ -19,5 +22,6 @@ export declare const handlers: {
     translate: typeof handleTranslate;
     loggingStream: typeof handleLoggingStream;
     rag: typeof handleRag;
+    cancel: typeof cancelHandler;
 };
 //# sourceMappingURL=index.d.ts.map

package/dist/server/rpc/handlers/index.js

@@ -8,9 +8,12 @@ import { provideHandler } from "./provideHandler.js";
 import { handleTranslate } from "./translate.js";
 import { handleLoggingStream } from "./logging-stream.js";
 import { handleRag } from "./rag.js";
+import { cancelHandler } from "./cancelHandler.js";
+import { handleDeleteCache } from "./delete-cache.js";
 export const handlers = {
     completionStream: handleCompletionStream,
     downloadAsset: handleDownloadAsset,
+    deleteCache: handleDeleteCache,
     loadModel: handleLoadModel,
     unloadModel: handleUnloadModel,
     embed: handleEmbed,
@@ -19,4 +22,5 @@ export const handlers = {
     translate: handleTranslate,
     loggingStream: handleLoggingStream,
     rag: handleRag,
+    cancel: cancelHandler,
 };

package/dist/server/rpc/handlers/load-model/download-manager.d.ts

@@ -4,28 +4,9 @@ export declare const registerDownload: (key: string, entry: DownloadEntry) => vo
 export declare const unregisterDownload: (key: string) => void;
 export declare const createHyperdriveDownloadKey: (hyperdriveKey: string, modelFileName: string) => string;
 export declare const createHttpDownloadKey: (url: string) => string;
-export declare const getAllActiveDownloads: () => Map<string, DownloadEntry>;
 export declare const setClearCacheFlag: (downloadKey: string, clearCache: boolean) => void;
 export declare const shouldClearCache: (downloadKey: string) => boolean;
 export declare const clearClearCacheFlag: (downloadKey: string) => void;
 export declare const createCancelFunction: (downloadKey: string, clearCache?: boolean) => () => void;
 export declare const cancelAllDownloads: () => void;
-export declare const getDownloadStats: () => {
-    activeCount: number;
-    downloads: ({
-        hyperdriveKey: string;
-        modelFileName: string;
-        key: string;
-        startTime: number;
-        duration: number;
-        type: "hyperdrive" | "http";
-    } | {
-        url: string;
-        modelPath: string;
-        key: string;
-        startTime: number;
-        duration: number;
-        type: "hyperdrive" | "http";
-    })[];
-};
 //# sourceMappingURL=download-manager.d.ts.map