@qvac/sdk 0.10.1 → 0.11.0

package/dist/schemas/unload-model.js

@@ -2,9 +2,12 @@ import { z } from "zod";
 export const unloadModelParamsSchema = z.object({
     modelId: z.string(),
     clearStorage: z.boolean().default(false),
+    autoClose: z.boolean().optional(),
 });
-export const unloadModelRequestSchema = unloadModelParamsSchema.extend({
+export const unloadModelRequestSchema = z.object({
     type: z.literal("unloadModel"),
+    modelId: z.string(),
+    clearStorage: z.boolean().default(false),
 });
 export const unloadModelResponseSchema = z.object({
     type: z.literal("unloadModel"),

package/dist/scripts/run-unit-tests.js

@@ -4,10 +4,23 @@ import { spawnSync } from "child_process";
 import { fileURLToPath } from "url";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const testDir = join(__dirname, "..", "test", "unit");
-const testFiles = readdirSync(testDir).filter((f) => f.endsWith(".test.ts"));
+function collectTestFiles(dir) {
+    const files = [];
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+        const fullPath = join(dir, entry.name);
+        if (entry.isDirectory()) {
+            files.push(...collectTestFiles(fullPath));
+        }
+        else if (entry.isFile() && entry.name.endsWith(".test.ts")) {
+            files.push(fullPath);
+        }
+    }
+    return files;
+}
+const testFiles = collectTestFiles(testDir);
 let hasFailure = false;
 for (const file of testFiles) {
-    const result = spawnSync("bun", ["run", join(testDir, file)], {
+    const result = spawnSync("bun", ["run", file], {
         stdio: "inherit",
     });
     if (result.status !== 0) {

package/dist/server/bare/delegate-rpc-client.js

@@ -136,14 +136,16 @@ async function ensureRPCConnection(publicKey, timeout, healthCheckTimeout) {
     let conn;
     try {
         logger.info(`🔗 Establishing direct DHT connection to peer: ${publicKey}${timeout ? `, timeout: ${timeout}ms` : ""}`);
-        // Ensure the DHT routing table is populated before attempting to look up
-        // the peer. On a cold swarm the table is nearly empty and findPeer can
-        // return PEER_NOT_FOUND immediately even when the provider is reachable.
-        const bootstrapStart = nowMs();
-        const swarm = getSwarm();
-        logger.info(`⏳ Waiting for DHT to fully bootstrap before connect...`);
-        await withTimeout(swarm.dht.fullyBootstrapped(), getRemainingTimeout());
-        logger.info(`✅ DHT bootstrapped in ${(nowMs() - bootstrapStart).toFixed(0)}ms`);
+        // We deliberately do NOT `await swarm.dht.fullyBootstrapped()` here. The
+        // earlier guard (added with #1729 to side-step a theoretical PEER_NOT_FOUND
+        // on a fully-cold swarm) added a serial 1-3s wait on every first delegated
+        // call — measurably regressing `loadModel.delegation.connection` vs 0.9.0
+        // (≈3.2× slower in local benches). `getSwarm()` is invoked early during
+        // SDK init (registry/runtime), so by the time the consumer reaches this
+        // path the routing table is already warm enough; `dht.connect()` also
+        // bootstraps on demand if it isn't, so we lose nothing by skipping the
+        // explicit await.
+        getSwarm();
         conn = openDhtConnection(publicKey);
         await waitForOpen(conn, getRemainingTimeout());
         logger.info(`🍺 Peer connection opened: ${publicKey}`);

package/dist/server/bare/ops/cancel.d.ts

@@ -1,3 +1,29 @@
 import { type CancelInferenceBaseParams } from "../../../schemas/index";
-export declare function cancel(params: CancelInferenceBaseParams): Promise<void>;
+import type { RequestKind } from "../runtime/index";
+/**
+ * Broad cancel: abort every in-flight request matching `modelId` (and
+ * optionally a `kind`). Maps onto `RequestRegistry.cancel({ modelId })`
+ * — the registry walks active contexts and aborts each one's signal,
+ * which each handler has wired to its own addon-level / async unwind
+ * via the registry's `await using ctx = registry.begin(...)` block.
+ *
+ * Kept as a stable surface alongside the new `cancel({ requestId })`
+ * path: the caller may not have a `requestId` to hand (model unload,
+ * app shutdown, admin sweeps), and the escape hatch is cheap because
+ * the registry already does the matching.
+ *
+ * Returns the number of contexts whose abort was triggered by *this*
+ * call (already-cancelled contexts are skipped so callers can rely on
+ * the count to log "n requests cancelled" once). Used by the RPC
+ * cancel handler to populate `CancelResponse.cancelled` and by
+ * internal server-side callers that want to know whether anything
+ * landed.
+ *
+ * The legacy pre-registry addon-cancel fallback was removed in 0.11.0
+ * once every handler had been migrated onto the registry; the function
+ * now does exactly one thing — a registry walk.
+ */
+export declare function cancel(params: CancelInferenceBaseParams, opts?: {
+    kind?: RequestKind;
+}): number;
 //# sourceMappingURL=cancel.d.ts.map

package/dist/server/bare/ops/cancel.js

@@ -1,20 +1,46 @@
 import { getModel } from "../registry/model-registry.js";
 import { cancelInferenceBaseSchema, } from "../../../schemas/index.js";
 import { ModelNotLoadedError } from "../../../utils/errors-server.js";
-import { noteCancelRequested } from "../plugins/llamacpp-completion/ops/kv-cache-state.js";
-export async function cancel(params) {
+import { getRequestRegistry } from "../runtime/index.js";
+import { getServerLogger } from "../../../logging/index.js";
+const logger = getServerLogger();
+/**
+ * Broad cancel: abort every in-flight request matching `modelId` (and
+ * optionally a `kind`). Maps onto `RequestRegistry.cancel({ modelId })`
+ * — the registry walks active contexts and aborts each one's signal,
+ * which each handler has wired to its own addon-level / async unwind
+ * via the registry's `await using ctx = registry.begin(...)` block.
+ *
+ * Kept as a stable surface alongside the new `cancel({ requestId })`
+ * path: the caller may not have a `requestId` to hand (model unload,
+ * app shutdown, admin sweeps), and the escape hatch is cheap because
+ * the registry already does the matching.
+ *
+ * Returns the number of contexts whose abort was triggered by *this*
+ * call (already-cancelled contexts are skipped so callers can rely on
+ * the count to log "n requests cancelled" once). Used by the RPC
+ * cancel handler to populate `CancelResponse.cancelled` and by
+ * internal server-side callers that want to know whether anything
+ * landed.
+ *
+ * The legacy pre-registry addon-cancel fallback was removed in 0.11.0
+ * once every handler had been migrated onto the registry; the function
+ * now does exactly one thing — a registry walk.
+ */
+export function cancel(params, opts) {
     const { modelId } = cancelInferenceBaseSchema.parse(params);
     const model = getModel(modelId);
     if (!model) {
         throw new ModelNotLoadedError(modelId);
     }
-    // Must be recorded *before* `addon.cancel()` so the in-flight
-    // `completion()` for this model sees the bumped counter when it
-    // snapshots after `processModelResponse` returns. This is the signal
-    // that tells `completion()` not to record a `savedCount` for the
-    // kv-cache on a cancelled turn.
-    noteCancelRequested(modelId);
-    if (model.addon && model.addon.cancel) {
-        await model.addon.cancel();
+    const registry = getRequestRegistry();
+    const target = opts?.kind ? { modelId, kind: opts.kind } : { modelId };
+    const cancelled = registry.cancel(target);
+    if (cancelled === 0) {
+        // Callers (workbench "Stop" button, app shutdown sweeps) often
+        // fire-and-forget; log so operators can see when a broad cancel
+        // landed against a registry with nothing in flight on this model.
+        logger.debug(`[cancel] no in-flight request matched modelId=${modelId}${opts?.kind ? ` kind=${opts.kind}` : ""}`);
     }
+    return cancelled;
 }

package/dist/server/bare/ops/embed.d.ts

@@ -6,11 +6,11 @@ export interface EmbedResult {
 export declare function embed(params: {
     modelId: string;
     text: string;
-}): Promise<EmbedResult>;
+}, requestId?: string): Promise<EmbedResult>;
 export declare function embed(params: {
     modelId: string;
     text: string[];
-}): Promise<EmbedResult>;
-export declare function embed(params: EmbedParams): Promise<EmbedResult>;
+}, requestId?: string): Promise<EmbedResult>;
+export declare function embed(params: EmbedParams, requestId?: string): Promise<EmbedResult>;
 export declare function normalizeVector(vector: Float32Array): number[];
 //# sourceMappingURL=embed.d.ts.map

package/dist/server/bare/ops/embed.js

@@ -1,40 +1,156 @@
+var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
+    if (value !== null && value !== void 0) {
+        if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
+        var dispose, inner;
+        if (async) {
+            if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
+            dispose = value[Symbol.asyncDispose];
+        }
+        if (dispose === void 0) {
+            if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
+            dispose = value[Symbol.dispose];
+            if (async) inner = dispose;
+        }
+        if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
+        if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
+        env.stack.push({ value: value, dispose: dispose, async: async });
+    }
+    else if (async) {
+        env.stack.push({ async: true });
+    }
+    return value;
+};
+var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
+    return function (env) {
+        function fail(e) {
+            env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
+            env.hasError = true;
+        }
+        var r, s = 0;
+        function next() {
+            while (r = env.stack.pop()) {
+                try {
+                    if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
+                    if (r.dispose) {
+                        var result = r.dispose.call(r.value);
+                        if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
+                    }
+                    else s |= 1;
+                }
+                catch (e) {
+                    fail(e);
+                }
+            }
+            if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
+            if (env.hasError) throw env.error;
+        }
+        return next();
+    };
+})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
+    var e = new Error(message);
+    return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
+});
 import { getModel } from "../registry/model-registry.js";
 import { embedParamsSchema } from "../../../schemas/index.js";
 import { buildUnaryResult } from "../../../profiling/model-execution.js";
-import { EmbedNoEmbeddingsError, EmbedFailedError, } from "../../../utils/errors-server.js";
+import { EmbedNoEmbeddingsError, EmbedFailedError, InferenceCancelledError, } from "../../../utils/errors-server.js";
 import { nowMs } from "../../../profiling/index.js";
-export async function embed(params) {
-    const { modelId, text } = embedParamsSchema.parse(params);
-    const model = getModel(modelId);
-    const modelStart = nowMs();
-    const response = (await model.run(text));
-    const rawEmbeddings = await response.await();
-    const modelExecutionMs = nowMs() - modelStart;
-    const stats = {
-        ...(response.stats?.total_time_ms !== undefined && { totalTime: response.stats.total_time_ms }),
-        ...(response.stats?.tokens_per_second !== undefined && { tokensPerSecond: response.stats.tokens_per_second }),
-        ...(response.stats?.total_tokens !== undefined && { totalTokens: response.stats.total_tokens }),
-        ...(response.stats?.backendDevice !== undefined && { backendDevice: response.stats.backendDevice }),
-    };
-    const embeddingsArray = rawEmbeddings[0];
-    if (Array.isArray(text)) {
-        if (!embeddingsArray || embeddingsArray.length === 0) {
-            throw new EmbedNoEmbeddingsError();
+import { getRequestRegistry, withRequestContext, } from "../runtime/index.js";
+import { generateServerRequestId } from "../runtime/request-id.js";
+import { getServerLogger } from "../../../logging/index.js";
+export async function embed(params, requestId) {
+    const env_1 = { stack: [], error: void 0, hasError: false };
+    try {
+        const { modelId, text } = embedParamsSchema.parse(params);
+        // Open a request-scoped lifecycle. The registry routes
+        // `cancel({ requestId })` and broad `cancel({ modelId, kind: "embeddings" })`
+        // straight to this context's signal. Falls back to a server-generated
+        // id if the client didn't send one (older releases).
+        const ctx = __addDisposableResource(env_1, getRequestRegistry().begin({
+            requestId: requestId ?? generateServerRequestId(),
+            kind: "embeddings",
+            modelId,
+        }), true);
+        // `requestLogger` is intentionally referenced once so the
+        // request-scoped logger is built at handler entry per the canonical
+        // shape, even when this op has no per-step emits beyond the
+        // registry's own lifecycle lines. Future addon-level warns inside
+        // this body should route through `requestLogger`.
+        const requestLogger = withRequestContext(getServerLogger(), ctx);
+        const model = getModel(modelId);
+        // Hard-cancel wiring: llamacpp-embedding declares
+        // `cancel: { scope: "model", hard: true }`. The signal listener
+        // forwards an abort to the addon so C++ work stops as soon as the
+        // user cancels — mirrors the canonical handler shape documented in
+        // `request-lifecycle-primitives.mdc`. The post-await
+        // `signal.aborted` checks below are the soft-cancel safety net for
+        // the case where the abort fires between `model.run(...)` returning
+        // and `response.await()` resolving (both can take seconds on large
+        // batches).
+        const onAbort = () => {
+            const addon = model.addon;
+            if (addon?.cancel) {
+                addon.cancel.call(addon).catch((err) => {
+                    requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
+                });
+            }
+        };
+        ctx.signal.addEventListener("abort", onAbort, { once: true });
+        // `{ once: true }` does not fire if the signal is already aborted at
+        // register time. The registry may abort synchronously when a
+        // cancel-before-begin race resolves or the parent signal was pre-
+        // aborted — re-use `onAbort` so the listener body is the single
+        // source of truth for "what cancel does."
+        if (ctx.signal.aborted)
+            onAbort();
+        ctx.scope.defer(() => {
+            ctx.signal.removeEventListener("abort", onAbort);
+        });
+        const modelStart = nowMs();
+        const response = (await model.run(text));
+        if (ctx.signal.aborted) {
+            throw new InferenceCancelledError(ctx.requestId);
+        }
+        const rawEmbeddings = await response.await();
+        if (ctx.signal.aborted) {
+            throw new InferenceCancelledError(ctx.requestId);
+        }
+        const modelExecutionMs = nowMs() - modelStart;
+        const stats = {
+            ...(response.stats?.total_time_ms !== undefined && { totalTime: response.stats.total_time_ms }),
+            ...(response.stats?.tokens_per_second !== undefined && { tokensPerSecond: response.stats.tokens_per_second }),
+            ...(response.stats?.total_tokens !== undefined && { totalTokens: response.stats.total_tokens }),
+            ...(response.stats?.backendDevice !== undefined && { backendDevice: response.stats.backendDevice }),
+        };
+        const embeddingsArray = rawEmbeddings[0];
+        if (Array.isArray(text)) {
+            if (!embeddingsArray || embeddingsArray.length === 0) {
+                throw new EmbedNoEmbeddingsError();
+            }
+            const embedding = embeddingsArray.map((embeddingVector) => {
+                if (!embeddingVector || embeddingVector.length === 0) {
+                    throw new EmbedNoEmbeddingsError();
+                }
+                return normalizeVector(embeddingVector);
+            });
+            return buildUnaryResult({ embedding }, modelExecutionMs, stats);
         }
-        const embedding = embeddingsArray.map((embeddingVector) => {
+        else {
+            const embeddingVector = embeddingsArray?.[0];
             if (!embeddingVector || embeddingVector.length === 0) {
                 throw new EmbedNoEmbeddingsError();
             }
-            return normalizeVector(embeddingVector);
-        });
-        return buildUnaryResult({ embedding }, modelExecutionMs, stats);
-    }
-    else {
-        const embeddingVector = embeddingsArray?.[0];
-        if (!embeddingVector || embeddingVector.length === 0) {
-            throw new EmbedNoEmbeddingsError();
+            return buildUnaryResult({ embedding: normalizeVector(embeddingVector) }, modelExecutionMs, stats);
         }
-        return buildUnaryResult({ embedding: normalizeVector(embeddingVector) }, modelExecutionMs, stats);
+    }
+    catch (e_1) {
+        env_1.error = e_1;
+        env_1.hasError = true;
+    }
+    finally {
+        const result_1 = __disposeResources(env_1);
+        if (result_1)
+            await result_1;
     }
 }
 export function normalizeVector(vector) {

package/dist/server/bare/ops/kv-cache-utils.d.ts

@@ -1,10 +1,4 @@
 import { type CacheMessage } from "../../utils/index";
-export declare function markCacheInitialized(modelId: string, configHash: string, cacheKey: string): void;
-export declare function isCacheInitialized(modelId: string, configHash: string, cacheKey: string): boolean;
-export declare function clearCacheRegistry(scope?: {
-    cacheKey?: string | undefined;
-    modelId?: string | undefined;
-}): void;
 export declare function extractSystemPrompt(messages: CacheMessage[]): string | null;
 export declare function generateConfigHash(systemPrompt: string | null, tools?: unknown): string;
 export declare function generateCacheKey(messages: CacheMessage[]): string;
@@ -18,7 +12,6 @@ export declare function getCurrentCacheInfo(modelId: string, configHash: string,
     cachePath: string;
 }>;
 export declare function renameCacheFile(oldPath: string, newPath: string): Promise<boolean>;
-export declare function customCacheExists(modelId: string, configHash: string, cacheKey: string): Promise<boolean>;
 export declare function deleteCache(options: {
     all: true;
 } | {

package/dist/server/bare/ops/kv-cache-utils.js

@@ -4,37 +4,10 @@ import path from "bare-path";
 import { getAutoCacheLookupHistory, getKVCacheDir, validateAndJoinPath, } from "../../utils/index.js";
 import { getServerLogger } from "../../../logging/index.js";
 const logger = getServerLogger();
-// In-memory registry tracks caches initialized this session (addon defers disk writes)
-const initializedCaches = new Set();
-function getCacheRegistryKey(modelId, configHash, cacheKey) {
-    return `${modelId}:${configHash}:${cacheKey}`;
-}
-export function markCacheInitialized(modelId, configHash, cacheKey) {
-    initializedCaches.add(getCacheRegistryKey(modelId, configHash, cacheKey));
-}
-export function isCacheInitialized(modelId, configHash, cacheKey) {
-    return initializedCaches.has(getCacheRegistryKey(modelId, configHash, cacheKey));
-}
-export function clearCacheRegistry(scope) {
-    if (!scope || (scope.cacheKey === undefined && scope.modelId === undefined)) {
-        initializedCaches.clear();
-        return;
-    }
-    // key format: "modelId:configHash:cacheKey"
-    for (const key of initializedCaches) {
-        const firstSep = key.indexOf(":");
-        const secondSep = key.indexOf(":", firstSep + 1);
-        if (firstSep === -1 || secondSep === -1)
-            continue;
-        const modelId = key.slice(0, firstSep);
-        const cacheKey = key.slice(secondSep + 1);
-        if (scope.cacheKey !== undefined && cacheKey !== scope.cacheKey)
-            continue;
-        if (scope.modelId !== undefined && modelId !== scope.modelId)
-            continue;
-        initializedCaches.delete(key);
-    }
-}
+// In-memory KV-cache state lives in `KvCacheSession` (the single
+// mutation point for all three KV-cache bookkeeping layers). This
+// module keeps only the pure path / hash utilities that don't touch
+// in-memory state.
 export function extractSystemPrompt(messages) {
     const systemMessage = messages.find((msg) => msg.role === "system");
     return systemMessage ? systemMessage.content : null;
@@ -107,22 +80,10 @@ export async function renameCacheFile(oldPath, newPath) {
         return false;
     }
 }
-export async function customCacheExists(modelId, configHash, cacheKey) {
-    // Check in-memory registry first (addon defers disk writes)
-    if (isCacheInitialized(modelId, configHash, cacheKey)) {
-        return true;
-    }
-    // Then check file system (for caches from previous runs)
-    const cachePath = await getCacheFilePath(modelId, configHash, cacheKey);
-    try {
-        await fsPromises.access(cachePath);
-        markCacheInitialized(modelId, configHash, cacheKey);
-        return true;
-    }
-    catch {
-        return false;
-    }
-}
+// Cache-existence probing (in-memory registry first, fall back to
+// `fs.access`) lives in `KvCacheSession.beginTurn(...)`. Keeping the
+// in-memory `initializedCaches` set private to the session module
+// avoids drift between the two layers.
 export async function deleteCache(options) {
     const cacheDir = getKVCacheDir();
     if ("all" in options) {

package/dist/server/bare/ops/transcribe.d.ts

@@ -1,4 +1,4 @@
-import { type TranscribeParams, type TranscribeSegment, type TranscribeStats } from "../../../schemas/index";
+import { type TranscribeParams, type TranscribeSegment, type TranscribeStats, type TranscribeStreamEvent } from "../../../schemas/index";
 import { assertMetadataSupported, toTranscribeSegment, type WhisperAddonSegment } from "../utils/transcribe-metadata";
 export { assertMetadataSupported, toTranscribeSegment, type WhisperAddonSegment, };
 type TranscribeReturn = {
@@ -7,8 +7,13 @@ type TranscribeReturn = {
 };
 export declare function transcribe(params: TranscribeParams & {
     metadata: true;
-}): AsyncGenerator<TranscribeSegment, TranscribeReturn, void>;
-export declare function transcribe(params: TranscribeParams): AsyncGenerator<string, TranscribeReturn, void>;
-export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt: string | undefined, metadata: true): AsyncGenerator<TranscribeSegment, void, void>;
-export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt?: string, metadata?: boolean): AsyncGenerator<string, void, void>;
+}, requestId?: string): AsyncGenerator<TranscribeSegment, TranscribeReturn, void>;
+export declare function transcribe(params: TranscribeParams, requestId?: string): AsyncGenerator<string, TranscribeReturn, void>;
+export interface TranscribeStreamOpts {
+    emitVadEvents?: boolean;
+    endOfTurnSilenceMs?: number;
+    vadRunIntervalMs?: number;
+}
+export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt: string | undefined, metadata: true, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<TranscribeSegment | TranscribeStreamEvent, void, void>;
+export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt?: string, metadata?: boolean, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<string | TranscribeStreamEvent, void, void>;
 //# sourceMappingURL=transcribe.d.ts.map