@qvac/sdk 0.12.2 → 0.13.0

package/dist/server/bare/runtime/request-registry.js

@@ -24,10 +24,30 @@ import { getServerLogger } from "../../../logging/index.js";
  */
 const CANCEL_BEFORE_BEGIN_MAX_ENTRIES = 128;
 const CANCEL_BEFORE_BEGIN_TTL_MS = 30_000;
+/**
+ * Default cap on waiters queued per `(kind, modelId)` when a policy uses
+ * `onOverflow: "queue"` without an explicit `maxQueueDepthPerModel`.
+ * Bounds memory so a misbehaving client firing a flood of same-model
+ * requests can't grow the queue without limit — the (depth + 1)th begin
+ * rejects with `RequestRejectedByPolicyError` instead of enqueuing.
+ */
+const DEFAULT_MAX_QUEUE_DEPTH_PER_MODEL = 64;
 export function createRequestRegistry(options) {
     const entries = new Map();
     const policies = new Map();
     const logger = options?.logger ?? getServerLogger();
+    /**
+     * Per-`(kind, modelId)` admission semaphores. Keyed by
+     * `slotKey(kind, modelId)`. Absent until the first gated `begin(...)`
+     * for that key; removed again the moment the key is idle.
+     */
+    const keyStates = new Map();
+    /**
+     * Reverse index from a queued waiter's `requestId` to its slot so
+     * `cancel({ requestId })` can pull a still-queued request out of the
+     * FIFO in O(1) without scanning every `KeyState`.
+     */
+    const waitersById = new Map();
     function logLifecycle(event, ctx, durationMs) {
         const modelId = ctx.modelId !== undefined ? ctx.modelId : "-";
         const base = `[request-lifecycle] ${event} requestId=${ctx.requestId} kind=${ctx.kind} modelId=${modelId} state=${ctx.state}`;
@@ -99,6 +119,124 @@ export function createRequestRegistry(options) {
         cancelledBeforeBegin.delete(requestId);
         return entry;
     }
+    function slotKey(kind, modelId) {
+        // NUL separator can't appear in a kind (closed union) or a modelId
+        // (sdkModelId is a generated handle), so the join is unambiguous.
+        return `${kind}\u0000${modelId}`;
+    }
+    /**
+     * Pull a waiter out of its queue and tidy up: clear its timeout, drop
+     * the reverse index, and delete the `KeyState` if that left it idle.
+     * Does NOT settle the waiter's promise — the caller decides whether the
+     * removal resolves (granted / graceful cancel) or rejects (timeout /
+     * teardown).
+     */
+    function removeWaiter(key, waiter) {
+        if (waiter.timer !== undefined) {
+            clearTimeout(waiter.timer);
+            waiter.timer = undefined;
+        }
+        waitersById.delete(waiter.requestId);
+        const st = keyStates.get(key);
+        if (!st)
+            return;
+        const i = st.waiters.indexOf(waiter);
+        if (i >= 0)
+            st.waiters.splice(i, 1);
+        if (st.active <= 0 && st.waiters.length === 0)
+            keyStates.delete(key);
+    }
+    /**
+     * Admission gate. Resolves to the slot key the caller must release on
+     * dispose, or `undefined` when the request is ungated (no policy, no
+     * `modelId`, an unbounded limit, or an already-aborted parent — which we
+     * let through so the normal abort path produces a clean cancelled
+     * outcome rather than queuing a doomed request).
+     *
+     * When the key is at capacity it either rejects (`onOverflow: "reject"`,
+     * queue-depth cap, or `queueTimeoutMs`) or enqueues a FIFO waiter and
+     * awaits it. The releaser hands a freed slot to the oldest waiter
+     * (`resolve(true)`), so `active` is never decremented and re-incremented
+     * across a hand-off — that's what guarantees FIFO fairness with no
+     * acquire/release race in the single-threaded event loop.
+     */
+    async function acquireSlot(opts) {
+        if (opts.modelId === undefined)
+            return { slotKey: undefined };
+        const policy = policies.get(opts.kind);
+        if (!policy || !Number.isFinite(policy.maxConcurrent)) {
+            return { slotKey: undefined };
+        }
+        // A parent (worker-shutdown) signal that's already aborted: don't
+        // queue behind live work that may never drain — let begin() proceed
+        // and abort immediately via the parentSignal path.
+        if (opts.parentSignal?.aborted)
+            return { slotKey: undefined };
+        const modelId = opts.modelId;
+        const key = slotKey(opts.kind, modelId);
+        let st = keyStates.get(key);
+        if (!st) {
+            st = { active: 0, waiters: [] };
+            keyStates.set(key, st);
+        }
+        if (st.active < policy.maxConcurrent) {
+            st.active++;
+            return { slotKey: key };
+        }
+        if (policy.onOverflow === "reject") {
+            throw new RequestRejectedByPolicyError(opts.requestId, opts.kind, modelId, `another ${opts.kind} request is already running on this model`);
+        }
+        if (st.waiters.length >= policy.maxQueueDepth) {
+            throw new RequestRejectedByPolicyError(opts.requestId, opts.kind, modelId, `request queue for this model is full (${st.waiters.length} waiting)`);
+        }
+        const queue = st;
+        const granted = await new Promise((resolve, reject) => {
+            const waiter = {
+                requestId: opts.requestId,
+                kind: opts.kind,
+                modelId,
+                enqueuedAt: Date.now(),
+                resolve,
+                reject,
+                timer: undefined,
+            };
+            if (policy.queueTimeoutMs !== undefined) {
+                const timeoutMs = policy.queueTimeoutMs;
+                waiter.timer = setTimeout(() => {
+                    removeWaiter(key, waiter);
+                    reject(new RequestRejectedByPolicyError(opts.requestId, opts.kind, modelId, `timed out waiting ${timeoutMs}ms for a slot on this model`));
+                }, timeoutMs);
+            }
+            queue.waiters.push(waiter);
+            waitersById.set(opts.requestId, { key, waiter });
+        });
+        return granted ? { slotKey: key } : { slotKey: undefined };
+    }
+    /**
+     * Hand a freed slot to the next FIFO waiter, or free it outright when
+     * none are queued. Called from `disposeEntry` after the scope has fully
+     * unwound, so the native context is genuinely free before the next
+     * same-model request is admitted.
+     */
+    function releaseSlot(key) {
+        const st = keyStates.get(key);
+        if (!st)
+            return;
+        const next = st.waiters.shift();
+        if (next) {
+            // Hand off: the waiter inherits the slot, so `active` is unchanged.
+            waitersById.delete(next.requestId);
+            if (next.timer !== undefined) {
+                clearTimeout(next.timer);
+                next.timer = undefined;
+            }
+            next.resolve(true);
+            return;
+        }
+        st.active--;
+        if (st.active <= 0 && st.waiters.length === 0)
+            keyStates.delete(key);
+    }
     function cancelEntry(entry, reason) {
         if (entry.controller.signal.aborted)
             return false;
@@ -116,23 +254,43 @@ export function createRequestRegistry(options) {
         // Pull the entry out before unwinding so observers (e.g. a `cancel(...)`
         // racing with dispose) don't see a half-disposed context.
         entries.delete(entry.ctx.requestId);
-        await entry.scope[Symbol.asyncDispose]();
+        try {
+            await entry.scope[Symbol.asyncDispose]();
+        }
+        finally {
+            // Release the admission slot only after the scope has fully unwound:
+            // the shared llama.cpp context / addon job is torn down in scope
+            // cleanup, so the next queued same-model request must not be admitted
+            // (and start decoding) until this one has actually let go. `finally`
+            // guarantees the slot is freed even if a deferred cleanup throws,
+            // otherwise a throwing teardown would strand the queue forever.
+            if (entry.slotKey !== undefined)
+                releaseSlot(entry.slotKey);
+        }
     }
-    function begin(opts) {
-        if (entries.has(opts.requestId)) {
+    async function begin(opts) {
+        // A request id is reserved for its whole lifecycle, including the time
+        // it spends *queued* for an admission slot. `waitersById` holds the
+        // still-queued begins (they aren't in `entries` yet), so a duplicate id
+        // must be rejected against both maps — otherwise a second begin with the
+        // same id would enqueue behind the first and overwrite its `waitersById`
+        // index, leaving the original waiter unreachable by `cancel({ requestId })`.
+        if (entries.has(opts.requestId) || waitersById.has(opts.requestId)) {
             throw new RequestIdConflictError(opts.requestId);
         }
-        // Admission control runs before allocation so a rejected begin
-        // leaves no controller / scope behind.
-        const policy = policies.get(opts.kind);
-        if (policy && policy.oneAtATimePerModel && opts.modelId !== undefined) {
-            for (const existing of entries.values()) {
-                if (existing.ctx.kind !== opts.kind)
-                    continue;
-                if (existing.ctx.modelId !== opts.modelId)
-                    continue;
-                throw new RequestRejectedByPolicyError(opts.requestId, opts.kind, opts.modelId, `another ${opts.kind} request (${existing.ctx.requestId}) is already running on this model`);
-            }
+        // Admission control runs before allocation so a rejected begin leaves
+        // no controller / scope behind. This may await: a gated `(kind,
+        // modelId)` at capacity queues the begin FIFO and resolves once a slot
+        // frees (or rejects on overflow / queue-depth cap / timeout).
+        const { slotKey } = await acquireSlot(opts);
+        // The only interleaving point in this otherwise synchronous body is the
+        // await above. A duplicate id could (astronomically unlikely) have
+        // landed while we were queued — re-check and hand the just-acquired
+        // slot back rather than stranding it on the conflicting throw.
+        if (entries.has(opts.requestId)) {
+            if (slotKey !== undefined)
+                releaseSlot(slotKey);
+            throw new RequestIdConflictError(opts.requestId);
         }
         const controller = new AbortController();
         const scope = createDisposableScope();
@@ -179,6 +337,7 @@ export function createRequestRegistry(options) {
             scope,
             detachParent,
             startedAt: Date.now(),
+            slotKey,
         };
         entries.set(opts.requestId, entry);
         logLifecycle("begin", ctx);
@@ -215,6 +374,18 @@ export function createRequestRegistry(options) {
     function list() {
         return Array.from(entries.values(), (e) => e.ctx);
     }
+    /**
+     * Cancel a still-queued waiter the same way the Stop-button race is
+     * handled: record a cancel-before-begin so its `begin(...)` — which
+     * resumes the instant we `resolve(false)` (without a slot) — observes the
+     * cancel and returns an already-aborted context. The request thus ends in
+     * a clean `cancelled` state rather than throwing.
+     */
+    function cancelQueuedWaiterGracefully(key, waiter, reason) {
+        recordCancelBeforeBegin(waiter.requestId, reason);
+        removeWaiter(key, waiter);
+        waiter.resolve(false);
+    }
     function cancel(target) {
         let cancelled = 0;
         if ("requestId" in target) {
@@ -224,6 +395,17 @@ export function createRequestRegistry(options) {
                     cancelled++;
                 return cancelled;
             }
+            // Queued (still waiting for an admission slot)? It has no controller
+            // yet, so cancel it through the cancel-before-begin tripwire: pull it
+            // from the FIFO and let its begin() resume into an aborted context.
+            // Counted as one — unlike the pure race below, this targets a known
+            // in-flight (queued) begin, so the Stop button feels responsive
+            // instead of waiting for the slot to free first.
+            const queued = waitersById.get(target.requestId);
+            if (queued) {
+                cancelQueuedWaiterGracefully(queued.key, queued.waiter, target.reason);
+                return cancelled + 1;
+            }
             // Stop-button race: the client beat its own
             // `begin(...)`. Record the cancel so the next matching `begin`
             // aborts immediately. The return value stays 0 — no in-flight
@@ -240,12 +422,35 @@ export function createRequestRegistry(options) {
             if (cancelEntry(entry, target.reason))
                 cancelled++;
         }
+        // Queued waiters aren't in `entries`; drain the ones matching this
+        // model (and optional kind) so a broad `cancel({ modelId })` doesn't
+        // strand them behind a request that's being cancelled out from under
+        // them. Snapshot first — `removeWaiter` mutates `waitersById`.
+        for (const { key, waiter } of Array.from(waitersById.values())) {
+            if (waiter.modelId !== target.modelId)
+                continue;
+            if (target.kind && waiter.kind !== target.kind)
+                continue;
+            cancelQueuedWaiterGracefully(key, waiter, target.reason);
+            cancelled++;
+        }
         return cancelled;
     }
     function cancelAll(reason) {
         for (const entry of entries.values()) {
             cancelEntry(entry, reason);
         }
+        // Drain every queued waiter so a shutdown / model-unload sweep can't
+        // leave a `begin(...)` promise hung forever — which would in turn block
+        // the unload waiting on a request that never resolves. Unlike the
+        // targeted cancels above these *reject* rather than resolve-into-
+        // aborted: the model / worker they queued against is being torn down,
+        // so there is nothing left for them to run. Snapshot first —
+        // `removeWaiter` mutates `waitersById`.
+        for (const { key, waiter } of Array.from(waitersById.values())) {
+            removeWaiter(key, waiter);
+            waiter.reject(new RequestRejectedByPolicyError(waiter.requestId, waiter.kind, waiter.modelId, `queued request cancelled (${reason})`));
+        }
         // The interface returns Promise<void> so we can later make this an
         // async sweep that awaits per-handler scope unwinding (e.g. join on
         // the disposers). Today every handler unwinds on its own dispose
@@ -259,7 +464,7 @@ export function createRequestRegistry(options) {
         await disposeEntry(entry, outcome);
     }
     function policy(opts) {
-        policies.set(opts.kind, opts);
+        policies.set(opts.kind, normalizePolicy(opts));
     }
     return {
         begin,
@@ -275,6 +480,43 @@ export function createRequestRegistry(options) {
         // interface (typed via the augmented return type below) so handler
         // code can't depend on it accidentally.
         __cancelBeforeBeginSize: () => cancelledBeforeBegin.size,
+        // Test-only: the number of live per-`(kind, modelId)` admission
+        // states. Lets the queue tests assert the map empties on drain /
+        // dispose (no `KeyState` leak). Also off the public interface.
+        __keyStateSize: () => keyStates.size,
+    };
+}
+/**
+ * Resolve a `ConcurrencyPolicy` to concrete numbers, applying defaults and
+ * the deprecated `oneAtATimePerModel` alias. Done once at registration so
+ * the `begin(...)` hot path reads plain numbers.
+ */
+function normalizePolicy(opts) {
+    let maxConcurrent;
+    let onOverflow;
+    if (opts.maxConcurrentPerModel !== undefined) {
+        maxConcurrent = opts.maxConcurrentPerModel;
+        onOverflow = opts.onOverflow ?? "queue";
+    }
+    else if (opts.oneAtATimePerModel === true) {
+        // Legacy alias: at most one in-flight per model, reject the rest.
+        maxConcurrent = 1;
+        onOverflow = opts.onOverflow ?? "reject";
+    }
+    else {
+        // Nothing configured (includes `oneAtATimePerModel: false`) ⇒ no gating.
+        maxConcurrent = Infinity;
+        onOverflow = opts.onOverflow ?? "queue";
+    }
+    // A finite limit below 1 would gate every request forever; clamp to the
+    // smallest sensible serial limit.
+    if (Number.isFinite(maxConcurrent) && maxConcurrent < 1)
+        maxConcurrent = 1;
+    return {
+        maxConcurrent,
+        onOverflow,
+        maxQueueDepth: opts.maxQueueDepthPerModel ?? DEFAULT_MAX_QUEUE_DEPTH_PER_MODEL,
+        queueTimeoutMs: opts.queueTimeoutMs,
     };
 }
 /**
@@ -287,6 +529,7 @@ export function createRequestRegistry(options) {
 export const __requestRegistryTestHooks = {
     cancelBeforeBeginMaxEntries: CANCEL_BEFORE_BEGIN_MAX_ENTRIES,
     cancelBeforeBeginTtlMs: CANCEL_BEFORE_BEGIN_TTL_MS,
+    defaultMaxQueueDepthPerModel: DEFAULT_MAX_QUEUE_DEPTH_PER_MODEL,
 };
 function derivedTerminalState(ctx) {
     if (ctx.state === "failed")

package/dist/server/bare/runtime/with-request-context.d.ts

@@ -13,7 +13,7 @@ export interface RequestLogContext {
  * the prefixed message.
  *
  * @example
- *   await using ctx = registry.begin({ requestId, kind: "completion", modelId });
+ *   await using ctx = await registry.begin({ requestId, kind: "completion", modelId });
  *   const log = withRequestContext(getServerLogger(), ctx);
  *   log.info("decoding token 7");
  *   // → "[request-lifecycle completion requestId=<id> modelId=<id>] decoding token 7"

package/dist/server/bare/runtime/with-request-context.js

@@ -7,7 +7,7 @@
  * the prefixed message.
  *
  * @example
- *   await using ctx = registry.begin({ requestId, kind: "completion", modelId });
+ *   await using ctx = await registry.begin({ requestId, kind: "completion", modelId });
  *   const log = withRequestContext(getServerLogger(), ctx);
  *   log.info("decoding token 7");
  *   // → "[request-lifecycle completion requestId=<id> modelId=<id>] decoding token 7"

package/dist/server/bare/types/addon-responses.d.ts

@@ -37,6 +37,7 @@ export interface EmbedStats {
     tokens_per_second?: number;
     total_tokens?: number;
     backendDevice?: "cpu" | "gpu";
+    context_size?: number;
 }
 export interface EmbedResponse {
     stats?: EmbedStats;
@@ -53,6 +54,10 @@ export interface TranscribeStats {
     encoderMs?: number;
     decoderMs?: number;
     melSpecMs?: number;
+    backendDevice?: number;
+    backendId?: number;
+    gpuMemTotalMb?: number;
+    gpuMemFreeMb?: number;
 }
 export interface TranscribeAddonSegment {
     text: string;

package/dist/server/rpc/handler-registry.js

@@ -9,6 +9,8 @@ import { handleUnloadModel } from "./handlers/unload-model.js";
 import { handleUnloadModelDelegated } from "./handlers/unload-model-delegated.js";
 import { handleTranscribe } from "./handlers/transcribe.js";
 import { handleTranscribeStream } from "./handlers/transcribe-stream.js";
+import { handleBciTranscribe } from "./handlers/bci-transcribe.js";
+import { handleBciTranscribeStream } from "./handlers/bci-transcribe-stream.js";
 import { handleEmbed } from "./handlers/embed.js";
 import { handleTranslate } from "./handlers/translate.js";
 import { handleLoggingStream } from "./handlers/logging-stream.js";
@@ -118,6 +120,8 @@ export const registry = {
     // Simple Stream handlers
     transcribe: { type: "stream", handler: handleTranscribe },
     transcribeStream: { type: "duplex", handler: handleTranscribeStream },
+    bciTranscribe: { type: "stream", handler: handleBciTranscribe },
+    bciTranscribeStream: { type: "duplex", handler: handleBciTranscribeStream },
     loggingStream: { type: "stream", handler: handleLoggingStream },
     translate: { type: "stream", handler: handleTranslate },
     textToSpeech: { type: "stream", handler: handleTextToSpeech },

package/dist/server/rpc/handlers/bci-transcribe-stream.d.ts

@@ -0,0 +1,3 @@
+import type { BciTranscribeStreamRequest, BciTranscribeStreamResponse } from "../../../schemas/index";
+export declare function handleBciTranscribeStream(request: BciTranscribeStreamRequest, inputStream: AsyncIterable<Buffer>): AsyncGenerator<BciTranscribeStreamResponse>;
+//# sourceMappingURL=bci-transcribe-stream.d.ts.map

package/dist/server/rpc/handlers/bci-transcribe-stream.js

@@ -0,0 +1,4 @@
+import { dispatchPluginStream } from "./plugin-dispatch.js";
+export async function* handleBciTranscribeStream(request, inputStream) {
+    yield* dispatchPluginStream(request.modelId, "bciTranscribeStream", request, inputStream);
+}

package/dist/server/rpc/handlers/bci-transcribe.d.ts

@@ -0,0 +1,3 @@
+import type { BciTranscribeRequest, BciTranscribeResponse } from "../../../schemas/index";
+export declare function handleBciTranscribe(request: BciTranscribeRequest): AsyncGenerator<BciTranscribeResponse>;
+//# sourceMappingURL=bci-transcribe.d.ts.map

package/dist/server/rpc/handlers/bci-transcribe.js

@@ -0,0 +1,4 @@
+import { dispatchPluginStream } from "./plugin-dispatch.js";
+export async function* handleBciTranscribe(request) {
+    yield* dispatchPluginStream(request.modelId, "bciTranscribe", request);
+}

package/dist/server/rpc/handlers/download-asset.js

@@ -70,7 +70,7 @@ export async function handleDownloadAsset(request, progressCallback) {
         // `modelId` to register on the registry entry. Cancel by `requestId`
         // is the primary path; `cancel({ modelId })` is intentionally a
         // non-match for this kind.
-        const ctx = __addDisposableResource(env_1, getRequestRegistry().begin({
+        const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
             requestId,
             kind: "downloadAsset",
         }), true);

package/dist/server/rpc/handlers/get-model-info.js

@@ -15,7 +15,7 @@ export async function handleGetModelInfo(request) {
     const cacheStatus = catalogEntry.shardMetadata && catalogEntry.shardMetadata.length > 0
         ? await handleShardedModel(cacheKey, catalogEntry.shardMetadata)
         : catalogEntry.companionSet
-            ? await handleCompanionSetModel(catalogEntry.companionSet, catalogEntry.registryPath)
+            ? await handleCompanionSetModel(catalogEntry.companionSet, catalogEntry.registryPath, catalogEntry.addon)
             : await handleSingleFileModel(catalogEntry.registryPath, catalogEntry.expectedSize, catalogEntry.sha256Checksum);
     const { cacheFiles, isCached, actualSize, cachedAt, primaryPath } = cacheStatus;
     const loadedModelIds = getAllModelIds();
@@ -73,7 +73,7 @@ async function handleShardedModel(cacheKey, shardMetadata) {
     const paths = shardMetadata.map((s) => getShardPath(cacheKey, s.filename));
     return checkCacheStatus(fileEntries, paths);
 }
-async function handleCompanionSetModel(companionSet, primaryRegistryPath) {
+async function handleCompanionSetModel(companionSet, primaryRegistryPath, addon) {
     const { setKey, primaryKey, files } = companionSet;
     const baseCache = getModelsCacheDir();
     const fileEntries = files.map((f) => ({
@@ -94,7 +94,7 @@ async function handleCompanionSetModel(companionSet, primaryRegistryPath) {
         if (legacyResult.isCached)
             return legacyResult;
     }
-    else {
+    else if (addon === "nmt") {
         // Legacy flat-cache compatibility for Bergamot-style companion sets.
         // This is valid only for families whose runtime still works with explicit
         // per-file absolute paths; it is not a generic rule for all companion sets.

package/dist/server/rpc/handlers/index.d.ts

@@ -5,6 +5,8 @@ import { handleUnloadModel } from "./unload-model";
 import { handleEmbed } from "./embed";
 import { handleTranscribe } from "./transcribe";
 import { handleTranscribeStream } from "./transcribe-stream";
+import { handleBciTranscribe } from "./bci-transcribe";
+import { handleBciTranscribeStream } from "./bci-transcribe-stream";
 import { provideHandler } from "./provideHandler";
 import { stopProvideHandler } from "./stopProvideHandler";
 import { handleTranslate } from "./translate";
@@ -37,6 +39,8 @@ export declare const handlers: {
     embed: typeof handleEmbed;
     transcribe: typeof handleTranscribe;
     transcribeStream: typeof handleTranscribeStream;
+    bciTranscribe: typeof handleBciTranscribe;
+    bciTranscribeStream: typeof handleBciTranscribeStream;
     provide: typeof provideHandler;
     stopProvide: typeof stopProvideHandler;
     translate: typeof handleTranslate;

package/dist/server/rpc/handlers/index.js

@@ -5,6 +5,8 @@ import { handleUnloadModel } from "./unload-model.js";
 import { handleEmbed } from "./embed.js";
 import { handleTranscribe } from "./transcribe.js";
 import { handleTranscribeStream } from "./transcribe-stream.js";
+import { handleBciTranscribe } from "./bci-transcribe.js";
+import { handleBciTranscribeStream } from "./bci-transcribe-stream.js";
 import { provideHandler } from "./provideHandler.js";
 import { stopProvideHandler } from "./stopProvideHandler.js";
 import { handleTranslate } from "./translate.js";
@@ -37,6 +39,8 @@ export const handlers = {
     embed: handleEmbed,
     transcribe: handleTranscribe,
     transcribeStream: handleTranscribeStream,
+    bciTranscribe: handleBciTranscribe,
+    bciTranscribeStream: handleBciTranscribeStream,
     provide: provideHandler,
     stopProvide: stopProvideHandler,
     translate: handleTranslate,

package/dist/server/rpc/handlers/load-model/download-manager.d.ts

@@ -5,7 +5,7 @@ import type { DownloadHooks } from "./types";
 /**
  * Per-subscriber binding to a registry-tracked request.
  * `startOrJoinDownload` is request-aware: each caller (the
- * `await using ctx = registry.begin(...)` inside `handleLoadModel` /
+ * `await using ctx = await registry.begin(...)` inside `handleLoadModel` /
  * `handleDownloadAsset`) registers a subscriber bound to its
  * `requestId`. A `cancel({ requestId })` against the registry aborts
  * the subscriber's `ctx.signal`, which:

package/dist/server/rpc/handlers/load-model/handler.js

@@ -82,7 +82,7 @@ export async function handleLoadModel(request, progressCallback) {
         // is opened with `modelId: undefined` so a cancel-by-modelId fired
         // before the load completes is a clean no-op (rather than matching a
         // half-built entry). Cancel-by-`requestId` works from `begin(...)` on.
-        const ctx = __addDisposableResource(env_1, getRequestRegistry().begin({
+        const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
             requestId,
             kind: "loadModel",
         }), true);

package/dist/server/rpc/handlers/load-model/http.js

@@ -1,7 +1,7 @@
 import fs, { promises as fsPromises } from "bare-fs";
 import path from "bare-path";
 import { Readable } from "bare-stream";
-import fetch from "bare-fetch";
+import fetch, { Headers } from "bare-fetch";
 import { withTimeout } from "../../../../utils/withTimeout.js";
 import { getModelsCacheDir, getShardedModelCacheDir, generateShortHash, detectShardedModel, parsePatternBasedShardUrl, extractTensorsFromShards, calculatePercentage, isArchiveUrl, sanitizePathComponent, extractAndValidateShardedArchive, validateShardedModelCache, checkAllShardsExist, generateShardFilenames, hasValidGGUFHeader, } from "../../../utils/index.js";
 import { getSDKConfig } from "../../../bare/registry/config-registry.js";
@@ -84,11 +84,11 @@ async function performHttpDownload(url, modelPath, downloadKey, progressCallback
         logger.info(`📥 Starting fresh download`);
     }
     // Prepare headers for resume if needed
-    const headers = {
+    const headers = new Headers({
         "User-Agent": "qvac-sdk",
-    };
+    });
     if (startOffset > 0) {
-        headers["Range"] = `bytes=${startOffset}-`;
+        headers.append("Range", `bytes=${startOffset}-`);
     }
     const config = getSDKConfig();
     const connectionTimeout = config.httpConnectionTimeoutMs ?? DEFAULT_HTTP_CONNECTION_TIMEOUT_MS;
@@ -134,9 +134,9 @@ async function performHttpDownload(url, modelPath, downloadKey, progressCallback
             // Retry without Range header
             response = await fetch(url, {
                 method: "GET",
-                headers: {
+                headers: new Headers({
                     "User-Agent": "qvac-sdk",
-                },
+                }),
                 ...(signal && { signal }),
             });
             if (!response.ok) {

package/dist/server/rpc/handlers/load-model/hyperdrive.js

@@ -12,6 +12,7 @@ import { getSDKConfig } from "../../../bare/registry/config-registry.js";
 import { FileNotFoundError, ChecksumValidationFailedError, DownloadCancelledError, HyperdriveDownloadFailedError, ModelLoadFailedError, NoBlobFoundError, } from "../../../../utils/errors-server.js";
 import { getServerLogger } from "../../../../logging/index.js";
 import { registerSwarm, unregisterSwarm, registerCorestore, unregisterCorestore, } from "../../../bare/runtime-lifecycle.js";
+import { Buffer } from "bare-buffer";
 const logger = getServerLogger();
 function getCorestoreDir(hyperdriveKey) {
     return getQvacPath("corestore", hyperdriveKey);

package/dist/server/rpc/handlers/load-model/registry-companion-set.d.ts

@@ -6,6 +6,7 @@ type CompanionSetMetadata = NonNullable<RegistryItem["companionSet"]>;
 export interface DownloadCompanionSetOptions {
     companionSet: CompanionSetMetadata;
     downloadKey: string;
+    allowLegacyFlatCache?: boolean | undefined;
     progressCallback?: ((progress: ModelProgressUpdate) => void) | undefined;
     signal?: AbortSignal | undefined;
     hooks?: DownloadHooks | undefined;

package/dist/server/rpc/handlers/load-model/registry-companion-set.js

@@ -13,7 +13,7 @@ const logger = getServerLogger();
  * Returns the path to the primary file.
  */
 export async function downloadCompanionSetFromRegistry(options) {
-    const { companionSet, downloadKey, progressCallback, signal, hooks, shouldClearCache, } = options;
+    const { companionSet, downloadKey, allowLegacyFlatCache = false, progressCallback, signal, hooks, shouldClearCache, } = options;
     const { files, setKey, primaryKey } = companionSet;
     validateCompanionSetMetadata(files, primaryKey);
     const primaryEntry = files.find((f) => f.key === primaryKey);
@@ -30,7 +30,7 @@ export async function downloadCompanionSetFromRegistry(options) {
     // Legacy flat-cache compatibility for Bergamot-style companion sets.
     // This is valid only for families whose runtime still works with explicit
     // per-file absolute paths; it is not a generic rule for all companion sets.
-    if (!isLegacyOnnxSet(files)) {
+    if (allowLegacyFlatCache && !isLegacyOnnxSet(files)) {
         const flatPath = await checkLegacyFlatCache(files, primaryKey, hooks);
         if (flatPath) {
             logger.info(`✅ Using legacy flat cache for companion set: ${flatPath}`);

package/dist/server/rpc/handlers/load-model/registry.js

@@ -251,6 +251,7 @@ export async function downloadModelFromRegistry(registryPath, registrySource, pr
             return await downloadCompanionSetFromRegistry({
                 companionSet: modelMetadata.companionSet,
                 downloadKey,
+                allowLegacyFlatCache: modelMetadata.addon === "nmt",
                 progressCallback: ctx.broadcastProgress,
                 signal: ctx.signal,
                 hooks: companionHooks,

package/dist/server/rpc/handlers/load-model/resolve-session.d.ts

@@ -7,7 +7,7 @@ export interface ResolveSessionOptions {
     profilingEnabled: boolean;
     /**
      * Optional cancel signal — typically `ctx.signal` from the surrounding
-     * `await using ctx = registry.begin(...)` block. When provided,
+     * `await using ctx = await registry.begin(...)` block. When provided,
      * `resolveModelPath` short-circuits with `InferenceCancelledError` if the
      * signal is already aborted on entry; the same signal also propagates
      * to in-progress transfers via the request binding below so cancel