@qvac/harness 0.4.0 → 0.6.0

package/dist-lib/index.js

@@ -21,11 +21,13 @@ import { buildWavBuffer } from './lib/wav-encode.js';
 import { createVoiceLane, packSamples, speakOnce, speakStreamOnce, transcribeOnce, transcribeStreaming, SUPERTONIC_SAMPLE_RATE } from './lib/capabilities/voice.js';
 import { pushQueue } from './lib/queue.js';
 import { AbortedError, raceAbort } from './lib/helpers/race-abort.js';
-import { audioGenDownloadMetadata, customProjectionMap, defaultProjectionMap, defaultTtsFor, defaultVoiceModel, diffusionDownloadMetadata, infraModelsFor, modelNameToCapabilityProfile, musicModelSet, setCustomModels } from './lib/models/model-catalog.js';
+import { audioGenDownloadMetadata, customProjectionMap, defaultProjectionMap, defaultTtsFor, defaultVoiceModel, diffusionDownloadMetadata, curatedModelAsset, infraModelsFor, modelAddonOf, modelNameToCapabilityProfile, musicModelSet, residentSrc, setCustomModels } from './lib/models/model-catalog.js';
 import { createModelStatusCache, customModelRows, invalidateModelStatusCache, mergeCustomRows, listModels } from './lib/models/model-list.js';
 import { deleteModelFiles, summarizeDelete } from './lib/models/model-delete.js';
-import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, awaitModelCapacity, evictIdle, evictModelWithin, isResident, leaseResidentModel, leaseCount, loadedModels, modelKey, releaseModel, resolveLoadOptions, progressState } from './lib/models/model-pool.js';
+import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, awaitModelCapacity, evictIdle, evictIdleWhere, evictModel, evictModelWithin, isResident, leaseResidentModel, leaseCount, loadedModels, modelKey, modelOf, releaseModel, resolveLoadOptions, progressState } from './lib/models/model-pool.js';
+import { defaultContextSize } from './lib/models/context.js';
 import { resolveMainGpu } from './lib/models/gpu-selection.js';
+import { createIdleReaper } from './lib/models/idle-unload.js';
 import { openclCacheDir } from './lib/models/opencl-cache.js';
 import { assertEmbedDevice, createEmbedConfig } from './lib/models/embed-config.js';
 import { createGpuFaultState, isSdkGpuDeviceLostError, isSdkModelWedgedError } from './lib/models/sdk-faults.js';
@@ -39,6 +41,7 @@ import { ensureBundledSkillsDir } from './lib/tools/skills/bundled-dir.js';
 import { ensureSkillsOverlayDir } from './lib/tools/skills/overlay-dir.js';
 import { copyBaseForEdit, deleteOverlaySkill, listManagedSkills, readSkillFile, writeSkillFile } from './lib/tools/skills/overlay-crud.js';
 import { watchSkillsOverlay } from './lib/tools/skills/overlay-watch.js';
+import { errorMessage } from './lib/guards.js';
 import { hlog } from './lib/log.js';
 import { harness } from './spec/index.js';
 import { installResolutionDetector, resolutionDetail } from './lib/resolution-detector.js';
@@ -79,6 +82,7 @@ const MUSIC_EVICT_WAIT_MS = 15_000;
 const WEDGED_EVICT_WAIT_MS = 10_000;
 const MODEL_ROOM_WAIT_MS = 300_000;
 const WARM_ROOM_WAIT_MS = 2_000;
+const UNLOAD_LEASE_WAIT_MS = 10_000;
 const DEFAULT_COMPLETION_MODEL_CAP = 1;
 const WEDGED_EVICT_LEASED_MS = 120_000;
 const TTS_MODEL_CONFIG = {
@@ -91,6 +95,8 @@ export class Harness extends ReadyResource {
     _tools;
     _sdk;
     _loads = createLoadTracker();
+    _pinned = new Set();
+    _replying = [];
     _modelStatus = createModelStatusCache();
     _models;
     _embedModels;
@@ -133,6 +139,7 @@ export class Harness extends ReadyResource {
     _capability = null;
     _capabilityInflight = null;
     _modelFaults = new Map();
+    _idle;
     constructor({ tools = [], sdkLoader, skillsDir, modelsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, completionModelCap, credentialStore, platform, gpuStats } = {}) {
         super();
         this._sdk = sdkLoader ?? createSdkLoader();
@@ -177,8 +184,9 @@ export class Harness extends ReadyResource {
                 },
                 onLoaded: (model) => {
                     const transfer = imageTransfers.get(model);
-                    if (!transfer)
+                    if (!transfer) {
                         return;
+                    }
                     imageTransfers.delete(model);
                     invalidateModelStatusCache(this._modelStatus);
                     this.emit('model', { ...transfer, status: 'downloaded' });
@@ -186,9 +194,14 @@ export class Harness extends ReadyResource {
                 onBeforeLoad: async (signal) => {
                     if (this._models.destroyed)
                         return;
-                    const evicted = await awaitModelCapacity(this._models, 1, DIFFUSION_EVICT_WAIT_MS, undefined, signal);
-                    if (signal?.aborted)
+                    const evicted = await awaitModelCapacity(this._models, {
+                        cap: 1,
+                        deadlineMs: DIFFUSION_EVICT_WAIT_MS,
+                        signal
+                    });
+                    if (signal?.aborted) {
                         throw new AbortedError();
+                    }
                     if (!evicted) {
                         throw new Error('the chat model is busy; try the image again once the current reply finishes');
                     }
@@ -205,9 +218,14 @@ export class Harness extends ReadyResource {
                 onBeforeLoad: async (signal) => {
                     if (this._models.destroyed)
                         return;
-                    const evicted = await awaitModelCapacity(this._models, 1, MUSIC_EVICT_WAIT_MS, undefined, signal);
-                    if (signal?.aborted)
+                    const evicted = await awaitModelCapacity(this._models, {
+                        cap: 1,
+                        deadlineMs: MUSIC_EVICT_WAIT_MS,
+                        signal
+                    });
+                    if (signal?.aborted) {
                         throw new AbortedError();
+                    }
                     if (!evicted) {
                         throw new Error('the chat model is busy; try the music again once the current reply finishes');
                     }
@@ -245,13 +263,18 @@ export class Harness extends ReadyResource {
         this._diffusionSdk = null;
         this._musicSdk = null;
         this._translationSdk = null;
-    }
-    async _open() {
-        installResolutionDetector(import.meta);
         this._models = this._createPool();
         this._embedModels = this._createEmbedPool();
         this._ttsModels = this._createTtsPool();
         this._transcriptionModels = this._createTranscriptionPool();
+        this._idle = createIdleReaper({
+            sweep: () => this._unloadIdleModels(),
+            waitMs: WEDGED_EVICT_WAIT_MS,
+            onError: (err) => hlog('idle unload failed', { err: errorMessage(err) })
+        });
+    }
+    async _open() {
+        installResolutionDetector(import.meta);
         await this._reloadSkills();
         await this._startOverlayWatch();
         hlog('open', { skills: this._supportedSkills.length, tools: this._tools.length });
@@ -269,10 +292,7 @@ export class Harness extends ReadyResource {
             if (!token)
                 continue;
             const storeKey = scopeKey({ agentId, skill, key });
-            const record = {
-                token,
-                ...(typeof expiresAt === 'number' && expiresAt > 0 ? { expiresAt } : {})
-            };
+            const record = typeof expiresAt === 'number' && expiresAt > 0 ? { token, expiresAt } : { token };
             const current = previous.get(storeKey);
             if (!current || current.token !== record.token || current.expiresAt !== record.expiresAt) {
                 resetCredentialHealthAfterSnapshot(key, agentId, skill);
@@ -310,48 +330,57 @@ export class Harness extends ReadyResource {
     }
     async writeSkillFile(name, content) {
         const overlay = await ensureSkillsOverlayDir();
-        if (!overlay)
+        if (!overlay) {
             throw new Error('no writable skills overlay is configured');
+        }
         await writeSkillFile(overlay, name, content);
         await this._reloadSkills();
         return { ok: true };
     }
     async copyBaseForEdit(name) {
         const overlay = await ensureSkillsOverlayDir();
-        if (!overlay)
+        if (!overlay) {
             throw new Error('no writable skills overlay is configured');
+        }
         const result = await copyBaseForEdit(await this._baseSkillsDir(), overlay, name);
-        if (result.copied)
+        if (result.copied) {
             await this._reloadSkills();
+        }
         return result;
     }
     async deleteOverlaySkill(name) {
         const overlay = await ensureSkillsOverlayDir();
-        if (!overlay)
+        if (!overlay) {
             throw new Error('no writable skills overlay is configured');
+        }
         const result = await deleteOverlaySkill(overlay, name);
-        if (result.removed)
+        if (result.removed) {
             await this._reloadSkills();
+        }
         return result;
     }
     resolveCredential(key, ctx) {
-        if (this._credentialStore)
+        if (this._credentialStore) {
             return this._credentialStore(key, ctx);
+        }
         const agentId = ctx?.agentId ?? null;
         const deviceShared = isDeviceSharedCredentialKey(key);
         for (const skill of ctx?.skills ?? []) {
             const owned = this._credentials.get(scopeKey({ agentId, skill, key })) ??
                 (agentId && deviceShared ? this._credentials.get(scopeKey({ skill, key })) : undefined);
-            if (owned)
+            if (owned) {
                 return owned;
+            }
         }
         if (agentId) {
             const scoped = this._credentials.get(scopeKey({ agentId, key }));
-            if (scoped)
+            if (scoped) {
                 return scoped;
+            }
         }
-        if (!agentId || deviceShared)
+        if (!agentId || deviceShared) {
             return this._credentials.get(scopeKey({ key })) ?? null;
+        }
         return null;
     }
     async _completionSkills(agentId, model) {
@@ -362,7 +391,8 @@ export class Harness extends ReadyResource {
         }));
     }
     async skills(agentId, modelName, options) {
-        await this.ready();
+        if (!this.opened)
+            await this.ready();
         const platform = this._platform ?? detectPlatform();
         const refresh = options?.refresh === true;
         return resolveSkillStatuses(this._allSkills, {
@@ -376,7 +406,7 @@ export class Harness extends ReadyResource {
                 : await this._probeBinVersions(platform, refresh),
             refresh,
             skipCredentialProbe: options?.skipCredentialProbe === true,
-            ...(modelName ? { activeProfile: modelNameToCapabilityProfile(modelName) } : {})
+            activeProfile: modelName ? modelNameToCapabilityProfile(modelName) : undefined
         });
     }
     async _reloadSkills() {
@@ -391,14 +421,15 @@ export class Harness extends ReadyResource {
             return;
         try {
             const overlay = await ensureSkillsOverlayDir();
-            if (!overlay)
+            if (!overlay) {
                 return;
+            }
             this._overlayWatcher = watchSkillsOverlay(overlay, () => {
-                this._reloadSkills().catch((err) => hlog('overlay reload failed', { err: err instanceof Error ? err.message : String(err) }));
+                this._reloadSkills().catch((err) => hlog('overlay reload failed', { err: errorMessage(err) }));
             });
         }
         catch (err) {
-            hlog('overlay watch unavailable', { err: err instanceof Error ? err.message : String(err) });
+            hlog('overlay watch unavailable', { err: errorMessage(err) });
         }
     }
     async tools() {
@@ -411,8 +442,9 @@ export class Harness extends ReadyResource {
         if (!refresh && this._binVersionCache && this._binVersionCache.expiresAt > now) {
             return this._binVersionCache.bySkill;
         }
-        if (this._binVersionInflight)
+        if (this._binVersionInflight) {
             return this._binVersionInflight;
+        }
         this._binVersionInflight = this._runBinVersionProbes(platform)
             .then((bySkill) => {
             this._binVersionCache = {
@@ -427,12 +459,14 @@ export class Harness extends ReadyResource {
         return this._binVersionInflight;
     }
     async _runBinVersionProbes(platform) {
-        if (process.env.QVAC_NO_SKILL_BIN_PROBE)
+        if (process.env.QVAC_NO_SKILL_BIN_PROBE) {
             return new Map();
+        }
         const installed = new Set(this._tools.map((tool) => tool.name));
         const candidates = versionProbeSkills(this._allSkills, installed, platform);
-        if (candidates.length === 0)
+        if (candidates.length === 0) {
             return new Map();
+        }
         let runtime;
         try {
             runtime = await bareRuntime();
@@ -452,8 +486,9 @@ export class Harness extends ReadyResource {
         const bySkill = new Map();
         for (const skill of candidates) {
             const states = await probeBinVersions(skill.requires, { platform, run });
-            if (states.size > 0)
+            if (states.size > 0) {
                 bySkill.set(skill.name, states);
+            }
         }
         return bySkill;
     }
@@ -472,31 +507,35 @@ export class Harness extends ReadyResource {
     }
     _projectionFor(model) {
         const custom = this._customProjection[model];
-        if (custom)
+        if (custom) {
             return custom;
+        }
         const projection = this._projectionModel;
-        if (!projection)
+        if (!projection) {
             return undefined;
+        }
         return typeof projection === 'string' ? projection : projection[model];
     }
     setCustomModels(entries) {
         setCustomModels(entries);
         this._customProjection = customProjectionMap();
     }
+    setModelIdleTimeout(ms) {
+        this._idle.configure(ms);
+    }
     previewModelSource(context) {
-        return previewModelSource(context, {
-            assessFit: async ({ sha256Checksum, name, contextTokens }) => {
-                const sdk = await this._sdk.load();
-                const result = await sdk.assessModelFit({
-                    models: [{ model: { sha256Checksum, name }, workload: { kind: 'llm', contextTokens } }]
-                });
-                const model = result.models[0];
-                return {
-                    fit: model?.verdict ?? result.verdict,
-                    fitReasons: [...(model?.reasons ?? result.reasons)]
-                };
-            }
+        return previewModelSource(context, { assessFit: (query) => this._assessFit(query) });
+    }
+    async _assessFit({ sha256Checksum, name, contextTokens }) {
+        const sdk = await this._sdk.load();
+        const result = await sdk.assessModelFit({
+            models: [{ model: { sha256Checksum, name }, workload: { kind: 'llm', contextTokens } }]
         });
+        const model = result.models[0];
+        return {
+            fit: model?.verdict ?? result.verdict,
+            fitReasons: [...(model?.reasons ?? result.reasons)]
+        };
     }
     async describe() {
         if (!this.opened)
@@ -519,11 +558,12 @@ export class Harness extends ReadyResource {
         };
     }
     _loadedModels() {
-        const pools = [this._embedModels, ...this._retiredEmbedModels];
-        if (this._ttsModels)
-            pools.push(this._ttsModels);
-        if (this._transcriptionModels)
-            pools.push(this._transcriptionModels);
+        const pools = [
+            this._embedModels,
+            ...this._retiredEmbedModels,
+            this._ttsModels,
+            this._transcriptionModels
+        ];
         return [...loadedModels(this._models), ...pools.flatMap((pool) => loadedModels(pool))];
     }
     _loadPluginSdk(current, store, importPlugin) {
@@ -557,8 +597,9 @@ export class Harness extends ReadyResource {
         const loadOpts = resolveLoadOptions({}, this._deviceConfig());
         const key = modelKey(model, loadOpts);
         const pool = this._models;
-        if (!isResident(pool, key))
-            await this._makeRoomForCompletion(signal);
+        if (!isResident(pool, key)) {
+            await this._makeRoomForCompletion(model, { ctxSize: loadOpts.ctxSize, signal });
+        }
         try {
             const { modelId } = await raceAbort(pool.open(key, model, loadOpts).load(), signal);
             const sdk = await this._sdk.load();
@@ -592,25 +633,51 @@ export class Harness extends ReadyResource {
     }
     _createPool() {
         const loadSdk = this._sdk.load;
-        return createModelPool(loadSdk, (state) => this._notifyModel(state, true), this._deviceConfig(), this._loads);
+        return createModelPool(loadSdk, (state) => this._notifyModel(state), {
+            deviceConfig: this._deviceConfig(),
+            loads: this._loads
+        });
     }
     _createEmbedPool() {
         const loadSdk = () => this._sdk.loadEmbed();
-        return createEmbedPool(loadSdk, (state) => this._notifyModel(state, false), this._embedFaults, this._embedBatchSize, openclCacheDir, this._loads);
+        return createEmbedPool(loadSdk, (state) => this._notifyModel(state), {
+            faults: this._embedFaults,
+            batchSize: this._embedBatchSize,
+            openclCacheDir,
+            loads: this._loads
+        });
     }
     _createTtsPool() {
         const loadSdk = () => this._sdk.loadVoice();
-        return createVoicePool(loadSdk, (state) => this._notifyModel(state, false), 'tts-ggml', 'tts', TTS_MODEL_CONFIG, undefined, this._loads);
+        return createVoicePool(loadSdk, (state) => this._notifyModel(state), {
+            modelType: 'tts-ggml',
+            addon: 'tts',
+            modelConfig: TTS_MODEL_CONFIG,
+            loads: this._loads
+        });
     }
     _createTranscriptionPool() {
         const loadSdk = () => this._sdk.loadVoice();
-        return createVoicePool(loadSdk, (state) => this._notifyModel({ ...state, model: TRANSCRIPTION_MODEL_LABEL }, false), 'parakeet-transcription', 'parakeet', { useGPU: this._transcriptionDevice !== 'cpu' }, openclCacheDir, this._loads);
+        return createVoicePool(loadSdk, (state) => this._notifyModel({ ...state, model: TRANSCRIPTION_MODEL_LABEL }), {
+            modelType: 'parakeet-transcription',
+            addon: 'parakeet',
+            modelConfig: { useGPU: this._transcriptionDevice !== 'cpu' },
+            openclCacheDir,
+            loads: this._loads
+        });
     }
-    _notifyModel(state, languageModel) {
-        const catalogChanged = languageModel && (state.status === 'ready' || state.status === 'unloaded');
+    _notifyModel(state) {
+        const catalogChanged = state.status === 'ready' || state.status === 'unloaded';
         if (catalogChanged) {
             invalidateModelStatusCache(this._modelStatus);
         }
+        if (state.status === 'unloaded') {
+            for (const key of this._pinned) {
+                if (!isResident(this._models, key)) {
+                    this._pinned.delete(key);
+                }
+            }
+        }
         this.emit('model', { ...state, catalogChanged, loaded: this._loadedModels() });
     }
     async _collectRetiredEmbedPool(pool) {
@@ -646,14 +713,17 @@ export class Harness extends ReadyResource {
         return this._enqueueEmbedControl(() => this._setEmbeddingDevice(device));
     }
     async _setEmbeddingDevice(device) {
-        await this.ready();
+        if (!this.opened)
+            await this.ready();
         if (this.closing)
             throw new Error('harness is closing');
-        if (this._embedFaults.device === device)
+        if (this._embedFaults.device === device) {
             return;
+        }
         this._embedFaults = createGpuFaultState({ device, gpuLayers: this._embedGpuLayers });
-        if (this.suspended)
+        if (this.suspended) {
             return;
+        }
         const retired = this._embedModels;
         this._retiredEmbedModels.add(retired);
         this._embedModels = this._createEmbedPool();
@@ -667,15 +737,18 @@ export class Harness extends ReadyResource {
     }
     async suspend() {
         hlog('suspend called');
-        if (this._state === HarnessState.SUSPEND)
+        if (this._state === HarnessState.SUSPEND) {
             return;
-        if (this.closing)
+        }
+        if (this.closing) {
             return;
+        }
         this._state = HarnessState.SUSPEND;
         hlog('suspending', { models: loadedModels(this._models).map((entry) => entry.model) });
         const pending = this._sdk.peek();
-        if (pending)
+        if (pending) {
             this._loads.drain(await pending);
+        }
         await this._releaseRuntimes();
         await this._sdkLifecycle('suspend');
         invalidateModelStatusCache(this._modelStatus);
@@ -683,18 +756,20 @@ export class Harness extends ReadyResource {
     }
     async _sdkLifecycle(op) {
         const sdk = await this._sdk.peek()?.catch(noop);
-        if (!sdk)
+        if (!sdk) {
             return;
+        }
         try {
             await sdk[op]();
         }
         catch (err) {
-            hlog(`sdk ${op} failed`, { err: err instanceof Error ? err.message : String(err) });
-            if (op === 'suspend')
+            hlog(`sdk ${op} failed`, { err: errorMessage(err) });
+            if (op === 'suspend') {
                 return;
+            }
             await sdk.resume().catch((retryErr) => {
                 hlog('sdk resume retry failed', {
-                    err: retryErr instanceof Error ? retryErr.message : String(retryErr)
+                    err: errorMessage(retryErr)
                 });
             });
         }
@@ -713,15 +788,38 @@ export class Harness extends ReadyResource {
             if (outcome.status !== 'rejected')
                 continue;
             const err = outcome.reason;
-            hlog('unload failed', { err: err instanceof Error ? err.message : String(err) });
+            hlog('unload failed', { err: errorMessage(err) });
         }
     }
+    async _unloadIdleModels() {
+        if (!this.opened || this.closing || this.suspended)
+            return;
+        hlog('idle unload', { models: this._loadedModels().map((entry) => entry.model) });
+        const released = await Promise.allSettled([
+            evictIdle(this._models),
+            evictIdle(this._embedModels),
+            ...[...this._retiredEmbedModels].map((pool) => this._collectRetiredEmbedPool(pool)),
+            evictIdle(this._ttsModels),
+            evictIdle(this._transcriptionModels),
+            this._imageRuntime?.unloadIdle(),
+            this._musicRuntime?.unloadIdle(),
+            this._translationRuntime.unload()
+        ]);
+        for (const outcome of released) {
+            if (outcome.status !== 'rejected')
+                continue;
+            hlog('idle unload failed', { err: errorMessage(outcome.reason) });
+        }
+        invalidateModelStatusCache(this._modelStatus);
+    }
     async resume() {
         hlog('resume called');
-        if (this._state === HarnessState.RESUME)
+        if (this._state === HarnessState.RESUME) {
             return;
-        if (this.closing)
+        }
+        if (this.closing) {
             return;
+        }
         if (this._state === HarnessState.PRESUSPEND) {
             this._state = HarnessState.RESUME;
             return;
@@ -744,22 +842,26 @@ export class Harness extends ReadyResource {
     }
     async _close() {
         hlog('closing');
+        this._idle.close();
         this._overlayWatcher?.stop();
         this._overlayWatcher = null;
         const pending = this._sdk.peek();
-        if (pending)
+        if (pending) {
             this._loads.drain(await pending);
+        }
         await Promise.all([this._releaseRuntimes(), closePythonWorker()]);
         const sdk = await this._sdk.peek()?.catch(noop);
-        if (sdk)
+        if (sdk) {
             await sdk.close();
+        }
     }
     download(args) {
         const projection = (args.addon ?? 'llm') === 'llm' ? this._projectionFor(args.model) : undefined;
         const multiFile = diffusionDownloadMetadata(args.model, args.addon) ??
             audioGenDownloadMetadata(args.model, args.addon);
         return download(async () => {
-            await this.ready();
+            if (!this.opened)
+                await this.ready();
             return this._sdk.load();
         }, {
             ...args,
@@ -768,23 +870,28 @@ export class Harness extends ReadyResource {
             assetSizes: multiFile?.assetSizes
         }, () => invalidateModelStatusCache(this._modelStatus));
     }
-    async embed({ model, texts, signal }) {
-        await this.ready();
+    embed(context) {
+        return this._busy(() => this._embed(context));
+    }
+    async _embed({ model, texts, signal }) {
+        if (!this.opened)
+            await this.ready();
         if (this.suspended)
             throw new Error('harness is suspended');
-        if (texts.length === 0)
+        if (texts.length === 0) {
             return [];
+        }
         const pool = this._embedModels;
         const faults = this._embedFaults;
         const generation = faults.generation;
-        const key = faults.key(model);
+        const key = faults.key(residentSrc(model, 'embeddings'));
         const handle = pool.open(key, model);
         let loaded = null;
         let job = null;
         let releaseAfter = null;
         try {
             loaded = await raceAbort(handle.load(), signal);
-            const sdk = (await this._sdk.loadEmbed());
+            const sdk = await this._sdk.loadEmbed();
             job = this._embedLane(sdk, { modelId: loaded.modelId, texts });
             const vectors = await raceAbort(job, signal);
             faults.reset();
@@ -807,15 +914,18 @@ export class Harness extends ReadyResource {
             throw err;
         }
         finally {
-            if (releaseAfter)
+            if (releaseAfter) {
                 this._releaseEmbedResourceAfter(releaseAfter, pool, key);
-            else
+            }
+            else {
                 await this._releaseEmbedResource(pool, key);
+            }
         }
     }
     async _defaultVoiceSrc(category) {
-        if (category !== 'tts')
+        if (category !== 'tts') {
             return defaultVoiceModel(category);
+        }
         try {
             const { capabilityProfile } = await this.capability();
             return defaultTtsFor(capabilityProfile).asset.src;
@@ -824,11 +934,13 @@ export class Harness extends ReadyResource {
             return defaultVoiceModel('tts');
         }
     }
-    async _voiceJob(pool, category, model, signal, run) {
-        await this.ready();
+    async _voiceJob(pool, { category, model, signal }, run) {
+        if (!this.opened)
+            await this.ready();
         if (this.suspended)
             throw new Error('harness is suspended');
-        const name = model || (await this._defaultVoiceSrc(category));
+        const addon = category === 'tts' ? 'tts' : 'parakeet';
+        const name = residentSrc(model || (await this._defaultVoiceSrc(category)), addon);
         const handle = pool.open(name, name);
         let job = null;
         let releaseAfter = null;
@@ -844,8 +956,9 @@ export class Harness extends ReadyResource {
         }
         catch (err) {
             if (err instanceof AbortedError) {
-                if (sdk && modelId)
+                if (sdk && modelId) {
                     void sdk.cancel({ modelId, operation: 'inference' }).catch(noop);
+                }
                 releaseAfter = job;
             }
             throw err;
@@ -854,19 +967,25 @@ export class Harness extends ReadyResource {
             const release = () => {
                 releaseModel(pool, name);
             };
-            if (releaseAfter)
+            if (releaseAfter) {
                 void releaseAfter.then(release, release);
-            else
+            }
+            else {
                 release();
+            }
         }
     }
-    async transcribe({ model, audio, signal }) {
-        return this._voiceJob(this._transcriptionModels, 'transcription', model, signal, async (sdk, id) => {
+    transcribe(context) {
+        return this._busy(() => this._transcribe(context));
+    }
+    async _transcribe({ model, audio, signal }) {
+        return this._voiceJob(this._transcriptionModels, { category: 'transcription', model, signal }, async (sdk, id) => {
             const pcm = await pcmForTranscription(audio);
-            if (signal?.aborted)
+            if (signal?.aborted) {
                 throw new AbortedError();
+            }
             if (pcm && pcm.length >= TRANSCRIBE_STREAM_MIN_BYTES) {
-                return transcribeStreaming(sdk, id, pcm, signal);
+                return transcribeStreaming(sdk, pcm, { modelId: id, signal });
             }
             return transcribeOnce(sdk, id, pcm ? buildWavBuffer(pcm) : audio);
         });
@@ -874,33 +993,43 @@ export class Harness extends ReadyResource {
     detectLanguage(context) {
         return detectLanguage(context);
     }
-    async speak({ model, text, signal }) {
-        const samples = await this._voiceJob(this._ttsModels, 'tts', model, signal, (sdk, id) => speakOnce(sdk, id, text));
+    speak(context) {
+        return this._busy(() => this._speak(context));
+    }
+    async _speak({ model, text, signal }) {
+        const samples = await this._voiceJob(this._ttsModels, { category: 'tts', model, signal }, (sdk, id) => speakOnce(sdk, id, text));
         return { audio: packSamples(samples), sampleRate: SUPERTONIC_SAMPLE_RATE };
     }
-    async *speakStream({ model, text, signal }) {
-        await this.ready();
+    speakStream(context) {
+        return this._busyStream(() => this._speakStream(context));
+    }
+    async *_speakStream({ model, text, signal }) {
+        if (!this.opened)
+            await this.ready();
         if (this.suspended)
             throw new Error('harness is suspended');
-        const name = model || (await this._defaultVoiceSrc('tts'));
+        const name = residentSrc(model || (await this._defaultVoiceSrc('tts')), 'tts');
         const pool = this._ttsModels;
         const handle = pool.open(name, name);
         const chunks = pushQueue();
         let sdk = null;
         let modelId = null;
         const cancel = () => {
-            if (sdk && modelId)
-                void sdk.cancel({ modelId, operation: 'inference' }).catch(() => { });
+            if (sdk && modelId) {
+                void sdk.cancel({ modelId, operation: 'inference' }).catch(noop);
+            }
         };
         signal?.addEventListener('abort', cancel, { once: true });
         const job = this._voiceLane(async () => {
             try {
                 modelId = (await handle.load()).modelId;
                 sdk = (await this._sdk.loadVoice());
-                if (signal?.aborted)
+                if (signal?.aborted) {
                     return;
-                for await (const chunk of speakStreamOnce(sdk, modelId, text))
+                }
+                for await (const chunk of speakStreamOnce(sdk, modelId, text)) {
                     chunks.push(chunk);
+                }
             }
             finally {
                 chunks.end();
@@ -914,18 +1043,23 @@ export class Harness extends ReadyResource {
         }
         finally {
             signal?.removeEventListener('abort', cancel);
-            if (!settled)
+            if (!settled) {
                 cancel();
+            }
             void job.catch(noop).finally(() => {
                 releaseModel(pool, name);
             });
         }
     }
     async list(options = {}) {
-        await this.ready();
-        const sdk = (await this._sdk.load());
+        if (!this.opened)
+            await this.ready();
+        const sdk = await this._sdk.load();
         const [rows, custom] = await Promise.all([
-            listModels(sdk, options, this._modelStatus, (name) => this._projectionFor(name)),
+            listModels(sdk, options, {
+                cache: this._modelStatus,
+                projectionFor: (name) => this._projectionFor(name)
+            }),
             customModelRows(options, {
                 modelsDir: this._modelsDir,
                 loaded: new Set(this._loadedModels().map((entry) => entry.model))
@@ -934,18 +1068,21 @@ export class Harness extends ReadyResource {
         return mergeCustomRows(rows, custom);
     }
     async deleteModel(name) {
-        await this.ready();
-        const sdk = (await this._sdk.load());
+        if (!this.opened)
+            await this.ready();
+        const sdk = await this._sdk.load();
         const result = await deleteModelFiles(sdk, name, { modelsDir: this._modelsDir });
         invalidateModelStatusCache(this._modelStatus);
         return summarizeDelete(result);
     }
     async clearKvCache() {
-        await this.ready();
+        if (!this.opened)
+            await this.ready();
         return deleteAllKvCaches(() => this._sdk.load());
     }
     async deleteKvCache(kvCacheKey) {
-        await this.ready();
+        if (!this.opened)
+            await this.ready();
         return deleteKvCacheKey(() => this._sdk.load(), kvCacheKey);
     }
     async infraModels(profile) {
@@ -989,37 +1126,160 @@ export class Harness extends ReadyResource {
         });
         return this._capabilityInflight;
     }
-    async warmModel({ model, agent, signal }) {
-        await this.ready();
+    warmModel(context) {
+        return this._busy(() => this._warmModel(context));
+    }
+    async _warmModel({ model, agent, signal }) {
+        if (!this.opened)
+            await this.ready();
         if (this.suspended)
             return;
-        const projectionModel = this._projectionFor(model);
-        const loadOpts = resolveLoadOptions({
-            ctxSize: agent?.ctxSize ?? undefined,
-            kvCacheK: agent?.kvCacheK ?? undefined,
-            kvCacheV: agent?.kvCacheV ?? undefined,
-            projectionModel
-        }, this._deviceConfig());
-        const key = modelKey(model, loadOpts);
-        const pool = this._models;
-        hlog('warm model', { model, resident: isResident(pool, key) });
-        if (!isResident(pool, key)) {
-            if (!(await this._makeRoomForCompletion(signal, WARM_ROOM_WAIT_MS)))
+        const target = this._residency(model, agent);
+        const resident = isResident(target.pool, target.key);
+        hlog('warm model', { model, resident });
+        if (target.pool === this._models && !resident) {
+            const roomed = await this._makeRoomForCompletion(model, {
+                ctxSize: target.ctxSize,
+                signal,
+                deadlineMs: WARM_ROOM_WAIT_MS
+            });
+            if (!roomed) {
                 return;
-            await this._imageRuntime?.unloadIdle().catch(noop);
+            }
+            await this._unloadIdleGenerators();
         }
+        await this._loadAndRelease(target, signal);
+    }
+    loadModel(context) {
+        return this._busy(() => this._loadModel(context));
+    }
+    async _loadModel({ model, agent, signal }) {
+        if (!this.opened)
+            await this.ready();
+        if (this.suspended) {
+            throw new Error('harness is suspended');
+        }
+        const target = this._residency(model, agent);
+        const resident = isResident(target.pool, target.key);
+        hlog('load model', { model, resident });
+        if (target.pool !== this._models) {
+            await this._loadAndRelease(target, signal);
+            return { loaded: true };
+        }
+        if (!resident) {
+            if (!(await this._fits(model, target.ctxSize))) {
+                return { loaded: false, reason: 'memory' };
+            }
+            await evictIdleWhere(this._models, (key) => modelOf(key) === model && !this._replying.includes(key));
+            await this._unloadIdleGenerators();
+        }
+        this._pinned.add(target.key);
+        try {
+            await this._loadAndRelease(target, signal);
+        }
+        catch (err) {
+            this._pinned.delete(target.key);
+            throw err;
+        }
+        return { loaded: true };
+    }
+    unloadModel(context) {
+        return this._busy(() => this._unloadModel(context));
+    }
+    async _unloadModel({ model }) {
+        if (!this.opened)
+            await this.ready();
+        const target = this._residency(model);
+        const name = modelOf(target.key);
+        const unloaded = await evictModel(target.pool, name, { deadlineMs: UNLOAD_LEASE_WAIT_MS });
+        hlog('unload model', { model, unloaded });
+        if (unloaded) {
+            for (const key of this._pinned) {
+                if (modelOf(key) === name) {
+                    this._pinned.delete(key);
+                }
+            }
+        }
+        return { unloaded };
+    }
+    async _loadAndRelease(target, signal) {
         try {
-            await raceAbort(pool.open(key, model, loadOpts).load(), signal);
+            await raceAbort(target.open().load(), signal);
         }
         finally {
-            releaseModel(pool, key);
+            releaseModel(target.pool, target.key);
         }
     }
-    async *invoke({ capability, args, enhance, agent, attach, fetchAttachment, progress, signal }) {
+    async _unloadIdleGenerators() {
+        await this._imageRuntime?.unloadIdle().catch(noop);
+        await this._musicRuntime?.unloadIdle().catch(noop);
+    }
+    async _fits(model, ctxSize) {
+        const asset = curatedModelAsset(model, 'llm');
+        if (!asset) {
+            return true;
+        }
+        const assessed = await this._assessFit({
+            sha256Checksum: asset.sha256Checksum,
+            name: asset.name,
+            contextTokens: ctxSize || defaultContextSize(asset)
+        }).catch((err) => {
+            hlog('fit estimate failed', { model, err: errorMessage(err) });
+            return null;
+        });
+        return assessed?.fit !== 'likely-too-large';
+    }
+    async _yieldPins(model, ctxSize) {
+        for (const key of this._pinned) {
+            if (leaseCount(this._models, key) > 0 || this._replying.includes(key)) {
+                continue;
+            }
+            if (await this._fits(model, ctxSize))
+                return;
+            this._pinned.delete(key);
+            await evictModelWithin(this._models, key, { deadlineMs: WEDGED_EVICT_WAIT_MS });
+        }
+    }
+    _residency(model, agent) {
+        const addon = modelAddonOf(model);
+        if (addon === 'llm') {
+            const loadOpts = this._completionLoadOptions(model, agent);
+            const key = modelKey(model, loadOpts);
+            return {
+                pool: this._models,
+                key,
+                ctxSize: loadOpts.ctxSize,
+                open: () => this._models.open(key, model, loadOpts)
+            };
+        }
+        if (addon === 'embeddings') {
+            const key = this._embedFaults.key(residentSrc(model, addon));
+            return { pool: this._embedModels, key, open: () => this._embedModels.open(key, model) };
+        }
+        if (addon === 'tts' || addon === 'parakeet') {
+            const pool = addon === 'tts' ? this._ttsModels : this._transcriptionModels;
+            const key = residentSrc(model, addon);
+            return { pool, key, open: () => pool.open(key, key) };
+        }
+        throw new Error(`${model} loads for each generation and is not kept in memory`);
+    }
+    _completionLoadOptions(model, agent) {
+        return resolveLoadOptions({
+            ctxSize: agent?.ctxSize ?? undefined,
+            kvCacheK: agent?.kvCacheK ?? undefined,
+            kvCacheV: agent?.kvCacheV ?? undefined,
+            projectionModel: this._projectionFor(model)
+        }, this._deviceConfig());
+    }
+    invoke(context) {
+        return this._busyStream(() => this._invoke(context));
+    }
+    async *_invoke({ capability, args, enhance, agent, attach, fetchAttachment, progress, signal }) {
         if (!this.opened)
             await this.ready();
-        if (this.suspended)
+        if (this.suspended) {
             return yield { type: 'aborted' };
+        }
         hlog('invoke start', { capability, enhance });
         try {
             yield* invoke({
@@ -1041,31 +1301,28 @@ export class Harness extends ReadyResource {
             hlog('invoke end', { capability });
         }
     }
-    async *completion({ model, agent, messages, cacheKey, thinkingDefault, toolConfigs, generationParams, responseFormat, captureInferenceCalls, grantedDirs, grantedDrives, driveOp, grantedWriteDirs, approve, attach, fetchAttachment, ragSearch, progress, askUser, signal }) {
-        await this.ready();
+    completion(context) {
+        return this._busyStream(() => this._completion(context));
+    }
+    async *_completion({ model, agent, messages, cacheKey, thinkingDefault, toolConfigs, generationParams, responseFormat, captureInferenceCalls, grantedDirs, grantedDrives, driveOp, grantedWriteDirs, approve, attach, fetchAttachment, ragSearch, progress, askUser, signal }) {
+        if (!this.opened)
+            await this.ready();
         if (this.suspended)
             throw new Error('harness is suspended');
-        const projectionModel = this._projectionFor(model);
-        const loadOpts = resolveLoadOptions({
-            ctxSize: agent?.ctxSize ?? undefined,
-            kvCacheK: agent?.kvCacheK ?? undefined,
-            kvCacheV: agent?.kvCacheV ?? undefined,
-            projectionModel
-        }, this._deviceConfig());
+        const loadOpts = this._completionLoadOptions(model, agent);
+        const projectionModel = loadOpts.projectionModel;
         const key = modelKey(model, loadOpts);
         const pool = this._models;
         const resident = isResident(pool, key);
         hlog('completion start', { model, resident, projection: Boolean(projectionModel) });
         if (!resident) {
-            await this._makeRoomForCompletion(signal);
-            await this._imageRuntime?.unloadIdle().catch(noop);
-            await this._musicRuntime?.unloadIdle().catch(noop);
+            await this._makeRoomForCompletion(model, { ctxSize: loadOpts.ctxSize, signal });
+            await this._unloadIdleGenerators();
         }
         const acquireRound = async () => {
             if (!isResident(pool, key)) {
-                await this._makeRoomForCompletion(signal);
-                await this._imageRuntime?.unloadIdle().catch(noop);
-                await this._musicRuntime?.unloadIdle().catch(noop);
+                await this._makeRoomForCompletion(model, { ctxSize: loadOpts.ctxSize, signal });
+                await this._unloadIdleGenerators();
             }
             try {
                 const { modelId } = await raceAbort(pool.open(key, model, loadOpts).load(), signal);
@@ -1089,6 +1346,7 @@ export class Harness extends ReadyResource {
             held = false;
             releaseModel(pool, key);
         };
+        this._replying.push(key);
         try {
             const { modelId, ctxSize } = await raceAbort(handle.load(), signal);
             hlog('model ready', { model });
@@ -1137,21 +1395,18 @@ export class Harness extends ReadyResource {
                 yield { type: 'aborted' };
             }
             else {
-                const message = err instanceof Error ? err.message : String(err);
+                const message = errorMessage(err);
                 hlog('completion failed', { model, ...resolutionDetail(err), err: message });
-                if (isSdkModelWedgedError(err)) {
-                    const wedged = await this._evictWedgedModel(model, key);
-                    yield { type: 'error', message: wedged.message };
-                    if (wedged.fatal)
-                        process.exit(1);
-                }
-                else {
-                    yield { type: 'error', message };
+                const wedged = isSdkModelWedgedError(err) ? await this._evictWedgedModel(model, key) : null;
+                yield { type: 'error', message: wedged ? wedged.message : message };
+                if (wedged?.fatal) {
+                    process.exit(1);
                 }
             }
         }
         finally {
             hlog('completion end', { model });
+            this._replying.splice(this._replying.indexOf(key), 1);
             releaseHeld();
             void this._imageRuntime?.unloadIdle().catch(noop);
             void this._musicRuntime?.unloadIdle().catch(noop);
@@ -1160,14 +1415,19 @@ export class Harness extends ReadyResource {
     _residentEnhancer() {
         return async (prompt, signal) => {
             const lease = await leaseResidentModel(this._models, (model) => !isTranslationModel(model));
-            if (!lease)
+            if (!lease) {
                 return null;
+            }
             try {
                 const sdk = await this._sdk.load();
-                return await expandImagePrompt(sdk, lease.modelId, lease.model, prompt, signal);
+                return await expandImagePrompt(sdk, prompt, {
+                    modelId: lease.modelId,
+                    model: lease.model,
+                    signal
+                });
             }
             catch (err) {
-                hlog('enhance failed', { err: err instanceof Error ? err.message : String(err) });
+                hlog('enhance failed', { err: errorMessage(err) });
                 return null;
             }
             finally {
@@ -1175,24 +1435,33 @@ export class Harness extends ReadyResource {
             }
         };
     }
-    async _makeRoomForCompletion(signal, deadlineMs = MODEL_ROOM_WAIT_MS) {
-        const roomed = await awaitModelCapacity(this._models, this._completionModelCap, deadlineMs, 250, signal);
+    async _makeRoomForCompletion(model, { ctxSize, signal, deadlineMs = MODEL_ROOM_WAIT_MS } = {}) {
+        const roomed = await awaitModelCapacity(this._models, {
+            cap: this._completionModelCap,
+            deadlineMs,
+            signal,
+            evictable: (key) => !this._pinned.has(key) || modelOf(key) === model
+        });
         if (!roomed) {
             hlog('model room wait gave up', {
                 cap: this._completionModelCap,
                 loaded: loadedModels(this._models).length
             });
         }
+        await this._yieldPins(model, ctxSize);
         return roomed;
     }
     async _evictWedgedModel(model, key) {
         const faults = (this._modelFaults.get(model) ?? 0) + 1;
         this._modelFaults.set(model, faults);
         const leased = leaseCount(this._models, key) > 0;
-        const evicted = leased ? false : await evictModelWithin(this._models, key, WEDGED_EVICT_WAIT_MS);
+        const evicted = leased
+            ? false
+            : await evictModelWithin(this._models, key, { deadlineMs: WEDGED_EVICT_WAIT_MS });
         hlog('wedged model evicted', { model, faults, evicted, leased });
-        if (leased)
+        if (leased) {
             this._evictAfterLeases(model, key);
+        }
         if (!evicted && !leased) {
             return {
                 message: `${model} is stuck in device memory — restarting the model runtime`,
@@ -1207,9 +1476,27 @@ export class Harness extends ReadyResource {
         };
     }
     _evictAfterLeases(model, key) {
-        void evictModelWithin(this._models, key, WEDGED_EVICT_LEASED_MS)
+        void evictModelWithin(this._models, key, { deadlineMs: WEDGED_EVICT_LEASED_MS })
             .then((evicted) => hlog('wedged model evicted after leases', { model, evicted }))
             .catch(noop);
     }
+    async _busy(run) {
+        await this._idle.enter();
+        try {
+            return await run();
+        }
+        finally {
+            this._idle.exit();
+        }
+    }
+    async *_busyStream(run) {
+        await this._idle.enter();
+        try {
+            yield* run();
+        }
+        finally {
+            this._idle.exit();
+        }
+    }
 }
 function noop() { }