@qvac/harness 0.3.3 → 0.5.0

package/dist-lib/index.js

@@ -8,6 +8,7 @@ import { closePythonWorker } from './lib/tools/exec/adapters/python/worker/clien
 import { completion, imageSettingsFrom } from './lib/capabilities/completion.js';
 import { detectLanguage } from './lib/capabilities/detect-language.js';
 import { invoke } from './lib/capabilities/invoke.js';
+import { previewModelSource } from './lib/capabilities/preview-model-source.js';
 import { expandImagePrompt } from './lib/capabilities/enhance.js';
 import { cleanAnswer } from './lib/helpers/clean-answer.js';
 import { isTranslationModel } from './lib/models/nmt-catalog.js';
@@ -21,10 +22,11 @@ import { createVoiceLane, packSamples, speakOnce, speakStreamOnce, transcribeOnc
 import { pushQueue } from './lib/queue.js';
 import { AbortedError, raceAbort } from './lib/helpers/race-abort.js';
 import { audioGenDownloadMetadata, customProjectionMap, defaultProjectionMap, defaultTtsFor, defaultVoiceModel, diffusionDownloadMetadata, infraModelsFor, modelNameToCapabilityProfile, musicModelSet, setCustomModels } from './lib/models/model-catalog.js';
-import { createModelStatusCache, invalidateModelStatusCache, listModels } from './lib/models/model-list.js';
+import { createModelStatusCache, customModelRows, invalidateModelStatusCache, mergeCustomRows, listModels } from './lib/models/model-list.js';
 import { deleteModelFiles, summarizeDelete } from './lib/models/model-delete.js';
 import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, awaitModelCapacity, evictIdle, evictModelWithin, isResident, leaseResidentModel, leaseCount, loadedModels, modelKey, releaseModel, resolveLoadOptions, progressState } from './lib/models/model-pool.js';
 import { resolveMainGpu } from './lib/models/gpu-selection.js';
+import { createIdleReaper } from './lib/models/idle-unload.js';
 import { openclCacheDir } from './lib/models/opencl-cache.js';
 import { assertEmbedDevice, createEmbedConfig } from './lib/models/embed-config.js';
 import { createGpuFaultState, isSdkGpuDeviceLostError, isSdkModelWedgedError } from './lib/models/sdk-faults.js';
@@ -38,6 +40,7 @@ import { ensureBundledSkillsDir } from './lib/tools/skills/bundled-dir.js';
 import { ensureSkillsOverlayDir } from './lib/tools/skills/overlay-dir.js';
 import { copyBaseForEdit, deleteOverlaySkill, listManagedSkills, readSkillFile, writeSkillFile } from './lib/tools/skills/overlay-crud.js';
 import { watchSkillsOverlay } from './lib/tools/skills/overlay-watch.js';
+import { errorMessage } from './lib/guards.js';
 import { hlog } from './lib/log.js';
 import { harness } from './spec/index.js';
 import { installResolutionDetector, resolutionDetail } from './lib/resolution-detector.js';
@@ -104,6 +107,7 @@ export class Harness extends ReadyResource {
     _voiceLane = createVoiceLane();
     _state;
     _skillsDir;
+    _modelsDir;
     _overlayWatcher = null;
     _supportedSkills;
     _projectionModel;
@@ -131,7 +135,8 @@ export class Harness extends ReadyResource {
     _capability = null;
     _capabilityInflight = null;
     _modelFaults = new Map();
-    constructor({ tools = [], sdkLoader, skillsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, completionModelCap, credentialStore, platform, gpuStats } = {}) {
+    _idle;
+    constructor({ tools = [], sdkLoader, skillsDir, modelsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, completionModelCap, credentialStore, platform, gpuStats } = {}) {
         super();
         this._sdk = sdkLoader ?? createSdkLoader();
         this._llmDevice = llmDevice;
@@ -175,8 +180,9 @@ export class Harness extends ReadyResource {
                 },
                 onLoaded: (model) => {
                     const transfer = imageTransfers.get(model);
-                    if (!transfer)
+                    if (!transfer) {
                         return;
+                    }
                     imageTransfers.delete(model);
                     invalidateModelStatusCache(this._modelStatus);
                     this.emit('model', { ...transfer, status: 'downloaded' });
@@ -184,9 +190,14 @@ export class Harness extends ReadyResource {
                 onBeforeLoad: async (signal) => {
                     if (this._models.destroyed)
                         return;
-                    const evicted = await awaitModelCapacity(this._models, 1, DIFFUSION_EVICT_WAIT_MS, undefined, signal);
-                    if (signal?.aborted)
+                    const evicted = await awaitModelCapacity(this._models, {
+                        cap: 1,
+                        deadlineMs: DIFFUSION_EVICT_WAIT_MS,
+                        signal
+                    });
+                    if (signal?.aborted) {
                         throw new AbortedError();
+                    }
                     if (!evicted) {
                         throw new Error('the chat model is busy; try the image again once the current reply finishes');
                     }
@@ -203,9 +214,14 @@ export class Harness extends ReadyResource {
                 onBeforeLoad: async (signal) => {
                     if (this._models.destroyed)
                         return;
-                    const evicted = await awaitModelCapacity(this._models, 1, MUSIC_EVICT_WAIT_MS, undefined, signal);
-                    if (signal?.aborted)
+                    const evicted = await awaitModelCapacity(this._models, {
+                        cap: 1,
+                        deadlineMs: MUSIC_EVICT_WAIT_MS,
+                        signal
+                    });
+                    if (signal?.aborted) {
                         throw new AbortedError();
+                    }
                     if (!evicted) {
                         throw new Error('the chat model is busy; try the music again once the current reply finishes');
                     }
@@ -235,6 +251,7 @@ export class Harness extends ReadyResource {
         this._projectionModel = projectionModel ?? defaultProjectionMap();
         this._state = HarnessState.RESUME;
         this._skillsDir = skillsDir;
+        this._modelsDir = modelsDir;
         this._supportedSkills = [];
         this._allSkills = [];
         this._credentialStore = credentialStore;
@@ -242,13 +259,18 @@ export class Harness extends ReadyResource {
         this._diffusionSdk = null;
         this._musicSdk = null;
         this._translationSdk = null;
-    }
-    async _open() {
-        installResolutionDetector(import.meta);
         this._models = this._createPool();
         this._embedModels = this._createEmbedPool();
         this._ttsModels = this._createTtsPool();
         this._transcriptionModels = this._createTranscriptionPool();
+        this._idle = createIdleReaper({
+            sweep: () => this._unloadIdleModels(),
+            waitMs: WEDGED_EVICT_WAIT_MS,
+            onError: (err) => hlog('idle unload failed', { err: errorMessage(err) })
+        });
+    }
+    async _open() {
+        installResolutionDetector(import.meta);
         await this._reloadSkills();
         await this._startOverlayWatch();
         hlog('open', { skills: this._supportedSkills.length, tools: this._tools.length });
@@ -266,10 +288,7 @@ export class Harness extends ReadyResource {
             if (!token)
                 continue;
             const storeKey = scopeKey({ agentId, skill, key });
-            const record = {
-                token,
-                ...(typeof expiresAt === 'number' && expiresAt > 0 ? { expiresAt } : {})
-            };
+            const record = typeof expiresAt === 'number' && expiresAt > 0 ? { token, expiresAt } : { token };
             const current = previous.get(storeKey);
             if (!current || current.token !== record.token || current.expiresAt !== record.expiresAt) {
                 resetCredentialHealthAfterSnapshot(key, agentId, skill);
@@ -307,48 +326,57 @@ export class Harness extends ReadyResource {
     }
     async writeSkillFile(name, content) {
         const overlay = await ensureSkillsOverlayDir();
-        if (!overlay)
+        if (!overlay) {
             throw new Error('no writable skills overlay is configured');
+        }
         await writeSkillFile(overlay, name, content);
         await this._reloadSkills();
         return { ok: true };
     }
     async copyBaseForEdit(name) {
         const overlay = await ensureSkillsOverlayDir();
-        if (!overlay)
+        if (!overlay) {
             throw new Error('no writable skills overlay is configured');
+        }
         const result = await copyBaseForEdit(await this._baseSkillsDir(), overlay, name);
-        if (result.copied)
+        if (result.copied) {
             await this._reloadSkills();
+        }
         return result;
     }
     async deleteOverlaySkill(name) {
         const overlay = await ensureSkillsOverlayDir();
-        if (!overlay)
+        if (!overlay) {
             throw new Error('no writable skills overlay is configured');
+        }
         const result = await deleteOverlaySkill(overlay, name);
-        if (result.removed)
+        if (result.removed) {
             await this._reloadSkills();
+        }
         return result;
     }
     resolveCredential(key, ctx) {
-        if (this._credentialStore)
+        if (this._credentialStore) {
             return this._credentialStore(key, ctx);
+        }
         const agentId = ctx?.agentId ?? null;
         const deviceShared = isDeviceSharedCredentialKey(key);
         for (const skill of ctx?.skills ?? []) {
             const owned = this._credentials.get(scopeKey({ agentId, skill, key })) ??
                 (agentId && deviceShared ? this._credentials.get(scopeKey({ skill, key })) : undefined);
-            if (owned)
+            if (owned) {
                 return owned;
+            }
         }
         if (agentId) {
             const scoped = this._credentials.get(scopeKey({ agentId, key }));
-            if (scoped)
+            if (scoped) {
                 return scoped;
+            }
         }
-        if (!agentId || deviceShared)
+        if (!agentId || deviceShared) {
             return this._credentials.get(scopeKey({ key })) ?? null;
+        }
         return null;
     }
     async _completionSkills(agentId, model) {
@@ -359,7 +387,8 @@ export class Harness extends ReadyResource {
         }));
     }
     async skills(agentId, modelName, options) {
-        await this.ready();
+        if (!this.opened)
+            await this.ready();
         const platform = this._platform ?? detectPlatform();
         const refresh = options?.refresh === true;
         return resolveSkillStatuses(this._allSkills, {
@@ -373,7 +402,7 @@ export class Harness extends ReadyResource {
                 : await this._probeBinVersions(platform, refresh),
             refresh,
             skipCredentialProbe: options?.skipCredentialProbe === true,
-            ...(modelName ? { activeProfile: modelNameToCapabilityProfile(modelName) } : {})
+            activeProfile: modelName ? modelNameToCapabilityProfile(modelName) : undefined
         });
     }
     async _reloadSkills() {
@@ -388,14 +417,15 @@ export class Harness extends ReadyResource {
             return;
         try {
             const overlay = await ensureSkillsOverlayDir();
-            if (!overlay)
+            if (!overlay) {
                 return;
+            }
             this._overlayWatcher = watchSkillsOverlay(overlay, () => {
-                this._reloadSkills().catch((err) => hlog('overlay reload failed', { err: err instanceof Error ? err.message : String(err) }));
+                this._reloadSkills().catch((err) => hlog('overlay reload failed', { err: errorMessage(err) }));
             });
         }
         catch (err) {
-            hlog('overlay watch unavailable', { err: err instanceof Error ? err.message : String(err) });
+            hlog('overlay watch unavailable', { err: errorMessage(err) });
         }
     }
     async tools() {
@@ -408,8 +438,9 @@ export class Harness extends ReadyResource {
         if (!refresh && this._binVersionCache && this._binVersionCache.expiresAt > now) {
             return this._binVersionCache.bySkill;
         }
-        if (this._binVersionInflight)
+        if (this._binVersionInflight) {
             return this._binVersionInflight;
+        }
         this._binVersionInflight = this._runBinVersionProbes(platform)
             .then((bySkill) => {
             this._binVersionCache = {
@@ -424,12 +455,14 @@ export class Harness extends ReadyResource {
         return this._binVersionInflight;
     }
     async _runBinVersionProbes(platform) {
-        if (process.env.QVAC_NO_SKILL_BIN_PROBE)
+        if (process.env.QVAC_NO_SKILL_BIN_PROBE) {
             return new Map();
+        }
         const installed = new Set(this._tools.map((tool) => tool.name));
         const candidates = versionProbeSkills(this._allSkills, installed, platform);
-        if (candidates.length === 0)
+        if (candidates.length === 0) {
             return new Map();
+        }
         let runtime;
         try {
             runtime = await bareRuntime();
@@ -449,8 +482,9 @@ export class Harness extends ReadyResource {
         const bySkill = new Map();
         for (const skill of candidates) {
             const states = await probeBinVersions(skill.requires, { platform, run });
-            if (states.size > 0)
+            if (states.size > 0) {
                 bySkill.set(skill.name, states);
+            }
         }
         return bySkill;
     }
@@ -469,17 +503,37 @@ export class Harness extends ReadyResource {
     }
     _projectionFor(model) {
         const custom = this._customProjection[model];
-        if (custom)
+        if (custom) {
             return custom;
+        }
         const projection = this._projectionModel;
-        if (!projection)
+        if (!projection) {
             return undefined;
+        }
         return typeof projection === 'string' ? projection : projection[model];
     }
     setCustomModels(entries) {
         setCustomModels(entries);
         this._customProjection = customProjectionMap();
     }
+    setModelIdleTimeout(ms) {
+        this._idle.configure(ms);
+    }
+    previewModelSource(context) {
+        return previewModelSource(context, {
+            assessFit: async ({ sha256Checksum, name, contextTokens }) => {
+                const sdk = await this._sdk.load();
+                const result = await sdk.assessModelFit({
+                    models: [{ model: { sha256Checksum, name }, workload: { kind: 'llm', contextTokens } }]
+                });
+                const model = result.models[0];
+                return {
+                    fit: model?.verdict ?? result.verdict,
+                    fitReasons: [...(model?.reasons ?? result.reasons)]
+                };
+            }
+        });
+    }
     async describe() {
         if (!this.opened)
             await this.ready();
@@ -501,11 +555,12 @@ export class Harness extends ReadyResource {
         };
     }
     _loadedModels() {
-        const pools = [this._embedModels, ...this._retiredEmbedModels];
-        if (this._ttsModels)
-            pools.push(this._ttsModels);
-        if (this._transcriptionModels)
-            pools.push(this._transcriptionModels);
+        const pools = [
+            this._embedModels,
+            ...this._retiredEmbedModels,
+            this._ttsModels,
+            this._transcriptionModels
+        ];
         return [...loadedModels(this._models), ...pools.flatMap((pool) => loadedModels(pool))];
     }
     _loadPluginSdk(current, store, importPlugin) {
@@ -539,8 +594,9 @@ export class Harness extends ReadyResource {
         const loadOpts = resolveLoadOptions({}, this._deviceConfig());
         const key = modelKey(model, loadOpts);
         const pool = this._models;
-        if (!isResident(pool, key))
+        if (!isResident(pool, key)) {
             await this._makeRoomForCompletion(signal);
+        }
         try {
             const { modelId } = await raceAbort(pool.open(key, model, loadOpts).load(), signal);
             const sdk = await this._sdk.load();
@@ -574,21 +630,40 @@ export class Harness extends ReadyResource {
     }
     _createPool() {
         const loadSdk = this._sdk.load;
-        return createModelPool(loadSdk, (state) => this._notifyModel(state, true), this._deviceConfig(), this._loads);
+        return createModelPool(loadSdk, (state) => this._notifyModel(state, { languageModel: true }), {
+            deviceConfig: this._deviceConfig(),
+            loads: this._loads
+        });
     }
     _createEmbedPool() {
         const loadSdk = () => this._sdk.loadEmbed();
-        return createEmbedPool(loadSdk, (state) => this._notifyModel(state, false), this._embedFaults, this._embedBatchSize, openclCacheDir, this._loads);
+        return createEmbedPool(loadSdk, (state) => this._notifyModel(state), {
+            faults: this._embedFaults,
+            batchSize: this._embedBatchSize,
+            openclCacheDir,
+            loads: this._loads
+        });
     }
     _createTtsPool() {
         const loadSdk = () => this._sdk.loadVoice();
-        return createVoicePool(loadSdk, (state) => this._notifyModel(state, false), 'tts-ggml', 'tts', TTS_MODEL_CONFIG, undefined, this._loads);
+        return createVoicePool(loadSdk, (state) => this._notifyModel(state), {
+            modelType: 'tts-ggml',
+            addon: 'tts',
+            modelConfig: TTS_MODEL_CONFIG,
+            loads: this._loads
+        });
     }
     _createTranscriptionPool() {
         const loadSdk = () => this._sdk.loadVoice();
-        return createVoicePool(loadSdk, (state) => this._notifyModel({ ...state, model: TRANSCRIPTION_MODEL_LABEL }, false), 'parakeet-transcription', 'parakeet', { useGPU: this._transcriptionDevice !== 'cpu' }, openclCacheDir, this._loads);
+        return createVoicePool(loadSdk, (state) => this._notifyModel({ ...state, model: TRANSCRIPTION_MODEL_LABEL }), {
+            modelType: 'parakeet-transcription',
+            addon: 'parakeet',
+            modelConfig: { useGPU: this._transcriptionDevice !== 'cpu' },
+            openclCacheDir,
+            loads: this._loads
+        });
     }
-    _notifyModel(state, languageModel) {
+    _notifyModel(state, { languageModel = false } = {}) {
         const catalogChanged = languageModel && (state.status === 'ready' || state.status === 'unloaded');
         if (catalogChanged) {
             invalidateModelStatusCache(this._modelStatus);
@@ -628,14 +703,17 @@ export class Harness extends ReadyResource {
         return this._enqueueEmbedControl(() => this._setEmbeddingDevice(device));
     }
     async _setEmbeddingDevice(device) {
-        await this.ready();
+        if (!this.opened)
+            await this.ready();
         if (this.closing)
             throw new Error('harness is closing');
-        if (this._embedFaults.device === device)
+        if (this._embedFaults.device === device) {
             return;
+        }
         this._embedFaults = createGpuFaultState({ device, gpuLayers: this._embedGpuLayers });
-        if (this.suspended)
+        if (this.suspended) {
             return;
+        }
         const retired = this._embedModels;
         this._retiredEmbedModels.add(retired);
         this._embedModels = this._createEmbedPool();
@@ -649,15 +727,18 @@ export class Harness extends ReadyResource {
     }
     async suspend() {
         hlog('suspend called');
-        if (this._state === HarnessState.SUSPEND)
+        if (this._state === HarnessState.SUSPEND) {
             return;
-        if (this.closing)
+        }
+        if (this.closing) {
             return;
+        }
         this._state = HarnessState.SUSPEND;
         hlog('suspending', { models: loadedModels(this._models).map((entry) => entry.model) });
         const pending = this._sdk.peek();
-        if (pending)
+        if (pending) {
             this._loads.drain(await pending);
+        }
         await this._releaseRuntimes();
         await this._sdkLifecycle('suspend');
         invalidateModelStatusCache(this._modelStatus);
@@ -665,18 +746,20 @@ export class Harness extends ReadyResource {
     }
     async _sdkLifecycle(op) {
         const sdk = await this._sdk.peek()?.catch(noop);
-        if (!sdk)
+        if (!sdk) {
             return;
+        }
         try {
             await sdk[op]();
         }
         catch (err) {
-            hlog(`sdk ${op} failed`, { err: err instanceof Error ? err.message : String(err) });
-            if (op === 'suspend')
+            hlog(`sdk ${op} failed`, { err: errorMessage(err) });
+            if (op === 'suspend') {
                 return;
+            }
             await sdk.resume().catch((retryErr) => {
                 hlog('sdk resume retry failed', {
-                    err: retryErr instanceof Error ? retryErr.message : String(retryErr)
+                    err: errorMessage(retryErr)
                 });
             });
         }
@@ -695,15 +778,38 @@ export class Harness extends ReadyResource {
             if (outcome.status !== 'rejected')
                 continue;
             const err = outcome.reason;
-            hlog('unload failed', { err: err instanceof Error ? err.message : String(err) });
+            hlog('unload failed', { err: errorMessage(err) });
+        }
+    }
+    async _unloadIdleModels() {
+        if (!this.opened || this.closing || this.suspended)
+            return;
+        hlog('idle unload', { models: this._loadedModels().map((entry) => entry.model) });
+        const released = await Promise.allSettled([
+            evictIdle(this._models),
+            evictIdle(this._embedModels),
+            ...[...this._retiredEmbedModels].map((pool) => this._collectRetiredEmbedPool(pool)),
+            evictIdle(this._ttsModels),
+            evictIdle(this._transcriptionModels),
+            this._imageRuntime?.unloadIdle(),
+            this._musicRuntime?.unloadIdle(),
+            this._translationRuntime.unload()
+        ]);
+        for (const outcome of released) {
+            if (outcome.status !== 'rejected')
+                continue;
+            hlog('idle unload failed', { err: errorMessage(outcome.reason) });
         }
+        invalidateModelStatusCache(this._modelStatus);
     }
     async resume() {
         hlog('resume called');
-        if (this._state === HarnessState.RESUME)
+        if (this._state === HarnessState.RESUME) {
             return;
-        if (this.closing)
+        }
+        if (this.closing) {
             return;
+        }
         if (this._state === HarnessState.PRESUSPEND) {
             this._state = HarnessState.RESUME;
             return;
@@ -726,22 +832,26 @@ export class Harness extends ReadyResource {
     }
     async _close() {
         hlog('closing');
+        this._idle.close();
         this._overlayWatcher?.stop();
         this._overlayWatcher = null;
         const pending = this._sdk.peek();
-        if (pending)
+        if (pending) {
             this._loads.drain(await pending);
+        }
         await Promise.all([this._releaseRuntimes(), closePythonWorker()]);
         const sdk = await this._sdk.peek()?.catch(noop);
-        if (sdk)
+        if (sdk) {
             await sdk.close();
+        }
     }
     download(args) {
         const projection = (args.addon ?? 'llm') === 'llm' ? this._projectionFor(args.model) : undefined;
         const multiFile = diffusionDownloadMetadata(args.model, args.addon) ??
             audioGenDownloadMetadata(args.model, args.addon);
         return download(async () => {
-            await this.ready();
+            if (!this.opened)
+                await this.ready();
             return this._sdk.load();
         }, {
             ...args,
@@ -750,12 +860,17 @@ export class Harness extends ReadyResource {
             assetSizes: multiFile?.assetSizes
         }, () => invalidateModelStatusCache(this._modelStatus));
     }
-    async embed({ model, texts, signal }) {
-        await this.ready();
+    embed(context) {
+        return this._busy(() => this._embed(context));
+    }
+    async _embed({ model, texts, signal }) {
+        if (!this.opened)
+            await this.ready();
         if (this.suspended)
             throw new Error('harness is suspended');
-        if (texts.length === 0)
+        if (texts.length === 0) {
             return [];
+        }
         const pool = this._embedModels;
         const faults = this._embedFaults;
         const generation = faults.generation;
@@ -766,7 +881,7 @@ export class Harness extends ReadyResource {
         let releaseAfter = null;
         try {
             loaded = await raceAbort(handle.load(), signal);
-            const sdk = (await this._sdk.loadEmbed());
+            const sdk = await this._sdk.loadEmbed();
             job = this._embedLane(sdk, { modelId: loaded.modelId, texts });
             const vectors = await raceAbort(job, signal);
             faults.reset();
@@ -789,15 +904,18 @@ export class Harness extends ReadyResource {
             throw err;
         }
         finally {
-            if (releaseAfter)
+            if (releaseAfter) {
                 this._releaseEmbedResourceAfter(releaseAfter, pool, key);
-            else
+            }
+            else {
                 await this._releaseEmbedResource(pool, key);
+            }
         }
     }
     async _defaultVoiceSrc(category) {
-        if (category !== 'tts')
+        if (category !== 'tts') {
             return defaultVoiceModel(category);
+        }
         try {
             const { capabilityProfile } = await this.capability();
             return defaultTtsFor(capabilityProfile).asset.src;
@@ -806,8 +924,9 @@ export class Harness extends ReadyResource {
             return defaultVoiceModel('tts');
         }
     }
-    async _voiceJob(pool, category, model, signal, run) {
-        await this.ready();
+    async _voiceJob(pool, { category, model, signal }, run) {
+        if (!this.opened)
+            await this.ready();
         if (this.suspended)
             throw new Error('harness is suspended');
         const name = model || (await this._defaultVoiceSrc(category));
@@ -826,8 +945,9 @@ export class Harness extends ReadyResource {
         }
         catch (err) {
             if (err instanceof AbortedError) {
-                if (sdk && modelId)
+                if (sdk && modelId) {
                     void sdk.cancel({ modelId, operation: 'inference' }).catch(noop);
+                }
                 releaseAfter = job;
             }
             throw err;
@@ -836,19 +956,25 @@ export class Harness extends ReadyResource {
             const release = () => {
                 releaseModel(pool, name);
             };
-            if (releaseAfter)
+            if (releaseAfter) {
                 void releaseAfter.then(release, release);
-            else
+            }
+            else {
                 release();
+            }
         }
     }
-    async transcribe({ model, audio, signal }) {
-        return this._voiceJob(this._transcriptionModels, 'transcription', model, signal, async (sdk, id) => {
+    transcribe(context) {
+        return this._busy(() => this._transcribe(context));
+    }
+    async _transcribe({ model, audio, signal }) {
+        return this._voiceJob(this._transcriptionModels, { category: 'transcription', model, signal }, async (sdk, id) => {
             const pcm = await pcmForTranscription(audio);
-            if (signal?.aborted)
+            if (signal?.aborted) {
                 throw new AbortedError();
+            }
             if (pcm && pcm.length >= TRANSCRIBE_STREAM_MIN_BYTES) {
-                return transcribeStreaming(sdk, id, pcm, signal);
+                return transcribeStreaming(sdk, pcm, { modelId: id, signal });
             }
             return transcribeOnce(sdk, id, pcm ? buildWavBuffer(pcm) : audio);
         });
@@ -856,12 +982,19 @@ export class Harness extends ReadyResource {
     detectLanguage(context) {
         return detectLanguage(context);
     }
-    async speak({ model, text, signal }) {
-        const samples = await this._voiceJob(this._ttsModels, 'tts', model, signal, (sdk, id) => speakOnce(sdk, id, text));
+    speak(context) {
+        return this._busy(() => this._speak(context));
+    }
+    async _speak({ model, text, signal }) {
+        const samples = await this._voiceJob(this._ttsModels, { category: 'tts', model, signal }, (sdk, id) => speakOnce(sdk, id, text));
         return { audio: packSamples(samples), sampleRate: SUPERTONIC_SAMPLE_RATE };
     }
-    async *speakStream({ model, text, signal }) {
-        await this.ready();
+    speakStream(context) {
+        return this._busyStream(() => this._speakStream(context));
+    }
+    async *_speakStream({ model, text, signal }) {
+        if (!this.opened)
+            await this.ready();
         if (this.suspended)
             throw new Error('harness is suspended');
         const name = model || (await this._defaultVoiceSrc('tts'));
@@ -871,18 +1004,21 @@ export class Harness extends ReadyResource {
         let sdk = null;
         let modelId = null;
         const cancel = () => {
-            if (sdk && modelId)
-                void sdk.cancel({ modelId, operation: 'inference' }).catch(() => { });
+            if (sdk && modelId) {
+                void sdk.cancel({ modelId, operation: 'inference' }).catch(noop);
+            }
         };
         signal?.addEventListener('abort', cancel, { once: true });
         const job = this._voiceLane(async () => {
             try {
                 modelId = (await handle.load()).modelId;
                 sdk = (await this._sdk.loadVoice());
-                if (signal?.aborted)
+                if (signal?.aborted) {
                     return;
-                for await (const chunk of speakStreamOnce(sdk, modelId, text))
+                }
+                for await (const chunk of speakStreamOnce(sdk, modelId, text)) {
                     chunks.push(chunk);
+                }
             }
             finally {
                 chunks.end();
@@ -896,31 +1032,46 @@ export class Harness extends ReadyResource {
         }
         finally {
             signal?.removeEventListener('abort', cancel);
-            if (!settled)
+            if (!settled) {
                 cancel();
+            }
             void job.catch(noop).finally(() => {
                 releaseModel(pool, name);
             });
         }
     }
     async list(options = {}) {
-        await this.ready();
-        const sdk = (await this._sdk.load());
-        return listModels(sdk, options, this._modelStatus, (name) => this._projectionFor(name));
+        if (!this.opened)
+            await this.ready();
+        const sdk = await this._sdk.load();
+        const [rows, custom] = await Promise.all([
+            listModels(sdk, options, {
+                cache: this._modelStatus,
+                projectionFor: (name) => this._projectionFor(name)
+            }),
+            customModelRows(options, {
+                modelsDir: this._modelsDir,
+                loaded: new Set(this._loadedModels().map((entry) => entry.model))
+            })
+        ]);
+        return mergeCustomRows(rows, custom);
     }
     async deleteModel(name) {
-        await this.ready();
-        const sdk = (await this._sdk.load());
-        const result = await deleteModelFiles(sdk, name);
+        if (!this.opened)
+            await this.ready();
+        const sdk = await this._sdk.load();
+        const result = await deleteModelFiles(sdk, name, { modelsDir: this._modelsDir });
         invalidateModelStatusCache(this._modelStatus);
         return summarizeDelete(result);
     }
     async clearKvCache() {
-        await this.ready();
+        if (!this.opened)
+            await this.ready();
         return deleteAllKvCaches(() => this._sdk.load());
     }
     async deleteKvCache(kvCacheKey) {
-        await this.ready();
+        if (!this.opened)
+            await this.ready();
         return deleteKvCacheKey(() => this._sdk.load(), kvCacheKey);
     }
     async infraModels(profile) {
@@ -964,8 +1115,12 @@ export class Harness extends ReadyResource {
         });
         return this._capabilityInflight;
     }
-    async warmModel({ model, agent, signal }) {
-        await this.ready();
+    warmModel(context) {
+        return this._busy(() => this._warmModel(context));
+    }
+    async _warmModel({ model, agent, signal }) {
+        if (!this.opened)
+            await this.ready();
         if (this.suspended)
             return;
         const projectionModel = this._projectionFor(model);
@@ -979,8 +1134,9 @@ export class Harness extends ReadyResource {
         const pool = this._models;
         hlog('warm model', { model, resident: isResident(pool, key) });
         if (!isResident(pool, key)) {
-            if (!(await this._makeRoomForCompletion(signal, WARM_ROOM_WAIT_MS)))
+            if (!(await this._makeRoomForCompletion(signal, WARM_ROOM_WAIT_MS))) {
                 return;
+            }
             await this._imageRuntime?.unloadIdle().catch(noop);
         }
         try {
@@ -990,11 +1146,15 @@ export class Harness extends ReadyResource {
             releaseModel(pool, key);
         }
     }
-    async *invoke({ capability, args, enhance, agent, attach, fetchAttachment, progress, signal }) {
+    invoke(context) {
+        return this._busyStream(() => this._invoke(context));
+    }
+    async *_invoke({ capability, args, enhance, agent, attach, fetchAttachment, progress, signal }) {
         if (!this.opened)
             await this.ready();
-        if (this.suspended)
+        if (this.suspended) {
             return yield { type: 'aborted' };
+        }
         hlog('invoke start', { capability, enhance });
         try {
             yield* invoke({
@@ -1016,8 +1176,12 @@ export class Harness extends ReadyResource {
             hlog('invoke end', { capability });
         }
     }
-    async *completion({ model, agent, messages, cacheKey, thinkingDefault, toolConfigs, generationParams, responseFormat, captureInferenceCalls, grantedDirs, grantedDrives, driveOp, grantedWriteDirs, approve, attach, fetchAttachment, ragSearch, progress, askUser, signal }) {
-        await this.ready();
+    completion(context) {
+        return this._busyStream(() => this._completion(context));
+    }
+    async *_completion({ model, agent, messages, cacheKey, thinkingDefault, toolConfigs, generationParams, responseFormat, captureInferenceCalls, grantedDirs, grantedDrives, driveOp, grantedWriteDirs, approve, attach, fetchAttachment, ragSearch, progress, askUser, signal }) {
+        if (!this.opened)
+            await this.ready();
         if (this.suspended)
             throw new Error('harness is suspended');
         const projectionModel = this._projectionFor(model);
@@ -1112,16 +1276,12 @@ export class Harness extends ReadyResource {
                 yield { type: 'aborted' };
             }
             else {
-                const message = err instanceof Error ? err.message : String(err);
+                const message = errorMessage(err);
                 hlog('completion failed', { model, ...resolutionDetail(err), err: message });
-                if (isSdkModelWedgedError(err)) {
-                    const wedged = await this._evictWedgedModel(model, key);
-                    yield { type: 'error', message: wedged.message };
-                    if (wedged.fatal)
-                        process.exit(1);
-                }
-                else {
-                    yield { type: 'error', message };
+                const wedged = isSdkModelWedgedError(err) ? await this._evictWedgedModel(model, key) : null;
+                yield { type: 'error', message: wedged ? wedged.message : message };
+                if (wedged?.fatal) {
+                    process.exit(1);
                 }
             }
         }
@@ -1135,14 +1295,19 @@ export class Harness extends ReadyResource {
     _residentEnhancer() {
         return async (prompt, signal) => {
             const lease = await leaseResidentModel(this._models, (model) => !isTranslationModel(model));
-            if (!lease)
+            if (!lease) {
                 return null;
+            }
             try {
                 const sdk = await this._sdk.load();
-                return await expandImagePrompt(sdk, lease.modelId, lease.model, prompt, signal);
+                return await expandImagePrompt(sdk, prompt, {
+                    modelId: lease.modelId,
+                    model: lease.model,
+                    signal
+                });
             }
             catch (err) {
-                hlog('enhance failed', { err: err instanceof Error ? err.message : String(err) });
+                hlog('enhance failed', { err: errorMessage(err) });
                 return null;
             }
             finally {
@@ -1151,7 +1316,11 @@ export class Harness extends ReadyResource {
         };
     }
     async _makeRoomForCompletion(signal, deadlineMs = MODEL_ROOM_WAIT_MS) {
-        const roomed = await awaitModelCapacity(this._models, this._completionModelCap, deadlineMs, 250, signal);
+        const roomed = await awaitModelCapacity(this._models, {
+            cap: this._completionModelCap,
+            deadlineMs,
+            signal
+        });
         if (!roomed) {
             hlog('model room wait gave up', {
                 cap: this._completionModelCap,
@@ -1164,10 +1333,13 @@ export class Harness extends ReadyResource {
         const faults = (this._modelFaults.get(model) ?? 0) + 1;
         this._modelFaults.set(model, faults);
         const leased = leaseCount(this._models, key) > 0;
-        const evicted = leased ? false : await evictModelWithin(this._models, key, WEDGED_EVICT_WAIT_MS);
+        const evicted = leased
+            ? false
+            : await evictModelWithin(this._models, key, { deadlineMs: WEDGED_EVICT_WAIT_MS });
         hlog('wedged model evicted', { model, faults, evicted, leased });
-        if (leased)
+        if (leased) {
             this._evictAfterLeases(model, key);
+        }
         if (!evicted && !leased) {
             return {
                 message: `${model} is stuck in device memory — restarting the model runtime`,
@@ -1182,9 +1354,27 @@ export class Harness extends ReadyResource {
         };
     }
     _evictAfterLeases(model, key) {
-        void evictModelWithin(this._models, key, WEDGED_EVICT_LEASED_MS)
+        void evictModelWithin(this._models, key, { deadlineMs: WEDGED_EVICT_LEASED_MS })
             .then((evicted) => hlog('wedged model evicted after leases', { model, evicted }))
             .catch(noop);
     }
+    async _busy(run) {
+        await this._idle.enter();
+        try {
+            return await run();
+        }
+        finally {
+            this._idle.exit();
+        }
+    }
+    async *_busyStream(run) {
+        await this._idle.enter();
+        try {
+            yield* run();
+        }
+        finally {
+            this._idle.exit();
+        }
+    }
 }
 function noop() { }