@qvac/harness 0.5.0 → 0.6.0

package/dist-lib/index.d.ts

@@ -10,7 +10,7 @@ import { type EmbedContext } from './lib/capabilities/embed.ts';
 import { type KvCacheDeletionResult } from './lib/capabilities/kv-cache.ts';
 import type { LlmGenerationParams } from './lib/models/llm-presets.ts';
 import { type SpeakContext, type SpeakResult, type SpeakStreamContext, type TranscribeContext } from './lib/capabilities/voice.ts';
-import type { CapabilityInfo, CredentialEntry, HarnessDiagnostics, InfraModel, SkillsOptions, SkillWriteResult } from './lib/contract-types.ts';
+import type { CapabilityInfo, CredentialEntry, HarnessDiagnostics, InfraModel, LoadModelResult, SkillsOptions, SkillWriteResult, UnloadModelContext, UnloadModelResult } from './lib/contract-types.ts';
 import { type CustomModelEntry } from './lib/models/model-catalog.ts';
 import { type ListModelsOptions, type ModelInfo } from './lib/models/model-list.ts';
 import { type DeleteModelSummary } from './lib/models/model-delete.ts';
@@ -138,6 +138,8 @@ export declare class Harness extends ReadyResource {
     private _tools;
     private _sdk;
     private readonly _loads;
+    private readonly _pinned;
+    private readonly _replying;
     private readonly _modelStatus;
     private _models;
     private _embedModels;
@@ -211,6 +213,7 @@ export declare class Harness extends ReadyResource {
     setCustomModels(entries: ReadonlyArray<CustomModelEntry>): void;
     setModelIdleTimeout(ms: number): void;
     previewModelSource(context: PreviewModelSourceContext): Promise<ModelSourcePreview>;
+    private _assessFit;
     describe(): Promise<HarnessInfo>;
     diagnostics(): HarnessDiagnostics;
     private _loadedModels;
@@ -261,6 +264,16 @@ export declare class Harness extends ReadyResource {
     capability(): Promise<CapabilityInfo>;
     warmModel(context: WarmModelOptions): Promise<void>;
     private _warmModel;
+    loadModel(context: WarmModelOptions): Promise<LoadModelResult>;
+    private _loadModel;
+    unloadModel(context: UnloadModelContext): Promise<UnloadModelResult>;
+    private _unloadModel;
+    private _loadAndRelease;
+    private _unloadIdleGenerators;
+    private _fits;
+    private _yieldPins;
+    private _residency;
+    private _completionLoadOptions;
     invoke(context: InvokeContext): AsyncGenerator<CompletionEvent>;
     private _invoke;
     completion(context: CompletionContext): AsyncGenerator<CompletionEvent>;

package/dist-lib/index.js

@@ -21,10 +21,11 @@ import { buildWavBuffer } from './lib/wav-encode.js';
 import { createVoiceLane, packSamples, speakOnce, speakStreamOnce, transcribeOnce, transcribeStreaming, SUPERTONIC_SAMPLE_RATE } from './lib/capabilities/voice.js';
 import { pushQueue } from './lib/queue.js';
 import { AbortedError, raceAbort } from './lib/helpers/race-abort.js';
-import { audioGenDownloadMetadata, customProjectionMap, defaultProjectionMap, defaultTtsFor, defaultVoiceModel, diffusionDownloadMetadata, infraModelsFor, modelNameToCapabilityProfile, musicModelSet, setCustomModels } from './lib/models/model-catalog.js';
+import { audioGenDownloadMetadata, customProjectionMap, defaultProjectionMap, defaultTtsFor, defaultVoiceModel, diffusionDownloadMetadata, curatedModelAsset, infraModelsFor, modelAddonOf, modelNameToCapabilityProfile, musicModelSet, residentSrc, setCustomModels } from './lib/models/model-catalog.js';
 import { createModelStatusCache, customModelRows, invalidateModelStatusCache, mergeCustomRows, listModels } from './lib/models/model-list.js';
 import { deleteModelFiles, summarizeDelete } from './lib/models/model-delete.js';
-import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, awaitModelCapacity, evictIdle, evictModelWithin, isResident, leaseResidentModel, leaseCount, loadedModels, modelKey, releaseModel, resolveLoadOptions, progressState } from './lib/models/model-pool.js';
+import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, awaitModelCapacity, evictIdle, evictIdleWhere, evictModel, evictModelWithin, isResident, leaseResidentModel, leaseCount, loadedModels, modelKey, modelOf, releaseModel, resolveLoadOptions, progressState } from './lib/models/model-pool.js';
+import { defaultContextSize } from './lib/models/context.js';
 import { resolveMainGpu } from './lib/models/gpu-selection.js';
 import { createIdleReaper } from './lib/models/idle-unload.js';
 import { openclCacheDir } from './lib/models/opencl-cache.js';
@@ -81,6 +82,7 @@ const MUSIC_EVICT_WAIT_MS = 15_000;
 const WEDGED_EVICT_WAIT_MS = 10_000;
 const MODEL_ROOM_WAIT_MS = 300_000;
 const WARM_ROOM_WAIT_MS = 2_000;
+const UNLOAD_LEASE_WAIT_MS = 10_000;
 const DEFAULT_COMPLETION_MODEL_CAP = 1;
 const WEDGED_EVICT_LEASED_MS = 120_000;
 const TTS_MODEL_CONFIG = {
@@ -93,6 +95,8 @@ export class Harness extends ReadyResource {
     _tools;
     _sdk;
     _loads = createLoadTracker();
+    _pinned = new Set();
+    _replying = [];
     _modelStatus = createModelStatusCache();
     _models;
     _embedModels;
@@ -520,19 +524,18 @@ export class Harness extends ReadyResource {
         this._idle.configure(ms);
     }
     previewModelSource(context) {
-        return previewModelSource(context, {
-            assessFit: async ({ sha256Checksum, name, contextTokens }) => {
-                const sdk = await this._sdk.load();
-                const result = await sdk.assessModelFit({
-                    models: [{ model: { sha256Checksum, name }, workload: { kind: 'llm', contextTokens } }]
-                });
-                const model = result.models[0];
-                return {
-                    fit: model?.verdict ?? result.verdict,
-                    fitReasons: [...(model?.reasons ?? result.reasons)]
-                };
-            }
+        return previewModelSource(context, { assessFit: (query) => this._assessFit(query) });
+    }
+    async _assessFit({ sha256Checksum, name, contextTokens }) {
+        const sdk = await this._sdk.load();
+        const result = await sdk.assessModelFit({
+            models: [{ model: { sha256Checksum, name }, workload: { kind: 'llm', contextTokens } }]
         });
+        const model = result.models[0];
+        return {
+            fit: model?.verdict ?? result.verdict,
+            fitReasons: [...(model?.reasons ?? result.reasons)]
+        };
     }
     async describe() {
         if (!this.opened)
@@ -595,7 +598,7 @@ export class Harness extends ReadyResource {
         const key = modelKey(model, loadOpts);
         const pool = this._models;
         if (!isResident(pool, key)) {
-            await this._makeRoomForCompletion(signal);
+            await this._makeRoomForCompletion(model, { ctxSize: loadOpts.ctxSize, signal });
         }
         try {
             const { modelId } = await raceAbort(pool.open(key, model, loadOpts).load(), signal);
@@ -630,7 +633,7 @@ export class Harness extends ReadyResource {
     }
     _createPool() {
         const loadSdk = this._sdk.load;
-        return createModelPool(loadSdk, (state) => this._notifyModel(state, { languageModel: true }), {
+        return createModelPool(loadSdk, (state) => this._notifyModel(state), {
             deviceConfig: this._deviceConfig(),
             loads: this._loads
         });
@@ -663,11 +666,18 @@ export class Harness extends ReadyResource {
             loads: this._loads
         });
     }
-    _notifyModel(state, { languageModel = false } = {}) {
-        const catalogChanged = languageModel && (state.status === 'ready' || state.status === 'unloaded');
+    _notifyModel(state) {
+        const catalogChanged = state.status === 'ready' || state.status === 'unloaded';
         if (catalogChanged) {
             invalidateModelStatusCache(this._modelStatus);
         }
+        if (state.status === 'unloaded') {
+            for (const key of this._pinned) {
+                if (!isResident(this._models, key)) {
+                    this._pinned.delete(key);
+                }
+            }
+        }
         this.emit('model', { ...state, catalogChanged, loaded: this._loadedModels() });
     }
     async _collectRetiredEmbedPool(pool) {
@@ -874,7 +884,7 @@ export class Harness extends ReadyResource {
         const pool = this._embedModels;
         const faults = this._embedFaults;
         const generation = faults.generation;
-        const key = faults.key(model);
+        const key = faults.key(residentSrc(model, 'embeddings'));
         const handle = pool.open(key, model);
         let loaded = null;
         let job = null;
@@ -929,7 +939,8 @@ export class Harness extends ReadyResource {
             await this.ready();
         if (this.suspended)
             throw new Error('harness is suspended');
-        const name = model || (await this._defaultVoiceSrc(category));
+        const addon = category === 'tts' ? 'tts' : 'parakeet';
+        const name = residentSrc(model || (await this._defaultVoiceSrc(category)), addon);
         const handle = pool.open(name, name);
         let job = null;
         let releaseAfter = null;
@@ -997,7 +1008,7 @@ export class Harness extends ReadyResource {
             await this.ready();
         if (this.suspended)
             throw new Error('harness is suspended');
-        const name = model || (await this._defaultVoiceSrc('tts'));
+        const name = residentSrc(model || (await this._defaultVoiceSrc('tts')), 'tts');
         const pool = this._ttsModels;
         const handle = pool.open(name, name);
         const chunks = pushQueue();
@@ -1123,29 +1134,143 @@ export class Harness extends ReadyResource {
             await this.ready();
         if (this.suspended)
             return;
-        const projectionModel = this._projectionFor(model);
-        const loadOpts = resolveLoadOptions({
-            ctxSize: agent?.ctxSize ?? undefined,
-            kvCacheK: agent?.kvCacheK ?? undefined,
-            kvCacheV: agent?.kvCacheV ?? undefined,
-            projectionModel
-        }, this._deviceConfig());
-        const key = modelKey(model, loadOpts);
-        const pool = this._models;
-        hlog('warm model', { model, resident: isResident(pool, key) });
-        if (!isResident(pool, key)) {
-            if (!(await this._makeRoomForCompletion(signal, WARM_ROOM_WAIT_MS))) {
+        const target = this._residency(model, agent);
+        const resident = isResident(target.pool, target.key);
+        hlog('warm model', { model, resident });
+        if (target.pool === this._models && !resident) {
+            const roomed = await this._makeRoomForCompletion(model, {
+                ctxSize: target.ctxSize,
+                signal,
+                deadlineMs: WARM_ROOM_WAIT_MS
+            });
+            if (!roomed) {
                 return;
             }
-            await this._imageRuntime?.unloadIdle().catch(noop);
+            await this._unloadIdleGenerators();
+        }
+        await this._loadAndRelease(target, signal);
+    }
+    loadModel(context) {
+        return this._busy(() => this._loadModel(context));
+    }
+    async _loadModel({ model, agent, signal }) {
+        if (!this.opened)
+            await this.ready();
+        if (this.suspended) {
+            throw new Error('harness is suspended');
+        }
+        const target = this._residency(model, agent);
+        const resident = isResident(target.pool, target.key);
+        hlog('load model', { model, resident });
+        if (target.pool !== this._models) {
+            await this._loadAndRelease(target, signal);
+            return { loaded: true };
+        }
+        if (!resident) {
+            if (!(await this._fits(model, target.ctxSize))) {
+                return { loaded: false, reason: 'memory' };
+            }
+            await evictIdleWhere(this._models, (key) => modelOf(key) === model && !this._replying.includes(key));
+            await this._unloadIdleGenerators();
+        }
+        this._pinned.add(target.key);
+        try {
+            await this._loadAndRelease(target, signal);
+        }
+        catch (err) {
+            this._pinned.delete(target.key);
+            throw err;
+        }
+        return { loaded: true };
+    }
+    unloadModel(context) {
+        return this._busy(() => this._unloadModel(context));
+    }
+    async _unloadModel({ model }) {
+        if (!this.opened)
+            await this.ready();
+        const target = this._residency(model);
+        const name = modelOf(target.key);
+        const unloaded = await evictModel(target.pool, name, { deadlineMs: UNLOAD_LEASE_WAIT_MS });
+        hlog('unload model', { model, unloaded });
+        if (unloaded) {
+            for (const key of this._pinned) {
+                if (modelOf(key) === name) {
+                    this._pinned.delete(key);
+                }
+            }
         }
+        return { unloaded };
+    }
+    async _loadAndRelease(target, signal) {
         try {
-            await raceAbort(pool.open(key, model, loadOpts).load(), signal);
+            await raceAbort(target.open().load(), signal);
         }
         finally {
-            releaseModel(pool, key);
+            releaseModel(target.pool, target.key);
         }
     }
+    async _unloadIdleGenerators() {
+        await this._imageRuntime?.unloadIdle().catch(noop);
+        await this._musicRuntime?.unloadIdle().catch(noop);
+    }
+    async _fits(model, ctxSize) {
+        const asset = curatedModelAsset(model, 'llm');
+        if (!asset) {
+            return true;
+        }
+        const assessed = await this._assessFit({
+            sha256Checksum: asset.sha256Checksum,
+            name: asset.name,
+            contextTokens: ctxSize || defaultContextSize(asset)
+        }).catch((err) => {
+            hlog('fit estimate failed', { model, err: errorMessage(err) });
+            return null;
+        });
+        return assessed?.fit !== 'likely-too-large';
+    }
+    async _yieldPins(model, ctxSize) {
+        for (const key of this._pinned) {
+            if (leaseCount(this._models, key) > 0 || this._replying.includes(key)) {
+                continue;
+            }
+            if (await this._fits(model, ctxSize))
+                return;
+            this._pinned.delete(key);
+            await evictModelWithin(this._models, key, { deadlineMs: WEDGED_EVICT_WAIT_MS });
+        }
+    }
+    _residency(model, agent) {
+        const addon = modelAddonOf(model);
+        if (addon === 'llm') {
+            const loadOpts = this._completionLoadOptions(model, agent);
+            const key = modelKey(model, loadOpts);
+            return {
+                pool: this._models,
+                key,
+                ctxSize: loadOpts.ctxSize,
+                open: () => this._models.open(key, model, loadOpts)
+            };
+        }
+        if (addon === 'embeddings') {
+            const key = this._embedFaults.key(residentSrc(model, addon));
+            return { pool: this._embedModels, key, open: () => this._embedModels.open(key, model) };
+        }
+        if (addon === 'tts' || addon === 'parakeet') {
+            const pool = addon === 'tts' ? this._ttsModels : this._transcriptionModels;
+            const key = residentSrc(model, addon);
+            return { pool, key, open: () => pool.open(key, key) };
+        }
+        throw new Error(`${model} loads for each generation and is not kept in memory`);
+    }
+    _completionLoadOptions(model, agent) {
+        return resolveLoadOptions({
+            ctxSize: agent?.ctxSize ?? undefined,
+            kvCacheK: agent?.kvCacheK ?? undefined,
+            kvCacheV: agent?.kvCacheV ?? undefined,
+            projectionModel: this._projectionFor(model)
+        }, this._deviceConfig());
+    }
     invoke(context) {
         return this._busyStream(() => this._invoke(context));
     }
@@ -1184,27 +1309,20 @@ export class Harness extends ReadyResource {
             await this.ready();
         if (this.suspended)
             throw new Error('harness is suspended');
-        const projectionModel = this._projectionFor(model);
-        const loadOpts = resolveLoadOptions({
-            ctxSize: agent?.ctxSize ?? undefined,
-            kvCacheK: agent?.kvCacheK ?? undefined,
-            kvCacheV: agent?.kvCacheV ?? undefined,
-            projectionModel
-        }, this._deviceConfig());
+        const loadOpts = this._completionLoadOptions(model, agent);
+        const projectionModel = loadOpts.projectionModel;
         const key = modelKey(model, loadOpts);
         const pool = this._models;
         const resident = isResident(pool, key);
         hlog('completion start', { model, resident, projection: Boolean(projectionModel) });
         if (!resident) {
-            await this._makeRoomForCompletion(signal);
-            await this._imageRuntime?.unloadIdle().catch(noop);
-            await this._musicRuntime?.unloadIdle().catch(noop);
+            await this._makeRoomForCompletion(model, { ctxSize: loadOpts.ctxSize, signal });
+            await this._unloadIdleGenerators();
         }
         const acquireRound = async () => {
             if (!isResident(pool, key)) {
-                await this._makeRoomForCompletion(signal);
-                await this._imageRuntime?.unloadIdle().catch(noop);
-                await this._musicRuntime?.unloadIdle().catch(noop);
+                await this._makeRoomForCompletion(model, { ctxSize: loadOpts.ctxSize, signal });
+                await this._unloadIdleGenerators();
             }
             try {
                 const { modelId } = await raceAbort(pool.open(key, model, loadOpts).load(), signal);
@@ -1228,6 +1346,7 @@ export class Harness extends ReadyResource {
             held = false;
             releaseModel(pool, key);
         };
+        this._replying.push(key);
         try {
             const { modelId, ctxSize } = await raceAbort(handle.load(), signal);
             hlog('model ready', { model });
@@ -1287,6 +1406,7 @@ export class Harness extends ReadyResource {
         }
         finally {
             hlog('completion end', { model });
+            this._replying.splice(this._replying.indexOf(key), 1);
             releaseHeld();
             void this._imageRuntime?.unloadIdle().catch(noop);
             void this._musicRuntime?.unloadIdle().catch(noop);
@@ -1315,11 +1435,12 @@ export class Harness extends ReadyResource {
             }
         };
     }
-    async _makeRoomForCompletion(signal, deadlineMs = MODEL_ROOM_WAIT_MS) {
+    async _makeRoomForCompletion(model, { ctxSize, signal, deadlineMs = MODEL_ROOM_WAIT_MS } = {}) {
         const roomed = await awaitModelCapacity(this._models, {
             cap: this._completionModelCap,
             deadlineMs,
-            signal
+            signal,
+            evictable: (key) => !this._pinned.has(key) || modelOf(key) === model
         });
         if (!roomed) {
             hlog('model room wait gave up', {
@@ -1327,6 +1448,7 @@ export class Harness extends ReadyResource {
                 loaded: loadedModels(this._models).length
             });
         }
+        await this._yieldPins(model, ctxSize);
         return roomed;
     }
     async _evictWedgedModel(model, key) {

package/dist-lib/lib/connect.d.ts

@@ -3,7 +3,7 @@ import { type Transport } from './rpc-session.ts';
 import { type KvCacheDeletionResult } from './capabilities/kv-cache.ts';
 import type { CustomModelEntry } from './models/model-catalog.ts';
 import { type ModelSourcePreview, type PreviewModelSourceContext } from './capabilities/preview-model-source.ts';
-import type { CapabilityInfo, CompletionContext, CredentialEntry, DetectedLanguage, DetectLanguageContext, DownloadContext, EmbedContext, HarnessContract, HarnessDiagnostics, HarnessEvent, HarnessInfo, InfraModel, InvokeContext, ListModelsOptions, ModelInfo, ResolvedSkillCatalogEntry, SkillsOptions, SkillWriteResult, SpeakContext, SpeakResult, SpeakStreamContext, ToolCatalogEntry, TranscribeContext, WarmModelContext } from './contract-types.ts';
+import type { CapabilityInfo, CompletionContext, CredentialEntry, DetectedLanguage, DetectLanguageContext, DownloadContext, EmbedContext, HarnessContract, HarnessDiagnostics, HarnessEvent, HarnessInfo, InfraModel, InvokeContext, ListModelsOptions, ModelInfo, ResolvedSkillCatalogEntry, SkillsOptions, SkillWriteResult, SpeakContext, SpeakResult, SpeakStreamContext, ToolCatalogEntry, TranscribeContext, LoadModelContext, LoadModelResult, UnloadModelContext, UnloadModelResult, WarmModelContext } from './contract-types.ts';
 import { type EmbedDevice } from './models/embed-config.ts';
 import type { DeleteModelSummary } from './models/model-delete.ts';
 import type { CopyBaseResult, DeleteOverlayResult, ManagedSkill, SkillFileContent } from './tools/skills/overlay-crud.ts';
@@ -46,6 +46,8 @@ export declare class RemoteHarness extends EventEmitter implements HarnessContra
     skills(agentId?: string | null, modelName?: string | null, options?: SkillsOptions): Promise<ResolvedSkillCatalogEntry[]>;
     tools(): Promise<ToolCatalogEntry[]>;
     warmModel(context: WarmModelContext): Promise<void>;
+    loadModel(context: LoadModelContext): Promise<LoadModelResult>;
+    unloadModel(context: UnloadModelContext): Promise<UnloadModelResult>;
     list(options?: ListModelsOptions): Promise<ModelInfo[]>;
     deleteModel(name: string): Promise<DeleteModelSummary>;
     clearKvCache(): Promise<KvCacheDeletionResult>;

package/dist-lib/lib/connect.js

@@ -540,6 +540,24 @@ export class RemoteHarness extends EventEmitter {
             agent: context.agent ? knobs(context.agent) : undefined
         }), context.signal);
     }
+    async loadModel(context) {
+        const { client } = await this._ensure();
+        const response = await abortable(client.loadModel({
+            model: context.model,
+            agent: context.agent ? knobs(context.agent) : undefined
+        }), context.signal);
+        return {
+            loaded: response.loaded === true,
+            reason: response.reason === 'memory' ? 'memory' : undefined
+        };
+    }
+    async unloadModel(context) {
+        const { client } = await this._ensure();
+        const response = await client.unloadModel({
+            model: context.model
+        });
+        return { unloaded: response.unloaded === true };
+    }
     async list(options = {}) {
         const { client } = await this._ensure();
         const response = await client.listModels({

package/dist-lib/lib/contract-types.d.ts

@@ -113,6 +113,17 @@ export interface WarmModelContext {
     agent?: AgentKnobs;
     signal?: CompletionContext['signal'];
 }
+export type LoadModelContext = WarmModelContext;
+export interface LoadModelResult {
+    loaded: boolean;
+    reason?: 'memory';
+}
+export interface UnloadModelContext {
+    model: string;
+}
+export interface UnloadModelResult {
+    unloaded: boolean;
+}
 export interface DownloadContext {
     model: string;
     addon?: string;
@@ -224,6 +235,8 @@ export interface HarnessContract {
     }>;
     list?(options?: ListModelsOptions): Promise<ModelInfo[]>;
     warmModel?(context: WarmModelContext): Promise<void>;
+    loadModel?(context: LoadModelContext): Promise<LoadModelResult>;
+    unloadModel?(context: UnloadModelContext): Promise<UnloadModelResult>;
     deleteModel?(name: string): Promise<DeleteModelSummary>;
     clearKvCache?(): Promise<KvCacheDeletionResult>;
     deleteKvCache?(kvCacheKey: string): Promise<KvCacheDeletionResult>;

package/dist-lib/lib/models/model-catalog.d.ts

@@ -95,6 +95,8 @@ export interface CuratedListable {
 }
 export declare function curatedListables(): CuratedListable[];
 export declare function curatedModelAsset(name: string, addon: ModelRegistryEntry['addon']): SdkModelAsset | undefined;
+export declare function modelAddonOf(model: string): ModelRegistryEntry['addon'];
+export declare function residentSrc(model: string, addon: ModelRegistryEntry['addon']): string;
 export declare const CUSTOM_TASK_TYPES: readonly ['llm'];
 export type CustomTaskType = (typeof CUSTOM_TASK_TYPES)[number];
 export interface CustomModelEntry {

package/dist-lib/lib/models/model-catalog.js

@@ -332,6 +332,13 @@ export function curatedListables() {
 export function curatedModelAsset(name, addon) {
     return curatedListables().find((entry) => entry.asset.addon === addon && (entry.name === name || entry.asset.name === name))?.asset;
 }
+export function modelAddonOf(model) {
+    const known = [...curatedListables(), ...musicListables()].find((entry) => entry.name === model || entry.asset.name === model || entry.asset.src === model);
+    return known?.asset.addon ?? 'llm';
+}
+export function residentSrc(model, addon) {
+    return curatedModelAsset(model, addon)?.src ?? model;
+}
 export const CUSTOM_TASK_TYPES = ['llm'];
 let CUSTOM_MODELS = new Map();
 export function setCustomModels(entries) {

package/dist-lib/lib/models/model-pool.d.ts

@@ -75,6 +75,7 @@ declare class ModelPuddle extends ResourcePuddle<string, ModelHandle> {
     readonly resources: Map<string, PuddleEntry>;
 }
 type PoolLike = ModelPuddle;
+export declare function modelOf(key: string): string;
 export declare function releaseModel(pool: PoolLike, key: string): void;
 export declare function loadedModels(pool: PoolLike): LoadedModel[];
 export interface ResidentLease {
@@ -86,6 +87,7 @@ export declare function leaseResidentModel(pool: PoolLike, wanted?: (model: stri
 export declare function isResident(pool: PoolLike, key: string): boolean;
 export declare function leaseCount(pool: PoolLike, key: string): number;
 export declare function evictIdle(pool: PoolLike): Promise<void>;
+export declare function evictIdleWhere(pool: PoolLike, wanted: (key: string) => boolean): Promise<void>;
 export interface DeadlineOptions {
     deadlineMs: number;
     tickMs?: number;
@@ -95,9 +97,11 @@ export interface ModelCapacityOptions extends DeadlineOptions {
     signal?: {
         aborted?: boolean;
     } | undefined;
+    evictable?: (key: string) => boolean;
 }
-export declare function awaitModelCapacity(pool: PoolLike, { cap, deadlineMs, tickMs, signal }: ModelCapacityOptions): Promise<boolean>;
-export declare function evictModelWithin(pool: PoolLike, key: string, { deadlineMs, tickMs }: DeadlineOptions): Promise<boolean>;
+export declare function awaitModelCapacity(pool: PoolLike, { cap, deadlineMs, tickMs, signal, evictable }: ModelCapacityOptions): Promise<boolean>;
+export declare function evictModelWithin(pool: PoolLike, key: string, options: DeadlineOptions): Promise<boolean>;
+export declare function evictModel(pool: PoolLike, model: string, options: DeadlineOptions): Promise<boolean>;
 export type LoadSdk = () => Promise<ModelLifecycleSdk>;
 export type EmbedLifecycleSdk = Pick<SdkModule, 'unloadModel' | 'modelRegistrySearch'> & Partial<CancelSdk> & {
     loadModel(params: {

package/dist-lib/lib/models/model-pool.js

@@ -69,6 +69,9 @@ class ModelPuddle extends ResourcePuddle {
 function entryOf(pool, key) {
     return pool.resources.get(key);
 }
+export function modelOf(key) {
+    return key.split('\0')[0];
+}
 export function releaseModel(pool, key) {
     if (pool.destroyed)
         return;
@@ -123,8 +126,43 @@ export async function evictIdle(pool) {
         await pool.gc();
     }
 }
-export async function awaitModelCapacity(pool, { cap, deadlineMs, tickMs = 250, signal }) {
-    const roomed = () => loadedModels(pool).length < cap;
+export async function evictIdleWhere(pool, wanted) {
+    for (let tick = 0; tick <= pool.gcIdleTicks; tick++) {
+        if (pool.destroyed)
+            return;
+        const held = [];
+        for (const entry of pool.resources.values()) {
+            if (entry.refs === 0 && !wanted(entry.id)) {
+                held.push(entry.id);
+            }
+        }
+        if (pool.idle === held.length) {
+            return;
+        }
+        for (const key of held) {
+            pool.open(key);
+        }
+        try {
+            await pool.gc();
+        }
+        finally {
+            for (const key of held) {
+                releaseSettled(pool, key);
+            }
+        }
+    }
+}
+function residentCount(pool, wanted) {
+    let count = 0;
+    for (const entry of pool.resources.values()) {
+        if (wanted(entry.id) && entry.value.peek() !== null) {
+            count++;
+        }
+    }
+    return count;
+}
+export async function awaitModelCapacity(pool, { cap, deadlineMs, tickMs = 250, signal, evictable = () => true }) {
+    const roomed = () => residentCount(pool, evictable) < cap;
     let done = false;
     let timer = null;
     const expired = new Promise((resolve) => {
@@ -134,7 +172,7 @@ export async function awaitModelCapacity(pool, { cap, deadlineMs, tickMs = 250,
         }, deadlineMs);
     });
     const evicting = (async () => {
-        await evictIdle(pool);
+        await evictIdleWhere(pool, evictable);
         while (!roomed()) {
             if (done || signal?.aborted)
                 return false;
@@ -142,7 +180,7 @@ export async function awaitModelCapacity(pool, { cap, deadlineMs, tickMs = 250,
             if (done || signal?.aborted) {
                 return false;
             }
-            await evictIdle(pool);
+            await evictIdleWhere(pool, evictable);
         }
         return true;
     })();
@@ -156,18 +194,36 @@ export async function awaitModelCapacity(pool, { cap, deadlineMs, tickMs = 250,
         }
     }
 }
-export async function evictModelWithin(pool, key, { deadlineMs, tickMs = 250 }) {
+export function evictModelWithin(pool, key, options) {
+    return evictWhereWithin(pool, (candidate) => candidate === key, options);
+}
+export function evictModel(pool, model, options) {
+    return evictWhereWithin(pool, (key) => modelOf(key) === model, options);
+}
+async function evictWhereWithin(pool, wanted, { deadlineMs, tickMs = 250 }) {
+    let done = false;
     const evicting = (async () => {
-        while (isResident(pool, key)) {
-            await evictIdle(pool);
-            if (!isResident(pool, key)) {
+        while (residentCount(pool, wanted) > 0) {
+            await evictIdleWhere(pool, wanted);
+            if (residentCount(pool, wanted) === 0) {
                 break;
             }
             await new Promise((resolve) => setTimeout(resolve, tickMs));
+            if (done) {
+                return false;
+            }
         }
         return true;
     })();
-    return raceTimeout(evicting, { ms: deadlineMs, fallback: false });
+    try {
+        return await raceTimeout(evicting, { ms: deadlineMs, fallback: false });
+    }
+    finally {
+        done = true;
+    }
+}
+function definedFields(config) {
+    return Object.fromEntries(Object.entries(config).filter(([, value]) => value !== undefined));
 }
 function llmVerbosity() {
     const raw = Number(process.env.QVAC_LLM_VERBOSITY ?? 0);
@@ -253,7 +309,7 @@ export function createModelPool(loadSdk, notify, { deviceConfig = {}, loads = cr
         const openclCache = deviceConfig.device !== 'cpu' && deviceConfig.openclCacheDir
             ? deviceConfig.openclCacheDir()
             : undefined;
-        const modelConfig = {
+        const modelConfig = definedFields({
             verbosity: llmVerbosity(),
             tools: true,
             stop_sequences: RUNAWAY_STOP_SEQUENCES,
@@ -265,7 +321,7 @@ export function createModelPool(loadSdk, notify, { deviceConfig = {}, loads = cr
             gpu_layers: deviceConfig.gpuLayers,
             'main-gpu': mainGpu,
             openclCacheDir: openclCache || undefined
-        };
+        });
         const { projectionModelSrc: projection, ...loggedConfig } = modelConfig;
         hlog('llm load config', { model, projection: Boolean(projection), ...loggedConfig });
         const loadStarted = Date.now();