@qvac/harness 0.8.5 → 0.8.7

package/dist-lib/index.js

@@ -1,4 +1,5 @@
 import fs from 'fs';
+import ScopeLock from 'scope-lock';
 import { platform, totalmem } from 'os';
 import process from 'process';
 import ReadyResource from 'ready-resource';
@@ -17,15 +18,18 @@ import { download } from './lib/capabilities/download.js';
 import { createEmbedLane } from './lib/capabilities/embed.js';
 import { deleteAllKvCaches, deleteKvCacheKey } from './lib/capabilities/kv-cache.js';
 import { pcmForTranscription } from './lib/capabilities/audio-resample.js';
-import { buildWavBuffer } from './lib/wav-encode.js';
-import { createVoiceLane, packSamples, speakOnce, speakStreamOnce, transcribeOnce, transcribeStreaming, SUPERTONIC_SAMPLE_RATE } from './lib/capabilities/voice.js';
+import { createVoiceLane, packSamples, speakOnce, speakStreamOnce, transcribeAudio, SUPERTONIC_SAMPLE_RATE } from './lib/capabilities/voice.js';
 import { pushQueue } from './lib/queue.js';
 import { AbortedError, raceAbort } from './lib/helpers/race-abort.js';
-import { audioGenDownloadMetadata, categoryModelConfig, customProjectionMap, defaultProjectionMap, defaultTtsFor, defaultVoiceModel, diffusionDownloadMetadata, curatedModelAsset, infraModelsFor, modelAddonOf, modelNameToCapabilityProfile, musicModelSet, residentSrc, setCustomModels } from './lib/models/model-catalog.js';
-import { createModelStatusCache, customModelRows, invalidateModelStatusCache, mergeCustomRows, listModels } from './lib/models/model-list.js';
+import { audioGenDownloadMetadata, categoryModelConfig, customProjectionMap, defaultProjectionMap, defaultTtsFor, defaultVoiceModel, diffusionDownloadMetadata, assetBytes, curatedModelAsset, customContextLength, customModelSrc, infraModelsFor, modelAddonOf, modelNameToCapabilityProfile, musicModelSet, catalogModelName, residentSrc, setCustomModels } from './lib/models/model-catalog.js';
+import { createModelStatusCache, customModelRows, invalidateModelStatusCache, isModelSourceCached, mergeCustomRows, listModels } from './lib/models/model-list.js';
+import { createDownloadGate } from './lib/models/download-gate.js';
 import { deleteModelFiles, summarizeDelete } from './lib/models/model-delete.js';
-import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, awaitModelCapacity, evictIdle, evictIdleWhere, evictModel, evictModelWithin, isResident, leaseResidentModel, leaseCount, loadedModels, loadingModels, settledModels, modelKey, modelOf, releaseModel, resolveLoadOptions, progressState } from './lib/models/model-pool.js';
+import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, awaitModelCapacity, evictIdle, evictIdleWhere, evictIdleWithin, evictModel, evictModelWithin, isResident, leaseResidentModel, leaseCount, loadedModels, loadingModels, settledEntries, settledModels, modelKey, modelOf, releaseModel, resolveLoadOptions, progressState } from './lib/models/model-pool.js';
+import { memoryCapBytes, nextVictim, VictimRank } from './lib/models/memory-cap.js';
 import { defaultContextSize } from './lib/models/context.js';
+import { ContextChoice } from './lib/models/context-choice.js';
+import { parseGgufHeader, readGgufHeaderBytes } from './lib/models/gguf-header.js';
 import { resolveMainGpu } from './lib/models/gpu-selection.js';
 import { createIdleReaper } from './lib/models/idle-unload.js';
 import { openclCacheDir } from './lib/models/opencl-cache.js';
@@ -41,7 +45,7 @@ import { ensureBundledSkillsDir } from './lib/tools/skills/bundled-dir.js';
 import { ensureSkillsOverlayDir } from './lib/tools/skills/overlay-dir.js';
 import { copyBaseForEdit, deleteOverlaySkill, listManagedSkills, readSkillFile, writeSkillFile } from './lib/tools/skills/overlay-crud.js';
 import { watchSkillsOverlay } from './lib/tools/skills/overlay-watch.js';
-import { errorMessage } from './lib/guards.js';
+import { errorMessage, isRecord } from './lib/guards.js';
 import { hlog } from './lib/log.js';
 import { harness } from './spec/index.js';
 import { installResolutionDetector, resolutionDetail } from './lib/resolution-detector.js';
@@ -75,7 +79,6 @@ export { harnessHooks } from './lib/hooks/registry.js';
 export { EgressGuardHook, SkillCatalogGuardHook } from './lib/hooks/built-in/index.js';
 const HarnessState = harness.getEnum('@harness/state');
 const BIN_VERSION_PROBE_TTL_MS = 45_000;
-const TRANSCRIBE_STREAM_MIN_BYTES = 16_000 * 2 * 60;
 const TRANSCRIPTION_MODEL_LABEL = 'transcription model';
 const DIFFUSION_EVICT_WAIT_MS = 15_000;
 const MUSIC_EVICT_WAIT_MS = 15_000;
@@ -91,6 +94,23 @@ const TTS_MODEL_CONFIG = {
     ttsSpeed: 1.175,
     ttsNumInferenceSteps: 5
 };
+function fitSrc(modelSrc) {
+    if (typeof modelSrc === 'string')
+        return modelSrc;
+    const { src, name, sha256Checksum } = modelSrc;
+    return { src, name, sha256Checksum };
+}
+function downloadBytes(modelSrc) {
+    if (typeof modelSrc === 'string') {
+        return assetBytes(modelSrc);
+    }
+    return isRecord(modelSrc) && typeof modelSrc.expectedSize === 'number'
+        ? modelSrc.expectedSize
+        : undefined;
+}
+function stagesBytes(srcs) {
+    return srcs.reduce((sum, src) => sum + (assetBytes(src) ?? 0), 0);
+}
 export class Harness extends ReadyResource {
     _tools;
     _sdk;
@@ -98,6 +118,8 @@ export class Harness extends ReadyResource {
     _pinned = new Set();
     _replying = [];
     _modelStatus = createModelStatusCache();
+    _downloads;
+    _hold;
     _models;
     _embedModels;
     _retiredEmbedModels;
@@ -140,6 +162,12 @@ export class Harness extends ReadyResource {
     _capabilityInflight = null;
     _modelFaults = new Map();
     _idle;
+    _sizeLoads = false;
+    _capBytes = null;
+    _reserved = new Map();
+    _contexts = new ContextChoice(() => this._contextProfile());
+    _admission = new ScopeLock();
+    _byHand = new Map();
     constructor({ tools = [], sdkLoader, skillsDir, modelsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, completionModelCap, credentialStore, platform, gpuStats } = {}) {
         super();
         this._sdk = sdkLoader ?? createSdkLoader();
@@ -164,11 +192,19 @@ export class Harness extends ReadyResource {
             device: embedConfig.device,
             gpuLayers: embedConfig.gpuLayers
         });
+        this._downloads = createDownloadGate({
+            isCached: (model, addon) => this._isModelCached(model, addon),
+            onHeld: ({ model, addon }) => this._notifyModel({ status: 'held', model, addon }),
+            onDropped: ({ model, addon }) => this._notifyModel({ status: 'error', model, addon, message: 'aborted' })
+        });
+        const hold = (model, addon, signal) => this._downloads.hold(catalogModelName(model, addon), addon, signal);
+        this._hold = hold;
         const imageTransfers = new Map();
         const image = typeof imageModel === 'string' ? { model: imageModel } : imageModel;
         this._imageRuntime = image
             ? createDiffusionRuntime({
                 ...image,
+                hold,
                 loads: this._loads,
                 loadSdk: () => this._loadDiffusionSdk(),
                 onProgress: (model, progress) => {
@@ -191,9 +227,13 @@ export class Harness extends ReadyResource {
                     invalidateModelStatusCache(this._modelStatus);
                     this.emit('model', { ...transfer, status: 'downloaded' });
                 },
-                onBeforeLoad: async (signal) => {
+                onBeforeLoad: async (signal, srcs) => {
                     if (this._models.destroyed)
                         return;
+                    if (this._capBytes !== null) {
+                        await this._makeRoomForGenerator(srcs, signal);
+                        return;
+                    }
                     const evicted = await awaitModelCapacity(this._models, {
                         cap: 1,
                         deadlineMs: DIFFUSION_EVICT_WAIT_MS,
@@ -212,12 +252,17 @@ export class Harness extends ReadyResource {
         this._musicRuntime = music
             ? createMusicRuntime({
                 ...music,
+                hold,
                 loads: this._loads,
                 loadSdk: () => this._loadMusicSdk(),
                 onProgress: (model, progress) => this.emit('model', progressState(model, progress)),
-                onBeforeLoad: async (signal) => {
+                onBeforeLoad: async (signal, srcs) => {
                     if (this._models.destroyed)
                         return;
+                    if (this._capBytes !== null) {
+                        await this._makeRoomForGenerator(srcs, signal);
+                        return;
+                    }
                     const evicted = await awaitModelCapacity(this._models, {
                         cap: 1,
                         deadlineMs: MUSIC_EVICT_WAIT_MS,
@@ -233,6 +278,7 @@ export class Harness extends ReadyResource {
             })
             : null;
         this._translationRuntime = createTranslationRuntime({
+            hold,
             loads: this._loads,
             loadSdk: () => this._loadTranslationSdk(),
             onProgress: (model, progress) => this.emit('model', progressState(model, progress))
@@ -523,13 +569,45 @@ export class Harness extends ReadyResource {
     setModelIdleTimeout(ms) {
         this._idle.configure(ms);
     }
+    async setModelMemoryCap(percent) {
+        this._sizeLoads = true;
+        const { mobile, memoryBudget } = await this.capability();
+        this._capBytes =
+            percent > 0 && !mobile && memoryBudget
+                ? memoryCapBytes(percent, memoryBudget.availableBytes)
+                : null;
+        hlog('memory cap', { percent, capBytes: this._capBytes });
+        if (this._capBytes !== null) {
+            await this._busy(() => this._admit(() => this._makeRoom(0, { generators: true })));
+        }
+    }
+    async memoryUsage() {
+        return { usedBytes: this._settledBytes() };
+    }
+    setDownloadsAllowed({ allowed }) {
+        this._downloads.setAllowed(allowed);
+    }
+    async _isModelCached(model, addon) {
+        const row = (await this.list({ type: addon })).find((entry) => entry.name === model);
+        if (row) {
+            return row.isCached;
+        }
+        const sdk = await this._sdk.load();
+        return isModelSourceCached(sdk, { model, addon, modelsDir: this._modelsDir });
+    }
     previewModelSource(context) {
         return previewModelSource(context, { assessFit: (query) => this._assessFit(query) });
     }
-    async _assessFit({ sha256Checksum, name, contextTokens }) {
+    async _assessFit({ src, sha256Checksum, name, contextTokens }) {
         const sdk = await this._sdk.load();
         const result = await sdk.assessModelFit({
-            models: [{ model: { sha256Checksum, name }, workload: { kind: 'llm', contextTokens } }]
+            models: [
+                {
+                    modelSrc: { src, sha256Checksum, name },
+                    modelType: 'llm',
+                    modelConfig: { ctx_size: contextTokens }
+                }
+            ]
         });
         const model = result.models[0];
         return {
@@ -607,7 +685,7 @@ export class Harness extends ReadyResource {
         const key = modelKey(model, loadOpts);
         const pool = this._models;
         if (!isResident(pool, key)) {
-            await this._makeRoomForCompletion(model, { ctxSize: loadOpts.ctxSize, signal });
+            await this._makeRoomForCompletion(model, { ctxSize: this._ctxSizeOf(key, loadOpts), signal });
         }
         try {
             const { modelId } = await raceAbort(pool.open(key, model, loadOpts).load(), signal);
@@ -644,7 +722,10 @@ export class Harness extends ReadyResource {
         const loadSdk = this._sdk.load;
         return createModelPool(loadSdk, (state) => this._notifyModel(state), {
             deviceConfig: this._deviceConfig(),
-            loads: this._loads
+            loads: this._loads,
+            beforeLoad: (load) => this._beforeLoad(load),
+            chooseContext: (load, tier) => this._chooseContext(load, tier),
+            hold: this._hold
         });
     }
     _createEmbedPool() {
@@ -653,7 +734,9 @@ export class Harness extends ReadyResource {
             faults: this._embedFaults,
             batchSize: this._embedBatchSize,
             openclCacheDir,
-            loads: this._loads
+            loads: this._loads,
+            beforeLoad: (load) => this._beforeLoad(load),
+            hold: this._hold
         });
     }
     _createTtsPool() {
@@ -663,7 +746,9 @@ export class Harness extends ReadyResource {
             addon: 'tts',
             modelConfig: TTS_MODEL_CONFIG,
             modelConfigFor: categoryModelConfig,
-            loads: this._loads
+            loads: this._loads,
+            beforeLoad: (load) => this._beforeLoad(load),
+            hold: this._hold
         });
     }
     _createTranscriptionPool() {
@@ -673,7 +758,9 @@ export class Harness extends ReadyResource {
             addon: 'parakeet',
             modelConfig: { useGPU: this._transcriptionDevice !== 'cpu' },
             openclCacheDir,
-            loads: this._loads
+            loads: this._loads,
+            beforeLoad: (load) => this._beforeLoad(load),
+            hold: this._hold
         });
     }
     _notifyModel(state) {
@@ -703,6 +790,10 @@ export class Harness extends ReadyResource {
         releaseModel(pool, key);
         await this._collectRetiredEmbedPool(pool);
     }
+    _collectRetiredEmbedPoolAfter(settled, pool) {
+        const collect = () => this._collectRetiredEmbedPool(pool);
+        settled.then(collect, collect).catch(noop);
+    }
     _releaseEmbedResourceAfter(settled, pool, key) {
         settled
             .then(() => this._releaseEmbedResource(pool, key), () => this._releaseEmbedResource(pool, key))
@@ -758,11 +849,14 @@ export class Harness extends ReadyResource {
         }
         this._state = HarnessState.SUSPEND;
         hlog('suspending', { models: loadedModels(this._models).map((entry) => entry.model) });
+        this._downloads.abandon();
         const pending = this._sdk.peek();
         if (pending) {
             this._loads.drain(await pending);
         }
         await this._releaseRuntimes();
+        this._reserved.clear();
+        this._byHand.clear();
         await this._sdkLifecycle('suspend');
         invalidateModelStatusCache(this._modelStatus);
         hlog('suspended');
@@ -856,6 +950,7 @@ export class Harness extends ReadyResource {
     async _close() {
         hlog('closing');
         this._idle.close();
+        this._downloads.abandon();
         this._overlayWatcher?.stop();
         this._overlayWatcher = null;
         const pending = this._sdk.peek();
@@ -881,7 +976,12 @@ export class Harness extends ReadyResource {
             projection,
             companions: multiFile?.companions,
             assetSizes: multiFile?.assetSizes
-        }, () => invalidateModelStatusCache(this._modelStatus));
+        }, (event) => {
+            invalidateModelStatusCache(this._modelStatus);
+            if (event.type === 'complete') {
+                this._downloads.downloaded();
+            }
+        });
     }
     embed(context) {
         return this._busy(() => this._embed(context));
@@ -902,6 +1002,7 @@ export class Harness extends ReadyResource {
         let loaded = null;
         let job = null;
         let releaseAfter = null;
+        let collectAfter = null;
         try {
             loaded = await raceAbort(handle.load(), signal);
             const sdk = await this._sdk.loadEmbed();
@@ -912,7 +1013,8 @@ export class Harness extends ReadyResource {
         }
         catch (err) {
             if (err instanceof AbortedError) {
-                releaseAfter = job ?? (this._retiredEmbedModels.has(pool) ? handle.peek() : null);
+                releaseAfter = job;
+                collectAfter = job === null && this._retiredEmbedModels.has(pool) ? handle.peek() : null;
             }
             if (isSdkGpuDeviceLostError(err)) {
                 faults.recordFault(generation);
@@ -930,6 +1032,10 @@ export class Harness extends ReadyResource {
             if (releaseAfter) {
                 this._releaseEmbedResourceAfter(releaseAfter, pool, key);
             }
+            else if (collectAfter) {
+                releaseModel(pool, key);
+                this._collectRetiredEmbedPoolAfter(collectAfter, pool);
+            }
             else {
                 await this._releaseEmbedResource(pool, key);
             }
@@ -992,16 +1098,7 @@ export class Harness extends ReadyResource {
         return this._busy(() => this._transcribe(context));
     }
     async _transcribe({ model, audio, signal }) {
-        return this._voiceJob(this._transcriptionModels, { category: 'transcription', model, signal }, async (sdk, id) => {
-            const pcm = await pcmForTranscription(audio);
-            if (signal?.aborted) {
-                throw new AbortedError();
-            }
-            if (pcm && pcm.length >= TRANSCRIBE_STREAM_MIN_BYTES) {
-                return transcribeStreaming(sdk, pcm, { modelId: id, signal });
-            }
-            return transcribeOnce(sdk, id, pcm ? buildWavBuffer(pcm) : audio);
-        });
+        return this._voiceJob(this._transcriptionModels, { category: 'transcription', model, signal }, (sdk, id) => transcribeAudio(sdk, pcmForTranscription(audio), { modelId: id, signal, audio }));
     }
     detectLanguage(context) {
         return detectLanguage(context);
@@ -1033,21 +1130,25 @@ export class Harness extends ReadyResource {
             }
         };
         signal?.addEventListener('abort', cancel, { once: true });
-        const job = this._voiceLane(async () => {
+        const job = (async () => {
             try {
-                modelId = (await handle.load()).modelId;
-                sdk = (await this._sdk.loadVoice());
-                if (signal?.aborted) {
-                    return;
-                }
-                for await (const chunk of speakStreamOnce(sdk, modelId, text)) {
-                    chunks.push(chunk);
-                }
+                const loadedId = (await raceAbort(handle.load(), signal)).modelId;
+                modelId = loadedId;
+                const voiceSdk = (await this._sdk.loadVoice());
+                sdk = voiceSdk;
+                await this._voiceLane(async () => {
+                    if (signal?.aborted) {
+                        return;
+                    }
+                    for await (const chunk of speakStreamOnce(voiceSdk, loadedId, text)) {
+                        chunks.push(chunk);
+                    }
+                });
             }
             finally {
                 chunks.end();
             }
-        });
+        })();
         let settled = false;
         try {
             yield* chunks;
@@ -1075,9 +1176,11 @@ export class Harness extends ReadyResource {
         const loaded = this._settledModels();
         const loading = this._loadingModels();
         const custom = await customModelRows(options, { modelsDir: this._modelsDir, loaded });
+        const memory = this._memoryByModel();
         return mergeCustomRows(rows, custom).map((row) => ({
             ...row,
-            isLoading: loading.has(row.name)
+            isLoading: loading.has(row.name),
+            memoryBytes: memory.get(row.name) ?? memory.get(residentSrc(row.name, row.type))
         }));
     }
     async deleteModel(name) {
@@ -1159,7 +1262,7 @@ export class Harness extends ReadyResource {
             if (!roomed) {
                 return;
             }
-            await this._unloadIdleGenerators();
+            await this._yieldGenerators();
         }
         await this._loadAndRelease(target, signal);
     }
@@ -1175,20 +1278,20 @@ export class Harness extends ReadyResource {
         const target = this._residency(model, agent);
         const resident = isResident(target.pool, target.key);
         hlog('load model', { model, resident });
+        if (!resident && !(await this._admitByHand(model, target))) {
+            return { loaded: false, reason: 'memory' };
+        }
         if (target.pool !== this._models) {
-            await this._loadAndRelease(target, signal);
+            await this._loadByHand(target, signal);
             return { loaded: true };
         }
         if (!resident) {
-            if (!(await this._fits(model, target.ctxSize))) {
-                return { loaded: false, reason: 'memory' };
-            }
-            await evictIdleWhere(this._models, (key) => modelOf(key) === model && !this._replying.includes(key));
-            await this._unloadIdleGenerators();
+            await evictIdleWhere(this._models, this._replacedBy(model));
+            await this._yieldGenerators();
         }
         this._pinned.add(target.key);
         try {
-            await this._loadAndRelease(target, signal);
+            await this._loadByHand(target, signal);
         }
         catch (err) {
             this._pinned.delete(target.key);
@@ -1228,11 +1331,15 @@ export class Harness extends ReadyResource {
         await this._musicRuntime?.unloadIdle().catch(noop);
     }
     async _fits(model, ctxSize) {
+        return (await this._curatedFit(model, ctxSize))?.fit !== 'likely-too-large';
+    }
+    async _curatedFit(model, ctxSize) {
         const asset = curatedModelAsset(model, 'llm');
         if (!asset) {
-            return true;
+            return null;
         }
-        const assessed = await this._assessFit({
+        return this._assessFit({
+            src: asset.src,
             sha256Checksum: asset.sha256Checksum,
             name: asset.name,
             contextTokens: ctxSize || defaultContextSize(asset)
@@ -1240,7 +1347,273 @@ export class Harness extends ReadyResource {
             hlog('fit estimate failed', { model, err: errorMessage(err) });
             return null;
         });
-        return assessed?.fit !== 'likely-too-large';
+    }
+    async _admitByHand(model, target) {
+        if (this._capBytes === null) {
+            return (target.pool !== this._models ||
+                this._alone(model, target) ||
+                this._fits(model, target.ctxSize));
+        }
+        return this._admit(async () => {
+            if (isResident(target.pool, target.key) || this._byHand.has(target.key)) {
+                return true;
+            }
+            const sized = await this._sizeByHand(model, target);
+            const capBytes = this._capBytes;
+            const replaced = this._replacedEntries(model, target);
+            const used = this._usedBytes() - replaced.reduce((sum, { bytes }) => sum + (bytes ?? 0), 0);
+            const alone = this._alone(model, target);
+            const fits = capBytes === null || alone || sized.bytes === undefined || used + sized.bytes <= capBytes;
+            if ((sized.tooLarge && !alone) || !fits) {
+                return false;
+            }
+            this._byHand.set(target.key, sized.ctxSize);
+            if (sized.bytes !== undefined) {
+                this._reserved.set(target.key, sized.bytes);
+            }
+            return true;
+        });
+    }
+    _replacedBy(model) {
+        return (key) => modelOf(key) === model && !this._replying.includes(key);
+    }
+    _replacedEntries(model, target) {
+        const replacedBy = this._replacedBy(model);
+        return target.pool === this._models
+            ? settledEntries(this._models).filter(({ key, refs }) => refs === 0 && replacedBy(key))
+            : [];
+    }
+    _alone(model, target) {
+        const settled = this._pools().flatMap((pool) => settledEntries(pool));
+        const loading = this._pools().some((pool) => loadingModels(pool).length > 0);
+        return (!loading &&
+            this._reserved.size === 0 &&
+            settled.length === this._replacedEntries(model, target).length);
+    }
+    async _sizeByHand(model, target) {
+        const addon = modelAddonOf(model);
+        const modelSrc = curatedModelAsset(model, addon) ?? customModelSrc(model, addon) ?? model;
+        const ctxSize = target.pool === this._models
+            ? target.ctxSize || defaultContextSize(modelSrc, customContextLength(model))
+            : undefined;
+        const sized = await this._sizeLoad({
+            key: target.key,
+            model,
+            modelSrc,
+            modelType: target.modelType,
+            modelConfig: ctxSize === undefined
+                ? undefined
+                : { ctx_size: ctxSize, projectionModelSrc: this._projectionFor(model) }
+        });
+        return { ...sized, ctxSize };
+    }
+    async _loadByHand(target, signal) {
+        const loading = target.open().load();
+        void loading.then(noop, noop).then(() => {
+            if (this._byHand.delete(target.key)) {
+                this._reserved.delete(target.key);
+            }
+        });
+        try {
+            await raceAbort(loading, signal);
+        }
+        finally {
+            releaseModel(target.pool, target.key);
+        }
+    }
+    async _admit(run) {
+        await this._admission.lock();
+        try {
+            return await run();
+        }
+        finally {
+            this._admission.unlock();
+        }
+    }
+    async _makeRoomForGenerator(srcs, signal) {
+        if (signal?.aborted) {
+            throw new AbortedError();
+        }
+        await this._makeRoom(stagesBytes(srcs), { replying: true });
+        if (signal?.aborted) {
+            throw new AbortedError();
+        }
+    }
+    async _yieldGenerators() {
+        if (this._capBytes === null) {
+            await this._unloadIdleGenerators();
+        }
+    }
+    async _beforeLoad(load) {
+        if (!this._sizeLoads)
+            return undefined;
+        const bytes = this._byHand.has(load.key)
+            ? await this._admit(() => this._reserveByHand(load))
+            : await this._admit(() => this._reserve(load));
+        return { bytes, release: () => this._reserved.delete(load.key) };
+    }
+    async _reserveByHand(load) {
+        const admittedAt = this._byHand.get(load.key);
+        this._byHand.delete(load.key);
+        if (load.modelConfig?.['ctx_size'] !== admittedAt) {
+            const { bytes } = await this._sizeLoad(load);
+            if (bytes !== undefined) {
+                this._reserved.set(load.key, bytes);
+            }
+        }
+        return this._reserved.get(load.key);
+    }
+    async _reserve(load) {
+        const sized = await this._sizeLoad(load);
+        if (this._capBytes !== null) {
+            await this._makeRoom(sized.bytes ?? 0, {
+                generators: true,
+                tooLarge: sized.tooLarge,
+                recheck: async () => (await this._sizeLoad(load)).tooLarge
+            });
+        }
+        if (sized.bytes !== undefined) {
+            this._reserved.set(load.key, sized.bytes);
+        }
+        return sized.bytes;
+    }
+    async _sizeLoad({ model, modelSrc, modelType, modelConfig }) {
+        const downloaded = downloadBytes(modelSrc);
+        const { projectionModelSrc, ...config } = modelConfig ?? {};
+        const projection = downloadBytes(projectionModelSrc) ?? 0;
+        try {
+            const sdk = await this._sdk.load();
+            const { models } = await sdk.assessModelFit({
+                models: [{ modelSrc: fitSrc(modelSrc), modelType, modelConfig: config }]
+            });
+            const [fit] = models;
+            const bytes = fit?.estimate?.upperBoundBytes ?? fit?.floorBytes ?? downloaded;
+            return {
+                bytes: bytes === undefined ? undefined : bytes + projection,
+                tooLarge: fit?.verdict === 'likely-too-large',
+                confirmed: fit?.verdict === 'likely-fits' &&
+                    (fit.evidence === 'calibration' || fit.evidence === 'native-fit')
+            };
+        }
+        catch (err) {
+            hlog('fit estimate failed', { model, err: errorMessage(err) });
+            return {
+                bytes: downloaded === undefined ? undefined : downloaded + projection,
+                tooLarge: false,
+                confirmed: false
+            };
+        }
+    }
+    async _chooseContext(load, tier) {
+        if (isTranslationModel(load.model)) {
+            return tier;
+        }
+        const ctxSize = await this._contexts.choose(load.key, tier, {
+            trained: () => this._trainedContext(load),
+            fits: async (rung) => {
+                const sized = await this._sizeLoad({
+                    ...load,
+                    modelConfig: { ...load.modelConfig, ctx_size: rung }
+                });
+                const room = this._capRoom(load.key);
+                return sized.confirmed && (room === null || (sized.bytes ?? Infinity) <= room);
+            }
+        });
+        hlog('context chosen', { model: load.model, tier, ctxSize });
+        return ctxSize;
+    }
+    _capRoom(key) {
+        const cap = this._capBytes;
+        return cap === null ? null : cap - (this._usedBytes() - (this._reserved.get(key) ?? 0));
+    }
+    async _contextProfile() {
+        const { capabilityProfile, memoryBudget } = await this.capability();
+        return memoryBudget ? capabilityProfile : undefined;
+    }
+    async _trainedContext({ model, modelSrc }) {
+        const custom = customContextLength(model);
+        if (custom || typeof modelSrc === 'string') {
+            return custom;
+        }
+        const sdk = await this._sdk.load();
+        const info = await sdk.getModelInfo({ name: modelSrc.name }).catch(() => null);
+        const file = info?.cacheFiles[0]?.path;
+        const bytes = file ? await readGgufHeaderBytes(file) : null;
+        return (bytes && parseGgufHeader(bytes)?.contextLength) || undefined;
+    }
+    _ctxSizeOf(key, loadOpts) {
+        return loadOpts.ctxSize || this._contexts.chosen(key);
+    }
+    async _makeRoom(need, room) {
+        const capBytes = this._capBytes;
+        if (capBytes === null) {
+            return;
+        }
+        const tried = new Set();
+        let tooLarge = room.tooLarge ?? false;
+        while (this._usedBytes() + need > capBytes || tooLarge) {
+            const victim = nextVictim(this._capVictims(room).filter(({ id }) => !tried.has(id)));
+            if (!victim) {
+                hlog('model over memory cap', { need, used: this._usedBytes(), capBytes, tooLarge });
+                return;
+            }
+            tried.add(victim.id);
+            await victim.evict();
+            if (tooLarge && room.recheck) {
+                tooLarge = await room.recheck();
+            }
+        }
+    }
+    _capVictims({ replying = false, generators = false }) {
+        const victims = [];
+        if (generators) {
+            victims.push({
+                id: 'generators',
+                rank: VictimRank.generator,
+                lastUsed: 0,
+                evict: () => this._unloadIdleGenerators()
+            });
+        }
+        for (const [index, pool] of this._pools().entries()) {
+            for (const { key, refs, lastUsed } of settledEntries(pool)) {
+                const inReply = this._replying.includes(key);
+                if (refs > 0 || (inReply && !replying)) {
+                    continue;
+                }
+                victims.push({
+                    id: `${index}:${key}`,
+                    rank: this._victimRank(pool, key, inReply),
+                    lastUsed,
+                    evict: () => evictIdleWithin(pool, key, { deadlineMs: WEDGED_EVICT_WAIT_MS })
+                });
+            }
+        }
+        return victims;
+    }
+    _victimRank(pool, key, inReply) {
+        if (pool !== this._models)
+            return VictimRank.companion;
+        if (inReply)
+            return VictimRank.replying;
+        return this._pinned.has(key) ? VictimRank.pinned : VictimRank.requested;
+    }
+    _usedBytes() {
+        return (this._settledBytes() + [...this._reserved.values()].reduce((sum, bytes) => sum + bytes, 0));
+    }
+    _settledBytes() {
+        return this._pools()
+            .flatMap((pool) => settledEntries(pool))
+            .reduce((sum, { bytes }) => sum + (bytes ?? 0), 0);
+    }
+    _memoryByModel() {
+        const memory = new Map();
+        for (const { key, bytes } of this._pools().flatMap((pool) => settledEntries(pool))) {
+            if (bytes === undefined)
+                continue;
+            const model = modelOf(key);
+            memory.set(model, (memory.get(model) ?? 0) + bytes);
+        }
+        return memory;
     }
     async _yieldPins(model, ctxSize) {
         for (const key of this._pinned) {
@@ -1261,18 +1634,25 @@ export class Harness extends ReadyResource {
             return {
                 pool: this._models,
                 key,
-                ctxSize: loadOpts.ctxSize,
+                modelType: 'llamacpp-completion',
+                ctxSize: this._ctxSizeOf(key, loadOpts),
                 open: () => this._models.open(key, model, loadOpts)
             };
         }
         if (addon === 'embeddings') {
             const key = this._embedFaults.key(residentSrc(model, addon));
-            return { pool: this._embedModels, key, open: () => this._embedModels.open(key, model) };
+            return {
+                pool: this._embedModels,
+                key,
+                modelType: 'llamacpp-embedding',
+                open: () => this._embedModels.open(key, model)
+            };
         }
         if (addon === 'tts' || addon === 'parakeet') {
             const pool = addon === 'tts' ? this._ttsModels : this._transcriptionModels;
             const key = residentSrc(model, addon);
-            return { pool, key, open: () => pool.open(key, key) };
+            const modelType = addon === 'tts' ? 'tts-ggml' : 'parakeet-transcription';
+            return { pool, key, modelType, open: () => pool.open(key, key) };
         }
         throw new Error(`${model} loads for each generation and is not kept in memory`);
     }
@@ -1317,7 +1697,7 @@ export class Harness extends ReadyResource {
     completion(context) {
         return this._busyStream(() => this._completion(context));
     }
-    async *_completion({ model, agent, messages, cacheKey, thinkingDefault, toolConfigs, generationParams, responseFormat, captureInferenceCalls, grantedDirs, grantedDrives, driveOp, grantedWriteDirs, approve, attach, fetchAttachment, ragSearch, progress, askUser, signal }) {
+    async *_completion({ model, agent, messages, typedText, cacheKey, session, thinkingDefault, toolConfigs, generationParams, responseFormat, captureInferenceCalls, grantedDirs, grantedDrives, driveOp, grantedWriteDirs, approve, attach, fetchAttachment, ragSearch, progress, askUser, signal }) {
         if (!this.opened)
             await this.ready();
         if (this.suspended)
@@ -1329,13 +1709,16 @@ export class Harness extends ReadyResource {
         const resident = isResident(pool, key);
         hlog('completion start', { model, resident, projection: Boolean(projectionModel) });
         if (!resident) {
-            await this._makeRoomForCompletion(model, { ctxSize: loadOpts.ctxSize, signal });
-            await this._unloadIdleGenerators();
+            await this._makeRoomForCompletion(model, { ctxSize: this._ctxSizeOf(key, loadOpts), signal });
+            await this._yieldGenerators();
         }
         const acquireRound = async () => {
             if (!isResident(pool, key)) {
-                await this._makeRoomForCompletion(model, { ctxSize: loadOpts.ctxSize, signal });
-                await this._unloadIdleGenerators();
+                await this._makeRoomForCompletion(model, {
+                    ctxSize: this._ctxSizeOf(key, loadOpts),
+                    signal
+                });
+                await this._yieldGenerators();
             }
             try {
                 const { modelId } = await raceAbort(pool.open(key, model, loadOpts).load(), signal);
@@ -1382,11 +1765,13 @@ export class Harness extends ReadyResource {
                 agent,
                 ctxSize,
                 cacheKey,
+                session,
                 thinkingDefault,
                 generationParams,
                 responseFormat,
                 captureInferenceCalls,
                 messages: turns,
+                typedText,
                 tools: this._tools,
                 skillCatalog: await this._completionSkills(agent?.id ?? null, model),
                 toolConfigs,
@@ -1449,6 +1834,10 @@ export class Harness extends ReadyResource {
         };
     }
     async _makeRoomForCompletion(model, { ctxSize, signal, deadlineMs = MODEL_ROOM_WAIT_MS } = {}) {
+        if (this._capBytes !== null) {
+            await evictIdleWhere(this._models, (key) => modelOf(key) === model && !this._replying.includes(key));
+            return true;
+        }
         const roomed = await awaitModelCapacity(this._models, {
             cap: this._completionModelCap,
             deadlineMs,