@qvac/harness 0.7.1 → 1.0.0-rc.1

package/dist-lib/index.js

@@ -5,6 +5,9 @@ import ReadyResource from 'ready-resource';
 import { probeGpuStats } from './lib/hardware/gpu-probe.js';
 import { probeCapabilityProfile } from './lib/hardware/capability-probe.js';
 import { closePythonWorker } from './lib/tools/exec/adapters/python/worker/active.js';
+import { trackCompletion } from './lib/completion-lifecycle.js';
+import { createHarnessLifecycle } from './lib/harness-lifecycle.js';
+import { didDrainTimeOut, raceDrain } from './lib/machine/index.js';
 import { completion, imageSettingsFrom } from './lib/capabilities/completion.js';
 import { detectLanguage } from './lib/capabilities/detect-language.js';
 import { invoke } from './lib/capabilities/invoke.js';
@@ -24,7 +27,7 @@ import { AbortedError, raceAbort } from './lib/helpers/race-abort.js';
 import { audioGenDownloadMetadata, customProjectionMap, defaultProjectionMap, defaultTtsFor, defaultVoiceModel, diffusionDownloadMetadata, curatedModelAsset, infraModelsFor, modelAddonOf, modelNameToCapabilityProfile, musicModelSet, residentSrc, setCustomModels } from './lib/models/model-catalog.js';
 import { createModelStatusCache, customModelRows, invalidateModelStatusCache, mergeCustomRows, listModels } from './lib/models/model-list.js';
 import { deleteModelFiles, summarizeDelete } from './lib/models/model-delete.js';
-import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, awaitModelCapacity, evictIdle, evictIdleWhere, evictModel, evictModelWithin, isResident, leaseResidentModel, leaseCount, loadedModels, loadingModels, settledModels, modelKey, modelOf, releaseModel, resolveLoadOptions, progressState } from './lib/models/model-pool.js';
+import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, ModelUnloadError, awaitModelCapacity, evictIdle, evictIdleWhere, evictModel, evictModelWithin, isResident, leaseResidentModel, leaseCount, loadedModels, loadingModels, settledModels, modelKey, modelOf, releaseModel, resolveCtxSize, resolveLoadOptions, progressState } from './lib/models/model-pool.js';
 import { defaultContextSize } from './lib/models/context.js';
 import { resolveMainGpu } from './lib/models/gpu-selection.js';
 import { createIdleReaper } from './lib/models/idle-unload.js';
@@ -32,6 +35,8 @@ import { openclCacheDir } from './lib/models/opencl-cache.js';
 import { assertEmbedDevice, createEmbedConfig } from './lib/models/embed-config.js';
 import { createGpuFaultState, isSdkGpuDeviceLostError, isSdkModelWedgedError } from './lib/models/sdk-faults.js';
 import { createSdkLoader } from './lib/sdk-loader.js';
+import { installAbortControllerPolyfill } from './lib/capabilities/completion-engine/abort-polyfill.js';
+import { prepareCompletionRuntime } from './lib/capabilities/completion-runtime.js';
 import { createEditImageTool } from './lib/tools/edit-image.js';
 import { toolCatalog } from './lib/tools/catalog.js';
 import { createDiffusionRuntime, createGenerateImageTool } from './lib/tools/generate-image.js';
@@ -41,9 +46,9 @@ import { ensureBundledSkillsDir } from './lib/tools/skills/bundled-dir.js';
 import { ensureSkillsOverlayDir } from './lib/tools/skills/overlay-dir.js';
 import { copyBaseForEdit, deleteOverlaySkill, listManagedSkills, readSkillFile, writeSkillFile } from './lib/tools/skills/overlay-crud.js';
 import { watchSkillsOverlay } from './lib/tools/skills/overlay-watch.js';
-import { errorMessage } from './lib/guards.js';
+import { errorMessage, isRecord } from './lib/guards.js';
+import { redactedRecoveryFields } from './lib/wire.js';
 import { hlog } from './lib/log.js';
-import { harness } from './spec/index.js';
 import { installResolutionDetector, resolutionDetail } from './lib/resolution-detector.js';
 import { probeBinVersions } from './lib/tools/skills/bin-check.js';
 import { bareRuntime } from './lib/tools/exec/bare-runtime.js';
@@ -73,8 +78,8 @@ export { serveHarness } from './lib/serve.js';
 export { hlog, setHarnessLog, setHarnessLogSink } from './lib/log.js';
 export { harnessHooks } from './lib/hooks/registry.js';
 export { EgressGuardHook, SkillCatalogGuardHook } from './lib/hooks/built-in/index.js';
-const HarnessState = harness.getEnum('@harness/state');
 const BIN_VERSION_PROBE_TTL_MS = 45_000;
+const SUSPEND_SETTLE_MS = 500;
 const TRANSCRIBE_STREAM_MIN_BYTES = 16_000 * 2 * 60;
 const TRANSCRIPTION_MODEL_LABEL = 'transcription model';
 const DIFFUSION_EVICT_WAIT_MS = 15_000;
@@ -101,6 +106,7 @@ export class Harness extends ReadyResource {
     _models;
     _embedModels;
     _retiredEmbedModels;
+    _faultedEmbedKeys = new WeakMap();
     _embedLane = createEmbedLane();
     _embedFaults;
     _embedGpuLayers;
@@ -109,7 +115,23 @@ export class Harness extends ReadyResource {
     _ttsModels;
     _transcriptionModels;
     _voiceLane = createVoiceLane();
-    _state;
+    _lifecycle = createHarnessLifecycle({
+        suspend: () => this._suspendRuntimes(),
+        resume: () => this._resumeRuntimes()
+    });
+    _detachedRuns = new Set();
+    _effectDrainWaiters = new Set();
+    _activeWork = 0;
+    _workDrainWaiters = new Set();
+    _runtimeDraining = false;
+    _runtimeUnloadFailed = false;
+    _poolUnloadFailed = false;
+    _poolCleanupFailed = false;
+    _failedModelUnloads = new Set();
+    _sdkClosed = false;
+    _runtimesReleased = false;
+    _closeRetryNeeded = false;
+    _closeRequested = false;
     _skillsDir;
     _modelsDir;
     _overlayWatcher = null;
@@ -134,6 +156,7 @@ export class Harness extends ReadyResource {
     _musicSdk;
     _translationSdk;
     _musicRuntime;
+    _completionRuns;
     _translationRuntime;
     _capabilityTools;
     _capability = null;
@@ -142,6 +165,7 @@ export class Harness extends ReadyResource {
     _idle;
     constructor({ tools = [], sdkLoader, skillsDir, modelsDir, projectionModel, imageModel, imageClamps, musicModel, musicClamps, llmDevice, llmGpuLayers, contextSize, kvCacheK, kvCacheV, embedDevice, embedGpuLayers, embedBatchSize, transcriptionDevice, completionModelCap, credentialStore, platform, gpuStats } = {}) {
         super();
+        installAbortControllerPolyfill();
         this._sdk = sdkLoader ?? createSdkLoader();
         this._llmDevice = llmDevice;
         this._gpuStats = gpuStats;
@@ -214,7 +238,10 @@ export class Harness extends ReadyResource {
                 ...music,
                 loads: this._loads,
                 loadSdk: () => this._loadMusicSdk(),
-                onProgress: (model, progress) => this.emit('model', progressState(model, progress)),
+                onProgress: (model, progress) => {
+                    const state = progressState(model, progress);
+                    this.emit('model', state);
+                },
                 onBeforeLoad: async (signal) => {
                     if (this._models.destroyed)
                         return;
@@ -253,7 +280,6 @@ export class Harness extends ReadyResource {
             ...(this._musicRuntime ? [createGenerateMusicTool(this._musicRuntime, musicClamps)] : [])
         ];
         this._projectionModel = projectionModel ?? defaultProjectionMap();
-        this._state = HarnessState.RESUME;
         this._skillsDir = skillsDir;
         this._modelsDir = modelsDir;
         this._supportedSkills = [];
@@ -262,6 +288,7 @@ export class Harness extends ReadyResource {
         this._platform = platform;
         this._diffusionSdk = null;
         this._musicSdk = null;
+        this._completionRuns = new Set();
         this._translationSdk = null;
         this._models = this._createPool();
         this._embedModels = this._createEmbedPool();
@@ -274,13 +301,18 @@ export class Harness extends ReadyResource {
         });
     }
     async _open() {
-        installResolutionDetector(import.meta);
-        await this._reloadSkills();
-        await this._startOverlayWatch();
-        hlog('open', { skills: this._supportedSkills.length, tools: this._tools.length });
+        try {
+            installResolutionDetector(import.meta);
+            await this._reloadSkills();
+            await this._startOverlayWatch();
+            hlog('open', { skills: this._supportedSkills.length, tools: this._tools.length });
+        }
+        catch (error) {
+            throw error;
+        }
     }
     get suspended() {
-        return this._state === HarnessState.SUSPEND;
+        return this._lifecycle.suspended;
     }
     get skillNames() {
         return this._supportedSkills.map((skill) => skill.name);
@@ -393,6 +425,7 @@ export class Harness extends ReadyResource {
     async skills(agentId, modelName, options) {
         if (!this.opened)
             await this.ready();
+        this._assertNotClosing();
         const platform = this._platform ?? detectPlatform();
         const refresh = options?.refresh === true;
         return resolveSkillStatuses(this._allSkills, {
@@ -641,19 +674,24 @@ export class Harness extends ReadyResource {
         };
     }
     _createPool() {
-        const loadSdk = this._sdk.load;
+        const loadSdk = () => this._loadModelLifecycleSdk();
         return createModelPool(loadSdk, (state) => this._notifyModel(state), {
             deviceConfig: this._deviceConfig(),
-            loads: this._loads
+            loads: this._loads,
+            onUnloadFailure: (modelId, error) => this._recordModelUnloadFailure(modelId, error)
         });
     }
+    _loadModelLifecycleSdk() {
+        return this._sdk.load();
+    }
     _createEmbedPool() {
         const loadSdk = () => this._sdk.loadEmbed();
         return createEmbedPool(loadSdk, (state) => this._notifyModel(state), {
             faults: this._embedFaults,
             batchSize: this._embedBatchSize,
             openclCacheDir,
-            loads: this._loads
+            loads: this._loads,
+            onUnloadFailure: (modelId, error) => this._recordModelUnloadFailure(modelId, error)
         });
     }
     _createTtsPool() {
@@ -662,7 +700,8 @@ export class Harness extends ReadyResource {
             modelType: 'tts-ggml',
             addon: 'tts',
             modelConfig: TTS_MODEL_CONFIG,
-            loads: this._loads
+            loads: this._loads,
+            onUnloadFailure: (modelId, error) => this._recordModelUnloadFailure(modelId, error)
         });
     }
     _createTranscriptionPool() {
@@ -672,9 +711,16 @@ export class Harness extends ReadyResource {
             addon: 'parakeet',
             modelConfig: { useGPU: this._transcriptionDevice !== 'cpu' },
             openclCacheDir,
-            loads: this._loads
+            loads: this._loads,
+            onUnloadFailure: (modelId, error) => this._recordModelUnloadFailure(modelId, error)
         });
     }
+    _recordModelUnloadFailure(modelId, error) {
+        this._failedModelUnloads.add(modelId);
+        this._poolUnloadFailed = true;
+        this._runtimeUnloadFailed = true;
+        hlog('model unload failed', { modelId, err: errorMessage(error) });
+    }
     _notifyModel(state) {
         if (state.status === 'ready' || state.status === 'unloaded' || state.status === 'error') {
             invalidateModelStatusCache(this._modelStatus);
@@ -695,28 +741,166 @@ export class Harness extends ReadyResource {
     async _collectRetiredEmbedPool(pool) {
         if (!this._retiredEmbedModels.has(pool) || pool.idle !== pool.size)
             return;
+        try {
+            await pool.destroy();
+        }
+        catch (error) {
+            if (!(error instanceof ModelUnloadError)) {
+                this._poolCleanupFailed = true;
+                this._poolUnloadFailed = true;
+                this._runtimeUnloadFailed = true;
+            }
+            throw error;
+        }
         this._retiredEmbedModels.delete(pool);
-        await pool.destroy();
     }
     async _releaseEmbedResource(pool, key) {
         releaseModel(pool, key);
+        const faulted = this._faultedEmbedKeys.get(pool);
+        if (faulted?.has(key)) {
+            await evictIdle(pool);
+            if (!isResident(pool, key)) {
+                faulted.delete(key);
+            }
+        }
         await this._collectRetiredEmbedPool(pool);
     }
-    _releaseEmbedResourceAfter(settled, pool, key) {
-        settled
-            .then(() => this._releaseEmbedResource(pool, key), () => this._releaseEmbedResource(pool, key))
+    async _releaseEmbedResourceBeforeWork(releasing, releaseWork) {
+        try {
+            await releasing;
+        }
+        finally {
+            releaseWork();
+        }
+    }
+    _markEmbedFault({ pool, key, faults, generation }) {
+        faults.recordFault(generation);
+        let faulted = this._faultedEmbedKeys.get(pool);
+        if (!faulted) {
+            faulted = new Set();
+            this._faultedEmbedKeys.set(pool, faulted);
+        }
+        faulted.add(key);
+    }
+    _releaseEmbedResourceAfter(settled, fault) {
+        return settled
+            .then(() => this._releaseEmbedResource(fault.pool, fault.key), (error) => {
+            if (isSdkGpuDeviceLostError(error)) {
+                this._markEmbedFault(fault);
+            }
+            return this._releaseEmbedResource(fault.pool, fault.key);
+        })
             .catch(noop);
     }
     async _destroyEmbedPools() {
-        const pools = [this._embedModels, ...this._retiredEmbedModels];
-        this._retiredEmbedModels.clear();
-        await Promise.all(pools.map((pool) => pool.destroy()));
+        const pools = [this._embedModels, ...this._retiredEmbedModels].filter((pool) => Boolean(pool));
+        const outcomes = await Promise.allSettled(pools.map(async (pool) => {
+            await pool.destroy();
+            this._retiredEmbedModels.delete(pool);
+        }));
+        const generic = outcomes.find((outcome) => outcome.status === 'rejected' && !(outcome.reason instanceof ModelUnloadError));
+        if (generic?.status === 'rejected') {
+            throw generic.reason;
+        }
+        const failed = outcomes.find((outcome) => outcome.status === 'rejected');
+        if (failed?.status === 'rejected') {
+            throw failed.reason;
+        }
     }
     _enqueueEmbedControl(run) {
         const next = this._embedControl.then(run, run);
         this._embedControl = next.catch(noop);
         return next;
     }
+    async _runLifecycle(type, { open = true } = {}) {
+        if (!this.opened) {
+            if (open) {
+                await this.ready();
+            }
+        }
+        if (this._closeRequested || this.closing) {
+            return;
+        }
+        await this._lifecycle.send(type);
+    }
+    _abortCompletions() {
+        for (const run of this._completionRuns) {
+            run.abort();
+        }
+    }
+    _assertNotClosing() {
+        if (this._closeRequested || this.closing || this.closed) {
+            throw new Error('harness is closing');
+        }
+    }
+    _enterWork() {
+        this._assertNotClosing();
+        if (this._runtimeDraining) {
+            throw new Error('harness is suspended');
+        }
+        if (this._runtimeUnloadFailed) {
+            throw new Error('Runtime unload failed; close the harness before loading models again');
+        }
+        this._activeWork++;
+        let released = false;
+        return () => {
+            if (released)
+                return;
+            released = true;
+            if (--this._activeWork !== 0) {
+                return;
+            }
+            for (const resolve of this._workDrainWaiters) {
+                resolve();
+            }
+            this._workDrainWaiters.clear();
+        };
+    }
+    _waitForWorkDrain() {
+        if (!this._activeWork) {
+            return Promise.resolve();
+        }
+        return new Promise((resolve) => this._workDrainWaiters.add(resolve));
+    }
+    _effectsSettled() {
+        return !this._detachedRuns.size && ![...this._completionRuns].some((run) => run.started);
+    }
+    _notifyEffectDrain() {
+        if (!this._effectsSettled())
+            return;
+        for (const resolve of this._effectDrainWaiters) {
+            resolve();
+        }
+        this._effectDrainWaiters.clear();
+    }
+    _waitForEffectDrain() {
+        if (this._effectsSettled()) {
+            return Promise.resolve();
+        }
+        return new Promise((resolve) => this._effectDrainWaiters.add(resolve));
+    }
+    _trackDetachedRun(pending, runId) {
+        const tracked = pending.catch((error) => {
+            hlog('detached completion effect failed', { runId, err: errorMessage(error) });
+        });
+        this._detachedRuns.add(tracked);
+        void tracked
+            .finally(() => {
+            this._detachedRuns.delete(tracked);
+            this._notifyEffectDrain();
+        })
+            .catch(noop);
+    }
+    async _closeCompletions(runs = this._completionRuns) {
+        const closed = await Promise.allSettled([...runs].map((run) => run.close()));
+        for (const outcome of closed) {
+            if (outcome.status === 'rejected') {
+                hlog('completion close failed', {
+                    err: errorMessage(outcome.reason)
+                });
+            }
+        }
+    }
     getEmbeddingDevice() {
         return this._embedFaults.device;
     }
@@ -727,43 +911,61 @@ export class Harness extends ReadyResource {
     async _setEmbeddingDevice(device) {
         if (!this.opened)
             await this.ready();
-        if (this.closing)
-            throw new Error('harness is closing');
-        if (this._embedFaults.device === device) {
-            return;
-        }
-        this._embedFaults = createGpuFaultState({ device, gpuLayers: this._embedGpuLayers });
         if (this.suspended) {
+            this._assertNotClosing();
+            if (this._embedFaults.device === device) {
+                return;
+            }
+            this._embedFaults = createGpuFaultState({ device, gpuLayers: this._embedGpuLayers });
             return;
         }
-        const retired = this._embedModels;
-        this._retiredEmbedModels.add(retired);
-        this._embedModels = this._createEmbedPool();
-        await this._collectRetiredEmbedPool(retired);
+        const releaseWork = this._enterWork();
+        try {
+            if (this._embedFaults.device === device) {
+                return;
+            }
+            this._embedFaults = createGpuFaultState({ device, gpuLayers: this._embedGpuLayers });
+            if (this.suspended) {
+                return;
+            }
+            const retired = this._embedModels;
+            this._retiredEmbedModels.add(retired);
+            this._embedModels = this._createEmbedPool();
+            await this._collectRetiredEmbedPool(retired);
+        }
+        finally {
+            releaseWork();
+        }
     }
     async presuspend() {
-        if (this.closing)
-            return;
-        this._state = HarnessState.PRESUSPEND;
-        hlog('presuspend');
+        await this._runLifecycle('PRESUSPEND', { open: false });
     }
     async suspend() {
-        hlog('suspend called');
-        if (this._state === HarnessState.SUSPEND) {
-            return;
+        const result = await raceDrain(this._runLifecycle('SUSPEND'), SUSPEND_SETTLE_MS);
+        if (didDrainTimeOut(result)) {
+            throw new Error('Harness suspend is still waiting for active work to settle');
         }
-        if (this.closing) {
-            return;
-        }
-        this._state = HarnessState.SUSPEND;
+    }
+    async _suspendRuntimes() {
+        this._runtimeDraining = true;
         hlog('suspending', { models: loadedModels(this._models).map((entry) => entry.model) });
+        this._abortCompletions();
+        this._loads.drain();
+        await this._closeCompletions([...this._completionRuns].filter((run) => run.started));
+        await this._waitForEffectDrain();
         const pending = this._sdk.peek();
         if (pending) {
-            this._loads.drain(await pending);
+            const sdk = await pending.catch(noop);
+            if (sdk) {
+                this._loads.drain(sdk);
+            }
+        }
+        await this._waitForWorkDrain();
+        if (!(await this._releaseRuntimes())) {
+            throw new Error('Runtime unload failed; close the harness before loading models again');
         }
-        await this._releaseRuntimes();
-        await this._sdkLifecycle('suspend');
         invalidateModelStatusCache(this._modelStatus);
+        await this._sdkLifecycle('suspend');
         hlog('suspended');
     }
     async _sdkLifecycle(op) {
@@ -777,35 +979,60 @@ export class Harness extends ReadyResource {
         catch (err) {
             hlog(`sdk ${op} failed`, { err: errorMessage(err) });
             if (op === 'suspend') {
-                return;
+                throw err;
             }
             await sdk.resume().catch((retryErr) => {
                 hlog('sdk resume retry failed', {
                     err: errorMessage(retryErr)
                 });
+                throw retryErr;
             });
         }
     }
     async _releaseRuntimes() {
         const released = await Promise.allSettled([
-            this._models.destroy(),
+            this._models?.destroy(),
             this._destroyEmbedPools(),
-            this._ttsModels.destroy(),
-            this._transcriptionModels.destroy(),
+            this._ttsModels?.destroy(),
+            this._transcriptionModels?.destroy(),
             this._imageRuntime?.unload(),
             this._musicRuntime?.unload(),
             this._translationRuntime.unload()
         ]);
-        for (const outcome of released) {
+        for (const [index, outcome] of released.entries()) {
             if (outcome.status !== 'rejected')
                 continue;
+            if (index < 4 && !(outcome.reason instanceof ModelUnloadError)) {
+                this._poolCleanupFailed = true;
+            }
             const err = outcome.reason;
             hlog('unload failed', { err: errorMessage(err) });
         }
+        const sdk = await this._sdk.peek()?.catch(noop);
+        if (sdk) {
+            for (const modelId of this._failedModelUnloads) {
+                try {
+                    await sdk.unloadModel({ modelId });
+                    this._failedModelUnloads.delete(modelId);
+                    invalidateModelStatusCache(this._modelStatus);
+                }
+                catch (error) {
+                    hlog('model unload retry failed', {
+                        modelId,
+                        err: errorMessage(error)
+                    });
+                }
+            }
+        }
+        this._poolUnloadFailed = this._poolCleanupFailed || this._failedModelUnloads.size > 0;
+        this._runtimeUnloadFailed =
+            this._poolUnloadFailed || released.slice(4).some((outcome) => outcome.status === 'rejected');
+        return !this._runtimeUnloadFailed;
     }
     async _unloadIdleModels() {
-        if (!this.opened || this.closing || this.suspended)
+        if (!this.opened || this.closing || this.suspended || this._runtimeDraining) {
             return;
+        }
         hlog('idle unload', { models: this._loadedModels().map((entry) => entry.model) });
         const released = await Promise.allSettled([
             evictIdle(this._models),
@@ -825,62 +1052,164 @@ export class Harness extends ReadyResource {
         invalidateModelStatusCache(this._modelStatus);
     }
     async resume() {
-        hlog('resume called');
-        if (this._state === HarnessState.RESUME) {
-            return;
-        }
-        if (this.closing) {
-            return;
-        }
-        if (this._state === HarnessState.PRESUSPEND) {
-            this._state = HarnessState.RESUME;
-            return;
+        await this._runLifecycle('RESUME');
+    }
+    async _resumeRuntimes() {
+        this._assertNotClosing();
+        if (this._runtimeUnloadFailed) {
+            throw new Error('Runtime unload failed; close the harness before loading models again');
         }
-        this._state = HarnessState.RESUME;
         hlog('resuming', { pools: 'recreated' });
         this._loads.undrain();
         this._models = this._createPool();
         this._embedModels = this._createEmbedPool();
         this._ttsModels = this._createTtsPool();
         this._transcriptionModels = this._createTranscriptionPool();
-        await this._sdkLifecycle('resume');
+        try {
+            await this._sdkLifecycle('resume');
+            this._runtimeDraining = false;
+        }
+        catch (error) {
+            const sdk = await this._sdk.peek()?.catch(noop);
+            if (sdk) {
+                this._loads.drain(sdk);
+            }
+            await this._releaseRuntimes();
+            throw error;
+        }
     }
     async wakeup() {
-        if (this.closing)
-            return;
-        this._state = HarnessState.WAKEUP;
-        hlog('wakeup');
-        await this.resume();
+        await this._runLifecycle('WAKEUP');
+    }
+    ready() {
+        if (this._closeRequested || this.closing || this.closed) {
+            return Promise.reject(new Error('harness is closing'));
+        }
+        return super.ready();
     }
     async _close() {
         hlog('closing');
         this._idle.close();
-        this._overlayWatcher?.stop();
-        this._overlayWatcher = null;
-        const pending = this._sdk.peek();
-        if (pending) {
-            this._loads.drain(await pending);
+        this._lifecycle.cancelPending();
+        this._abortCompletions();
+        this._loads.drain();
+        await this._lifecycle.settled();
+        await this._finishClose();
+    }
+    close() {
+        this._closeRequested = true;
+        if (this._closeRetryNeeded) {
+            this.closing = null;
+            this._closeRetryNeeded = false;
+        }
+        const pending = !this.opened && this.opening !== null ? this._closeWhileOpening() : super.close();
+        void pending.catch(() => {
+            if (this.closing === pending) {
+                this._closeRetryNeeded = true;
+            }
+        });
+        const bounded = raceDrain(pending).then((result) => {
+            if (didDrainTimeOut(result)) {
+                throw new Error('Harness close is still waiting for active work to settle');
+            }
+        });
+        void bounded.catch(noop);
+        return bounded;
+    }
+    _closeWhileOpening() {
+        if (this.closing)
+            return this.closing;
+        const opening = this.opening;
+        const pending = (async () => {
+            await opening?.catch(noop);
+            this._abortCompletions();
+            this._loads.drain();
+            await this._finishClose();
+            this.closed = true;
+            this.emit('close');
+        })();
+        this.closing = pending;
+        return pending;
+    }
+    async _finishClose() {
+        let completed = false;
+        try {
+            const pending = this._sdk.peek();
+            if (pending) {
+                const sdk = await pending.catch(noop);
+                if (sdk) {
+                    this._loads.drain(sdk);
+                }
+            }
+            await this._closeCompletions();
+            await this._waitForEffectDrain();
+            await this._waitForWorkDrain();
+            this._overlayWatcher?.stop();
+            this._overlayWatcher = null;
+            if (!this._runtimesReleased) {
+                let [released] = await Promise.all([this._releaseRuntimes(), closePythonWorker()]);
+                if (!released && !this._poolUnloadFailed) {
+                    released = await this._releaseRuntimes();
+                }
+                this._runtimesReleased = released;
+            }
+            let sdkCloseFailed = false;
+            let sdkCloseError;
+            const sdk = !this._sdkClosed && !this._runtimeUnloadFailed ? await this._sdk.peek()?.catch(noop) : null;
+            if (sdk) {
+                try {
+                    await sdk.close();
+                    this._sdkClosed = true;
+                }
+                catch (error) {
+                    sdkCloseFailed = true;
+                    sdkCloseError = error;
+                    hlog('sdk close failed', { err: errorMessage(error) });
+                }
+            }
+            if (sdkCloseFailed) {
+                throw sdkCloseError;
+            }
+            if (this._runtimeUnloadFailed) {
+                throw new Error('Runtime unload failed; model cleanup could not be confirmed');
+            }
+            completed = true;
         }
-        await Promise.all([this._releaseRuntimes(), closePythonWorker()]);
-        const sdk = await this._sdk.peek()?.catch(noop);
-        if (sdk) {
-            await sdk.close();
+        finally {
+            if (completed) {
+                this._lifecycle.stop();
+            }
         }
     }
     download(args) {
+        this._assertNotClosing();
+        let releaseWork = null;
         const projection = (args.addon ?? 'llm') === 'llm' ? this._projectionFor(args.model) : undefined;
         const multiFile = diffusionDownloadMetadata(args.model, args.addon) ??
             audioGenDownloadMetadata(args.model, args.addon);
         return download(async () => {
             if (!this.opened)
                 await this.ready();
-            return this._sdk.load();
+            releaseWork = this._enterWork();
+            try {
+                const sdk = await this._sdk.load();
+                this._assertNotClosing();
+                return sdk;
+            }
+            catch (error) {
+                releaseWork?.();
+                releaseWork = null;
+                throw error;
+            }
         }, {
             ...args,
             projection,
             companions: multiFile?.companions,
             assetSizes: multiFile?.assetSizes
-        }, () => invalidateModelStatusCache(this._modelStatus));
+        }, () => invalidateModelStatusCache(this._modelStatus), () => {
+            releaseWork?.();
+            releaseWork = null;
+        });
     }
     embed(context) {
         return this._busy(() => this._embed(context));
@@ -888,8 +1217,10 @@ export class Harness extends ReadyResource {
     async _embed({ model, texts, signal }) {
         if (!this.opened)
             await this.ready();
-        if (this.suspended)
+        this._assertNotClosing();
+        if (this.suspended) {
             throw new Error('harness is suspended');
+        }
         if (texts.length === 0) {
             return [];
         }
@@ -897,13 +1228,23 @@ export class Harness extends ReadyResource {
         const faults = this._embedFaults;
         const generation = faults.generation;
         const key = faults.key(residentSrc(model, 'embeddings'));
-        const handle = pool.open(key, model);
+        const releaseWork = this._enterWork();
+        let handle;
+        try {
+            handle = pool.open(key, model);
+        }
+        catch (error) {
+            releaseWork();
+            throw error;
+        }
         let loaded = null;
         let job = null;
         let releaseAfter = null;
+        let drainWorkAfter = null;
         try {
             loaded = await raceAbort(handle.load(), signal);
             const sdk = await this._sdk.loadEmbed();
+            this._assertNotClosing();
             job = this._embedLane(sdk, { modelId: loaded.modelId, texts });
             const vectors = await raceAbort(job, signal);
             faults.reset();
@@ -911,26 +1252,35 @@ export class Harness extends ReadyResource {
         }
         catch (err) {
             if (err instanceof AbortedError) {
-                releaseAfter = job ?? (this._retiredEmbedModels.has(pool) ? handle.peek() : null);
+                if (job || this._retiredEmbedModels.has(pool)) {
+                    releaseAfter = job ?? handle.peek();
+                }
+                else {
+                    drainWorkAfter = handle.peek();
+                }
             }
             if (isSdkGpuDeviceLostError(err)) {
-                faults.recordFault(generation);
-                if (loaded) {
-                    const dead = loaded.modelId;
-                    await this._sdk
-                        .loadEmbed()
-                        .then((sdk) => sdk.unloadModel({ modelId: dead }))
-                        .catch(noop);
-                }
+                this._markEmbedFault({ pool, key, faults, generation });
             }
             throw err;
         }
         finally {
             if (releaseAfter) {
-                this._releaseEmbedResourceAfter(releaseAfter, pool, key);
+                void this._releaseEmbedResourceAfter(releaseAfter, {
+                    pool,
+                    key,
+                    faults,
+                    generation
+                }).finally(releaseWork);
             }
             else {
-                await this._releaseEmbedResource(pool, key);
+                const releasing = this._releaseEmbedResource(pool, key);
+                if (drainWorkAfter) {
+                    void Promise.allSettled([releasing, drainWorkAfter]).then(releaseWork);
+                }
+                else {
+                    await this._releaseEmbedResourceBeforeWork(releasing, releaseWork);
+                }
             }
         }
     }
@@ -953,7 +1303,16 @@ export class Harness extends ReadyResource {
             throw new Error('harness is suspended');
         const addon = category === 'tts' ? 'tts' : 'parakeet';
         const name = residentSrc(model || (await this._defaultVoiceSrc(category)), addon);
-        const handle = pool.open(name, name);
+        this._assertNotClosing();
+        const releaseWork = this._enterWork();
+        let handle;
+        try {
+            handle = pool.open(name, name);
+        }
+        catch (error) {
+            releaseWork();
+            throw error;
+        }
         let job = null;
         let releaseAfter = null;
         let sdk = null;
@@ -961,6 +1320,7 @@ export class Harness extends ReadyResource {
         try {
             modelId = (await raceAbort(handle.load(), signal)).modelId;
             sdk = (await this._sdk.loadVoice());
+            this._assertNotClosing();
             const boundSdk = sdk;
             const boundId = modelId;
             job = this._voiceLane(() => run(boundSdk, boundId));
@@ -971,7 +1331,7 @@ export class Harness extends ReadyResource {
                 if (sdk && modelId) {
                     void sdk.cancel({ modelId, operation: 'inference' }).catch(noop);
                 }
-                releaseAfter = job;
+                releaseAfter = job ?? handle.peek();
             }
             throw err;
         }
@@ -980,10 +1340,15 @@ export class Harness extends ReadyResource {
                 releaseModel(pool, name);
             };
             if (releaseAfter) {
-                void releaseAfter.then(release, release);
+                void releaseAfter.then(release, release).finally(releaseWork).catch(noop);
             }
             else {
-                release();
+                try {
+                    release();
+                }
+                finally {
+                    releaseWork();
+                }
             }
         }
     }
@@ -1021,9 +1386,18 @@ export class Harness extends ReadyResource {
         if (this.suspended)
             throw new Error('harness is suspended');
         const name = residentSrc(model || (await this._defaultVoiceSrc('tts')), 'tts');
+        this._assertNotClosing();
         const pool = this._ttsModels;
-        const handle = pool.open(name, name);
         const chunks = pushQueue();
+        const releaseWork = this._enterWork();
+        let handle;
+        try {
+            handle = pool.open(name, name);
+        }
+        catch (error) {
+            releaseWork();
+            throw error;
+        }
         let sdk = null;
         let modelId = null;
         const cancel = () => {
@@ -1036,6 +1410,7 @@ export class Harness extends ReadyResource {
             try {
                 modelId = (await handle.load()).modelId;
                 sdk = (await this._sdk.loadVoice());
+                this._assertNotClosing();
                 if (signal?.aborted) {
                     return;
                 }
@@ -1058,53 +1433,98 @@ export class Harness extends ReadyResource {
             if (!settled) {
                 cancel();
             }
-            void job.catch(noop).finally(() => {
-                releaseModel(pool, name);
-            });
+            void job
+                .catch(noop)
+                .finally(() => {
+                try {
+                    releaseModel(pool, name);
+                }
+                finally {
+                    releaseWork();
+                }
+            })
+                .catch(noop);
         }
     }
     async list(options = {}) {
         if (!this.opened)
             await this.ready();
-        const sdk = await this._sdk.load();
-        const rows = await listModels(sdk, options, {
-            cache: this._modelStatus,
-            projectionFor: (name) => this._projectionFor(name)
-        });
-        const loaded = this._settledModels();
-        const loading = this._loadingModels();
-        const custom = await customModelRows(options, { modelsDir: this._modelsDir, loaded });
-        return mergeCustomRows(rows, custom).map((row) => ({
-            ...row,
-            isLoading: loading.has(row.name)
-        }));
+        const releaseWork = this._enterWork();
+        try {
+            const sdk = await this._sdk.load();
+            this._assertNotClosing();
+            const rows = await listModels(sdk, options, {
+                cache: this._modelStatus,
+                projectionFor: (name) => this._projectionFor(name)
+            });
+            const loaded = this._settledModels();
+            const loading = this._loadingModels();
+            const custom = await customModelRows(options, { modelsDir: this._modelsDir, loaded });
+            return mergeCustomRows(rows, custom).map((row) => ({
+                ...row,
+                isLoading: loading.has(row.name)
+            }));
+        }
+        finally {
+            releaseWork();
+        }
     }
     async deleteModel(name) {
         if (!this.opened)
             await this.ready();
-        const sdk = await this._sdk.load();
-        const result = await deleteModelFiles(sdk, name, { modelsDir: this._modelsDir });
-        invalidateModelStatusCache(this._modelStatus);
-        return summarizeDelete(result);
+        const releaseWork = this._enterWork();
+        try {
+            const sdk = await this._sdk.load();
+            this._assertNotClosing();
+            const result = await deleteModelFiles(sdk, name, { modelsDir: this._modelsDir });
+            invalidateModelStatusCache(this._modelStatus);
+            return summarizeDelete(result);
+        }
+        finally {
+            releaseWork();
+        }
     }
     async clearKvCache() {
         if (!this.opened)
             await this.ready();
-        return deleteAllKvCaches(() => this._sdk.load());
+        const releaseWork = this._enterWork();
+        try {
+            return await deleteAllKvCaches(() => {
+                this._assertNotClosing();
+                return this._sdk.load();
+            });
+        }
+        finally {
+            releaseWork();
+        }
     }
     async deleteKvCache(kvCacheKey) {
         if (!this.opened)
             await this.ready();
-        return deleteKvCacheKey(() => this._sdk.load(), kvCacheKey);
+        const releaseWork = this._enterWork();
+        try {
+            return await deleteKvCacheKey(() => {
+                this._assertNotClosing();
+                return this._sdk.load();
+            }, kvCacheKey);
+        }
+        finally {
+            releaseWork();
+        }
     }
     async infraModels(profile) {
         return infraModelsFor(profile);
     }
     async capability() {
-        if (this._capability)
+        if (!this.opened)
+            await this.ready();
+        this._assertNotClosing();
+        if (this._capability) {
             return this._capability;
-        if (this._capabilityInflight)
+        }
+        if (this._capabilityInflight) {
             return this._capabilityInflight;
+        }
         const reads = {
             totalmem,
             platform,
@@ -1130,6 +1550,7 @@ export class Harness extends ReadyResource {
         };
         this._capabilityInflight = probeCapabilityProfile(reads)
             .then((capability) => {
+            capability = { ...capability, hostRunLog: true, hostRunLogVersion: 1 };
             this._capability = capability;
             return capability;
         })
@@ -1144,23 +1565,26 @@ export class Harness extends ReadyResource {
     async _warmModel({ model, agent, signal }) {
         if (!this.opened)
             await this.ready();
-        if (this.suspended)
+        this._assertNotClosing();
+        if (this.suspended) {
             return;
+        }
         const target = this._residency(model, agent);
-        const resident = isResident(target.pool, target.key);
-        hlog('warm model', { model, resident });
-        if (target.pool === this._models && !resident) {
+        hlog('warm model', { model, resident: isResident(target.pool, target.key) });
+        await this._loadAndRelease(target, signal, async () => {
+            if (target.pool !== this._models || isResident(target.pool, target.key)) {
+                return true;
+            }
             const roomed = await this._makeRoomForCompletion(model, {
                 ctxSize: target.ctxSize,
                 signal,
                 deadlineMs: WARM_ROOM_WAIT_MS
             });
-            if (!roomed) {
-                return;
+            if (roomed) {
+                await this._unloadIdleGenerators();
             }
-            await this._unloadIdleGenerators();
-        }
-        await this._loadAndRelease(target, signal);
+            return roomed;
+        });
     }
     loadModel(context) {
         return this._busy(() => this._loadModel(context));
@@ -1214,13 +1638,9 @@ export class Harness extends ReadyResource {
         }
         return { unloaded };
     }
-    async _loadAndRelease(target, signal) {
-        try {
-            await raceAbort(target.open().load(), signal);
-        }
-        finally {
-            releaseModel(target.pool, target.key);
-        }
+    async _loadAndRelease({ pool, key, open }, signal, prepare = async () => true) {
+        const lease = await this._acquireModelLease({ pool, key, open, signal, prepare });
+        lease?.release();
     }
     async _unloadIdleGenerators() {
         await this._imageRuntime?.unloadIdle().catch(noop);
@@ -1283,6 +1703,57 @@ export class Harness extends ReadyResource {
             projectionModel: this._projectionFor(model)
         }, this._deviceConfig());
     }
+    async _acquireModelLease({ pool, key, open, signal, prepare }) {
+        const releaseWork = this._enterWork();
+        let pendingLoad = null;
+        let opened = false;
+        let released = false;
+        const release = () => {
+            if (released) {
+                return;
+            }
+            released = true;
+            try {
+                if (opened) {
+                    releaseModel(pool, key);
+                }
+            }
+            finally {
+                releaseWork();
+            }
+        };
+        try {
+            if (!(await prepare())) {
+                release();
+                return null;
+            }
+            if (signal?.aborted) {
+                throw new AbortedError();
+            }
+            if (this.suspended || this._runtimeDraining) {
+                throw new Error('harness is suspended');
+            }
+            this._assertNotClosing();
+            if (this._runtimeUnloadFailed) {
+                throw new Error('Runtime unload failed; close the harness before loading models again');
+            }
+            const handle = open();
+            opened = true;
+            const loading = handle.load();
+            pendingLoad = loading;
+            const loaded = await raceAbort(loading, signal);
+            return { modelId: loaded.modelId, release };
+        }
+        catch (error) {
+            if (pendingLoad) {
+                void pendingLoad.then(release, release).catch(noop);
+            }
+            else {
+                release();
+            }
+            throw error;
+        }
+    }
     invoke(context) {
         return this._busyStream(() => this._invoke(context));
     }
@@ -1290,8 +1761,10 @@ export class Harness extends ReadyResource {
         if (!this.opened)
             await this.ready();
         if (this.suspended) {
+            this._assertNotClosing();
             return yield { type: 'aborted' };
         }
+        const releaseWork = this._enterWork();
         hlog('invoke start', { capability, enhance });
         try {
             yield* invoke({
@@ -1311,117 +1784,101 @@ export class Harness extends ReadyResource {
         }
         finally {
             hlog('invoke end', { capability });
+            releaseWork();
         }
     }
     completion(context) {
-        return this._busyStream(() => this._completion(context));
+        this._assertNotClosing();
+        if (this._runtimeDraining) {
+            throw new Error('harness is suspended');
+        }
+        if (this._runtimeUnloadFailed) {
+            throw new Error('Runtime unload failed; close the harness before loading models again');
+        }
+        const tracked = trackCompletion((signal) => this._busyStream(() => this._runCompletion({ ...context, signal })), context.signal, (run) => {
+            this._completionRuns.delete(run);
+            this._notifyEffectDrain();
+        });
+        const { iterator, run } = tracked;
+        this._completionRuns.add(run);
+        return iterator;
     }
-    async *_completion({ model, agent, messages, cacheKey, thinkingDefault, toolConfigs, generationParams, responseFormat, captureInferenceCalls, grantedDirs, grantedDrives, driveOp, grantedWriteDirs, approve, attach, fetchAttachment, ragSearch, progress, askUser, signal }) {
+    async *_runCompletion(request) {
         if (!this.opened)
             await this.ready();
-        if (this.suspended)
-            throw new Error('harness is suspended');
-        const loadOpts = this._completionLoadOptions(model, agent);
-        const projectionModel = loadOpts.projectionModel;
-        const key = modelKey(model, loadOpts);
-        const pool = this._models;
-        const resident = isResident(pool, key);
-        hlog('completion start', { model, resident, projection: Boolean(projectionModel) });
-        if (!resident) {
-            await this._makeRoomForCompletion(model, { ctxSize: loadOpts.ctxSize, signal });
-            await this._unloadIdleGenerators();
+        this._assertNotClosing();
+        if (this.suspended || this._runtimeDraining) {
+            yield { type: 'aborted' };
+            return;
         }
-        const acquireRound = async () => {
-            if (!isResident(pool, key)) {
-                await this._makeRoomForCompletion(model, { ctxSize: loadOpts.ctxSize, signal });
-                await this._unloadIdleGenerators();
-            }
-            try {
-                const { modelId } = await raceAbort(pool.open(key, model, loadOpts).load(), signal);
-                return {
-                    modelId,
-                    release: () => {
-                        releaseModel(pool, key);
-                    }
-                };
+        const { boot, onStep, hostAppendTimeoutMs, requestDigestKey, ...provided } = request;
+        let modelId = request.model ?? '';
+        let key = null;
+        try {
+            const runtime = async (run) => {
+                modelId = run.model ?? run.modelId;
+                const runContext = { ...run, model: run.model ?? run.modelId };
+                const lease = await this._completionRuntime(runContext);
+                if (!key) {
+                    this._replying.push(lease.key);
+                }
+                key = lease.key;
+                return lease;
+            };
+            const args = { ...provided, lockOwner: this };
+            for await (const event of completion(runtime, args, {
+                boot,
+                onStep,
+                hostAppendTimeoutMs,
+                requestDigestKey,
+                resumeRequest: provided,
+                onDetached: (pending, runId) => this._trackDetachedRun(pending, runId)
+            })) {
+                yield event;
             }
-            catch (err) {
-                releaseModel(pool, key);
-                throw err;
+            if (key) {
+                this._modelFaults.delete(modelId);
             }
-        };
-        const handle = pool.open(key, model, loadOpts);
-        let held = true;
-        const releaseHeld = () => {
-            if (!held)
-                return;
-            held = false;
-            releaseModel(pool, key);
-        };
-        this._replying.push(key);
-        try {
-            const { modelId, ctxSize } = await raceAbort(handle.load(), signal);
-            hlog('model ready', { model });
-            const sdk = await this._sdk.load();
-            const turns = projectionModel ? messages : messages.map(({ attachments, ...turn }) => turn);
-            const remoteRoots = driveOp && grantedDrives?.length ? createRemoteRoots(grantedDrives, driveOp) : [];
-            const fsRoots = [
-                ...(grantedDirs?.length ? await createLocalRoots(grantedDirs) : []),
-                ...remoteRoots
-            ];
-            const fsWriteRoots = [
-                ...(grantedWriteDirs?.length ? await createLocalRoots(grantedWriteDirs) : []),
-                ...remoteRoots.filter((root) => root.writable)
-            ];
-            releaseHeld();
-            yield* completion(sdk, {
-                modelId,
-                acquireRound,
-                model,
-                agent,
-                ctxSize,
-                cacheKey,
-                thinkingDefault,
-                generationParams,
-                responseFormat,
-                captureInferenceCalls,
-                messages: turns,
-                tools: this._tools,
-                skillCatalog: await this._completionSkills(agent?.id ?? null, model),
-                toolConfigs,
-                fsRoots,
-                fsWriteRoots,
-                approve,
-                attach,
-                fetchAttachment,
-                ragSearch,
-                progress,
-                askUser,
-                signal
-            });
-            this._modelFaults.delete(model);
         }
-        catch (err) {
-            if (err instanceof AbortedError || this.suspended) {
-                hlog('completion aborted', { model, suspended: this.suspended });
-                yield { type: 'aborted' };
+        catch (error) {
+            yield* this._completionFailed(error, modelId, key);
+        }
+        finally {
+            if (key) {
+                hlog('completion end', { model: modelId });
+                this._replying.splice(this._replying.indexOf(key), 1);
+                void this._imageRuntime?.unloadIdle().catch(noop);
+                void this._musicRuntime?.unloadIdle().catch(noop);
             }
-            else {
-                const message = errorMessage(err);
-                hlog('completion failed', { model, ...resolutionDetail(err), err: message });
-                const wedged = isSdkModelWedgedError(err) ? await this._evictWedgedModel(model, key) : null;
-                yield { type: 'error', message: wedged ? wedged.message : message };
-                if (wedged?.fatal) {
-                    process.exit(1);
-                }
+        }
+    }
+    async *_completionFailed(err, model, key) {
+        if (err instanceof AbortedError || this.suspended || this._runtimeDraining) {
+            hlog('completion aborted', { model, suspended: this.suspended || this._runtimeDraining });
+            yield { type: 'aborted' };
+            return;
+        }
+        const message = errorMessage(err);
+        const name = err instanceof Error ? err.name : undefined;
+        hlog('completion failed', { model, ...resolutionDetail(err), err: message });
+        if (key && isSdkModelWedgedError(err)) {
+            const wedged = await this._evictWedgedModel(model, key);
+            yield completionErrorEvent(wedged.message, name, err);
+            if (wedged.fatal) {
+                process.exit(1);
             }
         }
+        else {
+            yield completionErrorEvent(message, name, err);
+        }
+    }
+    async _completionRuntime(request) {
+        const releaseWork = this._enterWork();
+        try {
+            return await prepareCompletionRuntime(this, request);
+        }
         finally {
-            hlog('completion end', { model });
-            this._replying.splice(this._replying.indexOf(key), 1);
-            releaseHeld();
-            void this._imageRuntime?.unloadIdle().catch(noop);
-            void this._musicRuntime?.unloadIdle().catch(noop);
+            releaseWork();
         }
     }
     _residentEnhancer() {
@@ -1511,4 +1968,32 @@ export class Harness extends ReadyResource {
         }
     }
 }
+function completionErrorEvent(message, name, error) {
+    const event = { type: 'error', message, ...completionErrorDetails(error) };
+    if (name) {
+        event.name = name;
+    }
+    return event;
+}
+function completionErrorDetails(error) {
+    if (!isRecord(error))
+        return {};
+    if (error.code === 'PENDING_ARGS_REDACTED') {
+        return { code: 'PENDING_ARGS_REDACTED', ...redactedRecoveryFields(error) };
+    }
+    if (error.name !== 'RunIncompatible') {
+        return {};
+    }
+    const match = errorMessage(error).match(/^Incompatible run log: (machine|log|xstate) version (\d{1,10}(?:\.\d{1,10}){0,3}(?:-[A-Za-z0-9.-]{1,16})?) \(expected (\d{1,10}(?:\.\d{1,10}){0,3}(?:-[A-Za-z0-9.-]{1,16})?)\)/);
+    if (!match) {
+        return {};
+    }
+    const kind = match[1];
+    const found = match[2];
+    const expected = match[3];
+    if ((kind !== 'machine' && kind !== 'log' && kind !== 'xstate') || !found || !expected) {
+        return {};
+    }
+    return { kind, found, expected };
+}
 function noop() { }