@bendyline/gezel-inference 0.0.0-stage → 0.1.0

package/dist/providers/native/provider-pool.js

@@ -0,0 +1,1166 @@
+/**
+ * ProviderPool — owns the lifecycle of every local-engine
+ * {@link LLMProvider} instance, keyed by `{provider, modelId,
+ * replicaIdx}`. Replaces the singleton "one supervisor per provider"
+ * model with a fleet so different gezels can run different models
+ * without cold-reload between turns.
+ *
+ * Capacity is enforced through {@link CapacityBroker}: every `ensure`
+ * reserves the model's `residentBytes`; when the budget is exhausted
+ * the pool LRU-evicts entries (shutting them down, releasing
+ * capacity, flushing any disk-cache work in the provider's
+ * `shutdown`) until the new entry fits.
+ *
+ * The pool is intentionally provider-agnostic — it doesn't know how
+ * to launch a llama-server or a Python MLX runtime. Callers inject a
+ * builder per `LocalProviderName`; tests inject a `MockProviderBuilder`
+ * that produces no-op LLMProviders.
+ */
+import { createLogger } from '@bendyline/gezel';
+import { AbortedWhileQueuedError } from '../queue.js';
+import { CapacityDeniedError, EngineBusyError, formatCapacityDenial, } from './capacity-broker.js';
+import { makeEngineKey, parseEngineKey, } from './engine-key.js';
+const log = createLogger('provider-pool');
+/**
+ * The stable, machine-shaped denial log line. Its shape is a contract
+ * with the eval harness — evals/src/failure-class.ts:118,
+ * evals/src/preflight.ts:196, and evals/src/runner.ts:981 all grep the
+ * daemon log for `capacity broker denied <key>: budget exhausted: …`
+ * to classify capacity denials as infra (not model) failures. Do not
+ * reword either side without updating the other.
+ */
+export function capacityDenialLogLine(key, reason) {
+    return `capacity broker denied ${key}: ${reason}`;
+}
+/**
+ * Slack allowed between the card's measured free memory and what a new engine
+ * needs on it. Covers the display framebuffer, driver allocations, and the lag
+ * between a `shutdown()` returning and the driver actually handing the memory
+ * back — all of which make a fresh reading pessimistic by a fraction of a GB.
+ * Large enough that only an unambiguous shortfall denies.
+ */
+const MEASURED_VRAM_TOLERANCE_BYTES = 1024 ** 3;
+/**
+ * A native child can be gone before a discrete-GPU driver publishes the
+ * released allocation as free again. Only pay this bounded wait after this
+ * pool actually evicted one of its own engines; a genuinely foreign tenant
+ * must still fail fast instead of adding latency to every denial.
+ */
+const MEASURED_VRAM_RECLAIM_POLL_MS = 1_000;
+const MEASURED_VRAM_RECLAIM_WAIT_MS = 5_000;
+function gb(bytes) {
+    return `${(bytes / 1024 ** 3).toFixed(1)} GB`;
+}
+/**
+ * Raw reason for a denial the device reading made, not the ledger. Keeps the
+ * `budget exhausted: would commit N against M` shape {@link capacityDenialLogLine}
+ * carries into the eval harness — this IS a capacity denial, measured rather
+ * than reserved, and any other shape would be reported as a model failure.
+ */
+export function measuredVramDenialReason(requiredBytes, freeBytes) {
+    const required = Math.round(requiredBytes);
+    const free = Math.round(freeBytes);
+    return `budget exhausted: would commit ${required} against ${free} (measured free graphics memory)`;
+}
+function emptyPooledQueueSummary() {
+    return {
+        running: 0,
+        runningInteractive: 0,
+        runningBackground: 0,
+        queuedInteractive: 0,
+        queuedBackground: 0,
+        ambientHeld: 0,
+        concurrency: 0,
+        interactiveConcurrency: 0,
+        backgroundConcurrency: 0,
+        maxConcurrency: 0,
+        active: [],
+        pending: [],
+    };
+}
+function positiveConcurrency(value) {
+    if (typeof value !== 'number' || !Number.isFinite(value) || value < 1)
+        return undefined;
+    return Math.floor(value);
+}
+/**
+ * Concurrent generation width the provider can serve right now.
+ *
+ * A supervised native launch is the strongest source: its retained `slots`
+ * diagnostic is the exact `--parallel` / server-width value of the live
+ * process. Older or wrapper providers may expose only `batch` or the queue's
+ * interactive cap, so use the widest of those compatible signals as the
+ * fallback. Never use total queue concurrency here — local queues carry an
+ * extra logical background lease that is not a native generation slot.
+ */
+export function liveProviderConcurrency(provider) {
+    const launchedSlots = positiveConcurrency(provider.engineLaunchSnapshot?.()?.diagnostics?.slots);
+    if (launchedSlots !== undefined)
+        return launchedSlots;
+    return Math.max(1, positiveConcurrency(provider.batch?.maxConcurrency) ?? 0, positiveConcurrency(provider.queue?.interactiveConcurrency) ?? 0);
+}
+/** Window over which {@link PoolEntry.recentHits} counts toward demand. */
+const DEMAND_WINDOW_MS = 60_000;
+const DEFAULT_DRAIN_WAIT_MS = 30_000;
+const DRAIN_POLL_MS = 500;
+/** Matches the provider queue's wait-notice cadence. */
+const DRAIN_NOTICE_REPEAT_MS = 5_000;
+/**
+ * In-flight work check, via the provider's own queue. A provider
+ * without a queue (cloud providers, test fakes) is treated as idle.
+ * Known blind spot: turns sent with `bypassQueue: true` (ask-specialist
+ * sub-sessions) never appear in the snapshot, so a drain can complete
+ * while one of those is still streaming — bounded harm, since those
+ * turns are short consultations with their own timeouts.
+ */
+function isBusy(entry) {
+    const snap = entry.provider.queue?.snapshot();
+    if (!snap)
+        return false;
+    return snap.running + snap.queuedInteractive + snap.queuedBackground > 0;
+}
+function throwIfAborted(signal) {
+    if (signal?.aborted)
+        throw new AbortedWhileQueuedError();
+}
+function drainWaitInfo(key, entry) {
+    const queue = entry.provider.queue;
+    const active = typeof queue?.describe === 'function' ? queue.describe().active : [];
+    return {
+        engineKey: key,
+        modelId: entry.parsed.modelId,
+        holders: active.map((a) => ({
+            ...(a.sessionId !== undefined ? { sessionId: a.sessionId } : {}),
+            ...(a.gezelId !== undefined ? { gezelId: a.gezelId } : {}),
+            ...(a.job !== undefined ? { job: a.job } : {}),
+            ...(a.actorLabel !== undefined ? { actorLabel: a.actorLabel } : {}),
+        })),
+    };
+}
+/** Native-child state for providers backed by a supervised local engine. */
+function engineLifecycle(entry) {
+    return entry.provider.engineLifecycleSnapshot?.();
+}
+/** Drop recent-hit timestamps older than the demand window. */
+function pruneHits(entry, now) {
+    const cutoff = now - DEMAND_WINDOW_MS;
+    if (entry.recentHits.length === 0 || entry.recentHits[0] >= cutoff)
+        return;
+    entry.recentHits = entry.recentHits.filter((t) => t >= cutoff);
+}
+export class ProviderPool {
+    broker;
+    builders;
+    now;
+    onChange;
+    drainWaitMs;
+    sleep;
+    gpuPanicGuard;
+    vramHeadroom;
+    serializeGpuLoads;
+    /** Tail of the serialized-load chain; each load awaits the previous. */
+    gpuLoadChain = Promise.resolve();
+    entries = new Map();
+    // Lock per key so two parallel `ensure` calls don't race to build
+    // the same engine. Hit path stays lock-free (Map.get).
+    buildLocks = new Map();
+    // In-flight evictions per key — parallel evict() calls join the
+    // same teardown instead of double-shutting-down the provider.
+    evicting = new Map();
+    /** Closed permanently once this user daemon adopts the machine broker. */
+    retiring = false;
+    retirementPromise = null;
+    constructor(opts) {
+        this.broker = opts.broker;
+        this.builders = opts.builders;
+        this.now = opts.now ?? Date.now;
+        if (opts.onChange)
+            this.onChange = opts.onChange;
+        this.drainWaitMs = opts.drainWaitMs ?? DEFAULT_DRAIN_WAIT_MS;
+        this.sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+        if (opts.gpuPanicGuard)
+            this.gpuPanicGuard = opts.gpuPanicGuard;
+        if (opts.vramHeadroom)
+            this.vramHeadroom = opts.vramHeadroom;
+        this.serializeGpuLoads =
+            opts.serializeGpuLoads ??
+                (process.platform === 'darwin' && process.env.GEZEL_SERIALIZE_GPU_LOADS !== 'off');
+    }
+    /**
+     * Run a model-LOAD (a `builder()` call, which maps weights onto the GPU)
+     * either immediately or serialized behind any in-flight load, per
+     * {@link ProviderPoolOptions.serializeGpuLoads}. Deadlock-free: the chain
+     * only ever waits for a load to finish, never for a turn or an eviction, and
+     * a load's failure doesn't wedge the chain (prior errors are swallowed).
+     */
+    async withGpuLoadSlot(load) {
+        if (!this.serializeGpuLoads)
+            return load();
+        const prior = this.gpuLoadChain;
+        let release;
+        this.gpuLoadChain = new Promise((r) => {
+            release = r;
+        });
+        await prior.catch(() => { });
+        try {
+            return await load();
+        }
+        finally {
+            release();
+        }
+    }
+    /**
+     * Resolve an engine key to a live, initialized provider. Hits the
+     * cache; on miss, evicts LRU as needed to clear `residentBytes`,
+     * then builds via the injected builder. The cost of eviction
+     * (provider.shutdown) is awaited so the caller never returns to a
+     * partially-spawned state.
+     */
+    async ensure(provider, modelId, replicaIdx, residentBytes, opts = {}) {
+        if (this.retiring) {
+            throw new Error('local engine pool retired after machine engine adoption');
+        }
+        await this.sweepFailedStarts();
+        const key = makeEngineKey(provider, modelId, replicaIdx);
+        const hit = this.entries.get(key);
+        if (hit && !hit.draining) {
+            hit.lastUsedAt = this.now();
+            return hit.provider;
+        }
+        if (hit?.draining) {
+            // The entry is being torn down to make room elsewhere. Handing
+            // it out would keep it busy forever (starvation); instead wait,
+            // on this caller's terms, for the eviction to settle, then
+            // rebuild below. If every drain gave up, or this caller's own
+            // wait ran out first, the entry survives — return it rather
+            // than double-building over a still-reserved key.
+            await this.waitForDrainToSettle(key, hit, opts);
+            const survived = this.entries.get(key);
+            if (survived) {
+                survived.lastUsedAt = this.now();
+                return survived.provider;
+            }
+            // Retirement may have started while this ensure was parked behind the
+            // drain. Never resurrect a user-owned engine after broker adoption.
+            if (this.retiring) {
+                throw new Error('local engine pool retired after machine engine adoption');
+            }
+        }
+        if (this.retiring) {
+            throw new Error('local engine pool retired after machine engine adoption');
+        }
+        const pending = this.buildLocks.get(key);
+        if (pending)
+            return pending;
+        const build = this.buildEntry(provider, modelId, replicaIdx, residentBytes, opts).finally(() => {
+            // Identity-guarded like `evicting` below: a retry build started after
+            // this one settled must not be evicted by this one's cleanup.
+            if (this.buildLocks.get(key) === build)
+                this.buildLocks.delete(key);
+        });
+        this.buildLocks.set(key, build);
+        return build;
+    }
+    async buildEntry(provider, modelId, replicaIdx, residentBytes, opts) {
+        const builder = this.builders[provider];
+        if (!builder) {
+            throw new Error(`no ProviderBuilder registered for '${provider}'`);
+        }
+        // GPU-panic guard: refuse to spawn a fresh local GPU engine right after a
+        // macOS GPU-driver kernel panic so we don't re-trigger the crash loop.
+        // Denied the same way as a capacity denial (throw a user-readable reason);
+        // no-op off macOS / when no guard is wired. See gpu-panic-guard.ts.
+        const panicDecision = this.gpuPanicGuard?.check(provider);
+        if (panicDecision?.blocked) {
+            throw new Error(panicDecision.reason ?? 'Local models are paused after a recent GPU kernel panic.');
+        }
+        const evictedOwnEngine = await this.makeRoom(residentBytes, opts);
+        // A caller may wait a whole turn for room; the service can begin
+        // shutting down in that time, and must not get a new engine for it.
+        if (this.retiring) {
+            throw new Error('local engine pool retired after machine engine adoption');
+        }
+        const key = makeEngineKey(provider, modelId, replicaIdx);
+        // Pre-flight: makeRoom evicts what it can, but on a memory-tight
+        // machine a model larger than the whole budget can never fit (no
+        // amount of eviction helps). Deny BEFORE spawning so we don't load
+        // gigabytes of weights only to immediately tear them down — and
+        // surface a human-readable reason instead of a raw byte comparison.
+        if (!this.broker.canReserve(residentBytes)) {
+            const c = this.broker.committed();
+            log.error(capacityDenialLogLine(key, this.broker.denialReason(residentBytes)));
+            throw new CapacityDeniedError(formatCapacityDenial({
+                modelLabel: modelId,
+                requestedBytes: residentBytes,
+                budgetBytes: c.budgetBytes,
+                committedBytes: c.committedBytes,
+                systemRamBytes: c.systemRamBytes,
+                pools: c.pools,
+                coResidencyBytes: c.ramSpillover.coResidencyBytes,
+            }));
+        }
+        // A model larger than the card streams part of itself from system RAM,
+        // and how much stays on the card is the build's plan. Checked against
+        // the whole card instead, it was refused whenever anything else used the
+        // GPU, so it is checked after the build, against the plan.
+        const pools = this.broker.committed().pools;
+        const largerThanCard = pools.kind === 'discrete-gpu' && residentBytes > pools.fastBytes;
+        if (!largerThanCard) {
+            const measured = await this.measuredVramDenial(modelId, residentBytes, evictedOwnEngine);
+            if (measured) {
+                log.error(capacityDenialLogLine(key, measured.reason));
+                throw new CapacityDeniedError(measured.message);
+            }
+        }
+        let built;
+        try {
+            built = await this.withGpuLoadSlot(() => builder({ modelId, replicaIdx }));
+        }
+        catch (err) {
+            if (!(err instanceof CapacityDeniedError))
+                throw err;
+            // A supervised child may have released its physical memory while its
+            // provider entry deliberately retained a worst-case reservation for a
+            // lazy restart. Usually that paper guarantee is harmless. It must not,
+            // however, make a NEW model's context planner conclude the machine is
+            // full before ProviderPool gets the builder's context-inclusive byte
+            // estimate and can run its ordinary makeRoom pass.
+            //
+            // Reclaim stopped, idle entries only after a real capacity denial, then
+            // retry the non-binding build once. Running/warm engines stay intact;
+            // an intrinsically too-large target simply returns the retried denial.
+            const reclaimed = await this.reclaimReleasedReservations();
+            if (reclaimed <= 0)
+                throw err;
+            log.info(`reclaimed ${gb(reclaimed)} of released model reservations for ${key}; retrying admission`);
+            built = await this.withGpuLoadSlot(() => builder({ modelId, replicaIdx }));
+        }
+        if (largerThanCard) {
+            const gpuNeed = built.provider.plannedGpuBytes?.() ?? residentBytes;
+            const measured = await this.measuredVramDenial(modelId, gpuNeed, evictedOwnEngine);
+            if (measured) {
+                await built.provider.shutdown().catch((err) => {
+                    log.warn(`shutdown after a graphics-memory denial for ${key}: ${err}`);
+                });
+                log.error(capacityDenialLogLine(key, measured.reason));
+                throw new CapacityDeniedError(measured.message);
+            }
+        }
+        // The builder's actual residentBytes wins if it differs from the
+        // pre-flight estimate (e.g. catalog override). Reserve the truth.
+        const finalBytes = built.residentBytes;
+        const r = this.broker.reserve(key, finalBytes);
+        if (!r.granted) {
+            // Reservation race: another concurrent build consumed budget
+            // between our makeRoom and reserve. Shut down what we just
+            // built, retry one more makeRoom + reserve loop.
+            await built.provider.shutdown().catch((err) => {
+                log.warn(`shutdown after failed reserve for ${key}: ${err}`);
+            });
+            await this.makeRoom(finalBytes, opts);
+            const r2 = this.broker.reserve(key, finalBytes);
+            if (!r2.granted) {
+                const c = this.broker.committed();
+                log.error(capacityDenialLogLine(key, r2.reason ?? this.broker.denialReason(finalBytes)));
+                throw new CapacityDeniedError(formatCapacityDenial({
+                    modelLabel: modelId,
+                    requestedBytes: finalBytes,
+                    budgetBytes: c.budgetBytes,
+                    committedBytes: c.committedBytes,
+                    systemRamBytes: c.systemRamBytes,
+                    pools: c.pools,
+                    coResidencyBytes: c.ramSpillover.coResidencyBytes,
+                }));
+            }
+            // Re-build under the now-clear budget.
+            const rebuilt = await this.withGpuLoadSlot(() => builder({ modelId, replicaIdx }));
+            this.installEntry(key, provider, modelId, replicaIdx, rebuilt.provider, rebuilt.residentBytes, rebuilt.modelWeightsBytes);
+            return rebuilt.provider;
+        }
+        this.installEntry(key, provider, modelId, replicaIdx, built.provider, finalBytes, built.modelWeightsBytes);
+        return built.provider;
+    }
+    installEntry(key, provider, modelId, replicaIdx, instance, residentBytes, modelWeightsBytes) {
+        const at = this.now();
+        this.entries.set(key, {
+            parsed: { provider, modelId, replicaIdx },
+            provider: instance,
+            residentBytes,
+            ...(typeof modelWeightsBytes === 'number' && Number.isFinite(modelWeightsBytes)
+                ? { modelWeightsBytes: Math.max(0, modelWeightsBytes) }
+                : {}),
+            lastUsedAt: at,
+            createdAt: at,
+            draining: false,
+            drainClaims: 0,
+            recentHits: [at],
+        });
+        // Registration is lazy: most providers do not launch their native child
+        // until the first request. Calling this "spawned" made preflight logs
+        // claim an engine existed when only a pool entry had been installed.
+        log.info(`registered provider ${key} (${residentBytes} bytes)`);
+        this.fireChange();
+    }
+    /**
+     * Mark a key as touched. Called by ChatManager after each turn
+     * so the LRU evictor doesn't kill the engine the user is
+     * actively using.
+     */
+    touch(key) {
+        const entry = this.entries.get(key);
+        if (!entry)
+            return;
+        const now = this.now();
+        entry.lastUsedAt = now;
+        entry.recentHits.push(now);
+        pruneHits(entry, now);
+    }
+    /** Returns true if the pool currently holds an entry for `key`. */
+    has(key) {
+        return this.entries.has(key);
+    }
+    /**
+     * True while a waiting request is draining `key` so another model can
+     * load. A turn bound to it that can rebind should step aside rather than
+     * start — see `SendAndWaitOpts.queue.yieldToSwap`.
+     */
+    isDraining(key) {
+        return this.entries.get(key)?.draining === true;
+    }
+    /**
+     * Drop entries whose engine failed to start, releasing the budget they hold.
+     *
+     * The reservation is taken when the builder returns — before the engine
+     * process exists, because the engine starts lazily on the first turn. A
+     * start that then fails (memory admission refused, spawn error, panic
+     * guard) leaves an entry that owns its whole `residentBytes` while running
+     * nothing, and it is never reclaimed on its own: the idle timer that would
+     * eventually release it is only ever armed by an engine that reached
+     * `running`. Observed as a 9.7 GB reservation on a machine whose engine
+     * process count was zero, against an 11.2 GB budget.
+     *
+     * Eviction is the whole repair. The next `ensure` rebuilds, which re-plans
+     * the context window against memory as it stands then rather than handing
+     * back a provider that will fail the same way.
+     */
+    async sweepFailedStarts() {
+        const dead = [];
+        for (const [key, entry] of this.entries) {
+            if (entry.draining || isBusy(entry))
+                continue;
+            const lifecycle = engineLifecycle(entry);
+            if (lifecycle?.startFailedAt !== undefined && !lifecycle.running)
+                dead.push(key);
+        }
+        for (const key of dead) {
+            try {
+                log.info(`releasing ${key}: engine never started`);
+                await this.evict(key);
+            }
+            catch (err) {
+                // A racing turn made it busy — the next sweep gets it.
+                log.warn(`sweepFailedStarts: ${key} not released: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+    }
+    /**
+     * Evict pool entries whose supervised native child has already stopped.
+     * Their bytes are capacity guarantees, not physical residency. Keeping the
+     * wrapper cached is useful until another model actually needs that promise;
+     * a capacity denial is the pressure signal that makes the released entries
+     * expendable.
+     */
+    async reclaimReleasedReservations() {
+        const candidates = [];
+        for (const [key, entry] of this.entries) {
+            if (entry.draining || isBusy(entry))
+                continue;
+            const lifecycle = engineLifecycle(entry);
+            if (lifecycle && !lifecycle.running) {
+                candidates.push({ key, bytes: entry.residentBytes });
+            }
+        }
+        if (candidates.length === 0)
+            return 0;
+        const reclaimed = await Promise.all(candidates.map(async ({ key, bytes }) => {
+            try {
+                return (await this.unloadIdle(key)) ? bytes : 0;
+            }
+            catch (err) {
+                // A new turn may have claimed the provider after the lifecycle
+                // sample. Never drain that racing work merely to repair accounting;
+                // leave the entry resident and let the retried denial describe what
+                // is still unavailable.
+                log.warn(`released reservation ${key} could not be reclaimed: ${err instanceof Error ? err.message : String(err)}`);
+                return 0;
+            }
+        }));
+        return reclaimed.reduce((sum, bytes) => sum + bytes, 0);
+    }
+    /**
+     * Immediately unload one resident engine, but only while it is idle.
+     *
+     * This is the user-triggered counterpart to the idle-retention timer. It
+     * deliberately refuses a busy engine instead of entering the ordinary
+     * bounded drain path: an "Unload now" click must never turn into a delayed
+     * teardown of work that started just after the status surface was sampled.
+     * Returns false when the engine has already gone away.
+     */
+    async unloadIdle(key) {
+        const existingEviction = this.evicting.get(key);
+        if (existingEviction) {
+            await existingEviction;
+            return true;
+        }
+        const entry = this.entries.get(key);
+        if (!entry)
+            return false;
+        if (isBusy(entry)) {
+            throw new EngineBusyError(`engine ${key} is currently serving requests and cannot be unloaded yet. Wait for current turns to finish.`);
+        }
+        await this.evict(key);
+        return true;
+    }
+    /**
+     * Unload every resident engine that is idle right now, leaving busy ones
+     * running. Returns the keys that stayed resident because they were serving
+     * a turn — the caller reports them; the ordinary LRU / idle-retention path
+     * collects them once they go quiet.
+     *
+     * This is what a live config reset uses. {@link shutdown} is the other
+     * shape — it force-evicts busy engines too — and is only correct when the
+     * process is going away (service shutdown, emergency stop), because a
+     * forced eviction kills whatever turn is streaming.
+     */
+    async releaseIdle() {
+        const busy = [];
+        for (const key of [...this.entries.keys()]) {
+            try {
+                await this.unloadIdle(key);
+            }
+            catch (err) {
+                if (err instanceof EngineBusyError) {
+                    busy.push(key);
+                    continue;
+                }
+                log.warn(`releaseIdle: ${key} failed to unload: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+        return busy;
+    }
+    /**
+     * Evict a specific entry. Awaits the provider's `shutdown` so the
+     * disk-cache flush (Tier 1 prior plan) completes before the broker
+     * releases capacity.
+     *
+     * When the engine has in-flight or queued turns and `force` is not
+     * set, the entry is marked `draining` (no new `ensure` hits, no new
+     * binds) and the eviction waits up to the caller's `drainWaitMs` for
+     * the queue to empty. On timeout it throws a "model is busy" error
+     * instead of killing live work — the caller (typically `makeRoom` on
+     * behalf of a request for a *different* model) surfaces that as a
+     * clear temporary-failure to its own caller. `force: true` (service
+     * shutdown) skips the wait entirely.
+     *
+     * Parallel `evict` calls for the same key wait independently, each
+     * with its own deadline and signal, and join the same teardown.
+     */
+    async evict(key, opts = {}) {
+        if (opts.force) {
+            const entry = this.entries.get(key);
+            return entry ? this.teardown(key, entry) : this.evicting.get(key);
+        }
+        const waitMs = opts.drainWaitMs ?? this.drainWaitMs;
+        const deadline = this.now() + waitMs;
+        for (;;) {
+            const entry = this.entries.get(key);
+            if (!entry) {
+                await this.evicting.get(key);
+                return;
+            }
+            // Idle check and teardown share a tick, so no turn can start between them.
+            if (!isBusy(entry))
+                return this.teardown(key, entry);
+            await this.drainFor(key, entry, waitMs, deadline, opts);
+        }
+    }
+    /**
+     * Hold `entry` draining until its queue empties, the entry is replaced,
+     * or this caller's deadline or signal ends the wait. The last caller to
+     * give up clears `draining`, so the engine serves hits again; a caller
+     * that saw it drain leaves the flag set for the teardown that follows.
+     */
+    async drainFor(key, entry, waitMs, deadline, opts) {
+        entry.drainClaims += 1;
+        if (!entry.draining) {
+            entry.draining = true;
+            this.fireChange();
+        }
+        log.info(`evict ${key}: engine busy — draining (${Number.isFinite(waitMs) ? `cap ${waitMs}ms` : 'until idle'})`);
+        const notice = this.drainNotifier(key, entry, opts);
+        let drained = false;
+        try {
+            while (isBusy(entry)) {
+                // A force-evict (service shutdown) may have torn the entry
+                // down underneath this drain — if our entry is no longer the
+                // resident one, the eviction is done.
+                if (this.entries.get(key) !== entry)
+                    return;
+                throwIfAborted(opts.signal);
+                if (this.now() >= deadline) {
+                    throw new EngineBusyError(`engine ${key} is busy serving requests and did not drain within ${Math.round(waitMs / 1000)}s — not evicting. Retry shortly, or wait for current turns to finish.`);
+                }
+                notice();
+                await this.sleep(DRAIN_POLL_MS);
+            }
+            drained = true;
+        }
+        finally {
+            entry.drainClaims -= 1;
+            if (!drained && entry.drainClaims === 0 && this.entries.get(key) === entry) {
+                entry.draining = false;
+                this.fireChange();
+            }
+        }
+    }
+    /**
+     * Wait for someone else's drain of `entry` to end — evicted, or given
+     * up — within this caller's own deadline and signal. Returning while
+     * the entry is still draining is the deadline: the caller takes the
+     * resident engine as it is.
+     */
+    async waitForDrainToSettle(key, entry, opts) {
+        const deadline = this.now() + (opts.drainWaitMs ?? this.drainWaitMs);
+        const notice = this.drainNotifier(key, entry, opts);
+        while (this.entries.get(key) === entry && entry.draining) {
+            throwIfAborted(opts.signal);
+            if (this.now() >= deadline)
+                return;
+            notice();
+            await this.sleep(DRAIN_POLL_MS);
+        }
+        await this.evicting.get(key);
+    }
+    /** Rate-limited {@link EngineWaitOpts.onDrainWait}: once at the start, then every few seconds. */
+    drainNotifier(key, entry, opts) {
+        const notify = opts.onDrainWait;
+        if (!notify)
+            return () => { };
+        let lastAt = Number.NEGATIVE_INFINITY;
+        return () => {
+            const now = this.now();
+            if (now - lastAt < DRAIN_NOTICE_REPEAT_MS)
+                return;
+            lastAt = now;
+            try {
+                notify(drainWaitInfo(key, entry));
+            }
+            catch (err) {
+                log.warn(`onDrainWait threw: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        };
+    }
+    /**
+     * Remove `entry` and shut its provider down. The removal is synchronous,
+     * so nothing can bind to it once this is called; the shutdown is shared
+     * by every caller evicting the same entry.
+     */
+    teardown(key, entry) {
+        if (this.entries.get(key) !== entry)
+            return this.evicting.get(key) ?? Promise.resolve();
+        this.entries.delete(key);
+        const prior = this.evicting.get(key);
+        const run = (async () => {
+            await prior;
+            try {
+                await entry.provider.shutdown();
+            }
+            catch (err) {
+                log.warn(`shutdown threw for ${key}: ${err instanceof Error ? err.message : String(err)}`);
+            }
+            this.broker.release(key);
+            log.info(`evicted ${key}`);
+            this.fireChange();
+        })().finally(() => {
+            if (this.evicting.get(key) === run)
+                this.evicting.delete(key);
+        });
+        this.evicting.set(key, run);
+        return run;
+    }
+    /**
+     * Deny a spawn the card cannot physically take, whatever the ledger says.
+     *
+     * Runs after `makeRoom`, so everything this daemon was willing to release is
+     * already gone and the reading reflects what is genuinely left. Discrete
+     * cards only: on a unified host there is one pool, the RAM share already
+     * governs it, and the OS pages normally rather than falling off a cliff.
+     *
+     * A model larger than the card is deliberately streamed from system RAM, so
+     * the requirement is capped at the fast pool — otherwise this would refuse
+     * exactly the big-MoE case the two-pool budget was built to allow.
+     */
+    async measuredVramDenial(modelId, residentBytes, waitForOwnedReclaim) {
+        const vramHeadroom = this.vramHeadroom;
+        if (!vramHeadroom)
+            return null;
+        const committed = this.broker.committed();
+        if (!committed.enforced || committed.pools.kind !== 'discrete-gpu')
+            return null;
+        const required = Math.min(residentBytes, committed.pools.fastBytes);
+        if (required <= 0)
+            return null;
+        const probe = async () => await vramHeadroom().catch((err) => {
+            log.debug(`vram headroom probe failed: ${err instanceof Error ? err.message : String(err)}`);
+            return null;
+        });
+        let reading = await probe();
+        let freeBytes = reading?.freeBytes;
+        if (typeof freeBytes !== 'number' || !Number.isFinite(freeBytes))
+            return null;
+        if (freeBytes + MEASURED_VRAM_TOLERANCE_BYTES >= required)
+            return null;
+        if (waitForOwnedReclaim) {
+            const attempts = Math.ceil(MEASURED_VRAM_RECLAIM_WAIT_MS / MEASURED_VRAM_RECLAIM_POLL_MS);
+            log.info(`graphics memory still reports ${gb(freeBytes)} free after an owned engine eviction; ` +
+                `waiting up to ${MEASURED_VRAM_RECLAIM_WAIT_MS / 1_000}s for driver reclamation`);
+            for (let attempt = 1; attempt <= attempts; attempt += 1) {
+                await this.sleep(MEASURED_VRAM_RECLAIM_POLL_MS);
+                reading = await probe();
+                freeBytes = reading?.freeBytes;
+                // Preserve the guard's fail-open behavior when the device probe stops
+                // producing a usable reading during the reclaim window.
+                if (typeof freeBytes !== 'number' || !Number.isFinite(freeBytes))
+                    return null;
+                if (freeBytes + MEASURED_VRAM_TOLERANCE_BYTES >= required) {
+                    log.info(`graphics memory reclaimed after ${attempt * MEASURED_VRAM_RECLAIM_POLL_MS}ms ` +
+                        `(${gb(freeBytes)} free)`);
+                    return null;
+                }
+            }
+        }
+        const total = reading?.totalBytes;
+        const reason = measuredVramDenialReason(required, freeBytes);
+        const capacity = [
+            `Not enough graphics memory to load ${modelId} (about ${gb(required)} on the card). `,
+            `The graphics card has ${gb(freeBytes)} free`,
+            typeof total === 'number' && Number.isFinite(total) ? ` of ${gb(total)}` : '',
+            '. ',
+        ].join('');
+        const message = waitForOwnedReclaim
+            ? [
+                capacity,
+                `Gezel unloaded its previous model and waited ${MEASURED_VRAM_RECLAIM_WAIT_MS / 1_000} seconds, `,
+                'but the graphics driver has not made enough memory available yet. Wait a few seconds ',
+                'and try again. If this keeps happening, another program or a second Gezel engine may ',
+                'also be using the graphics card.',
+            ].join('')
+            : [
+                capacity,
+                'Gezel has no other idle model in this engine left to unload, so another program, a ',
+                'second Gezel engine, or the graphics driver may be holding that memory. Close the other ',
+                'graphics-card workload, or wait a few seconds and try again.',
+            ].join('');
+        return { reason, message };
+    }
+    /**
+     * Evict entries until the broker reports headroom for `needed`
+     * bytes (or the pool is empty). Victim order: oldest *idle* entries
+     * first; only when every candidate is busy does it pick a busy one —
+     * one already draining for another caller first — whose eviction
+     * then drains on this caller's terms rather than killing in-flight
+     * turns. Pool-internal; callers never invoke directly.
+     *
+     * Idle victims are evicted *concurrently* (a model switch on a
+     * memory-tight box often has to clear several idle resident models
+     * to fit the incoming one): idle `shutdown()` doesn't drain, so the
+     * per-engine disk-cache flush of N victims runs in parallel instead
+     * of paying each one serially before the new model can spawn. The
+     * busy path stays serial to preserve the bounded-drain semantics.
+     */
+    async makeRoom(needed, opts = {}) {
+        let evictedOwnEngine = false;
+        while (!this.broker.canReserve(needed) && this.entries.size > 0) {
+            const idleBatch = this.idleVictimsFor(needed);
+            if (idleBatch.length > 0) {
+                const evicted = await Promise.all(idleBatch.map(async (k) => {
+                    try {
+                        await this.evict(k);
+                        return true;
+                    }
+                    catch (err) {
+                        // Idle evictions shouldn't throw (no drain wait), but a
+                        // shutdown() rejection must not abort the sibling evicts
+                        // or the whole makeRoom — log and let the loop re-check.
+                        log.warn(`makeRoom: idle evict ${k} failed: ${err instanceof Error ? err.message : String(err)}`);
+                        return false;
+                    }
+                }));
+                evictedOwnEngine ||= evicted.some(Boolean);
+                continue;
+            }
+            // No idle candidates — evict a single busy/draining victim. This
+            // path waits on the caller's terms (and may throw on a busy engine
+            // that won't drain in time, which propagates to the caller).
+            const victim = this.lruKey();
+            if (!victim) {
+                // Only teardowns are left in flight — wait for one to release
+                // its memory and re-check the budget.
+                if (this.evicting.size === 0)
+                    break;
+                await Promise.race([...this.evicting.values()]).catch(() => { });
+                continue;
+            }
+            await this.evict(victim, opts);
+            evictedOwnEngine = true;
+        }
+        return evictedOwnEngine;
+    }
+    /**
+     * The idle (non-busy, non-draining) eviction victims — oldest-first —
+     * whose combined `residentBytes`, once released, would give `needed`
+     * room to fit under the budget. Returns `[]` when there are no idle
+     * candidates (the caller then falls back to a single busy eviction)
+     * or when no eviction is required. Lets {@link makeRoom} batch idle
+     * evictions in parallel instead of one-at-a-time.
+     */
+    idleVictimsFor(needed) {
+        // The broker owns the arithmetic: two constraints bind here (the total
+        // budget and, when RAM spillover is off, the co-residency ceiling) and
+        // sizing the batch against only the first left this returning no victims
+        // while `canReserve` stayed false — makeRoom would then fall through to
+        // evicting a BUSY engine instead of the idle ones sitting right there.
+        let deficit = this.broker.shortfallFor(needed);
+        if (deficit <= 0)
+            return [];
+        const now = this.now();
+        const idle = [];
+        for (const [key, entry] of this.entries) {
+            if (entry.draining)
+                continue;
+            if (isBusy(entry))
+                continue;
+            pruneHits(entry, now);
+            idle.push({
+                key,
+                at: entry.lastUsedAt,
+                bytes: entry.residentBytes,
+                demand: entry.recentHits.length,
+            });
+        }
+        // Demand-weighted LRU: evict the LEAST-recently-DEMANDED idle model first
+        // (fewest hits in the window), breaking ties by oldest `lastUsedAt`. Keeps
+        // a model that several tenants hit intermittently resident instead of
+        // dropping it just because one caller idled. Falls back to pure-oldest
+        // when demand is uniform (e.g. all single-hit), preserving prior behavior.
+        idle.sort((a, b) => a.demand - b.demand || a.at - b.at);
+        const out = [];
+        for (const v of idle) {
+            if (deficit <= 0)
+                break;
+            out.push(v.key);
+            deficit -= v.bytes;
+        }
+        return out;
+    }
+    lruKey() {
+        let oldestIdle = null;
+        let draining = null;
+        let oldestBusy = null;
+        for (const [key, entry] of this.entries) {
+            // Joining a drain already under way frees memory that is leaving
+            // anyway; starting a second one would unload more than needed.
+            if (entry.draining) {
+                draining ??= key;
+                continue;
+            }
+            if (isBusy(entry)) {
+                if (oldestBusy === null || entry.lastUsedAt < oldestBusy.at) {
+                    oldestBusy = { key, at: entry.lastUsedAt };
+                }
+            }
+            else if (oldestIdle === null || entry.lastUsedAt < oldestIdle.at) {
+                oldestIdle = { key, at: entry.lastUsedAt };
+            }
+        }
+        return oldestIdle?.key ?? draining ?? oldestBusy?.key ?? null;
+    }
+    /**
+     * Shut down everything the pool owns. Used at service shutdown —
+     * forces eviction without waiting for busy engines to drain.
+     */
+    async shutdown() {
+        this.retiring = true;
+        const keys = [...this.entries.keys()];
+        await Promise.all(keys.map((k) => this.evict(k, { force: true })));
+    }
+    /**
+     * Permanently close the pool to new work, wait for builds already admitted,
+     * then drain every interactive and background queue before shutdown. Unlike
+     * service shutdown this never force-kills a turn and has no turn-length cap.
+     */
+    async retire() {
+        if (this.retirementPromise)
+            return this.retirementPromise;
+        this.retiring = true;
+        const run = (async () => {
+            while (this.buildLocks.size > 0) {
+                await Promise.allSettled(Array.from(this.buildLocks.values()));
+            }
+            while (this.entries.size > 0) {
+                const keys = [...this.entries.keys()];
+                await Promise.all(keys.map((key) => this.evict(key, { drainWaitMs: Number.POSITIVE_INFINITY })));
+            }
+        })();
+        this.retirementPromise = run;
+        try {
+            await run;
+        }
+        catch (error) {
+            if (this.retirementPromise === run)
+                this.retirementPromise = null;
+            throw error;
+        }
+    }
+    /**
+     * Reconcile the resident set toward `target` — a map of
+     * `(provider, modelId) → desiredReplicaCount`. Spawns missing
+     * replicas (subject to capacity) and evicts replicas at indices
+     * `>= count`. Used by the UX layer when the user changes the
+     * clone-count picker.
+     *
+     * `ensureRes` resolves residentBytes for `(provider, modelId)`;
+     * typically a closure over the catalog.
+     */
+    async reconcile(target, ensureRes) {
+        // 1. Evict replicas at indices >= desired clone count.
+        const wanted = new Set();
+        for (const { provider, modelId, clones } of target) {
+            for (let i = 0; i < clones; i++) {
+                wanted.add(makeEngineKey(provider, modelId, i));
+            }
+        }
+        // Be careful: only evict entries whose (provider, modelId)
+        // appears in `target` at all. Entries the caller didn't enumerate
+        // are pre-existing and left alone (typical chat-driven spawns).
+        const namedSet = new Set(target.map((t) => `${t.provider}:${t.modelId}`));
+        const toEvict = [];
+        for (const [key, entry] of this.entries) {
+            const fp = `${entry.parsed.provider}:${entry.parsed.modelId}`;
+            if (namedSet.has(fp) && !wanted.has(key))
+                toEvict.push(key);
+        }
+        for (const key of toEvict) {
+            try {
+                await this.evict(key);
+            }
+            catch (err) {
+                // Busy engine that didn't drain in time — leave it resident
+                // and carry on; the user can re-apply the clone change once
+                // its turns finish.
+                log.warn(`reconcile: could not evict ${key}: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+        // 2. Spawn missing replicas (best-effort under capacity).
+        for (const { provider, modelId, clones } of target) {
+            for (let i = 0; i < clones; i++) {
+                const key = makeEngineKey(provider, modelId, i);
+                if (this.entries.has(key))
+                    continue;
+                try {
+                    await this.ensure(provider, modelId, i, ensureRes(provider, modelId));
+                }
+                catch (err) {
+                    log.warn(`reconcile: failed to spawn ${key}: ${err instanceof Error ? err.message : String(err)}`);
+                }
+            }
+        }
+    }
+    /**
+     * Pick a replica index to bind a new session to. Returns the
+     * least-loaded existing replica of `(provider, modelId)` based on
+     * an external `sessionsPerKey` map (the chat manager owns the
+     * tally; the pool stays state-free about sessions). If no replica
+     * exists, returns 0 so the caller spawns the first one.
+     */
+    pickReplicaForBind(provider, modelId, sessionsPerKey) {
+        let bestIdx = -1;
+        let bestLoad = Number.POSITIVE_INFINITY;
+        for (const [key, entry] of this.entries) {
+            if (entry.parsed.provider !== provider)
+                continue;
+            if (entry.parsed.modelId !== modelId)
+                continue;
+            if (entry.draining)
+                continue; // being evicted — don't bind new sessions to it
+            const load = sessionsPerKey.get(key) ?? 0;
+            if (load < bestLoad) {
+                bestLoad = load;
+                bestIdx = entry.parsed.replicaIdx;
+            }
+        }
+        return bestIdx >= 0 ? bestIdx : 0;
+    }
+    /**
+     * Enumerate `(provider, modelId, replicaIdx)` triples currently
+     * resident. The chat layer uses this to validate a session's
+     * persisted `engineKey` after restart.
+     */
+    resolveExisting(key) {
+        const entry = this.entries.get(key);
+        if (entry)
+            return entry.parsed;
+        return parseEngineKey(key);
+    }
+    /**
+     * Existing replica indices for a `(provider, modelId)` pair,
+     * sorted ascending. Used to discover "is replica 1 already
+     * resident?" without scanning the whole pool from the outside.
+     */
+    existingReplicas(provider, modelId) {
+        const out = [];
+        for (const entry of this.entries.values()) {
+            if (entry.parsed.provider === provider && entry.parsed.modelId === modelId) {
+                out.push(entry.parsed.replicaIdx);
+            }
+        }
+        return out.sort((a, b) => a - b);
+    }
+    snapshot() {
+        const committed = this.broker.committed();
+        const entries = [];
+        for (const [key, e] of this.entries) {
+            const lifecycle = engineLifecycle(e);
+            entries.push({
+                key,
+                provider: e.parsed.provider,
+                modelId: e.parsed.modelId,
+                replicaIdx: e.parsed.replicaIdx,
+                residentBytes: e.residentBytes,
+                ...(e.modelWeightsBytes !== undefined ? { modelWeightsBytes: e.modelWeightsBytes } : {}),
+                lastUsedAt: e.lastUsedAt,
+                createdAt: e.createdAt,
+                draining: e.draining,
+                ...(lifecycle ? { lifecycle } : {}),
+            });
+        }
+        return {
+            entries: entries.sort((a, b) => a.key.localeCompare(b.key)),
+            committedBytes: committed.committedBytes,
+            budgetBytes: committed.budgetBytes,
+            enforced: committed.enforced,
+            systemRamBytes: committed.systemRamBytes,
+            autoBudgetBytes: committed.autoBudgetBytes,
+            overridden: committed.overridden,
+            pools: committed.pools,
+            ramSpillover: committed.ramSpillover,
+        };
+    }
+    /**
+     * Return already-created replicas for one exact model without allocating,
+     * initializing, touching LRU state, or evicting anything. Cache warming and
+     * telemetry use this peek so a read-like request cannot cold-start an engine.
+     */
+    peekProvidersForModel(provider, modelId) {
+        return [...this.entries.values()]
+            .filter((entry) => !entry.draining && entry.parsed.provider === provider && entry.parsed.modelId === modelId)
+            .sort((a, b) => b.lastUsedAt - a.lastUsedAt)
+            .map((entry) => entry.provider);
+    }
+    /**
+     * Fold every resident replica's live queue into one summary per
+     * {@link LocalProviderName}. Under the pool architecture local
+     * providers aren't seeded as singletons in `ChatManager.providers`, so
+     * `/api/queues` — which reads only `getProviderIfReady(name).queue` —
+     * otherwise misses ALL pooled work: chat turns and background one-shots
+     * alike (weekly digest, about.md drafts, memory extraction). That blind
+     * spot is what made the engine pill read "Idle" while a one-shot decoded
+     * on the GPU.
+     *
+     * Counts sum across a provider's replicas; `active` / `pending` (with
+     * their job labels) concatenate. Draining replicas are included — they
+     * can still be finishing in-flight turns. Replicas whose provider
+     * exposes no queue (cloud, test fakes) contribute nothing.
+     */
+    /**
+     * Launch provenance of every pooled provider whose engine process is
+     * live right now — provider + model attribution from the pool key, the
+     * snapshot (pid, start time, granted context) from the provider's
+     * supervisor. Providers without a supervised engine contribute nothing.
+     */
+    engineLaunchSnapshots() {
+        const out = [];
+        for (const entry of this.entries.values()) {
+            const snapshot = entry.provider.engineLaunchSnapshot?.();
+            if (!snapshot)
+                continue;
+            out.push({ provider: entry.parsed.provider, modelId: entry.parsed.modelId, snapshot });
+        }
+        return out;
+    }
+    queueSummaries() {
+        const out = new Map();
+        for (const entry of this.entries.values()) {
+            const q = entry.provider.queue;
+            if (!q)
+                continue;
+            const d = q.describe();
+            const name = entry.parsed.provider;
+            const cur = out.get(name) ?? emptyPooledQueueSummary();
+            cur.running += d.running;
+            cur.runningInteractive += d.runningInteractive;
+            cur.runningBackground += d.runningBackground;
+            cur.queuedInteractive += d.queuedInteractive;
+            cur.queuedBackground += d.queuedBackground;
+            cur.ambientHeld += d.ambientHeld;
+            cur.concurrency += d.concurrency;
+            cur.interactiveConcurrency += d.interactiveConcurrency;
+            cur.backgroundConcurrency += d.backgroundConcurrency;
+            cur.maxConcurrency += liveProviderConcurrency(entry.provider);
+            cur.active.push(...d.active);
+            cur.pending.push(...d.pending);
+            out.set(name, cur);
+        }
+        return out;
+    }
+    /**
+     * Cancel a pending queue entry on whichever resident replica of
+     * `provider` holds it — the write side of {@link queueSummaries},
+     * which the singleton `getProviderIfReady` path can't reach for
+     * pool-routed local engines. Queue ids are replica-local (each
+     * {@link ProviderQueue} counts from 1), so we cancel on the first
+     * replica of this provider whose queue knows the id; in the common
+     * single-replica case that is unambiguous, and a stale id simply
+     * matches nothing. Returns true when an entry was removed.
+     */
+    cancelPendingQueueItem(provider, id) {
+        for (const entry of this.entries.values()) {
+            if (entry.parsed.provider !== provider)
+                continue;
+            if (entry.provider.queue?.cancelPending(id))
+                return true;
+        }
+        return false;
+    }
+    /** Reorder a pending queue entry across this provider's replicas. See
+     *  {@link cancelPendingQueueItem} for the id-matching contract. */
+    movePendingQueueItem(provider, id, direction) {
+        for (const entry of this.entries.values()) {
+            if (entry.parsed.provider !== provider)
+                continue;
+            if (entry.provider.queue?.movePending(id, direction))
+                return true;
+        }
+        return false;
+    }
+    fireChange() {
+        if (!this.onChange)
+            return;
+        try {
+            this.onChange(this.snapshot());
+        }
+        catch (err) {
+            log.warn(`onChange listener threw: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+}