@otto-code/brain 0.8.12 → 0.8.14

package/dist/service/activity.js

@@ -9,7 +9,7 @@ var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (
     if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
     return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
 };
-var _ReasoningTracker_instances, _ReasoningTracker_requests, _ReasoningTracker_slots, _ReasoningTracker_tails, _ReasoningTracker_inlineReasoning, _ReasoningTracker_listeners, _ReasoningTracker_lastSnapshot, _ReasoningTracker_announce;
+var _ReasoningTracker_instances, _ReasoningTracker_requests, _ReasoningTracker_listeners, _ReasoningTracker_lastSnapshot, _ReasoningTracker_counter, _ReasoningTracker_announce, _ReasoningTracker_setSlot, _ReasoningTracker_observe, _ReasoningTracker_touch, _ReasoningTracker_drop;
 /**
  * What long-running work currently owns the brain, and which stage each live
  * inference request has reached.
@@ -179,6 +179,23 @@ export function chunkHasContent(text) {
         /"tool_calls"\s*:\s*\[\s*\{/u.test(text) ||
         /"type"\s*:\s*"(?:tool_use|input_json_delta)"/u.test(text));
 }
+/**
+ * How long a request may be silent before the reaper will consider it at all.
+ *
+ * Long enough to cover the gap between proxy dispatch and llama-server picking
+ * the task up, which is the one window where a healthy request and an idle
+ * engine legitimately coexist.
+ */
+const INFERENCE_QUIET_MS = 5000;
+/**
+ * How many consecutive contradicting samples clear a request.
+ *
+ * More than one because a single sample can catch a real dispatch mid-flight;
+ * small because the status sampler runs about once a second while anything
+ * claims to be busy, so a genuine leak is gone in seconds rather than surviving
+ * until someone restarts the service.
+ */
+const INFERENCE_STRIKES = 3;
 /**
  * Which in-flight completions are currently mid-thought.
  *
@@ -197,18 +214,9 @@ export class ReasoningTracker {
     constructor() {
         _ReasoningTracker_instances.add(this);
         _ReasoningTracker_requests.set(this, new Map());
-        /**
-         * The llama-server slot a request was pinned to at dispatch, so its proxy-side
-         * stage can be attributed to the engine row the panel actually shows. Set once
-         * per request (see `setSlot`), never on the per-chunk path.
-         */
-        _ReasoningTracker_slots.set(this, new Map());
-        /** Tail of the last transport chunk, so a field name split by TCP is still detected. */
-        _ReasoningTracker_tails.set(this, new Map());
-        /** Models/runtimes that leave reasoning inline as `<think>…</think>`. */
-        _ReasoningTracker_inlineReasoning.set(this, new Set());
         _ReasoningTracker_listeners.set(this, new Set());
-        _ReasoningTracker_lastSnapshot.set(this, "0:0:0:0");
+        _ReasoningTracker_lastSnapshot.set(this, "");
+        _ReasoningTracker_counter.set(this, 0);
     }
     /**
      * Watch stage counts, not the per-chunk traffic behind them.
@@ -221,74 +229,142 @@ export class ReasoningTracker {
         __classPrivateFieldGet(this, _ReasoningTracker_listeners, "f").add(listener);
         return () => __classPrivateFieldGet(this, _ReasoningTracker_listeners, "f").delete(listener);
     }
-    /** A completion was dispatched to llama-server and awaits its first output delta. */
-    begin(requestId) {
-        if (__classPrivateFieldGet(this, _ReasoningTracker_requests, "f").has(requestId))
-            return;
-        __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "processing");
-        __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
-    }
     /**
-     * Record the engine slot this request was pinned to. Called exactly once per
-     * request, at dispatch - the pin is injected into the outbound body before
-     * the request goes out, so the association exists before the first chunk and
-     * `observe` never has to learn about it.
+     * Open a lease for a completion that has just been dispatched to
+     * llama-server and awaits its first output delta.
+     *
+     * A lease rather than an id the caller carries around, because every stuck
+     * "thinking" this tracker has produced was a release that did not happen on
+     * some branch of the proxy's event wiring. A lease makes both halves of that
+     * bug unrepresentable: nothing can advance a request without holding its
+     * lease, and a released lease is inert, so a chunk that lands after the
+     * release cannot resurrect the request it belongs to. (The same shape as
+     * `beginActivity` above, for the same reason.)
      *
-     * Idempotent and self-cleaning: a repeat for the same slot is a no-op, and a
-     * different slot replaces it, so a request that somehow moves slots (a
-     * restarted engine hands a task out again) reports where it is now.
+     * Ids are minted here rather than by the caller: two callers sharing one id
+     * would silently share one request's state.
      */
-    setSlot(requestId, slotId) {
-        if (!Number.isInteger(slotId) || slotId < 0)
-            return;
-        if (__classPrivateFieldGet(this, _ReasoningTracker_slots, "f").get(requestId) === slotId)
-            return;
-        __classPrivateFieldGet(this, _ReasoningTracker_slots, "f").set(requestId, slotId);
+    begin() {
+        __classPrivateFieldSet(this, _ReasoningTracker_counter, __classPrivateFieldGet(this, _ReasoningTracker_counter, "f") + 1, "f");
+        const id = `s${__classPrivateFieldGet(this, _ReasoningTracker_counter, "f")}`;
+        const now = Date.now();
+        __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(id, {
+            stage: "processing",
+            slotId: null,
+            tail: "",
+            inlineReasoning: false,
+            startedAt: now,
+            lastSignalAt: now,
+            strikes: 0,
+        });
         __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
+        let open = true;
+        return {
+            id,
+            observe: (text) => {
+                if (!open)
+                    return;
+                __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_observe).call(this, id, text);
+            },
+            setSlot: (slotId) => {
+                if (!open)
+                    return;
+                __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_setSlot).call(this, id, slotId);
+            },
+            end: () => {
+                if (!open)
+                    return;
+                open = false;
+                __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_drop).call(this, [id]);
+            },
+        };
     }
-    /** Note a chunk of `requestId`'s stream. Cheap enough to call per chunk. */
-    observe(requestId, text) {
-        const current = __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").get(requestId);
-        if (current === "generating")
-            return;
-        if (!current)
-            __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "processing");
-        // Node can split an SSE JSON field name at any byte. Keeping a small tail
-        // makes stage recognition independent of transport chunk boundaries without
-        // parsing or retaining the generated content itself.
-        const combined = `${__classPrivateFieldGet(this, _ReasoningTracker_tails, "f").get(requestId) ?? ""}${text}`;
-        __classPrivateFieldGet(this, _ReasoningTracker_tails, "f").set(requestId, combined.slice(-128));
-        if (__classPrivateFieldGet(this, _ReasoningTracker_inlineReasoning, "f").has(requestId)) {
-            if (combined.includes("</think>")) {
-                __classPrivateFieldGet(this, _ReasoningTracker_inlineReasoning, "f").delete(requestId);
-                __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "generating");
-                __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
-            }
-            return;
-        }
-        if (combined.includes("<think>")) {
-            __classPrivateFieldGet(this, _ReasoningTracker_inlineReasoning, "f").add(requestId);
-            __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "thinking");
-            __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
-            return;
-        }
-        if (chunkHasContent(combined)) {
-            __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "generating");
-            __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
-            return;
+    /**
+     * Forget every slot pin, because the engine's slots did not survive its
+     * relaunch.
+     *
+     * The mirror of `Scheduler.forgetSlots`, and required for the same reason: a
+     * pin that outlives the process it named is no longer evidence. Worse than
+     * useless, in fact - a stale pin can collide with a NEW request's slot id,
+     * and the reaper would read that unrelated busy row as proof the dead request
+     * is still alive. Dropping the pins demotes those requests to the
+     * conservative unpinned rule, which clears them once the engine is quiet.
+     */
+    forgetSlots() {
+        let changed = false;
+        for (const state of __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").values()) {
+            if (state.slotId === null)
+                continue;
+            state.slotId = null;
+            changed = true;
         }
-        if (chunkHasReasoning(combined) && current !== "thinking") {
-            __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").set(requestId, "thinking");
+        if (changed)
             __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
-        }
     }
-    /** Forget the request. Must be called on end *and* on error, or the flag sticks. */
-    end(requestId) {
-        __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").delete(requestId);
-        __classPrivateFieldGet(this, _ReasoningTracker_slots, "f").delete(requestId);
-        __classPrivateFieldGet(this, _ReasoningTracker_tails, "f").delete(requestId);
-        __classPrivateFieldGet(this, _ReasoningTracker_inlineReasoning, "f").delete(requestId);
-        __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
+    /**
+     * Drop tracked requests the engine's own account of itself contradicts.
+     *
+     * The safety net under the lease, and it exists because `active` outranks
+     * every engine signal on the rail: one release that never happened claims
+     * "thinking" until the service restarts. The ops tracker already refuses that
+     * bargain by probing the recorded pid, on the principle that a status stuck
+     * on "calibrating" forever is worse than no status at all. This is the
+     * inference half of the same rule.
+     *
+     * **It must never clear valid work**, so it acts only on positive evidence,
+     * and only on evidence a live request could not produce:
+     *
+     *  1. A request that has sent a chunk (or been pinned, or been dispatched)
+     *     within `INFERENCE_QUIET_MS` is alive. A streaming request is therefore
+     *     never a candidate at all, whatever the engine says this instant.
+     *  2. A PINNED request is checked against its own slot. llama-server marks a
+     *     slot processing for the whole task, prefill included, so a request that
+     *     is genuinely running makes its row busy. That row being idle is the
+     *     contradiction. This is what lets one chat's leak be cleared while
+     *     another chat keeps generating.
+     *  3. An UNPINNED request - or any request when the engine reports no
+     *     per-slot rows - cannot be attributed to a row, so it is cleared only
+     *     when the engine reports nothing running at all. Ambiguity is not
+     *     evidence.
+     *  4. The contradiction has to hold for `INFERENCE_STRIKES` samples in a row.
+     *     A single sample can race the dispatch window, where a request has been
+     *     begun and the engine has not picked it up yet; a run of them cannot.
+     *
+     * A failed slot sample is not evidence either, and reconciles nothing.
+     *
+     * Returns what it reaped, so the caller can log a leak rather than silently
+     * paper over it.
+     */
+    reconcile(truth) {
+        if (truth.busyCount === null)
+            return [];
+        const now = Date.now();
+        const quietBefore = now - INFERENCE_QUIET_MS;
+        const reaped = [];
+        for (const [id, state] of __classPrivateFieldGet(this, _ReasoningTracker_requests, "f")) {
+            if (state.lastSignalAt > quietBefore) {
+                state.strikes = 0;
+                continue;
+            }
+            const contradicted = state.slotId !== null && truth.busySlots
+                ? !truth.busySlots.has(state.slotId)
+                : truth.busyCount === 0;
+            if (!contradicted) {
+                state.strikes = 0;
+                continue;
+            }
+            state.strikes += 1;
+            if (state.strikes < INFERENCE_STRIKES)
+                continue;
+            reaped.push({
+                id,
+                stage: state.stage,
+                slotId: state.slotId,
+                ageMs: now - state.startedAt,
+            });
+        }
+        __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_drop).call(this, reaped.map((entry) => entry.id));
+        return reaped;
     }
     get active() {
         return this.snapshot.thinking > 0;
@@ -305,19 +381,18 @@ export class ReasoningTracker {
             generating: 0,
         };
         let slotStages;
-        for (const [requestId, stage] of __classPrivateFieldGet(this, _ReasoningTracker_requests, "f")) {
-            result[stage] += 1;
-            const slot = __classPrivateFieldGet(this, _ReasoningTracker_slots, "f").get(requestId);
-            if (slot === undefined)
+        for (const state of __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").values()) {
+            result[state.stage] += 1;
+            if (state.slotId === null)
                 continue;
-            (slotStages ?? (slotStages = {}))[String(slot)] = stage;
+            (slotStages ?? (slotStages = {}))[String(state.slotId)] = state.stage;
         }
         if (slotStages)
             result.slotStages = slotStages;
         return result;
     }
 }
-_ReasoningTracker_requests = new WeakMap(), _ReasoningTracker_slots = new WeakMap(), _ReasoningTracker_tails = new WeakMap(), _ReasoningTracker_inlineReasoning = new WeakMap(), _ReasoningTracker_listeners = new WeakMap(), _ReasoningTracker_lastSnapshot = new WeakMap(), _ReasoningTracker_instances = new WeakSet(), _ReasoningTracker_announce = function _ReasoningTracker_announce() {
+_ReasoningTracker_requests = new WeakMap(), _ReasoningTracker_listeners = new WeakMap(), _ReasoningTracker_lastSnapshot = new WeakMap(), _ReasoningTracker_counter = new WeakMap(), _ReasoningTracker_instances = new WeakSet(), _ReasoningTracker_announce = function _ReasoningTracker_announce() {
     const snapshot = this.snapshot;
     // The slot join rides in the key too: pinning a request to a slot is a
     // state change even when no stage count moves, and it is the field the
@@ -341,5 +416,58 @@ _ReasoningTracker_requests = new WeakMap(), _ReasoningTracker_slots = new WeakMa
             // Status reporting must never break a proxied completion.
         }
     }
+}, _ReasoningTracker_setSlot = function _ReasoningTracker_setSlot(id, slotId) {
+    if (!Number.isInteger(slotId) || slotId < 0)
+        return;
+    const state = __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").get(id);
+    if (!state || state.slotId === slotId)
+        return;
+    state.slotId = slotId;
+    __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_touch).call(this, state);
+    __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
+}, _ReasoningTracker_observe = function _ReasoningTracker_observe(id, text) {
+    const state = __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").get(id);
+    if (!state)
+        return;
+    __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_touch).call(this, state);
+    if (state.stage === "generating")
+        return;
+    // Node can split an SSE JSON field name at any byte. Keeping a small tail
+    // makes stage recognition independent of transport chunk boundaries without
+    // parsing or retaining the generated content itself.
+    const combined = `${state.tail}${text}`;
+    state.tail = combined.slice(-128);
+    if (state.inlineReasoning) {
+        if (combined.includes("</think>")) {
+            state.inlineReasoning = false;
+            state.stage = "generating";
+            __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
+        }
+        return;
+    }
+    if (combined.includes("<think>")) {
+        state.inlineReasoning = true;
+        state.stage = "thinking";
+        __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
+        return;
+    }
+    if (chunkHasContent(combined)) {
+        state.stage = "generating";
+        __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
+        return;
+    }
+    if (chunkHasReasoning(combined) && state.stage !== "thinking") {
+        state.stage = "thinking";
+        __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
+    }
+}, _ReasoningTracker_touch = function _ReasoningTracker_touch(state) {
+    state.lastSignalAt = Date.now();
+    state.strikes = 0;
+}, _ReasoningTracker_drop = function _ReasoningTracker_drop(ids) {
+    let removed = false;
+    for (const id of ids)
+        removed = __classPrivateFieldGet(this, _ReasoningTracker_requests, "f").delete(id) || removed;
+    if (removed)
+        __classPrivateFieldGet(this, _ReasoningTracker_instances, "m", _ReasoningTracker_announce).call(this);
 };
 //# sourceMappingURL=activity.js.map

package/dist/service/host-api.d.ts

@@ -28,7 +28,7 @@ import * as vram from "../vram.js";
 import type { SystemSample } from "../sysmon.js";
 import type { BrainLogPublisher, BrainStatusPublisher } from "./status-events.js";
 import type { Supervisor } from "./supervisor.js";
-import type { Scheduler } from "./scheduler.js";
+import type { ModelScheduler } from "./scheduler.js";
 import type { BrainRunLog } from "./run-log.js";
 import type { BrainLogArea } from "./log-format.js";
 /**
@@ -83,6 +83,8 @@ export interface HostCapabilities {
     jobs: boolean;
     /** POST /__host/restart delegates a restart to the service owner. */
     restart: boolean;
+    /** Multiple independently supervised model processes are supported. */
+    processPool: boolean;
 }
 /** A long-running operation owned by this brain host, not its caller. */
 export interface HostJob {
@@ -122,8 +124,9 @@ export interface HostApiDeps {
     queryGpuInfo: () => Promise<GpuInfo | null>;
     getRanking: () => RankedModel[];
     loadModel: (model: Model) => Promise<void>;
-    /** The single model queue shared by completions and resident operations. */
-    scheduler?: Scheduler | null;
+    unloadModels?: () => Promise<void>;
+    /** The process-pool scheduler shared by completions and resident operations. */
+    scheduler?: ModelScheduler<Supervisor> | null;
     /** Mirrors POST /__host/config's gate: may a network caller change things? */
     getAllowWrite: () => boolean;
     /** The managed models directory, for disk accounting. Null when unresolvable. */
@@ -204,7 +207,7 @@ export declare function buildInventoryRow(params: {
     gpu: GpuInfo | null;
     ranking: RankedModel[];
     supervisor: Supervisor;
-    scheduler?: Scheduler | null;
+    scheduler?: ModelScheduler<Supervisor> | null;
     runtimeBuild?: number | null;
 }): InventoryRow;
 export interface HostApi {

package/dist/service/host-api.js

@@ -1,7 +1,7 @@
 import { randomUUID } from "node:crypto";
 import { calibrationInfo, profileFieldDescriptors, profileWarnings, sanitizeProfilePatch, } from "../config/profile-edit.js";
 import { familyHostingProfileId, hostingFamily, removeHostingProfileMaterialization, } from "../config/hosting-profiles.js";
-import { forModel, getCalibration, put } from "../config/profiles.js";
+import { forModel, getCalibrationForBudget, put } from "../config/profiles.js";
 import { HostingProfileSchema, } from "../config/schema.js";
 import { deleteComponentFile, deleteModelFiles, diskUsage, planDelete, totalModelBytes, } from "../models/manage.js";
 import { deleteDisplayName, updateDisplayName } from "../models/rename-map.js";
@@ -171,11 +171,13 @@ function hostingProfilesFor(store, model) {
     return Object.values(store.hostingProfiles).filter((candidate) => candidate.family === family);
 }
 function stateOf(supervisor, scheduler, model) {
-    const resident = supervisor.model?.id === model.id;
+    const residentSupervisor = (typeof scheduler?.supervisorFor === "function" ? scheduler.supervisorFor(model.id) : null) ??
+        (supervisor.model?.id === model.id ? supervisor : null);
+    const resident = residentSupervisor !== null;
     if (resident) {
-        if (supervisor.state === "starting")
+        if (residentSupervisor.state === "starting")
             return "loading";
-        if (supervisor.state === "stopping")
+        if (residentSupervisor.state === "stopping")
             return "unloading";
     }
     const stats = scheduler?.stats();
@@ -183,7 +185,7 @@ function stateOf(supervisor, scheduler, model) {
         return "active";
     if ((stats?.waitingModelIds[model.id] ?? 0) > 0)
         return "queued";
-    if (resident && supervisor.state === "ready")
+    if (resident && residentSupervisor.state === "ready")
         return "loaded";
     return "not-loaded";
 }
@@ -197,7 +199,7 @@ function stateOf(supervisor, scheduler, model) {
 export function buildInventoryRow(params) {
     const { model, store, defaults, gpu, ranking, supervisor, scheduler = null, runtimeBuild: activeRuntimeBuild = null, } = params;
     const profile = forModel(store, model, defaults);
-    const calibration = profile.calibrationRequired ? null : getCalibration(store, model, profile);
+    const calibration = getCalibrationForBudget(store, model, profile);
     const budgetOptions = gpu
         ? { model, profile, calibration, totalVramBytes: gpu.totalBytes }
         : null;
@@ -344,6 +346,7 @@ export function createHostApi(deps) {
         writable: deps.getAllowWrite(),
         jobs: Boolean(deps.jobs),
         restart: Boolean(deps.restart),
+        processPool: true,
     });
     /** Refuse a write unless the owner opted into remote configuration. */
     const guardWrite = (res) => {
@@ -413,7 +416,9 @@ export function createHostApi(deps) {
                     // A setting is only unapplied when it was changed on the currently
                     // resident model. Edits to an unloaded model take effect naturally
                     // when it is next loaded and do not earn a misleading reload badge.
-                    const requiresRestart = deps.supervisor.model?.id === model.id;
+                    const requiresRestart = Boolean((typeof deps.scheduler?.supervisorFor === "function"
+                        ? deps.scheduler.supervisorFor(model.id)
+                        : null) ?? (deps.supervisor.model?.id === model.id ? deps.supervisor : null));
                     if (requiresRestart)
                         store.pendingReloadModelIds[model.id] = true;
                     deps.saveProfiles(store);
@@ -421,9 +426,7 @@ export function createHostApi(deps) {
                     // Return the recomputed budget so an edit costs one round trip rather
                     // than a write followed by a read the UI has to sequence.
                     const gpu = await deps.queryGpuInfo();
-                    const calibration = profile.calibrationRequired
-                        ? null
-                        : getCalibration(store, model, profile);
+                    const calibration = getCalibrationForBudget(store, model, profile);
                     const options = gpu
                         ? { model, profile, calibration, totalVramBytes: gpu.totalBytes }
                         : null;
@@ -522,7 +525,7 @@ export function createHostApi(deps) {
                 const options = {
                     model,
                     profile,
-                    calibration: profile.calibrationRequired ? null : getCalibration(store, model, profile),
+                    calibration: getCalibrationForBudget(store, model, profile),
                     totalVramBytes: gpu.totalBytes,
                 };
                 sendJson(res, {
@@ -553,11 +556,14 @@ export function createHostApi(deps) {
                 deps.log?.("model", `loading ${model.displayName}`);
                 await deps.loadModel(model);
                 deps.log?.("model", `loaded ${model.displayName}`);
+                const resident = (typeof deps.scheduler?.supervisorFor === "function"
+                    ? deps.scheduler.supervisorFor(model.id)
+                    : null) ?? deps.supervisor;
                 sendJson(res, {
-                    status: deps.supervisor.status(),
+                    status: resident.status(),
                     // What actually got used: loadModel fits the profile to VRAM, so the
                     // context here may be lower than the one saved.
-                    profile: deps.supervisor.profile,
+                    profile: resident.profile,
                 });
             }
             catch (error) {
@@ -570,7 +576,10 @@ export function createHostApi(deps) {
         void (async () => {
             try {
                 deps.log?.("model", "unloading resident model");
-                await deps.supervisor.stop();
+                if (deps.unloadModels)
+                    await deps.unloadModels();
+                else
+                    await deps.supervisor.stop();
                 deps.log?.("model", "resident model unloaded");
                 sendJson(res, { status: deps.supervisor.status() });
             }
@@ -580,7 +589,10 @@ export function createHostApi(deps) {
         })();
     };
     const handleDelete = (res, model) => {
-        if (deps.supervisor.model?.id === model.id && deps.supervisor.state !== "stopped") {
+        const resident = typeof deps.scheduler?.supervisorFor === "function"
+            ? deps.scheduler.supervisorFor(model.id)
+            : null;
+        if (resident && resident.state !== "stopped") {
             sendError(res, 409, "stop the model before deleting it");
             return;
         }
@@ -600,7 +612,10 @@ export function createHostApi(deps) {
         }
     };
     const handleComponentDelete = (res, model, componentId) => {
-        if (deps.supervisor.model?.id === model.id && deps.supervisor.state !== "stopped") {
+        const resident = typeof deps.scheduler?.supervisorFor === "function"
+            ? deps.scheduler.supervisorFor(model.id)
+            : null;
+        if (resident && resident.state !== "stopped") {
             sendError(res, 409, "stop the model before removing a bundle component");
             return;
         }

package/dist/service/process-pool.d.ts

@@ -0,0 +1,45 @@
+import type { Model } from "../types.js";
+import type { ModelScheduler, SchedulerStats, SchedulerSubmitOptions } from "./scheduler.js";
+import { Scheduler } from "./scheduler.js";
+import type { Supervisor } from "./supervisor.js";
+export interface ModelProcessPoolOptions {
+    initialSupervisor: Supervisor;
+    maxModels: number;
+    createSupervisor: (index: number) => Supervisor;
+    createScheduler: (supervisor: Supervisor, loadModel: (model: Model) => Promise<void>, onChange: () => void) => Scheduler<Supervisor>;
+    /** Returns the complete VRAM reservation for the process after it is ready. */
+    loadModel: (supervisor: Supervisor, model: Model, reservedElsewhereBytes: number) => Promise<number>;
+    onChange?: (() => void) | null;
+    logger?: ((message: string) => void) | null;
+}
+/**
+ * Global admission and eviction boundary for independently hosted models.
+ *
+ * Each resident model owns one Supervisor/Scheduler pair and therefore one
+ * llama-server process, port, KV pool, and lifecycle. Requests for a resident
+ * model go directly to that process. An unloaded model claims a free process
+ * slot, or evicts the least-recently-used idle process when the configured
+ * model limit is full. Busy processes are never evicted; the request remains
+ * queued until one becomes idle.
+ */
+export declare class ModelProcessPool implements ModelScheduler<Supervisor> {
+    #private;
+    constructor(options: ModelProcessPoolOptions);
+    get maxModels(): number;
+    submit(model: Model, run: (supervisor: Supervisor) => Promise<unknown>, options?: SchedulerSubmitOptions): Promise<unknown>;
+    /** Load a model and leave it resident without consuming an inference slot. */
+    preload(model: Model): Promise<void>;
+    /** Apply the host-owned process limit and retire excess idle processes. */
+    configure(maxModels: number): Promise<void>;
+    supervisorFor(modelId: string): Supervisor | null;
+    supervisors(): Supervisor[];
+    /** Complete statuses for every process that currently owns a model. */
+    residentSupervisors(): Supervisor[];
+    reservationFor(modelId: string): number;
+    unload(modelId?: string | null): Promise<void>;
+    stop(): Promise<void>;
+    forgetSlots(): void;
+    stats(): SchedulerStats;
+}
+export declare function normalizeModelLimit(value: number): number;
+//# sourceMappingURL=process-pool.d.ts.map