@otto-code/brain 0.8.12 → 0.8.13

package/dist/service/process-pool.js

@@ -0,0 +1,271 @@
+var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
+    if (kind === "m") throw new TypeError("Private method is not writable");
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
+    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
+};
+var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
+    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
+};
+var _ModelProcessPool_instances, _ModelProcessPool_createSupervisor, _ModelProcessPool_createScheduler, _ModelProcessPool_loadModel, _ModelProcessPool_onChange, _ModelProcessPool_logger, _ModelProcessPool_slots, _ModelProcessPool_queue, _ModelProcessPool_maxModels, _ModelProcessPool_busy, _ModelProcessPool_dirty, _ModelProcessPool_clock, _ModelProcessPool_makeSlot, _ModelProcessPool_announce, _ModelProcessPool_dispatch, _ModelProcessPool_pass, _ModelProcessPool_slotFor, _ModelProcessPool_nextSlotIndex, _ModelProcessPool_releaseSlot, _ModelProcessPool_trimIdleSlots;
+/**
+ * Global admission and eviction boundary for independently hosted models.
+ *
+ * Each resident model owns one Supervisor/Scheduler pair and therefore one
+ * llama-server process, port, KV pool, and lifecycle. Requests for a resident
+ * model go directly to that process. An unloaded model claims a free process
+ * slot, or evicts the least-recently-used idle process when the configured
+ * model limit is full. Busy processes are never evicted; the request remains
+ * queued until one becomes idle.
+ */
+export class ModelProcessPool {
+    constructor(options) {
+        _ModelProcessPool_instances.add(this);
+        _ModelProcessPool_createSupervisor.set(this, void 0);
+        _ModelProcessPool_createScheduler.set(this, void 0);
+        _ModelProcessPool_loadModel.set(this, void 0);
+        _ModelProcessPool_onChange.set(this, void 0);
+        _ModelProcessPool_logger.set(this, void 0);
+        _ModelProcessPool_slots.set(this, []);
+        _ModelProcessPool_queue.set(this, []);
+        _ModelProcessPool_maxModels.set(this, void 0);
+        _ModelProcessPool_busy.set(this, false);
+        _ModelProcessPool_dirty.set(this, false);
+        _ModelProcessPool_clock.set(this, 0);
+        __classPrivateFieldSet(this, _ModelProcessPool_createSupervisor, options.createSupervisor, "f");
+        __classPrivateFieldSet(this, _ModelProcessPool_createScheduler, options.createScheduler, "f");
+        __classPrivateFieldSet(this, _ModelProcessPool_loadModel, options.loadModel, "f");
+        __classPrivateFieldSet(this, _ModelProcessPool_onChange, options.onChange ?? null, "f");
+        __classPrivateFieldSet(this, _ModelProcessPool_logger, options.logger ?? null, "f");
+        __classPrivateFieldSet(this, _ModelProcessPool_maxModels, normalizeModelLimit(options.maxModels), "f");
+        __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").push(__classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_makeSlot).call(this, 0, options.initialSupervisor));
+    }
+    get maxModels() {
+        return __classPrivateFieldGet(this, _ModelProcessPool_maxModels, "f");
+    }
+    submit(model, run, options = {}) {
+        return new Promise((resolve, reject) => {
+            __classPrivateFieldGet(this, _ModelProcessPool_queue, "f").push({ model, run, options, resolve, reject });
+            __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_announce).call(this);
+            queueMicrotask(() => void __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_dispatch).call(this));
+        });
+    }
+    /** Load a model and leave it resident without consuming an inference slot. */
+    async preload(model) {
+        await this.submit(model, async () => undefined);
+    }
+    /** Apply the host-owned process limit and retire excess idle processes. */
+    async configure(maxModels) {
+        __classPrivateFieldSet(this, _ModelProcessPool_maxModels, normalizeModelLimit(maxModels), "f");
+        await __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_trimIdleSlots).call(this);
+        __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_announce).call(this);
+        void __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_dispatch).call(this);
+    }
+    supervisorFor(modelId) {
+        return (__classPrivateFieldGet(this, _ModelProcessPool_slots, "f").find((slot) => slot.assignedModelId === modelId ||
+            (slot.supervisor.model?.id === modelId && slot.supervisor.state !== "stopped"))?.supervisor ?? null);
+    }
+    supervisors() {
+        return __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").map((slot) => slot.supervisor);
+    }
+    /** Complete statuses for every process that currently owns a model. */
+    residentSupervisors() {
+        return __classPrivateFieldGet(this, _ModelProcessPool_slots, "f")
+            .filter((slot) => slot.assignedModelId !== null && slot.supervisor.state !== "stopped")
+            .map((slot) => slot.supervisor);
+    }
+    reservationFor(modelId) {
+        return __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").find((slot) => slot.assignedModelId === modelId)?.reservationBytes ?? 0;
+    }
+    async unload(modelId) {
+        const targets = modelId
+            ? __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").filter((slot) => slot.assignedModelId === modelId)
+            : [...__classPrivateFieldGet(this, _ModelProcessPool_slots, "f")];
+        for (const slot of targets) {
+            if (!slot.scheduler.isIdle || slot.pending > 0) {
+                throw new Error(modelId ? "the model is still serving requests" : "a model is still serving requests");
+            }
+        }
+        await Promise.all(targets.map((slot) => __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_releaseSlot).call(this, slot)));
+        await __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_trimIdleSlots).call(this);
+        __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_announce).call(this);
+        void __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_dispatch).call(this);
+    }
+    async stop() {
+        const error = new Error("Brain process pool stopped");
+        for (const job of __classPrivateFieldGet(this, _ModelProcessPool_queue, "f").splice(0))
+            job.reject(error);
+        await Promise.all(__classPrivateFieldGet(this, _ModelProcessPool_slots, "f").map((slot) => slot.supervisor.stop()));
+        __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_announce).call(this);
+    }
+    forgetSlots() {
+        for (const slot of __classPrivateFieldGet(this, _ModelProcessPool_slots, "f"))
+            slot.scheduler.forgetSlots();
+    }
+    stats() {
+        const waiting = {};
+        const waitingModelIds = {};
+        let queued = 0;
+        let lastTurn = null;
+        let active = null;
+        for (const job of __classPrivateFieldGet(this, _ModelProcessPool_queue, "f")) {
+            queued += 1;
+            waiting[job.model.displayName] = (waiting[job.model.displayName] ?? 0) + 1;
+            waitingModelIds[job.model.id] = (waitingModelIds[job.model.id] ?? 0) + 1;
+        }
+        for (const slot of __classPrivateFieldGet(this, _ModelProcessPool_slots, "f")) {
+            const stats = slot.scheduler.stats();
+            queued += stats.queued;
+            for (const [name, count] of Object.entries(stats.waiting)) {
+                waiting[name] = (waiting[name] ?? 0) + count;
+            }
+            for (const [id, count] of Object.entries(stats.waitingModelIds)) {
+                waitingModelIds[id] = (waitingModelIds[id] ?? 0) + count;
+            }
+            lastTurn = stats.lastTurn ?? lastTurn;
+            active = active ?? stats.active;
+        }
+        return { queued, waiting, waitingModelIds, lastTurn, active };
+    }
+}
+_ModelProcessPool_createSupervisor = new WeakMap(), _ModelProcessPool_createScheduler = new WeakMap(), _ModelProcessPool_loadModel = new WeakMap(), _ModelProcessPool_onChange = new WeakMap(), _ModelProcessPool_logger = new WeakMap(), _ModelProcessPool_slots = new WeakMap(), _ModelProcessPool_queue = new WeakMap(), _ModelProcessPool_maxModels = new WeakMap(), _ModelProcessPool_busy = new WeakMap(), _ModelProcessPool_dirty = new WeakMap(), _ModelProcessPool_clock = new WeakMap(), _ModelProcessPool_instances = new WeakSet(), _ModelProcessPool_makeSlot = function _ModelProcessPool_makeSlot(index, supervisor) {
+    var _a;
+    const slot = {
+        index,
+        supervisor,
+        scheduler: null,
+        assignedModelId: supervisor.model?.id ?? null,
+        pending: 0,
+        reservationBytes: 0,
+        lastUsedAt: __classPrivateFieldSet(this, _ModelProcessPool_clock, (_a = __classPrivateFieldGet(this, _ModelProcessPool_clock, "f"), ++_a), "f"),
+    };
+    slot.scheduler = __classPrivateFieldGet(this, _ModelProcessPool_createScheduler, "f").call(this, supervisor, async (model) => {
+        var _a;
+        const reservedElsewhereBytes = __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").reduce((total, candidate) => (candidate === slot ? total : total + candidate.reservationBytes), 0);
+        slot.reservationBytes = await __classPrivateFieldGet(this, _ModelProcessPool_loadModel, "f").call(this, supervisor, model, reservedElsewhereBytes);
+        slot.assignedModelId = model.id;
+        slot.lastUsedAt = __classPrivateFieldSet(this, _ModelProcessPool_clock, (_a = __classPrivateFieldGet(this, _ModelProcessPool_clock, "f"), ++_a), "f");
+        __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_announce).call(this);
+    }, () => {
+        __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_announce).call(this);
+        void __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_dispatch).call(this);
+    });
+    return slot;
+}, _ModelProcessPool_announce = function _ModelProcessPool_announce() {
+    try {
+        __classPrivateFieldGet(this, _ModelProcessPool_onChange, "f")?.call(this);
+    }
+    catch {
+        // Status observers are not allowed to fail admission or eviction.
+    }
+}, _ModelProcessPool_dispatch = async function _ModelProcessPool_dispatch() {
+    if (__classPrivateFieldGet(this, _ModelProcessPool_busy, "f")) {
+        __classPrivateFieldSet(this, _ModelProcessPool_dirty, true, "f");
+        return;
+    }
+    __classPrivateFieldSet(this, _ModelProcessPool_busy, true, "f");
+    try {
+        do {
+            __classPrivateFieldSet(this, _ModelProcessPool_dirty, false, "f");
+            await __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_pass).call(this);
+        } while (__classPrivateFieldGet(this, _ModelProcessPool_dirty, "f"));
+    }
+    finally {
+        __classPrivateFieldSet(this, _ModelProcessPool_busy, false, "f");
+    }
+}, _ModelProcessPool_pass = async function _ModelProcessPool_pass() {
+    var _a;
+    await __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_trimIdleSlots).call(this);
+    for (;;) {
+        let selectedIndex = -1;
+        let selectedSlot = null;
+        for (let index = 0; index < __classPrivateFieldGet(this, _ModelProcessPool_queue, "f").length; index += 1) {
+            const candidate = __classPrivateFieldGet(this, _ModelProcessPool_queue, "f")[index];
+            const slot = await __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_slotFor).call(this, candidate.model);
+            if (!slot)
+                continue;
+            selectedIndex = index;
+            selectedSlot = slot;
+            break;
+        }
+        if (selectedIndex < 0 || !selectedSlot)
+            return;
+        const [job] = __classPrivateFieldGet(this, _ModelProcessPool_queue, "f").splice(selectedIndex, 1);
+        if (!job)
+            return;
+        const slot = selectedSlot;
+        slot.pending += 1;
+        slot.lastUsedAt = __classPrivateFieldSet(this, _ModelProcessPool_clock, (_a = __classPrivateFieldGet(this, _ModelProcessPool_clock, "f"), ++_a), "f");
+        __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_announce).call(this);
+        void slot.scheduler
+            .submit(job.model, job.run, job.options)
+            .then(job.resolve, job.reject)
+            .finally(() => {
+            var _a;
+            slot.pending = Math.max(0, slot.pending - 1);
+            slot.lastUsedAt = __classPrivateFieldSet(this, _ModelProcessPool_clock, (_a = __classPrivateFieldGet(this, _ModelProcessPool_clock, "f"), ++_a), "f");
+            if (slot.supervisor.state === "failed" && slot.pending === 0) {
+                void __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_releaseSlot).call(this, slot).finally(() => void __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_dispatch).call(this));
+            }
+            __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_announce).call(this);
+            void __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_dispatch).call(this);
+        });
+    }
+}, _ModelProcessPool_slotFor = async function _ModelProcessPool_slotFor(model) {
+    const resident = __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").find((slot) => slot.assignedModelId === model.id);
+    if (resident)
+        return resident;
+    let empty = __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").find((slot) => slot.assignedModelId === null && slot.scheduler.isIdle && slot.pending === 0);
+    if (!empty && __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").length < __classPrivateFieldGet(this, _ModelProcessPool_maxModels, "f")) {
+        const index = __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_nextSlotIndex).call(this);
+        empty = __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_makeSlot).call(this, index, __classPrivateFieldGet(this, _ModelProcessPool_createSupervisor, "f").call(this, index));
+        __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").push(empty);
+    }
+    if (empty) {
+        empty.assignedModelId = model.id;
+        return empty;
+    }
+    const evictable = __classPrivateFieldGet(this, _ModelProcessPool_slots, "f")
+        .filter((slot) => slot.scheduler.isIdle && slot.pending === 0)
+        .sort((left, right) => left.lastUsedAt - right.lastUsedAt)[0];
+    if (!evictable)
+        return null;
+    __classPrivateFieldGet(this, _ModelProcessPool_logger, "f")?.call(this, `evicting ${evictable.supervisor.model?.displayName ?? "idle model"} for ${model.displayName}`);
+    await __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_releaseSlot).call(this, evictable);
+    evictable.assignedModelId = model.id;
+    return evictable;
+}, _ModelProcessPool_nextSlotIndex = function _ModelProcessPool_nextSlotIndex() {
+    const used = new Set(__classPrivateFieldGet(this, _ModelProcessPool_slots, "f").map((slot) => slot.index));
+    let index = 0;
+    while (used.has(index))
+        index += 1;
+    return index;
+}, _ModelProcessPool_releaseSlot = async function _ModelProcessPool_releaseSlot(slot) {
+    var _a;
+    await slot.supervisor.stop();
+    slot.scheduler.forgetSlots();
+    slot.assignedModelId = null;
+    slot.reservationBytes = 0;
+    slot.lastUsedAt = __classPrivateFieldSet(this, _ModelProcessPool_clock, (_a = __classPrivateFieldGet(this, _ModelProcessPool_clock, "f"), ++_a), "f");
+}, _ModelProcessPool_trimIdleSlots = async function _ModelProcessPool_trimIdleSlots() {
+    if (__classPrivateFieldGet(this, _ModelProcessPool_slots, "f").length <= __classPrivateFieldGet(this, _ModelProcessPool_maxModels, "f"))
+        return;
+    const removable = __classPrivateFieldGet(this, _ModelProcessPool_slots, "f")
+        .filter((slot) => slot.scheduler.isIdle && slot.pending === 0)
+        .sort((left, right) => left.lastUsedAt - right.lastUsedAt);
+    while (__classPrivateFieldGet(this, _ModelProcessPool_slots, "f").length > __classPrivateFieldGet(this, _ModelProcessPool_maxModels, "f") && removable.length > 0) {
+        const slot = removable.shift();
+        await __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_releaseSlot).call(this, slot);
+        const index = __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").indexOf(slot);
+        if (index >= 0)
+            __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").splice(index, 1);
+    }
+};
+export function normalizeModelLimit(value) {
+    if (!Number.isFinite(value))
+        return 1;
+    return Math.max(1, Math.min(16, Math.floor(value)));
+}
+//# sourceMappingURL=process-pool.js.map

package/dist/service/router.d.ts

@@ -1,5 +1,5 @@
 import http from "node:http";
-import { Scheduler } from "./scheduler.js";
+import { type ModelScheduler } from "./scheduler.js";
 import type { Supervisor } from "./supervisor.js";
 import { type RankedModel } from "../ops/results.js";
 import type { GpuInfo, Model } from "../types.js";
@@ -108,7 +108,7 @@ export declare function describeModel(model: Model | null, options?: DescribeOpt
  * the supervisor is running marked 'loaded'. Falls back to just the running
  * model when no catalog provider is wired in.
  */
-export declare function buildModelList(supervisor: Supervisor, getCatalog: GetCatalog): ModelEntry[];
+export declare function buildModelList(supervisor: Supervisor, getCatalog: GetCatalog, scheduler?: ModelScheduler<Supervisor> | null): ModelEntry[];
 /**
  * Map an OpenAI-compatible effort request onto a model's own chat-template
  * arguments. llama.cpp does not know every model's dialect: Qwen3.8 calls the
@@ -207,6 +207,7 @@ export declare function decideModelGate(params: {
     lockModel: boolean;
     requestedName: string | null;
     pinned: Model | null;
+    pinnedModels?: Model[];
     resolved: Model | null;
 }): ModelGateResult;
 export interface RouterOptions {
@@ -227,6 +228,8 @@ export interface RouterOptions {
     getLockModel?: () => boolean;
     /** Live: the configured default model, the pin target before one is resident. */
     getDefaultModel?: () => string | null;
+    /** Live: the complete model set served while locking is enabled. */
+    getLockedModels?: () => string[];
     /**
      * Apply an editable config patch (write config.json, live-switch the model,
      * update the lock), for POST /__host/config. Absent = the write endpoint is
@@ -262,8 +265,8 @@ export interface RouterOptions {
      */
     statusEvents?: BrainStatusPublisher | null;
     /** A service shares this scheduler with host-owned model operations. */
-    scheduler?: Scheduler | null;
+    scheduler?: ModelScheduler<Supervisor> | null;
 }
-export declare function createRouter({ supervisor, telemetry, logger, getCatalog, loadModel, loadRanking, queryGpuInfo, version, getConfig, getEvals, getLockModel, getDefaultModel, applyConfigPatch, getAllowConfigWrite, hostApi, getResources, statusEvents, scheduler: suppliedScheduler, }: RouterOptions): (req: http.IncomingMessage, res: http.ServerResponse) => void;
+export declare function createRouter({ supervisor, telemetry, logger, getCatalog, loadModel, loadRanking, queryGpuInfo, version, getConfig, getEvals, getLockModel, getDefaultModel, getLockedModels, applyConfigPatch, getAllowConfigWrite, hostApi, getResources, statusEvents, scheduler: suppliedScheduler, }: RouterOptions): (req: http.IncomingMessage, res: http.ServerResponse) => void;
 export {};
 //# sourceMappingURL=router.d.ts.map

package/dist/service/router.js

@@ -205,30 +205,39 @@ function resolveCatalog(getCatalog) {
  * the supervisor is running marked 'loaded'. Falls back to just the running
  * model when no catalog provider is wired in.
  */
-export function buildModelList(supervisor, getCatalog) {
-    const loadedId = supervisor.model ? supervisor.model.id : null;
+export function buildModelList(supervisor, getCatalog, scheduler = null) {
+    const supervisors = scheduler && typeof scheduler.supervisors === "function"
+        ? scheduler.supervisors()
+        : [supervisor];
+    const residentFor = (modelId) => supervisors.find((candidate) => candidate.model?.id === modelId && candidate.state !== "stopped") ?? null;
     const stateOf = (model) => {
-        if (!loadedId || model.id !== loadedId)
+        const resident = residentFor(model.id);
+        if (!resident)
             return "not-loaded";
-        if (supervisor.state === "ready")
+        if (resident.state === "ready")
             return "loaded";
-        if (supervisor.state === "starting")
+        if (resident.state === "starting")
             return "loading";
         return "not-loaded";
     };
     let catalog = resolveCatalog(getCatalog);
     // Guarantee the running model appears even if the snapshot predates it.
-    if (supervisor.model && !catalog.some((m) => m.id === loadedId)) {
-        catalog = [supervisor.model, ...catalog];
+    for (const resident of supervisors) {
+        if (resident.model && !catalog.some((model) => model.id === resident.model?.id)) {
+            catalog.unshift(resident.model);
+        }
     }
-    return catalog.map((model) => describeModel(model, {
-        state: stateOf(model),
-        profile: model.id === loadedId ? supervisor.profile : null,
-        createdAt: model.id === loadedId ? supervisor.startedAt : null,
-    }));
+    return catalog.map((model) => {
+        const resident = residentFor(model.id);
+        return describeModel(model, {
+            state: stateOf(model),
+            profile: resident?.profile ?? null,
+            createdAt: resident?.startedAt ?? null,
+        });
+    });
 }
 /** Handle the model-discovery endpoints ourselves; returns true if it did. */
-function handleModelsRoute(req, res, supervisor, getCatalog) {
+function handleModelsRoute(req, res, supervisor, getCatalog, scheduler = null) {
     if (req.method !== "GET")
         return false;
     const url = (req.url || "").split("?")[0];
@@ -238,7 +247,7 @@ function handleModelsRoute(req, res, supervisor, getCatalog) {
         : null;
     if (!isList && single === null)
         return false;
-    const list = buildModelList(supervisor, getCatalog);
+    const list = buildModelList(supervisor, getCatalog, scheduler);
     let payload;
     if (single !== null) {
         const entry = list.find((e) => e.id === single);
@@ -658,19 +667,24 @@ export function injectSystemAddendum(body, addendum, shape) {
  * a switch; an unnamed request rides the pin.
  */
 export function decideModelGate(params) {
-    const { lockModel, requestedName, pinned, resolved } = params;
+    const { lockModel, requestedName, pinned, pinnedModels = pinned ? [pinned] : [], resolved, } = params;
     if (lockModel) {
-        if (!pinned) {
-            return { ok: false, status: 503, message: "no model is loaded yet on this locked host" };
+        if (pinnedModels.length === 0) {
+            return { ok: false, status: 503, message: "no models are selected on this locked host" };
         }
-        if (requestedName && requestedName !== pinned.id && requestedName !== pinned.displayName) {
+        const selected = requestedName
+            ? pinnedModels.find((model) => model.id === requestedName || model.displayName === requestedName)
+            : pinnedModels[0];
+        if (!selected) {
             return {
                 ok: false,
                 status: 409,
-                message: `model switching is disabled on this host; only "${pinned.displayName}" is served`,
+                message: `model switching is disabled on this host; served models: ${pinnedModels
+                    .map((model) => `"${model.displayName}"`)
+                    .join(", ")}`,
             };
         }
-        return { ok: true, model: pinned };
+        return { ok: true, model: selected };
     }
     if (!resolved) {
         return {
@@ -684,7 +698,7 @@ export function decideModelGate(params) {
     return { ok: true, model: resolved };
 }
 /** Buffer a completion request, resolve its target model, and queue it. */
-function scheduleCompletion({ req, res, agent, supervisor, telemetry, logger, scheduler, modelGate, }) {
+function scheduleCompletion({ req, res, agent, telemetry, logger, scheduler, modelGate, }) {
     const chunks = [];
     let size = 0;
     let tooBig = false;
@@ -737,10 +751,10 @@ function scheduleCompletion({ req, res, agent, supervisor, telemetry, logger, sc
         // relaunched - and not at dispatch time either, where a sibling admitted in
         // the same pass could see the same slot free and pin it too.
         let slot = null;
-        const queued = scheduler.submit(model, () => proxyBuffered({
+        const queued = scheduler.submit(model, (resident) => proxyBuffered({
             agent,
             model,
-            supervisor,
+            supervisor: resident,
             telemetry,
             logger,
             req,
@@ -758,7 +772,7 @@ function scheduleCompletion({ req, res, agent, supervisor, telemetry, logger, sc
 // (rare), so the router caches the ranking and re-reads it at most once per
 // window - the cheap time-based trigger.
 const RANKING_TTL_MS = 60000;
-export function createRouter({ supervisor, telemetry, logger, getCatalog = null, loadModel = null, loadRanking = () => rankModels(), queryGpuInfo = queryGpu, version = null, getConfig = null, getEvals = null, getLockModel = () => false, getDefaultModel = () => null, applyConfigPatch = null, getAllowConfigWrite = () => false, hostApi = null, getResources = null, statusEvents = null, scheduler: suppliedScheduler = null, }) {
+export function createRouter({ supervisor, telemetry, logger, getCatalog = null, loadModel = null, loadRanking = () => rankModels(), queryGpuInfo = queryGpu, version = null, getConfig = null, getEvals = null, getLockModel = () => false, getDefaultModel = () => null, getLockedModels = () => [], applyConfigPatch = null, getAllowConfigWrite = () => false, hostApi = null, getResources = null, statusEvents = null, scheduler: suppliedScheduler = null, }) {
     // llama-server may close an idle response socket while this scheduler holds
     // the next request in queue. A reused keep-alive socket then fails as
     // ECONNRESET ("socket hang up") before the queued request reaches inference.
@@ -851,11 +865,20 @@ export function createRouter({ supervisor, telemetry, logger, getCatalog = null,
     // The single model this host will serve when switching is locked: the
     // resident model if one is up, else the configured default resolved through
     // the catalog. Null means nothing is loadable yet.
-    const pinnedModel = () => {
-        if (supervisor.model)
-            return supervisor.model;
-        const def = getDefaultModel();
-        return def ? resolveModel(def) : null;
+    const pinnedModels = () => {
+        const configured = getLockedModels();
+        const names = configured.length > 0 ? configured : [getDefaultModel()].filter(Boolean);
+        const selected = names
+            .map((name) => resolveModel(name ?? null))
+            .filter((model) => model !== null);
+        if (selected.length > 0)
+            return selected;
+        const residents = scheduler && typeof scheduler.supervisors === "function"
+            ? scheduler.supervisors()
+            : [supervisor];
+        return residents
+            .map((candidate) => candidate.model)
+            .filter((model) => model !== null);
     };
     // Decide whether a completion for `name` may run. The lock/default are read
     // live so a POST /__host/config change takes effect without a restart. Only
@@ -865,7 +888,8 @@ export function createRouter({ supervisor, telemetry, logger, getCatalog = null,
         return decideModelGate({
             lockModel: lock,
             requestedName: name,
-            pinned: lock ? pinnedModel() : null,
+            pinned: null,
+            pinnedModels: lock ? pinnedModels() : [],
             resolved: lock ? null : resolveModel(name),
         });
     };
@@ -880,6 +904,10 @@ export function createRouter({ supervisor, telemetry, logger, getCatalog = null,
      */
     const buildCheapStatus = async () => {
         const schedulerStats = scheduler ? scheduler.stats() : null;
+        const residentSupervisors = scheduler && typeof scheduler.supervisors === "function"
+            ? scheduler.supervisors()
+            : [supervisor];
+        const residents = residentSupervisors.filter((candidate) => candidate.model !== null && candidate.state !== "stopped");
         // Slots come from a loopback GET on the resident llama-server. That is
         // cheap enough to pay on every sample - unlike GPU sampling, which spawns
         // `nvidia-smi` and stays opt-in. Skipped entirely unless a model is
@@ -895,6 +923,7 @@ export function createRouter({ supervisor, telemetry, logger, getCatalog = null,
             // package version.
             apiVersion: HOST_API_VERSION,
             ...supervisor.status(),
+            residents: residents.map((resident) => resident.status()),
             telemetry: { ...telemetry.totals, warning: telemetry.warning },
             scheduler: schedulerStats,
             recent: telemetry.records.slice(-10),
@@ -983,7 +1012,7 @@ export function createRouter({ supervisor, telemetry, logger, getCatalog = null,
             return;
         // Answer model discovery ourselves so ids are real names (not paths), the
         // whole catalog is listed, and each carries LM Studio's context fields.
-        if (handleModelsRoute(req, res, supervisor, getCatalog))
+        if (handleModelsRoute(req, res, supervisor, getCatalog, scheduler))
             return;
         // With a scheduler wired in, completion requests are queued and served in
         // turns - including loading/switching to the model they ask for - instead
@@ -993,7 +1022,6 @@ export function createRouter({ supervisor, telemetry, logger, getCatalog = null,
                 req,
                 res,
                 agent,
-                supervisor,
                 telemetry,
                 logger,
                 scheduler,

package/dist/service/scheduler.d.ts

@@ -84,6 +84,18 @@ export interface SchedulerSupervisor {
     model: Model | null;
     profile: Profile | null;
 }
+/**
+ * The scheduler surface consumed by the router and host operations. A managed
+ * process pool implements the same contract while choosing which resident
+ * supervisor owns each submitted model.
+ */
+export interface ModelScheduler<TSupervisor extends SchedulerSupervisor = SchedulerSupervisor> {
+    submit(model: Model, run: (supervisor: TSupervisor) => Promise<unknown>, options?: SchedulerSubmitOptions): Promise<unknown>;
+    stats(): SchedulerStats;
+    forgetSlots(): void;
+    supervisorFor(modelId: string): TSupervisor | null;
+    supervisors(): TSupervisor[];
+}
 /**
  * The answer to a live slot measurement: how many sequence slots are free, and
  * optionally WHICH ones. `idle` drives admission (see CAPACITY); `ids` lets
@@ -115,8 +127,8 @@ export interface SlotMeasurement {
  * behavior but must never fail the completion that is about to run.
  */
 export type SlotEraser = (slotId: number) => Promise<void>;
-export interface SchedulerOptions {
-    supervisor: SchedulerSupervisor;
+export interface SchedulerOptions<TSupervisor extends SchedulerSupervisor = SchedulerSupervisor> {
+    supervisor: TSupervisor;
     loadModel: (model: Model) => Promise<void>;
     logger?: ((message: string) => void) | null;
     /**
@@ -192,7 +204,7 @@ export interface SchedulerSubmitOptions {
     onSlotFree?: ((slotId: number | null) => void) | null;
 }
 /** A queued request or host operation bound to a resolved catalog model. */
-export interface QueuedJob {
+export interface QueuedJob<TSupervisor extends SchedulerSupervisor = SchedulerSupervisor> {
     modelId: string;
     model: Model;
     kind: SchedulerJobKind;
@@ -209,7 +221,7 @@ export interface QueuedJob {
      * busy behavior would serialize the pair onto it while another slot sat empty.
      */
     slotId: number | null;
-    run: () => Promise<unknown>;
+    run: (supervisor: TSupervisor) => Promise<unknown>;
     resolve: (value: unknown) => void;
     reject: (error: unknown) => void;
 }
@@ -224,13 +236,13 @@ export interface SchedulerStats {
         kind: Exclude<SchedulerJobKind, "completion">;
     } | null;
 }
-export declare class Scheduler {
+export declare class Scheduler<TSupervisor extends SchedulerSupervisor = SchedulerSupervisor> implements ModelScheduler<TSupervisor> {
     #private;
-    supervisor: SchedulerSupervisor;
+    supervisor: TSupervisor;
     loadModel: (model: Model) => Promise<void>;
     logger: ((message: string) => void) | null;
     /** Submitted, not yet claimed by a turn. */
-    queue: QueuedJob[];
+    queue: QueuedJob<TSupervisor>[];
     lastTurnId: string | null;
     /**
      * Per model: the session whose jobs most recently filled its slots. That
@@ -239,7 +251,7 @@ export declare class Scheduler {
      */
     hotSessions: Map<string, string>;
     /** The exclusive operation in flight, if any. Reported by `stats()`. */
-    activeJob: QueuedJob | null;
+    activeJob: QueuedJob<TSupervisor> | null;
     onChange: (() => void) | null;
     freeSlots: SchedulerOptions["freeSlots"];
     slotPollMs: number;
@@ -252,7 +264,7 @@ export declare class Scheduler {
      * engine reloads a model - slot ids do not survive the relaunch.
      */
     slotOwners: Map<number, string | null>;
-    constructor({ supervisor, loadModel, logger, onChange, freeSlots, slotPollMs, eraseSlot, }: SchedulerOptions);
+    constructor({ supervisor, loadModel, logger, onChange, freeSlots, slotPollMs, eraseSlot, }: SchedulerOptions<TSupervisor>);
     /** Id of the model that is actually loaded and ready, or null. */
     get loadedId(): string | null;
     /** How many requests may run at once against the resident model (static ceiling). */
@@ -263,7 +275,7 @@ export declare class Scheduler {
      * Dispatch is deferred a microtask so a burst of requests submitted together
      * shares one turn rather than the first one taking a turn by itself.
      */
-    submit(model: Model, run: () => Promise<unknown>, { kind, exclusive, onStart, session, onSlotFree, }?: SchedulerSubmitOptions): Promise<unknown>;
+    submit(model: Model, run: (supervisor: TSupervisor) => Promise<unknown>, { kind, exclusive, onStart, session, onSlotFree, }?: SchedulerSubmitOptions): Promise<unknown>;
     /**
      * Drop every recorded slot owner. The engine's slots do not survive a model
      * (re)launch, so their owners do not either - a stale entry would make the
@@ -279,6 +291,10 @@ export declare class Scheduler {
      * supervisor says the slots are gone. Both paths are idempotent.
      */
     forgetSlots(): void;
+    /** True only when this engine has no claimed, queued, or running work. */
+    get isIdle(): boolean;
+    supervisorFor(modelId: string): TSupervisor | null;
+    supervisors(): TSupervisor[];
     /** Queue snapshot for the status endpoint / UI. */
     stats(): SchedulerStats;
 }

package/dist/service/scheduler.js

@@ -118,6 +118,19 @@ export class Scheduler {
     forgetSlots() {
         this.slotOwners.clear();
     }
+    /** True only when this engine has no claimed, queued, or running work. */
+    get isIdle() {
+        return (this.queue.length === 0 &&
+            __classPrivateFieldGet(this, _Scheduler_batch, "f").length === 0 &&
+            __classPrivateFieldGet(this, _Scheduler_running, "f").size === 0 &&
+            this.activeJob === null);
+    }
+    supervisorFor(modelId) {
+        return this.supervisor.model?.id === modelId ? this.supervisor : null;
+    }
+    supervisors() {
+        return [this.supervisor];
+    }
     /** Queue snapshot for the status endpoint / UI. */
     stats() {
         const waiting = {};
@@ -269,7 +282,7 @@ async function _Scheduler_start(job) {
     void (async () => {
         try {
             job.onStart?.();
-            job.resolve(await job.run());
+            job.resolve(await job.run(this.supervisor));
         }
         catch (error) {
             job.reject(error);