@otto-code/brain 0.8.12 → 0.8.14

package/dist/service/process-pool.js

@@ -0,0 +1,271 @@
+var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
+    if (kind === "m") throw new TypeError("Private method is not writable");
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
+    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
+};
+var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
+    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
+};
+var _ModelProcessPool_instances, _ModelProcessPool_createSupervisor, _ModelProcessPool_createScheduler, _ModelProcessPool_loadModel, _ModelProcessPool_onChange, _ModelProcessPool_logger, _ModelProcessPool_slots, _ModelProcessPool_queue, _ModelProcessPool_maxModels, _ModelProcessPool_busy, _ModelProcessPool_dirty, _ModelProcessPool_clock, _ModelProcessPool_makeSlot, _ModelProcessPool_announce, _ModelProcessPool_dispatch, _ModelProcessPool_pass, _ModelProcessPool_slotFor, _ModelProcessPool_nextSlotIndex, _ModelProcessPool_releaseSlot, _ModelProcessPool_trimIdleSlots;
+/**
+ * Global admission and eviction boundary for independently hosted models.
+ *
+ * Each resident model owns one Supervisor/Scheduler pair and therefore one
+ * llama-server process, port, KV pool, and lifecycle. Requests for a resident
+ * model go directly to that process. An unloaded model claims a free process
+ * slot, or evicts the least-recently-used idle process when the configured
+ * model limit is full. Busy processes are never evicted; the request remains
+ * queued until one becomes idle.
+ */
+export class ModelProcessPool {
+    constructor(options) {
+        _ModelProcessPool_instances.add(this);
+        _ModelProcessPool_createSupervisor.set(this, void 0);
+        _ModelProcessPool_createScheduler.set(this, void 0);
+        _ModelProcessPool_loadModel.set(this, void 0);
+        _ModelProcessPool_onChange.set(this, void 0);
+        _ModelProcessPool_logger.set(this, void 0);
+        _ModelProcessPool_slots.set(this, []);
+        _ModelProcessPool_queue.set(this, []);
+        _ModelProcessPool_maxModels.set(this, void 0);
+        _ModelProcessPool_busy.set(this, false);
+        _ModelProcessPool_dirty.set(this, false);
+        _ModelProcessPool_clock.set(this, 0);
+        __classPrivateFieldSet(this, _ModelProcessPool_createSupervisor, options.createSupervisor, "f");
+        __classPrivateFieldSet(this, _ModelProcessPool_createScheduler, options.createScheduler, "f");
+        __classPrivateFieldSet(this, _ModelProcessPool_loadModel, options.loadModel, "f");
+        __classPrivateFieldSet(this, _ModelProcessPool_onChange, options.onChange ?? null, "f");
+        __classPrivateFieldSet(this, _ModelProcessPool_logger, options.logger ?? null, "f");
+        __classPrivateFieldSet(this, _ModelProcessPool_maxModels, normalizeModelLimit(options.maxModels), "f");
+        __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").push(__classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_makeSlot).call(this, 0, options.initialSupervisor));
+    }
+    get maxModels() {
+        return __classPrivateFieldGet(this, _ModelProcessPool_maxModels, "f");
+    }
+    submit(model, run, options = {}) {
+        return new Promise((resolve, reject) => {
+            __classPrivateFieldGet(this, _ModelProcessPool_queue, "f").push({ model, run, options, resolve, reject });
+            __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_announce).call(this);
+            queueMicrotask(() => void __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_dispatch).call(this));
+        });
+    }
+    /** Load a model and leave it resident without consuming an inference slot. */
+    async preload(model) {
+        await this.submit(model, async () => undefined);
+    }
+    /** Apply the host-owned process limit and retire excess idle processes. */
+    async configure(maxModels) {
+        __classPrivateFieldSet(this, _ModelProcessPool_maxModels, normalizeModelLimit(maxModels), "f");
+        await __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_trimIdleSlots).call(this);
+        __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_announce).call(this);
+        void __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_dispatch).call(this);
+    }
+    supervisorFor(modelId) {
+        return (__classPrivateFieldGet(this, _ModelProcessPool_slots, "f").find((slot) => slot.assignedModelId === modelId ||
+            (slot.supervisor.model?.id === modelId && slot.supervisor.state !== "stopped"))?.supervisor ?? null);
+    }
+    supervisors() {
+        return __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").map((slot) => slot.supervisor);
+    }
+    /** Complete statuses for every process that currently owns a model. */
+    residentSupervisors() {
+        return __classPrivateFieldGet(this, _ModelProcessPool_slots, "f")
+            .filter((slot) => slot.assignedModelId !== null && slot.supervisor.state !== "stopped")
+            .map((slot) => slot.supervisor);
+    }
+    reservationFor(modelId) {
+        return __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").find((slot) => slot.assignedModelId === modelId)?.reservationBytes ?? 0;
+    }
+    async unload(modelId) {
+        const targets = modelId
+            ? __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").filter((slot) => slot.assignedModelId === modelId)
+            : [...__classPrivateFieldGet(this, _ModelProcessPool_slots, "f")];
+        for (const slot of targets) {
+            if (!slot.scheduler.isIdle || slot.pending > 0) {
+                throw new Error(modelId ? "the model is still serving requests" : "a model is still serving requests");
+            }
+        }
+        await Promise.all(targets.map((slot) => __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_releaseSlot).call(this, slot)));
+        await __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_trimIdleSlots).call(this);
+        __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_announce).call(this);
+        void __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_dispatch).call(this);
+    }
+    async stop() {
+        const error = new Error("Brain process pool stopped");
+        for (const job of __classPrivateFieldGet(this, _ModelProcessPool_queue, "f").splice(0))
+            job.reject(error);
+        await Promise.all(__classPrivateFieldGet(this, _ModelProcessPool_slots, "f").map((slot) => slot.supervisor.stop()));
+        __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_announce).call(this);
+    }
+    forgetSlots() {
+        for (const slot of __classPrivateFieldGet(this, _ModelProcessPool_slots, "f"))
+            slot.scheduler.forgetSlots();
+    }
+    stats() {
+        const waiting = {};
+        const waitingModelIds = {};
+        let queued = 0;
+        let lastTurn = null;
+        let active = null;
+        for (const job of __classPrivateFieldGet(this, _ModelProcessPool_queue, "f")) {
+            queued += 1;
+            waiting[job.model.displayName] = (waiting[job.model.displayName] ?? 0) + 1;
+            waitingModelIds[job.model.id] = (waitingModelIds[job.model.id] ?? 0) + 1;
+        }
+        for (const slot of __classPrivateFieldGet(this, _ModelProcessPool_slots, "f")) {
+            const stats = slot.scheduler.stats();
+            queued += stats.queued;
+            for (const [name, count] of Object.entries(stats.waiting)) {
+                waiting[name] = (waiting[name] ?? 0) + count;
+            }
+            for (const [id, count] of Object.entries(stats.waitingModelIds)) {
+                waitingModelIds[id] = (waitingModelIds[id] ?? 0) + count;
+            }
+            lastTurn = stats.lastTurn ?? lastTurn;
+            active = active ?? stats.active;
+        }
+        return { queued, waiting, waitingModelIds, lastTurn, active };
+    }
+}
+_ModelProcessPool_createSupervisor = new WeakMap(), _ModelProcessPool_createScheduler = new WeakMap(), _ModelProcessPool_loadModel = new WeakMap(), _ModelProcessPool_onChange = new WeakMap(), _ModelProcessPool_logger = new WeakMap(), _ModelProcessPool_slots = new WeakMap(), _ModelProcessPool_queue = new WeakMap(), _ModelProcessPool_maxModels = new WeakMap(), _ModelProcessPool_busy = new WeakMap(), _ModelProcessPool_dirty = new WeakMap(), _ModelProcessPool_clock = new WeakMap(), _ModelProcessPool_instances = new WeakSet(), _ModelProcessPool_makeSlot = function _ModelProcessPool_makeSlot(index, supervisor) {
+    var _a;
+    const slot = {
+        index,
+        supervisor,
+        scheduler: null,
+        assignedModelId: supervisor.model?.id ?? null,
+        pending: 0,
+        reservationBytes: 0,
+        lastUsedAt: __classPrivateFieldSet(this, _ModelProcessPool_clock, (_a = __classPrivateFieldGet(this, _ModelProcessPool_clock, "f"), ++_a), "f"),
+    };
+    slot.scheduler = __classPrivateFieldGet(this, _ModelProcessPool_createScheduler, "f").call(this, supervisor, async (model) => {
+        var _a;
+        const reservedElsewhereBytes = __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").reduce((total, candidate) => (candidate === slot ? total : total + candidate.reservationBytes), 0);
+        slot.reservationBytes = await __classPrivateFieldGet(this, _ModelProcessPool_loadModel, "f").call(this, supervisor, model, reservedElsewhereBytes);
+        slot.assignedModelId = model.id;
+        slot.lastUsedAt = __classPrivateFieldSet(this, _ModelProcessPool_clock, (_a = __classPrivateFieldGet(this, _ModelProcessPool_clock, "f"), ++_a), "f");
+        __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_announce).call(this);
+    }, () => {
+        __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_announce).call(this);
+        void __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_dispatch).call(this);
+    });
+    return slot;
+}, _ModelProcessPool_announce = function _ModelProcessPool_announce() {
+    try {
+        __classPrivateFieldGet(this, _ModelProcessPool_onChange, "f")?.call(this);
+    }
+    catch {
+        // Status observers are not allowed to fail admission or eviction.
+    }
+}, _ModelProcessPool_dispatch = async function _ModelProcessPool_dispatch() {
+    if (__classPrivateFieldGet(this, _ModelProcessPool_busy, "f")) {
+        __classPrivateFieldSet(this, _ModelProcessPool_dirty, true, "f");
+        return;
+    }
+    __classPrivateFieldSet(this, _ModelProcessPool_busy, true, "f");
+    try {
+        do {
+            __classPrivateFieldSet(this, _ModelProcessPool_dirty, false, "f");
+            await __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_pass).call(this);
+        } while (__classPrivateFieldGet(this, _ModelProcessPool_dirty, "f"));
+    }
+    finally {
+        __classPrivateFieldSet(this, _ModelProcessPool_busy, false, "f");
+    }
+}, _ModelProcessPool_pass = async function _ModelProcessPool_pass() {
+    var _a;
+    await __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_trimIdleSlots).call(this);
+    for (;;) {
+        let selectedIndex = -1;
+        let selectedSlot = null;
+        for (let index = 0; index < __classPrivateFieldGet(this, _ModelProcessPool_queue, "f").length; index += 1) {
+            const candidate = __classPrivateFieldGet(this, _ModelProcessPool_queue, "f")[index];
+            const slot = await __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_slotFor).call(this, candidate.model);
+            if (!slot)
+                continue;
+            selectedIndex = index;
+            selectedSlot = slot;
+            break;
+        }
+        if (selectedIndex < 0 || !selectedSlot)
+            return;
+        const [job] = __classPrivateFieldGet(this, _ModelProcessPool_queue, "f").splice(selectedIndex, 1);
+        if (!job)
+            return;
+        const slot = selectedSlot;
+        slot.pending += 1;
+        slot.lastUsedAt = __classPrivateFieldSet(this, _ModelProcessPool_clock, (_a = __classPrivateFieldGet(this, _ModelProcessPool_clock, "f"), ++_a), "f");
+        __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_announce).call(this);
+        void slot.scheduler
+            .submit(job.model, job.run, job.options)
+            .then(job.resolve, job.reject)
+            .finally(() => {
+            var _a;
+            slot.pending = Math.max(0, slot.pending - 1);
+            slot.lastUsedAt = __classPrivateFieldSet(this, _ModelProcessPool_clock, (_a = __classPrivateFieldGet(this, _ModelProcessPool_clock, "f"), ++_a), "f");
+            if (slot.supervisor.state === "failed" && slot.pending === 0) {
+                void __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_releaseSlot).call(this, slot).finally(() => void __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_dispatch).call(this));
+            }
+            __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_announce).call(this);
+            void __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_dispatch).call(this);
+        });
+    }
+}, _ModelProcessPool_slotFor = async function _ModelProcessPool_slotFor(model) {
+    const resident = __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").find((slot) => slot.assignedModelId === model.id);
+    if (resident)
+        return resident;
+    let empty = __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").find((slot) => slot.assignedModelId === null && slot.scheduler.isIdle && slot.pending === 0);
+    if (!empty && __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").length < __classPrivateFieldGet(this, _ModelProcessPool_maxModels, "f")) {
+        const index = __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_nextSlotIndex).call(this);
+        empty = __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_makeSlot).call(this, index, __classPrivateFieldGet(this, _ModelProcessPool_createSupervisor, "f").call(this, index));
+        __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").push(empty);
+    }
+    if (empty) {
+        empty.assignedModelId = model.id;
+        return empty;
+    }
+    const evictable = __classPrivateFieldGet(this, _ModelProcessPool_slots, "f")
+        .filter((slot) => slot.scheduler.isIdle && slot.pending === 0)
+        .sort((left, right) => left.lastUsedAt - right.lastUsedAt)[0];
+    if (!evictable)
+        return null;
+    __classPrivateFieldGet(this, _ModelProcessPool_logger, "f")?.call(this, `evicting ${evictable.supervisor.model?.displayName ?? "idle model"} for ${model.displayName}`);
+    await __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_releaseSlot).call(this, evictable);
+    evictable.assignedModelId = model.id;
+    return evictable;
+}, _ModelProcessPool_nextSlotIndex = function _ModelProcessPool_nextSlotIndex() {
+    const used = new Set(__classPrivateFieldGet(this, _ModelProcessPool_slots, "f").map((slot) => slot.index));
+    let index = 0;
+    while (used.has(index))
+        index += 1;
+    return index;
+}, _ModelProcessPool_releaseSlot = async function _ModelProcessPool_releaseSlot(slot) {
+    var _a;
+    await slot.supervisor.stop();
+    slot.scheduler.forgetSlots();
+    slot.assignedModelId = null;
+    slot.reservationBytes = 0;
+    slot.lastUsedAt = __classPrivateFieldSet(this, _ModelProcessPool_clock, (_a = __classPrivateFieldGet(this, _ModelProcessPool_clock, "f"), ++_a), "f");
+}, _ModelProcessPool_trimIdleSlots = async function _ModelProcessPool_trimIdleSlots() {
+    if (__classPrivateFieldGet(this, _ModelProcessPool_slots, "f").length <= __classPrivateFieldGet(this, _ModelProcessPool_maxModels, "f"))
+        return;
+    const removable = __classPrivateFieldGet(this, _ModelProcessPool_slots, "f")
+        .filter((slot) => slot.scheduler.isIdle && slot.pending === 0)
+        .sort((left, right) => left.lastUsedAt - right.lastUsedAt);
+    while (__classPrivateFieldGet(this, _ModelProcessPool_slots, "f").length > __classPrivateFieldGet(this, _ModelProcessPool_maxModels, "f") && removable.length > 0) {
+        const slot = removable.shift();
+        await __classPrivateFieldGet(this, _ModelProcessPool_instances, "m", _ModelProcessPool_releaseSlot).call(this, slot);
+        const index = __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").indexOf(slot);
+        if (index >= 0)
+            __classPrivateFieldGet(this, _ModelProcessPool_slots, "f").splice(index, 1);
+    }
+};
+export function normalizeModelLimit(value) {
+    if (!Number.isFinite(value))
+        return 1;
+    return Math.max(1, Math.min(16, Math.floor(value)));
+}
+//# sourceMappingURL=process-pool.js.map

package/dist/service/router.d.ts

@@ -1,5 +1,5 @@
 import http from "node:http";
-import { Scheduler } from "./scheduler.js";
+import { type ModelScheduler } from "./scheduler.js";
 import type { Supervisor } from "./supervisor.js";
 import { type RankedModel } from "../ops/results.js";
 import type { GpuInfo, Model } from "../types.js";
@@ -108,7 +108,7 @@ export declare function describeModel(model: Model | null, options?: DescribeOpt
  * the supervisor is running marked 'loaded'. Falls back to just the running
  * model when no catalog provider is wired in.
  */
-export declare function buildModelList(supervisor: Supervisor, getCatalog: GetCatalog): ModelEntry[];
+export declare function buildModelList(supervisor: Supervisor, getCatalog: GetCatalog, scheduler?: ModelScheduler<Supervisor> | null): ModelEntry[];
 /**
  * Map an OpenAI-compatible effort request onto a model's own chat-template
  * arguments. llama.cpp does not know every model's dialect: Qwen3.8 calls the
@@ -207,6 +207,7 @@ export declare function decideModelGate(params: {
     lockModel: boolean;
     requestedName: string | null;
     pinned: Model | null;
+    pinnedModels?: Model[];
     resolved: Model | null;
 }): ModelGateResult;
 export interface RouterOptions {
@@ -227,6 +228,8 @@ export interface RouterOptions {
     getLockModel?: () => boolean;
     /** Live: the configured default model, the pin target before one is resident. */
     getDefaultModel?: () => string | null;
+    /** Live: the complete model set served while locking is enabled. */
+    getLockedModels?: () => string[];
     /**
      * Apply an editable config patch (write config.json, live-switch the model,
      * update the lock), for POST /__host/config. Absent = the write endpoint is
@@ -262,8 +265,8 @@ export interface RouterOptions {
      */
     statusEvents?: BrainStatusPublisher | null;
     /** A service shares this scheduler with host-owned model operations. */
-    scheduler?: Scheduler | null;
+    scheduler?: ModelScheduler<Supervisor> | null;
 }
-export declare function createRouter({ supervisor, telemetry, logger, getCatalog, loadModel, loadRanking, queryGpuInfo, version, getConfig, getEvals, getLockModel, getDefaultModel, applyConfigPatch, getAllowConfigWrite, hostApi, getResources, statusEvents, scheduler: suppliedScheduler, }: RouterOptions): (req: http.IncomingMessage, res: http.ServerResponse) => void;
+export declare function createRouter({ supervisor, telemetry, logger, getCatalog, loadModel, loadRanking, queryGpuInfo, version, getConfig, getEvals, getLockModel, getDefaultModel, getLockedModels, applyConfigPatch, getAllowConfigWrite, hostApi, getResources, statusEvents, scheduler: suppliedScheduler, }: RouterOptions): (req: http.IncomingMessage, res: http.ServerResponse) => void;
 export {};
 //# sourceMappingURL=router.d.ts.map