@otto-code/brain 0.8.12 → 0.8.13

package/dist/service/serve.js

@@ -13,7 +13,7 @@ import { execFile, spawn } from "node:child_process";
 import { randomUUID } from "node:crypto";
 import http from "node:http";
 import https from "node:https";
-import { getCalibration, forModel, loadPersistedConfig, loadProfilesStore, put, putCalibration, saveBrainConfig, saveProfilesStore, } from "../config/index.js";
+import { getCalibrationForBudget, forModel, loadPersistedConfig, loadProfilesStore, put, putCalibration, saveBrainConfig, saveProfilesStore, } from "../config/index.js";
 import { resolveBrainPaths } from "../config/paths.js";
 import { query as queryGpu } from "../gpu.js";
 import { managedModelsDir, pickAutoModel, pickModel, scanModels } from "../models/index.js";
@@ -31,6 +31,7 @@ import { createHostApi } from "./host-api.js";
 import { errorMessage } from "./http-util.js";
 import { createRouter, createSlotEraser, Telemetry } from "./router.js";
 import { Scheduler } from "./scheduler.js";
+import { ModelProcessPool } from "./process-pool.js";
 import { BrainLogPublisher, BrainStatusPublisher } from "./status-events.js";
 import { Supervisor } from "./supervisor.js";
 import * as tailscale from "./tailscale.js";
@@ -171,9 +172,9 @@ class ServiceJobRunner {
             job.queuePosition = this.scheduler.stats().queued + 1;
             job.message = `Queued for ${model.displayName}`;
             void this.scheduler
-                .submit(model, () => controller.signal.aborted
+                .submit(model, (supervisor) => controller.signal.aborted
                 ? Promise.reject(new Error("Operation canceled."))
-                : this.runResidentJob(kind, model.id, {
+                : this.runResidentJob(supervisor, kind, model.id, {
                     message: (value) => {
                         if (job.status === "running")
                             job.message = value.slice(-1000);
@@ -469,7 +470,7 @@ export async function startService({ config, modelNeedle, env = process.env, onL
         const fit = vram.fitToBudget({
             model,
             profile,
-            calibration: getCalibration(store, model, profile),
+            calibration: getCalibrationForBudget(store, model, profile),
             totalVramBytes: gpu.totalBytes,
         });
         if (!fit.adjusted && !fit.budget.fits) {
@@ -495,41 +496,43 @@ export async function startService({ config, modelNeedle, env = process.env, onL
     });
     supervisor.on("log", (line) => log("server", line));
     supervisor.on("crashed", (error) => log("model", `FATAL ${error}`));
-    // Serialize model switches: the router queues request-driven switches, but the
-    // config path (POST /__host/config) calls loadModel directly. Chaining here
-    // guarantees two switches (e.g. a config write racing a request-driven switch)
-    // can never overlap two supervisor.start() calls, whichever caller triggers them.
-    let modelSwitchChain = Promise.resolve();
-    const loadModelUnsafe = async (target) => {
+    const loadModelInto = async (resident, target, reservedElsewhereBytes) => {
         // A runtime can be installed from the Library tab after this service starts.
         // Resolve it at load time so the user does not have to restart the brain.
-        supervisor.runtime = resolveRuntime(config, env);
-        if (!supervisor.runtime) {
+        resident.runtime = resolveRuntime(config, env);
+        if (!resident.runtime) {
             throw new Error("no llama.cpp runtime available; install one from the Library tab");
         }
         const gpuInfo = await queryGpu();
         let fitProfile = forModel(store, target, config.defaults);
+        let reservationBytes = 0;
         if (gpuInfo) {
             const fit = vram.fitToBudget({
                 model: target,
                 profile: fitProfile,
-                calibration: getCalibration(store, target, fitProfile),
-                totalVramBytes: gpuInfo.totalBytes,
+                calibration: getCalibrationForBudget(store, target, fitProfile),
+                // Every resident process keeps its complete budget reserved. Fit this
+                // process against the capacity left after those independent allocations.
+                totalVramBytes: Math.max(0, gpuInfo.totalBytes - reservedElsewhereBytes),
             });
             if (!fit.adjusted && !fit.budget.fits)
                 throw new Error(fit.reason ?? "does not fit");
             fitProfile = fit.profile;
+            reservationBytes = fit.budget.totalBytes;
         }
-        await supervisor.start(target, fitProfile);
+        await resident.start(target, fitProfile);
         delete store.pendingReloadModelIds[target.id];
         store.lastModelId = target.id;
         saveProfilesStore(store, paths);
+        return reservationBytes;
     };
-    const loadModel = (target) => {
-        const run = modelSwitchChain.then(() => loadModelUnsafe(target));
-        // Keep the chain alive even if this switch fails, so a later switch still runs.
-        modelSwitchChain = run.catch(() => undefined);
-        return run;
+    let processPool = null;
+    const loadModel = async (target) => {
+        if (processPool) {
+            await processPool.preload(target);
+            return;
+        }
+        await loadModelInto(supervisor, target, 0);
     };
     // Apply an editable config patch from POST /__host/config: mutate the live
     // config (so the lock/default getters and future starts see it), persist it to
@@ -555,15 +558,42 @@ export async function startService({ config, modelNeedle, env = process.env, onL
                 throw new Error("lockModel must be a boolean");
             config.lockModel = p.lockModel;
         }
+        if ("maxLoadedModels" in p) {
+            const next = p.maxLoadedModels;
+            if (!Number.isInteger(next) || next < 1 || next > 16) {
+                throw new Error("maxLoadedModels must be an integer from 1 to 16");
+            }
+            config.maxLoadedModels = next;
+        }
+        if ("lockedModels" in p) {
+            const next = p.lockedModels;
+            if (!Array.isArray(next) || !next.every((value) => typeof value === "string")) {
+                throw new Error("lockedModels must be an array of model ids");
+            }
+            config.lockedModels = [...new Set(next)].slice(0, config.maxLoadedModels);
+        }
+        if (config.lockedModels.length > config.maxLoadedModels) {
+            config.lockedModels = config.lockedModels.slice(0, config.maxLoadedModels);
+        }
         const persisted = loadPersistedConfig(paths);
         persisted.defaultModel = config.defaultModel;
         persisted.lockModel = config.lockModel;
+        persisted.maxLoadedModels = config.maxLoadedModels;
+        persisted.lockedModels = config.lockedModels;
         saveBrainConfig(persisted, paths);
+        await processPool?.configure(config.maxLoadedModels);
         if (switchTo) {
             const target = catalog.find((m) => m.displayName === switchTo || m.id === switchTo);
             if (target)
                 await loadModel(target);
         }
+        if (config.lockModel) {
+            for (const id of config.lockedModels) {
+                const target = catalog.find((candidate) => candidate.id === id || candidate.displayName === id);
+                if (target)
+                    await loadModel(target);
+            }
+        }
         return redactConfig(config);
     };
     // One CPU sampler for the lifetime of the service: it reports a busy fraction
@@ -574,7 +604,7 @@ export async function startService({ config, modelNeedle, env = process.env, onL
     // /__host/events. One instance, so `capabilities.events` and the stream can
     // never disagree about whether this brain publishes.
     const statusEvents = new BrainStatusPublisher();
-    const runResidentJob = async (kind, target, update, signal) => {
+    const runResidentJob = async (supervisor, kind, target, update, signal) => {
         const ensureActive = () => {
             if (signal.aborted)
                 throw new Error("Operation canceled.");
@@ -659,7 +689,7 @@ export async function startService({ config, modelNeedle, env = process.env, onL
         supervisor.recordLog(`operation benchmark: resident model ready`);
         const profile = supervisor.profile ?? forModel(store, targetModel, config.defaults);
         const gpuInfo = await queryGpu();
-        const calibration = getCalibration(store, targetModel, profile);
+        const calibration = getCalibrationForBudget(store, targetModel, profile);
         const fit = gpuInfo
             ? vram.fitToBudget({
                 model: targetModel,
@@ -703,47 +733,54 @@ export async function startService({ config, modelNeedle, env = process.env, onL
         update.percent(100);
         supervisor.recordLog(`operation benchmark: saved result for ${targetModel.displayName}`);
     };
-    const scheduler = new Scheduler({
-        supervisor,
-        loadModel,
+    const attachSupervisorLogs = (resident) => {
+        if (resident === supervisor)
+            return;
+        resident.on("log", (line) => log("server", line));
+        resident.on("crashed", (error) => log("model", `FATAL ${error}`));
+        resident.on("state", () => statusEvents.notify());
+    };
+    const createPooledSupervisor = (index) => {
+        const resident = new Supervisor({
+            runtime: resolveRuntime(config, env),
+            internalPort: supervisor.internalPort + index,
+            paths,
+            getProfilesStore: () => store,
+            logVerbosity: config.runtime.logVerbosity,
+        });
+        attachSupervisorLogs(resident);
+        return resident;
+    };
+    const createResidentScheduler = (resident, loadResidentModel, onChange) => new Scheduler({
+        supervisor: resident,
+        loadModel: loadResidentModel,
         logger: (message) => log("api", `WARN ${message}`),
-        onChange: () => statusEvents.notify(),
-        // Live admission: the engine's own /slots report is the source of truth
-        // for how many sequence slots are actually free right now. The profile's
-        // parallelSlots sizes the KV pool at launch; this keeps dispatch honest
-        // while llama-server is mid-eviction or saturated, and degrades to the
-        // static count when the sample is unavailable.
-        // Answers with the free slots NAMED, not just counted. The count gates
-        // admission; the ids let the scheduler pin each admitted completion to a
-        // distinct slot (`id_slot`), which is what lets the proxy attribute a
-        // request's stage - "thinking" above all, which llama-server cannot report -
-        // to the exact slot row the Overview panel draws. Handing back only a count
-        // (as this did before) leaves every request unpinned and the panel unable to
-        // say which slot is thinking and which is emitting tokens.
+        onChange,
         freeSlots: async () => {
-            if (supervisor.state !== "ready")
+            if (resident.state !== "ready")
                 return null;
             try {
-                const slots = await sampleSlots({
-                    host: supervisor.host,
-                    port: supervisor.internalPort,
-                });
+                const slots = await sampleSlots({ host: resident.host, port: resident.internalPort });
                 return slots ? { idle: slots.idle, ids: slots.idleSlots } : null;
             }
             catch {
                 return null;
             }
         },
-        // Erase a slot's retained KV when it is handed to a different chat, so one
-        // chat's KV never bleeds into the next chat's thinking. The scheduler
-        // decides the handoff from the owner map; this is the engine-side wipe on
-        // the private port, resolved once the engine acknowledges it (see
-        // Scheduler.OWNERSHIP). The router that shares this scheduler clears the
-        // owner map on the supervisor's `starting` state.
-        eraseSlot: createSlotEraser(supervisor.host, supervisor.internalPort),
+        eraseSlot: createSlotEraser(resident.host, resident.internalPort),
     });
+    const scheduler = new ModelProcessPool({
+        initialSupervisor: supervisor,
+        maxModels: config.maxLoadedModels,
+        createSupervisor: createPooledSupervisor,
+        createScheduler: createResidentScheduler,
+        loadModel: loadModelInto,
+        logger: (message) => log("model", message),
+        onChange: () => statusEvents.notify(),
+    });
+    processPool = scheduler;
     const jobs = new ServiceJobRunner(rescanCatalog, runResidentJob, scheduler, (target) => {
-        const modelId = target ?? supervisor.model?.id ?? store.lastModelId ?? null;
+        const modelId = target ?? scheduler.residentSupervisors()[0]?.model?.id ?? store.lastModelId ?? null;
         return modelId
             ? (catalog.find((candidate) => candidate.id === modelId || candidate.displayName === modelId) ?? null)
             : null;
@@ -768,6 +805,7 @@ export async function startService({ config, modelNeedle, env = process.env, onL
             }
         },
         loadModel,
+        unloadModels: () => scheduler.unload(),
         scheduler,
         // The same gate as POST /__host/config. Deleting someone's model files over
         // the network is strictly more dangerous than changing their default model,
@@ -796,6 +834,7 @@ export async function startService({ config, modelNeedle, env = process.env, onL
         getEvals: collectEvals,
         getLockModel: () => config.lockModel,
         getDefaultModel: () => config.defaultModel,
+        getLockedModels: () => config.lockedModels,
         applyConfigPatch,
         getAllowConfigWrite: allowWrite,
         hostApi,
@@ -846,11 +885,17 @@ export async function startService({ config, modelNeedle, env = process.env, onL
         throw error;
     }
     certManager?.start();
-    if (model && profile && runtime) {
-        await supervisor.start(model, profile);
-        delete store.pendingReloadModelIds[model.id];
-        store.lastModelId = model.id;
-        saveProfilesStore(store, paths);
+    if (runtime && config.lockModel && config.lockedModels.length > 0) {
+        for (const lockedId of config.lockedModels.slice(0, config.maxLoadedModels)) {
+            const locked = catalog.find((candidate) => candidate.id === lockedId || candidate.displayName === lockedId);
+            if (locked)
+                await scheduler.preload(locked);
+        }
+    }
+    else if (model && profile && runtime) {
+        // Default model remains one auto-load. Additional process slots stay empty
+        // until a request names another model.
+        await scheduler.preload(model);
     }
     else if (!runtime) {
         log("server", "ready: no llama.cpp runtime installed; use the Library tab to download one");
@@ -866,11 +911,15 @@ export async function startService({ config, modelNeedle, env = process.env, onL
         secure: Boolean(tlsOptions),
         displayHost,
     }, env);
-    log("server", `ready: ${supervisor.model?.displayName ?? "no model loaded"} on ${bindHost}:${port}; run log ${runLog.path}`);
+    log("server", `ready: ${scheduler
+        .residentSupervisors()
+        .map((resident) => resident.model?.displayName)
+        .filter(Boolean)
+        .join(", ") || "no model loaded"} on ${bindHost}:${port}; run log ${runLog.path}`);
     const stop = async () => {
         certManager?.stop();
         log("server", "Brain service stopping");
-        await supervisor.stop();
+        await scheduler.stop();
         // Publish the terminal outcome before closing the SSE responses below.
         // Once the listener is closed, the daemon can still report the child exit,
         // but it cannot receive this service-owned, durable session-log entry.

package/dist/service/status-events.js

@@ -97,6 +97,7 @@ export function statusChangeKey(snapshot) {
         state: snapshot.state ?? null,
         model: snapshot.model ?? null,
         modelId: snapshot.modelId ?? null,
+        residents: snapshot.residents ?? null,
         pid: snapshot.pid ?? null,
         vramBytes: snapshot.vramBytes ?? null,
         loadSeconds: snapshot.loadSeconds ?? null,

package/dist/service/supervisor.js

@@ -11,7 +11,7 @@ import { spawn } from "node:child_process";
 import { EventEmitter } from "node:events";
 import { buildArgs, buildEnv, formatCommand } from "../runtime/index.js";
 import { resolveHostingProfileForLaunch } from "../config/hosting-profiles.js";
-import { getCalibration } from "../config/profiles.js";
+import { getCalibrationForBudget } from "../config/profiles.js";
 import { resolveBrainPaths } from "../config/paths.js";
 import { loadProfilesStore } from "../config/store.js";
 import { usedBytes } from "../gpu.js";
@@ -115,7 +115,7 @@ export class Supervisor extends EventEmitter {
         // The prompt-cache budget is derived from measured KV bytes/token, so the
         // launch boundary is where it has to be resolved - nothing downstream of
         // here can reach the calibration store.
-        getCalibration(this.getProfilesStore(), model, launchProfile));
+        getCalibrationForBudget(this.getProfilesStore(), model, launchProfile));
         this.args = args;
         this.command = formatCommand(runtime, args);
         __classPrivateFieldGet(this, _Supervisor_instances, "m", _Supervisor_log).call(this, formatBrainLog("model", `launching: ${this.command}`));

package/dist/tui/app.js

@@ -246,7 +246,7 @@ export class App {
             const fit = vram.fitToBudget({
                 model: target,
                 profile,
-                calibration: profiles.getCalibration(this.store, target, profile),
+                calibration: profiles.getCalibrationForBudget(this.store, target, profile),
                 totalVramBytes: info.totalBytes,
             });
             if (!fit.adjusted && !fit.budget.fits)
@@ -430,7 +430,7 @@ export class App {
         const profile = this.profile;
         if (!model || !profile)
             return null;
-        return profiles.getCalibration(this.store, model, profile);
+        return profiles.getCalibrationForBudget(this.store, model, profile);
     }
     get budget() {
         const model = this.model;
@@ -1398,7 +1398,9 @@ export class App {
                         // Only when it belongs to the model actually being benchmarked - the
                         // model may have been resident since before this fit was computed.
                         fit: this.lastFit?.modelId === model.id ? this.lastFit.fit : null,
-                        calibration: profile ? profiles.getCalibration(this.store, model, profile) : null,
+                        calibration: profile
+                            ? profiles.getCalibrationForBudget(this.store, model, profile)
+                            : null,
                         suite: {
                             // The TUI runs the full static suite; only concurrency varies, and
                             // it tracks the profile's slot count the same way runSuite is called.
@@ -1818,8 +1820,12 @@ export class App {
             const start = Math.max(0, Math.min(focusLine - Math.floor(rows / 2), content.length - rows));
             lines = content.slice(start, start + rows);
         }
+        const stale = Boolean(this.profile?.calibrationRequired ||
+            (this.model &&
+                this.profile &&
+                profiles.hasStaleCalibration(this.store, this.model, this.profile)));
         return box({
-            title: `Configuration${this.calibration ? `  ${style.brightGreen}calibrated${style.reset}` : `  ${style.yellow}not calibrated${style.reset}`}`,
+            title: `Configuration${stale ? `  ${style.yellow}recalibrate${style.reset}` : this.calibration ? `  ${style.brightGreen}calibrated${style.reset}` : `  ${style.yellow}not calibrated${style.reset}`}`,
             lines,
             innerWidth,
             footer: `${style.grey}←→ change · enter edit${style.reset}`,
@@ -1847,18 +1853,21 @@ export class App {
             ` = ${style.bold}${vram.formatGiB(b.totalBytes)}${style.reset}`;
         const kvLabel = `kv ${(b.kvBytesPerToken / 1024).toFixed(1)} KB/token`;
         const cal = this.calibration;
+        const stale = Boolean(this.profile?.calibrationRequired ||
+            (this.model &&
+                this.profile &&
+                profiles.hasStaleCalibration(this.store, this.model, this.profile)));
         let sourceNote;
         if (b.source === "measured") {
             sourceNote = cal?.inherited
                 ? `${style.yellow}${kvLabel} (measured on a relative - press c to calibrate this model)${style.reset}`
-                : `${style.grey}${kvLabel} (measured)${style.reset}`;
+                : stale
+                    ? `${style.yellow}${kvLabel} (last measurement - press c to recalibrate)${style.reset}`
+                    : `${style.grey}${kvLabel} (measured)${style.reset}`;
         }
         else {
-            const stale = this.model && this.profile
-                ? profiles.hasStaleCalibration(this.store, this.model, this.profile)
-                : false;
             const hint = stale
-                ? "cache types changed - press c to recalibrate"
+                ? "recalibrate to refresh this value"
                 : "press c to calibrate; usually unlocks more context";
             sourceNote = `${style.yellow}${kvLabel} (theoretical - ${hint})${style.reset}`;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@otto-code/brain",
-  "version": "0.8.12",
+  "version": "0.8.13",
   "description": "Otto Brain - self-contained host for local GGUF models, with measured VRAM budgeting and reasoning-budget control",
   "license": "AGPL-3.0-or-later",
   "bin": {