@otto-code/brain 0.8.9 → 0.8.12

package/dist/service/router.js

@@ -180,6 +180,10 @@ export function describeModel(model, options = {}) {
         reasoningEfforts.every((value) => typeof value === "string")) {
         entry.reasoning_efforts = reasoningEfforts;
     }
+    const reasoningEffortDefault = md["reasoning_effort_default"] ?? model.reasoningEffortDefault;
+    if (typeof reasoningEffortDefault === "string") {
+        entry.reasoning_effort_default = reasoningEffortDefault;
+    }
     if (state === "loaded" && profile && profile.contextSize) {
         // llama-server splits -c across --parallel slots, so the window a single
         // request actually gets is the total divided by the concurrency.
@@ -277,13 +281,152 @@ function nextStreamId() {
  * bench command builds its own throwaway router and simply never reads it.
  */
 const reasoningTracker = new ReasoningTracker();
+/**
+ * Map an OpenAI-compatible effort request onto a model's own chat-template
+ * arguments. llama.cpp does not know every model's dialect: Qwen3.8 calls the
+ * controls `enable_thinking` and `reasoning_effort`, for example. Only catalog
+ * entries that declare these names are rewritten, so generic models and GPT-OSS
+ * keep their existing server-native request handling.
+ */
+export function applyModelReasoningTemplate(body, model) {
+    const template = model.reasoningTemplate;
+    const advertised = model.reasoningEfforts;
+    if (!template || !Array.isArray(advertised))
+        return body;
+    let parsed;
+    try {
+        parsed = JSON.parse(body.toString("utf8"));
+    }
+    catch {
+        return body;
+    }
+    if (!isRecord(parsed) || typeof parsed.reasoning_effort !== "string")
+        return body;
+    const requested = parsed.reasoning_effort.toLowerCase();
+    const knownEfforts = new Set(advertised.map((value) => value.toLowerCase()));
+    const isDisabled = requested === "off" || requested === "none";
+    const isEnabled = requested === "on" || knownEfforts.has(requested);
+    if (!isDisabled && !isEnabled)
+        return body;
+    const suppliedKwargs = parsed.chat_template_kwargs;
+    if (suppliedKwargs !== undefined && !isRecord(suppliedKwargs))
+        return body;
+    const templateKwargs = { ...suppliedKwargs };
+    templateKwargs[template.enableThinkingArgument] = !isDisabled;
+    if (knownEfforts.has(requested)) {
+        templateKwargs[template.effortArgument] = requested;
+    }
+    else {
+        // The generic On selection means the model's native default, not a stale
+        // explicit effort that happened to be supplied by a previous client.
+        delete templateKwargs[template.effortArgument];
+    }
+    const { reasoning_effort: _reasoningEffort, ...withoutReasoningEffort } = parsed;
+    return Buffer.from(JSON.stringify({ ...withoutReasoningEffort, chat_template_kwargs: templateKwargs }), "utf8");
+}
+/**
+ * Pin the request to one llama-server slot by adding the engine's own
+ * `id_slot` field to the request body (host API v3).
+ *
+ * Why pin instead of guess: the OpenAI-compatible stream chunks never carry the
+ * slot id, so without a pin the router can only correlate a request to a slot
+ * by elimination, and with several concurrent requests that guess is exactly
+ * the lie the Overview panel used to tell. llama-server honors the pin on every
+ * completion endpoint: if the named slot is free the task lands there, and if
+ * it is busy the engine DEFERS the task internally - it never reassigns the
+ * task elsewhere and never fails the request - so the slot this router names is
+ * always the slot the request ends up on (possibly after waiting on it).
+ *
+ * `null` returns the body untouched: no pin, and therefore no join data for
+ * this request, which is the same degraded state as an older brain. A body this
+ * cannot parse is forwarded exactly as-is - an unfamiliar request must reach
+ * llama-server and get llama-server's own answer, not a 400 invented here.
+ */
+export function pinSlot(body, slotId) {
+    if (slotId === null)
+        return body;
+    let parsed;
+    try {
+        parsed = JSON.parse(body.toString("utf8"));
+    }
+    catch {
+        return body;
+    }
+    if (!isRecord(parsed))
+        return body;
+    parsed.id_slot = slotId;
+    return Buffer.from(JSON.stringify(parsed), "utf8");
+}
+/**
+ * Wipe one llama-server slot's retained KV state, and RESOLVE only once the
+ * engine has acknowledged the wipe.
+ *
+ * This is the engine-side half of the scheduler's OWNERSHIP fix. The engine
+ * never clears a released slot's prompt, so a slot handed to a different chat
+ * would keep the previous chat's KV and bleed its topics into the new chat's
+ * thinking. The router erases the slot the moment the scheduler hands it off;
+ * the engine's task queue runs in arrival order, so resolving on the
+ * acknowledgment is what guarantees the clean state sits in the queue ahead of
+ * the completion the scheduler posts right after.
+ *
+ * The route is `POST /slots?action=erase&id_slot=N` - llama.cpp's own slot
+ * action. It answers 200 `{id, id_slot, n_erased}` on success and a
+ * `NOT_SUPPORTED` error when the server was not launched with a slot-save path;
+ * either way this resolves (never rejects), because an erase that cannot be
+ * performed degrades to the old behavior rather than failing the completion.
+ *
+ * NOTE: `action` and `id_slot` MUST travel in the query string, not the JSON
+ * body. llama-server's `POST /slots` handler reads both via `req.get_param()`,
+ * which is built only from query + path params (b10441 tools/server/server-http.cpp,
+ * `server_http_req::params` = "path_params + query_params"; the body is a
+ * separate field the handler never parses for this route). A body-only request
+ * reaches `std::stoi("")` and answers 400 "Invalid slot ID" - the erase then
+ * silently no-ops and the bleed survives. The body must stay empty for the
+ * same reason `handle_slots_erase` ignores it entirely.
+ */
+export function eraseSlot(host, port, slotId) {
+    return new Promise((resolve) => {
+        const req = http.request({
+            host,
+            port,
+            path: `/slots?action=erase&id_slot=${slotId}`,
+            method: "POST",
+            timeout: 3000,
+        }, (res) => {
+            res.resume();
+            res.on("end", () => resolve());
+        });
+        req.on("timeout", () => req.destroy());
+        req.on("error", () => resolve());
+        req.end();
+    });
+}
+/**
+ * The eraser the scheduler needs, bound to one engine endpoint. Extracted so
+ * the router (which builds its own scheduler) and the service (which builds a
+ * shared one and passes it in) hand the scheduler the SAME transport rather
+ * than each spelling the request.
+ */
+export function createSlotEraser(host, port) {
+    return (slotId) => eraseSlot(host, port, slotId);
+}
 /**
  * Forward a buffered completion body to the resident llama-server and stream the
  * reply back, teeing non-streaming bodies for classification. Resolves once the
  * client response is fully concluded (it owns the response in every outcome,
  * including upstream errors), so the scheduler can move to the next turn.
+ *
+ * The slot this request is pinned to arrives as `options.slot`, named by the
+ * scheduler at the moment of admission (see `Scheduler.onSlotFree`). That is
+ * the only race-free place to choose it: by then the model is resident (slot
+ * ids do not survive a model switch), the admission sample has just counted
+ * the slot free, and every sibling admitted in the same pass has been named a
+ * DISTINCT id. `null` means the engine could not be sampled or reported no
+ * per-slot rows - the request then runs unpinned with no join data, exactly
+ * the degraded state of an older brain.
  */
-function proxyBuffered({ agent, supervisor, telemetry, logger, req, res, body, reasoning = null, }) {
+function proxyBuffered({ agent, model, supervisor, telemetry, logger, req, res, body, reasoning = null, slot, }) {
+    const slotId = slot ?? null;
     return new Promise((resolve) => {
         let settled = false;
         const streamId = nextStreamId();
@@ -300,7 +443,8 @@ function proxyBuffered({ agent, supervisor, telemetry, logger, req, res, body, r
         // Injected here, not at queue time: the scheduler may switch models between
         // buffering and dispatch, and the addendum belongs to whichever model ends
         // up resident, which is the one `supervisor.profile` now describes.
-        const outbound = injectSystemAddendum(body, supervisor.profile?.chatSystemAddendum ?? null, completionShape(req.url));
+        const withSystemAddendum = injectSystemAddendum(body, supervisor.profile?.chatSystemAddendum ?? null, completionShape(req.url));
+        const outbound = pinSlot(applyModelReasoningTemplate(withSystemAddendum, model), slotId);
         const headers = {};
         for (const [name, value] of Object.entries(req.headers)) {
             if (!HOP_BY_HOP.has(name.toLowerCase()))
@@ -309,6 +453,7 @@ function proxyBuffered({ agent, supervisor, telemetry, logger, req, res, body, r
         headers.host = `${supervisor.host}:${supervisor.internalPort}`;
         headers["content-length"] = Buffer.byteLength(outbound);
         const started = Date.now();
+        logger?.info?.(`dispatching ${req.method ?? "POST"} ${req.url ?? "/v1/chat/completions"} to ${model.displayName}`);
         const upstream = http.request({
             host: supervisor.host,
             port: supervisor.internalPort,
@@ -362,6 +507,7 @@ function proxyBuffered({ agent, supervisor, telemetry, logger, req, res, body, r
                         streamed: true,
                         verdict: !sawContent && sawReasoning ? "reasoning-only" : "ok",
                     });
+                    logger?.info?.(`completed streamed ${req.method ?? "POST"} ${req.url ?? "/v1/chat/completions"} for ${model.displayName}`);
                     done();
                 });
                 upstreamRes.pipe(res);
@@ -389,6 +535,7 @@ function proxyBuffered({ agent, supervisor, telemetry, logger, req, res, body, r
                     ...(analysis ?? { verdict: "ok" }),
                 };
                 telemetry.record(entry);
+                logger?.info?.(`completed ${req.method ?? "POST"} ${req.url ?? "/v1/chat/completions"} for ${model.displayName} in ${entry.ms}ms (${entry.verdict})`);
                 if (entry.verdict === "reasoning-only") {
                     logger?.warn?.(`reasoning-only response: ${entry.outputTokens} tokens, ${entry.reasoningChars} reasoning chars, 0 content`);
                 }
@@ -398,6 +545,12 @@ function proxyBuffered({ agent, supervisor, telemetry, logger, req, res, body, r
         res.on("close", () => {
             if (!res.writableFinished && !upstream.destroyed)
                 upstream.destroy();
+            // The client is gone - interrupt, chat switch, socket death. Destroying
+            // the upstream request does not reliably fire "aborted"/"error" on the
+            // already-open response stream, so this is the authoritative release:
+            // without it a departed client's job never resolves and pins its model's
+            // slot forever, wedging the whole queue behind it (needs a reboot).
+            done();
         });
         res.on("error", () => {
             if (!upstream.destroyed)
@@ -410,13 +563,21 @@ function proxyBuffered({ agent, supervisor, telemetry, logger, req, res, body, r
                 verdict: "failed",
                 error: error.message,
             });
-            sendError(res, 502, `Upstream llama-server error: ${error.message}`);
+            logger?.warn(`request to ${model.displayName} failed: ${error.message}`);
+            // The client may already be gone; writing a 502 into a dead socket
+            // throws and would kill this handler before done() releases the slot.
+            if (!res.writableEnded && !res.destroyed) {
+                sendError(res, 502, `Upstream llama-server error: ${error.message}`);
+            }
             done();
         });
         // This is the first authoritative inference-stage signal: the request is
         // being dispatched to llama-server and is waiting for prompt processing or
-        // its first output delta.
+        // its first output delta. The slot association is set once here, at the
+        // same moment, so `observe` stays free of any per-chunk slot work.
         reasoning?.begin(streamId);
+        if (slotId !== null)
+            reasoning?.setSlot(streamId, slotId);
         upstream.end(outbound);
     });
 }
@@ -544,22 +705,41 @@ function scheduleCompletion({ req, res, agent, supervisor, telemetry, logger, sc
         }
         const body = Buffer.concat(chunks);
         let modelName = null;
+        let session = null;
         try {
             const parsed = JSON.parse(body.toString("utf8"));
-            modelName = isRecord(parsed) && typeof parsed.model === "string" ? parsed.model : null;
+            if (isRecord(parsed)) {
+                modelName = typeof parsed.model === "string" ? parsed.model : null;
+                // The standard prompt_cache_key is the chat's stable identity: Otto's
+                // provider sends its session id there, and a third-party client that
+                // uses it for prompt caching sends one too. Brain reads it for
+                // session-affine scheduling; the OpenAI-compatible contract is
+                // untouched either way.
+                session =
+                    typeof parsed.prompt_cache_key === "string" && parsed.prompt_cache_key.length > 0
+                        ? parsed.prompt_cache_key
+                        : null;
+            }
         }
         catch {
             /* leave null */
         }
         const gate = modelGate(modelName);
         if (!gate.ok) {
+            logger?.warn(`refused ${req.method ?? "POST"} ${req.url ?? "completion"}: ${gate.message}`);
             sendError(res, gate.status, gate.message);
             return;
         }
         const model = gate.model;
-        scheduler
-            .submit(model, () => proxyBuffered({
+        // The slot is named by the scheduler at admission time (one distinct id per
+        // job, drawn from the pass's own free-slot sample). It is not sampled here
+        // at queue time - that could name a slot of an engine about to be
+        // relaunched - and not at dispatch time either, where a sibling admitted in
+        // the same pass could see the same slot free and pin it too.
+        let slot = null;
+        const queued = scheduler.submit(model, () => proxyBuffered({
             agent,
+            model,
             supervisor,
             telemetry,
             logger,
@@ -567,8 +747,10 @@ function scheduleCompletion({ req, res, agent, supervisor, telemetry, logger, sc
             res,
             body,
             reasoning: reasoningTracker,
-        }))
-            .catch((error) => sendError(res, 502, `could not serve ${model.displayName}: ${errorMessage(error)}`));
+            slot,
+        }), { session, onSlotFree: (id) => (slot = id) });
+        logger?.info?.(`queued ${req.method ?? "POST"} ${req.url ?? "completion"} for ${model.displayName}; queue depth ${scheduler.stats().queued}`);
+        queued.catch((error) => sendError(res, 502, `could not serve ${model.displayName}: ${errorMessage(error)}`));
     });
 }
 // The bench ranking is read from disk (one JSON per run). A completion request
@@ -576,29 +758,41 @@ function scheduleCompletion({ req, res, agent, supervisor, telemetry, logger, sc
 // (rare), so the router caches the ranking and re-reads it at most once per
 // window - the cheap time-based trigger.
 const RANKING_TTL_MS = 60000;
-export function createRouter({ supervisor, telemetry, logger, getCatalog = null, loadModel = null, loadRanking = () => rankModels(), queryGpuInfo = queryGpu, version = null, getConfig = null, getEvals = null, getLockModel = () => false, getDefaultModel = () => null, applyConfigPatch = null, getAllowConfigWrite = () => false, hostApi = null, getResources = null, statusEvents = null, }) {
+export function createRouter({ supervisor, telemetry, logger, getCatalog = null, loadModel = null, loadRanking = () => rankModels(), queryGpuInfo = queryGpu, version = null, getConfig = null, getEvals = null, getLockModel = () => false, getDefaultModel = () => null, applyConfigPatch = null, getAllowConfigWrite = () => false, hostApi = null, getResources = null, statusEvents = null, scheduler: suppliedScheduler = null, }) {
     // llama-server may close an idle response socket while this scheduler holds
     // the next request in queue. A reused keep-alive socket then fails as
     // ECONNRESET ("socket hang up") before the queued request reaches inference.
     // Inference time dwarfs localhost connection setup, so isolate each request
     // instead of letting a second client inherit a stale upstream connection.
     const agent = new http.Agent({ keepAlive: false, maxSockets: 32 });
-    const scheduler = loadModel
-        ? new Scheduler({
-            supervisor,
-            loadModel,
-            logger: (m) => logger?.warn?.(m),
-            onChange: statusEvents ? () => statusEvents.notify() : null,
-        })
-        : null;
+    // Erase an engine slot's KV when it is handed to a different chat. The
+    // endpoint is the engine's own, on the private port - never on the public
+    // one - so a remote client cannot wipe a chat's cache out from under it.
+    const eraseSlot = createSlotEraser(supervisor.host, supervisor.internalPort);
+    const scheduler = suppliedScheduler ??
+        (loadModel
+            ? new Scheduler({
+                supervisor,
+                loadModel,
+                logger: (m) => logger?.warn?.(m),
+                onChange: statusEvents ? () => statusEvents.notify() : null,
+                eraseSlot,
+            })
+            : null);
     // A (re)start means whatever produced the current warning no longer applies -
     // either a different model is now resident, or the same one just picked up an
     // edited profile (e.g. a lowered reasoning budget). Either way the recent
     // window is stale, so start it clean rather than let old records blame a
-    // config that is no longer running.
+    // config that is no longer running. The scheduler's slot owners go the same
+    // way: the engine's slots do not survive the relaunch, so a stale owner entry
+    // would make the next admission erase a FRESH slot or mistake a keyless job
+    // for one it owns. The model-switch case is cleared by the scheduler itself;
+    // this catches the relaunch that keeps the same model resident.
     supervisor.on("state", ({ state }) => {
-        if (state === "starting")
+        if (state === "starting") {
             telemetry.reset();
+            scheduler?.forgetSlots();
+        }
     });
     // GPU total VRAM is static hardware, so it is queried once at startup and
     // cached. Absent (no nvidia-smi) or not-yet-resolved leaves the fit predicate
@@ -763,8 +957,12 @@ export function createRouter({ supervisor, telemetry, logger, getCatalog = null,
                     sendError(res, 400, result.error);
                     return;
                 }
+                logger?.info?.("received host configuration update");
                 applyConfigPatch(result.body)
-                    .then((cfg) => sendJson(res, cfg))
+                    .then((cfg) => {
+                    logger?.info?.("applied host configuration update");
+                    return sendJson(res, cfg);
+                })
                     .catch((err) => sendError(res, 500, `could not apply config: ${errorMessage(err)}`));
             });
             return;

package/dist/service/run-log.d.ts

@@ -1,6 +1,11 @@
 export interface BrainRunLog {
     path: string;
-    write(line: string): void;
+    write(line: string): string[];
+    /** The current Brain-service session's durable tail, including every source. */
+    tail(limit: number): {
+        lines: string[];
+        total: number;
+    };
 }
 /** Start a fresh Brain log and prune only expired Brain run logs. */
 export declare function createBrainRunLog(env?: NodeJS.ProcessEnv): BrainRunLog;

package/dist/service/run-log.js

@@ -1,7 +1,8 @@
 /** Durable, per-service-run diagnostics for Otto Brain. */
-import { appendFileSync, mkdirSync, readdirSync, rmSync, statSync } from "node:fs";
+import { appendFileSync, mkdirSync, openSync, readSync, closeSync, readdirSync, rmSync, statSync, } from "node:fs";
 import path from "node:path";
 import { resolveBrainPaths } from "../config/paths.js";
+import { timestampBrainLogLine } from "./log-format.js";
 const RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
 const RUN_LOG_SUFFIX = "-brain.log";
 /** Start a fresh Brain log and prune only expired Brain run logs. */
@@ -18,14 +19,55 @@ export function createBrainRunLog(env = process.env) {
         pruneBrainRunLogs(logsDir, startedAt.getTime());
     }
     catch { }
+    let lineCount = 0;
     const write = (line) => {
+        const lines = line.split(/\r?\n|\r/u).filter((entry) => entry.length > 0);
+        if (lines.length === 0)
+            return [];
         try {
-            appendFileSync(filePath, `${new Date().toISOString()} ${line}\n`, "utf8");
+            const timestamp = new Date().toISOString().slice(11, 23);
+            const entries = lines.map((entry) => timestampBrainLogLine(timestamp, entry));
+            appendFileSync(filePath, `${entries.join("\n")}\n`, "utf8");
+            lineCount += lines.length;
+            return entries;
+        }
+        catch {
+            return [];
         }
-        catch { }
     };
     write(`Brain service started (pid ${process.pid})`);
-    return { path: filePath, write };
+    return {
+        path: filePath,
+        write,
+        tail(limit) {
+            const count = Math.max(1, Math.floor(limit));
+            // The Brain log is append-only for the service lifetime. Read only enough
+            // of its end for the live viewer, rather than loading an unbounded bench
+            // or download log every two-second poll.
+            try {
+                const size = statSync(filePath).size;
+                const bytes = Math.min(size, Math.max(64 * 1024, count * 1024));
+                const fd = openSync(filePath, "r");
+                try {
+                    const data = Buffer.alloc(bytes);
+                    readSync(fd, data, 0, bytes, size - bytes);
+                    const text = data.toString("utf8");
+                    const firstNewline = text.indexOf("\n");
+                    const completeLines = bytes < size && firstNewline !== -1 ? text.slice(firstNewline + 1) : text;
+                    return {
+                        lines: completeLines.split(/\r?\n/u).filter(Boolean).slice(-count),
+                        total: lineCount,
+                    };
+                }
+                finally {
+                    closeSync(fd);
+                }
+            }
+            catch {
+                return { lines: [], total: lineCount };
+            }
+        },
+    };
 }
 export function pruneBrainRunLogs(logsDir, now = Date.now()) {
     for (const entry of readdirSync(logsDir, { withFileTypes: true })) {