@otto-code/brain 0.8.8 → 0.8.9

package/dist/service/host-api.js

@@ -1,11 +1,24 @@
+import { randomUUID } from "node:crypto";
 import { calibrationInfo, profileFieldDescriptors, profileWarnings, sanitizeProfilePatch, } from "../config/profile-edit.js";
+import { familyHostingProfileId, hostingFamily, removeHostingProfileMaterialization, } from "../config/hosting-profiles.js";
 import { forModel, getCalibration, put } from "../config/profiles.js";
+import { HostingProfileSchema, } from "../config/schema.js";
 import { deleteComponentFile, deleteModelFiles, diskUsage, planDelete, totalModelBytes, } from "../models/manage.js";
 import { deleteDisplayName, updateDisplayName } from "../models/rename-map.js";
+import { runtimeBuild } from "../runtime/index.js";
 import * as vram from "../vram.js";
 import { errorMessage, readJsonBody, sendError, sendJson } from "./http-util.js";
 const MAX_PATCH_BYTES = 256 * 1024;
 const MAX_DISPLAY_NAME = 200;
+const MAX_HOSTING_PROFILE_NAME = 80;
+const MAX_HOSTING_PROFILE_TEXT = 128 * 1024;
+const MAX_HOSTING_PROFILES = 100;
+const MAX_HOSTING_PROFILE_ID = 80;
+const HOSTING_PROFILE_ID = /^[a-zA-Z0-9_-]+$/u;
+// Keep product-owned records immutable through this API. Their source lives in
+// builtin-hosting-profiles.ts, so an update would otherwise look successful but
+// be replaced at the next Brain start with no user-facing restore action.
+const BUILTIN_HOSTING_PROFILE_IDS = new Set(["qwen-sharp-v21.3"]);
 const DEFAULT_LOG_LINES = 200;
 /**
  * The management API's own version, additive to the capability flags.
@@ -24,6 +37,139 @@ export const HOST_API_VERSION = 3;
  * fine. Comments are ignored by every SSE parser, so no reader sees them.
  */
 const SSE_KEEPALIVE_MS = 20000;
+/**
+ * Apply the hosting-profile half of a profile patch, mutating both `profile`
+ * (this model's selection) and `store` (the shared library and family default).
+ *
+ * Separate from `sanitizeProfilePatch` because these keys are not profile
+ * fields: three of the five write to the store rather than the profile.
+ * Exported for testing - the ordering and the cross-profile cleanup are the
+ * parts worth pinning down, and they are unreachable through the HTTP surface
+ * without standing up a service.
+ *
+ * Throws on any invalid input; the caller turns that into a 400.
+ */
+export function applyHostingProfilePatch(store, model, profile, patch, onDelete = undefined) {
+    const family = hostingFamily(model.family);
+    if ("hostingProfileId" in patch) {
+        const selected = patch.hostingProfileId;
+        if (selected !== null && typeof selected !== "string") {
+            throw new Error("hostingProfileId must be a string or null");
+        }
+        if (selected && !store.hostingProfiles[selected]) {
+            throw new Error("selected hosting profile does not exist");
+        }
+        profile.hostingProfileId = selected || null;
+        profile.hostingProfileMode = selected ? "custom" : "off";
+    }
+    // After the id, so a client can send both and have the explicit mode win.
+    if ("hostingProfileMode" in patch) {
+        const mode = patch.hostingProfileMode;
+        if (mode !== "inherit" && mode !== "off" && mode !== "custom") {
+            throw new Error("hostingProfileMode must be inherit, off, or custom");
+        }
+        if (mode === "custom" && !profile.hostingProfileId) {
+            throw new Error("select a custom profile before using custom mode");
+        }
+        profile.hostingProfileMode = mode;
+    }
+    if ("familyHostingProfileId" in patch) {
+        const selected = patch.familyHostingProfileId;
+        if (selected !== null && typeof selected !== "string") {
+            throw new Error("familyHostingProfileId must be a string or null");
+        }
+        if (selected && !store.hostingProfiles[selected]) {
+            throw new Error("selected family hosting profile does not exist");
+        }
+        if (selected && store.hostingProfiles[selected].family !== family) {
+            throw new Error("selected family hosting profile must match the selected model");
+        }
+        // Null is a real instruction: it is the only way off a family default.
+        store.familyHostingProfileIds[family] = selected || null;
+    }
+    if ("hostingProfile" in patch) {
+        const candidate = HostingProfileSchema.safeParse(patch.hostingProfile);
+        if (!candidate.success)
+            throw new Error("hosting profile is invalid");
+        const item = candidate.data;
+        // Name the field that is wrong. One shared "name or text is too long" for an
+        // empty name, a missing template and an oversized addendum told a remote or
+        // CLI caller nothing about what to change.
+        if (item.name.trim().length === 0)
+            throw new Error("hosting profile needs a name");
+        if (item.name.length > MAX_HOSTING_PROFILE_NAME) {
+            throw new Error(`hosting profile name must be ${MAX_HOSTING_PROFILE_NAME} characters or fewer`);
+        }
+        if (!item.template?.trim())
+            throw new Error("hosting profile needs a Jinja chat template");
+        if (item.template.length > MAX_HOSTING_PROFILE_TEXT) {
+            throw new Error("hosting profile chat template is too long");
+        }
+        if ((item.systemPromptAddendum?.length ?? 0) > MAX_HOSTING_PROFILE_TEXT) {
+            throw new Error("hosting profile system prompt is too long");
+        }
+        if (item.family !== family) {
+            throw new Error("hosting profile family must match the selected model");
+        }
+        const isNew = item.id.length === 0;
+        const id = isNew ? `hosting_${randomUUID()}` : item.id;
+        if (!isNew && BUILTIN_HOSTING_PROFILE_IDS.has(id)) {
+            throw new Error("built-in hosting profiles cannot be edited");
+        }
+        if (!isNew && (id.length > MAX_HOSTING_PROFILE_ID || !HOSTING_PROFILE_ID.test(id))) {
+            throw new Error(`hosting profile id must use only letters, numbers, underscores, or hyphens and be ${MAX_HOSTING_PROFILE_ID} characters or fewer`);
+        }
+        if (!isNew && !store.hostingProfiles[id]) {
+            throw new Error("hosting profile does not exist; create profiles without an id");
+        }
+        if (isNew && Object.keys(store.hostingProfiles).length >= MAX_HOSTING_PROFILES) {
+            throw new Error(`hosting profile limit of ${MAX_HOSTING_PROFILES} reached`);
+        }
+        store.hostingProfiles[id] = { ...item, id, name: item.name.trim() };
+        // Only a freshly created profile is auto-selected. Editing the text of an
+        // existing one must not silently convert a model that was on System default
+        // into a custom override of it.
+        if (isNew) {
+            profile.hostingProfileId = id;
+            profile.hostingProfileMode = "custom";
+        }
+    }
+    if (typeof patch.deleteHostingProfileId === "string") {
+        const id = patch.deleteHostingProfileId;
+        if (BUILTIN_HOSTING_PROFILE_IDS.has(id)) {
+            throw new Error("built-in hosting profiles cannot be deleted");
+        }
+        delete store.hostingProfiles[id];
+        onDelete?.(id);
+        if (profile.hostingProfileId === id) {
+            profile.hostingProfileId = null;
+            profile.hostingProfileMode = "off";
+        }
+        // Clear the mode alongside the id on every other model. Leaving `custom`
+        // behind with no id left those models unsavable: the mode guard above
+        // rejects the next write each of them makes.
+        for (const saved of Object.values(store.profiles)) {
+            if (saved.hostingProfileId !== id)
+                continue;
+            saved.hostingProfileId = null;
+            saved.hostingProfileMode = "off";
+        }
+        for (const [key, selected] of Object.entries(store.familyHostingProfileIds)) {
+            if (selected === id)
+                store.familyHostingProfileIds[key] = null;
+        }
+    }
+    // One invariant, enforced after every branch: an id belongs only to `custom`.
+    // It is what lets `forModel`'s legacy migration read a stored id as an
+    // unambiguous "this profile predates hostingProfileMode".
+    if (profile.hostingProfileMode !== "custom")
+        profile.hostingProfileId = null;
+}
+/** The hosting profiles a model may choose from: its family's bucket, nothing else. */
+function hostingProfilesFor(store, model) {
+    const family = hostingFamily(model.family);
+    return Object.values(store.hostingProfiles).filter((candidate) => candidate.family === family);
+}
 function stateOf(supervisor, model) {
     if (!supervisor.model || supervisor.model.id !== model.id)
         return "not-loaded";
@@ -41,9 +187,9 @@ function stateOf(supervisor, model) {
  * client would otherwise have to correlate three unrelated lists by display name.
  */
 export function buildInventoryRow(params) {
-    const { model, store, defaults, gpu, ranking, supervisor } = params;
+    const { model, store, defaults, gpu, ranking, supervisor, runtimeBuild: activeRuntimeBuild = null, } = params;
     const profile = forModel(store, model, defaults);
-    const calibration = getCalibration(store, model, profile);
+    const calibration = profile.calibrationRequired ? null : getCalibration(store, model, profile);
     const budgetOptions = gpu
         ? { model, profile, calibration, totalVramBytes: gpu.totalBytes }
         : null;
@@ -51,6 +197,7 @@ export function buildInventoryRow(params) {
     return {
         id: model.id,
         displayName: model.displayName,
+        family: model.family ?? null,
         publisher: model.publisher ?? null,
         quant: model.quant,
         sizeBytes: model.sizeBytes,
@@ -77,7 +224,19 @@ export function buildInventoryRow(params) {
         score: ranked,
         state: stateOf(supervisor, model),
         warnings: profileWarnings(profile, model, store),
-        components: model.components ?? null,
+        components: model.components?.map((component) => {
+            if (component.minRuntimeBuild === undefined ||
+                (activeRuntimeBuild !== null && activeRuntimeBuild >= component.minRuntimeBuild)) {
+                return component;
+            }
+            const active = activeRuntimeBuild === null ? "unknown" : `b${activeRuntimeBuild}`;
+            return {
+                ...component,
+                available: false,
+                unavailableReason: `Requires llama.cpp build b${component.minRuntimeBuild} or newer ` +
+                    `(active build: ${active})`,
+            };
+        }) ?? null,
     };
 }
 /**
@@ -126,7 +285,7 @@ function profileFromQuery(base, params, model) {
     }
     if (Object.keys(patch).length === 0)
         return base;
-    return sanitizeProfilePatch(base, patch, model).profile;
+    return sanitizeProfilePatch(base, patch, model, runtimeBuild(null)).profile;
 }
 /**
  * Build the `/__host/*` management handler.
@@ -162,9 +321,15 @@ export function createHostApi(deps) {
         const [gpu, store] = [await deps.queryGpuInfo(), deps.getProfilesStore()];
         const defaults = deps.getProfileDefaults();
         const ranking = deps.getRanking();
-        return deps
-            .getCatalog()
-            .map((model) => buildInventoryRow({ model, store, defaults, gpu, ranking, supervisor: deps.supervisor }));
+        return deps.getCatalog().map((model) => buildInventoryRow({
+            model,
+            store,
+            defaults,
+            gpu,
+            ranking,
+            supervisor: deps.supervisor,
+            runtimeBuild: runtimeBuild(deps.supervisor.runtime),
+        }));
     };
     const handleModelsList = (res) => {
         void (async () => {
@@ -187,9 +352,12 @@ export function createHostApi(deps) {
         const profile = forModel(store, model, deps.getProfileDefaults());
         sendJson(res, {
             profile,
-            fields: profileFieldDescriptors(model),
+            fields: profileFieldDescriptors(model, profile),
             warnings: profileWarnings(profile, model, store),
             calibration: calibrationInfo(store, model, profile),
+            requiresRestart: store.pendingReloadModelIds[model.id] === true,
+            hostingProfiles: hostingProfilesFor(store, model),
+            familyHostingProfileId: familyHostingProfileId(store, model.family),
         });
     };
     const handleProfileSet = (req, res, model) => {
@@ -202,24 +370,38 @@ export function createHostApi(deps) {
                 try {
                     const store = deps.getProfilesStore();
                     const current = forModel(store, model, deps.getProfileDefaults());
-                    const { profile, adjustments } = sanitizeProfilePatch(current, result.body, model);
-                    deps.saveProfiles(put(store, model, profile));
+                    const activeRuntimeBuild = runtimeBuild(deps.supervisor.runtime);
+                    const { profile, adjustments } = sanitizeProfilePatch(current, result.body, model, activeRuntimeBuild);
+                    applyHostingProfilePatch(store, model, profile, result.body, (id) => removeHostingProfileMaterialization(deps.supervisor.paths, id));
+                    put(store, model, profile);
+                    // A setting is only unapplied when it was changed on the currently
+                    // resident model. Edits to an unloaded model take effect naturally
+                    // when it is next loaded and do not earn a misleading reload badge.
+                    const requiresRestart = deps.supervisor.model?.id === model.id;
+                    if (requiresRestart)
+                        store.pendingReloadModelIds[model.id] = true;
+                    deps.saveProfiles(store);
                     // Return the recomputed budget so an edit costs one round trip rather
                     // than a write followed by a read the UI has to sequence.
                     const gpu = await deps.queryGpuInfo();
-                    const calibration = getCalibration(store, model, profile);
+                    const calibration = profile.calibrationRequired
+                        ? null
+                        : getCalibration(store, model, profile);
                     const options = gpu
                         ? { model, profile, calibration, totalVramBytes: gpu.totalBytes }
                         : null;
                     sendJson(res, {
                         profile,
+                        fields: profileFieldDescriptors(model, profile),
                         adjustments,
                         warnings: profileWarnings(profile, model, store),
                         calibration: calibrationInfo(store, model, profile),
                         budget: options ? vram.budget(options) : null,
                         maxContextThatFits: options ? vram.maxContextThatFits(options) : null,
                         /** True when the running model is the one just edited: a restart applies it. */
-                        requiresRestart: deps.supervisor.model?.id === model.id,
+                        requiresRestart,
+                        hostingProfiles: hostingProfilesFor(store, model),
+                        familyHostingProfileId: familyHostingProfileId(store, model.family),
                     });
                 }
                 catch (error) {
@@ -301,7 +483,7 @@ export function createHostApi(deps) {
                 const options = {
                     model,
                     profile,
-                    calibration: getCalibration(store, model, profile),
+                    calibration: profile.calibrationRequired ? null : getCalibration(store, model, profile),
                     totalVramBytes: gpu.totalBytes,
                 };
                 sendJson(res, {
@@ -824,6 +1006,7 @@ export function createHostApi(deps) {
                         gpu,
                         ranking: deps.getRanking(),
                         supervisor: deps.supervisor,
+                        runtimeBuild: runtimeBuild(deps.supervisor.runtime),
                     }));
                 }
                 catch (error) {

package/dist/service/router.d.ts

@@ -73,6 +73,7 @@ export interface ModelEntry {
     id: string;
     /** Brain's editable human-facing name; `id` remains the stable model key. */
     name: string;
+    family?: string;
     object: "model";
     created: number;
     owned_by: string;
@@ -104,6 +105,23 @@ export declare function describeModel(model: Model | null, options?: DescribeOpt
  * model when no catalog provider is wired in.
  */
 export declare function buildModelList(supervisor: Supervisor, getCatalog: GetCatalog): ModelEntry[];
+/** Which shape a completion path uses to carry its system turn. */
+export type CompletionShape = "anthropic" | "openai";
+export declare function completionShape(url: string | null | undefined): CompletionShape;
+/**
+ * Append the active hosting profile's system-prompt addendum to a buffered
+ * completion body.
+ *
+ * Appending rather than prepending or replacing is the whole point: the agent's
+ * own system prompt still leads, and the profile's instructions are read last.
+ * A body this cannot understand is forwarded untouched - a malformed or
+ * unfamiliar request must still reach llama-server and get llama-server's own
+ * error, not a 400 invented here.
+ *
+ * Cost is one extra parse/serialize per request, paid only by models whose
+ * profile actually sets an addendum.
+ */
+export declare function injectSystemAddendum(body: Buffer, addendum: string | null, shape: CompletionShape): Buffer;
 export type ModelGateResult = {
     ok: true;
     model: Model;

package/dist/service/router.js

@@ -149,6 +149,7 @@ export function describeModel(model, options = {}) {
         // OpenAI-compatible clients send `id`; Otto uses `name` for presentation.
         id: model.id,
         name: model.displayName,
+        ...(model.family ? { family: model.family } : {}),
         object: "model",
         created: Math.floor((createdAt ? createdAt.getTime() : Date.now()) / 1000),
         owned_by: model.publisher || "local",
@@ -159,7 +160,13 @@ export function describeModel(model, options = {}) {
         compatibility_type: "gguf",
         quantization: model.quant || null,
         state,
-        max_context_length: md.contextLength ?? null,
+        // The GGUF header remains the native limit, but an actively loaded YaRN
+        // profile intentionally extends the server's usable maximum. Publish that
+        // effective ceiling so OpenAI-compatible clients do not reject a context
+        // that this very llama-server instance has been configured to accept.
+        max_context_length: typeof md.contextLength === "number"
+            ? md.contextLength * (profile?.contextMultiplier ?? 1)
+            : null,
         // GGUF template detection is deliberately conservative. A false result
         // means "not detected", not proof that a catalog-marked reasoner is not
         // one, so preserve the catalog's positive capability metadata.
@@ -290,13 +297,17 @@ function proxyBuffered({ agent, supervisor, telemetry, logger, req, res, body, r
                 resolve();
             }
         };
+        // Injected here, not at queue time: the scheduler may switch models between
+        // buffering and dispatch, and the addendum belongs to whichever model ends
+        // up resident, which is the one `supervisor.profile` now describes.
+        const outbound = injectSystemAddendum(body, supervisor.profile?.chatSystemAddendum ?? null, completionShape(req.url));
         const headers = {};
         for (const [name, value] of Object.entries(req.headers)) {
             if (!HOP_BY_HOP.has(name.toLowerCase()))
                 headers[name] = value;
         }
         headers.host = `${supervisor.host}:${supervisor.internalPort}`;
-        headers["content-length"] = Buffer.byteLength(body);
+        headers["content-length"] = Buffer.byteLength(outbound);
         const started = Date.now();
         const upstream = http.request({
             host: supervisor.host,
@@ -406,9 +417,78 @@ function proxyBuffered({ agent, supervisor, telemetry, logger, req, res, body, r
         // being dispatched to llama-server and is waiting for prompt processing or
         // its first output delta.
         reasoning?.begin(streamId);
-        upstream.end(body);
+        upstream.end(outbound);
     });
 }
+export function completionShape(url) {
+    return /\/v1\/messages/.test(url ?? "") ? "anthropic" : "openai";
+}
+/** One text block, as both API shapes spell it inside a structured content array. */
+function textBlock(text) {
+    return { type: "text", text };
+}
+/**
+ * Append the active hosting profile's system-prompt addendum to a buffered
+ * completion body.
+ *
+ * Appending rather than prepending or replacing is the whole point: the agent's
+ * own system prompt still leads, and the profile's instructions are read last.
+ * A body this cannot understand is forwarded untouched - a malformed or
+ * unfamiliar request must still reach llama-server and get llama-server's own
+ * error, not a 400 invented here.
+ *
+ * Cost is one extra parse/serialize per request, paid only by models whose
+ * profile actually sets an addendum.
+ */
+export function injectSystemAddendum(body, addendum, shape) {
+    if (!addendum)
+        return body;
+    let parsed;
+    try {
+        parsed = JSON.parse(body.toString("utf8"));
+    }
+    catch {
+        return body;
+    }
+    if (!isRecord(parsed))
+        return body;
+    if (shape === "anthropic") {
+        // Anthropic carries the system turn beside `messages`, never inside it.
+        const system = parsed.system;
+        if (system === undefined || system === null || system === "")
+            parsed.system = addendum;
+        else if (typeof system === "string")
+            parsed.system = `${system}\n\n${addendum}`;
+        else if (Array.isArray(system))
+            parsed.system = [...system, textBlock(addendum)];
+        else
+            return body;
+        return Buffer.from(JSON.stringify(parsed), "utf8");
+    }
+    const messages = parsed.messages;
+    if (!Array.isArray(messages))
+        return body;
+    // `developer` is the newer OpenAI spelling of the same turn; either one is
+    // the message this addendum belongs on.
+    const index = messages.findIndex((message) => isRecord(message) && (message.role === "system" || message.role === "developer"));
+    if (index === -1) {
+        parsed.messages = [{ role: "system", content: addendum }, ...messages];
+        return Buffer.from(JSON.stringify(parsed), "utf8");
+    }
+    const existing = messages[index];
+    const content = existing.content;
+    let merged;
+    if (content === undefined || content === null || content === "")
+        merged = addendum;
+    else if (typeof content === "string")
+        merged = `${content}\n\n${addendum}`;
+    else if (Array.isArray(content))
+        merged = [...content, textBlock(addendum)];
+    else
+        return body;
+    parsed.messages = messages.map((message, at) => at === index ? { ...existing, content: merged } : message);
+    return Buffer.from(JSON.stringify(parsed), "utf8");
+}
 /**
  * Pure model-admission decision, factored out of the router so it is unit
  * testable. `pinned` is the single model a locked host serves; `resolved` is the
@@ -497,7 +577,12 @@ function scheduleCompletion({ req, res, agent, supervisor, telemetry, logger, sc
 // window - the cheap time-based trigger.
 const RANKING_TTL_MS = 60000;
 export function createRouter({ supervisor, telemetry, logger, getCatalog = null, loadModel = null, loadRanking = () => rankModels(), queryGpuInfo = queryGpu, version = null, getConfig = null, getEvals = null, getLockModel = () => false, getDefaultModel = () => null, applyConfigPatch = null, getAllowConfigWrite = () => false, hostApi = null, getResources = null, statusEvents = null, }) {
-    const agent = new http.Agent({ keepAlive: true, maxSockets: 32 });
+    // llama-server may close an idle response socket while this scheduler holds
+    // the next request in queue. A reused keep-alive socket then fails as
+    // ECONNRESET ("socket hang up") before the queued request reaches inference.
+    // Inference time dwarfs localhost connection setup, so isolate each request
+    // instead of letting a second client inherit a stale upstream connection.
+    const agent = new http.Agent({ keepAlive: false, maxSockets: 32 });
     const scheduler = loadModel
         ? new Scheduler({
             supervisor,