@otto-code/brain 0.8.10 → 0.8.12

package/dist/config/profile-edit.js

@@ -17,8 +17,33 @@
  *  - Changing a cache type invalidates a measured calibration, because KV
  *    bytes/token is a function of the cache types (see `vram.ts`).
  */
-import { CACHE_TYPE_BYTES } from "../vram.js";
+import os from "node:os";
+import { CACHE_TYPE_BYTES, formatGiB, promptCacheSize } from "../vram.js";
 import { getCalibration, hasStaleCalibration } from "./profiles.js";
+/**
+ * Sampler ranges. llama.cpp itself bounds almost none of these - it will take a
+ * temperature of 50 - so the bounds are the useful range rather than the legal
+ * one, which is what a stepper wants. The write path clamps to exactly these,
+ * so a value the editor cannot reach is a value the brain will not store.
+ */
+export const SAMPLING_RANGES = {
+    temperature: { min: 0, max: 2, step: 0.05, precision: 2 },
+    topP: { min: 0, max: 1, step: 0.05, precision: 2 },
+    topK: { min: 0, max: 200, step: 1 },
+    minP: { min: 0, max: 1, step: 0.01, precision: 2 },
+    presencePenalty: { min: -2, max: 2, step: 0.05, precision: 2 },
+    repeatPenalty: { min: 0.5, max: 2, step: 0.01, precision: 2 },
+};
+/** Tri-state preservation, in the order a cycle should offer it. */
+export const PRESERVE_REASONING_CYCLE = ["default", "on", "off"];
+/** The stored value a cycle option maps to. */
+export function preserveReasoningFromOption(option) {
+    return option === "on" ? true : option === "off" ? false : null;
+}
+/** The cycle option a stored value maps to. */
+export function preserveReasoningOption(value) {
+    return value === true ? "on" : value === false ? "off" : "default";
+}
 /**
  * The cache types the editor cycles through. `CACHE_TYPE_BYTES` knows more
  * (f32, bf16, q5_0, q4_1) and a write naming one of those is accepted; these
@@ -44,6 +69,19 @@ export function formatReasoningBudget(budget) {
     return budget === UNRESTRICTED_REASONING_BUDGET ? "unrestricted" : String(budget);
 }
 export const MAX_PARALLEL_SLOTS = 16;
+/**
+ * Ceiling on `cachedChats`. Generous on purpose: entries are small for a small
+ * model and huge for a large one, so the honest guard is the RAM figure the
+ * warning shows, not an arbitrary count.
+ */
+export const MAX_CACHED_CHATS = 64;
+/**
+ * What llama.cpp allows by default when no `--cache-ram` is emitted, in bytes.
+ * `cachedChats` of 0 emits no flag, so the estimate for the Default option is
+ * this fixed figure against the installed RAM - not the model's measured KV
+ * cost, which is irrelevant here because the size does not depend on the model.
+ */
+export const ENGINE_DEFAULT_CACHE_RAM_BYTES = 8192 * 1024 * 1024;
 export const MAX_GPU_LAYERS = 999;
 export const MIN_CONTEXT_SIZE = 1024;
 export const CONTEXT_STEP = 8192;
@@ -62,13 +100,25 @@ export function profileFieldDescriptors(model, profile) {
     const hasProjector = model?.components
         ? Boolean(projector?.available)
         : Boolean(model?.mmprojPath);
+    // The template exposes a thinking channel at all - the gate for both reasoning
+    // controls. Preservation used to be gated on `reasoningPreservation
+    // .templateArgument` instead, which is detected by grepping the chat template
+    // for a literal `preserve_thinking`/`preserve_reasoning` kwarg. Almost no
+    // template spells it that way: llama.cpp decides the same question by probing
+    // the rendered template, so it happily logs "chat template supports preserving
+    // reasoning, consider enabling it via --reasoning-preserve" for a model whose
+    // toggle Otto was hiding. The flag exists on every build we ship and defaults
+    // to the template's own behavior when absent, so offering it wherever there is
+    // reasoning to preserve costs nothing and stops hiding a working setting.
+    const hasReasoning = Boolean(model?.metadata?.reasoning || model?.reasoningPreservation);
     const fields = [
         {
             key: "contextMultiplier",
             label: "Context multiplier",
             kind: "cycle",
+            description: "Stretches the context window past the model's native size with RoPE scaling.",
             options: CONTEXT_MULTIPLIERS,
-            optionLabels: ["Off", "2× (YaRN)", "4× (YaRN)"],
+            optionLabels: ["Off", "2×", "4×"],
             available: Boolean(model?.metadata?.contextLength),
             ...(model?.metadata?.contextLength ? {} : { unavailableReason: "native context unknown" }),
         },
@@ -76,6 +126,7 @@ export function profileFieldDescriptors(model, profile) {
             key: "contextSize",
             label: "Context",
             kind: "number",
+            description: "How many tokens of conversation the model can hold at once.",
             step: CONTEXT_STEP,
             min: MIN_CONTEXT_SIZE,
             max: contextLimit(model, profile?.contextMultiplier ?? 1),
@@ -85,6 +136,7 @@ export function profileFieldDescriptors(model, profile) {
             key: "cacheTypeK",
             label: "KV cache K",
             kind: "cycle",
+            description: "Precision of the cached keys; lower saves VRAM and costs a little accuracy.",
             options: CACHE_TYPE_CYCLE,
             available: true,
         },
@@ -92,54 +144,147 @@ export function profileFieldDescriptors(model, profile) {
             key: "cacheTypeV",
             label: "KV cache V",
             kind: "cycle",
+            description: "Precision of the cached values; lower saves VRAM and costs a little accuracy.",
             options: CACHE_TYPE_CYCLE,
             available: true,
         },
-        { key: "flashAttention", label: "Flash attention", kind: "toggle", available: true },
+        // The five KV/context settings that size and split the cache sit together,
+        // in the order the math reads them: the window (multiplier, size), its
+        // quantisation (K, V), how it is split (slots), and the RAM fallback
+        // (cached chats). Flash attention is related but leaves the KV byte total
+        // untouched, so it stays with the remaining options below.
+        {
+            key: "parallelSlots",
+            label: "Parallel slots",
+            kind: "number",
+            description: "How many chats the model serves at once, each taking a share of the context.",
+            step: 1,
+            min: 1,
+            max: MAX_PARALLEL_SLOTS,
+            available: true,
+        },
+        {
+            key: "cachedChats",
+            label: "Cached KVs",
+            kind: "number",
+            description: "How many idle chats keep their state in system RAM so returning to one skips a re-read.",
+            step: 1,
+            min: 0,
+            max: MAX_CACHED_CHATS,
+            available: true,
+        },
+        {
+            key: "flashAttention",
+            label: "Flash attention",
+            kind: "toggle",
+            description: "A faster attention kernel, and what a quantised value cache requires.",
+            available: true,
+        },
+        // Bundle models expose the projector in the component section below. Keep
+        // the legacy profile field for hand-scanned single-file models, but do not
+        // render two controls that write the same vision setting for bundles. It
+        // belongs with the other options, right after flash attention.
+        ...(model?.components
+            ? []
+            : [
+                {
+                    key: "vision",
+                    label: "Vision",
+                    kind: "toggle",
+                    description: "Loads the vision projector so the model can read images.",
+                    available: hasProjector,
+                    ...(hasProjector
+                        ? {}
+                        : {
+                            unavailableReason: projector
+                                ? "download the vision component first"
+                                : "no projector",
+                        }),
+                },
+            ]),
         {
             key: "reasoningBudget",
             label: "Reasoning budget",
             kind: "cycle",
+            description: "Caps how many tokens the model may think before it is told to answer.",
             options: REASONING_BUDGET_CYCLE,
             optionLabels: ["Thinking Off", "512", "1024", "1536", "3072", "Unrestricted"],
             min: -1,
             available: true,
         },
+        // Extends the reasoning group, immediately after the budget it qualifies.
+        {
+            key: "preserveReasoning",
+            label: "Preserve reasoning",
+            kind: "cycle",
+            description: "Keeps earlier thinking in the history instead of only the latest reply's.",
+            options: [...PRESERVE_REASONING_CYCLE],
+            optionLabels: ["Template default", "On", "Off"],
+            optionValues: PRESERVE_REASONING_CYCLE.map(preserveReasoningFromOption),
+            available: hasReasoning,
+            ...(hasReasoning ? {} : { unavailableReason: "no thinking channel in this template" }),
+        },
         {
             key: "gpuLayers",
             label: "GPU layers",
             kind: "number",
+            description: "How many layers run on the GPU; any remainder runs on the CPU.",
             step: 1,
             min: 0,
             max: MAX_GPU_LAYERS,
             available: true,
         },
+        // Sampling. These change what the model writes rather than what it costs, so
+        // they sit after the hosting fields and contribute nothing to the budget.
         {
-            key: "parallelSlots",
-            label: "Parallel slots",
+            key: "temperature",
+            label: "Temperature",
             kind: "number",
-            step: 1,
-            min: 1,
-            max: MAX_PARALLEL_SLOTS,
+            description: "How adventurous each next-token choice is; lower is more predictable.",
+            ...SAMPLING_RANGES.temperature,
+            available: true,
+        },
+        {
+            key: "topP",
+            label: "Top P",
+            kind: "number",
+            description: "Considers only the likeliest tokens whose probabilities add up to this share.",
+            ...SAMPLING_RANGES.topP,
+            available: true,
+        },
+        {
+            key: "topK",
+            label: "Top K",
+            kind: "number",
+            description: "Considers only this many of the likeliest tokens; 0 turns the limit off.",
+            ...SAMPLING_RANGES.topK,
+            available: true,
+        },
+        {
+            key: "minP",
+            label: "Min P",
+            kind: "number",
+            description: "Drops tokens less likely than this fraction of the best one; 0 turns it off.",
+            ...SAMPLING_RANGES.minP,
+            available: true,
+        },
+        {
+            key: "presencePenalty",
+            label: "Presence penalty",
+            kind: "number",
+            description: "Pushes toward new subjects by penalising tokens already used; 0 turns it off.",
+            ...SAMPLING_RANGES.presencePenalty,
+            available: true,
+        },
+        {
+            key: "repeatPenalty",
+            label: "Repetition penalty",
+            kind: "number",
+            description: "Discourages repeating recent tokens; 1 turns it off.",
+            ...SAMPLING_RANGES.repeatPenalty,
             available: true,
         },
     ];
-    // Bundle models expose the projector in the component section below. Keep
-    // the legacy profile field for hand-scanned single-file models, but do not
-    // render two controls that write the same vision setting for bundles.
-    if (!model?.components) {
-        fields.splice(4, 0, {
-            key: "vision",
-            label: "Vision",
-            kind: "toggle",
-            available: hasProjector,
-            ...(hasProjector
-                ? {}
-                : {
-                    unavailableReason: projector ? "download the vision component first" : "no projector",
-                }),
-        });
-    }
     return fields;
 }
 /** Whether a KV cache type is quantised (anything that is not a float type). */
@@ -166,10 +311,47 @@ export function profileWarnings(profile, model, store) {
         });
     }
     if (profile.parallelSlots > 1) {
+        const perSlot = profile.contextSize > 0 ? Math.floor(profile.contextSize / profile.parallelSlots) : null;
         warnings.push({
             field: "parallelSlots",
             severity: "info",
-            message: `${profile.parallelSlots} concurrent requests, sharing one KV pool.`,
+            message: perSlot !== null
+                ? `${profile.parallelSlots} concurrent chats: ~${Math.round(perSlot / 1000)}K context each, all resident.`
+                : `${profile.parallelSlots} concurrent requests, sharing one KV pool.`,
+            blocksStart: false,
+        });
+    }
+    // The estimate is shown whenever this field is available, including at the
+    // Default of 0. That is the one value the user could not otherwise price:
+    // 0 emits no flag, and llama.cpp then parks up to its own 8 GiB cache-ram
+    // default in system RAM, which is not the same as caching nothing.
+    //
+    // A count above 0 can only be priced from a measurement: without one the
+    // theoretical KV cost runs to multiples of the real one, and naming a figure
+    // derived from it would invite reserving several times the RAM actually
+    // needed. So the unmeasured count falls back to the calibration request.
+    const calibration = model && store ? getCalibration(store, model, profile) : null;
+    const cache = model ? promptCacheSize(model, profile, calibration) : null;
+    const hasCount = (profile.cachedChats ?? 0) > 0;
+    if (hasCount && (!cache || cache.source !== "measured")) {
+        warnings.push({
+            field: "cachedChats",
+            severity: "warn",
+            message: "Calibrate this model first: without a measured KV cost the RAM this needs cannot be sized, so llama.cpp's own cache limit stays in effect.",
+            blocksStart: false,
+        });
+    }
+    else {
+        const totalBytes = hasCount ? cache.totalBytes : ENGINE_DEFAULT_CACHE_RAM_BYTES;
+        const installed = formatGiB(os.totalmem());
+        warnings.push({
+            field: "cachedChats",
+            // Yellow at half the machine's RAM, red once the parked state would use
+            // at least all of it.
+            severity: totalBytes >= os.totalmem() ? "error" : totalBytes >= os.totalmem() / 2 ? "warn" : "info",
+            message: hasCount
+                ? `${formatGiB(cache.perChatBytes)} each becomes ${formatGiB(totalBytes)} of ${installed}.`
+                : `llama.cpp's own limit applies: about ${formatGiB(totalBytes)} of ${installed}.`,
             blocksStart: false,
         });
     }
@@ -230,15 +412,37 @@ export function calibrationInfo(store, model, profile) {
 function clamp(value, min, max) {
     return Math.max(min, Math.min(max, value));
 }
-/** The settings whose value changes what a calibration would measure. */
+/**
+ * The settings whose value changes what a calibration would measure - the KV
+ * cache system (cache types, attention configuration, loaded components) or the
+ * evaluation (the extended RoPE shape). Only these may set
+ * `calibrationRequired`.
+ *
+ * `contextSize` is deliberately absent. A calibration measures bytes *per
+ * token* - the context size is the independent variable it varies to get the
+ * slope, so the result is by construction the same at any context, and
+ * `calibrationKey` does not record it either. Listing it here invalidated the
+ * measurement on every context edit, including the one "Fit to VRAM" makes
+ * itself: the fit picked a context from the measured figure, the write that
+ * saved it dropped back to the (~4x higher) theoretical estimate, and the
+ * saved context was then far past the budget it had just been sized against.
+ *
+ * `gpuLayers` and `parallelSlots` are absent for the same differential reason:
+ * they only move weights between devices or split the one KV pool across slots.
+ * The calibration is the GPU-delta between two loads at different contexts, so
+ * every fixed term - weights wherever they sit, the CUDA context, compute
+ * buffers - cancels out of the slope, and a load whose KV split to CPU is
+ * rejected as unusable rather than measured low. Neither the bytes/token nor the
+ * evaluation changes, so neither may discard a real measurement.
+ *
+ * `cachedChats` is absent because it spends host RAM, not VRAM, and never
+ * reaches the load a calibration measures.
+ */
 const CALIBRATION_INPUTS = [
     "contextMultiplier",
-    "contextSize",
     "cacheTypeK",
     "cacheTypeV",
     "flashAttention",
-    "gpuLayers",
-    "parallelSlots",
     "vision",
     "enabledComponents",
 ];
@@ -287,6 +491,27 @@ export function sanitizeProfilePatch(current, patch, model, runtimeBuild = null)
             throw new Error(`${key} must be a boolean`);
         next[key] = p[key];
     };
+    /**
+     * A sampler value: clamped to its useful range and rounded to the precision
+     * the editor advertises. Rounding matters as much as clamping - a stepper that
+     * added 0.05 six times sends 0.30000000000000004, and without this the profile
+     * would store that and hand it to llama-server on the command line.
+     */
+    const takeSampling = (key) => {
+        if (!(key in p))
+            return;
+        const raw = p[key];
+        if (typeof raw !== "number" || !Number.isFinite(raw)) {
+            throw new Error(`${key} must be a number`);
+        }
+        const range = SAMPLING_RANGES[key];
+        const precision = "precision" in range ? range.precision : 0;
+        const rounded = Number(raw.toFixed(precision));
+        const clamped = clamp(rounded, range.min, range.max);
+        if (clamped !== rounded)
+            adjustments.push(`${key} clamped to ${clamped}`);
+        next[key] = clamped;
+    };
     const takeCacheType = (key) => {
         if (!(key in p))
             return;
@@ -312,6 +537,7 @@ export function sanitizeProfilePatch(current, patch, model, runtimeBuild = null)
     takeBoolean("flashAttention");
     takeNumber("gpuLayers", 0, MAX_GPU_LAYERS);
     takeNumber("parallelSlots", 1, MAX_PARALLEL_SLOTS);
+    takeNumber("cachedChats", 0, MAX_CACHED_CHATS);
     // -1 (unrestricted) is the floor; anything below it is meaningless to
     // llama-server. There is no upper bound worth inventing: a budget larger than
     // the context is simply never reached.
@@ -326,6 +552,31 @@ export function sanitizeProfilePatch(current, patch, model, runtimeBuild = null)
             adjustments.push(`reasoningBudget clamped to ${value}`);
         next.reasoningBudget = value;
     }
+    // Tri-state, and the third state is the point: null means "leave the template's
+    // own behavior alone", which is not the same as false ("trim the trace"). The
+    // string forms are accepted because the editor renders this as a cycle whose
+    // options are strings on the wire.
+    if ("preserveReasoning" in p) {
+        const raw = p.preserveReasoning;
+        if (raw === null || raw === "default") {
+            next.preserveReasoning = null;
+        }
+        else if (typeof raw === "boolean") {
+            next.preserveReasoning = raw;
+        }
+        else if (raw === "on" || raw === "off") {
+            next.preserveReasoning = preserveReasoningFromOption(raw);
+        }
+        else {
+            throw new Error('preserveReasoning must be a boolean, null, or one of "default"/"on"/"off"');
+        }
+    }
+    takeSampling("temperature");
+    takeSampling("topP");
+    takeSampling("topK");
+    takeSampling("minP");
+    takeSampling("presencePenalty");
+    takeSampling("repeatPenalty");
     // Vision is only real when the model actually has a projector paired with it.
     if ("vision" in p) {
         if (typeof p.vision !== "boolean")

package/dist/config/profiles.js

@@ -32,7 +32,20 @@ export function defaultProfile(model, defaults) {
             : Boolean(model?.mmprojPath),
         reasoningBudget: defaults?.reasoningBudget ?? 1536,
         reasoningBudgetMessage: DEFAULT_REASONING_MESSAGE,
+        // Null, not false: llama-server's own default is "whatever the template
+        // wants", and a profile that has never been touched must not override it.
+        // A template that declares a preservation kwarg still gets its own default
+        // through that path (see buildArgs).
+        preserveReasoning: model?.reasoningPreservation?.default ?? null,
+        // llama.cpp's sampler defaults; see the note on ProfileSchema.
+        temperature: 0.8,
+        topP: 0.95,
+        topK: 40,
+        minP: 0.05,
+        presencePenalty: 0,
+        repeatPenalty: 1,
         parallelSlots: defaults?.parallelSlots ?? 1, // one agent at a time: max context per request
+        cachedChats: 0, // llama.cpp's own --cache-ram default until the user sizes it
         batchSize: null,
         ubatchSize: null,
         extraArgs: [],
@@ -67,6 +80,9 @@ export function forModel(store, model, defaults) {
     return {
         ...base,
         ...stored,
+        // `preserveReasoning` was optional in the store so adding the Qwen default
+        // does not rewrite existing profiles as false merely because they predate it.
+        preserveReasoning: stored.preserveReasoning ?? base.preserveReasoning,
         // COMPAT(hostingProfileMode): added in v0.8.8, remove after 2027-02-12.
         // The first hosting-profile store only had an id, and a non-null id there
         // was an explicit custom choice. Such a profile parses as `inherit` (the