ollanet 0.6.2 → 0.6.3

package/README.md

@@ -192,7 +192,7 @@ optimizeDeps: { exclude: ["ollanet"] },
 | `ollanet pull <machine> <model>` | Ask that host to download / update a library model |
 | `ollanet show <machine> <model>` | Inspect Modelfile / params / capabilities (`[tuned]` when it looks like Finetuna) |
 | `ollanet rm <machine> <model> --yes` | Delete a model from that host’s disk |
-| `ollanet ps [machine]` | Models loaded in VRAM (omit machine = every discovered host) |
+| `ollanet ps [machine]` | Models loaded in VRAM, with CPU/GPU % (omit machine = every discovered host) |
 | `ollanet prompt …` | Send a prompt / continue a chat (`--file` for `.txt` / `.md`) |
 | `ollanet compare …` | Same prompt on 2–5 models; summary + `compares/<id>.md` |
 | `ollanet chats` | List or inspect saved transcripts |
@@ -340,10 +340,12 @@ ollanet bench localhost gemma4:12b --hot --runs 5
 ```
 
 Runs a built-in suite (`quick` default, or `--suite full`) against one or more **completion**
-models: instruction checks once, throughput case repeated (`--runs`, default 3) with
+models: instruction checks once, 256-token throughput repeated (`--runs`, default 3) with
 pinned `num_predict` / `seed` / `temperature`. Reports median tok/s + spread.
-`--hot` discards the first throughput run and leaves models loaded so the counted
-repeats are already warm (Finetuna steady-state). Under `--all`,
+`--hot` discards the first 256-token run and leaves models loaded so the counted
+repeats are already warm (Finetuna steady-state). `--suite full` also runs json/reason
+checks and **one** 1024-token prose generation (`tok/s long`) — closer to a long chat
+reply. Under `--all`,
 skips non-completion models (embeddings); multimodal `completion`+`vision` models stay in
 (early-stop / pass_rate handle text-suite failures). Use `--exclude-vision` to drop them.
 Saves `benchmarks/<id>.json` (or `~/.ollanet/benchmarks/` when installed).
@@ -351,7 +353,7 @@ See [`docs/bench-spec.md`](docs/bench-spec.md) for measurement rules.
 
 `quick` has only two scored checks (`ping`, `math`), so `pass` is a coarse
 liveness/sanity gauge (`0/2`, `1/2`, `2/2`) — not “50% broken.” Use `--suite full`
-when you care more about the quality signal.
+when you care more about the quality signal or want the long tok/s column.
 
 ## Discovery
 

package/dist/bench-suite.d.ts

@@ -3,14 +3,21 @@ export interface BenchCase {
     id: string;
     role: CaseRole;
     prompt: string;
-    /** Cap for this case; throughput uses settings.throughput_num_predict. */
+    /** Cap for this case. Peak throughput uses `--num-predict` when omitted. */
     num_predict?: number;
+    /**
+     * Throughput attempts. Omit to use `--runs` (and `--hot` discard).
+     * Set `1` for a single large shot that does not repeat.
+     */
+    repeats?: number;
     format?: "json";
     check?: (content: string) => {
         ok: boolean;
         detail?: string;
     };
 }
+/** Pinned length for `full`'s prose throughput case (`throughput_long`). */
+export declare const LONG_THROUGHPUT_NUM_PREDICT = 1024;
 export type SuiteName = "quick" | "full";
 export declare function getSuiteCases(suite: SuiteName): BenchCase[];
 export declare function suiteRevision(suite: SuiteName): string;

package/dist/bench-suite.js

@@ -2,6 +2,8 @@
  * Built-in ollanet bench suites, checkers, and revision hashes.
  */
 import { createHash } from "node:crypto";
+/** Pinned length for `full`'s prose throughput case (`throughput_long`). */
+export const LONG_THROUGHPUT_NUM_PREDICT = 1024;
 const MATH_ANSWER = 323;
 const REASON_ANSWER = 42;
 function lastTokenNormalized(content) {
@@ -104,18 +106,23 @@ const FULL_EXTRA = [
     {
         id: "reason",
         role: "check",
-        prompt: "A box has 10 apples. You eat 3, then buy 5 more, then give away 2 times as many as you ate. " +
-            "How many apples are left? Reply with only the final number.",
+        prompt: "Start with 40. Add 7. Subtract 5. Reply with only the final number.",
         num_predict: 48,
-        // 10 - 3 + 5 - 6 = 6... wait: gave away 2*3=6, so 10-3+5-6=6. Spec said 42 as placeholder.
-        // Use a problem that clearly ends at 42:
-        // Actually rewrite for 42: "Start with 40. Add 7. Subtract 5. Reply with only the number." => 42
         check: checkExpectedInt(REASON_ANSWER),
     },
+    {
+        id: "throughput_long",
+        role: "throughput",
+        // Chat-shaped tokens (markdown, lists, a table) at a long pin. Anti-EOS
+        // so done_reason=length; one shot (not --runs) so full stays usable.
+        prompt: "Write a detailed trade-off briefing for a product team choosing a web UI stack. " +
+            "Compare at least two options. Use markdown headings, bullet lists, and at least one table. " +
+            "Keep adding sections (testing, hiring, migration, failure modes, cost) without wrapping up. " +
+            "Do not write a closing recommendation that ends the piece. Do not stop early.",
+        num_predict: LONG_THROUGHPUT_NUM_PREDICT,
+        repeats: 1,
+    },
 ];
-// Fix reason prompt to match REASON_ANSWER
-FULL_EXTRA[FULL_EXTRA.length - 1].prompt =
-    "Start with 40. Add 7. Subtract 5. Reply with only the final number.";
 export function getSuiteCases(suite) {
     if (suite === "full")
         return [...QUICK_CASES, ...FULL_EXTRA];

package/dist/bench.js

@@ -2,7 +2,7 @@
  * ollanet bench — speed + lightweight quality suite against Ollama models.
  */
 import { defaultModelForHost, loadConfig, machineSettingsForHost, mergeSettings, settingsFromEnv, } from "./config.js";
-import { comparabilityKey, getSuiteCases, median, suiteRevision, } from "./bench-suite.js";
+import { comparabilityKey, getSuiteCases, LONG_THROUGHPUT_NUM_PREDICT, median, suiteRevision, } from "./bench-suite.js";
 import { benchmarksDir, newBenchId, saveBenchmark } from "./bench-store.js";
 import { readFile } from "node:fs/promises";
 import { envInt, ollamaBaseUrl, shortName } from "./hosts.js";
@@ -23,19 +23,20 @@ Examples:
   ollanet bench studio gemma3:12b --hot --runs 5
   ollanet help bench
 
-Fixed suite (not compare). Checks run once; throughput repeats --runs times.
+Fixed suite (not compare). Checks run once; 256-token throughput repeats --runs times.
 Median tok/s excludes early-stops. Unload is between models, not between repeats.
+full adds json/reason checks plus one 1024-token prose generation (tok/s long).
 
 Options:
   --all                 Every completion-capable model from /api/tags
   --exclude-vision      With --all, skip models that advertise vision
   --suite quick|full    Prompt suite (default quick)
-  --runs <n>            Counted throughput repeats (default 3)
-  --hot                 Discard first throughput run; keep models loaded
+  --runs <n>            Counted 256-token throughput repeats (default 3)
+  --hot                 Discard first 256-token throughput run; keep models loaded
   --warmup              Short discarded call before timed cases (default on)
   --no-warmup           Skip the short warmup (--hot implies this)
   --cold-load           Measure cold load via unload + /api/ps
-  --num-predict <n>     Pin throughput length (default 256)
+  --num-predict <n>     Pin 256-token throughput length (default 256; not the long case)
   --num-ctx <n>         Context window override
   --keep-alive <value>  Keep model loaded (e.g. 5m, 0, -1)
   --think / --no-think  Thinking tokens (default off)
@@ -65,6 +66,26 @@ function resolveModelName(tags, query) {
     const prefix = tags.find((t) => t.name.toLowerCase().startsWith(`${q}:`) || q.startsWith(`${t.name.toLowerCase()}:`));
     return prefix?.name;
 }
+function pinnedNumPredict(caseDef, throughputNumPredict) {
+    if (caseDef.role === "throughput") {
+        return caseDef.num_predict ?? throughputNumPredict;
+    }
+    return caseDef.num_predict ?? 32;
+}
+/** At least the global timeout; longer pins need room to finish at ~8 tok/s. */
+function attemptTimeoutMs(numPredict) {
+    return Math.max(BENCH_TIMEOUT_MS, Math.ceil(numPredict / 8) * 1000);
+}
+function caseReps(caseDef, runs, hot) {
+    if (caseDef.role !== "throughput")
+        return 1;
+    if (caseDef.repeats != null)
+        return caseDef.repeats;
+    return runs + (hot ? 1 : 0);
+}
+function discardFirstThroughput(caseDef, hot) {
+    return hot && caseDef.role === "throughput" && caseDef.repeats == null;
+}
 function tokPerSec(chunk) {
     if (chunk.eval_count == null || chunk.eval_duration == null || chunk.eval_duration <= 0) {
         return undefined;
@@ -285,9 +306,7 @@ async function runColdLoad(baseUrl, model, keepAlive) {
     }
 }
 async function runCaseAttempt(opts) {
-    const numPredict = opts.caseDef.role === "throughput"
-        ? opts.throughputNumPredict
-        : (opts.caseDef.num_predict ?? 32);
+    const numPredict = pinnedNumPredict(opts.caseDef, opts.throughputNumPredict);
     const settings = mergeSettings(opts.settings, {
         temperature: 0,
         seed: 0,
@@ -303,7 +322,7 @@ async function runCaseAttempt(opts) {
             messages: [{ role: "user", content: opts.caseDef.prompt }],
             stream: false,
             writeStdout: false,
-            timeoutMs: BENCH_TIMEOUT_MS,
+            timeoutMs: attemptTimeoutMs(numPredict),
             settings,
         });
         const wallMs = Date.now() - wall0;
@@ -373,12 +392,23 @@ async function runCaseAttempt(opts) {
 function countedAttempts(attempts) {
     return attempts.filter((a) => !a.discarded);
 }
-function summarizeModel(cases, cold, selfJudge) {
-    const throughput = cases.find((c) => c.id === "throughput");
-    const counted = countedAttempts(throughput?.attempts ?? []);
+function speedFrom(cases, id) {
+    const counted = countedAttempts(cases.find((c) => c.id === id)?.attempts ?? []);
     const fullLen = counted.filter((a) => !a.error && !a.early_stop && a.tok_s != null);
     const toks = fullLen.map((a) => a.tok_s);
-    const earlyStopCount = counted.filter((a) => a.early_stop || (a.done_reason && a.done_reason !== "length")).length;
+    return {
+        median: median(toks) ?? null,
+        min: toks.length ? Math.min(...toks) : null,
+        max: toks.length ? Math.max(...toks) : null,
+    };
+}
+function summarizeModel(cases, cold, selfJudge) {
+    const peak = speedFrom(cases, "throughput");
+    const long = speedFrom(cases, "throughput_long");
+    const throughputAttempts = cases
+        .filter((c) => c.role === "throughput")
+        .flatMap((c) => countedAttempts(c.attempts));
+    const earlyStopCount = throughputAttempts.filter((a) => a.early_stop || (a.done_reason && a.done_reason !== "length")).length;
     const checkAttempts = cases
         .filter((c) => c.role === "check")
         .flatMap((c) => c.attempts)
@@ -386,9 +416,12 @@ function summarizeModel(cases, cold, selfJudge) {
     const passed = checkAttempts.filter((a) => a.quality?.pass).length;
     const passRate = checkAttempts.length > 0 ? passed / checkAttempts.length : null;
     return {
-        tok_s_median: median(toks) ?? null,
-        tok_s_min: toks.length ? Math.min(...toks) : null,
-        tok_s_max: toks.length ? Math.max(...toks) : null,
+        tok_s_median: peak.median,
+        tok_s_min: peak.min,
+        tok_s_max: peak.max,
+        tok_s_long_median: long.median,
+        tok_s_long_min: long.min,
+        tok_s_long_max: long.max,
         early_stop_count: earlyStopCount,
         pass_rate: passRate,
         load_ms: cold && cold.error == null && cold.load_ms > 0 ? cold.load_ms : cold?.error ? null : cold?.load_ms ?? null,
@@ -404,17 +437,22 @@ function printTable(machine, suite, runs, revision, compKey, models, skipped, co
         lines.push("Cold-load: on");
     if (hot)
         lines.push("Hot: first throughput run discarded; models stay loaded");
+    if (suite === "full")
+        lines.push("full: tok/s long is one 1024-token prose generation");
     lines.push("");
+    const showLong = suite === "full";
     const header = coldLoad
         ? pad("Model", MODEL_COL) +
             pad("tok/s (med)", 12) +
             pad("spread", 12) +
+            (showLong ? pad("tok/s long", 12) : "") +
             pad("load", 8) +
             pad("pass", 8) +
             "notes"
         : pad("Model", MODEL_COL) +
             pad("tok/s (med)", 12) +
             pad("spread", 12) +
+            (showLong ? pad("tok/s long", 12) : "") +
             pad("pass", 8) +
             "notes";
     lines.push(header);
@@ -448,6 +486,7 @@ function printTable(machine, suite, runs, revision, compKey, models, skipped, co
         const spread = m.summary.tok_s_min != null && m.summary.tok_s_max != null
             ? `${m.summary.tok_s_min.toFixed(1)}–${m.summary.tok_s_max.toFixed(1)}`
             : "—";
+        const tokLong = m.summary.tok_s_long_median != null ? m.summary.tok_s_long_median.toFixed(1) : "—";
         const pass = m.summary.pass_rate != null
             ? `${Math.round(m.summary.pass_rate * checkDenom(m))}/${checkDenom(m)}`
             : "—";
@@ -467,12 +506,14 @@ function printTable(machine, suite, runs, revision, compKey, models, skipped, co
             ? pad(m.name, MODEL_COL) +
                 pad(tok, 12) +
                 pad(spread, 12) +
+                (showLong ? pad(tokLong, 12) : "") +
                 pad(load, 8) +
                 pad(pass, 8) +
                 notes.join("; ")
             : pad(m.name, MODEL_COL) +
                 pad(tok, 12) +
                 pad(spread, 12) +
+                (showLong ? pad(tokLong, 12) : "") +
                 pad(pass, 8) +
                 notes.join("; ");
         lines.push(row);
@@ -585,10 +626,13 @@ export async function main() {
         hot: parsed.hot,
     });
     const suiteCases = getSuiteCases(parsed.suite);
-    const throughputReps = parsed.runs + (parsed.hot ? 1 : 0);
     const caseCount = selected.length *
-        (suiteCases.filter((c) => c.role !== "throughput").length +
-            suiteCases.filter((c) => c.role === "throughput").length * throughputReps);
+        suiteCases.reduce((n, c) => n + caseReps(c, parsed.runs, parsed.hot), 0);
+    const worstCaseMs = selected.length *
+        suiteCases.reduce((n, c) => {
+            const pin = pinnedNumPredict(c, parsed.throughputNumPredict);
+            return n + caseReps(c, parsed.runs, parsed.hot) * attemptTimeoutMs(pin);
+        }, 0);
     if (!parsed.json) {
         const skipBits = [
             skipped.filter((s) => s.reason === "non-completion").length
@@ -603,7 +647,10 @@ export async function main() {
             ` × suite=${parsed.suite} × runs=${parsed.runs}` +
             (parsed.hot ? " (hot: +1 discarded)" : ""));
         console.error(`Cases: ~${caseCount} chat calls   timeout: ${BENCH_TIMEOUT_MS}ms/case` +
-            `   worst-case ceiling: ~${Math.ceil((caseCount * BENCH_TIMEOUT_MS) / 60000)}m if every call times out`);
+            (parsed.suite === "full"
+                ? ` (long case ≥${attemptTimeoutMs(LONG_THROUGHPUT_NUM_PREDICT)}ms)`
+                : "") +
+            `   worst-case ceiling: ~${Math.ceil(worstCaseMs / 60000)}m if every call times out`);
         if (!parsed.hot && !isLoopback(host) && (selected.length >= 2 || parsed.coldLoad)) {
             const bits = [
                 selected.length >= 2 ? "unloads each model before the next" : "",
@@ -665,8 +712,8 @@ export async function main() {
         let modelError;
         for (const caseDef of suiteCases) {
             const attempts = [];
-            const discardFirst = parsed.hot && caseDef.role === "throughput";
-            const reps = caseDef.role === "throughput" ? parsed.runs + (discardFirst ? 1 : 0) : 1;
+            const discardFirst = discardFirstThroughput(caseDef, parsed.hot);
+            const reps = caseReps(caseDef, parsed.runs, parsed.hot);
             for (let i = 0; i < reps; i += 1) {
                 const discarded = discardFirst && i === 0;
                 const attempt = await runCaseAttempt({
@@ -758,6 +805,7 @@ export async function main() {
             temperature: 0,
             seed: 0,
             throughput_num_predict: parsed.throughputNumPredict,
+            throughput_long_num_predict: parsed.suite === "full" ? LONG_THROUGHPUT_NUM_PREDICT : null,
             num_ctx: baseSettings.num_ctx ?? null,
             warmup: parsed.warmup,
             hot: parsed.hot,

package/dist/mcp.js

@@ -200,7 +200,7 @@ const TOOLS = [
     {
         name: "ollanet_ps",
         description: "List models currently loaded in VRAM. Optional machine; omit to probe every discovered host. " +
-            "scan is on-disk inventory; ps is what is resident right now.",
+            "scan is on-disk inventory; ps is what is resident right now, with CPU/GPU %.",
         inputSchema: {
             type: "object",
             properties: {

package/dist/ollama-chat.d.ts

@@ -74,6 +74,8 @@ export declare function ollamaVersion(baseUrl: string, timeoutMs: number): Promi
 export interface PsModel {
     name?: string;
     model?: string;
+    /** Total resident bytes (VRAM + CPU/RAM). */
+    size?: number;
     size_vram?: number;
     context_length?: number;
     digest?: string;

package/dist/ps.d.ts

@@ -8,7 +8,12 @@ import { type AppConfig } from "./config.js";
 export declare function helpText(): string;
 export interface LoadedModel {
     name: string;
+    size?: number;
     size_vram?: number;
+    cpu_percent?: number;
+    gpu_percent?: number;
+    /** Same wording as `ollama ps` PROCESSOR (e.g. "100% GPU", "83%/17% CPU/GPU"). */
+    processor?: string;
     context_length?: number;
     digest?: string;
     tuned: boolean;
@@ -28,5 +33,11 @@ export interface PsOptions {
 export interface PsResult {
     hosts: LoadedHost[];
 }
+/** Match `ollama ps` PROCESSOR: CPU% from (size - size_vram) / size. */
+export declare function processorSplit(size?: number, sizeVram?: number): {
+    cpu_percent?: number;
+    gpu_percent?: number;
+    processor?: string;
+};
 export declare function listLoaded(opts?: PsOptions): Promise<PsResult>;
 export declare function main(): Promise<void>;

package/dist/ps.js

@@ -23,7 +23,8 @@ Examples:
   ollanet ps studio
   ollanet ps studio --json
 
-scan lists models on disk. ps lists what is loaded in VRAM right now.
+scan lists models on disk. ps lists what is loaded and the CPU/GPU split
+(same rule as ollama ps: size_vram / size).
 
 Options:
   --machine <name>   One host (default: every discovered host)
@@ -76,11 +77,31 @@ function formatBytes(bytes) {
     }
     return `${value.toFixed(unit === 0 ? 0 : 1)} ${units[unit]}`;
 }
+/** Match `ollama ps` PROCESSOR: CPU% from (size - size_vram) / size. */
+export function processorSplit(size, sizeVram) {
+    const total = size ?? 0;
+    const vram = sizeVram ?? 0;
+    if (vram === 0 && total === 0)
+        return {};
+    if (vram === 0)
+        return { cpu_percent: 100, gpu_percent: 0, processor: "100% CPU" };
+    if (total > 0 && vram === total) {
+        return { cpu_percent: 0, gpu_percent: 100, processor: "100% GPU" };
+    }
+    if (vram > total || total === 0)
+        return { processor: "Unknown" };
+    const cpu = Math.round(((total - vram) / total) * 100);
+    const gpu = 100 - cpu;
+    return { cpu_percent: cpu, gpu_percent: gpu, processor: `${cpu}%/${gpu}% CPU/GPU` };
+}
 function toLoaded(m) {
     const name = (m.name ?? m.model ?? "").trim();
+    const split = processorSplit(m.size, m.size_vram);
     return {
         name,
+        size: m.size,
         size_vram: m.size_vram,
+        ...split,
         context_length: m.context_length,
         digest: m.digest,
         tuned: looksTuned(name),
@@ -143,7 +164,7 @@ function printPs(result) {
                 const vram = formatBytes(m.size_vram);
                 const ctx = m.context_length != null ? `ctx ${m.context_length}` : "";
                 const tag = m.tuned ? " [tuned]" : "";
-                const meta = [vram ? `VRAM ${vram}` : "", ctx].filter(Boolean).join("  ");
+                const meta = [vram ? `VRAM ${vram}` : "", m.processor, ctx].filter(Boolean).join("  ");
                 process.stdout.write(meta ? `  ${m.name}  ${meta}${tag}\n` : `  ${m.name}${tag}\n`);
             }
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ollanet",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "description": "Discover and talk to Ollama on any reachable network — CLI, MCP, and a Node library",
   "type": "module",
   "bin": {