ollanet 0.1.1 → 0.2.1

package/README.md

@@ -102,6 +102,7 @@ ollanet chats --id a1b2c3d4e5f6
 | `ollanet scan` | Probe known/discovered hosts for Ollama + list models |
 | `ollanet prompt …` | Send a prompt / continue a chat |
 | `ollanet chats` | List or inspect saved transcripts |
+| `ollanet bench …` | Benchmark models for tok/s + lightweight quality checks |
 
 ### `scan` options
 
@@ -139,6 +140,26 @@ Machine can be a discovered name, hostname, FQDN, or IP (`192.168.1.50`, `host:1
 - `--json` — list summary JSON
 - `--id <hash>` — show one chat (full JSON with `--json`)
 
+### `bench`
+
+```bash
+ollanet bench <machine|ip> [model...]
+ollanet bench <machine|ip> --all
+ollanet bench localhost gemma4:12b --runs 5 --cold-load --json
+```
+
+Runs a built-in suite (`quick` default, or `--suite full`) against one or more **completion**
+models: instruction checks once, throughput case repeated (`--runs`, default 3) with
+pinned `num_predict` / `seed` / `temperature`. Reports median tok/s + spread. Under `--all`,
+skips non-completion models (embeddings); multimodal `completion`+`vision` models stay in
+(early-stop / pass_rate handle text-suite failures). Use `--exclude-vision` to drop them.
+Saves `benchmarks/<id>.json` (or `~/.ollanet/benchmarks/` when installed).
+See [`docs/bench-spec.md`](docs/bench-spec.md) for measurement rules.
+
+`quick` has only two scored checks (`ping`, `math`), so `pass` is a coarse
+liveness/sanity gauge (`0/2`, `1/2`, `2/2`) — not “50% broken.” Use `--suite full`
+when you care more about the quality signal.
+
 ## Discovery
 
 ollanet combines several sources (deduped by `ip:port`):
@@ -205,6 +226,8 @@ Edit `~/.ollanet/config.json` (installed) or `config.json` (checkout) — or poi
 | `OLLANET_THINK` / `OLLAMA_THINK` | Enable thinking (`true`/`false`; default off) |
 | `OLLAMA_TIMEOUT_MS` | HTTP scan/probe timeout |
 | `OLLAMA_PROMPT_TIMEOUT_MS` | Prompt/chat HTTP timeout (default `600000`; `0` = none) |
+| `OLLAMA_BENCH_TIMEOUT_MS` | Per-case bench timeout (default `60000`) |
+| `OLLANET_BENCHMARKS_DIR` | Benchmark result directory |
 | `OLLAMA_CONCURRENCY` | Scan concurrency |
 
 ## Chat transcripts
@@ -233,9 +256,15 @@ ollanet/
     chat-store.ts
     config.ts
     paths.ts          # checkout vs installed path resolution
+    ollama-chat.ts    # shared /api/chat (+ tags/show/ps helpers)
+    bench.ts          # ollanet bench
+    bench-suite.ts
+    bench-store.ts
   dist/               # compiled output (published to npm)
+  docs/bench-spec.md  # bench measurement spec
   config.json
   responses/          # gitignored chat history
+  benchmarks/         # gitignored bench results
 ```
 
 ## Development
@@ -243,10 +272,14 @@ ollanet/
 ```bash
 pnpm install          # builds dist/ via prepare
 pnpm typecheck
+pnpm test             # build dist/, then run node:test suite
 pnpm ollanet -- scan  # runs from source via tsx
 pnpm build            # compile dist/ manually
 ```
 
+Tests use Node’s built-in test runner against `dist/` (what gets published), with an
+in-process mock Ollama server — no real models or Tailscale required.
+
 ## License
 
 MIT

package/dist/bench-store.js

@@ -0,0 +1,20 @@
+import { randomBytes } from "node:crypto";
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { defaultBenchmarksDir } from "./paths.js";
+const BENCH_DIR = process.env.OLLANET_BENCHMARKS_DIR ?? defaultBenchmarksDir();
+export function benchmarksDir() {
+    return BENCH_DIR;
+}
+export function newBenchId() {
+    return randomBytes(6).toString("hex");
+}
+export async function saveBenchmark(result) {
+    await mkdir(BENCH_DIR, { recursive: true });
+    const id = result && typeof result === "object" && "id" in result && typeof result.id === "string"
+        ? result.id
+        : newBenchId();
+    const file = path.join(BENCH_DIR, `${id}.json`);
+    await writeFile(file, `${JSON.stringify(result, null, 2)}\n`, "utf8");
+    return file;
+}

package/dist/bench-suite.js

@@ -0,0 +1,149 @@
+/**
+ * Built-in ollanet bench suites, checkers, and revision hashes.
+ */
+import { createHash } from "node:crypto";
+const MATH_ANSWER = 323;
+const REASON_ANSWER = 42;
+function lastTokenNormalized(content) {
+    const tokens = content.trim().split(/\s+/).filter(Boolean);
+    const last = tokens[tokens.length - 1] ?? "";
+    return last.toLowerCase().replace(/^[^a-z0-9]+|[^a-z0-9]+$/gi, "");
+}
+function integersInLastLine(content) {
+    const lines = content
+        .split(/\r?\n/)
+        .map((l) => l.trim())
+        .filter(Boolean);
+    const last = lines[lines.length - 1] ?? content.trim();
+    const matches = last.match(/-?\d+/g) ?? [];
+    return matches.map((m) => Number(m)).filter((n) => Number.isFinite(n));
+}
+function stripCodeFences(content) {
+    const trimmed = content.trim();
+    const fenced = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i);
+    return (fenced?.[1] ?? trimmed).trim();
+}
+function checkPing(content) {
+    const token = lastTokenNormalized(content);
+    return token === "ok"
+        ? { ok: true, detail: token }
+        : { ok: false, detail: `last token=${JSON.stringify(token)}` };
+}
+function checkExpectedInt(expected) {
+    return (content) => {
+        const ints = integersInLastLine(content);
+        return ints.includes(expected)
+            ? { ok: true, detail: String(expected) }
+            : { ok: false, detail: `ints=${JSON.stringify(ints)}` };
+    };
+}
+function checkHaikuLive(content) {
+    const lines = content
+        .split(/\r?\n/)
+        .map((l) => l.trim())
+        .filter(Boolean);
+    const ok = content.trim().length > 0 && lines.length >= 3;
+    return ok ? { ok: true, detail: `${lines.length} lines` } : { ok: false, detail: `${lines.length} lines` };
+}
+function checkJsonStructure(content) {
+    try {
+        const parsed = JSON.parse(stripCodeFences(content));
+        const animal = String(parsed.animal ?? "").toLowerCase();
+        const legs = Number(parsed.legs);
+        const ok = animal.includes("ferret") && legs === 4;
+        return ok
+            ? { ok: true }
+            : { ok: false, detail: `got animal=${JSON.stringify(parsed.animal)} legs=${JSON.stringify(parsed.legs)}` };
+    }
+    catch (err) {
+        return { ok: false, detail: err instanceof Error ? err.message : "parse error" };
+    }
+}
+const QUICK_CASES = [
+    {
+        id: "ping",
+        role: "check",
+        prompt: "Reply with exactly: OK",
+        num_predict: 16,
+        check: checkPing,
+    },
+    {
+        id: "math",
+        role: "check",
+        prompt: "What is 17 * 19? Reply with only the number.",
+        num_predict: 32,
+        check: checkExpectedInt(MATH_ANSWER),
+    },
+    {
+        id: "haiku",
+        role: "live",
+        prompt: "Write a haiku about ferrets.",
+        num_predict: 64,
+        check: checkHaikuLive,
+    },
+    {
+        id: "throughput",
+        role: "throughput",
+        // Enumerative on purpose: at temperature 0, prose prompts often EOS well
+        // before num_predict. Counting resists early stop so done_reason=length.
+        prompt: "Count from 1 to 400. Write one integer per line and nothing else. " +
+            "Do not stop early. Do not write words.",
+        // num_predict pinned by runner
+    },
+];
+const FULL_EXTRA = [
+    {
+        id: "json",
+        role: "check",
+        prompt: 'Return a JSON object describing an animal that is a ferret and has 4 legs. ' +
+            'Use keys "animal" (string) and "legs" (number).',
+        num_predict: 64,
+        format: "json",
+        check: checkJsonStructure,
+    },
+    {
+        id: "reason",
+        role: "check",
+        prompt: "A box has 10 apples. You eat 3, then buy 5 more, then give away 2 times as many as you ate. " +
+            "How many apples are left? Reply with only the final number.",
+        num_predict: 48,
+        // 10 - 3 + 5 - 6 = 6... wait: gave away 2*3=6, so 10-3+5-6=6. Spec said 42 as placeholder.
+        // Use a problem that clearly ends at 42:
+        // Actually rewrite for 42: "Start with 40. Add 7. Subtract 5. Reply with only the number." => 42
+        check: checkExpectedInt(REASON_ANSWER),
+    },
+];
+// Fix reason prompt to match REASON_ANSWER
+FULL_EXTRA[FULL_EXTRA.length - 1].prompt =
+    "Start with 40. Add 7. Subtract 5. Reply with only the final number.";
+export function getSuiteCases(suite) {
+    if (suite === "full")
+        return [...QUICK_CASES, ...FULL_EXTRA];
+    return [...QUICK_CASES];
+}
+export function suiteRevision(suite) {
+    const cases = getSuiteCases(suite);
+    const canonical = cases.map((c) => `${c.id}\n${c.role}\n${c.prompt}`).join("\n---\n");
+    return createHash("sha256").update(canonical).digest("hex").slice(0, 16);
+}
+export function comparabilityKey(opts) {
+    const payload = [
+        suiteRevision(opts.suite),
+        `num_predict=${opts.throughputNumPredict}`,
+        `seed=${opts.seed}`,
+        `temperature=${opts.temperature}`,
+        `think=${opts.think ? 1 : 0}`,
+        `num_ctx=${opts.numCtx ?? ""}`,
+    ].join("|");
+    return createHash("sha256").update(payload).digest("hex").slice(0, 16);
+}
+export function median(values) {
+    if (values.length === 0)
+        return undefined;
+    const sorted = [...values].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    if (sorted.length % 2 === 0) {
+        return (sorted[mid - 1] + sorted[mid]) / 2;
+    }
+    return sorted[mid];
+}