ollanet 0.5.0 → 0.6.0

package/dist/compare.js

@@ -0,0 +1,338 @@
+/**
+ * Head-to-head: one prompt, 2–5 models on the same host.
+ *
+ *   ollanet compare studio gemma3:12b llama3.2:3b --prompt "Explain MagicDNS"
+ *   ollanet compare studio m1 m2 --file task.md
+ */
+import { configFromPartial, loadConfig, machineSettingsForHost, mergeSettings, settingsFromEnv, } from "./config.js";
+import { discoverHosts, envInt, ollamaBaseUrl, resolveHost, shortName } from "./hosts.js";
+import { ollamaChat, ollamaTags, ollamaUnload, waitUntilUnloaded, } from "./ollama-chat.js";
+import { assemblePrompt, readPromptFile } from "./prompt-input.js";
+import { comparesDir, newCompareId, saveCompare } from "./compare-store.js";
+const MIN_MODELS = 2;
+const MAX_MODELS = 5;
+/** Used when --prompt, --file, and stdin are all omitted. */
+export const DEFAULT_COMPARE_PROMPT = "In 6–10 sentences, explain how you’d pick an Ollama host on a mixed LAN/Tailscale mesh " +
+    "for a 12B chat model. Then give one failure mode (wrong host, cold load, or VRAM spill) " +
+    "and how you’d confirm it. Be specific.";
+function compareTimeoutMs() {
+    return envInt("OLLAMA_COMPARE_TIMEOUT_MS", envInt("OLLAMA_PROMPT_TIMEOUT_MS", 600_000));
+}
+function unloadWaitMs() {
+    return envInt("OLLAMA_COMPARE_UNLOAD_WAIT_MS", 60_000);
+}
+function usage() {
+    console.error(`Usage:
+  ollanet compare <machine> <model> <model> [model...]
+  ollanet compare <machine> <model> <model> [model...] --prompt <text>
+  ollanet compare <machine> <model> <model> [model...] --file <path.txt|.md>
+
+Examples:
+  ollanet compare studio gemma3:12b llama3.2:3b
+  ollanet compare studio gemma3:12b llama3.2:3b --prompt "Explain MagicDNS"
+  ollanet compare studio gemma3:12b qwen2.5:7b llama3.2:3b --file ./task.md
+
+Runs the same prompt on 2–5 models on one host. Prints a summary table and
+writes compares/<id>.md plus .json (prompt, replies, stats). Not a bench suite.
+Omit --prompt/--file to use the built-in mesh-host tasting prompt.
+
+Options:
+  --prompt <text>        Prompt text (default: built-in mesh-host prompt)
+  --file <path>          Prompt from a .txt or .md file
+  --system <text>        System prompt
+  --temperature <n>
+  --num-predict <n>
+  --num-ctx <n>
+  --keep-alive <value>
+  --think / --no-think
+  --unload               Unload each model before the next (fairer tok/s)
+  --no-save              Do not write compares/*
+  --json                 Emit the full record on stdout`);
+    process.exit(1);
+}
+function takeValue(args, flag) {
+    const value = args.shift();
+    if (!value) {
+        console.error(`${flag} requires a value`);
+        usage();
+    }
+    return value;
+}
+function parseNumberFlag(raw, flag) {
+    const n = Number(raw);
+    if (!Number.isFinite(n)) {
+        console.error(`${flag} must be a number (got "${raw}")`);
+        usage();
+    }
+    return n;
+}
+function parseArgs(argv) {
+    const args = [...argv];
+    let prompt;
+    let file;
+    let json = false;
+    let save = true;
+    let unload = false;
+    const settings = {};
+    const positional = [];
+    while (args.length > 0) {
+        const arg = args.shift();
+        if (arg === "--")
+            continue;
+        if (arg === "--help" || arg === "-h")
+            usage();
+        if (arg === "--json") {
+            json = true;
+            continue;
+        }
+        if (arg === "--no-save") {
+            save = false;
+            continue;
+        }
+        if (arg === "--unload") {
+            unload = true;
+            continue;
+        }
+        if (arg === "--think") {
+            settings.think = true;
+            continue;
+        }
+        if (arg === "--no-think") {
+            settings.think = false;
+            continue;
+        }
+        if (arg === "--prompt" || arg === "-p" || arg.startsWith("--prompt=")) {
+            prompt = arg.includes("=") ? arg.slice("--prompt=".length) : takeValue(args, "--prompt");
+            continue;
+        }
+        if (arg === "--file" || arg === "-f" || arg.startsWith("--file=")) {
+            file = arg.includes("=") ? arg.slice("--file=".length) : takeValue(args, "--file");
+            continue;
+        }
+        if (arg === "--system" || arg.startsWith("--system=")) {
+            settings.system = arg.includes("=")
+                ? arg.slice("--system=".length)
+                : takeValue(args, "--system");
+            continue;
+        }
+        if (arg === "--temperature" || arg === "-t" || arg.startsWith("--temperature=")) {
+            const raw = arg.includes("=") ? arg.slice("--temperature=".length) : takeValue(args, "--temperature");
+            settings.temperature = parseNumberFlag(raw, "--temperature");
+            continue;
+        }
+        if (arg === "--num-predict" || arg.startsWith("--num-predict=")) {
+            const raw = arg.includes("=") ? arg.slice("--num-predict=".length) : takeValue(args, "--num-predict");
+            settings.num_predict = Math.trunc(parseNumberFlag(raw, "--num-predict"));
+            continue;
+        }
+        if (arg === "--num-ctx" || arg.startsWith("--num-ctx=")) {
+            const raw = arg.includes("=") ? arg.slice("--num-ctx=".length) : takeValue(args, "--num-ctx");
+            settings.num_ctx = Math.trunc(parseNumberFlag(raw, "--num-ctx"));
+            continue;
+        }
+        if (arg === "--keep-alive" || arg.startsWith("--keep-alive=")) {
+            const raw = arg.includes("=") ? arg.slice("--keep-alive=".length) : takeValue(args, "--keep-alive");
+            const asNum = Number(raw);
+            settings.keep_alive =
+                raw.trim() !== "" && Number.isFinite(asNum) && String(asNum) === raw ? asNum : raw;
+            continue;
+        }
+        if (arg.startsWith("-")) {
+            console.error(`Unknown flag: ${arg}`);
+            usage();
+        }
+        positional.push(arg);
+    }
+    const machine = positional[0];
+    const models = positional.slice(1);
+    return { machine, models, prompt, file, json, save, unload, settings };
+}
+function tokPerSec(chunk) {
+    if (chunk.eval_count == null || chunk.eval_duration == null || chunk.eval_duration <= 0) {
+        return undefined;
+    }
+    return chunk.eval_count / (chunk.eval_duration / 1e9);
+}
+function resolveModelName(available, query) {
+    const q = query.toLowerCase();
+    const exact = available.find((n) => n.toLowerCase() === q);
+    if (exact)
+        return exact;
+    const prefix = available.filter((n) => n.toLowerCase().startsWith(`${q}:`) || q.startsWith(`${n.toLowerCase()}:`));
+    return prefix.length === 1 ? prefix[0] : undefined;
+}
+function printSummary(record) {
+    const nameWidth = Math.max(8, ...record.results.map((r) => r.model.length));
+    process.stderr.write(`Compare on ${record.machine}  ${record.results.length} models  ${record.prompt.length} char prompt\n`);
+    process.stderr.write(`${"model".padEnd(nameWidth)}  ${"tok/s".padStart(7)}  ${"tokens".padStart(6)}  ${"wall".padStart(7)}  done\n`);
+    for (const r of record.results) {
+        if (r.error) {
+            process.stderr.write(`${r.model.padEnd(nameWidth)}  ERROR  ${r.error}\n`);
+            continue;
+        }
+        const toks = r.tok_s != null ? r.tok_s.toFixed(1) : "—";
+        const tokens = r.eval_count != null ? String(r.eval_count) : "—";
+        const wall = r.wall_ms != null ? `${(r.wall_ms / 1000).toFixed(2)}s` : "—";
+        process.stderr.write(`${r.model.padEnd(nameWidth)}  ${toks.padStart(7)}  ${tokens.padStart(6)}  ${wall.padStart(7)}  ${r.done_reason ?? "—"}\n`);
+    }
+}
+export async function runCompare(opts) {
+    const models = opts.models.map((m) => m.trim()).filter(Boolean);
+    if (models.length < MIN_MODELS || models.length > MAX_MODELS) {
+        throw new Error(`Compare needs ${MIN_MODELS}–${MAX_MODELS} models (got ${models.length}).`);
+    }
+    let fileText;
+    if (opts.file) {
+        fileText = await readPromptFile(opts.file);
+    }
+    const assembled = assemblePrompt({ argv: opts.prompt, file: fileText });
+    const defaulted = !assembled;
+    const prompt = assembled || DEFAULT_COMPARE_PROMPT;
+    const config = opts.config ? configFromPartial(opts.config) : await loadConfig();
+    const { hosts } = await discoverHosts({
+        hosts: config.hosts,
+        discovery: config.discovery,
+    });
+    const host = resolveHost(hosts, opts.machine);
+    if (!host.online && !host.isSelf && host.source === "tailscale") {
+        throw new Error(`Machine "${shortName(host)}" appears offline.`);
+    }
+    const baseUrl = ollamaBaseUrl(host);
+    const timeoutMs = opts.timeoutMs ?? compareTimeoutMs();
+    let available = null;
+    try {
+        available = (await ollamaTags(baseUrl, Math.min(timeoutMs || 5000, 10_000))).map((t) => t.name);
+    }
+    catch {
+        available = null;
+    }
+    const resolved = [];
+    for (const name of models) {
+        if (available) {
+            const hit = resolveModelName(available, name);
+            if (!hit) {
+                throw new Error(`Unknown model "${name}" on ${shortName(host)}. Available: ${available.join(", ")}`);
+            }
+            resolved.push(hit);
+        }
+        else {
+            resolved.push(name);
+        }
+    }
+    const settings = mergeSettings({ think: false }, config.defaults, machineSettingsForHost(config, host), settingsFromEnv(), opts.settings);
+    const quiet = opts.quiet === true || opts.writeStdout === false;
+    const results = [];
+    for (let i = 0; i < resolved.length; i += 1) {
+        const model = resolved[i];
+        if (!quiet) {
+            process.stderr.write(`→ ${model} (${i + 1}/${resolved.length})\n`);
+        }
+        const started = Date.now();
+        try {
+            const { content, thinking, chunk } = await ollamaChat({
+                baseUrl,
+                model,
+                messages: [
+                    ...(settings.system ? [{ role: "system", content: settings.system }] : []),
+                    { role: "user", content: prompt },
+                ],
+                stream: false,
+                writeStdout: false,
+                timeoutMs,
+                settings,
+            });
+            results.push({
+                model,
+                content,
+                thinking: thinking || undefined,
+                tok_s: tokPerSec(chunk),
+                eval_count: chunk.eval_count,
+                eval_duration: chunk.eval_duration,
+                total_duration: chunk.total_duration,
+                load_duration: chunk.load_duration,
+                wall_ms: Date.now() - started,
+                done_reason: chunk.done_reason,
+            });
+        }
+        catch (err) {
+            results.push({
+                model,
+                wall_ms: Date.now() - started,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+        if (opts.unload && i < resolved.length - 1) {
+            try {
+                await ollamaUnload(baseUrl, model, Math.min(timeoutMs || 30_000, 30_000));
+                await waitUntilUnloaded(baseUrl, model, unloadWaitMs());
+            }
+            catch {
+                // Fairness hint only — do not fail the compare.
+            }
+        }
+    }
+    const record = {
+        id: newCompareId(),
+        created_at: new Date().toISOString(),
+        machine: shortName(host),
+        endpoint: baseUrl,
+        prompt,
+        default_prompt: defaulted,
+        system: settings.system,
+        results,
+    };
+    if (opts.save !== false) {
+        record.files = await saveCompare(record);
+    }
+    return record;
+}
+export async function main() {
+    const parsed = parseArgs(process.argv.slice(2));
+    if (!parsed.machine) {
+        console.error("Machine is required.");
+        usage();
+    }
+    const { stdin: stdinStream } = await import("node:process");
+    let fromStdin = "";
+    if (!stdinStream.isTTY) {
+        const chunks = [];
+        for await (const chunk of stdinStream)
+            chunks.push(chunk);
+        fromStdin = Buffer.concat(chunks).toString("utf8").trim();
+    }
+    let fileText;
+    if (parsed.file) {
+        try {
+            fileText = await readPromptFile(parsed.file);
+        }
+        catch (err) {
+            console.error(err instanceof Error ? err.message : err);
+            process.exitCode = 1;
+            return;
+        }
+    }
+    const prompt = assemblePrompt({ argv: parsed.prompt, file: fileText, stdin: fromStdin });
+    const record = await runCompare({
+        machine: parsed.machine,
+        models: parsed.models,
+        prompt: prompt || undefined,
+        settings: parsed.settings,
+        save: parsed.save,
+        unload: parsed.unload,
+        writeStdout: !parsed.json,
+        quiet: parsed.json,
+    });
+    if (record.default_prompt) {
+        process.stderr.write("Using default compare prompt (pass --prompt or --file to override).\n");
+    }
+    if (parsed.json) {
+        process.stdout.write(`${JSON.stringify(record, null, 2)}\n`);
+        return;
+    }
+    printSummary(record);
+    if (record.files) {
+        process.stderr.write(`saved ${record.files.md}\n`);
+        process.stderr.write(`      ${record.files.json}\n`);
+        process.stderr.write(`compares dir: ${comparesDir()}\n`);
+    }
+}

package/dist/index.d.ts

@@ -8,13 +8,18 @@
  *   const { servers } = await scanNetwork({ lanScan: false });
  */
 export { scanNetwork } from "./scan.js";
-export type { ScanOptions, ScanPayload, ScannedServer } from "./scan.js";
+export type { ScanOptions, ScanPayload, ScannedServer, ScanFailure } from "./scan.js";
+export { lastScan, lastScanPath, loadLastScan } from "./scan-store.js";
+export type { StoredScan } from "./scan-store.js";
 export { discoverHosts, resolveHost, ollamaBaseUrl, shortName, ollamaPort, } from "./hosts.js";
 export type { DiscoverOptions, DiscoveryResult, HostTarget, HostSource, HostConfigEntry, } from "./hosts.js";
 export { loadConfig, defaultModelForHost, mergeSettings, configFromPartial, } from "./config.js";
 export type { AppConfig, DiscoveryConfig, GenerateSettings } from "./config.js";
 export { runPrompt } from "./prompt.js";
 export type { PromptRunOptions, PromptRunResult } from "./prompt.js";
+export { runCompare, DEFAULT_COMPARE_PROMPT } from "./compare.js";
+export type { CompareOptions, CompareRecord, CompareModelResult } from "./compare.js";
+export { assemblePrompt, readPromptFile } from "./prompt-input.js";
 export { pullModel } from "./pull.js";
 export type { PullOptions, PullResult } from "./pull.js";
 export { showModel } from "./show.js";

package/dist/index.js

@@ -8,9 +8,12 @@
  *   const { servers } = await scanNetwork({ lanScan: false });
  */
 export { scanNetwork } from "./scan.js";
+export { lastScan, lastScanPath, loadLastScan } from "./scan-store.js";
 export { discoverHosts, resolveHost, ollamaBaseUrl, shortName, ollamaPort, } from "./hosts.js";
 export { loadConfig, defaultModelForHost, mergeSettings, configFromPartial, } from "./config.js";
 export { runPrompt } from "./prompt.js";
+export { runCompare, DEFAULT_COMPARE_PROMPT } from "./compare.js";
+export { assemblePrompt, readPromptFile } from "./prompt-input.js";
 export { pullModel } from "./pull.js";
 export { showModel } from "./show.js";
 export { removeModel } from "./rm.js";

package/dist/mcp.js

@@ -12,10 +12,13 @@ import { createInterface } from "node:readline";
 import { readFile } from "node:fs/promises";
 import { listChats, loadChat } from "./chat-store.js";
 import { projectPath } from "./paths.js";
+import { runCompare } from "./compare.js";
 import { listLoaded } from "./ps.js";
 import { pullModel } from "./pull.js";
+import { assemblePrompt, readPromptFile } from "./prompt-input.js";
 import { runPrompt } from "./prompt.js";
 import { removeModel } from "./rm.js";
+import { lastScan } from "./scan-store.js";
 import { scanNetwork } from "./scan.js";
 import { showModel } from "./show.js";
 // Pin to the oldest widely-supported MCP revision. Clients may request a newer
@@ -38,6 +41,10 @@ const TOOLS = [
                     type: "boolean",
                     description: "Also probe offline Tailscale peers (default false)",
                 },
+                last: {
+                    type: "boolean",
+                    description: "Return the last saved scan without probing the network (default false)",
+                },
             },
             additionalProperties: false,
         },
@@ -84,8 +91,51 @@ const TOOLS = [
                     type: "boolean",
                     description: "Persist the transcript (default true)",
                 },
+                file: {
+                    type: "string",
+                    description: "Read prompt body from a local .txt or .md file (joined with prompt if both set)",
+                },
             },
-            required: ["prompt"],
+            additionalProperties: false,
+        },
+    },
+    {
+        name: "ollanet_compare",
+        description: "Head-to-head: run the same prompt on 2–5 models on one Ollama host. " +
+            "Returns a summary plus saved markdown/JSON paths. " +
+            "Omit prompt/file to use the built-in mesh-host tasting prompt. " +
+            "Machine is a discovered name, MagicDNS name, hostname, or IP[:port].",
+        inputSchema: {
+            type: "object",
+            properties: {
+                machine: { type: "string", description: "Host name/IP" },
+                models: {
+                    type: "array",
+                    items: { type: "string" },
+                    minItems: 2,
+                    maxItems: 5,
+                    description: "2–5 model names on that host",
+                },
+                prompt: {
+                    type: "string",
+                    description: "Prompt text (optional; default is the built-in mesh-host prompt)",
+                },
+                file: {
+                    type: "string",
+                    description: "Read prompt from a local .txt or .md file",
+                },
+                system: { type: "string" },
+                temperature: { type: "number" },
+                num_predict: { type: "integer" },
+                num_ctx: { type: "integer" },
+                think: { type: "boolean" },
+                unload: {
+                    type: "boolean",
+                    description: "Unload each model before the next (fairer tok/s)",
+                },
+                save: { type: "boolean", description: "Write compares/*.md and .json (default true)" },
+            },
+            required: ["machine", "models"],
             additionalProperties: false,
         },
     },
@@ -218,6 +268,13 @@ async function packageVersion() {
 async function callTool(name, args) {
     switch (name) {
         case "ollanet_scan": {
+            if (args.last === true) {
+                const stored = await lastScan();
+                if (!stored) {
+                    return textResult({ error: "No saved scan. Run ollanet_scan without last=true first." }, true);
+                }
+                return textResult(stored);
+            }
             const payload = await scanNetwork({
                 lanScan: Boolean(args.lan),
                 includeOffline: Boolean(args.all),
@@ -225,9 +282,21 @@ async function callTool(name, args) {
             return textResult(payload);
         }
         case "ollanet_prompt": {
-            const prompt = typeof args.prompt === "string" ? args.prompt : "";
+            let fileText = "";
+            if (typeof args.file === "string" && args.file.trim()) {
+                try {
+                    fileText = await readPromptFile(args.file);
+                }
+                catch (err) {
+                    return textResult({ error: err instanceof Error ? err.message : String(err) }, true);
+                }
+            }
+            const prompt = assemblePrompt({
+                argv: typeof args.prompt === "string" ? args.prompt : "",
+                file: fileText,
+            });
             if (!prompt.trim()) {
-                return textResult({ error: "prompt is required" }, true);
+                return textResult({ error: "prompt or file is required" }, true);
             }
             const settings = {};
             if (typeof args.system === "string")
@@ -263,6 +332,52 @@ async function callTool(name, args) {
             });
             return textResult(result);
         }
+        case "ollanet_compare": {
+            const machine = typeof args.machine === "string" ? args.machine : "";
+            const models = Array.isArray(args.models)
+                ? args.models.filter((m) => typeof m === "string")
+                : [];
+            if (!machine.trim())
+                return textResult({ error: "machine is required" }, true);
+            if (models.length < 2 || models.length > 5) {
+                return textResult({ error: "models must be an array of 2–5 names" }, true);
+            }
+            let fileText = "";
+            if (typeof args.file === "string" && args.file.trim()) {
+                try {
+                    fileText = await readPromptFile(args.file);
+                }
+                catch (err) {
+                    return textResult({ error: err instanceof Error ? err.message : String(err) }, true);
+                }
+            }
+            const prompt = assemblePrompt({
+                argv: typeof args.prompt === "string" ? args.prompt : "",
+                file: fileText,
+            });
+            const settings = {};
+            if (typeof args.system === "string")
+                settings.system = args.system;
+            if (typeof args.temperature === "number")
+                settings.temperature = args.temperature;
+            if (typeof args.num_predict === "number")
+                settings.num_predict = Math.trunc(args.num_predict);
+            if (typeof args.num_ctx === "number")
+                settings.num_ctx = Math.trunc(args.num_ctx);
+            if (typeof args.think === "boolean")
+                settings.think = args.think;
+            const result = await runCompare({
+                machine,
+                models,
+                prompt: prompt || undefined,
+                settings,
+                save: args.save === undefined ? true : Boolean(args.save),
+                unload: args.unload === true,
+                writeStdout: false,
+                quiet: true,
+            });
+            return textResult(result);
+        }
         case "ollanet_pull": {
             const machine = typeof args.machine === "string" ? args.machine : "";
             const model = typeof args.model === "string" ? args.model : "";

package/dist/paths.d.ts

@@ -15,3 +15,7 @@ export declare function defaultConfigPath(): string;
 export declare function defaultResponsesDir(): string;
 /** Default benchmark results dir (mirrors responses layout). */
 export declare function defaultBenchmarksDir(): string;
+/** Last successful scan snapshot (single file). */
+export declare function defaultLastScanPath(): string;
+/** Head-to-head compare records (markdown + JSON). */
+export declare function defaultComparesDir(): string;

package/dist/paths.js

@@ -26,3 +26,11 @@ export function defaultResponsesDir() {
 export function defaultBenchmarksDir() {
     return IS_INSTALLED ? path.join(USER_DIR, "benchmarks") : projectPath("benchmarks");
 }
+/** Last successful scan snapshot (single file). */
+export function defaultLastScanPath() {
+    return IS_INSTALLED ? path.join(USER_DIR, "last-scan.json") : projectPath("last-scan.json");
+}
+/** Head-to-head compare records (markdown + JSON). */
+export function defaultComparesDir() {
+    return IS_INSTALLED ? path.join(USER_DIR, "compares") : projectPath("compares");
+}

package/dist/prompt-input.d.ts

@@ -0,0 +1,7 @@
+export declare function assertPromptFilename(filePath: string): string;
+export declare function readPromptFile(filePath: string): Promise<string>;
+export declare function assemblePrompt(parts: {
+    argv?: string;
+    file?: string;
+    stdin?: string;
+}): string;

package/dist/prompt-input.js

@@ -0,0 +1,36 @@
+/**
+ * Shared prompt assembly: argv, --file (.txt / .md), and piped stdin.
+ */
+import { readFile, stat } from "node:fs/promises";
+import path from "node:path";
+const ALLOWED = new Set([".txt", ".md"]);
+const MAX_BYTES = 1_000_000;
+export function assertPromptFilename(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    if (!ALLOWED.has(ext)) {
+        throw new Error(`Prompt file must be .txt or .md (got "${ext || "no extension"}": ${filePath})`);
+    }
+    return path.resolve(filePath);
+}
+export async function readPromptFile(filePath) {
+    const resolved = assertPromptFilename(filePath);
+    const info = await stat(resolved);
+    if (!info.isFile()) {
+        throw new Error(`Prompt path is not a file: ${resolved}`);
+    }
+    if (info.size > MAX_BYTES) {
+        throw new Error(`Prompt file is too large (${info.size} bytes; max ${MAX_BYTES}): ${resolved}`);
+    }
+    const text = (await readFile(resolved, "utf8")).replace(/^\uFEFF/, "");
+    if (!text.trim()) {
+        throw new Error(`Prompt file is empty: ${resolved}`);
+    }
+    return text;
+}
+export function assemblePrompt(parts) {
+    return [parts.argv, parts.file, parts.stdin]
+        .map((s) => (s ?? "").trim())
+        .filter(Boolean)
+        .join("\n\n")
+        .trim();
+}