@otto-code/brain 0.7.5

package/dist/commands/bench.js

@@ -0,0 +1,222 @@
+/**
+ * `otto brain bench` — score a model's agentic coding ability on this machine.
+ * Either benchmarks an endpoint that is already serving (`--endpoint`) or loads
+ * each requested model itself. A long streaming run, so it prints a formatted
+ * report directly rather than going through the output layer.
+ */
+import http from "node:http";
+import path from "node:path";
+import { forModel, getCalibration, loadBrainConfig, loadProfilesStore } from "../config/index.js";
+import { query as queryGpu } from "../gpu.js";
+import { pickModel, scanModels } from "../models/index.js";
+import { CommandError } from "../output/types.js";
+import { resolveRuntime } from "../runtime/index.js";
+import { createRouter, Telemetry } from "../service/router.js";
+import { Supervisor } from "../service/supervisor.js";
+import * as bench from "../bench/index.js";
+import { loadRepoTasks } from "../bench/repo-task.js";
+import { describeCuratedRepos, findCuratedRepo } from "../bench/curated-repos.js";
+import * as archive from "../ops/archive.js";
+import * as results from "../ops/results.js";
+import * as vram from "../vram.js";
+function progress(p) {
+    if (p.phase === "start")
+        process.stderr.write(`  ${(p.title ?? "").padEnd(20)} running…\n`);
+    if (p.phase === "done") {
+        process.stderr.write(`  ${(p.title ?? "").padEnd(20)} ${((p.score ?? 0) * 100).toFixed(0)}%  ${p.summary}\n`);
+    }
+    if (p.phase === "failed")
+        process.stderr.write(`  failed: ${p.summary}\n`);
+}
+export function addBenchOptions(cmd) {
+    return cmd
+        .description("Score agentic coding ability on this machine")
+        .option("--model <fragments>", "model name fragment(s), comma-separated")
+        .option("--endpoint <host:port>", "benchmark an endpoint already serving")
+        .option("--only <a,b>", "run only these tasks")
+        .option("--depths <a,b,c>", "prompt depths for the depth-scaling task")
+        .option("--concurrency <n>", "requests in flight for the concurrency task", "3")
+        .option("--no-execute", "syntax-check generated code but do not run it")
+        .option("--port <n>", "internal port for the loaded model", "1251")
+        .option("--repo-workspace-dir <dir>", "mine SWE-bench tasks from a workspace, e.g. packages/protocol")
+        .option("--repo-workspace <name>", "npm workspace name for mined tasks, e.g. @otto-code/protocol")
+        .option("--repo-dir <dir>", "repo working copy to mine (default: current directory)")
+        .option("--repo-ref <ref>", "git ref to mine from", "origin/main")
+        .option("--repo-max <n>", "max mined tasks to run", "5")
+        .option("--curated <name>", "run a curated mined-repo preset (needs --repo-dir); pass an unknown name to list presets");
+}
+export async function runBenchCommand(options, _command) {
+    const config = loadBrainConfig();
+    const execute = options.execute !== false;
+    const depths = options.depths
+        ? options.depths.split(",").map((s) => Number(s.trim()))
+        : undefined;
+    const only = options.only ? options.only.split(",").map((s) => s.trim()) : null;
+    const concurrency = options.concurrency ? Math.max(1, Number(options.concurrency)) : 3;
+    // Opt-in: mining a repo replaces the static suite with real SWE-bench tasks.
+    // Absent these flags nothing touches a repo checkout.
+    let repoTasks;
+    if (options.curated) {
+        const preset = findCuratedRepo(options.curated);
+        if (!preset) {
+            throw new CommandError({
+                code: "NO_CURATED",
+                message: `unknown curated preset "${options.curated}". Available:\n${describeCuratedRepos()}`,
+            });
+        }
+        // The harness resets the working copy hard between tasks, so require an
+        // explicit --repo-dir rather than silently mutating the current checkout.
+        if (!options.repoDir) {
+            throw new CommandError({
+                code: "NO_REPO_DIR",
+                message: "--repo-dir is required with --curated: the working copy is reset hard " +
+                    "(git reset --hard + git clean -fd) between tasks, so point it at a spare " +
+                    "checkout, never one with uncommitted work.",
+            });
+        }
+        const dir = path.resolve(options.repoDir);
+        process.stderr.write(`\ncurated: mining ${preset.workspace} @ ${preset.ref} in ${dir}\n` +
+            `  note: this resets the working copy (git reset --hard + git clean -fd)\n`);
+        const tasks = await loadRepoTasks({
+            dir,
+            workspace: preset.workspace,
+            workspaceDir: preset.workspaceDir,
+            ref: preset.ref,
+            maxTasks: preset.maxTasks,
+        });
+        if (!tasks.length) {
+            throw new CommandError({
+                code: "NO_TASKS",
+                message: `no mineable fix commits found in ${preset.workspaceDir}`,
+            });
+        }
+        process.stderr.write(`  ${tasks.length} mined task(s): ${tasks.map((t) => t.id).join(", ")}\n`);
+        repoTasks = tasks;
+    }
+    else if (options.repoWorkspaceDir) {
+        if (!options.repoWorkspace) {
+            throw new CommandError({
+                code: "NO_WORKSPACE",
+                message: "--repo-workspace <name> is required with --repo-workspace-dir",
+            });
+        }
+        const dir = options.repoDir ? path.resolve(options.repoDir) : process.cwd();
+        process.stderr.write(`\nmining tasks from ${options.repoWorkspaceDir} @ ${options.repoRef}\n`);
+        const tasks = await loadRepoTasks({
+            dir,
+            workspace: options.repoWorkspace,
+            workspaceDir: options.repoWorkspaceDir,
+            ref: options.repoRef,
+            maxTasks: options.repoMax ? Math.max(1, Number(options.repoMax)) : 5,
+        });
+        if (!tasks.length) {
+            throw new CommandError({
+                code: "NO_TASKS",
+                message: `no mineable fix commits found in ${options.repoWorkspaceDir}`,
+            });
+        }
+        process.stderr.write(`  ${tasks.length} mined task(s): ${tasks.map((t) => t.id).join(", ")}\n`);
+        repoTasks = tasks;
+    }
+    if (options.endpoint) {
+        const [host, portStr] = options.endpoint.split(":");
+        const port = Number(portStr || config.listen.port);
+        process.stderr.write(`\nbenchmarking endpoint ${host}:${port}\n`);
+        const report = await bench.runSuite({
+            host,
+            port,
+            execute,
+            depths,
+            only,
+            concurrency,
+            tasks: repoTasks,
+            onProgress: progress,
+        });
+        process.stdout.write(`${bench.formatReport(report, { modelName: `${host}:${port}` })}\n`);
+        return;
+    }
+    const store = loadProfilesStore();
+    const catalog = scanModels(config);
+    const needles = options.model
+        ? options.model.split(",").map((s) => s.trim())
+        : [store.lastModelId].filter((v) => Boolean(v));
+    if (!needles.length) {
+        throw new CommandError({
+            code: "NO_MODEL",
+            message: "specify --model <fragment>[,<fragment>...] or --endpoint host:port",
+        });
+    }
+    const runtime = resolveRuntime(config);
+    if (!runtime) {
+        throw new CommandError({
+            code: "NO_RUNTIME",
+            message: "no llama.cpp runtime; run `otto brain runtime install`",
+        });
+    }
+    const port = Number(options.port || 1251);
+    const entries = [];
+    for (const needle of needles) {
+        const model = pickModel(catalog, needle);
+        let profile = forModel(store, model, config.defaults);
+        const gpu = await queryGpu();
+        if (gpu) {
+            const fit = vram.fitToBudget({
+                model,
+                profile,
+                calibration: getCalibration(store, model, profile),
+                totalVramBytes: gpu.totalBytes,
+            });
+            if (!fit.adjusted && !fit.budget.fits) {
+                process.stderr.write(`\nskipping ${model.displayName}: ${fit.reason}\n`);
+                continue;
+            }
+            profile = fit.profile;
+        }
+        process.stdout.write(`\n${"=".repeat(74)}\n${model.displayName}\n${"=".repeat(74)}\n`);
+        const supervisor = new Supervisor({ runtime, internalPort: port });
+        const telemetry = new Telemetry();
+        const server = http.createServer(createRouter({ supervisor, telemetry, getCatalog: () => catalog }));
+        await new Promise((resolve, reject) => {
+            server.once("error", reject);
+            server.listen(port + 1, "127.0.0.1", resolve);
+        });
+        try {
+            await supervisor.start(model, profile);
+            const archiveId = archive.runId(model);
+            const report = await bench.runSuite({
+                host: "127.0.0.1",
+                port: port + 1,
+                execute,
+                depths,
+                only,
+                concurrency,
+                reasoningBudget: profile.reasoningBudget ?? null,
+                contextWindow: profile.contextSize ?? null,
+                archiveId,
+                tasks: repoTasks,
+                onProgress: progress,
+            });
+            process.stdout.write(`${bench.formatReport(report, { modelName: model.displayName, profile })}\n`);
+            const runtimeLabel = `${runtime.label} v${runtime.version}`;
+            const { file } = results.save({
+                model,
+                profile,
+                report,
+                gpu,
+                runtime: runtimeLabel,
+                archiveId,
+            });
+            process.stderr.write(`  saved to results/${path.basename(file)}\n`);
+            entries.push({ modelName: model.displayName, report });
+        }
+        finally {
+            await supervisor.stop();
+            await new Promise((resolve) => server.close(() => resolve()));
+        }
+    }
+    if (entries.length > 1) {
+        process.stdout.write(`\n${"=".repeat(74)}\n  COMPARISON\n${"=".repeat(74)}\n`);
+        process.stdout.write(`${bench.formatComparison(entries)}\n`);
+    }
+}
+//# sourceMappingURL=bench.js.map

package/dist/commands/calibrate.d.ts

@@ -0,0 +1,19 @@
+/**
+ * `otto brain calibrate` — measure real KV bytes/token for a model and persist it,
+ * so the VRAM budget uses a measured figure instead of the (over-estimating)
+ * theoretical formula.
+ */
+import type { Command } from "commander";
+import type { AnyCommandResult } from "../output/index.js";
+export interface CalibrateRow {
+    model: string;
+    measuredKvKB: string;
+    formulaKvKB: string;
+    overhead: string;
+    maxContext: string;
+}
+export declare function addCalibrateOptions(cmd: Command): Command;
+export declare function runCalibrateCommand(options: {
+    model?: string;
+}, _command: Command): Promise<AnyCommandResult<CalibrateRow>>;
+//# sourceMappingURL=calibrate.d.ts.map

package/dist/commands/calibrate.js

@@ -0,0 +1,75 @@
+import { getCalibration, forModel, loadBrainConfig, loadProfilesStore, putCalibration, saveProfilesStore, } from "../config/index.js";
+import { query as queryGpu } from "../gpu.js";
+import { pickModel, scanModels } from "../models/index.js";
+import { CommandError } from "../output/types.js";
+import { calibrate } from "../ops/calibrate.js";
+import { resolveRuntime } from "../runtime/index.js";
+import * as vram from "../vram.js";
+const calibrateSchema = {
+    idField: "model",
+    columns: [
+        { header: "MODEL", field: "model", width: 40 },
+        { header: "MEASURED KV", field: "measuredKvKB", width: 12, align: "right" },
+        { header: "FORMULA KV", field: "formulaKvKB", width: 12, align: "right" },
+        { header: "OVERHEAD", field: "overhead", width: 9, align: "right" },
+        { header: "MAX CTX", field: "maxContext", width: 12, align: "right" },
+    ],
+};
+export function addCalibrateOptions(cmd) {
+    return cmd
+        .description("Measure real KV bytes/token and save it")
+        .option("--model <fragment>", "model name fragment or catalog id");
+}
+export async function runCalibrateCommand(options, _command) {
+    const config = loadBrainConfig();
+    const runtime = resolveRuntime(config);
+    if (!runtime) {
+        throw new CommandError({
+            code: "NO_RUNTIME",
+            message: "no llama.cpp runtime available",
+            details: "run `otto brain runtime install`",
+        });
+    }
+    const store = loadProfilesStore();
+    const catalog = scanModels(config);
+    const model = pickModel(catalog, options.model ?? store.lastModelId ?? undefined);
+    const profile = forModel(store, model, config.defaults);
+    const measurement = await calibrate({
+        runtime,
+        model,
+        profile,
+        onProgress: (p) => {
+            if (p.phase === "loading")
+                process.stderr.write(`  loading at ${p.contextSize?.toLocaleString()} ctx…\n`);
+            if (p.phase === "measured")
+                process.stderr.write(`    used ${vram.formatGiB(p.deltaBytes ?? 0)}\n`);
+            if (p.phase === "skip")
+                process.stderr.write(`  skipped ${p.contextSize}: ${p.reason}\n`);
+        },
+    });
+    putCalibration(store, model, profile, measurement);
+    saveProfilesStore(store);
+    const gpu = await queryGpu();
+    const maxCtx = gpu
+        ? vram.maxContextThatFits({
+            model,
+            profile,
+            calibration: getCalibration(store, model, profile),
+            totalVramBytes: gpu.totalBytes,
+        })
+        : null;
+    return {
+        type: "single",
+        data: {
+            model: model.displayName,
+            measuredKvKB: `${(measurement.kvBytesPerToken / 1024).toFixed(2)} KB`,
+            formulaKvKB: measurement.theoreticalKvBytesPerToken
+                ? `${(measurement.theoreticalKvBytesPerToken / 1024).toFixed(2)} KB`
+                : "-",
+            overhead: vram.formatGiB(measurement.baseOverheadBytes),
+            maxContext: maxCtx ? maxCtx.toLocaleString() : "unknown",
+        },
+        schema: calibrateSchema,
+    };
+}
+//# sourceMappingURL=calibrate.js.map

package/dist/commands/catalog.d.ts

@@ -0,0 +1,32 @@
+/**
+ * `otto brain catalog` — list the downloadable model catalog, each entry
+ * annotated with whether it is already installed. The installed flag reuses the
+ * authoritative catalog↔model join (enrichWithCatalog sets `catalogId` back-
+ * references on scanned models), so it agrees with `scan`/`pull` and works with
+ * no llama-server running. The daemon shells this out (`--json`) to power the
+ * Brain "Models" UI, which is why the row carries rich fields (repo, size,
+ * params, tier, use-cases) that only the JSON output surfaces.
+ */
+import type { Command } from "commander";
+import type { AnyCommandResult, OutputSchema } from "../output/index.js";
+export interface CatalogRow {
+    id: string;
+    name: string;
+    installed: boolean;
+    publisher: string;
+    repo: string;
+    quant: string;
+    params: string;
+    sizeBytes: number | null;
+    size: string;
+    vision: boolean;
+    thinking: boolean;
+    contextMax: number | null;
+    tier: string;
+    useCases: string[];
+    why: string;
+}
+export declare const catalogSchema: OutputSchema<CatalogRow>;
+export declare function addCatalogOptions(cmd: Command): Command;
+export declare function runCatalogCommand(_options: unknown, _command: Command): Promise<AnyCommandResult<CatalogRow>>;
+//# sourceMappingURL=catalog.d.ts.map

package/dist/commands/catalog.js

@@ -0,0 +1,48 @@
+import { loadBrainConfig, loadCatalog } from "../config/index.js";
+import { formatBytes, scanModels } from "../models/index.js";
+export const catalogSchema = {
+    idField: "id",
+    columns: [
+        { header: "NAME", field: "name", width: 34 },
+        { header: "PARAMS", field: "params", width: 8 },
+        { header: "SIZE", field: "size", width: 9, align: "right" },
+        { header: "TIER", field: "tier", width: 8 },
+        {
+            // Format the boolean for the table only; JSON keeps the real `installed`.
+            header: "INSTALLED",
+            field: (row) => (row.installed ? "yes" : ""),
+            width: 9,
+            color: (v) => (v === "yes" ? "green" : undefined),
+        },
+    ],
+};
+export function addCatalogOptions(cmd) {
+    return cmd.description("List downloadable models from the catalog");
+}
+export async function runCatalogCommand(_options, _command) {
+    const config = loadBrainConfig();
+    const catalog = loadCatalog();
+    // Metadata-free scan: the catalog join is by path/repo, not GGUF headers, so
+    // skip the header reads for speed.
+    const installed = scanModels(config, process.env, { withMetadata: false });
+    const installedCatalogIds = new Set(installed.map((model) => model.catalogId).filter((id) => Boolean(id)));
+    const rows = catalog.models.map((model) => ({
+        id: model.id,
+        name: model.name,
+        installed: installedCatalogIds.has(model.id),
+        publisher: model.publisher ?? "",
+        repo: model.hfRepo,
+        quant: model.quant,
+        params: model.params ?? "",
+        sizeBytes: model.approxWeightsBytes ?? null,
+        size: model.approxWeightsBytes ? formatBytes(model.approxWeightsBytes) : "",
+        vision: model.vision ?? false,
+        thinking: model.thinking ?? false,
+        contextMax: model.contextMax ?? null,
+        tier: model.tier ?? "",
+        useCases: model.useCases ?? [],
+        why: model.why ?? "",
+    }));
+    return { type: "list", data: rows, schema: catalogSchema };
+}
+//# sourceMappingURL=catalog.js.map

package/dist/commands/config.d.ts

@@ -0,0 +1,13 @@
+/**
+ * `otto brain config show|set` — inspect and edit the persisted config at
+ * $OTTO_HOME/otto-brain/config.json. Opt-in flags (`enabled`, `autoStart`) live
+ * here; the Otto daemon reads them to decide whether to manage the brain.
+ */
+import type { Command } from "commander";
+import type { BrainConfig } from "../config/schema.js";
+import type { AnyCommandResult } from "../output/index.js";
+export declare function addConfigShowOptions(cmd: Command): Command;
+export declare function runConfigShowCommand(_options: unknown, _command: Command): Promise<AnyCommandResult<BrainConfig>>;
+export declare function addConfigSetOptions(cmd: Command): Command;
+export declare function runConfigSetCommand(key: string, value: string, _options: unknown, _command: Command): Promise<AnyCommandResult<BrainConfig>>;
+//# sourceMappingURL=config.d.ts.map

package/dist/commands/config.js

@@ -0,0 +1,90 @@
+import { loadBrainConfig, loadPersistedConfig, saveBrainConfig } from "../config/index.js";
+import { CommandError } from "../output/types.js";
+const configSchema = {
+    idField: () => "config",
+    columns: [{ header: "CONFIG", field: () => "config" }],
+    renderHuman: (data) => JSON.stringify(data, null, 2),
+    serialize: (data) => data,
+};
+export function addConfigShowOptions(cmd) {
+    return cmd.description("Show the effective brain config");
+}
+export async function runConfigShowCommand(_options, _command) {
+    return { type: "single", data: loadBrainConfig(), schema: configSchema };
+}
+function parseValue(raw) {
+    if (raw === "true")
+        return true;
+    if (raw === "false")
+        return false;
+    const num = Number(raw);
+    return Number.isFinite(num) && raw.trim() !== "" ? num : raw;
+}
+/** Apply a dotted key (e.g. listen.port, runtime.source) to a mutable config. */
+function applyKey(config, key, value) {
+    switch (key) {
+        case "enabled":
+            config.enabled = Boolean(value);
+            return;
+        case "autoStart":
+            config.autoStart = Boolean(value);
+            return;
+        case "listen.host":
+            config.listen.host = String(value);
+            return;
+        case "listen.port":
+            config.listen.port = Number(value);
+            return;
+        case "defaultModel":
+            config.defaultModel = String(value);
+            return;
+        case "hfToken":
+            config.hfToken = String(value);
+            return;
+        case "runtime.source":
+            config.runtime.source = value;
+            return;
+        case "auth.mode":
+            config.auth.mode = value;
+            return;
+        case "auth.token":
+            config.auth.token = String(value);
+            return;
+        case "tls.mode":
+            config.tls.mode = value;
+            return;
+        case "tls.certFile":
+            config.tls.certFile = String(value);
+            return;
+        case "tls.keyFile":
+            config.tls.keyFile = String(value);
+            return;
+        case "tls.hostname":
+            config.tls.hostname = String(value);
+            return;
+        case "tls.certDir":
+            config.tls.certDir = String(value);
+            return;
+        case "tls.renewBeforeDays":
+            config.tls.renewBeforeDays = Number(value);
+            return;
+        case "tls.tailscaleExe":
+            config.tls.tailscaleExe = String(value);
+            return;
+        default:
+            throw new CommandError({ code: "UNKNOWN_KEY", message: `unknown config key "${key}"` });
+    }
+}
+export function addConfigSetOptions(cmd) {
+    return cmd
+        .description("Set a config value")
+        .argument("<key>", "config key (e.g. enabled, listen.port, runtime.source)")
+        .argument("<value>", "new value");
+}
+export async function runConfigSetCommand(key, value, _options, _command) {
+    const config = loadPersistedConfig();
+    applyKey(config, key, parseValue(value));
+    saveBrainConfig(config);
+    return { type: "single", data: config, schema: configSchema };
+}
+//# sourceMappingURL=config.js.map

package/dist/commands/lifecycle.d.ts

@@ -0,0 +1,29 @@
+import type { Command } from "commander";
+import type { AnyCommandResult, OutputSchema } from "../output/index.js";
+export declare function addServeOptions(cmd: Command): Command;
+export declare function runServeCommand(options: {
+    model?: string;
+}, _command: Command): Promise<void>;
+export interface LifecycleRow {
+    status: string;
+    pid: number | string;
+    host: string;
+    port: number | string;
+}
+export declare const lifecycleSchema: OutputSchema<LifecycleRow>;
+export declare function addStartOptions(cmd: Command): Command;
+export declare function runStartCommand(options: {
+    model?: string;
+}, command: Command): Promise<AnyCommandResult<LifecycleRow>>;
+export declare function addStopOptions(cmd: Command): Command;
+export declare function runStopCommand(_options: unknown, _command: Command): Promise<AnyCommandResult<LifecycleRow>>;
+export declare function addRestartOptions(cmd: Command): Command;
+export declare function runRestartCommand(options: {
+    model?: string;
+}, command: Command): Promise<AnyCommandResult<LifecycleRow>>;
+export interface StatusRow extends LifecycleRow {
+    health: string;
+}
+export declare function addStatusOptions(cmd: Command): Command;
+export declare function runStatusCommand(_options: unknown, _command: Command): Promise<AnyCommandResult<StatusRow>>;
+//# sourceMappingURL=lifecycle.d.ts.map