@otto-code/brain 0.7.5

package/dist/ops/results.d.ts

@@ -0,0 +1,207 @@
+import type { GpuInfo, Model, ModelFeatures } from "../types.js";
+import type { Profile } from "../config/schema.js";
+declare const RESULTS_DIR: string;
+/** Aggregate of a numeric health series (nvidia-smi samples during a run). */
+export interface AggStat {
+    avg: number;
+    min: number;
+    max: number;
+}
+/** System health summary stored alongside a run's score. */
+export interface SystemHealth {
+    samples: number;
+    gpuUtilPct: AggStat | null;
+    tempC: AggStat | null;
+    powerW: AggStat | null;
+    clockMHz: AggStat | null;
+    vramUsedMiB: AggStat | null;
+    cpuPct: AggStat | null;
+    ramUsedBytes: AggStat | null;
+    thermalThrottle: boolean;
+    powerThrottle: boolean;
+}
+/** One depth-scaling measurement point (from the context-depth task detail). */
+export interface DepthPoint {
+    promptTokens?: number;
+    ttftSeconds?: number;
+    generatePerSecond?: number;
+}
+/** One graded task inside a live benchmark report (input to `save`). */
+export interface BenchTaskResult {
+    id: string;
+    category: string;
+    weight: number;
+    score: number;
+    summary: string;
+    seconds: number;
+    detail?: unknown;
+    error?: string | null;
+}
+/** A completed benchmark report, as produced by the bench suite. */
+export interface BenchReport {
+    overall: number;
+    grade: string;
+    seconds: number;
+    executedCode: boolean;
+    results: BenchTaskResult[];
+    vramBytes?: number | null;
+    loadSeconds?: number | null;
+    archiveId?: string | null;
+    system?: SystemHealth | null;
+}
+/** The model identity persisted with a run. */
+export interface RecordModel {
+    id: string | null;
+    displayName: string;
+    quant: string | null;
+    arch: string | null;
+    sizeBytes: number | null;
+    publisher: string | null;
+    features: ModelFeatures | null;
+}
+/** The subset of a profile persisted with a run. */
+export interface RecordProfile {
+    contextSize: number;
+    cacheTypeK: string;
+    cacheTypeV: string;
+    reasoningBudget: number;
+    vision: boolean;
+    flashAttention: boolean;
+}
+/** The GPU identity persisted with a run. */
+export interface RecordGpu {
+    name: string;
+    totalBytes: number;
+    driver: string;
+}
+/** One task's score persisted with a run. */
+export interface RecordTask {
+    id: string;
+    category: string;
+    weight: number;
+    score: number;
+    summary: string;
+    seconds: number;
+    detail: unknown;
+    error: string | null;
+}
+/** One stored benchmark run — the shared shape used across results and report. */
+export interface RunRecord {
+    schema: number;
+    ranAt: string;
+    model: RecordModel;
+    profile: RecordProfile | null;
+    configKey: string;
+    archiveId: string | null;
+    gpu: RecordGpu | null;
+    system: SystemHealth | null;
+    runtime: string | null;
+    overall: number;
+    grade: string;
+    seconds: number;
+    executedCode: boolean;
+    vramBytes: number | null;
+    loadSeconds: number | null;
+    tasks: RecordTask[];
+    file?: string;
+    runIndex?: number;
+}
+/** Arguments to `save`. */
+export interface SaveOptions {
+    model: Model | null;
+    profile: Profile | null;
+    report: BenchReport;
+    gpu?: GpuInfo | null;
+    runtime?: string | null;
+    archiveId?: string | null;
+    system?: SystemHealth | null;
+    timestamp?: Date;
+}
+/** Result of `save`: the written file path plus the persisted record. */
+export interface SaveResult {
+    file: string;
+    record: RunRecord;
+}
+/** count / mean / sample-std / min / max of a numeric series. */
+export interface Stats {
+    count: number;
+    mean: number;
+    std: number;
+    min: number;
+    max: number;
+}
+/** A model+config grouping of runs. */
+export interface Group {
+    key: string;
+    model: RecordModel;
+    configKey: string;
+    runs: RunRecord[];
+    count: number;
+}
+/** Per-task variance stats within a group (category plus optional stats). */
+export interface VarianceTask {
+    category: string;
+    count?: number;
+    mean?: number;
+    std?: number;
+    min?: number;
+    max?: number;
+}
+/** Per-config spread of overall and per-task scores across repeated runs. */
+export interface VarianceRow {
+    key: string;
+    model: RecordModel;
+    configKey: string;
+    count: number;
+    overall: Stats | null;
+    tasks: Record<string, VarianceTask>;
+}
+/** One ranked entry per model, scored by the mean of its runs. */
+export interface RankedModel {
+    id: string | null;
+    displayName: string;
+    overall: number;
+    runs: number;
+    std: number;
+    grade: string;
+    rank?: number;
+}
+/** A task column: an id paired with its display category. */
+export interface TaskColumn {
+    id: string;
+    category: string;
+}
+declare function slugify(text: string): string;
+/** Stable identity for "same model, same settings", so reruns can be grouped. */
+declare function configKey(profile: Profile | null): string;
+declare function save({ model, profile, report, gpu, runtime, archiveId, system, timestamp, }: SaveOptions): SaveResult;
+/** Every stored run, newest first. */
+declare function loadAll(): RunRecord[];
+/**
+ * Collapse history to the most recent run per model+config, which is what you
+ * want when comparing: earlier runs of the same setup are superseded.
+ */
+declare function latestPerConfig(records?: RunRecord[]): RunRecord[];
+/**
+ * Group every run by model+config (all runs kept, not just the latest), and
+ * number them oldest→newest so a run can be plotted against run number. Wall
+ * time is deliberately not the axis — only how many times we have measured it.
+ */
+declare function grouped(records?: RunRecord[]): Group[];
+/** count / mean / sample-std / min / max of a numeric series. */
+declare function stats(values: Array<number | undefined>): Stats | null;
+/**
+ * Per-config spread of overall and per-task scores across repeated runs — the
+ * measure of whether the benchmark method is itself consistent.
+ */
+declare function variance(records?: RunRecord[]): VarianceRow[];
+/**
+ * One ranked entry per model, scored by the MEAN of all its runs (so a single
+ * noisy or throttled run cannot set a model's rank), best first. `runs` and
+ * `std` travel with it so the app can show how well-backed a ranking is.
+ */
+declare function rankModels(records?: RunRecord[]): RankedModel[];
+/** All task ids seen across a set of records, in a stable order. */
+declare function taskColumns(records: RunRecord[]): TaskColumn[];
+export { RESULTS_DIR, save, loadAll, latestPerConfig, grouped, variance, stats, rankModels, taskColumns, configKey, slugify, };
+//# sourceMappingURL=results.d.ts.map

package/dist/ops/results.js

@@ -0,0 +1,250 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+/**
+ * Persistent benchmark history.
+ *
+ * One JSON file per run, so results accumulate across sessions and can be
+ * compared and charted later. Runs record the configuration they were measured
+ * under, because a score is meaningless without the quant, context size and
+ * reasoning budget that produced it.
+ */
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.resolve(HERE, "..", "..");
+const RESULTS_DIR = path.join(ROOT, "results");
+function slugify(text) {
+    return String(text)
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-|-$/g, "")
+        .slice(0, 80);
+}
+/** Stable identity for "same model, same settings", so reruns can be grouped. */
+function configKey(profile) {
+    if (!profile)
+        return "unknown";
+    return [
+        `ctx${profile.contextSize}`,
+        `kv${profile.cacheTypeK}-${profile.cacheTypeV}`,
+        `rb${profile.reasoningBudget}`,
+        profile.vision ? "vision" : "novision",
+    ].join("_");
+}
+function save({ model, profile, report, gpu = null, runtime = null, archiveId = null, system = null, timestamp = new Date(), }) {
+    fs.mkdirSync(RESULTS_DIR, { recursive: true });
+    const record = {
+        schema: 1,
+        ranAt: timestamp.toISOString(),
+        model: {
+            id: model?.id ?? null,
+            displayName: model?.displayName ?? "unknown",
+            quant: model?.quant ?? null,
+            arch: model?.metadata?.arch ?? null,
+            sizeBytes: model?.sizeBytes ?? null,
+            publisher: model?.publisher ?? null,
+            features: model?.features ?? null,
+        },
+        profile: profile
+            ? {
+                contextSize: profile.contextSize,
+                cacheTypeK: profile.cacheTypeK,
+                cacheTypeV: profile.cacheTypeV,
+                reasoningBudget: profile.reasoningBudget,
+                vision: profile.vision,
+                flashAttention: profile.flashAttention,
+            }
+            : null,
+        configKey: configKey(profile),
+        // Links this result to its stored transcripts so `rescore` can re-grade it.
+        archiveId: archiveId ?? report.archiveId ?? null,
+        gpu: gpu ? { name: gpu.name, totalBytes: gpu.totalBytes, driver: gpu.driver } : null,
+        // System health sampled during the run, so a score can be read against the
+        // machine state that produced it (throttling, power cap, VRAM pressure).
+        system: system ?? report.system ?? null,
+        runtime,
+        overall: report.overall,
+        grade: report.grade,
+        seconds: report.seconds,
+        executedCode: report.executedCode,
+        vramBytes: report.vramBytes ?? null,
+        loadSeconds: report.loadSeconds ?? null,
+        tasks: report.results.map((r) => ({
+            id: r.id,
+            category: r.category,
+            weight: r.weight,
+            score: r.score,
+            summary: r.summary,
+            seconds: r.seconds,
+            detail: r.detail ?? null,
+            error: r.error ?? null,
+        })),
+    };
+    const stamp = record.ranAt.replace(/[:.]/g, "-");
+    const file = path.join(RESULTS_DIR, `${stamp}_${slugify(record.model.displayName)}.json`);
+    fs.writeFileSync(file, `${JSON.stringify(record, null, 2)}\n`);
+    return { file, record };
+}
+/** Every stored run, newest first. */
+function loadAll() {
+    if (!fs.existsSync(RESULTS_DIR))
+        return [];
+    return fs
+        .readdirSync(RESULTS_DIR)
+        .filter((f) => f.endsWith(".json"))
+        .map((f) => {
+        try {
+            const record = JSON.parse(fs.readFileSync(path.join(RESULTS_DIR, f), "utf8"));
+            return { ...record, file: f };
+        }
+        catch {
+            return null;
+        }
+    })
+        .filter((r) => Boolean(r))
+        .sort((a, b) => (a.ranAt < b.ranAt ? 1 : -1));
+}
+/**
+ * Collapse history to the most recent run per model+config, which is what you
+ * want when comparing: earlier runs of the same setup are superseded.
+ */
+function latestPerConfig(records = loadAll()) {
+    const best = new Map();
+    for (const record of records) {
+        const key = `${record.model.displayName}::${record.configKey}`;
+        if (!best.has(key))
+            best.set(key, record);
+    }
+    return [...best.values()].sort((a, b) => b.overall - a.overall);
+}
+/**
+ * Group every run by model+config (all runs kept, not just the latest), and
+ * number them oldest→newest so a run can be plotted against run number. Wall
+ * time is deliberately not the axis — only how many times we have measured it.
+ */
+function grouped(records = loadAll()) {
+    const groups = new Map();
+    for (const record of records) {
+        const key = `${record.model.displayName}::${record.configKey}`;
+        let group = groups.get(key);
+        if (!group) {
+            group = { key, model: record.model, configKey: record.configKey, runs: [], count: 0 };
+            groups.set(key, group);
+        }
+        group.runs.push(record);
+    }
+    for (const group of groups.values()) {
+        // loadAll is newest-first; number oldest→newest for a left-to-right x-axis.
+        [...group.runs].reverse().forEach((run, i) => {
+            run.runIndex = i + 1;
+        });
+        group.count = group.runs.length;
+    }
+    return [...groups.values()];
+}
+/** count / mean / sample-std / min / max of a numeric series. */
+function stats(values) {
+    const nums = values.filter((v) => typeof v === "number" && Number.isFinite(v));
+    if (!nums.length)
+        return null;
+    const mean = nums.reduce((a, b) => a + b, 0) / nums.length;
+    const varc = nums.length > 1 ? nums.reduce((a, b) => a + (b - mean) ** 2, 0) / (nums.length - 1) : 0;
+    return {
+        count: nums.length,
+        mean,
+        std: Math.sqrt(varc),
+        min: Math.min(...nums),
+        max: Math.max(...nums),
+    };
+}
+/**
+ * Per-config spread of overall and per-task scores across repeated runs — the
+ * measure of whether the benchmark method is itself consistent.
+ */
+function variance(records = loadAll()) {
+    return grouped(records)
+        .map((group) => {
+        const overall = stats(group.runs.map((r) => r.overall));
+        const taskMeta = new Map();
+        for (const run of group.runs) {
+            for (const task of run.tasks)
+                if (!taskMeta.has(task.id))
+                    taskMeta.set(task.id, task.category);
+        }
+        const tasks = {};
+        for (const [id, category] of taskMeta) {
+            tasks[id] = {
+                category,
+                ...stats(group.runs.map((r) => r.tasks.find((t) => t.id === id)?.score)),
+            };
+        }
+        return {
+            key: group.key,
+            model: group.model,
+            configKey: group.configKey,
+            count: group.count,
+            overall,
+            tasks,
+        };
+    })
+        .sort((a, b) => (b.overall?.mean ?? 0) - (a.overall?.mean ?? 0));
+}
+function gradeFor(score) {
+    if (score >= 0.9)
+        return "excellent";
+    if (score >= 0.75)
+        return "strong";
+    if (score >= 0.55)
+        return "usable";
+    if (score >= 0.35)
+        return "weak";
+    return "unusable";
+}
+/**
+ * One ranked entry per model, scored by the MEAN of all its runs (so a single
+ * noisy or throttled run cannot set a model's rank), best first. `runs` and
+ * `std` travel with it so the app can show how well-backed a ranking is.
+ */
+function rankModels(records = loadAll()) {
+    const byModel = new Map();
+    for (const record of records) {
+        const key = record.model.id || record.model.displayName;
+        if (!byModel.has(key)) {
+            byModel.set(key, {
+                id: record.model.id ?? null,
+                displayName: record.model.displayName,
+                scores: [],
+            });
+        }
+        byModel.get(key)?.scores.push(record.overall);
+    }
+    const ranked = [...byModel.values()]
+        .map((m) => {
+        const s = stats(m.scores) ?? { count: 0, mean: 0, std: 0, min: 0, max: 0 };
+        return {
+            id: m.id,
+            displayName: m.displayName,
+            overall: s.mean,
+            runs: s.count,
+            std: s.std,
+            grade: gradeFor(s.mean),
+        };
+    })
+        .sort((a, b) => b.overall - a.overall);
+    ranked.forEach((m, i) => {
+        m.rank = i + 1;
+    });
+    return ranked;
+}
+/** All task ids seen across a set of records, in a stable order. */
+function taskColumns(records) {
+    const seen = new Map();
+    for (const record of records) {
+        for (const task of record.tasks) {
+            if (!seen.has(task.id))
+                seen.set(task.id, task.category);
+        }
+    }
+    return [...seen.entries()].map(([id, category]) => ({ id, category }));
+}
+export { RESULTS_DIR, save, loadAll, latestPerConfig, grouped, variance, stats, rankModels, taskColumns, configKey, slugify, };
+//# sourceMappingURL=results.js.map

package/dist/ops/sweep.d.ts

@@ -0,0 +1,77 @@
+import { Supervisor } from "../service/supervisor.js";
+import type { Model, Runtime } from "../types.js";
+import type { Profile } from "../config/schema.js";
+/**
+ * Find the reasoning budget that delivers the most useful output for a model.
+ *
+ * Thinking models default to an unrestricted budget (-1) and will happily burn
+ * an entire token allowance reasoning, returning no content at all. The right
+ * cap is model-specific, so measure it: run one long-horizon task per candidate
+ * budget and score by delivered content per second.
+ */
+export declare const DEFAULT_BUDGETS: number[];
+export declare const LONG_TASK: string;
+export declare const EXPECTED_FILES: string[];
+/** The measured outcome of a single generation trial. */
+export interface Trial {
+    finishReason: string | null;
+    outputTokens: number | null;
+    contentChars: number;
+    reasoningChars: number;
+    filesDelivered: number;
+    elapsedSeconds: number;
+    tokensPerSecond: number | null;
+    contentPerSecond: number;
+}
+interface RunTrialOptions {
+    supervisor: Supervisor;
+    maxTokens: number;
+    temperature: number;
+}
+export declare function runTrial({ supervisor, maxTokens, temperature, }: RunTrialOptions): Promise<Trial>;
+/** Progress event emitted while sweeping reasoning budgets. */
+export interface SweepProgress {
+    phase: "loading" | "generating" | "done" | "failed";
+    budget: number;
+    error?: string;
+    finishReason?: string | null;
+    outputTokens?: number | null;
+    contentChars?: number;
+    reasoningChars?: number;
+    filesDelivered?: number;
+    elapsedSeconds?: number;
+    tokensPerSecond?: number | null;
+    contentPerSecond?: number;
+}
+/** A single budget's trial, plus its error (null on success). */
+export interface SweepResult {
+    budget: number;
+    error: string | null;
+    finishReason?: string | null;
+    outputTokens?: number | null;
+    contentChars: number;
+    reasoningChars?: number;
+    filesDelivered: number;
+    elapsedSeconds?: number;
+    tokensPerSecond?: number | null;
+    contentPerSecond: number;
+}
+export interface SweepReport {
+    results: SweepResult[];
+    recommended: number | null;
+    ranked: SweepResult[];
+    sweptAt: string;
+}
+export interface SweepOptions {
+    runtime: Runtime;
+    model: Model;
+    profile: Profile;
+    budgets?: number[];
+    maxTokens?: number;
+    temperature?: number;
+    internalPort?: number;
+    onProgress?: (event: SweepProgress) => void;
+}
+export declare function sweep({ runtime, model, profile, budgets, maxTokens, temperature, internalPort, onProgress, }: SweepOptions): Promise<SweepReport>;
+export {};
+//# sourceMappingURL=sweep.d.ts.map

package/dist/ops/sweep.js

@@ -0,0 +1,126 @@
+import { request } from "node:http";
+import { DEFAULT_INTERNAL_PORT, Supervisor } from "../service/supervisor.js";
+/**
+ * Find the reasoning budget that delivers the most useful output for a model.
+ *
+ * Thinking models default to an unrestricted budget (-1) and will happily burn
+ * an entire token allowance reasoning, returning no content at all. The right
+ * cap is model-specific, so measure it: run one long-horizon task per candidate
+ * budget and score by delivered content per second.
+ */
+export const DEFAULT_BUDGETS = [0, 512, 1536, 3072, -1];
+export const LONG_TASK = "Write a complete Python implementation of a thread-safe LRU cache with TTL " +
+    "expiry. Produce FOUR separate complete files, each fully implemented with no " +
+    "placeholders or elisions:\n" +
+    "1. lru.py - the cache with get/put/delete/clear, OrderedDict-based, RLock, " +
+    "per-entry TTL, and a background sweeper thread\n" +
+    "2. metrics.py - hit/miss/eviction counters with a snapshot() method\n" +
+    "3. test_lru.py - at least 12 unittest cases covering eviction order, TTL " +
+    "expiry, concurrent access, and edge cases\n" +
+    "4. README.md - full usage documentation with examples\n" +
+    "Write every file out in full. Do not abbreviate anything.";
+export const EXPECTED_FILES = ["lru.py", "metrics.py", "test_lru.py", "README.md"];
+function postJson({ host, port, path: urlPath, payload, timeoutMs = 900000, }) {
+    return new Promise((resolve, reject) => {
+        const body = JSON.stringify(payload);
+        const req = request({
+            host,
+            port,
+            path: urlPath,
+            method: "POST",
+            headers: { "content-type": "application/json", "content-length": Buffer.byteLength(body) },
+            timeout: timeoutMs,
+        }, (res) => {
+            let text = "";
+            res.on("data", (c) => (text += c));
+            res.on("end", () => {
+                try {
+                    resolve(JSON.parse(text));
+                }
+                catch (error) {
+                    reject(error);
+                }
+            });
+        });
+        req.on("timeout", () => {
+            req.destroy();
+            reject(new Error("request timed out"));
+        });
+        req.on("error", reject);
+        req.end(body);
+    });
+}
+export async function runTrial({ supervisor, maxTokens, temperature, }) {
+    const started = Date.now();
+    const result = (await postJson({
+        host: supervisor.host,
+        port: supervisor.internalPort,
+        path: "/v1/chat/completions",
+        payload: {
+            messages: [{ role: "user", content: LONG_TASK }],
+            max_tokens: maxTokens,
+            temperature,
+            top_k: 20,
+            top_p: 0.95,
+        },
+    }));
+    const elapsedSeconds = (Date.now() - started) / 1000;
+    const choice = result.choices?.[0];
+    const message = choice?.message;
+    const content = message?.content || "";
+    const reasoning = message?.reasoning_content || "";
+    return {
+        finishReason: choice?.finish_reason ?? null,
+        outputTokens: result.usage?.completion_tokens ?? null,
+        contentChars: content.length,
+        reasoningChars: reasoning.length,
+        filesDelivered: EXPECTED_FILES.filter((name) => content.includes(name)).length,
+        elapsedSeconds,
+        tokensPerSecond: result.timings?.predicted_per_second ?? null,
+        // The metric that matters: useful output per unit of wall time.
+        contentPerSecond: elapsedSeconds > 0 ? content.length / elapsedSeconds : 0,
+    };
+}
+export async function sweep({ runtime, model, profile, budgets = DEFAULT_BUDGETS, maxTokens = 8192, temperature = 0.7, internalPort = DEFAULT_INTERNAL_PORT + 2, onProgress = () => { }, }) {
+    const results = [];
+    for (const budget of budgets) {
+        const supervisor = new Supervisor({ runtime, internalPort });
+        onProgress({ phase: "loading", budget });
+        try {
+            await supervisor.start(model, { ...profile, reasoningBudget: budget });
+            onProgress({ phase: "generating", budget });
+            const trial = await runTrial({ supervisor, maxTokens, temperature });
+            results.push({ budget, ...trial, error: null });
+            onProgress({ phase: "done", budget, ...trial });
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            results.push({
+                budget,
+                error: message,
+                contentChars: 0,
+                contentPerSecond: 0,
+                filesDelivered: 0,
+            });
+            onProgress({ phase: "failed", budget, error: message });
+        }
+        finally {
+            await supervisor.stop();
+            await new Promise((resolve) => setTimeout(resolve, 2500));
+        }
+    }
+    // Prefer runs that delivered every file; break ties on content per second.
+    const viable = results.filter((r) => !r.error && r.contentChars > 0);
+    const ranked = [...viable].sort((a, b) => {
+        if (b.filesDelivered !== a.filesDelivered)
+            return b.filesDelivered - a.filesDelivered;
+        return b.contentPerSecond - a.contentPerSecond;
+    });
+    return {
+        results,
+        recommended: ranked.length ? ranked[0].budget : null,
+        ranked,
+        sweptAt: new Date().toISOString(),
+    };
+}
+//# sourceMappingURL=sweep.js.map

package/dist/output/index.d.ts

@@ -0,0 +1,5 @@
+/** Barrel for the output layer. */
+export * from "./types.js";
+export { renderResult, renderError } from "./render.js";
+export { withOutput } from "./with-output.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/output/index.js

@@ -0,0 +1,5 @@
+/** Barrel for the output layer. */
+export * from "./types.js";
+export { renderResult, renderError } from "./render.js";
+export { withOutput } from "./with-output.js";
+//# sourceMappingURL=index.js.map

package/dist/output/render.d.ts

@@ -0,0 +1,4 @@
+import type { AnyCommandResult, CommandErrorShape, OutputOptions } from "./types.js";
+export declare function renderResult<T>(result: AnyCommandResult<T>, options: OutputOptions): string;
+export declare function renderError(error: CommandErrorShape, options: OutputOptions): string;
+//# sourceMappingURL=render.d.ts.map