@otto-code/brain 0.7.5

package/dist/bench/repo.js

@@ -0,0 +1,195 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import crypto from "node:crypto";
+import { execFile } from "node:child_process";
+/**
+ * Repository control for SWE-bench-style tasks.
+ *
+ * One working copy is reused for every task rather than re-cloning: reset the
+ * tracked files, keep the ignored ones. That is deliberate and cheap -
+ * `git clean -fd` (crucially WITHOUT -x) removes the model's stray files while
+ * preserving node_modules and dist. Measured on otto-code: reset under a
+ * second, warm install 1.7s, build 6.1s, full protocol suite 4.3s.
+ *
+ * Tasks therefore serialise on the working copy, which costs nothing real:
+ * only one model fits in VRAM at a time, so they were already serialised.
+ */
+const DEFAULT_TIMEOUT = 300000;
+export function run(command, args, { cwd, timeout = DEFAULT_TIMEOUT, env } = {}) {
+    return new Promise((resolve) => {
+        execFile(command, args, {
+            cwd,
+            timeout,
+            windowsHide: true,
+            maxBuffer: 32 * 1024 * 1024,
+            env: env || process.env,
+            shell: process.platform === "win32", // npm/npx are .cmd shims on Windows
+        }, (error, stdout, stderr) => {
+            resolve({
+                ok: !error,
+                code: error?.code ?? 0,
+                timedOut: Boolean(error?.killed),
+                stdout: String(stdout),
+                stderr: String(stderr),
+            });
+        });
+    });
+}
+export class Repo {
+    /**
+     * @param {object} options
+     * @param {string} options.dir           working copy
+     * @param {string} options.workspace     npm workspace name, e.g. @otto-code/protocol
+     * @param {string} options.workspaceDir  its path relative to dir
+     */
+    constructor({ dir, workspace, workspaceDir }) {
+        this.dir = path.resolve(dir);
+        this.workspace = workspace;
+        this.workspaceDir = workspaceDir;
+        this.lastLockHash = null;
+    }
+    git(args, options = {}) {
+        return run("git", args, { cwd: this.dir, ...options });
+    }
+    /** Discard all local modifications and untracked files, keeping ignored ones. */
+    async reset(sha) {
+        const hard = await this.git(["reset", "--hard", sha]);
+        if (!hard.ok)
+            throw new Error(`git reset --hard ${sha} failed: ${hard.stderr.trim()}`);
+        // No -x: node_modules and dist are ignored and must survive.
+        const clean = await this.git(["clean", "-fd"]);
+        if (!clean.ok)
+            throw new Error(`git clean failed: ${clean.stderr.trim()}`);
+        return true;
+    }
+    /** Bring specific paths in from another commit - used to apply the oracle. */
+    async checkoutPaths(sha, paths) {
+        if (!paths.length)
+            return true;
+        const result = await this.git(["checkout", sha, "--", ...paths]);
+        if (!result.ok)
+            throw new Error(`git checkout ${sha} -- <paths> failed: ${result.stderr.trim()}`);
+        return true;
+    }
+    async headSha() {
+        const result = await this.git(["rev-parse", "HEAD"]);
+        return result.stdout.trim();
+    }
+    /**
+     * Install only when the lockfile actually changed since the last install.
+     * Across neighbouring commits this is almost always a no-op.
+     */
+    async ensureDependencies() {
+        const lock = path.join(this.dir, "package-lock.json");
+        let hash = "none";
+        try {
+            hash = crypto.createHash("sha1").update(fs.readFileSync(lock)).digest("hex");
+        }
+        catch {
+            /* no lockfile: fall through and install once */
+        }
+        if (hash === this.lastLockHash)
+            return { installed: false };
+        // --ignore-scripts: the root `prepare` hook runs lefthook (git hooks),
+        // which fails here and is not wanted in a benchmark harness anyway.
+        const result = await run("npm", ["install", "--ignore-scripts", "--no-audit", "--no-fund"], {
+            cwd: this.dir,
+            timeout: 900000,
+        });
+        if (!result.ok) {
+            throw new Error(`npm install failed: ${(result.stderr || result.stdout).slice(-500)}`);
+        }
+        this.lastLockHash = hash;
+        return { installed: true };
+    }
+    /**
+     * Build the workspace. Required, not optional: some tests import through the
+     * package's own `exports` map into dist/, and fail at import time otherwise.
+     */
+    async build() {
+        const result = await run("npm", ["run", "build", "--workspace", this.workspace], {
+            cwd: this.dir,
+            timeout: 600000,
+        });
+        return {
+            ok: result.ok,
+            output: (result.stderr || result.stdout).slice(-2000),
+        };
+    }
+    /**
+     * Run the workspace test suite and return per-test outcomes.
+     *
+     * `files` restricts the run to those test paths (passed to vitest as filters).
+     * The SWE-bench oracle only ever scores its own test files, so scoping to them
+     * avoids running the whole workspace suite - faster, and it keeps the harness
+     * within the "never run the full suite" rule.
+     *
+     * vitest's --outputFile needs a native path; a POSIX-style /tmp path silently
+     * produces no file on Windows.
+     */
+    async test({ timeout = 600000, files, } = {}) {
+        const outFile = path.join(os.tmpdir(), `vitest-${process.pid}-${Date.now()}.json`);
+        const cwd = path.join(this.dir, this.workspaceDir);
+        const args = ["vitest", "run", "--reporter=json", `--outputFile=${outFile}`];
+        if (files && files.length)
+            args.push(...files);
+        const result = await run("npx", args, { cwd, timeout });
+        let report;
+        try {
+            report = JSON.parse(fs.readFileSync(outFile, "utf8"));
+        }
+        catch {
+            return {
+                ok: false,
+                parsed: false,
+                passed: new Set(),
+                failed: new Set(),
+                fileFailures: [],
+                error: (result.stderr || result.stdout).slice(-1500),
+            };
+        }
+        finally {
+            fs.rmSync(outFile, { force: true });
+        }
+        return { ...parseVitest(report, this.dir), ok: result.ok, parsed: true };
+    }
+}
+/**
+ * Turn a vitest JSON report into pass/fail sets keyed by a stable test id.
+ *
+ * A file that fails to import has no assertion results at all - those are
+ * recorded separately, because "no tests ran" is not the same as "tests failed"
+ * and must not be mistaken for a passing baseline.
+ */
+export function parseVitest(report, repoDir) {
+    const passed = new Set();
+    const failed = new Set();
+    const fileFailures = [];
+    for (const suite of report.testResults || []) {
+        const file = path.relative(repoDir, suite.name || "").replace(/\\/g, "/");
+        const assertions = suite.assertionResults || [];
+        if (!assertions.length && suite.status !== "passed") {
+            fileFailures.push({ file, message: (suite.message || "").slice(0, 500) });
+            continue;
+        }
+        for (const assertion of assertions) {
+            const id = `${file}::${assertion.fullName}`;
+            if (assertion.status === "passed")
+                passed.add(id);
+            else
+                failed.add(id);
+        }
+    }
+    return {
+        passed,
+        failed,
+        fileFailures,
+        totals: {
+            total: report.numTotalTests ?? passed.size + failed.size,
+            passed: report.numPassedTests ?? passed.size,
+            failed: report.numFailedTests ?? failed.size,
+        },
+    };
+}
+//# sourceMappingURL=repo.js.map

package/dist/bench/rescore.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Re-grade archived runs with the current scorer.
+ *
+ * Only tasks whose score is a pure function of the stored exchange can be
+ * replayed. Timing-based tasks (depth scaling) cannot - their result depends on
+ * the hardware state at the time - and are reported as not re-scorable rather
+ * than silently reused or zeroed.
+ */
+declare const RESCORABLE: Set<string>;
+/** A re-scored task entry within a run. */
+export interface RescoreTaskResult {
+    taskId: string;
+    score: number | null;
+    summary: string;
+    detail?: unknown;
+    skipped?: boolean;
+}
+/** The re-scored result of one archived run. */
+export interface RescoreRunResult {
+    archiveId: string;
+    tasks: RescoreTaskResult[];
+}
+/** Progress event emitted while re-scoring. */
+export interface RescoreProgress {
+    phase: "start" | "done";
+    archiveId?: string;
+    tasks?: RescoreTaskResult[];
+}
+/**
+ * @returns {{archiveId:string, tasks:Array, changed:Array}}
+ */
+export declare function rescoreRun(archiveId: string, { execute }?: {
+    execute?: boolean;
+}): Promise<RescoreRunResult>;
+/** Re-score every archived run and report where the new scorer disagrees. */
+export declare function rescoreAll({ execute, onProgress, }?: {
+    execute?: boolean;
+    onProgress?: (event: RescoreProgress) => void;
+}): Promise<RescoreRunResult[]>;
+export { RESCORABLE };
+//# sourceMappingURL=rescore.d.ts.map

package/dist/bench/rescore.js

@@ -0,0 +1,127 @@
+import * as archive from "../ops/archive.js";
+import { TOOL_CASES } from "./tasks.js";
+import { verifyPython, verifyToolCall } from "./verify.js";
+/**
+ * Re-grade archived runs with the current scorer.
+ *
+ * Only tasks whose score is a pure function of the stored exchange can be
+ * replayed. Timing-based tasks (depth scaling) cannot - their result depends on
+ * the hardware state at the time - and are reported as not re-scorable rather
+ * than silently reused or zeroed.
+ */
+const RESCORABLE = new Set(["long-horizon", "tool-calling"]);
+const LONG_HORIZON_FILES = ["lru.py", "metrics.py", "test_lru.py"];
+function firstUserContent(request) {
+    const messages = request?.messages || [];
+    const user = messages.find((m) => m.role === "user");
+    return typeof user?.content === "string" ? user.content : "";
+}
+async function rescoreLongHorizon(exchanges, { execute }) {
+    const last = exchanges[exchanges.length - 1];
+    const choice = last?.response?.choices?.[0] || {};
+    const content = choice.message?.content || "";
+    const reasoning = choice.message?.reasoning_content || "";
+    if (!content) {
+        return {
+            score: 0,
+            summary: reasoning ? `no content - all tokens went to reasoning` : "empty response",
+        };
+    }
+    const verification = await verifyPython(content, {
+        expectedFiles: LONG_HORIZON_FILES,
+        execute,
+    });
+    const fileScore = (LONG_HORIZON_FILES.length - verification.filesMissing.length) / LONG_HORIZON_FILES.length;
+    const compileScore = verification.compiled ? 1 : 0;
+    let testScore = 0;
+    if (verification.testsRun && verification.testsTotal) {
+        testScore = (verification.testsPassed ?? 0) / verification.testsTotal;
+    }
+    else if (!execute && verification.compiled) {
+        testScore = null;
+    }
+    const parts = [fileScore * 0.3, compileScore * 0.3];
+    let weight = 0.6;
+    if (testScore !== null) {
+        parts.push(testScore * 0.4);
+        weight = 1.0;
+    }
+    const penalty = Math.min(0.2, verification.placeholders * 0.1);
+    const score = Math.max(0, parts.reduce((a, b) => a + b, 0) / weight - penalty);
+    const bits = [
+        `${LONG_HORIZON_FILES.length - verification.filesMissing.length}/${LONG_HORIZON_FILES.length} files`,
+        verification.compiled ? "compiles" : "syntax errors",
+    ];
+    if (verification.testsRun)
+        bits.push(`${verification.testsPassed}/${verification.testsTotal} tests pass`);
+    else if (!execute)
+        bits.push("tests not executed");
+    if (verification.nameCollisions?.length)
+        bits.push(`${verification.nameCollisions.length} name collisions`);
+    return { score, summary: bits.join(", ") };
+}
+function rescoreToolCalling(exchanges) {
+    const outcomes = [];
+    for (const exchange of exchanges) {
+        const prompt = firstUserContent(exchange.request);
+        // Match the archived exchange back to its case by prompt text, so a
+        // reordering of TOOL_CASES cannot silently mis-grade history.
+        const testCase = TOOL_CASES.find((c) => c.prompt === prompt);
+        if (!testCase)
+            continue;
+        const message = exchange.response?.choices?.[0]?.message || {};
+        const verdict = verifyToolCall(message.tool_calls, testCase.expect);
+        outcomes.push({ id: testCase.id, ok: verdict.ok, reason: verdict.reason || null });
+    }
+    if (!outcomes.length)
+        return { score: null, summary: "no matching cases in archive" };
+    const passed = outcomes.filter((o) => o.ok).length;
+    const failed = outcomes
+        .filter((o) => !o.ok)
+        .map((o) => `${o.id.replace(/^tools\//, "")} (${o.reason ?? "failed"})`);
+    return {
+        score: passed / outcomes.length,
+        summary: `${passed}/${outcomes.length} tool calls correct` +
+            (failed.length ? ` - missed: ${failed.join("; ")}` : ""),
+        detail: outcomes,
+    };
+}
+/**
+ * @returns {{archiveId:string, tasks:Array, changed:Array}}
+ */
+export async function rescoreRun(archiveId, { execute = true } = {}) {
+    const byTask = archive.load(archiveId);
+    const tasks = [];
+    for (const [taskId, exchanges] of Object.entries(byTask)) {
+        if (!RESCORABLE.has(taskId)) {
+            tasks.push({
+                taskId,
+                score: null,
+                summary: "not re-scorable from a transcript",
+                skipped: true,
+            });
+            continue;
+        }
+        if (taskId === "long-horizon") {
+            tasks.push({ taskId, ...(await rescoreLongHorizon(exchanges, { execute })) });
+        }
+        else if (taskId === "tool-calling") {
+            tasks.push({ taskId, ...rescoreToolCalling(exchanges) });
+        }
+    }
+    return { archiveId, tasks };
+}
+/** Re-score every archived run and report where the new scorer disagrees. */
+export async function rescoreAll({ execute = true, onProgress = () => { }, } = {}) {
+    const runs = archive.list();
+    const results = [];
+    for (const id of runs) {
+        onProgress({ phase: "start", archiveId: id });
+        const result = await rescoreRun(id, { execute });
+        results.push(result);
+        onProgress({ phase: "done", ...result });
+    }
+    return results;
+}
+export { RESCORABLE };
+//# sourceMappingURL=rescore.js.map

package/dist/bench/tasks.d.ts

@@ -0,0 +1,124 @@
+import type { ToolCall, ToolCallExpectation } from "./verify.js";
+/**
+ * Benchmark tasks for agentic coding on this machine.
+ *
+ * Every task scores 0..1 from something checkable - a tool call with the right
+ * name and arguments, code the interpreter accepts, tests that actually pass -
+ * rather than a subjective read of the output.
+ */
+/** A single JSON-schema parameter of a tool. */
+export interface ToolParameterSpec {
+    type: string;
+    description?: string;
+}
+/** The parameters object of a tool function. */
+export interface ToolFunctionParameters {
+    type: string;
+    properties: Record<string, ToolParameterSpec>;
+    required: string[];
+}
+/** An OpenAI-style function tool. */
+export interface ToolFunction {
+    name: string;
+    description: string;
+    parameters: ToolFunctionParameters;
+}
+/** A tool exposed to the model. */
+export interface Tool {
+    type: string;
+    function: ToolFunction;
+}
+/** A message sent to the chat endpoint. */
+export interface ChatRequestMessage {
+    role: string;
+    content: string | null;
+    tool_calls?: ToolCall[];
+    tool_call_id?: string;
+}
+/** The payload POSTed to /v1/chat/completions. */
+export interface ChatPayload {
+    messages: ChatRequestMessage[];
+    tools?: Tool[];
+    max_tokens?: number;
+    temperature?: number;
+}
+/** A message returned by the model. */
+export interface ChatMessage {
+    role?: string;
+    content?: string | null;
+    reasoning_content?: string;
+    tool_calls?: ToolCall[];
+}
+/** Token accounting reported by the endpoint. */
+export interface ChatUsage {
+    prompt_tokens?: number;
+    completion_tokens?: number;
+}
+/** Timing information reported by llama-server. */
+export interface ChatTimings {
+    prompt_ms?: number;
+    prompt_per_second?: number;
+    predicted_per_second?: number;
+}
+/** One choice in a chat completion response. */
+export interface ChatChoice {
+    message?: ChatMessage;
+    finish_reason?: string;
+}
+/** A chat completion response (with an `error` slot for a failed request). */
+export interface ChatResponse {
+    choices?: ChatChoice[];
+    usage?: ChatUsage;
+    timings?: ChatTimings;
+    error?: string;
+}
+/** The function each task uses to talk to the endpoint. */
+export interface ChatFn {
+    (payload: ChatPayload): Promise<ChatResponse>;
+}
+/** Context handed to each task's run(). */
+export interface TaskRunContext {
+    chat: ChatFn;
+    execute?: boolean;
+    depths?: number[];
+    concurrency?: number;
+    /**
+     * The model's reasoning-token budget, so tasks can size their response cap
+     * above it. A thinking model given a max_tokens smaller than its budget spends
+     * the whole allowance reasoning and returns empty content. Null/unset means
+     * unknown (e.g. an arbitrary --endpoint), and tasks fall back to a safe cap.
+     */
+    reasoningBudget?: number | null;
+    /**
+     * The model's loaded context window (tokens). Long-horizon tasks report peak
+     * prompt-token usage as a fraction of this, so a run says not just whether the
+     * model fixed the bug but how much context it held while doing it. Null/unset
+     * means unknown (e.g. an arbitrary --endpoint).
+     */
+    contextWindow?: number | null;
+}
+/** What a task returns after running. */
+export interface TaskOutcome {
+    score: number;
+    summary: string;
+    detail?: unknown;
+}
+/** A single benchmark task. */
+export interface Task {
+    id: string;
+    category: string;
+    weight: number;
+    description: string;
+    run(ctx: TaskRunContext): Promise<TaskOutcome>;
+}
+/** A single tool-calling test case. */
+export interface ToolCase {
+    id: string;
+    prompt: string;
+    expect: ToolCallExpectation;
+}
+export declare const TOOLS: Tool[];
+export declare const TOOL_CASES: ToolCase[];
+export declare const LONG_TASK_PROMPT: string;
+export declare const TASKS: Task[];
+//# sourceMappingURL=tasks.d.ts.map