@tachikomagundam/abathur 0.2.3 → 0.2.4

package/config/genomes/historian.example.jsonc

@@ -8,6 +8,11 @@
 //
 //   ${ABATHUR_HISTORIAN_REPO}  absolute path of the historian bench repo
 //                              (contains scenarios/, rubric.md, seed_sandbox.sh)
+//                              — repoPath ONLY: the S4 engine seam delivers the
+//                              bench's ACTIVE TREE (incumbent repoPath /
+//                              candidate worktree) to the commands below as the
+//                              {repoRoot} template var, so unit/seed paths must
+//                              NOT bake a materialized repo literal here.
 //   ${ABATHUR_REPO}            absolute path of THIS Abathur checkout
 //                              (hosts graders/historian/*)
 //   ${ABATHUR_WIKI_BASE}       canonical wiki root, e.g. http://localhost:3000
@@ -47,11 +52,11 @@
       { "id": "scenario-08", "path": "scenarios/08-current-state-g5.md", "split": "train" },
       { "id": "scenario-09", "path": "scenarios/09-timeline-week-groups.md", "split": "val" }
     ],
-    "seedCommand": "bash ${ABATHUR_REPO}/graders/historian/seed-wrapped.sh ${ABATHUR_HISTORIAN_REPO}/seed_sandbox.sh",
+    "seedCommand": "bash ${ABATHUR_REPO}/graders/historian/seed-wrapped.sh {repoRoot}/seed_sandbox.sh",
     "resetCommand": "bash ${ABATHUR_REPO}/graders/historian/reset-sandbox.sh",
-    "runCommand": "bash ${ABATHUR_REPO}/graders/historian/run-scenario.sh {unit.id} ${ABATHUR_HISTORIAN_REPO}/{unit.path}",
-    "graderCommand": "node ${ABATHUR_REPO}/graders/historian/grader.mjs {unit.id} ${ABATHUR_HISTORIAN_REPO}/{unit.path} ${ABATHUR_WIKI_BASE}",
-    "agentModel": "bailian-token-plan/qwen3.8-flash",
+    "runCommand": "bash ${ABATHUR_REPO}/graders/historian/run-scenario.sh {unit.id} {repoRoot}/{unit.path}",
+    "graderCommand": "node ${ABATHUR_REPO}/graders/historian/grader.mjs {unit.id} {repoRoot}/{unit.path} ${ABATHUR_WIKI_BASE}",
+    "agentModel": "<provider>/<model-id>", // "<provider>/<model-id> as registered in your opencode config"
     // Real agent runs, not unit tests: 1200s caps one stuck scenario without
     // poisoning the generation (timeout => unscored/inconclusive, excluded by
     // the stats layer). seed+reset hooks share a FIXED 60s adapter budget

package/dist/bench/adapter.js

@@ -10,11 +10,29 @@
 //                 excluded from score distributions (hr discipline), not zeroed.
 import { spawn } from "node:child_process";
 import { cannotAnswer } from "../exit.js";
-export function unitVars(unit, sandboxDir) {
-    return { "unit.path": unit.path, "unit.id": unit.id, sandbox: sandboxDir, workdir: sandboxDir };
+/**
+ * `{repoRoot}` = the bench's ACTIVE TREE: the genome repoPath for the incumbent,
+ * the sealed candidate worktree for a candidate (run-loop.ts swaps spec.repoPath
+ * per target; adapters resolve it through effectiveRepoPath at construction).
+ * Omitting repoRoot keeps `{repoRoot}` an UNKNOWN placeholder — fail-closed
+ * exit 2, never a silently half-substituted command.
+ */
+export function unitVars(unit, sandboxDir, repoRoot) {
+    const vars = {
+        "unit.path": unit.path,
+        "unit.id": unit.id,
+        sandbox: sandboxDir,
+        workdir: sandboxDir,
+    };
+    if (repoRoot !== undefined)
+        vars["repoRoot"] = repoRoot;
+    return vars;
 }
-export function sandboxVars(sandboxDir) {
-    return { sandbox: sandboxDir, workdir: sandboxDir };
+export function sandboxVars(sandboxDir, repoRoot) {
+    const vars = { sandbox: sandboxDir, workdir: sandboxDir };
+    if (repoRoot !== undefined)
+        vars["repoRoot"] = repoRoot;
+    return vars;
 }
 const PLACEHOLDER = /\{([A-Za-z0-9_.]+)\}/g;
 /**

package/dist/bench/fixture.js

@@ -32,6 +32,7 @@ import { cannotAnswer, ExitSignal } from "../exit.js";
 import { resolveConfigDir } from "../config.js";
 import { fingerprint } from "../core/ids.js";
 import { acquireGenomeLock, Ledger } from "../core/ledger.js";
+import { effectiveRepoPath } from "../core/spec.js";
 import { childStatus, firstLine, inconclusive, parseGraderLine, renderCommand, runChild, sandboxVars, unitVars, ZERO_METRICS, } from "./adapter.js";
 import { buildManifest, gateVal, mirrorSandboxHome, parseRunMeta, sandboxHomeDir, transcriptPathFor, writeManifest, } from "./fixture-support.js";
 import { probeEngines, resolveOpencodeBin } from "./fixture-probe.js";
@@ -52,7 +53,10 @@ export class FixtureScenariosAdapter {
         if (spec.bench.type !== "opencode-fixture-scenarios") {
             cannotAnswer(`fixture adapter: bench.type '${spec.bench.type}' is not "opencode-fixture-scenarios"`, "use ToyBenchAdapter for bench.type 'toy'");
         }
-        const root = path.resolve(spec.repoPath);
+        // Active-tree resolution follows the run-loop notion (run-loop.ts:142): a
+        // concrete repoPath resolves exactly as before; a `${VAR}` env literal
+        // resolves at THIS fs boundary or exits 2 naming the variable.
+        const root = effectiveRepoPath(spec.repoPath);
         if (!statSync(root, { throwIfNoEntry: false })?.isDirectory()) {
             cannotAnswer(`fixture: repoPath '${spec.repoPath}' is not a readable directory`);
         }
@@ -84,7 +88,7 @@ export class FixtureScenariosAdapter {
         const transcript = transcriptPathFor(sandboxDir, unit.id);
         mkdirSync(path.dirname(transcript), { recursive: true });
         const outcome = await runChild({
-            argv: renderCommand(this.spec.bench.runCommand, unitVars(unit, sandboxDir)),
+            argv: renderCommand(this.spec.bench.runCommand, unitVars(unit, sandboxDir, this.repoRoot)),
             cwd: sandboxDir,
             timeoutS,
             env: this.sandboxEnv(sandboxDir, unit),
@@ -109,7 +113,7 @@ export class FixtureScenariosAdapter {
             cannotAnswer("fixture adapter: score() called before any reset()/seed()/run() — no sandbox yet", "drive the adapter in reset→seed→run→score order");
         }
         const outcome = await runChild({
-            argv: renderCommand(this.spec.bench.graderCommand, unitVars(unit, sandbox)),
+            argv: renderCommand(this.spec.bench.graderCommand, unitVars(unit, sandbox, this.repoRoot)),
             cwd: sandbox,
             timeoutS: this.spec.bench.timeoutS,
             env: this.sandboxEnv(sandbox),
@@ -211,7 +215,7 @@ export class FixtureScenariosAdapter {
         if (template === undefined)
             return;
         const outcome = await runChild({
-            argv: renderCommand(template, sandboxVars(sandboxDir)),
+            argv: renderCommand(template, sandboxVars(sandboxDir, this.repoRoot)),
             cwd: sandboxDir,
             timeoutS: HOOK_TIMEOUT_S,
             env: this.sandboxEnv(sandboxDir),

package/dist/bench/toy.js

@@ -53,7 +53,7 @@ export class ToyBenchAdapter {
     async run(unit, sandboxDir, timeoutS) {
         this.activeSandbox = sandboxDir;
         const outcome = await runChild({
-            argv: renderCommand(this.spec.bench.runCommand, unitVars(unit, sandboxDir)),
+            argv: renderCommand(this.spec.bench.runCommand, unitVars(unit, sandboxDir, this.repoRoot)),
             cwd: sandboxDir,
             timeoutS,
             ...(this.opts.onChild === undefined ? {} : { onChild: this.opts.onChild }),
@@ -75,7 +75,7 @@ export class ToyBenchAdapter {
             cannotAnswer("toy adapter: score() called before any reset()/seed()/run() — no sandbox yet", "drive the adapter in reset→seed→run→score order");
         }
         const outcome = await runChild({
-            argv: renderCommand(this.spec.bench.graderCommand, unitVars(unit, sandbox)),
+            argv: renderCommand(this.spec.bench.graderCommand, unitVars(unit, sandbox, this.repoRoot)),
             cwd: sandbox,
             timeoutS: this.spec.bench.timeoutS,
             ...(this.opts.onChild === undefined ? {} : { onChild: this.opts.onChild }),
@@ -109,7 +109,7 @@ export class ToyBenchAdapter {
         if (template === undefined)
             return;
         const outcome = await runChild({
-            argv: renderCommand(template, sandboxVars(sandboxDir)),
+            argv: renderCommand(template, sandboxVars(sandboxDir, this.repoRoot)),
             cwd: sandboxDir,
             timeoutS: HOOK_TIMEOUT_S,
             ...(this.opts.onChild === undefined ? {} : { onChild: this.opts.onChild }),

package/dist/commands/status.js

@@ -15,6 +15,7 @@ import { tryGit } from "../util/git.js";
 import { readRegistry } from "../core/genome.js";
 import { LEDGER_KIND_GRAFT_IMPORT, decodeGraftImport, listGraftQueue, } from "../core/graft-support.js";
 import { resolveUniqueEntry } from "./run.js";
+import { effectiveRepoPath } from "../core/spec.js";
 const KIND_GENERATION = "generation_complete";
 const KIND_PROMOTE = "promote";
 const KIND_TOMBSTONE = "tombstone";
@@ -113,7 +114,11 @@ async function runStatus(args) {
         renderQueue(queue);
         return EXIT_OK;
     }
-    const repo = entry.spec.repoPath;
+    // Resolve at the fs seam exactly like run/genome/kernel (run-loop.ts:142):
+    // feeding the stored UNRESOLVED `${VAR}` literal to git/ledger paths makes
+    // Node report the missing cwd as the misleading `spawn git ENOENT`; an unset
+    // variable is now a clean exit 2 that names it.
+    const repo = effectiveRepoPath(entry.spec.repoPath);
     const records = readLedgerRecords(ledgerPath(repo));
     const gens = generations(records);
     writeStdout(`genome: ${flat(entry.label)} (${entry.fingerprint})  repo: ${flat(repo, 200)}`);

package/dist/core/evolve/run-bench.js

@@ -4,6 +4,7 @@ import path from "node:path";
 import { cannotAnswer } from "../../exit.js";
 import { runId } from "../ids.js";
 import { buildManifest } from "../kernel.js";
+import { effectiveRepoPath } from "../spec.js";
 import { budgetExhausted } from "../stats.js";
 import { ToyBenchAdapter } from "../../bench/toy.js";
 import { FixtureScenariosAdapter } from "../../bench/fixture.js";
@@ -164,7 +165,7 @@ export async function benchTarget(o) {
         spent,
         provenance: provenance ?? { benchType: o.spec.bench.type, versions: [] },
         complete,
-        manifest: buildManifest(path.resolve(o.spec.repoPath), o.spec.kernel.immutableGlobs),
+        manifest: buildManifest(effectiveRepoPath(o.spec.repoPath), o.spec.kernel.immutableGlobs),
         failures,
     };
 }

package/dist/test/bench-adapter.test.js

@@ -31,3 +31,27 @@ test("renderCommand: quote-aware argv, placeholders, fail-closed junk (exit 2)",
     expectExit2(() => renderCommand("   ", unitVars(unit, "/s")), "empty");
     expectExit2(() => renderCommand('node "oops', unitVars(unit, "/s")), "unbalanced quote");
 });
+// S4 engine seam: {repoRoot} = the bench's ACTIVE TREE, threaded through the
+// shared CommandVars constructors so both adapters resolve it identically.
+// Without a threaded repoRoot the placeholder must stay FAIL-CLOSED (exit 2),
+// exactly like any unknown placeholder — never a half-substituted command.
+test("{repoRoot}: unitVars/sandboxVars carry the active tree into run/grader/hook templates", () => {
+    const unit = { id: "u1", path: "scenarios/a.md", split: "train" };
+    assert.deepEqual(renderCommand("bash run-scenario.sh {unit.id} {repoRoot}/{unit.path}", unitVars(unit, "/sbx", "/active/tree")), ["bash", "run-scenario.sh", "u1", "/active/tree/scenarios/a.md"]);
+    assert.deepEqual(renderCommand("bash seed-wrapped.sh {repoRoot}/seed_sandbox.sh", sandboxVars("/sbx", "/active/tree")), ["bash", "seed-wrapped.sh", "/active/tree/seed_sandbox.sh"]);
+    // pre-existing keys untouched by the seam
+    assert.deepEqual(unitVars(unit, "/sbx", "/r"), {
+        "unit.path": "scenarios/a.md",
+        "unit.id": "u1",
+        sandbox: "/sbx",
+        workdir: "/sbx",
+        repoRoot: "/r",
+    });
+    assert.deepEqual(sandboxVars("/sbx", "/r"), { sandbox: "/sbx", workdir: "/sbx", repoRoot: "/r" });
+});
+test("{repoRoot} unthreaded ⇒ fail-closed exit 2 (unknown placeholder behavior UNCHANGED)", () => {
+    const unit = { id: "u1", path: "p", split: "train" };
+    expectExit2(() => renderCommand("node x {repoRoot}", unitVars(unit, "/s")), "unknown placeholder");
+    expectExit2(() => renderCommand("node x {repoRoot}", sandboxVars("/s")), "unknown placeholder");
+    expectExit2(() => renderCommand("node x {repoRoot}", { worktree: "/w", brief: "/b" }), "unknown placeholder");
+});

package/dist/test/bench-reporoot.test.js

@@ -0,0 +1,197 @@
+// S4 engine seam — the {repoRoot} command variable. The mutation channel's
+// delivery hinge: run/grader/seed/reset templates must resolve {repoRoot} to
+// the bench's ACTIVE TREE — the genome repoPath for the incumbent, the sealed
+// candidate worktree for a candidate (run-loop.ts swaps `spec.repoPath` before
+// benching each candidate; the adapter must follow that same notion, including
+// effectiveRepoPath's env-literal resolution at the FS boundary).
+// Unknown placeholders stay fail-closed (exit 2). Offline: capture scripts
+// record the rendered argv; the fake opencode only answers --version.
+import assert from "node:assert/strict";
+import { chmodSync, cpSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import * as os from "node:os";
+import path from "node:path";
+import { test } from "node:test";
+import { ExitSignal } from "../exit.js";
+import { parseGenomeSpecDocument } from "../core/spec.js";
+import { FixtureScenariosAdapter } from "../bench/fixture.js";
+// ------------------------------------------------------------------- helpers
+async function freshDir(t, prefix) {
+    const dir = await mkdtemp(path.join(os.tmpdir(), `abathur-${prefix}-`));
+    t.after(() => rm(dir, { recursive: true, force: true }));
+    return dir;
+}
+async function writeScript(dir, name, body) {
+    const filePath = path.join(dir, name);
+    await writeFile(filePath, body, "utf8");
+    chmodSync(filePath, 0o755);
+    return filePath;
+}
+/** Records `argv[mode, out, ...rest]`: rest joined per line into `out`; prints the metric/score line. */
+const CAPTURE = `#!/usr/bin/env node
+import { writeFileSync } from "node:fs";
+const [mode, out, ...rest] = process.argv.slice(2);
+writeFileSync(out, rest.join("\\n") + "\\n", "utf8");
+if (mode === "score") {
+  process.stdout.write(JSON.stringify({ unit: "one", score: 1, pass: true, metrics: { tokensEst: 3, turns: 1 } }) + "\\n");
+} else {
+  process.stdout.write(JSON.stringify({ tokensEst: 3, turns: 1 }) + "\\n");
+}
+`;
+const FAKE_OC = `#!/usr/bin/env bash
+if [ "\${1:-}" = "--version" ]; then echo "1.2.3"; exit 0; fi
+exit 0
+`;
+const UNIT = { id: "one", path: "scenarios/one.md", split: "train" };
+const VAL_UNIT = { id: "v1", path: "scenarios/v1.md", split: "val" };
+async function seamFixture(t, genDir) {
+    const root = await freshDir(t, "seam-ad");
+    const capPath = await writeScript(root, "cap.mjs", CAPTURE);
+    const binDir = await freshDir(t, "seam-bin");
+    const binPath = await writeScript(binDir, "oc-fake", FAKE_OC);
+    const home = await freshDir(t, "seam-home");
+    const configDir = await freshDir(t, "seam-cfg");
+    const outDir = path.join(root, "caps");
+    mkdirSync(outDir, { recursive: true });
+    const out = (name) => path.join(outDir, `${name}.txt`);
+    const spec = parseGenomeSpecDocument({
+        label: "seam-germ",
+        repoPath: genDir,
+        bench: {
+            type: "opencode-fixture-scenarios",
+            units: [UNIT, VAL_UNIT],
+            runCommand: `node ${capPath} run ${out("run")} {repoRoot} {unit.path}`,
+            seedCommand: `node ${capPath} seed ${out("seed")} {repoRoot}`,
+            resetCommand: `node ${capPath} reset ${out("reset")} {repoRoot}`,
+            graderCommand: `node ${capPath} score ${out("grader")} {repoRoot}`,
+            agentModel: "test/agent",
+            timeoutS: 30,
+            stats: { halfWidth: 0.15, minEffect: 0.1, nReps: { initial: 1, max: 2 } },
+        },
+        budget: { maxCandidates: 2, maxModelCalls: 4, maxTokens: 10000, maxWallS: 600 },
+        kernel: { immutableGlobs: [] },
+        requires: [{ cmd: "node", args: ["--version"], probeExit: 0 }],
+        opencodeBinVersion: { minVersion: "1.0.0" },
+    }, "<test>");
+    const sandbox = path.join(root, "sandbox");
+    return {
+        spec,
+        genDir,
+        sandbox,
+        home,
+        cap: capPath,
+        binPath,
+        out,
+        adapter: (s) => new FixtureScenariosAdapter(s, { env: { HOME: home }, home, configDir, opencodeBin: binPath }),
+    };
+}
+/** Minimal genome dir (the fixture adapter only requires repoPath to be a directory). */
+async function genomeTree(t, name = "germ") {
+    const genRoot = await freshDir(t, `seam-${name}`);
+    const genDir = path.join(genRoot, "germ");
+    mkdirSync(path.join(genDir, "scenarios"), { recursive: true });
+    writeFileSync(path.join(genDir, "scenarios", "one.md"), "# one\n\n## Brief\ndo it\n", "utf8");
+    writeFileSync(path.join(genDir, "scenarios", "v1.md"), "# v1\n\n## Brief\ndo it val\n", "utf8");
+    return genDir;
+}
+function readCap(file) {
+    return readFileSync(file, "utf8").trim();
+}
+function isExit2Matching(needle) {
+    return (cause) => cause instanceof ExitSignal && cause.code === 2 && needle.test(cause.message);
+}
+// --------------------------------------------------------------------- pins
+test("{repoRoot} renders to the incumbent genome repoPath in run + grader + hooks", async (t) => {
+    const genDir = await genomeTree(t);
+    const f = await seamFixture(t, genDir);
+    const adapter = f.adapter(f.spec);
+    try {
+        await adapter.reset(f.sandbox);
+        await adapter.seed(f.sandbox);
+        const run = await adapter.run(UNIT, f.sandbox, 30);
+        assert.equal(run.status, "ok");
+        const score = await adapter.score(UNIT);
+        assert.equal(score.kind, "scored");
+        assert.equal(readCap(f.out("run")), `${genDir}\nscenarios/one.md`);
+        assert.equal(readCap(f.out("grader")), genDir);
+        assert.equal(readCap(f.out("seed")), genDir);
+        assert.equal(readCap(f.out("reset")), genDir);
+    }
+    finally {
+        adapter.release();
+    }
+});
+test("{repoRoot} follows the candidate worktree when spec.repoPath is swapped (run-loop.ts pattern)", async (t) => {
+    const genDir = await genomeTree(t);
+    const f = await seamFixture(t, genDir);
+    // simulate run-loop.ts:279 — benchTarget receives { ...spec, repoPath: worktreePath }
+    const worktree = path.join(path.dirname(genDir), "wt");
+    cpSync(genDir, worktree, { recursive: true });
+    mkdirSync(path.join(worktree, "abathur-notes"), { recursive: true });
+    writeFileSync(path.join(worktree, "abathur-notes", "card.md"), "planted run card\n", "utf8");
+    const candidate = f.adapter({ ...f.spec, repoPath: worktree });
+    try {
+        const sandbox = path.join(f.sandbox, "cand");
+        await candidate.reset(sandbox);
+        await candidate.seed(sandbox);
+        const run = await candidate.run(UNIT, sandbox, 30);
+        assert.equal(run.status, "ok");
+        const score = await candidate.score(UNIT);
+        assert.equal(score.kind, "scored");
+        assert.equal(readCap(f.out("run")), `${worktree}\nscenarios/one.md`);
+        assert.equal(readCap(f.out("grader")), worktree);
+    }
+    finally {
+        candidate.release();
+    }
+});
+test("fixture ctor resolves an env-literal repoPath via effectiveRepoPath (run-loop.ts:142 pattern)", async (t) => {
+    const genDir = await genomeTree(t, "lit");
+    const f = await seamFixture(t, genDir);
+    const litSpec = { ...f.spec, repoPath: "${ABATHUR_SEAM_LIT_REPO}" };
+    const base = {
+        env: { HOME: f.home },
+        home: f.home,
+        configDir: path.join(f.home, ".config", "abathur"),
+        opencodeBin: f.binPath,
+    };
+    delete process.env["ABATHUR_SEAM_LIT_REPO"];
+    try {
+        // unset env ⇒ clean exit 2 NAMING the variable (never a garbage-path spawn error)
+        assert.throws(() => new FixtureScenariosAdapter(litSpec, base), isExit2Matching(/requires env var ABATHUR_SEAM_LIT_REPO/));
+        // set env ⇒ resolves at the FS boundary; {repoRoot} renders to the resolved tree
+        process.env["ABATHUR_SEAM_LIT_REPO"] = genDir;
+        const adapter = new FixtureScenariosAdapter(litSpec, base);
+        try {
+            const sandbox = path.join(f.sandbox, "lit");
+            await adapter.reset(sandbox);
+            await adapter.seed(sandbox);
+            const run = await adapter.run(UNIT, sandbox, 30);
+            assert.equal(run.status, "ok");
+            assert.equal(readCap(f.out("run")), `${genDir}\nscenarios/one.md`);
+        }
+        finally {
+            adapter.release();
+        }
+    }
+    finally {
+        delete process.env["ABATHUR_SEAM_LIT_REPO"];
+    }
+});
+test("fail-closed UNCHANGED: unknown placeholder through the adapter is still exit 2", async (t) => {
+    const genDir = await genomeTree(t, "unk");
+    const f = await seamFixture(t, genDir);
+    const bad = {
+        ...f.spec,
+        bench: { ...f.spec.bench, runCommand: `node ${f.cap} run ${f.out("bad")} {nope}` },
+    };
+    const adapter = f.adapter(bad);
+    try {
+        await adapter.reset(f.sandbox);
+        await adapter.seed(f.sandbox);
+        await assert.rejects(() => adapter.run(UNIT, f.sandbox, 30), isExit2Matching(/unknown placeholder/));
+    }
+    finally {
+        adapter.release();
+    }
+});