@tachikomagundam/abathur 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/.github/workflows/ci.yml +29 -0
  2. package/.github/workflows/publish.yml +74 -0
  3. package/LICENSE +21 -0
  4. package/README.md +461 -0
  5. package/config/abathur.jsonc +17 -0
  6. package/config/genomes/historian.example.jsonc +124 -0
  7. package/dist/bench/adapter.js +201 -0
  8. package/dist/bench/fixture-probe.js +92 -0
  9. package/dist/bench/fixture-support.js +173 -0
  10. package/dist/bench/fixture.js +236 -0
  11. package/dist/bench/toy.js +152 -0
  12. package/dist/cli.js +110 -0
  13. package/dist/commands/bundle.js +79 -0
  14. package/dist/commands/genome.js +94 -0
  15. package/dist/commands/graft.js +71 -0
  16. package/dist/commands/kernel.js +47 -0
  17. package/dist/commands/promote.js +25 -0
  18. package/dist/commands/run.js +145 -0
  19. package/dist/commands/self-eval.js +240 -0
  20. package/dist/commands/status.js +186 -0
  21. package/dist/commands/tombstone.js +72 -0
  22. package/dist/config.js +161 -0
  23. package/dist/core/bundle-common.js +119 -0
  24. package/dist/core/bundle-export.js +212 -0
  25. package/dist/core/bundle-inspect.js +143 -0
  26. package/dist/core/bundle-manifest.js +105 -0
  27. package/dist/core/bundle-mask.js +75 -0
  28. package/dist/core/bundle-tar.js +240 -0
  29. package/dist/core/bundle.js +9 -0
  30. package/dist/core/evolve/brief.js +45 -0
  31. package/dist/core/evolve/candidate.js +140 -0
  32. package/dist/core/evolve/child-track.js +197 -0
  33. package/dist/core/evolve/friction.js +150 -0
  34. package/dist/core/evolve/reflect.js +191 -0
  35. package/dist/core/evolve/run-bench.js +170 -0
  36. package/dist/core/evolve/run-friction.js +63 -0
  37. package/dist/core/evolve/run-loop.js +282 -0
  38. package/dist/core/evolve/run-plan.js +39 -0
  39. package/dist/core/evolve/run-rows.js +145 -0
  40. package/dist/core/evolve/self-overlay.js +213 -0
  41. package/dist/core/evolve/self-snapshot.js +170 -0
  42. package/dist/core/evolve/stub-mutators.mjs +105 -0
  43. package/dist/core/evolve/udiff.js +189 -0
  44. package/dist/core/genome-paths.js +76 -0
  45. package/dist/core/genome.js +176 -0
  46. package/dist/core/glob.js +106 -0
  47. package/dist/core/graft-gates.js +184 -0
  48. package/dist/core/graft-rebench.js +187 -0
  49. package/dist/core/graft-support.js +181 -0
  50. package/dist/core/graft.js +218 -0
  51. package/dist/core/ids.js +154 -0
  52. package/dist/core/incumbent.js +46 -0
  53. package/dist/core/kernel.js +112 -0
  54. package/dist/core/ledger.js +198 -0
  55. package/dist/core/locks.js +172 -0
  56. package/dist/core/promote.js +119 -0
  57. package/dist/core/snapshot.js +61 -0
  58. package/dist/core/spec.js +178 -0
  59. package/dist/core/stats-math.js +102 -0
  60. package/dist/core/stats-pareto.js +57 -0
  61. package/dist/core/stats.js +184 -0
  62. package/dist/core/worktree.js +190 -0
  63. package/dist/exit.js +32 -0
  64. package/dist/genomes/toy-smoke/genome.jsonc +30 -0
  65. package/dist/genomes/toy-smoke/grader.mjs +61 -0
  66. package/dist/genomes/toy-smoke/init.mjs +63 -0
  67. package/dist/genomes/toy-smoke/units/add.mjs +17 -0
  68. package/dist/genomes/toy-smoke/units/explode.mjs +4 -0
  69. package/dist/genomes/toy-smoke/units/hang.mjs +16 -0
  70. package/dist/genomes/toy-smoke/units/mul.mjs +16 -0
  71. package/dist/genomes/toy-smoke/units/mutate.mjs +18 -0
  72. package/dist/genomes/toy-smoke/units/sub.mjs +16 -0
  73. package/dist/jsonc.js +77 -0
  74. package/dist/out.js +5 -0
  75. package/dist/test/bench-adapter.test.js +33 -0
  76. package/dist/test/bench-fixture.test.js +407 -0
  77. package/dist/test/bench-toy.test.js +251 -0
  78. package/dist/test/bundle.test.js +659 -0
  79. package/dist/test/config.test.js +185 -0
  80. package/dist/test/d7-gate.test.js +56 -0
  81. package/dist/test/fixture-loop.test.js +267 -0
  82. package/dist/test/fixtures/friction-writer.js +16 -0
  83. package/dist/test/fixtures-historian.js +82 -0
  84. package/dist/test/fixtures-self.js +143 -0
  85. package/dist/test/fixtures-wt.js +64 -0
  86. package/dist/test/friction.test.js +398 -0
  87. package/dist/test/genome.test.js +453 -0
  88. package/dist/test/git.test.js +69 -0
  89. package/dist/test/graft.test.js +567 -0
  90. package/dist/test/historian-genome.test.js +134 -0
  91. package/dist/test/historian-grader-io.test.js +148 -0
  92. package/dist/test/historian-grader.test.js +209 -0
  93. package/dist/test/ids.test.js +116 -0
  94. package/dist/test/include-val.test.js +120 -0
  95. package/dist/test/ledger-lock.test.js +99 -0
  96. package/dist/test/ledger.test.js +102 -0
  97. package/dist/test/promote.test.js +394 -0
  98. package/dist/test/reflect.test.js +410 -0
  99. package/dist/test/run-loop.test.js +433 -0
  100. package/dist/test/self-snapshot.test.js +328 -0
  101. package/dist/test/snapshot.test.js +86 -0
  102. package/dist/test/stats.test.js +423 -0
  103. package/dist/test/stub-mutators.test.js +17 -0
  104. package/dist/test/testutil.js +30 -0
  105. package/dist/test/worktree.test.js +198 -0
  106. package/dist/util/freeze.js +30 -0
  107. package/dist/util/git.js +85 -0
  108. package/docs/federation.md +184 -0
  109. package/docs/immutable-kernel.md +87 -0
  110. package/graders/historian/grader-core.d.mts +53 -0
  111. package/graders/historian/grader-core.mjs +276 -0
  112. package/graders/historian/grader-support.d.mts +57 -0
  113. package/graders/historian/grader-support.mjs +137 -0
  114. package/graders/historian/grader.mjs +113 -0
  115. package/graders/historian/mutate.sh +114 -0
  116. package/graders/historian/reset-sandbox.sh +60 -0
  117. package/graders/historian/run-scenario.sh +49 -0
  118. package/graders/historian/seed-wrapped.sh +32 -0
  119. package/package.json +42 -0
package/dist/jsonc.js ADDED
@@ -0,0 +1,77 @@
1
+ // Minimal dependency-free JSONC reader: // and /* */ comments plus trailing
2
+ // commas outside string literals. Kept separate from config so the parser can
3
+ // be unit-tested in isolation (config tests pin it through this seam).
4
+ /** Index just past the closing quote of the string literal starting at `start`. */
5
+ function findStringEnd(source, start) {
6
+ let i = start + 1;
7
+ while (i < source.length) {
8
+ if (source[i] === "\\") {
9
+ i += 2; // escaped char (incl. escaped quote) — skip both
10
+ continue;
11
+ }
12
+ if (source[i] === '"')
13
+ return i + 1;
14
+ i += 1;
15
+ }
16
+ return source.length; // unterminated — JSON.parse will report it
17
+ }
18
+ function dropComments(source) {
19
+ let out = "";
20
+ let cursor = 0;
21
+ while (cursor < source.length) {
22
+ const char = source[cursor];
23
+ if (char === '"') {
24
+ const end = findStringEnd(source, cursor);
25
+ out += source.slice(cursor, end);
26
+ cursor = end;
27
+ continue;
28
+ }
29
+ if (source.startsWith("//", cursor)) {
30
+ const newline = source.indexOf("\n", cursor);
31
+ cursor = newline === -1 ? source.length : newline; // keep the newline itself
32
+ continue;
33
+ }
34
+ if (source.startsWith("/*", cursor)) {
35
+ const close = source.indexOf("*/", cursor + 2);
36
+ cursor = close === -1 ? source.length : close + 2;
37
+ continue;
38
+ }
39
+ out += char;
40
+ cursor += 1;
41
+ }
42
+ return out;
43
+ }
44
+ function dropTrailingCommas(source) {
45
+ let out = "";
46
+ let cursor = 0;
47
+ while (cursor < source.length) {
48
+ const char = source[cursor];
49
+ if (char === '"') {
50
+ const end = findStringEnd(source, cursor);
51
+ out += source.slice(cursor, end);
52
+ cursor = end;
53
+ continue;
54
+ }
55
+ if (char === ",") {
56
+ let probe = cursor + 1;
57
+ while (probe < source.length && /\s/.test(source[probe] ?? ""))
58
+ probe += 1;
59
+ const next = source[probe];
60
+ if (next === "}" || next === "]") {
61
+ cursor += 1; // drop the comma
62
+ continue;
63
+ }
64
+ }
65
+ out += char;
66
+ cursor += 1;
67
+ }
68
+ return out;
69
+ }
70
+ /** Normalize JSONC to strict JSON text (comments + trailing commas removed). */
71
+ export function stripJsonc(source) {
72
+ return dropTrailingCommas(dropComments(source));
73
+ }
74
+ /** Parse JSONC text; throws SyntaxError on malformed input (caller rewraps). */
75
+ export function parseJsonc(source) {
76
+ return JSON.parse(stripJsonc(source));
77
+ }
package/dist/out.js ADDED
@@ -0,0 +1,5 @@
1
+ // stdout funnel (todo 1 seam): command handlers are pure except for this one line —
2
+ // they never touch process.exitCode / process.exit; the CLI boundary owns exit.
3
+ export function writeStdout(text) {
4
+ process.stdout.write(text.endsWith("\n") ? text : `${text}\n`);
5
+ }
@@ -0,0 +1,33 @@
1
+ // Shared adapter-layer contract (todo 6 reuses these exports): fail-closed
2
+ // command rendering. Pure functions, no fixture needed.
3
+ import assert from "node:assert/strict";
4
+ import { test } from "node:test";
5
+ import { ExitSignal } from "../exit.js";
6
+ import { renderCommand, sandboxVars, unitVars } from "../bench/adapter.js";
7
+ function expectExit2(action, ...needles) {
8
+ try {
9
+ action();
10
+ }
11
+ catch (cause) {
12
+ assert.ok(cause instanceof ExitSignal, `expected ExitSignal, got ${String(cause)}`);
13
+ assert.equal(cause.code, 2);
14
+ for (const needle of needles)
15
+ assert.match(cause.message, new RegExp(needle));
16
+ return;
17
+ }
18
+ assert.fail("expected ExitSignal(2), nothing thrown");
19
+ }
20
+ test("renderCommand: quote-aware argv, placeholders, fail-closed junk (exit 2)", () => {
21
+ const unit = { id: "u1", path: "dir with sp/u.mjs", split: "train" };
22
+ assert.deepEqual(renderCommand("node {unit.path} --id {unit.id}", unitVars(unit, "/sbx")), [
23
+ "node",
24
+ "dir with sp/u.mjs",
25
+ "--id",
26
+ "u1",
27
+ ]);
28
+ assert.deepEqual(renderCommand("sh -c 'echo hi'", sandboxVars("/sbx")), ["sh", "-c", "echo hi"]);
29
+ assert.deepEqual(renderCommand('run "{sandbox}"', sandboxVars("/tmp/s b")), ["run", "/tmp/s b"]);
30
+ expectExit2(() => renderCommand("node {nope}", unitVars(unit, "/s")), "unknown placeholder");
31
+ expectExit2(() => renderCommand(" ", unitVars(unit, "/s")), "empty");
32
+ expectExit2(() => renderCommand('node "oops', unitVars(unit, "/s")), "unbalanced quote");
33
+ });
@@ -0,0 +1,407 @@
1
+ // Todo 6 acceptance pins for the opencode-fixture-scenarios adapter. The whole
2
+ // bench runs against FAKE opencodeBins (bash fixtures written to tmp by these
3
+ // tests): argv-spawned shebang scripts, never a shell string. Pins:
4
+ // (A) 2-unit matrix runs end to end (provenance, metrics, transcript, HOME env);
5
+ // (B) val scenario paths never appear in the mutator-readable manifest, and a
6
+ // val run without includeVal is exit 2 (operator flag gates it);
7
+ // (C) --version missing/garbage/minVersion mismatch ⇒ exit 2 BEFORE any unit;
8
+ // (D) requires[] probes: missing cmd / probeExit mismatch ⇒ exit 2 naming it;
9
+ // (E) hanging unit ⇒ process-group kill at timeoutS, zero orphans (pgrep-level);
10
+ // (F) infra_failed is recorded distinctly, never looks like a score;
11
+ // (G) garbage grader / garbage run stdout ⇒ inconclusive / zeroed metrics, no crash;
12
+ // (H) crash → reset → seed digest == clean start (stale-state proof);
13
+ // (I) single-flight lock: second concurrent adapter gets exit 2 "another bench active";
14
+ // (J) sandbox HOME: ONLY .opencode/{plugin,skills,node_modules} + copied-then-mutated
15
+ // config, never symlinks, real HOME untouched.
16
+ import assert from "node:assert/strict";
17
+ import { spawnSync } from "node:child_process";
18
+ import { chmodSync, existsSync, lstatSync, readFileSync } from "node:fs";
19
+ import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
20
+ import * as os from "node:os";
21
+ import path from "node:path";
22
+ import { test } from "node:test";
23
+ import { ExitSignal } from "../exit.js";
24
+ import { treeDigestAt } from "../core/ids.js";
25
+ import { parseGenomeSpecDocument } from "../core/spec.js";
26
+ import { FixtureScenariosAdapter } from "../bench/fixture.js";
27
+ import { compareSemver, parseSemver, sandboxHomeDir } from "../bench/fixture.js";
28
+ // ------------------------------------------------------------------- helpers
29
+ function keep(t, dir) {
30
+ t.after(() => rm(dir, { recursive: true, force: true }));
31
+ return dir;
32
+ }
33
+ async function freshDir(t, prefix) {
34
+ return keep(t, await mkdtemp(path.join(os.tmpdir(), `abathur-${prefix}-`)));
35
+ }
36
+ async function writeScript(dir, name, body) {
37
+ const filePath = path.join(dir, name);
38
+ await writeFile(filePath, body, "utf8");
39
+ chmodSync(filePath, 0o755);
40
+ return filePath;
41
+ }
42
+ function isExit2(cause) {
43
+ return cause instanceof ExitSignal && cause.code === 2;
44
+ }
45
+ // Fake opencode bins. Contract consumed by the adapter: `--version` prints a
46
+ // semver line; `run <unit.path>` executes the scenario (writes $ABATHUR_TRANSCRIPT,
47
+ // echoes a final JSON metrics line). Hang/no-version/garbage variants probe the
48
+ // adversarial classes.
49
+ const BIN_GOOD = `#!/usr/bin/env bash
50
+ set -euo pipefail
51
+ if [ "\${1:-}" = "--version" ]; then echo "1.2.3"; exit 0; fi
52
+ unit="\${2:-}"
53
+ case "$unit" in
54
+ *poison*) echo dirty > poison.txt ;;
55
+ esac
56
+ if [ -n "\${ABATHUR_TRANSCRIPT:-}" ]; then
57
+ mkdir -p "$(dirname "$ABATHUR_TRANSCRIPT")"
58
+ printf 'transcript %s model=%s judge=%s home=%s\\n' \\
59
+ "$unit" "\${ABATHUR_AGENT_MODEL:-none}" "\${ABATHUR_JUDGE_MODEL:-none}" "$HOME" \\
60
+ > "$ABATHUR_TRANSCRIPT"
61
+ fi
62
+ echo "ran $unit"
63
+ echo '{"tokensEst":123,"turns":4}'
64
+ `;
65
+ const BIN_HANG = `#!/usr/bin/env bash
66
+ if [ "\${1:-}" = "--version" ]; then echo "1.2.3"; exit 0; fi
67
+ exec sleep 31.7
68
+ `;
69
+ const BIN_NO_VERSION = `#!/usr/bin/env bash
70
+ if [ "\${1:-}" = "--version" ]; then echo "unsupported flag" >&2; exit 1; fi
71
+ echo "ran \${2:-}"
72
+ `;
73
+ const BIN_GARBAGE_VERSION = `#!/usr/bin/env bash
74
+ if [ "\${1:-}" = "--version" ]; then echo "banana split deluxe"; exit 0; fi
75
+ echo "ran \${2:-}"
76
+ `;
77
+ const BIN_GARBAGE_RUN = `#!/usr/bin/env bash
78
+ if [ "\${1:-}" = "--version" ]; then echo "1.2.3"; exit 0; fi
79
+ echo 'garbage {{{ not json at all'
80
+ `;
81
+ /** Genome repo: opaque scenario files + reset/seed scripts + sandbox grader. */
82
+ async function fixtureGenome(t) {
83
+ const genDir = path.join(await freshDir(t, "fixgen"), "germ");
84
+ await mkdir(genDir, { recursive: true });
85
+ await writeFile(path.join(genDir, "scenarios-one.md"), "alpha train scenario\n", "utf8");
86
+ await writeFile(path.join(genDir, "scenarios-two.md"), "beta train scenario\n", "utf8");
87
+ await writeFile(path.join(genDir, "scenarios-poison.md"), "gamma state-mutating train scenario\n", "utf8");
88
+ await writeFile(path.join(genDir, "scenarios-val-canary-ZZHIDDEN.md"), "holdout val scenario ZZHIDDEN\n", "utf8");
89
+ await writeScript(genDir, "reset.sh", `#!/usr/bin/env bash
90
+ set -euo pipefail
91
+ rm -rf "\${1:?sandbox}"
92
+ mkdir -p "\${1:?sandbox}"
93
+ `);
94
+ await writeScript(genDir, "seed.sh", `#!/usr/bin/env bash
95
+ set -euo pipefail
96
+ here=$(cd "$(dirname "$0")" && pwd)
97
+ sandbox="\${1:?sandbox}"
98
+ mkdir -p "$sandbox/scenarios"
99
+ cp "$here"/scenarios-*.md "$sandbox/scenarios/"
100
+ cp "$here"/grader.mjs "$sandbox/grader.mjs"
101
+ touch "$sandbox/seeded.flag"
102
+ `);
103
+ await writeFile(path.join(genDir, "grader.mjs"), `const [unitId] = process.argv.slice(2);
104
+ const table = {
105
+ one: { score: 1, pass: true },
106
+ two: { score: 0.5, pass: false },
107
+ poison: { score: 0, pass: false },
108
+ val: { score: 0.25, pass: false },
109
+ };
110
+ const row = table[unitId];
111
+ if (!row) { console.error("unknown unit " + unitId); process.exit(1); }
112
+ console.log(JSON.stringify({ unit: unitId, score: row.score, pass: row.pass, metrics: { tokensEst: 42, turns: 2 } }));
113
+ `, "utf8");
114
+ return genDir;
115
+ }
116
+ const TRAIN_UNITS = [
117
+ { id: "one", path: "scenarios/scenarios-one.md", split: "train" },
118
+ { id: "two", path: "scenarios/scenarios-two.md", split: "train" },
119
+ ];
120
+ const VAL_UNIT = {
121
+ id: "val",
122
+ path: "scenarios/scenarios-val-canary-ZZHIDDEN.md",
123
+ split: "val",
124
+ };
125
+ const POISON_UNIT = {
126
+ id: "poison",
127
+ path: "scenarios/scenarios-poison.md",
128
+ split: "train",
129
+ };
130
+ function fixtureSpec(genDir, binPath, over = {}) {
131
+ return parseGenomeSpecDocument({
132
+ label: "fixture-germ",
133
+ repoPath: genDir,
134
+ bench: {
135
+ type: "opencode-fixture-scenarios",
136
+ units: [...TRAIN_UNITS, VAL_UNIT],
137
+ runCommand: over.runCommand ?? `${binPath} run {unit.path}`,
138
+ seedCommand: `${genDir}/seed.sh {sandbox}`,
139
+ resetCommand: `${genDir}/reset.sh {sandbox}`,
140
+ graderCommand: "node grader.mjs {unit.id}",
141
+ judgeCommand: "node judge.mjs {unit.id}",
142
+ judgeModel: "test/judge",
143
+ agentModel: "test/agent",
144
+ timeoutS: over.timeoutS ?? 5,
145
+ stats: { halfWidth: 0.1, minEffect: 0.05, nReps: { initial: 1, max: 2 } },
146
+ },
147
+ budget: { maxCandidates: 2, maxModelCalls: 10, maxTokens: 1000, maxWallS: 60 },
148
+ kernel: { immutableGlobs: [] },
149
+ requires: over.requires ?? [{ cmd: "node", args: ["--version"], probeExit: 0 }],
150
+ opencodeBinVersion: { minVersion: over.minVersion ?? "1.0.0" },
151
+ }, "<test>");
152
+ }
153
+ /** Home with the mirrored subset, decoys that must never be copied, and a config. */
154
+ async function fakeHome(t) {
155
+ const home = await freshDir(t, "fixhome");
156
+ for (const rel of [
157
+ ".opencode/plugin/x.js",
158
+ ".opencode/skills/s.md",
159
+ ".opencode/node_modules/m/package.json",
160
+ ".opencode/secrets.txt",
161
+ ".bashrc",
162
+ ".config/opencode/opencode.json",
163
+ ".config/opencode/prompts/p.md",
164
+ ]) {
165
+ const p = path.join(home, rel);
166
+ await mkdir(path.dirname(p), { recursive: true });
167
+ await writeFile(p, rel === ".config/opencode/opencode.json" ? '{"theme":"dark"}\n' : `${rel}\n`, "utf8");
168
+ }
169
+ return home;
170
+ }
171
+ async function adapterFixture(t, body = BIN_GOOD, over = {}, opts = {}) {
172
+ const binDir = await freshDir(t, "fixbin");
173
+ const binPath = await writeScript(binDir, "oc-fake", body);
174
+ const genDir = await fixtureGenome(t);
175
+ const home = await fakeHome(t);
176
+ const configDir = path.join(home, ".config", "abathur");
177
+ const spec = fixtureSpec(genDir, binPath, over);
178
+ const sandbox = path.join(await freshDir(t, "fixsbx"), "sb");
179
+ const adapter = new FixtureScenariosAdapter(spec, {
180
+ env: { HOME: home },
181
+ home,
182
+ configDir,
183
+ opencodeBin: binPath,
184
+ ...opts,
185
+ });
186
+ return { adapter, spec, sandbox, home, configDir };
187
+ }
188
+ function manifestPath(sandbox) {
189
+ return path.join(sandbox, ".bench", "manifest.json");
190
+ }
191
+ function scored(outcome) {
192
+ if (outcome.kind === "scored")
193
+ return outcome.result;
194
+ assert.fail(`expected scored, got inconclusive: ${outcome.reason}`);
195
+ }
196
+ // ------------------------------------------------------------------ AC (A)
197
+ test("(A) 2-unit run: statuses, provenance versions, metrics, transcript, HOME env", async (t) => {
198
+ const f = await adapterFixture(t, BIN_GOOD, {}, { includeVal: true });
199
+ await f.adapter.reset(f.sandbox);
200
+ await f.adapter.seed(f.sandbox);
201
+ const rows = [];
202
+ for (const unit of [TRAIN_UNITS[0], VAL_UNIT]) {
203
+ rows.push({
204
+ run: await f.adapter.run(unit, f.sandbox, f.spec.bench.timeoutS),
205
+ score: await f.adapter.score(unit),
206
+ });
207
+ }
208
+ for (const { run } of rows) {
209
+ assert.equal(run.status, "ok");
210
+ assert.equal(run.exitCode, 0);
211
+ assert.equal(run.benchProvenance.benchType, "opencode-fixture-scenarios");
212
+ const versions = new Map(run.benchProvenance.versions.map((v) => [v.bin, v.version]));
213
+ assert.equal(versions.get("node"), process.version);
214
+ }
215
+ const [a, b] = rows;
216
+ assert.equal(scored(a.score).score, 1);
217
+ assert.equal(scored(b.score).score, 0.25);
218
+ assert.equal(scored(b.score).pass, false);
219
+ assert.equal(a.run.metrics.tokensEst, 123); // parsed from the fake bin's run metadata
220
+ assert.equal(a.run.metrics.turns, 4);
221
+ const sbHome = sandboxHomeDir(f.sandbox);
222
+ const transcript = readFileSync(a.run.transcriptPath ?? "", "utf8");
223
+ assert.match(transcript, /model=test\/agent/);
224
+ assert.match(transcript, /judge=test\/judge/);
225
+ assert.ok(transcript.includes(`home=${sbHome}`), `transcript home must be the sandbox HOME: ${transcript}`);
226
+ // the observed opencode version string is present in EVERY RunResult provenance
227
+ for (const { run } of rows) {
228
+ assert.ok(run.benchProvenance.versions.some((v) => v.version === "1.2.3" && v.bin.includes("oc-fake")), "observed fake-bin version missing from provenance");
229
+ }
230
+ // seed copied the scenario files
231
+ assert.ok(existsSync(path.join(f.sandbox, "seeded.flag")));
232
+ });
233
+ // ------------------------------------------------------------------ AC (B)
234
+ test("(B) val paths hidden from the mutator manifest; val run needs includeVal", async (t) => {
235
+ const f = await adapterFixture(t);
236
+ await f.adapter.reset(f.sandbox);
237
+ await f.adapter.seed(f.sandbox);
238
+ const raw = readFileSync(manifestPath(f.sandbox), "utf8");
239
+ assert.ok(!raw.includes("ZZHIDDEN"), "canary scenario filename leaked into manifest");
240
+ assert.ok(!raw.includes("val-canary"), "val path leaked into manifest");
241
+ assert.ok(raw.includes("scenarios/scenarios-one.md"), "train path must stay visible");
242
+ const entries = JSON.parse(raw)["scenarios"];
243
+ const valEntry = entries.find((e) => e.split === "val");
244
+ assert.ok(valEntry !== undefined, "manifest must still count the val scenario");
245
+ assert.equal(valEntry.id, undefined);
246
+ assert.equal(valEntry.path, undefined);
247
+ assert.match(valEntry.alias, /^scenario-\d{2}$/);
248
+ // default adapter: running the val unit is a tool error naming the operator flag
249
+ await assert.rejects(() => f.adapter.run(VAL_UNIT, f.sandbox, f.spec.bench.timeoutS), (cause) => isExit2(cause) && /--include-val/.test(cause.message));
250
+ // operator-flagged adapter may run it end to end
251
+ const op = await adapterFixture(t, BIN_GOOD, {}, { includeVal: true });
252
+ await op.adapter.reset(op.sandbox);
253
+ await op.adapter.seed(op.sandbox);
254
+ const run = await op.adapter.run(VAL_UNIT, op.sandbox, op.spec.bench.timeoutS);
255
+ assert.equal(run.status, "ok");
256
+ assert.ok(readFileSync(manifestPath(op.sandbox), "utf8").includes("ZZHIDDEN")); // operator sees the truth
257
+ });
258
+ // ------------------------------------------------------------------ AC (C)
259
+ test("(C1) --version unsupported ⇒ exit 2 before ANY unit executed", async (t) => {
260
+ const f = await adapterFixture(t, BIN_NO_VERSION);
261
+ await assert.rejects(() => f.adapter.reset(f.sandbox), isExit2);
262
+ assert.ok(!existsSync(manifestPath(f.sandbox)), "manifest must never be written pre-probe");
263
+ const marker = spawnSync("pgrep", ["-f", "oc-fake run"], { encoding: "utf8" });
264
+ assert.equal((marker.stdout ?? "").trim(), "", "a unit ran despite the failed probe");
265
+ });
266
+ test("(C2) garbage --version output ⇒ exit 2, message names the parse failure", async (t) => {
267
+ const f = await adapterFixture(t, BIN_GARBAGE_VERSION);
268
+ await assert.rejects(() => f.adapter.reset(f.sandbox), (cause) => isExit2(cause) && /banana/.test(cause.message));
269
+ });
270
+ test("(C3) minVersion mismatch ⇒ exit 2 quoting observed and required", async (t) => {
271
+ const f = await adapterFixture(t, BIN_GOOD, { minVersion: "9.9.9" });
272
+ await assert.rejects(() => f.adapter.reset(f.sandbox), (cause) => isExit2(cause) && /1\.2\.3/.test(cause.message) && /9\.9\.9/.test(cause.message));
273
+ });
274
+ // ------------------------------------------------------------------ AC (D)
275
+ test("(D1) missing requires[] binary ⇒ exit 2 naming the prerequisite", async (t) => {
276
+ const f = await adapterFixture(t, BIN_GOOD, {
277
+ requires: [{ cmd: "abathur-no-such-engine", probeExit: 0 }],
278
+ });
279
+ await assert.rejects(() => f.adapter.reset(f.sandbox), (cause) => isExit2(cause) && /abathur-no-such-engine/.test(cause.message));
280
+ });
281
+ test("(D2) requires probeExit mismatch ⇒ exit 2", async (t) => {
282
+ const f = await adapterFixture(t, BIN_GOOD, {
283
+ requires: [{ cmd: "node", args: ["--version"], probeExit: 3 }],
284
+ });
285
+ await assert.rejects(() => f.adapter.reset(f.sandbox), isExit2);
286
+ });
287
+ // ------------------------------------------------------------------ AC (E)
288
+ test("(E) hanging unit: group kill at timeoutS, zero orphans", async (t) => {
289
+ const f = await adapterFixture(t, BIN_HANG, { timeoutS: 1 });
290
+ await f.adapter.reset(f.sandbox);
291
+ await f.adapter.seed(f.sandbox);
292
+ const started = Date.now();
293
+ const run = await f.adapter.run(TRAIN_UNITS[0], f.sandbox, 1);
294
+ assert.ok(Date.now() - started < 8000);
295
+ assert.equal(run.status, "timeout");
296
+ assert.equal(run.exitCode, null);
297
+ assert.match(run.note ?? "", /killed after 1s: process group SIGKILL/);
298
+ const ps = spawnSync("ps", ["-eo", "args"], { encoding: "utf8" });
299
+ assert.ok(!ps.stdout.includes("sleep 31.7"), "orphaned sleep survived the group kill");
300
+ });
301
+ // ------------------------------------------------------------------ AC (F)
302
+ test("(F) infra_failed is recorded with a distinct status/note, never a silent 0", async (t) => {
303
+ const f = await adapterFixture(t, BIN_GOOD, {
304
+ runCommand: "abathur-no-such-bin run {unit.path}",
305
+ });
306
+ await f.adapter.reset(f.sandbox);
307
+ await f.adapter.seed(f.sandbox);
308
+ const run = await f.adapter.run(TRAIN_UNITS[0], f.sandbox, 5);
309
+ assert.equal(run.status, "infra_failed");
310
+ assert.equal(run.exitCode, null);
311
+ assert.match(run.note ?? "", /spawn failed/);
312
+ });
313
+ // ------------------------------------------------------------------ AC (G)
314
+ test("(G1) garbage run stdout ⇒ ok with zeroed metrics, no parse crash", async (t) => {
315
+ const f = await adapterFixture(t, BIN_GARBAGE_RUN);
316
+ await f.adapter.reset(f.sandbox);
317
+ await f.adapter.seed(f.sandbox);
318
+ const run = await f.adapter.run(TRAIN_UNITS[0], f.sandbox, 5);
319
+ assert.equal(run.status, "ok");
320
+ assert.deepEqual(run.metrics, { tokensEst: 0, turns: 0 });
321
+ });
322
+ test("(G2) failing / garbage grader ⇒ inconclusive-for-unit, never a crash", async (t) => {
323
+ const f = await adapterFixture(t);
324
+ await f.adapter.reset(f.sandbox);
325
+ await f.adapter.seed(f.sandbox);
326
+ const unknown = await f.adapter.score({ id: "ghost", path: "scenarios/none.md", split: "train" });
327
+ assert.equal(unknown.kind, "inconclusive");
328
+ if (unknown.kind === "inconclusive")
329
+ assert.match(unknown.reason, /unknown unit ghost/);
330
+ await writeFile(path.join(f.sandbox, "grader.mjs"), 'process.stdout.write("not json at all\\n");\n', "utf8");
331
+ const bad = await f.adapter.score(TRAIN_UNITS[0]);
332
+ assert.equal(bad.kind, "inconclusive");
333
+ if (bad.kind === "inconclusive")
334
+ assert.match(bad.reason, /not a score JSON line/);
335
+ });
336
+ // ------------------------------------------------------------------ AC (H)
337
+ test("(H) crash→reset→seed restores the exact clean-start digest", async (t) => {
338
+ const f = await adapterFixture(t);
339
+ await f.adapter.reset(f.sandbox);
340
+ await f.adapter.seed(f.sandbox);
341
+ const clean = treeDigestAt(f.sandbox);
342
+ const run = await f.adapter.run(POISON_UNIT, f.sandbox, 5);
343
+ assert.equal(run.status, "ok");
344
+ assert.ok(existsSync(path.join(f.sandbox, "poison.txt")), "poison scenario must dirty the sandbox");
345
+ assert.notEqual(treeDigestAt(f.sandbox), clean);
346
+ await f.adapter.reset(f.sandbox);
347
+ await f.adapter.seed(f.sandbox);
348
+ assert.ok(!existsSync(path.join(f.sandbox, "poison.txt")));
349
+ assert.equal(treeDigestAt(f.sandbox), clean);
350
+ });
351
+ // ------------------------------------------------------------------ AC (I)
352
+ test("(I) single-flight: second adapter exit 2 'another bench active', ok after release", async (t) => {
353
+ const binDir = await freshDir(t, "fixbin");
354
+ const binPath = await writeScript(binDir, "oc-fake", BIN_GOOD);
355
+ const genDir = await fixtureGenome(t);
356
+ const home = await fakeHome(t);
357
+ const configDir = path.join(home, ".config", "abathur");
358
+ const spec = fixtureSpec(genDir, binPath);
359
+ const a = new FixtureScenariosAdapter(spec, { env: { HOME: home }, home, configDir, opencodeBin: binPath });
360
+ const b = new FixtureScenariosAdapter(spec, { env: { HOME: home }, home, configDir, opencodeBin: binPath });
361
+ const sandboxA = path.join(await freshDir(t, "fixsbx"), "sb-a");
362
+ await a.reset(sandboxA); // holds the fingerprint-keyed lease
363
+ const sandboxB = path.join(await freshDir(t, "fixsbx"), "sb-b");
364
+ await assert.rejects(() => b.reset(sandboxB), (cause) => isExit2(cause) && /another bench active/.test(cause.message));
365
+ a.release();
366
+ await b.reset(sandboxB); // lease free again
367
+ b.release();
368
+ });
369
+ // ------------------------------------------------------------------ AC (J)
370
+ test("(J) sandbox HOME copies only the .opencode subset + mutated config copy; real HOME untouched", async (t) => {
371
+ const f = await adapterFixture(t);
372
+ const openDigest = treeDigestAt(path.join(f.home, ".opencode"));
373
+ const cfgDigest = treeDigestAt(path.join(f.home, ".config", "opencode"));
374
+ await f.adapter.reset(f.sandbox);
375
+ await f.adapter.seed(f.sandbox);
376
+ const sbHome = sandboxHomeDir(f.sandbox);
377
+ assert.ok(existsSync(path.join(sbHome, ".opencode", "plugin", "x.js")));
378
+ assert.ok(existsSync(path.join(sbHome, ".opencode", "skills", "s.md")));
379
+ assert.ok(existsSync(path.join(sbHome, ".opencode", "node_modules", "m", "package.json")));
380
+ assert.ok(!existsSync(path.join(sbHome, ".opencode", "secrets.txt")), "secrets must not be mirrored");
381
+ assert.ok(!existsSync(path.join(sbHome, ".bashrc")), "dotfiles outside the subset must not be mirrored");
382
+ const cfg = path.join(sbHome, ".config", "opencode", "opencode.json");
383
+ assert.ok(lstatSync(cfg).isFile(), "opencode config must be a copied regular file, never a symlink");
384
+ const doc = JSON.parse(readFileSync(cfg, "utf8"));
385
+ assert.equal(doc["model"], "test/agent"); // mutated per scenario
386
+ assert.equal(doc["theme"], "dark"); // copied content preserved
387
+ assert.ok(existsSync(path.join(sbHome, ".config", "opencode", "prompts", "p.md")));
388
+ assert.equal(treeDigestAt(path.join(f.home, ".opencode")), openDigest, "real .opencode must be untouched");
389
+ assert.equal(treeDigestAt(path.join(f.home, ".config", "opencode")), cfgDigest, "real opencode config must be untouched");
390
+ const realCfg = JSON.parse(readFileSync(path.join(f.home, ".config", "opencode", "opencode.json"), "utf8"));
391
+ assert.equal(realCfg["model"], undefined);
392
+ });
393
+ // -------------------------------------------------------------- semver edges
394
+ test("semver helpers: 1.10 > 1.9, prerelease < release, garbage ⇒ null", () => {
395
+ const cmp = (a, b) => {
396
+ const va = parseSemver(a);
397
+ const vb = parseSemver(b);
398
+ if (va === null || vb === null)
399
+ throw new Error(`unparsable: ${a} / ${b}`);
400
+ return compareSemver(va, vb);
401
+ };
402
+ assert.ok(cmp("1.10.0", "1.9.0") > 0);
403
+ assert.equal(cmp("1.2.3", "1.2.3"), 0);
404
+ assert.ok(cmp("1.2.3-beta", "1.2.3") < 0);
405
+ assert.ok(cmp("v2.0.0", "1.99.99") > 0);
406
+ assert.equal(parseSemver("banana"), null);
407
+ });