@tachikomagundam/abathur 0.2.4 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/config/genomes/historian.example.jsonc +1 -1
  2. package/dist/commands/genome.js +2 -2
  3. package/dist/commands/graft.js +1 -1
  4. package/dist/commands/run.js +1 -1
  5. package/dist/commands/self-eval.js +1 -1
  6. package/dist/commands/tombstone.js +2 -1
  7. package/dist/core/evolve/run-bench.js +4 -3
  8. package/dist/core/evolve/run-loop.js +24 -4
  9. package/dist/core/evolve/run-plan.js +7 -3
  10. package/dist/core/evolve/score-bank.js +214 -0
  11. package/dist/core/graft-rebench.js +3 -0
  12. package/dist/core/ledger.js +6 -1
  13. package/dist/core/promote.js +2 -1
  14. package/dist/core/spec.js +5 -1
  15. package/dist/core/stats-math.js +39 -0
  16. package/dist/core/stats.js +125 -33
  17. package/dist/test/historian-grader-integrity.test.js +1494 -2
  18. package/dist/test/historian-grader-io.test.js +4 -0
  19. package/dist/test/historian-run-scenario.test.js +42 -2
  20. package/dist/test/repopath-seams.test.js +54 -0
  21. package/dist/test/score-bank.test.js +235 -0
  22. package/dist/test/seam-gates.test.js +78 -0
  23. package/dist/test/selfmode-literal.test.js +153 -0
  24. package/dist/test/stats-acceptance.test.js +329 -0
  25. package/dist/test/stats.test.js +18 -0
  26. package/graders/historian/grader-core.d.mts +107 -2
  27. package/graders/historian/grader-core.mjs +1046 -3
  28. package/graders/historian/grader.mjs +21 -2
  29. package/graders/historian/judge-poststage-19.mjs +212 -0
  30. package/graders/historian/judge-poststage.mjs +195 -0
  31. package/graders/historian/mutate.sh +59 -19
  32. package/graders/historian/run-scenario-17.sh +34 -0
  33. package/graders/historian/run-scenario-19.sh +43 -0
  34. package/graders/historian/run-scenario.sh +24 -1
  35. package/package.json +1 -1
  36. package/plugin/abathur.ts +1 -1
@@ -215,6 +215,7 @@ function integrityFixturesReady() {
215
215
  return (existsSync(path.join(FIXDIR, "scenario-10", "state-donothing.json")) &&
216
216
  existsSync(path.join(FIXDIR, "scenario-11", "state-fabricate.json")) &&
217
217
  existsSync(path.join(FIXDIR, "scenario-12", "seed-state.json")) &&
218
+ existsSync(path.join(FIXDIR, "scenario-13", "seed-state.json")) &&
218
219
  existsSync(path.join(HISTORIAN_REPO, "scenarios", "10-status-contradiction-audit.md")));
219
220
  }
220
221
  const integritySkip = integrityFixturesReady()
@@ -232,6 +233,9 @@ const INTEGRITY_CASES = [
232
233
  { unit: "scenario-12", scenarioRel: "scenarios/12-desc-junk-detection.md", shape: "pass", score: 1, pass: true, total: 6, weight: 6, dims: { G: 1, H: 1, I: 1, J: 1 } },
233
234
  { unit: "scenario-12", scenarioRel: "scenarios/12-desc-junk-detection.md", shape: "fail", score: 2 / 6, pass: false, total: 2, weight: 6, dims: { G: 1, H: 1, I: 0, J: 0 } },
234
235
  { unit: "scenario-12", scenarioRel: "scenarios/12-desc-junk-detection.md", shape: "donothing", score: 2 / 6, pass: false, total: 2, weight: 6, dims: { G: 1, H: 1, I: 0, J: 0 } },
236
+ { unit: "scenario-13", scenarioRel: "scenarios/13-translation-honesty-audit.md", shape: "pass", score: 1, pass: true, total: 8, weight: 8, dims: { D: 1, G: 1, H: 1, I: 1, J: 1 } },
237
+ { unit: "scenario-13", scenarioRel: "scenarios/13-translation-honesty-audit.md", shape: "fail", score: 0, pass: false, total: 0, weight: 8, dims: { D: 0, G: 0, H: 0, I: 0, J: 0 } },
238
+ { unit: "scenario-13", scenarioRel: "scenarios/13-translation-honesty-audit.md", shape: "donothing", score: 3 / 8, pass: false, total: 3, weight: 8, dims: { D: 1, G: 1, H: 0, I: 0, J: 0 } },
235
239
  ];
236
240
  function integritySandbox(unit, shape) {
237
241
  const fix = path.join(FIXDIR, unit);
@@ -42,6 +42,9 @@ const STUB_OPENCODE = `#!/usr/bin/env node
42
42
  // transcript on stdout (run-scenario.sh redirects it to $ABATHUR_TRANSCRIPT).
43
43
  // The finalMessage is the prompt echoed back — so a planted instruction in the
44
44
  // brief provably flows into the transcript.
45
+ // STUB_BANNER=1 / STUB_JUNK=1 reproduce campaign-1's channel contamination:
46
+ // a plugin startup banner / an arbitrary non-JSON line printed to stdout
47
+ // BEFORE the JSONL. Default off ⇒ clean output stays byte-identical.
45
48
  import { writeFileSync } from "node:fs";
46
49
  const args = process.argv.slice(2);
47
50
  const i = args.indexOf("--message");
@@ -51,6 +54,9 @@ if (capture !== undefined && capture.length > 0) writeFileSync(capture, message,
51
54
  const argvCapture = process.env.STUB_ARGV_FILE;
52
55
  if (argvCapture !== undefined && argvCapture.length > 0) writeFileSync(argvCapture, args.join("\\n") + "\\n", "utf8");
53
56
  const out = (doc) => process.stdout.write(JSON.stringify(doc) + "\\n");
57
+ if (process.env.STUB_BANNER === "1") out0("[persistence-watchdog] tier=R armed grace=2500ms minOutput=100 alerts=/tmp/abathur-test-alerts.log");
58
+ if (process.env.STUB_JUNK === "1") out0("this line is not json at all");
59
+ function out0(line) { process.stdout.write(line + "\\n"); }
54
60
  out({ type: "step_start", part: { type: "step-start" } });
55
61
  out({ type: "step_finish", part: { type: "step-finish", reason: "stop", tokens: { total: 42 } } });
56
62
  out({ type: "text", part: { type: "text", text: message } });
@@ -182,12 +188,12 @@ test("model pin: ABATHUR_AGENT_MODEL set ⇒ argv carries --model <value>, brief
182
188
  const argvFile = path.join(path.dirname(h.promptFile), "argv-pinned.txt");
183
189
  await runScenario(h, {
184
190
  argvFile,
185
- env: { ABATHUR_AGENT_MODEL: "testprov/testmodel" },
191
+ env: { ABATHUR_AGENT_MODEL: "test-provider/test-model" },
186
192
  });
187
193
  const tokens = argvTokens(argvFile);
188
194
  const i = tokens.indexOf("--model");
189
195
  assert.notEqual(i, -1, `--model missing from argv: ${JSON.stringify(tokens)}`);
190
- assert.equal(tokens[i + 1], "testprov/testmodel");
196
+ assert.equal(tokens[i + 1], "test-provider/test-model");
191
197
  assert.equal(tokens.at(-2), "--message", "pin must not disturb the brief channel");
192
198
  assert.equal(prompt(h), BRIEF_TEXT);
193
199
  });
@@ -203,3 +209,37 @@ test("model pin: ABATHUR_AGENT_MODEL empty string ⇒ treated as unset (same arg
203
209
  await runScenario(h, { argvFile, env: { ABATHUR_AGENT_MODEL: "" } });
204
210
  assert.deepEqual(argvTokens(argvFile), BASELINE_ARGV);
205
211
  });
212
+ // Channel contamination (campaign-1 root cause, historian evidence
213
+ // task-07-launch-blockers.md): the user's global persistence-watchdog plugin
214
+ // prints `[persistence-watchdog] tier=R armed …` to the agent's STDOUT — into
215
+ // the captured transcript — contaminating 6/8 candidate transcripts while the
216
+ // incumbent bench (pre-plugin) stayed clean. Contract of the fix: strip ONLY
217
+ // that known banner line; keep a verbatim `.raw` audit copy; unknown non-JSON
218
+ // lines must SURVIVE (the grader's corruption fail-closed stays honest); and a
219
+ // clean stream must pass through byte-identical (F1 invariant).
220
+ test("banner-strip: watchdog banner removed from transcript, .raw keeps it verbatim, grader-view parses", async (t) => {
221
+ const h = harness(t);
222
+ await runScenario(h, { env: { STUB_BANNER: "1" } });
223
+ const text = readFileSync(h.transcript, "utf8");
224
+ assert.ok(!text.includes("[persistence-watchdog]"), "banner must not reach the grader-facing transcript");
225
+ for (const line of text.split("\n")) {
226
+ if (line.length > 0)
227
+ assert.doesNotThrow(() => JSON.parse(line), `grader-facing line is not JSON: ${line}`);
228
+ }
229
+ const raw = readFileSync(`${h.transcript}.raw`, "utf8");
230
+ assert.ok(raw.startsWith("[persistence-watchdog] tier=R armed"), "raw audit copy preserves the banner verbatim");
231
+ const meta = parseTranscript(text);
232
+ assert.equal(meta.finalMessage, BRIEF_TEXT, "the stripped transcript must still yield the delivery truth");
233
+ });
234
+ test("banner-strip: clean stream ⇒ transcript byte-identical to raw (F1 invariant)", async (t) => {
235
+ const h = harness(t);
236
+ await runScenario(h);
237
+ assert.equal(readFileSync(h.transcript, "utf8"), readFileSync(`${h.transcript}.raw`, "utf8"));
238
+ });
239
+ test("banner-strip: UNKNOWN non-JSON lines are NOT stripped (grader fail-closed preserved)", async (t) => {
240
+ const h = harness(t);
241
+ await runScenario(h, { env: { STUB_JUNK: "1" } });
242
+ const text = readFileSync(h.transcript, "utf8");
243
+ assert.ok(text.includes("this line is not json at all"), "never silently launder unknown corruption");
244
+ assert.throws(() => parseTranscript(text), /transcript line is not JSON/);
245
+ });
@@ -0,0 +1,54 @@
1
+ // Tripwire (postmortem 863e6c2): the env-literal repoPath defect class was
2
+ // caused by per-call-site resolution being a CONVENTION — every new FS
3
+ // consumer had to remember to call effectiveRepoPath, and promote/tombstone
4
+ // forgot for 18 days. This test makes the convention mechanical: any line
5
+ // on the command/human-gate surface that touches `spec.repoPath` must either
6
+ // resolve it through effectiveRepoPath(...) on the same line, or carry an
7
+ // inline waiver comment `raw-repoPath: <reason>` documenting WHY the raw
8
+ // stored form is correct there (display strings, self-mode literal
9
+ // detection, post-swap pipeline paths). A new command that forgets now
10
+ // fails CI instead of failing a human gate 18 days later.
11
+ //
12
+ // Scope note: engine-internal pipeline modules (src/core/evolve/**,
13
+ // src/core/graft*.ts, src/core/bundle-*.ts, src/bench/**) operate on
14
+ // candidate-worktree-swapped absolute paths by design (run-loop.ts swaps
15
+ // spec.repoPath before invoking them) and are out of scope here; the ledger
16
+ // FS seam additionally resolves unconditionally in ledgerPath().
17
+ import assert from "node:assert/strict";
18
+ import { readdirSync, readFileSync, statSync } from "node:fs";
19
+ import path from "node:path";
20
+ import { fileURLToPath } from "node:url";
21
+ import { test } from "node:test";
22
+ // dist/test/*.js -> repo src/ (tests run from the built tree but guard SOURCE).
23
+ const SRC = fileURLToPath(new URL("../../src/", import.meta.url));
24
+ const GUARDED_FILES = [
25
+ ...readdirSync(path.join(SRC, "commands"))
26
+ .filter((f) => f.endsWith(".ts"))
27
+ .map((f) => path.join(SRC, "commands", f)),
28
+ path.join(SRC, "core", "promote.ts"),
29
+ path.join(SRC, "core", "tombstone.ts"),
30
+ ];
31
+ function collectHits(file) {
32
+ const hits = [];
33
+ const lines = readFileSync(file, "utf8").split("\n");
34
+ lines.forEach((line, i) => {
35
+ if (!line.includes("spec.repoPath"))
36
+ return;
37
+ if (line.includes("effectiveRepoPath("))
38
+ return;
39
+ if (line.includes("raw-repoPath:"))
40
+ return;
41
+ hits.push(`${path.relative(SRC, file)}:${String(i + 1)}: ${line.trim()}`);
42
+ });
43
+ return hits;
44
+ }
45
+ test("command/gate surface: every spec.repoPath use resolves via effectiveRepoPath or waives in writing", () => {
46
+ const violations = [];
47
+ for (const file of GUARDED_FILES) {
48
+ if (!statSync(file, { throwIfNoEntry: false }))
49
+ continue;
50
+ violations.push(...collectHits(file));
51
+ }
52
+ assert.deepEqual(violations, [], "raw spec.repoPath uses without effectiveRepoPath() or a `raw-repoPath:` waiver:\n" +
53
+ violations.join("\n"));
54
+ });
@@ -0,0 +1,235 @@
1
+ // Score-bank loader tests (acceptance-semantics redesign, 2026-09-24): the
2
+ // fail-closed activation law, exact shrinkage/quantum-floor arithmetic, row
3
+ // skipping with notices, corrupt-tail/backup exclusion and the gate-time
4
+ // candidate-tree exclusion hook the A9 backtest replays through.
5
+ import assert from "node:assert/strict";
6
+ import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
7
+ import os from "node:os";
8
+ import path from "node:path";
9
+ import { describe, it } from "node:test";
10
+ import { BANK_QUANTUM, BANK_THIN_DF, loadScoreBank } from "../core/evolve/score-bank.js";
11
+ const STATE_DIR = path.join(".state", "abathur");
12
+ function generationLine(row) {
13
+ const data = {
14
+ source: row.source,
15
+ ...(row.source === "candidate" ? { candidateId: `c-${row.tree}`, treeSha: row.tree, commitSha: row.tree ?? "c" } : {}),
16
+ headCommit: row.head,
17
+ complete: row.complete ?? true,
18
+ reps: 2,
19
+ units: row.units.map((u) => ({ ...u, runIds: u.scores.map((_, i) => `r${String(i)}`), failures: [] })),
20
+ counters: { candidates: row.source === "candidate" ? 1 : 0, modelCalls: 0, tokens: 1, wallS: 1 },
21
+ manifest: [],
22
+ benchProvenance: { benchType: "toy", versions: [] },
23
+ };
24
+ return `${JSON.stringify({ v: 1, ts: row.ts ?? "2026-09-24T00:00:00.000Z", kind: "generation_complete", genId: "g-1", runId: "a-1", data })}\n`;
25
+ }
26
+ function withRepo(write) {
27
+ const repo = mkdtempSync(path.join(os.tmpdir(), "abathur-bank-"));
28
+ try {
29
+ write(repo);
30
+ }
31
+ finally {
32
+ rmSync(repo, { recursive: true, force: true });
33
+ }
34
+ }
35
+ function stateDir(repo) {
36
+ const dir = path.join(repo, STATE_DIR);
37
+ mkdirSync(dir, { recursive: true });
38
+ return dir;
39
+ }
40
+ describe("loadScoreBank fail-closed activation", () => {
41
+ it("no .state dir at all ⇒ null — and the loader never creates one (peekPlanState discipline)", () => {
42
+ withRepo((repo) => {
43
+ assert.equal(loadScoreBank(repo), null);
44
+ assert.equal(existsSync(path.join(repo, STATE_DIR)), false);
45
+ });
46
+ });
47
+ it("live ledger only ⇒ null — rotation is the activation law", () => {
48
+ withRepo((repo) => {
49
+ const dir = stateDir(repo);
50
+ writeFileSync(path.join(dir, "ledger.jsonl"), generationLine({ source: "incumbent", head: "a".repeat(40), units: [{ unitId: "u", split: "train", scores: [0, 2, 1, 3] }] }));
51
+ assert.equal(loadScoreBank(repo), null);
52
+ });
53
+ });
54
+ it("ledger.corrupt-* is NOT a rotated archive; an empty dir ⇒ null", () => {
55
+ withRepo((repo) => {
56
+ const dir = stateDir(repo);
57
+ writeFileSync(path.join(dir, "ledger.corrupt-20260924-9.jsonl"), generationLine({ source: "incumbent", head: "a".repeat(40), units: [{ unitId: "u", split: "train", scores: [0, 2, 1, 3] }] }));
58
+ assert.equal(loadScoreBank(repo), null);
59
+ rmSync(path.join(dir, "ledger.corrupt-20260924-9.jsonl"));
60
+ writeFileSync(path.join(dir, "notes.txt"), "unrelated");
61
+ assert.equal(loadScoreBank(repo), null);
62
+ });
63
+ });
64
+ it("archive + live load; totalDf=0 (only single-rep groups ever) ⇒ null", () => {
65
+ withRepo((repo) => {
66
+ const dir = stateDir(repo);
67
+ writeFileSync(path.join(dir, "ledger.campaign1-x.jsonl"), generationLine({ source: "incumbent", head: "a".repeat(40), units: [{ unitId: "u", split: "train", scores: [1] }, { unitId: "v", split: "val", scores: [0.5] }] }));
68
+ writeFileSync(path.join(dir, "ledger.jsonl"), generationLine({ source: "incumbent", head: "b".repeat(40), units: [{ unitId: "u", split: "train", scores: [2] }] }));
69
+ assert.equal(loadScoreBank(repo), null);
70
+ });
71
+ });
72
+ });
73
+ describe("loadScoreBank arithmetic", () => {
74
+ const scatterRow = (head, tree, source, scores) => generationLine({ source, head, ...(tree === undefined ? {} : { tree }), units: [{ unitId: "u", split: "train", scores }] });
75
+ it("pools within-group SS across lineages; thin df floors σ at the quantum; df≥6 keeps the shrunk σ", () => {
76
+ withRepo((repo) => {
77
+ const dir = stateDir(repo);
78
+ // group g1 (campaign1 archive): [0,2] → mean 1, SS 2, df 1 — the ONLY scatter in the bank.
79
+ writeFileSync(path.join(dir, "ledger.campaign1-x.jsonl"), `${scatterRow("a".repeat(40), undefined, "incumbent", [0, 2])}` +
80
+ // group g2 (live): same unit, different lineage head, zero scatter
81
+ scatterRow("b".repeat(40), undefined, "incumbent", [1, 1]));
82
+ writeFileSync(path.join(dir, "ledger.jsonl"), generationLine({ source: "candidate", head: "b".repeat(40), tree: "t1", units: [{ unitId: "q", split: "train", scores: [1, 1, 1, 1] }] }));
83
+ const bank = loadScoreBank(repo);
84
+ assert.ok(bank !== null);
85
+ // totals: u groups df 1+1 (SS 2+0), q group df 3 (SS 0) ⇒ total df 5, prior σ² = 2/5.
86
+ assert.equal(bank.totalDf, 5);
87
+ assert.ok(Math.abs(bank.priorSigma - Math.sqrt(0.4)) < 1e-12);
88
+ // unit u: σ̂² = 2/2 = 1; shrunk = (2·1 + 2·0.4)/4 = 0.7 ⇒ σ = √0.7 (floor 0.0884 does not bind).
89
+ const u = bank.units.get("u");
90
+ assert.ok(u !== undefined);
91
+ assert.equal(u.df, 2);
92
+ assert.ok(Math.abs(u.sigma - Math.sqrt(0.7)) < 1e-12, `sigma ${String(u.sigma)}`);
93
+ // unit q: df 3, zero SS ⇒ shrunk toward the prior: (3·0 + 2·0.4)/5 = 0.16 ⇒ σ = 0.4 > floor.
94
+ const q = bank.units.get("q");
95
+ assert.ok(q !== undefined);
96
+ assert.ok(Math.abs(q.sigma - 0.4) < 1e-12);
97
+ });
98
+ });
99
+ it("zero-scatter bank: thin units sit exactly on BANK_QUANTUM, proven-quiet units (df≥6) on their shrunk σ", () => {
100
+ withRepo((repo) => {
101
+ const dir = stateDir(repo);
102
+ const thin = generationLine({ source: "incumbent", head: "a".repeat(40), units: [{ unitId: "thin", split: "train", scores: [0.5, 0.5] }] });
103
+ const wide = generationLine({ source: "incumbent", head: "b".repeat(40), units: [{ unitId: "wide", split: "train", scores: [1, 1, 1, 1, 1, 1] }] }) + generationLine({ source: "candidate", head: "b".repeat(40), tree: "x", units: [{ unitId: "wide", split: "train", scores: [1, 1, 1] }] });
104
+ writeFileSync(path.join(dir, "ledger.campaign1-x.jsonl"), thin + wide);
105
+ const bank = loadScoreBank(repo);
106
+ assert.ok(bank !== null);
107
+ assert.equal(bank.priorSigma, 0);
108
+ assert.equal(bank.units.get("thin")?.sigma, BANK_QUANTUM); // shrunk 0 → floored
109
+ assert.equal(bank.units.get("thin")?.df, 1);
110
+ assert.equal(bank.units.get("wide")?.sigma, 0); // df 4+1+… ≥ BANK_THIN_DF, zero prior ⇒ stays 0
111
+ assert.ok((bank.units.get("wide")?.df ?? 0) >= BANK_THIN_DF);
112
+ });
113
+ });
114
+ });
115
+ describe("loadScoreBank row hygiene", () => {
116
+ it("corrupt JSON lines and schema-invalid rows are skipped with notices; valid rows still price", () => {
117
+ withRepo((repo) => {
118
+ const dir = stateDir(repo);
119
+ const good = generationLine({ source: "incumbent", head: "a".repeat(40), units: [{ unitId: "u", split: "train", scores: [0, 2, 2, 0] }] });
120
+ writeFileSync(path.join(dir, "ledger.campaign1-x.jsonl"), `${good}{"v":1,"ts":"x\n${generationLine({ source: "incumbent", head: "a".repeat(40), units: [{ unitId: "bad", split: "nope", scores: [1, 2] }] })}`);
121
+ const bank = loadScoreBank(repo);
122
+ assert.ok(bank !== null);
123
+ assert.ok(bank.units.has("u"));
124
+ assert.equal(bank.units.has("bad"), false);
125
+ assert.equal(bank.notices.length, 2);
126
+ assert.match(bank.notices.join("\n"), /not JSON/);
127
+ assert.match(bank.notices.join("\n"), /unreadable/);
128
+ });
129
+ });
130
+ it("budget-truncated rows (complete:false) still contribute their real replicate scores", () => {
131
+ withRepo((repo) => {
132
+ const dir = stateDir(repo);
133
+ writeFileSync(path.join(dir, "ledger.campaign1-x.jsonl"), generationLine({ source: "candidate", head: "a".repeat(40), tree: "t", complete: false, units: [{ unitId: "u", split: "train", scores: [0, 2] }] }));
134
+ const bank = loadScoreBank(repo);
135
+ assert.equal(bank?.units.get("u")?.df, 1);
136
+ });
137
+ });
138
+ it("excludeCandidateTrees replays the gate-time world (rows written after evaluate)", () => {
139
+ withRepo((repo) => {
140
+ const dir = stateDir(repo);
141
+ writeFileSync(path.join(dir, "ledger.campaign1-x.jsonl"), generationLine({ source: "incumbent", head: "a".repeat(40), units: [{ unitId: "u", split: "train", scores: [0, 2] }] }) +
142
+ generationLine({ source: "candidate", head: "a".repeat(40), tree: "to-exclude", units: [{ unitId: "u", split: "train", scores: [4, 0] }] }));
143
+ const withIt = loadScoreBank(repo);
144
+ const without = loadScoreBank(repo, { excludeCandidateTrees: ["to-exclude"] });
145
+ assert.equal(withIt?.units.get("u")?.df, 2);
146
+ assert.equal(without?.units.get("u")?.df, 1);
147
+ assert.equal(withIt?.totalDf, 2);
148
+ assert.equal(without?.totalDf, 1);
149
+ });
150
+ });
151
+ it("excludeFiles hides a future campaign's ledger; null once no rotated archive remains", () => {
152
+ withRepo((repo) => {
153
+ const dir = stateDir(repo);
154
+ writeFileSync(path.join(dir, "ledger.campaign1-x.jsonl"), generationLine({ source: "incumbent", head: "a".repeat(40), units: [{ unitId: "u", split: "train", scores: [0, 2, 2, 0] }] }));
155
+ writeFileSync(path.join(dir, "ledger.campaign2-y.jsonl"), generationLine({ source: "incumbent", head: "b".repeat(40), units: [{ unitId: "u", split: "train", scores: [1, 5] }] }));
156
+ assert.equal(loadScoreBank(repo)?.units.get("u")?.df, 4);
157
+ const hidden = loadScoreBank(repo, { excludeFiles: ["ledger.campaign2-y.jsonl"] });
158
+ assert.equal(hidden?.units.get("u")?.df, 3);
159
+ assert.equal(hidden?.totalDf, 3);
160
+ assert.equal(loadScoreBank(repo, { excludeFiles: ["ledger.campaign1-x.jsonl", "ledger.campaign2-y.jsonl"] }), null);
161
+ });
162
+ });
163
+ });
164
+ describe("loadScoreBank bank-epoch lineage quarantine", () => {
165
+ const quietNew = {
166
+ source: "incumbent",
167
+ head: "h1",
168
+ ts: "2026-09-25T00:00:00.000Z",
169
+ units: [{ unitId: "scenario-9", split: "val", scores: [1, 1] }],
170
+ };
171
+ const noisyOld = {
172
+ source: "incumbent",
173
+ head: "h1",
174
+ ts: "2026-09-20T00:00:00.000Z",
175
+ units: [{ unitId: "scenario-9", split: "val", scores: [0, 1, 0, 1] }],
176
+ };
177
+ it("groups with ts before the epoch are quarantined: df drops, notice names the count and why", () => {
178
+ withRepo((repo) => {
179
+ const dir = stateDir(repo);
180
+ writeFileSync(path.join(dir, "ledger.campaign1-2026-09-20.jsonl"), generationLine(noisyOld) + generationLine(quietNew));
181
+ writeFileSync(path.join(dir, "bank-epoch.json"), JSON.stringify({ "scenario-9": { sinceIso: "2026-09-24T00:00:00Z", why: "pin era voided" } }));
182
+ const bank = loadScoreBank(repo);
183
+ assert.ok(bank !== null);
184
+ const stat = bank.units.get("scenario-9");
185
+ assert.ok(stat !== undefined);
186
+ assert.equal(stat.df, 1); // only the quiet new group counts
187
+ assert.ok(stat.sigma >= BANK_QUANTUM - 1e-12); // thin history floor holds the prior
188
+ assert.ok(bank.notices.some((n) => n.includes("bank-epoch 'scenario-9' since") && n.includes("1 group(s) quarantined") && n.includes("pin era voided")));
189
+ assert.ok(bank.notices.some((n) => n.includes("since 2026-09-24T00:00:00Z") && !n.includes("undefined")), "notice must print the ISO epoch, never undefined");
190
+ });
191
+ });
192
+ it("quarantined-only units leave zero pooled df ⇒ bank null ⇒ caller runs the legacy gate", () => {
193
+ withRepo((repo) => {
194
+ const dir = stateDir(repo);
195
+ writeFileSync(path.join(dir, "ledger.campaign1-2026-09-20.jsonl"), generationLine(noisyOld));
196
+ writeFileSync(path.join(dir, "bank-epoch.json"), JSON.stringify({ "scenario-9": { sinceIso: "2026-09-24T00:00:00Z" } }));
197
+ assert.equal(loadScoreBank(repo), null);
198
+ });
199
+ });
200
+ it("unreadable row ts fails closed for quarantined units but leaves unruled units untouched", () => {
201
+ withRepo((repo) => {
202
+ const dir = stateDir(repo);
203
+ const ruled = { ...noisyOld, ts: "not-a-date", units: [
204
+ { unitId: "scenario-9", split: "val", scores: [0, 1] },
205
+ { unitId: "scenario-8", split: "val", scores: [1, 0] },
206
+ ] };
207
+ writeFileSync(path.join(dir, "ledger.campaign1-2026-09-20.jsonl"), generationLine(ruled));
208
+ writeFileSync(path.join(dir, "bank-epoch.json"), JSON.stringify({ "scenario-9": { sinceIso: "2026-09-24T00:00:00Z" } }));
209
+ const bank = loadScoreBank(repo);
210
+ assert.ok(bank !== null);
211
+ assert.equal(bank.units.has("scenario-9"), false); // provably-at-or-after failed ⇒ quarantined
212
+ assert.ok(bank.units.has("scenario-8")); // no rule ⇒ untouched
213
+ });
214
+ });
215
+ it("malformed quarantine sidecars disable the WHOLE bank (never a silently ignored quarantine)", () => {
216
+ const bad = ["{not json", JSON.stringify({ "scenario-9": { sinceIso: "not-a-date" } }), JSON.stringify({ "scenario-9": { sinceIso: "2026-09-24T00:00:00Z", extra: 1 } }), JSON.stringify(["scenario-9"])];
217
+ for (const body of bad) {
218
+ withRepo((repo) => {
219
+ const dir = stateDir(repo);
220
+ writeFileSync(path.join(dir, "ledger.campaign1-2026-09-20.jsonl"), generationLine(noisyOld) + generationLine(quietNew));
221
+ writeFileSync(path.join(dir, "bank-epoch.json"), body);
222
+ assert.equal(loadScoreBank(repo), null, `sidecar must disable bank: ${body.slice(0, 24)}`);
223
+ });
224
+ }
225
+ });
226
+ it("absent sidecar = no quarantine, byte-identical legacy behavior", () => {
227
+ withRepo((repo) => {
228
+ const dir = stateDir(repo);
229
+ writeFileSync(path.join(dir, "ledger.campaign1-2026-09-20.jsonl"), generationLine(noisyOld) + generationLine(quietNew));
230
+ const bank = loadScoreBank(repo);
231
+ assert.ok(bank !== null);
232
+ assert.equal(bank.units.get("scenario-9")?.df, 4); // 3 df old + 1 df new — nothing skipped
233
+ });
234
+ });
235
+ });
@@ -0,0 +1,78 @@
1
+ // Seam test (postmortem 863e6c2): the human-gate commands `promote` and
2
+ // `tombstone` must resolve an env-literal repoPath exactly like status/run
3
+ // (4997618 pattern). Pre-fix, promote.ts opened the ledger under the STORED
4
+ // UNRESOLVED `${VAR}` literal, read an empty ledger, and reported the
5
+ // misleading "no generation_complete ledger row" with the literal still in
6
+ // the hint — the same fail-open seam class as the status spawn-ENOENT.
7
+ // Pins, for both gates:
8
+ // (1) env exported ⇒ gate reads the REAL ledger at the resolved path:
9
+ // the refusal names the resolved repo, never the `${VAR}` literal;
10
+ // (2) env unset ⇒ clean exit 2 NAMING the variable;
11
+ // (3) stderr never contains the unresolved literal or a spawn-ENOENT.
12
+ import assert from "node:assert/strict";
13
+ import { spawnSync } from "node:child_process";
14
+ import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
15
+ import { mkdtempSync, rmSync } from "node:fs";
16
+ import * as os from "node:os";
17
+ import path from "node:path";
18
+ import { fileURLToPath } from "node:url";
19
+ import { test } from "node:test";
20
+ import { registerGenome } from "../core/genome.js";
21
+ import { prepareToyGenome } from "../bench/toy.js";
22
+ const CLI = fileURLToPath(new URL("../../dist/cli.js", import.meta.url));
23
+ const LIT_VAR = "ABATHUR_SEAM_GATES_REPO";
24
+ async function fixture(t, label) {
25
+ const root = mkdtempSync(path.join(os.tmpdir(), "seam-gates-"));
26
+ t.after(() => rmSync(root, { recursive: true, force: true }));
27
+ const configDir = path.join(root, "config");
28
+ mkdirSync(configDir, { recursive: true });
29
+ writeFileSync(path.join(configDir, "config.jsonc"), '{ "opencodeBin": null }\n', "utf8");
30
+ const repo = await prepareToyGenome(path.join(root, "genome"));
31
+ const doc = JSON.parse(readFileSync(path.join(repo, "genome.jsonc"), "utf8"));
32
+ doc["label"] = label;
33
+ doc["repoPath"] = `\${${LIT_VAR}}`;
34
+ const specFile = path.join(root, "private.jsonc");
35
+ writeFileSync(specFile, `${JSON.stringify(doc, null, 2)}\n`, "utf8");
36
+ process.env[LIT_VAR] = repo;
37
+ try {
38
+ registerGenome(configDir, specFile);
39
+ }
40
+ finally {
41
+ delete process.env[LIT_VAR];
42
+ }
43
+ return { root, configDir, repo, label };
44
+ }
45
+ function cli(f, args, env) {
46
+ return spawnSync(process.execPath, [CLI, ...args], {
47
+ env: { ...process.env, ABATHUR_CONFIG: path.join(f.configDir, "config.jsonc"), ...env },
48
+ encoding: "utf8",
49
+ timeout: 60_000,
50
+ });
51
+ }
52
+ function assertNoLiteralOrEnoent(r) {
53
+ const blob = `${r.stdout}\n${r.stderr}`;
54
+ assert.ok(!blob.includes(`\${${LIT_VAR}}`), `unresolved literal leaked into output:\n${blob}`);
55
+ assert.ok(!blob.includes("ENOENT"), `misleading ENOENT leaked into output:\n${blob}`);
56
+ }
57
+ test("promote with env-literal repoPath: exported ⇒ blocked naming the RESOLVED repo path", async (t) => {
58
+ const f = await fixture(t, "seam-gate-promote");
59
+ const r = cli(f, ["promote", f.label, "g-nonexistent"], { [LIT_VAR]: f.repo });
60
+ assert.equal(r.status, 1, `stdout: ${r.stdout}\nstderr: ${r.stderr}`);
61
+ assertNoLiteralOrEnoent(r);
62
+ assert.ok(r.stderr.includes(f.repo), `refusal must name the resolved repo path so the operator can debug:\n${r.stderr}`);
63
+ });
64
+ test("tombstone with env-literal repoPath: exported ⇒ blocked, no literal leak", async (t) => {
65
+ const f = await fixture(t, "seam-gate-tombstone");
66
+ const r = cli(f, ["tombstone", f.label, "g-nonexistent", "--reason", "seam test"], { [LIT_VAR]: f.repo });
67
+ assert.equal(r.status, 1, `stdout: ${r.stdout}\nstderr: ${r.stderr}`);
68
+ assertNoLiteralOrEnoent(r);
69
+ });
70
+ test("promote with env-literal repoPath: unset ⇒ clean exit 2 naming the variable", async (t) => {
71
+ const f = await fixture(t, "seam-gate-promote-unset");
72
+ const env = {};
73
+ env[LIT_VAR] = undefined;
74
+ const r = cli(f, ["promote", f.label, "g-nonexistent"], env);
75
+ assert.equal(r.status, 2, `stdout: ${r.stdout}\nstderr: ${r.stderr}`);
76
+ assert.ok(r.stderr.includes(LIT_VAR), `exit-2 message must name the env var:\n${r.stderr}`);
77
+ assert.ok(!r.stderr.includes("ENOENT"), `misleading ENOENT leaked into output:\n${r.stderr}`);
78
+ });
@@ -0,0 +1,153 @@
1
+ // Campaign-1 defect pins (historian evidence task-07, live symptom rc=2):
2
+ // a `${VAR}` env-literal repoPath is a PATH-INJECTION convenience, NOT an
3
+ // engine-self marker. run-loop.ts routed selfMode on isEnvRepoLiteral — ANY
4
+ // literal — so the historian genome (`${ABATHUR_HISTORIAN_REPO}`) was forced
5
+ // into the tsc-build/toy-replay snapshot-overlay bench and died fail-closed
6
+ // demanding `selfbench/expected.json`. selfMode must mean
7
+ // `envRepoName(repoPath) === "ABATHUR_SELF_REPO"`. Secondary pin: the dry-run
8
+ // plan must display the RESOLVED repo path, never the raw literal joined to
9
+ // cwd (run-plan.ts printed `/…/historian/${ABATHUR_HISTORIAN_REPO}`).
10
+ // Harness mirrors run-loop.test.ts: toy genome + stub mutator, no model calls.
11
+ import assert from "node:assert/strict";
12
+ import { spawnSync } from "node:child_process";
13
+ import { existsSync, mkdirSync, readFileSync, writeFileSync, chmodSync } from "node:fs";
14
+ import { mkdtemp } from "node:fs/promises";
15
+ import * as os from "node:os";
16
+ import path from "node:path";
17
+ import test, {} from "node:test";
18
+ import { ExitSignal } from "../exit.js";
19
+ import { registerGenome, requireGenomesByLabel } from "../core/genome.js";
20
+ import { Ledger } from "../core/ledger.js";
21
+ import { prepareToyGenome } from "../bench/toy.js";
22
+ import { runEvolution } from "../core/evolve/run-loop.js";
23
+ // Same three-candidate stub script as run-loop.test.ts (annotate-add seals,
24
+ // touch-kernel path-rejected, fix-add nominated).
25
+ const ADD_ANCHOR = "return a - b; // seeded bug: must be `return a + b;`";
26
+ const STUB_SOURCE = `#!/usr/bin/env node
27
+ import { readFileSync } from "node:fs";
28
+ const args = process.argv.slice(2);
29
+ const opt = (n) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : undefined; };
30
+ const mode = opt("--mode");
31
+ const dir = opt("--dir");
32
+ const read = (f) => readFileSync(dir + "/" + f, "utf8");
33
+ function lineDiff(file, anchor, replacement) {
34
+ const lines = read(file).split("\\n");
35
+ const i = lines.findIndex((l) => l.includes(anchor));
36
+ if (i < 0) { process.stderr.write("stub: no anchor in " + file + "\\n"); process.exit(1); }
37
+ const changed = lines[i].replace(anchor, replacement);
38
+ return "--- a/" + file + "\\n+++ b/" + file + "\\n@@ -" + String(i + 1) + ",1 +" + String(i + 1) + ",1 @@\\n-" + lines[i] + "\\n+" + changed + "\\n";
39
+ }
40
+ const FIX = ${JSON.stringify(ADD_ANCHOR)};
41
+ const out = (candidates) => process.stdout.write(JSON.stringify({ candidates }) + "\\n");
42
+ if (mode === "three") {
43
+ out([
44
+ { id: "annotate-add", rationale: "annotate add()", diffs: [lineDiff("units/add.mjs", "export function add(", "export function add /* patched */(")] },
45
+ { id: "touch-kernel", rationale: "subvert grading", diffs: [lineDiff("grader.mjs", "model-free", "model-free-ish")] },
46
+ { id: "fix-add", rationale: "fix add(): subtraction to addition", diffs: [lineDiff("units/add.mjs", FIX, "return a + b;")] },
47
+ ]);
48
+ } else {
49
+ process.stderr.write("stub: unknown mode\\n");
50
+ process.exit(2);
51
+ }
52
+ `;
53
+ /**
54
+ * Toy genome registered with repoPath stored as the UNRESOLVED literal
55
+ * `${<envName>}` (the self-seed trick, pointed at the toy repo itself), env
56
+ * var set for the duration of the test. Fresh configDir per test.
57
+ */
58
+ async function literalFixture(t, envName) {
59
+ const root = await mkdtemp(path.join(os.tmpdir(), "abathur-selfmode-"));
60
+ t.after(() => {
61
+ // trusted snapshots are chmod-protected read-only: restore writability first.
62
+ spawnSync("chmod", ["-R", "u+rwX", root], { encoding: "utf8" });
63
+ spawnSync("rm", ["-rf", root], { encoding: "utf8" });
64
+ });
65
+ const configDir = path.join(root, "config");
66
+ mkdirSync(configDir, { recursive: true });
67
+ writeFileSync(path.join(configDir, "config.jsonc"), '{ "opencodeBin": null }\n', "utf8");
68
+ const repo = await prepareToyGenome(path.join(root, "genome"));
69
+ const specPath = path.join(repo, "genome.jsonc");
70
+ const doc = JSON.parse(readFileSync(specPath, "utf8"));
71
+ doc.repoPath = `\${${envName}}`;
72
+ writeFileSync(specPath, `${JSON.stringify(doc, null, 2)}\n`, "utf8");
73
+ const saved = process.env[envName];
74
+ process.env[envName] = repo;
75
+ t.after(() => {
76
+ if (saved === undefined)
77
+ delete process.env[envName];
78
+ else
79
+ process.env[envName] = saved;
80
+ });
81
+ registerGenome(configDir, specPath);
82
+ const entry = requireGenomesByLabel(configDir, "toy-smoke").entries[0];
83
+ if (entry === undefined)
84
+ throw new Error("fixture: toy-smoke registration vanished");
85
+ const stub = path.join(root, "stub.mjs");
86
+ writeFileSync(stub, STUB_SOURCE, "utf8");
87
+ chmodSync(stub, 0o755);
88
+ return {
89
+ root,
90
+ configDir,
91
+ repo,
92
+ entry,
93
+ mutatorCommand: `node ${stub} --mode three --dir {worktree} --brief {brief}`,
94
+ env: { XDG_CACHE_HOME: path.join(root, "xdg-cache"), HOME: path.join(root, "home") },
95
+ sandboxRoot: path.join(root, "sandboxes"),
96
+ };
97
+ }
98
+ function generationRowCount(f) {
99
+ return Ledger.open(f.repo)
100
+ .readAll()
101
+ .filter((r) => r.kind === "generation_complete").length;
102
+ }
103
+ // ------------------------------------------------- non-self literal: normal bench
104
+ test("campaign-1 defect: a non-self env-literal repoPath must NOT enter the self-bench overlay", async (t) => {
105
+ const f = await literalFixture(t, "ABATHUR_HISTORIAN_REPO");
106
+ const out = await runEvolution({
107
+ entry: f.entry,
108
+ configDir: f.configDir,
109
+ mutatorCommand: f.mutatorCommand,
110
+ env: f.env,
111
+ sandboxRoot: f.sandboxRoot,
112
+ });
113
+ const plan = out.lines.join("\n");
114
+ assert.equal(out.exitCode, 0, `expected exit 0, lines: ${plan}`);
115
+ assert.ok(!/selfbench|self-snapshot|self-bench|overlay/i.test(plan), `non-self literal must never touch the self-bench machinery: ${plan}`);
116
+ assert.ok(generationRowCount(f) >= 2, "incumbent + candidate rows benched through the NORMAL toy path");
117
+ assert.equal(existsSync(path.join(f.root, "selfbench")), false);
118
+ });
119
+ // ------------------------------------- self literal: overlay path preserved
120
+ test("regression pin: the ${ABATHUR_SELF_REPO} literal still routes into the self-bench path", async (t) => {
121
+ // The toy repo carries no selfbench/expected.json, so entering the overlay
122
+ // bench FAILS CLOSED with exactly that demand — the observable proof that
123
+ // selfMode is still true for the engine-self literal (live symptom shape).
124
+ const f = await literalFixture(t, "ABATHUR_SELF_REPO");
125
+ await assert.rejects(runEvolution({
126
+ entry: f.entry,
127
+ configDir: f.configDir,
128
+ mutatorCommand: f.mutatorCommand,
129
+ env: f.env,
130
+ sandboxRoot: f.sandboxRoot,
131
+ }), (cause) => {
132
+ assert.ok(cause instanceof ExitSignal, `expected ExitSignal, got ${String(cause)}`);
133
+ assert.equal(cause.code, 2);
134
+ assert.match(cause.message, /self-snapshot: trusted selfbench\/expected\.json missing/);
135
+ return true;
136
+ });
137
+ });
138
+ // ----------------------------------------------------- dry-run plan display
139
+ test("--dry-run plan displays the RESOLVED repo path, never the literal against cwd", async (t) => {
140
+ const f = await literalFixture(t, "ABATHUR_HISTORIAN_REPO");
141
+ const out = await runEvolution({
142
+ entry: f.entry,
143
+ configDir: f.configDir,
144
+ mutatorCommand: f.mutatorCommand,
145
+ dryRun: true,
146
+ env: f.env,
147
+ sandboxRoot: f.sandboxRoot,
148
+ });
149
+ assert.equal(out.exitCode, 0, out.lines.join("\n"));
150
+ const plan = out.lines.join("\n");
151
+ assert.ok(plan.includes(`repo: ${f.repo}`), `plan must show the resolved path:\n${plan}`);
152
+ assert.ok(!plan.includes("${"), `plan must never echo the raw literal (resolved against cwd garbage):\n${plan}`);
153
+ });