@tachikomagundam/abathur 0.2.4 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/genomes/historian.example.jsonc +1 -1
- package/dist/commands/genome.js +2 -2
- package/dist/commands/graft.js +1 -1
- package/dist/commands/run.js +1 -1
- package/dist/commands/self-eval.js +1 -1
- package/dist/commands/tombstone.js +2 -1
- package/dist/core/evolve/run-bench.js +4 -3
- package/dist/core/evolve/run-loop.js +24 -4
- package/dist/core/evolve/run-plan.js +7 -3
- package/dist/core/evolve/score-bank.js +214 -0
- package/dist/core/graft-rebench.js +3 -0
- package/dist/core/ledger.js +6 -1
- package/dist/core/promote.js +2 -1
- package/dist/core/spec.js +5 -1
- package/dist/core/stats-math.js +39 -0
- package/dist/core/stats.js +125 -33
- package/dist/test/historian-grader-integrity.test.js +1494 -2
- package/dist/test/historian-grader-io.test.js +4 -0
- package/dist/test/historian-run-scenario.test.js +42 -2
- package/dist/test/repopath-seams.test.js +54 -0
- package/dist/test/score-bank.test.js +235 -0
- package/dist/test/seam-gates.test.js +78 -0
- package/dist/test/selfmode-literal.test.js +153 -0
- package/dist/test/stats-acceptance.test.js +329 -0
- package/dist/test/stats.test.js +18 -0
- package/graders/historian/grader-core.d.mts +107 -2
- package/graders/historian/grader-core.mjs +1046 -3
- package/graders/historian/grader.mjs +21 -2
- package/graders/historian/judge-poststage-19.mjs +212 -0
- package/graders/historian/judge-poststage.mjs +195 -0
- package/graders/historian/mutate.sh +59 -19
- package/graders/historian/run-scenario-17.sh +34 -0
- package/graders/historian/run-scenario-19.sh +43 -0
- package/graders/historian/run-scenario.sh +24 -1
- package/package.json +1 -1
- package/plugin/abathur.ts +1 -1
|
@@ -215,6 +215,7 @@ function integrityFixturesReady() {
|
|
|
215
215
|
return (existsSync(path.join(FIXDIR, "scenario-10", "state-donothing.json")) &&
|
|
216
216
|
existsSync(path.join(FIXDIR, "scenario-11", "state-fabricate.json")) &&
|
|
217
217
|
existsSync(path.join(FIXDIR, "scenario-12", "seed-state.json")) &&
|
|
218
|
+
existsSync(path.join(FIXDIR, "scenario-13", "seed-state.json")) &&
|
|
218
219
|
existsSync(path.join(HISTORIAN_REPO, "scenarios", "10-status-contradiction-audit.md")));
|
|
219
220
|
}
|
|
220
221
|
const integritySkip = integrityFixturesReady()
|
|
@@ -232,6 +233,9 @@ const INTEGRITY_CASES = [
|
|
|
232
233
|
{ unit: "scenario-12", scenarioRel: "scenarios/12-desc-junk-detection.md", shape: "pass", score: 1, pass: true, total: 6, weight: 6, dims: { G: 1, H: 1, I: 1, J: 1 } },
|
|
233
234
|
{ unit: "scenario-12", scenarioRel: "scenarios/12-desc-junk-detection.md", shape: "fail", score: 2 / 6, pass: false, total: 2, weight: 6, dims: { G: 1, H: 1, I: 0, J: 0 } },
|
|
234
235
|
{ unit: "scenario-12", scenarioRel: "scenarios/12-desc-junk-detection.md", shape: "donothing", score: 2 / 6, pass: false, total: 2, weight: 6, dims: { G: 1, H: 1, I: 0, J: 0 } },
|
|
236
|
+
{ unit: "scenario-13", scenarioRel: "scenarios/13-translation-honesty-audit.md", shape: "pass", score: 1, pass: true, total: 8, weight: 8, dims: { D: 1, G: 1, H: 1, I: 1, J: 1 } },
|
|
237
|
+
{ unit: "scenario-13", scenarioRel: "scenarios/13-translation-honesty-audit.md", shape: "fail", score: 0, pass: false, total: 0, weight: 8, dims: { D: 0, G: 0, H: 0, I: 0, J: 0 } },
|
|
238
|
+
{ unit: "scenario-13", scenarioRel: "scenarios/13-translation-honesty-audit.md", shape: "donothing", score: 3 / 8, pass: false, total: 3, weight: 8, dims: { D: 1, G: 1, H: 0, I: 0, J: 0 } },
|
|
235
239
|
];
|
|
236
240
|
function integritySandbox(unit, shape) {
|
|
237
241
|
const fix = path.join(FIXDIR, unit);
|
|
@@ -42,6 +42,9 @@ const STUB_OPENCODE = `#!/usr/bin/env node
|
|
|
42
42
|
// transcript on stdout (run-scenario.sh redirects it to $ABATHUR_TRANSCRIPT).
|
|
43
43
|
// The finalMessage is the prompt echoed back — so a planted instruction in the
|
|
44
44
|
// brief provably flows into the transcript.
|
|
45
|
+
// STUB_BANNER=1 / STUB_JUNK=1 reproduce campaign-1's channel contamination:
|
|
46
|
+
// a plugin startup banner / an arbitrary non-JSON line printed to stdout
|
|
47
|
+
// BEFORE the JSONL. Default off ⇒ clean output stays byte-identical.
|
|
45
48
|
import { writeFileSync } from "node:fs";
|
|
46
49
|
const args = process.argv.slice(2);
|
|
47
50
|
const i = args.indexOf("--message");
|
|
@@ -51,6 +54,9 @@ if (capture !== undefined && capture.length > 0) writeFileSync(capture, message,
|
|
|
51
54
|
const argvCapture = process.env.STUB_ARGV_FILE;
|
|
52
55
|
if (argvCapture !== undefined && argvCapture.length > 0) writeFileSync(argvCapture, args.join("\\n") + "\\n", "utf8");
|
|
53
56
|
const out = (doc) => process.stdout.write(JSON.stringify(doc) + "\\n");
|
|
57
|
+
if (process.env.STUB_BANNER === "1") out0("[persistence-watchdog] tier=R armed grace=2500ms minOutput=100 alerts=/tmp/abathur-test-alerts.log");
|
|
58
|
+
if (process.env.STUB_JUNK === "1") out0("this line is not json at all");
|
|
59
|
+
function out0(line) { process.stdout.write(line + "\\n"); }
|
|
54
60
|
out({ type: "step_start", part: { type: "step-start" } });
|
|
55
61
|
out({ type: "step_finish", part: { type: "step-finish", reason: "stop", tokens: { total: 42 } } });
|
|
56
62
|
out({ type: "text", part: { type: "text", text: message } });
|
|
@@ -182,12 +188,12 @@ test("model pin: ABATHUR_AGENT_MODEL set ⇒ argv carries --model <value>, brief
|
|
|
182
188
|
const argvFile = path.join(path.dirname(h.promptFile), "argv-pinned.txt");
|
|
183
189
|
await runScenario(h, {
|
|
184
190
|
argvFile,
|
|
185
|
-
env: { ABATHUR_AGENT_MODEL: "
|
|
191
|
+
env: { ABATHUR_AGENT_MODEL: "test-provider/test-model" },
|
|
186
192
|
});
|
|
187
193
|
const tokens = argvTokens(argvFile);
|
|
188
194
|
const i = tokens.indexOf("--model");
|
|
189
195
|
assert.notEqual(i, -1, `--model missing from argv: ${JSON.stringify(tokens)}`);
|
|
190
|
-
assert.equal(tokens[i + 1], "
|
|
196
|
+
assert.equal(tokens[i + 1], "test-provider/test-model");
|
|
191
197
|
assert.equal(tokens.at(-2), "--message", "pin must not disturb the brief channel");
|
|
192
198
|
assert.equal(prompt(h), BRIEF_TEXT);
|
|
193
199
|
});
|
|
@@ -203,3 +209,37 @@ test("model pin: ABATHUR_AGENT_MODEL empty string ⇒ treated as unset (same arg
|
|
|
203
209
|
await runScenario(h, { argvFile, env: { ABATHUR_AGENT_MODEL: "" } });
|
|
204
210
|
assert.deepEqual(argvTokens(argvFile), BASELINE_ARGV);
|
|
205
211
|
});
|
|
212
|
+
// Channel contamination (campaign-1 root cause, historian evidence
|
|
213
|
+
// task-07-launch-blockers.md): the user's global persistence-watchdog plugin
|
|
214
|
+
// prints `[persistence-watchdog] tier=R armed …` to the agent's STDOUT — into
|
|
215
|
+
// the captured transcript — contaminating 6/8 candidate transcripts while the
|
|
216
|
+
// incumbent bench (pre-plugin) stayed clean. Contract of the fix: strip ONLY
|
|
217
|
+
// that known banner line; keep a verbatim `.raw` audit copy; unknown non-JSON
|
|
218
|
+
// lines must SURVIVE (the grader's corruption fail-closed stays honest); and a
|
|
219
|
+
// clean stream must pass through byte-identical (F1 invariant).
|
|
220
|
+
test("banner-strip: watchdog banner removed from transcript, .raw keeps it verbatim, grader-view parses", async (t) => {
|
|
221
|
+
const h = harness(t);
|
|
222
|
+
await runScenario(h, { env: { STUB_BANNER: "1" } });
|
|
223
|
+
const text = readFileSync(h.transcript, "utf8");
|
|
224
|
+
assert.ok(!text.includes("[persistence-watchdog]"), "banner must not reach the grader-facing transcript");
|
|
225
|
+
for (const line of text.split("\n")) {
|
|
226
|
+
if (line.length > 0)
|
|
227
|
+
assert.doesNotThrow(() => JSON.parse(line), `grader-facing line is not JSON: ${line}`);
|
|
228
|
+
}
|
|
229
|
+
const raw = readFileSync(`${h.transcript}.raw`, "utf8");
|
|
230
|
+
assert.ok(raw.startsWith("[persistence-watchdog] tier=R armed"), "raw audit copy preserves the banner verbatim");
|
|
231
|
+
const meta = parseTranscript(text);
|
|
232
|
+
assert.equal(meta.finalMessage, BRIEF_TEXT, "the stripped transcript must still yield the delivery truth");
|
|
233
|
+
});
|
|
234
|
+
test("banner-strip: clean stream ⇒ transcript byte-identical to raw (F1 invariant)", async (t) => {
|
|
235
|
+
const h = harness(t);
|
|
236
|
+
await runScenario(h);
|
|
237
|
+
assert.equal(readFileSync(h.transcript, "utf8"), readFileSync(`${h.transcript}.raw`, "utf8"));
|
|
238
|
+
});
|
|
239
|
+
test("banner-strip: UNKNOWN non-JSON lines are NOT stripped (grader fail-closed preserved)", async (t) => {
|
|
240
|
+
const h = harness(t);
|
|
241
|
+
await runScenario(h, { env: { STUB_JUNK: "1" } });
|
|
242
|
+
const text = readFileSync(h.transcript, "utf8");
|
|
243
|
+
assert.ok(text.includes("this line is not json at all"), "never silently launder unknown corruption");
|
|
244
|
+
assert.throws(() => parseTranscript(text), /transcript line is not JSON/);
|
|
245
|
+
});
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// Tripwire (postmortem 863e6c2): the env-literal repoPath defect class was
|
|
2
|
+
// caused by per-call-site resolution being a CONVENTION — every new FS
|
|
3
|
+
// consumer had to remember to call effectiveRepoPath, and promote/tombstone
|
|
4
|
+
// forgot for 18 days. This test makes the convention mechanical: any line
|
|
5
|
+
// on the command/human-gate surface that touches `spec.repoPath` must either
|
|
6
|
+
// resolve it through effectiveRepoPath(...) on the same line, or carry an
|
|
7
|
+
// inline waiver comment `raw-repoPath: <reason>` documenting WHY the raw
|
|
8
|
+
// stored form is correct there (display strings, self-mode literal
|
|
9
|
+
// detection, post-swap pipeline paths). A new command that forgets now
|
|
10
|
+
// fails CI instead of failing a human gate 18 days later.
|
|
11
|
+
//
|
|
12
|
+
// Scope note: engine-internal pipeline modules (src/core/evolve/**,
|
|
13
|
+
// src/core/graft*.ts, src/core/bundle-*.ts, src/bench/**) operate on
|
|
14
|
+
// candidate-worktree-swapped absolute paths by design (run-loop.ts swaps
|
|
15
|
+
// spec.repoPath before invoking them) and are out of scope here; the ledger
|
|
16
|
+
// FS seam additionally resolves unconditionally in ledgerPath().
|
|
17
|
+
import assert from "node:assert/strict";
|
|
18
|
+
import { readdirSync, readFileSync, statSync } from "node:fs";
|
|
19
|
+
import path from "node:path";
|
|
20
|
+
import { fileURLToPath } from "node:url";
|
|
21
|
+
import { test } from "node:test";
|
|
22
|
+
// dist/test/*.js -> repo src/ (tests run from the built tree but guard SOURCE).
|
|
23
|
+
const SRC = fileURLToPath(new URL("../../src/", import.meta.url));
|
|
24
|
+
const GUARDED_FILES = [
|
|
25
|
+
...readdirSync(path.join(SRC, "commands"))
|
|
26
|
+
.filter((f) => f.endsWith(".ts"))
|
|
27
|
+
.map((f) => path.join(SRC, "commands", f)),
|
|
28
|
+
path.join(SRC, "core", "promote.ts"),
|
|
29
|
+
path.join(SRC, "core", "tombstone.ts"),
|
|
30
|
+
];
|
|
31
|
+
function collectHits(file) {
|
|
32
|
+
const hits = [];
|
|
33
|
+
const lines = readFileSync(file, "utf8").split("\n");
|
|
34
|
+
lines.forEach((line, i) => {
|
|
35
|
+
if (!line.includes("spec.repoPath"))
|
|
36
|
+
return;
|
|
37
|
+
if (line.includes("effectiveRepoPath("))
|
|
38
|
+
return;
|
|
39
|
+
if (line.includes("raw-repoPath:"))
|
|
40
|
+
return;
|
|
41
|
+
hits.push(`${path.relative(SRC, file)}:${String(i + 1)}: ${line.trim()}`);
|
|
42
|
+
});
|
|
43
|
+
return hits;
|
|
44
|
+
}
|
|
45
|
+
test("command/gate surface: every spec.repoPath use resolves via effectiveRepoPath or waives in writing", () => {
|
|
46
|
+
const violations = [];
|
|
47
|
+
for (const file of GUARDED_FILES) {
|
|
48
|
+
if (!statSync(file, { throwIfNoEntry: false }))
|
|
49
|
+
continue;
|
|
50
|
+
violations.push(...collectHits(file));
|
|
51
|
+
}
|
|
52
|
+
assert.deepEqual(violations, [], "raw spec.repoPath uses without effectiveRepoPath() or a `raw-repoPath:` waiver:\n" +
|
|
53
|
+
violations.join("\n"));
|
|
54
|
+
});
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
// Score-bank loader tests (acceptance-semantics redesign, 2026-09-24): the
|
|
2
|
+
// fail-closed activation law, exact shrinkage/quantum-floor arithmetic, row
|
|
3
|
+
// skipping with notices, corrupt-tail/backup exclusion and the gate-time
|
|
4
|
+
// candidate-tree exclusion hook the A9 backtest replays through.
|
|
5
|
+
import assert from "node:assert/strict";
|
|
6
|
+
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
7
|
+
import os from "node:os";
|
|
8
|
+
import path from "node:path";
|
|
9
|
+
import { describe, it } from "node:test";
|
|
10
|
+
import { BANK_QUANTUM, BANK_THIN_DF, loadScoreBank } from "../core/evolve/score-bank.js";
|
|
11
|
+
const STATE_DIR = path.join(".state", "abathur");
|
|
12
|
+
function generationLine(row) {
|
|
13
|
+
const data = {
|
|
14
|
+
source: row.source,
|
|
15
|
+
...(row.source === "candidate" ? { candidateId: `c-${row.tree}`, treeSha: row.tree, commitSha: row.tree ?? "c" } : {}),
|
|
16
|
+
headCommit: row.head,
|
|
17
|
+
complete: row.complete ?? true,
|
|
18
|
+
reps: 2,
|
|
19
|
+
units: row.units.map((u) => ({ ...u, runIds: u.scores.map((_, i) => `r${String(i)}`), failures: [] })),
|
|
20
|
+
counters: { candidates: row.source === "candidate" ? 1 : 0, modelCalls: 0, tokens: 1, wallS: 1 },
|
|
21
|
+
manifest: [],
|
|
22
|
+
benchProvenance: { benchType: "toy", versions: [] },
|
|
23
|
+
};
|
|
24
|
+
return `${JSON.stringify({ v: 1, ts: row.ts ?? "2026-09-24T00:00:00.000Z", kind: "generation_complete", genId: "g-1", runId: "a-1", data })}\n`;
|
|
25
|
+
}
|
|
26
|
+
function withRepo(write) {
|
|
27
|
+
const repo = mkdtempSync(path.join(os.tmpdir(), "abathur-bank-"));
|
|
28
|
+
try {
|
|
29
|
+
write(repo);
|
|
30
|
+
}
|
|
31
|
+
finally {
|
|
32
|
+
rmSync(repo, { recursive: true, force: true });
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
function stateDir(repo) {
|
|
36
|
+
const dir = path.join(repo, STATE_DIR);
|
|
37
|
+
mkdirSync(dir, { recursive: true });
|
|
38
|
+
return dir;
|
|
39
|
+
}
|
|
40
|
+
describe("loadScoreBank fail-closed activation", () => {
|
|
41
|
+
it("no .state dir at all ⇒ null — and the loader never creates one (peekPlanState discipline)", () => {
|
|
42
|
+
withRepo((repo) => {
|
|
43
|
+
assert.equal(loadScoreBank(repo), null);
|
|
44
|
+
assert.equal(existsSync(path.join(repo, STATE_DIR)), false);
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
it("live ledger only ⇒ null — rotation is the activation law", () => {
|
|
48
|
+
withRepo((repo) => {
|
|
49
|
+
const dir = stateDir(repo);
|
|
50
|
+
writeFileSync(path.join(dir, "ledger.jsonl"), generationLine({ source: "incumbent", head: "a".repeat(40), units: [{ unitId: "u", split: "train", scores: [0, 2, 1, 3] }] }));
|
|
51
|
+
assert.equal(loadScoreBank(repo), null);
|
|
52
|
+
});
|
|
53
|
+
});
|
|
54
|
+
it("ledger.corrupt-* is NOT a rotated archive; an empty dir ⇒ null", () => {
|
|
55
|
+
withRepo((repo) => {
|
|
56
|
+
const dir = stateDir(repo);
|
|
57
|
+
writeFileSync(path.join(dir, "ledger.corrupt-20260924-9.jsonl"), generationLine({ source: "incumbent", head: "a".repeat(40), units: [{ unitId: "u", split: "train", scores: [0, 2, 1, 3] }] }));
|
|
58
|
+
assert.equal(loadScoreBank(repo), null);
|
|
59
|
+
rmSync(path.join(dir, "ledger.corrupt-20260924-9.jsonl"));
|
|
60
|
+
writeFileSync(path.join(dir, "notes.txt"), "unrelated");
|
|
61
|
+
assert.equal(loadScoreBank(repo), null);
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
it("archive + live load; totalDf=0 (only single-rep groups ever) ⇒ null", () => {
|
|
65
|
+
withRepo((repo) => {
|
|
66
|
+
const dir = stateDir(repo);
|
|
67
|
+
writeFileSync(path.join(dir, "ledger.campaign1-x.jsonl"), generationLine({ source: "incumbent", head: "a".repeat(40), units: [{ unitId: "u", split: "train", scores: [1] }, { unitId: "v", split: "val", scores: [0.5] }] }));
|
|
68
|
+
writeFileSync(path.join(dir, "ledger.jsonl"), generationLine({ source: "incumbent", head: "b".repeat(40), units: [{ unitId: "u", split: "train", scores: [2] }] }));
|
|
69
|
+
assert.equal(loadScoreBank(repo), null);
|
|
70
|
+
});
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
describe("loadScoreBank arithmetic", () => {
|
|
74
|
+
const scatterRow = (head, tree, source, scores) => generationLine({ source, head, ...(tree === undefined ? {} : { tree }), units: [{ unitId: "u", split: "train", scores }] });
|
|
75
|
+
it("pools within-group SS across lineages; thin df floors σ at the quantum; df≥6 keeps the shrunk σ", () => {
|
|
76
|
+
withRepo((repo) => {
|
|
77
|
+
const dir = stateDir(repo);
|
|
78
|
+
// group g1 (campaign1 archive): [0,2] → mean 1, SS 2, df 1 — the ONLY scatter in the bank.
|
|
79
|
+
writeFileSync(path.join(dir, "ledger.campaign1-x.jsonl"), `${scatterRow("a".repeat(40), undefined, "incumbent", [0, 2])}` +
|
|
80
|
+
// group g2 (live): same unit, different lineage head, zero scatter
|
|
81
|
+
scatterRow("b".repeat(40), undefined, "incumbent", [1, 1]));
|
|
82
|
+
writeFileSync(path.join(dir, "ledger.jsonl"), generationLine({ source: "candidate", head: "b".repeat(40), tree: "t1", units: [{ unitId: "q", split: "train", scores: [1, 1, 1, 1] }] }));
|
|
83
|
+
const bank = loadScoreBank(repo);
|
|
84
|
+
assert.ok(bank !== null);
|
|
85
|
+
// totals: u groups df 1+1 (SS 2+0), q group df 3 (SS 0) ⇒ total df 5, prior σ² = 2/5.
|
|
86
|
+
assert.equal(bank.totalDf, 5);
|
|
87
|
+
assert.ok(Math.abs(bank.priorSigma - Math.sqrt(0.4)) < 1e-12);
|
|
88
|
+
// unit u: σ̂² = 2/2 = 1; shrunk = (2·1 + 2·0.4)/4 = 0.7 ⇒ σ = √0.7 (floor 0.0884 does not bind).
|
|
89
|
+
const u = bank.units.get("u");
|
|
90
|
+
assert.ok(u !== undefined);
|
|
91
|
+
assert.equal(u.df, 2);
|
|
92
|
+
assert.ok(Math.abs(u.sigma - Math.sqrt(0.7)) < 1e-12, `sigma ${String(u.sigma)}`);
|
|
93
|
+
// unit q: df 3, zero SS ⇒ shrunk toward the prior: (3·0 + 2·0.4)/5 = 0.16 ⇒ σ = 0.4 > floor.
|
|
94
|
+
const q = bank.units.get("q");
|
|
95
|
+
assert.ok(q !== undefined);
|
|
96
|
+
assert.ok(Math.abs(q.sigma - 0.4) < 1e-12);
|
|
97
|
+
});
|
|
98
|
+
});
|
|
99
|
+
it("zero-scatter bank: thin units sit exactly on BANK_QUANTUM, proven-quiet units (df≥6) on their shrunk σ", () => {
|
|
100
|
+
withRepo((repo) => {
|
|
101
|
+
const dir = stateDir(repo);
|
|
102
|
+
const thin = generationLine({ source: "incumbent", head: "a".repeat(40), units: [{ unitId: "thin", split: "train", scores: [0.5, 0.5] }] });
|
|
103
|
+
const wide = generationLine({ source: "incumbent", head: "b".repeat(40), units: [{ unitId: "wide", split: "train", scores: [1, 1, 1, 1, 1, 1] }] }) + generationLine({ source: "candidate", head: "b".repeat(40), tree: "x", units: [{ unitId: "wide", split: "train", scores: [1, 1, 1] }] });
|
|
104
|
+
writeFileSync(path.join(dir, "ledger.campaign1-x.jsonl"), thin + wide);
|
|
105
|
+
const bank = loadScoreBank(repo);
|
|
106
|
+
assert.ok(bank !== null);
|
|
107
|
+
assert.equal(bank.priorSigma, 0);
|
|
108
|
+
assert.equal(bank.units.get("thin")?.sigma, BANK_QUANTUM); // shrunk 0 → floored
|
|
109
|
+
assert.equal(bank.units.get("thin")?.df, 1);
|
|
110
|
+
assert.equal(bank.units.get("wide")?.sigma, 0); // df 4+1+… ≥ BANK_THIN_DF, zero prior ⇒ stays 0
|
|
111
|
+
assert.ok((bank.units.get("wide")?.df ?? 0) >= BANK_THIN_DF);
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
describe("loadScoreBank row hygiene", () => {
|
|
116
|
+
it("corrupt JSON lines and schema-invalid rows are skipped with notices; valid rows still price", () => {
|
|
117
|
+
withRepo((repo) => {
|
|
118
|
+
const dir = stateDir(repo);
|
|
119
|
+
const good = generationLine({ source: "incumbent", head: "a".repeat(40), units: [{ unitId: "u", split: "train", scores: [0, 2, 2, 0] }] });
|
|
120
|
+
writeFileSync(path.join(dir, "ledger.campaign1-x.jsonl"), `${good}{"v":1,"ts":"x\n${generationLine({ source: "incumbent", head: "a".repeat(40), units: [{ unitId: "bad", split: "nope", scores: [1, 2] }] })}`);
|
|
121
|
+
const bank = loadScoreBank(repo);
|
|
122
|
+
assert.ok(bank !== null);
|
|
123
|
+
assert.ok(bank.units.has("u"));
|
|
124
|
+
assert.equal(bank.units.has("bad"), false);
|
|
125
|
+
assert.equal(bank.notices.length, 2);
|
|
126
|
+
assert.match(bank.notices.join("\n"), /not JSON/);
|
|
127
|
+
assert.match(bank.notices.join("\n"), /unreadable/);
|
|
128
|
+
});
|
|
129
|
+
});
|
|
130
|
+
it("budget-truncated rows (complete:false) still contribute their real replicate scores", () => {
|
|
131
|
+
withRepo((repo) => {
|
|
132
|
+
const dir = stateDir(repo);
|
|
133
|
+
writeFileSync(path.join(dir, "ledger.campaign1-x.jsonl"), generationLine({ source: "candidate", head: "a".repeat(40), tree: "t", complete: false, units: [{ unitId: "u", split: "train", scores: [0, 2] }] }));
|
|
134
|
+
const bank = loadScoreBank(repo);
|
|
135
|
+
assert.equal(bank?.units.get("u")?.df, 1);
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
it("excludeCandidateTrees replays the gate-time world (rows written after evaluate)", () => {
|
|
139
|
+
withRepo((repo) => {
|
|
140
|
+
const dir = stateDir(repo);
|
|
141
|
+
writeFileSync(path.join(dir, "ledger.campaign1-x.jsonl"), generationLine({ source: "incumbent", head: "a".repeat(40), units: [{ unitId: "u", split: "train", scores: [0, 2] }] }) +
|
|
142
|
+
generationLine({ source: "candidate", head: "a".repeat(40), tree: "to-exclude", units: [{ unitId: "u", split: "train", scores: [4, 0] }] }));
|
|
143
|
+
const withIt = loadScoreBank(repo);
|
|
144
|
+
const without = loadScoreBank(repo, { excludeCandidateTrees: ["to-exclude"] });
|
|
145
|
+
assert.equal(withIt?.units.get("u")?.df, 2);
|
|
146
|
+
assert.equal(without?.units.get("u")?.df, 1);
|
|
147
|
+
assert.equal(withIt?.totalDf, 2);
|
|
148
|
+
assert.equal(without?.totalDf, 1);
|
|
149
|
+
});
|
|
150
|
+
});
|
|
151
|
+
it("excludeFiles hides a future campaign's ledger; null once no rotated archive remains", () => {
|
|
152
|
+
withRepo((repo) => {
|
|
153
|
+
const dir = stateDir(repo);
|
|
154
|
+
writeFileSync(path.join(dir, "ledger.campaign1-x.jsonl"), generationLine({ source: "incumbent", head: "a".repeat(40), units: [{ unitId: "u", split: "train", scores: [0, 2, 2, 0] }] }));
|
|
155
|
+
writeFileSync(path.join(dir, "ledger.campaign2-y.jsonl"), generationLine({ source: "incumbent", head: "b".repeat(40), units: [{ unitId: "u", split: "train", scores: [1, 5] }] }));
|
|
156
|
+
assert.equal(loadScoreBank(repo)?.units.get("u")?.df, 4);
|
|
157
|
+
const hidden = loadScoreBank(repo, { excludeFiles: ["ledger.campaign2-y.jsonl"] });
|
|
158
|
+
assert.equal(hidden?.units.get("u")?.df, 3);
|
|
159
|
+
assert.equal(hidden?.totalDf, 3);
|
|
160
|
+
assert.equal(loadScoreBank(repo, { excludeFiles: ["ledger.campaign1-x.jsonl", "ledger.campaign2-y.jsonl"] }), null);
|
|
161
|
+
});
|
|
162
|
+
});
|
|
163
|
+
});
|
|
164
|
+
describe("loadScoreBank bank-epoch lineage quarantine", () => {
|
|
165
|
+
const quietNew = {
|
|
166
|
+
source: "incumbent",
|
|
167
|
+
head: "h1",
|
|
168
|
+
ts: "2026-09-25T00:00:00.000Z",
|
|
169
|
+
units: [{ unitId: "scenario-9", split: "val", scores: [1, 1] }],
|
|
170
|
+
};
|
|
171
|
+
const noisyOld = {
|
|
172
|
+
source: "incumbent",
|
|
173
|
+
head: "h1",
|
|
174
|
+
ts: "2026-09-20T00:00:00.000Z",
|
|
175
|
+
units: [{ unitId: "scenario-9", split: "val", scores: [0, 1, 0, 1] }],
|
|
176
|
+
};
|
|
177
|
+
it("groups with ts before the epoch are quarantined: df drops, notice names the count and why", () => {
|
|
178
|
+
withRepo((repo) => {
|
|
179
|
+
const dir = stateDir(repo);
|
|
180
|
+
writeFileSync(path.join(dir, "ledger.campaign1-2026-09-20.jsonl"), generationLine(noisyOld) + generationLine(quietNew));
|
|
181
|
+
writeFileSync(path.join(dir, "bank-epoch.json"), JSON.stringify({ "scenario-9": { sinceIso: "2026-09-24T00:00:00Z", why: "pin era voided" } }));
|
|
182
|
+
const bank = loadScoreBank(repo);
|
|
183
|
+
assert.ok(bank !== null);
|
|
184
|
+
const stat = bank.units.get("scenario-9");
|
|
185
|
+
assert.ok(stat !== undefined);
|
|
186
|
+
assert.equal(stat.df, 1); // only the quiet new group counts
|
|
187
|
+
assert.ok(stat.sigma >= BANK_QUANTUM - 1e-12); // thin history floor holds the prior
|
|
188
|
+
assert.ok(bank.notices.some((n) => n.includes("bank-epoch 'scenario-9' since") && n.includes("1 group(s) quarantined") && n.includes("pin era voided")));
|
|
189
|
+
assert.ok(bank.notices.some((n) => n.includes("since 2026-09-24T00:00:00Z") && !n.includes("undefined")), "notice must print the ISO epoch, never undefined");
|
|
190
|
+
});
|
|
191
|
+
});
|
|
192
|
+
it("quarantined-only units leave zero pooled df ⇒ bank null ⇒ caller runs the legacy gate", () => {
|
|
193
|
+
withRepo((repo) => {
|
|
194
|
+
const dir = stateDir(repo);
|
|
195
|
+
writeFileSync(path.join(dir, "ledger.campaign1-2026-09-20.jsonl"), generationLine(noisyOld));
|
|
196
|
+
writeFileSync(path.join(dir, "bank-epoch.json"), JSON.stringify({ "scenario-9": { sinceIso: "2026-09-24T00:00:00Z" } }));
|
|
197
|
+
assert.equal(loadScoreBank(repo), null);
|
|
198
|
+
});
|
|
199
|
+
});
|
|
200
|
+
it("unreadable row ts fails closed for quarantined units but leaves unruled units untouched", () => {
|
|
201
|
+
withRepo((repo) => {
|
|
202
|
+
const dir = stateDir(repo);
|
|
203
|
+
const ruled = { ...noisyOld, ts: "not-a-date", units: [
|
|
204
|
+
{ unitId: "scenario-9", split: "val", scores: [0, 1] },
|
|
205
|
+
{ unitId: "scenario-8", split: "val", scores: [1, 0] },
|
|
206
|
+
] };
|
|
207
|
+
writeFileSync(path.join(dir, "ledger.campaign1-2026-09-20.jsonl"), generationLine(ruled));
|
|
208
|
+
writeFileSync(path.join(dir, "bank-epoch.json"), JSON.stringify({ "scenario-9": { sinceIso: "2026-09-24T00:00:00Z" } }));
|
|
209
|
+
const bank = loadScoreBank(repo);
|
|
210
|
+
assert.ok(bank !== null);
|
|
211
|
+
assert.equal(bank.units.has("scenario-9"), false); // provably-at-or-after failed ⇒ quarantined
|
|
212
|
+
assert.ok(bank.units.has("scenario-8")); // no rule ⇒ untouched
|
|
213
|
+
});
|
|
214
|
+
});
|
|
215
|
+
it("malformed quarantine sidecars disable the WHOLE bank (never a silently ignored quarantine)", () => {
|
|
216
|
+
const bad = ["{not json", JSON.stringify({ "scenario-9": { sinceIso: "not-a-date" } }), JSON.stringify({ "scenario-9": { sinceIso: "2026-09-24T00:00:00Z", extra: 1 } }), JSON.stringify(["scenario-9"])];
|
|
217
|
+
for (const body of bad) {
|
|
218
|
+
withRepo((repo) => {
|
|
219
|
+
const dir = stateDir(repo);
|
|
220
|
+
writeFileSync(path.join(dir, "ledger.campaign1-2026-09-20.jsonl"), generationLine(noisyOld) + generationLine(quietNew));
|
|
221
|
+
writeFileSync(path.join(dir, "bank-epoch.json"), body);
|
|
222
|
+
assert.equal(loadScoreBank(repo), null, `sidecar must disable bank: ${body.slice(0, 24)}`);
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
});
|
|
226
|
+
it("absent sidecar = no quarantine, byte-identical legacy behavior", () => {
|
|
227
|
+
withRepo((repo) => {
|
|
228
|
+
const dir = stateDir(repo);
|
|
229
|
+
writeFileSync(path.join(dir, "ledger.campaign1-2026-09-20.jsonl"), generationLine(noisyOld) + generationLine(quietNew));
|
|
230
|
+
const bank = loadScoreBank(repo);
|
|
231
|
+
assert.ok(bank !== null);
|
|
232
|
+
assert.equal(bank.units.get("scenario-9")?.df, 4); // 3 df old + 1 df new — nothing skipped
|
|
233
|
+
});
|
|
234
|
+
});
|
|
235
|
+
});
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
// Seam test (postmortem 863e6c2): the human-gate commands `promote` and
|
|
2
|
+
// `tombstone` must resolve an env-literal repoPath exactly like status/run
|
|
3
|
+
// (4997618 pattern). Pre-fix, promote.ts opened the ledger under the STORED
|
|
4
|
+
// UNRESOLVED `${VAR}` literal, read an empty ledger, and reported the
|
|
5
|
+
// misleading "no generation_complete ledger row" with the literal still in
|
|
6
|
+
// the hint — the same fail-open seam class as the status spawn-ENOENT.
|
|
7
|
+
// Pins, for both gates:
|
|
8
|
+
// (1) env exported ⇒ gate reads the REAL ledger at the resolved path:
|
|
9
|
+
// the refusal names the resolved repo, never the `${VAR}` literal;
|
|
10
|
+
// (2) env unset ⇒ clean exit 2 NAMING the variable;
|
|
11
|
+
// (3) stderr never contains the unresolved literal or a spawn-ENOENT.
|
|
12
|
+
import assert from "node:assert/strict";
|
|
13
|
+
import { spawnSync } from "node:child_process";
|
|
14
|
+
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
15
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
16
|
+
import * as os from "node:os";
|
|
17
|
+
import path from "node:path";
|
|
18
|
+
import { fileURLToPath } from "node:url";
|
|
19
|
+
import { test } from "node:test";
|
|
20
|
+
import { registerGenome } from "../core/genome.js";
|
|
21
|
+
import { prepareToyGenome } from "../bench/toy.js";
|
|
22
|
+
const CLI = fileURLToPath(new URL("../../dist/cli.js", import.meta.url));
|
|
23
|
+
const LIT_VAR = "ABATHUR_SEAM_GATES_REPO";
|
|
24
|
+
async function fixture(t, label) {
|
|
25
|
+
const root = mkdtempSync(path.join(os.tmpdir(), "seam-gates-"));
|
|
26
|
+
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
27
|
+
const configDir = path.join(root, "config");
|
|
28
|
+
mkdirSync(configDir, { recursive: true });
|
|
29
|
+
writeFileSync(path.join(configDir, "config.jsonc"), '{ "opencodeBin": null }\n', "utf8");
|
|
30
|
+
const repo = await prepareToyGenome(path.join(root, "genome"));
|
|
31
|
+
const doc = JSON.parse(readFileSync(path.join(repo, "genome.jsonc"), "utf8"));
|
|
32
|
+
doc["label"] = label;
|
|
33
|
+
doc["repoPath"] = `\${${LIT_VAR}}`;
|
|
34
|
+
const specFile = path.join(root, "private.jsonc");
|
|
35
|
+
writeFileSync(specFile, `${JSON.stringify(doc, null, 2)}\n`, "utf8");
|
|
36
|
+
process.env[LIT_VAR] = repo;
|
|
37
|
+
try {
|
|
38
|
+
registerGenome(configDir, specFile);
|
|
39
|
+
}
|
|
40
|
+
finally {
|
|
41
|
+
delete process.env[LIT_VAR];
|
|
42
|
+
}
|
|
43
|
+
return { root, configDir, repo, label };
|
|
44
|
+
}
|
|
45
|
+
function cli(f, args, env) {
|
|
46
|
+
return spawnSync(process.execPath, [CLI, ...args], {
|
|
47
|
+
env: { ...process.env, ABATHUR_CONFIG: path.join(f.configDir, "config.jsonc"), ...env },
|
|
48
|
+
encoding: "utf8",
|
|
49
|
+
timeout: 60_000,
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
function assertNoLiteralOrEnoent(r) {
|
|
53
|
+
const blob = `${r.stdout}\n${r.stderr}`;
|
|
54
|
+
assert.ok(!blob.includes(`\${${LIT_VAR}}`), `unresolved literal leaked into output:\n${blob}`);
|
|
55
|
+
assert.ok(!blob.includes("ENOENT"), `misleading ENOENT leaked into output:\n${blob}`);
|
|
56
|
+
}
|
|
57
|
+
test("promote with env-literal repoPath: exported ⇒ blocked naming the RESOLVED repo path", async (t) => {
|
|
58
|
+
const f = await fixture(t, "seam-gate-promote");
|
|
59
|
+
const r = cli(f, ["promote", f.label, "g-nonexistent"], { [LIT_VAR]: f.repo });
|
|
60
|
+
assert.equal(r.status, 1, `stdout: ${r.stdout}\nstderr: ${r.stderr}`);
|
|
61
|
+
assertNoLiteralOrEnoent(r);
|
|
62
|
+
assert.ok(r.stderr.includes(f.repo), `refusal must name the resolved repo path so the operator can debug:\n${r.stderr}`);
|
|
63
|
+
});
|
|
64
|
+
test("tombstone with env-literal repoPath: exported ⇒ blocked, no literal leak", async (t) => {
|
|
65
|
+
const f = await fixture(t, "seam-gate-tombstone");
|
|
66
|
+
const r = cli(f, ["tombstone", f.label, "g-nonexistent", "--reason", "seam test"], { [LIT_VAR]: f.repo });
|
|
67
|
+
assert.equal(r.status, 1, `stdout: ${r.stdout}\nstderr: ${r.stderr}`);
|
|
68
|
+
assertNoLiteralOrEnoent(r);
|
|
69
|
+
});
|
|
70
|
+
test("promote with env-literal repoPath: unset ⇒ clean exit 2 naming the variable", async (t) => {
|
|
71
|
+
const f = await fixture(t, "seam-gate-promote-unset");
|
|
72
|
+
const env = {};
|
|
73
|
+
env[LIT_VAR] = undefined;
|
|
74
|
+
const r = cli(f, ["promote", f.label, "g-nonexistent"], env);
|
|
75
|
+
assert.equal(r.status, 2, `stdout: ${r.stdout}\nstderr: ${r.stderr}`);
|
|
76
|
+
assert.ok(r.stderr.includes(LIT_VAR), `exit-2 message must name the env var:\n${r.stderr}`);
|
|
77
|
+
assert.ok(!r.stderr.includes("ENOENT"), `misleading ENOENT leaked into output:\n${r.stderr}`);
|
|
78
|
+
});
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
// Campaign-1 defect pins (historian evidence task-07, live symptom rc=2):
|
|
2
|
+
// a `${VAR}` env-literal repoPath is a PATH-INJECTION convenience, NOT an
|
|
3
|
+
// engine-self marker. run-loop.ts routed selfMode on isEnvRepoLiteral — ANY
|
|
4
|
+
// literal — so the historian genome (`${ABATHUR_HISTORIAN_REPO}`) was forced
|
|
5
|
+
// into the tsc-build/toy-replay snapshot-overlay bench and died fail-closed
|
|
6
|
+
// demanding `selfbench/expected.json`. selfMode must mean
|
|
7
|
+
// `envRepoName(repoPath) === "ABATHUR_SELF_REPO"`. Secondary pin: the dry-run
|
|
8
|
+
// plan must display the RESOLVED repo path, never the raw literal joined to
|
|
9
|
+
// cwd (run-plan.ts printed `/…/historian/${ABATHUR_HISTORIAN_REPO}`).
|
|
10
|
+
// Harness mirrors run-loop.test.ts: toy genome + stub mutator, no model calls.
|
|
11
|
+
import assert from "node:assert/strict";
|
|
12
|
+
import { spawnSync } from "node:child_process";
|
|
13
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync, chmodSync } from "node:fs";
|
|
14
|
+
import { mkdtemp } from "node:fs/promises";
|
|
15
|
+
import * as os from "node:os";
|
|
16
|
+
import path from "node:path";
|
|
17
|
+
import test, {} from "node:test";
|
|
18
|
+
import { ExitSignal } from "../exit.js";
|
|
19
|
+
import { registerGenome, requireGenomesByLabel } from "../core/genome.js";
|
|
20
|
+
import { Ledger } from "../core/ledger.js";
|
|
21
|
+
import { prepareToyGenome } from "../bench/toy.js";
|
|
22
|
+
import { runEvolution } from "../core/evolve/run-loop.js";
|
|
23
|
+
// Same three-candidate stub script as run-loop.test.ts (annotate-add seals,
|
|
24
|
+
// touch-kernel path-rejected, fix-add nominated).
|
|
25
|
+
const ADD_ANCHOR = "return a - b; // seeded bug: must be `return a + b;`";
|
|
26
|
+
const STUB_SOURCE = `#!/usr/bin/env node
|
|
27
|
+
import { readFileSync } from "node:fs";
|
|
28
|
+
const args = process.argv.slice(2);
|
|
29
|
+
const opt = (n) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : undefined; };
|
|
30
|
+
const mode = opt("--mode");
|
|
31
|
+
const dir = opt("--dir");
|
|
32
|
+
const read = (f) => readFileSync(dir + "/" + f, "utf8");
|
|
33
|
+
function lineDiff(file, anchor, replacement) {
|
|
34
|
+
const lines = read(file).split("\\n");
|
|
35
|
+
const i = lines.findIndex((l) => l.includes(anchor));
|
|
36
|
+
if (i < 0) { process.stderr.write("stub: no anchor in " + file + "\\n"); process.exit(1); }
|
|
37
|
+
const changed = lines[i].replace(anchor, replacement);
|
|
38
|
+
return "--- a/" + file + "\\n+++ b/" + file + "\\n@@ -" + String(i + 1) + ",1 +" + String(i + 1) + ",1 @@\\n-" + lines[i] + "\\n+" + changed + "\\n";
|
|
39
|
+
}
|
|
40
|
+
const FIX = ${JSON.stringify(ADD_ANCHOR)};
|
|
41
|
+
const out = (candidates) => process.stdout.write(JSON.stringify({ candidates }) + "\\n");
|
|
42
|
+
if (mode === "three") {
|
|
43
|
+
out([
|
|
44
|
+
{ id: "annotate-add", rationale: "annotate add()", diffs: [lineDiff("units/add.mjs", "export function add(", "export function add /* patched */(")] },
|
|
45
|
+
{ id: "touch-kernel", rationale: "subvert grading", diffs: [lineDiff("grader.mjs", "model-free", "model-free-ish")] },
|
|
46
|
+
{ id: "fix-add", rationale: "fix add(): subtraction to addition", diffs: [lineDiff("units/add.mjs", FIX, "return a + b;")] },
|
|
47
|
+
]);
|
|
48
|
+
} else {
|
|
49
|
+
process.stderr.write("stub: unknown mode\\n");
|
|
50
|
+
process.exit(2);
|
|
51
|
+
}
|
|
52
|
+
`;
|
|
53
|
+
/**
|
|
54
|
+
* Toy genome registered with repoPath stored as the UNRESOLVED literal
|
|
55
|
+
* `${<envName>}` (the self-seed trick, pointed at the toy repo itself), env
|
|
56
|
+
* var set for the duration of the test. Fresh configDir per test.
|
|
57
|
+
*/
|
|
58
|
+
async function literalFixture(t, envName) {
|
|
59
|
+
const root = await mkdtemp(path.join(os.tmpdir(), "abathur-selfmode-"));
|
|
60
|
+
t.after(() => {
|
|
61
|
+
// trusted snapshots are chmod-protected read-only: restore writability first.
|
|
62
|
+
spawnSync("chmod", ["-R", "u+rwX", root], { encoding: "utf8" });
|
|
63
|
+
spawnSync("rm", ["-rf", root], { encoding: "utf8" });
|
|
64
|
+
});
|
|
65
|
+
const configDir = path.join(root, "config");
|
|
66
|
+
mkdirSync(configDir, { recursive: true });
|
|
67
|
+
writeFileSync(path.join(configDir, "config.jsonc"), '{ "opencodeBin": null }\n', "utf8");
|
|
68
|
+
const repo = await prepareToyGenome(path.join(root, "genome"));
|
|
69
|
+
const specPath = path.join(repo, "genome.jsonc");
|
|
70
|
+
const doc = JSON.parse(readFileSync(specPath, "utf8"));
|
|
71
|
+
doc.repoPath = `\${${envName}}`;
|
|
72
|
+
writeFileSync(specPath, `${JSON.stringify(doc, null, 2)}\n`, "utf8");
|
|
73
|
+
const saved = process.env[envName];
|
|
74
|
+
process.env[envName] = repo;
|
|
75
|
+
t.after(() => {
|
|
76
|
+
if (saved === undefined)
|
|
77
|
+
delete process.env[envName];
|
|
78
|
+
else
|
|
79
|
+
process.env[envName] = saved;
|
|
80
|
+
});
|
|
81
|
+
registerGenome(configDir, specPath);
|
|
82
|
+
const entry = requireGenomesByLabel(configDir, "toy-smoke").entries[0];
|
|
83
|
+
if (entry === undefined)
|
|
84
|
+
throw new Error("fixture: toy-smoke registration vanished");
|
|
85
|
+
const stub = path.join(root, "stub.mjs");
|
|
86
|
+
writeFileSync(stub, STUB_SOURCE, "utf8");
|
|
87
|
+
chmodSync(stub, 0o755);
|
|
88
|
+
return {
|
|
89
|
+
root,
|
|
90
|
+
configDir,
|
|
91
|
+
repo,
|
|
92
|
+
entry,
|
|
93
|
+
mutatorCommand: `node ${stub} --mode three --dir {worktree} --brief {brief}`,
|
|
94
|
+
env: { XDG_CACHE_HOME: path.join(root, "xdg-cache"), HOME: path.join(root, "home") },
|
|
95
|
+
sandboxRoot: path.join(root, "sandboxes"),
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
function generationRowCount(f) {
|
|
99
|
+
return Ledger.open(f.repo)
|
|
100
|
+
.readAll()
|
|
101
|
+
.filter((r) => r.kind === "generation_complete").length;
|
|
102
|
+
}
|
|
103
|
+
// ------------------------------------------------- non-self literal: normal bench
|
|
104
|
+
test("campaign-1 defect: a non-self env-literal repoPath must NOT enter the self-bench overlay", async (t) => {
|
|
105
|
+
const f = await literalFixture(t, "ABATHUR_HISTORIAN_REPO");
|
|
106
|
+
const out = await runEvolution({
|
|
107
|
+
entry: f.entry,
|
|
108
|
+
configDir: f.configDir,
|
|
109
|
+
mutatorCommand: f.mutatorCommand,
|
|
110
|
+
env: f.env,
|
|
111
|
+
sandboxRoot: f.sandboxRoot,
|
|
112
|
+
});
|
|
113
|
+
const plan = out.lines.join("\n");
|
|
114
|
+
assert.equal(out.exitCode, 0, `expected exit 0, lines: ${plan}`);
|
|
115
|
+
assert.ok(!/selfbench|self-snapshot|self-bench|overlay/i.test(plan), `non-self literal must never touch the self-bench machinery: ${plan}`);
|
|
116
|
+
assert.ok(generationRowCount(f) >= 2, "incumbent + candidate rows benched through the NORMAL toy path");
|
|
117
|
+
assert.equal(existsSync(path.join(f.root, "selfbench")), false);
|
|
118
|
+
});
|
|
119
|
+
// ------------------------------------- self literal: overlay path preserved
|
|
120
|
+
test("regression pin: the ${ABATHUR_SELF_REPO} literal still routes into the self-bench path", async (t) => {
|
|
121
|
+
// The toy repo carries no selfbench/expected.json, so entering the overlay
|
|
122
|
+
// bench FAILS CLOSED with exactly that demand — the observable proof that
|
|
123
|
+
// selfMode is still true for the engine-self literal (live symptom shape).
|
|
124
|
+
const f = await literalFixture(t, "ABATHUR_SELF_REPO");
|
|
125
|
+
await assert.rejects(runEvolution({
|
|
126
|
+
entry: f.entry,
|
|
127
|
+
configDir: f.configDir,
|
|
128
|
+
mutatorCommand: f.mutatorCommand,
|
|
129
|
+
env: f.env,
|
|
130
|
+
sandboxRoot: f.sandboxRoot,
|
|
131
|
+
}), (cause) => {
|
|
132
|
+
assert.ok(cause instanceof ExitSignal, `expected ExitSignal, got ${String(cause)}`);
|
|
133
|
+
assert.equal(cause.code, 2);
|
|
134
|
+
assert.match(cause.message, /self-snapshot: trusted selfbench\/expected\.json missing/);
|
|
135
|
+
return true;
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
// ----------------------------------------------------- dry-run plan display
|
|
139
|
+
test("--dry-run plan displays the RESOLVED repo path, never the literal against cwd", async (t) => {
|
|
140
|
+
const f = await literalFixture(t, "ABATHUR_HISTORIAN_REPO");
|
|
141
|
+
const out = await runEvolution({
|
|
142
|
+
entry: f.entry,
|
|
143
|
+
configDir: f.configDir,
|
|
144
|
+
mutatorCommand: f.mutatorCommand,
|
|
145
|
+
dryRun: true,
|
|
146
|
+
env: f.env,
|
|
147
|
+
sandboxRoot: f.sandboxRoot,
|
|
148
|
+
});
|
|
149
|
+
assert.equal(out.exitCode, 0, out.lines.join("\n"));
|
|
150
|
+
const plan = out.lines.join("\n");
|
|
151
|
+
assert.ok(plan.includes(`repo: ${f.repo}`), `plan must show the resolved path:\n${plan}`);
|
|
152
|
+
assert.ok(!plan.includes("${"), `plan must never echo the raw literal (resolved against cwd garbage):\n${plan}`);
|
|
153
|
+
});
|