@tachikomagundam/abathur 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/ci.yml +29 -0
- package/.github/workflows/publish.yml +74 -0
- package/LICENSE +21 -0
- package/README.md +461 -0
- package/config/abathur.jsonc +17 -0
- package/config/genomes/historian.example.jsonc +124 -0
- package/dist/bench/adapter.js +201 -0
- package/dist/bench/fixture-probe.js +92 -0
- package/dist/bench/fixture-support.js +173 -0
- package/dist/bench/fixture.js +236 -0
- package/dist/bench/toy.js +152 -0
- package/dist/cli.js +110 -0
- package/dist/commands/bundle.js +79 -0
- package/dist/commands/genome.js +94 -0
- package/dist/commands/graft.js +71 -0
- package/dist/commands/kernel.js +47 -0
- package/dist/commands/promote.js +25 -0
- package/dist/commands/run.js +145 -0
- package/dist/commands/self-eval.js +240 -0
- package/dist/commands/status.js +186 -0
- package/dist/commands/tombstone.js +72 -0
- package/dist/config.js +161 -0
- package/dist/core/bundle-common.js +119 -0
- package/dist/core/bundle-export.js +212 -0
- package/dist/core/bundle-inspect.js +143 -0
- package/dist/core/bundle-manifest.js +105 -0
- package/dist/core/bundle-mask.js +75 -0
- package/dist/core/bundle-tar.js +240 -0
- package/dist/core/bundle.js +9 -0
- package/dist/core/evolve/brief.js +45 -0
- package/dist/core/evolve/candidate.js +140 -0
- package/dist/core/evolve/child-track.js +197 -0
- package/dist/core/evolve/friction.js +150 -0
- package/dist/core/evolve/reflect.js +191 -0
- package/dist/core/evolve/run-bench.js +170 -0
- package/dist/core/evolve/run-friction.js +63 -0
- package/dist/core/evolve/run-loop.js +282 -0
- package/dist/core/evolve/run-plan.js +39 -0
- package/dist/core/evolve/run-rows.js +145 -0
- package/dist/core/evolve/self-overlay.js +213 -0
- package/dist/core/evolve/self-snapshot.js +170 -0
- package/dist/core/evolve/stub-mutators.mjs +105 -0
- package/dist/core/evolve/udiff.js +189 -0
- package/dist/core/genome-paths.js +76 -0
- package/dist/core/genome.js +176 -0
- package/dist/core/glob.js +106 -0
- package/dist/core/graft-gates.js +184 -0
- package/dist/core/graft-rebench.js +187 -0
- package/dist/core/graft-support.js +181 -0
- package/dist/core/graft.js +218 -0
- package/dist/core/ids.js +154 -0
- package/dist/core/incumbent.js +46 -0
- package/dist/core/kernel.js +112 -0
- package/dist/core/ledger.js +198 -0
- package/dist/core/locks.js +172 -0
- package/dist/core/promote.js +119 -0
- package/dist/core/snapshot.js +61 -0
- package/dist/core/spec.js +178 -0
- package/dist/core/stats-math.js +102 -0
- package/dist/core/stats-pareto.js +57 -0
- package/dist/core/stats.js +184 -0
- package/dist/core/worktree.js +190 -0
- package/dist/exit.js +32 -0
- package/dist/genomes/toy-smoke/genome.jsonc +30 -0
- package/dist/genomes/toy-smoke/grader.mjs +61 -0
- package/dist/genomes/toy-smoke/init.mjs +63 -0
- package/dist/genomes/toy-smoke/units/add.mjs +17 -0
- package/dist/genomes/toy-smoke/units/explode.mjs +4 -0
- package/dist/genomes/toy-smoke/units/hang.mjs +16 -0
- package/dist/genomes/toy-smoke/units/mul.mjs +16 -0
- package/dist/genomes/toy-smoke/units/mutate.mjs +18 -0
- package/dist/genomes/toy-smoke/units/sub.mjs +16 -0
- package/dist/jsonc.js +77 -0
- package/dist/out.js +5 -0
- package/dist/test/bench-adapter.test.js +33 -0
- package/dist/test/bench-fixture.test.js +407 -0
- package/dist/test/bench-toy.test.js +251 -0
- package/dist/test/bundle.test.js +659 -0
- package/dist/test/config.test.js +185 -0
- package/dist/test/d7-gate.test.js +56 -0
- package/dist/test/fixture-loop.test.js +267 -0
- package/dist/test/fixtures/friction-writer.js +16 -0
- package/dist/test/fixtures-historian.js +82 -0
- package/dist/test/fixtures-self.js +143 -0
- package/dist/test/fixtures-wt.js +64 -0
- package/dist/test/friction.test.js +398 -0
- package/dist/test/genome.test.js +453 -0
- package/dist/test/git.test.js +69 -0
- package/dist/test/graft.test.js +567 -0
- package/dist/test/historian-genome.test.js +134 -0
- package/dist/test/historian-grader-io.test.js +148 -0
- package/dist/test/historian-grader.test.js +209 -0
- package/dist/test/ids.test.js +116 -0
- package/dist/test/include-val.test.js +120 -0
- package/dist/test/ledger-lock.test.js +99 -0
- package/dist/test/ledger.test.js +102 -0
- package/dist/test/promote.test.js +394 -0
- package/dist/test/reflect.test.js +410 -0
- package/dist/test/run-loop.test.js +433 -0
- package/dist/test/self-snapshot.test.js +328 -0
- package/dist/test/snapshot.test.js +86 -0
- package/dist/test/stats.test.js +423 -0
- package/dist/test/stub-mutators.test.js +17 -0
- package/dist/test/testutil.js +30 -0
- package/dist/test/worktree.test.js +198 -0
- package/dist/util/freeze.js +30 -0
- package/dist/util/git.js +85 -0
- package/docs/federation.md +184 -0
- package/docs/immutable-kernel.md +87 -0
- package/graders/historian/grader-core.d.mts +53 -0
- package/graders/historian/grader-core.mjs +276 -0
- package/graders/historian/grader-support.d.mts +57 -0
- package/graders/historian/grader-support.mjs +137 -0
- package/graders/historian/grader.mjs +113 -0
- package/graders/historian/mutate.sh +114 -0
- package/graders/historian/reset-sandbox.sh +60 -0
- package/graders/historian/run-scenario.sh +49 -0
- package/graders/historian/seed-wrapped.sh +32 -0
- package/package.json +42 -0
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
// Todo 11 snapshot-overlay self-bench pins (plan 157-164). One shared TRIMMED
|
|
2
|
+
// harness copy (fixtures-self.ts) carries the fixture cost: it is a real git
|
|
3
|
+
// repo of the harness tree minus src/test, with tiny trusted tests, the seed
|
|
4
|
+
// genome and a captured golden-replay expected digest — the real repo is never
|
|
5
|
+
// touched by any bench here.
|
|
6
|
+
// AC-5: golden replay digest identical across two invocations and == expected;
|
|
7
|
+
// AC-4: candidate deleting a trusted test file (and editing another) ⇒ zero
|
|
8
|
+
// score effect, drops reported as 'trusted-tests';
|
|
9
|
+
// sealed-core candidate never overlays (AC-3 first half); the vacuous-green
|
|
10
|
+
// overlay-empty guard downgrades nominations to indeterminate;
|
|
11
|
+
// benign unsealed candidate overlays and stays nominated (non-degenerate);
|
|
12
|
+
// broken-build candidate scores 0 and culls; hung suite is killed at cap;
|
|
13
|
+
// build-timeout guard => inconclusive; timeout caps pinned;
|
|
14
|
+
// run --genome abathur-self end-to-end: storm stub (sealed reject + schema
|
|
15
|
+
// reject + benign candidate), ledger rows, friction digest, exit 0;
|
|
16
|
+
// self-eval CLI reports verdicts/drops/digest + friction, writes no ledger.
|
|
17
|
+
import assert from "node:assert/strict";
|
|
18
|
+
import { spawnSync } from "node:child_process";
|
|
19
|
+
import { mkdirSync, readFileSync, writeFileSync, chmodSync, rmSync } from "node:fs";
|
|
20
|
+
import path from "node:path";
|
|
21
|
+
import test, {} from "node:test";
|
|
22
|
+
import { genId, fingerprint } from "../core/ids.js";
|
|
23
|
+
import { registerGenome, requireGenomesByLabel, fingerprint16 } from "../core/genome.js";
|
|
24
|
+
import { Ledger } from "../core/ledger.js";
|
|
25
|
+
import { decodeGenerationRecord } from "../core/evolve/run-bench.js";
|
|
26
|
+
import { runEvolution } from "../core/evolve/run-loop.js";
|
|
27
|
+
import { DEFAULT_BUILD_TIMEOUT_S, DEFAULT_REPLAY_TIMEOUT_S, DEFAULT_TEST_TIMEOUT_S, SELF_BUILD_TIMEOUT, SELF_OVERLAY_EMPTY, selfBench, selfGuardVerdict, } from "../core/evolve/self-snapshot.js";
|
|
28
|
+
import { appendRunFriction } from "../core/evolve/friction.js";
|
|
29
|
+
import { prepareToyGenome } from "../bench/toy.js";
|
|
30
|
+
import { captureReplayDigest, editFile, headSha, makeSelfHarness, seedExpectedDigest } from "./fixtures-self.js";
|
|
31
|
+
const HARNESS_ROOT = path.resolve(new URL("../../", import.meta.url).pathname);
|
|
32
|
+
export const STUB_SELF = `#!/usr/bin/env node
|
|
33
|
+
import { readFileSync } from "node:fs";
|
|
34
|
+
const args = process.argv.slice(2);
|
|
35
|
+
const opt = (n) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : undefined; };
|
|
36
|
+
const dir = opt("--dir");
|
|
37
|
+
const read = (f) => readFileSync(dir + "/" + f, "utf8");
|
|
38
|
+
function lineDiff(file, anchor, replacement) {
|
|
39
|
+
const lines = read(file).split("\\n");
|
|
40
|
+
const i = lines.findIndex((l) => l.includes(anchor));
|
|
41
|
+
if (i < 0) { process.stderr.write("stub: no anchor in " + file + "\\n"); process.exit(1); }
|
|
42
|
+
const changed = lines[i].replace(anchor, replacement);
|
|
43
|
+
return "--- a/" + file + "\\n+++ b/" + file + "\\n@@ -" + String(i + 1) + ",1 +" + String(i + 1) + ",1 @@\\n-" + lines[i] + "\\n+" + changed + "\\n";
|
|
44
|
+
}
|
|
45
|
+
process.stdout.write(JSON.stringify({ candidates: [
|
|
46
|
+
{ id: "stats-tamper", rationale: "sealed-core rewrite", diffs: [lineDiff("src/core/stats.ts", "export const FAMILY_ALPHA", "export const FAMILY_ALPHA // tampered")] },
|
|
47
|
+
{ id: "no-diffs", rationale: "schema damage" },
|
|
48
|
+
{ id: "annotate-out", rationale: "comment-only unsealed edit", diffs: [lineDiff("src/out.ts", "// stdout funnel (todo 1 seam)", "// stdout funnel (todo 1 seam, patched)")] },
|
|
49
|
+
] }) + "\\n");
|
|
50
|
+
`;
|
|
51
|
+
function selfBenchReq(fx, over) {
|
|
52
|
+
return {
|
|
53
|
+
spec: fx.entry.spec,
|
|
54
|
+
genomeRepo: fx.harness.repo,
|
|
55
|
+
genomeFp: fx.genomeFp,
|
|
56
|
+
incumbentCommit: fx.head,
|
|
57
|
+
candidateCommit: over.candidateCommit,
|
|
58
|
+
genId: over.genId,
|
|
59
|
+
reps: 2,
|
|
60
|
+
env: fx.harness.env,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
async function buildFixture(t) {
|
|
64
|
+
const harness = makeSelfHarness(t);
|
|
65
|
+
mkdirSync(harness.configDir, { recursive: true });
|
|
66
|
+
writeFileSync(path.join(harness.configDir, "config.jsonc"), '{ "opencodeBin": null }\n', "utf8");
|
|
67
|
+
const digest = captureReplayDigest(harness.repo);
|
|
68
|
+
const head = seedExpectedDigest(harness, digest);
|
|
69
|
+
const stub = path.join(harness.root, "stub-self.mjs");
|
|
70
|
+
writeFileSync(stub, STUB_SELF, "utf8");
|
|
71
|
+
chmodSync(stub, 0o755);
|
|
72
|
+
const saved = process.env.ABATHUR_SELF_REPO;
|
|
73
|
+
try {
|
|
74
|
+
process.env.ABATHUR_SELF_REPO = harness.repo;
|
|
75
|
+
registerGenome(harness.configDir, harness.specPath);
|
|
76
|
+
// a NON-self toy genome in the same home: self-eval must refuse to grade it.
|
|
77
|
+
const toyRepo = await prepareToyGenome(path.join(harness.root, "toy-genome"));
|
|
78
|
+
registerGenome(harness.configDir, path.join(toyRepo, "genome.jsonc"));
|
|
79
|
+
const entry = requireGenomesByLabel(harness.configDir, "abathur-self").entries[0];
|
|
80
|
+
if (entry === undefined)
|
|
81
|
+
throw new Error("fixture: abathur-self vanished from registry");
|
|
82
|
+
const genomeFp = fingerprint16(entry.spec);
|
|
83
|
+
const incumbent = await selfBench({
|
|
84
|
+
spec: entry.spec,
|
|
85
|
+
genomeRepo: harness.repo,
|
|
86
|
+
genomeFp,
|
|
87
|
+
incumbentCommit: head,
|
|
88
|
+
candidateCommit: null,
|
|
89
|
+
genId: genId(fingerprint({ seed: head }), new Date()),
|
|
90
|
+
reps: 2,
|
|
91
|
+
env: harness.env,
|
|
92
|
+
});
|
|
93
|
+
return { harness, head, entry, genomeFp, expected: digest, incumbent, stub };
|
|
94
|
+
}
|
|
95
|
+
finally {
|
|
96
|
+
if (saved === undefined)
|
|
97
|
+
delete process.env.ABATHUR_SELF_REPO;
|
|
98
|
+
else
|
|
99
|
+
process.env.ABATHUR_SELF_REPO = saved;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
test("self-snapshot suite", async (t) => {
|
|
103
|
+
const fx = await buildFixture(t);
|
|
104
|
+
const saved = process.env.ABATHUR_SELF_REPO;
|
|
105
|
+
process.env.ABATHUR_SELF_REPO = fx.harness.repo;
|
|
106
|
+
t.after(() => {
|
|
107
|
+
if (saved === undefined)
|
|
108
|
+
delete process.env.ABATHUR_SELF_REPO;
|
|
109
|
+
else
|
|
110
|
+
process.env.ABATHUR_SELF_REPO = saved;
|
|
111
|
+
});
|
|
112
|
+
await t.test("fixture baseline: incumbent suite green, replay digest == trusted expected", () => {
|
|
113
|
+
assert.equal(fx.incumbent.buildStatus, "ok");
|
|
114
|
+
assert.equal(fx.incumbent.replayDigest, fx.expected);
|
|
115
|
+
assert.equal(fx.incumbent.replayExpected, fx.expected);
|
|
116
|
+
for (const run of fx.incumbent.suiteRuns) {
|
|
117
|
+
assert.deepEqual([run.tests, run.fail], [4, 0]);
|
|
118
|
+
}
|
|
119
|
+
assert.deepEqual(fx.incumbent.dropped, []);
|
|
120
|
+
assert.deepEqual(fx.incumbent.overlaid, []);
|
|
121
|
+
assert.equal(fx.incumbent.complete, true);
|
|
122
|
+
assert.equal(fx.incumbent.spent.modelCalls, 0);
|
|
123
|
+
});
|
|
124
|
+
await t.test("AC-5: golden replay digest identical across two invocations", async () => {
|
|
125
|
+
const again = await selfBench(selfBenchReq(fx, { genId: genId(fingerprint({ rerun: 1 }), new Date()), candidateCommit: null }));
|
|
126
|
+
const third = await selfBench(selfBenchReq(fx, { genId: genId(fingerprint({ rerun: 2 }), new Date()), candidateCommit: null }));
|
|
127
|
+
assert.equal(again.replayDigest, fx.expected);
|
|
128
|
+
assert.equal(third.replayDigest, fx.expected);
|
|
129
|
+
assert.equal(again.replayDigest, third.replayDigest);
|
|
130
|
+
assert.deepEqual(again.units.map((u) => u.scores), fx.incumbent.units.map((u) => u.scores));
|
|
131
|
+
});
|
|
132
|
+
await t.test("AC-4: candidate deleting a trusted test + editing another ⇒ zero score effect", async () => {
|
|
133
|
+
const cand = commitSurgery(fx, "drop+edit trusted tests", () => {
|
|
134
|
+
rmSync(path.join(fx.harness.repo, "src/test/self-mutators.test.ts"));
|
|
135
|
+
editFile(fx.harness.repo, "src/test/self-sanity.test.ts", (s) => `${s}\n// candidate tampering\n`);
|
|
136
|
+
});
|
|
137
|
+
const res = await selfBench(selfBenchReq(fx, { genId: genId(fingerprint({ ac4: cand }), new Date()), candidateCommit: cand }));
|
|
138
|
+
assert.ok(res.dropped.some((d) => d.path === "src/test/self-sanity.test.ts" && d.reason === "trusted-tests"));
|
|
139
|
+
assert.deepEqual(res.overlaid, []);
|
|
140
|
+
assert.ok(res.failures.some((f) => f.startsWith(SELF_OVERLAY_EMPTY)));
|
|
141
|
+
assert.deepEqual(res.units.map((u) => u.scores), fx.incumbent.units.map((u) => u.scores));
|
|
142
|
+
for (const run of res.suiteRuns)
|
|
143
|
+
assert.deepEqual([run.tests, run.fail], [4, 0]);
|
|
144
|
+
assert.equal(res.replayDigest, fx.expected);
|
|
145
|
+
assert.equal(selfGuardVerdict(res.failures, "nominated"), "indeterminate");
|
|
146
|
+
});
|
|
147
|
+
await t.test("sealed-core candidate never overlays (stats.ts edit dropped as sealed)", async () => {
|
|
148
|
+
const cand = commitSurgery(fx, "tamper sealed stats", () => {
|
|
149
|
+
editFile(fx.harness.repo, "src/core/stats.ts", (s) => s.replace("export const FAMILY_ALPHA", "export const FAMILY_ALPHA // tampered"));
|
|
150
|
+
});
|
|
151
|
+
const res = await selfBench(selfBenchReq(fx, { genId: genId(fingerprint({ sealed: cand }), new Date()), candidateCommit: cand }));
|
|
152
|
+
assert.ok(res.dropped.some((d) => d.path === "src/core/stats.ts" && d.reason === "sealed"));
|
|
153
|
+
assert.deepEqual(res.overlaid, []);
|
|
154
|
+
assert.equal(res.buildStatus, "ok");
|
|
155
|
+
assert.equal(res.replayDigest, fx.expected);
|
|
156
|
+
});
|
|
157
|
+
await t.test("benign unsealed candidate overlays and stays nominated (non-degenerate)", async () => {
|
|
158
|
+
const cand = commitSurgery(fx, "comment edit src/out.ts", () => {
|
|
159
|
+
editFile(fx.harness.repo, "src/out.ts", (s) => s.replace("// stdout funnel (todo 1 seam)", "// stdout funnel (todo 1 seam, patched)"));
|
|
160
|
+
});
|
|
161
|
+
const res = await selfBench(selfBenchReq(fx, { genId: genId(fingerprint({ benign: cand }), new Date()), candidateCommit: cand }));
|
|
162
|
+
assert.deepEqual(res.overlaid, ["src/out.ts"]);
|
|
163
|
+
assert.deepEqual(res.dropped, []);
|
|
164
|
+
assert.equal(res.buildStatus, "ok");
|
|
165
|
+
assert.equal(res.replayDigest, fx.expected);
|
|
166
|
+
assert.equal(selfGuardVerdict(res.failures, "nominated"), "nominated");
|
|
167
|
+
});
|
|
168
|
+
await t.test("broken build scores zero and culls; caps + guards pinned", async () => {
|
|
169
|
+
const cand = commitSurgery(fx, "type error in out.ts", () => {
|
|
170
|
+
editFile(fx.harness.repo, "src/out.ts", () => 'export const writeStdout: (t: string) => void = 42;\n');
|
|
171
|
+
});
|
|
172
|
+
const res = await selfBench(selfBenchReq(fx, { genId: genId(fingerprint({ broken: cand }), new Date()), candidateCommit: cand }));
|
|
173
|
+
assert.equal(res.buildStatus, "failed");
|
|
174
|
+
assert.ok(res.failures.some((f) => f.includes("tsc")));
|
|
175
|
+
for (const u of res.units)
|
|
176
|
+
assert.deepEqual(u.scores, [0, 0]);
|
|
177
|
+
assert.equal(selfGuardVerdict(res.failures, "culled"), "culled");
|
|
178
|
+
assert.equal(selfGuardVerdict([`${SELF_BUILD_TIMEOUT}: tsc killed at 180s`], "culled"), "inconclusive");
|
|
179
|
+
assert.equal(selfGuardVerdict([`${SELF_OVERLAY_EMPTY}: nothing`], "nominated"), "indeterminate");
|
|
180
|
+
assert.deepEqual([DEFAULT_BUILD_TIMEOUT_S, DEFAULT_TEST_TIMEOUT_S, DEFAULT_REPLAY_TIMEOUT_S], [180, 900, 120]);
|
|
181
|
+
});
|
|
182
|
+
await t.test("hung suite is killed at the cap: no suite samples, inconclusive", async () => {
|
|
183
|
+
const cand = commitSurgery(fx, "event-loop-never-drains hang in out.ts", () => {
|
|
184
|
+
editFile(fx.harness.repo, "src/out.ts", () => [
|
|
185
|
+
"const keepAlive: ReturnType<typeof setInterval> = setInterval(() => {}, 60_000);",
|
|
186
|
+
"void keepAlive;",
|
|
187
|
+
"export function writeStdout(text: string): void {",
|
|
188
|
+
" process.stdout.write(text);",
|
|
189
|
+
"}",
|
|
190
|
+
"",
|
|
191
|
+
].join("\n"));
|
|
192
|
+
});
|
|
193
|
+
const res = await selfBench({
|
|
194
|
+
...selfBenchReq(fx, { genId: genId(fingerprint({ hung: cand }), new Date()), candidateCommit: cand }),
|
|
195
|
+
testTimeoutS: 5,
|
|
196
|
+
replayTimeoutS: 5,
|
|
197
|
+
});
|
|
198
|
+
assert.equal(res.buildStatus, "ok");
|
|
199
|
+
const suite = res.units.find((u) => u.unitId === "suite");
|
|
200
|
+
const replay = res.units.find((u) => u.unitId === "golden-replay");
|
|
201
|
+
assert.deepEqual(suite?.scores ?? ["missing"], []);
|
|
202
|
+
assert.ok(suite?.failures.some((f) => f.includes("timeout")) ?? false, JSON.stringify({ failures: suite?.failures, buildNote: res.buildNote, suiteRuns: res.suiteRuns }));
|
|
203
|
+
assert.deepEqual(replay?.scores ?? ["missing"], [1, 1]); // helper runs reps: 2; replay path never imports out.js
|
|
204
|
+
assert.equal(res.complete, false);
|
|
205
|
+
assert.equal(selfGuardVerdict(res.failures, "nominated"), "inconclusive");
|
|
206
|
+
});
|
|
207
|
+
await t.test("run --genome abathur-self end-to-end: storm + benign nomination + friction digest", async () => {
|
|
208
|
+
exec("git", ["-C", fx.harness.repo, "reset", "--hard", fx.head]);
|
|
209
|
+
const before = ledgerCount(fx);
|
|
210
|
+
const seen = [];
|
|
211
|
+
const outcome = await runEvolution({
|
|
212
|
+
entry: fx.entry,
|
|
213
|
+
configDir: fx.harness.configDir,
|
|
214
|
+
mutatorCommand: `node ${fx.stub} --dir {worktree} --brief {brief}`,
|
|
215
|
+
env: fx.harness.env,
|
|
216
|
+
friction: (input) => {
|
|
217
|
+
seen.push(input);
|
|
218
|
+
appendRunFriction(fx.harness.configDir, input);
|
|
219
|
+
},
|
|
220
|
+
});
|
|
221
|
+
assert.equal(seen.length, 1);
|
|
222
|
+
const summary = seen[0];
|
|
223
|
+
assert.equal(summary?.counts.applied, 1, "one candidate sealed through");
|
|
224
|
+
assert.equal(summary?.rejected.length, 2, "both rejects carried stage+reason");
|
|
225
|
+
assert.equal(summary?.counts.nominated, 1);
|
|
226
|
+
assert.equal(outcome.exitCode, 0);
|
|
227
|
+
assert.ok(outcome.lines.some((l) => l.includes("stats-tamper rejected (path)")), outcome.lines.join("\n"));
|
|
228
|
+
assert.ok(outcome.lines.some((l) => l.includes("no-diffs rejected (schema)")));
|
|
229
|
+
assert.ok(outcome.lines.some((l) => l.includes("annotate-out") && l.includes("nominated")));
|
|
230
|
+
assert.equal(ledgerCount(fx), before + 2, "incumbent row + candidate row");
|
|
231
|
+
const rows = candidateRows(fx);
|
|
232
|
+
const cand = rows.find((r) => r.candidateId === "annotate-out");
|
|
233
|
+
assert.ok(cand !== undefined && cand.verdict === "nominated");
|
|
234
|
+
const queue = readFileSync(path.join(fx.harness.configDir, "friction.jsonl"), "utf8");
|
|
235
|
+
assert.ok(queue.includes('"run-summary"'));
|
|
236
|
+
assert.ok(!queue.includes("stats-tamper"), "candidateId strings are not required in the digest");
|
|
237
|
+
});
|
|
238
|
+
await t.test("self-eval CLI reports verdict + drops + digest, appends friction, writes no ledger", () => {
|
|
239
|
+
const genRow = latestGenIdFor(fx, "annotate-out");
|
|
240
|
+
const before = ledgerCount(fx);
|
|
241
|
+
const run = spawnSync(process.execPath, [path.join(HARNESS_ROOT, "dist/cli.js"), "self-eval", "--genome", "abathur-self", "--gen", genRow, "--reps", "2"], {
|
|
242
|
+
encoding: "utf8",
|
|
243
|
+
env: {
|
|
244
|
+
...process.env,
|
|
245
|
+
ABATHUR_CONFIG: path.join(fx.harness.configDir, "config.jsonc"),
|
|
246
|
+
ABATHUR_SELF_REPO: fx.harness.repo,
|
|
247
|
+
HOME: path.join(fx.harness.root, "home"),
|
|
248
|
+
XDG_CACHE_HOME: path.join(fx.harness.root, "xdg-selfeval"),
|
|
249
|
+
},
|
|
250
|
+
timeout: 280_000,
|
|
251
|
+
});
|
|
252
|
+
assert.equal(run.status, 0, `${run.stdout}\n${run.stderr}`);
|
|
253
|
+
assert.ok(run.stdout.includes("verdict nominated"), run.stdout);
|
|
254
|
+
assert.ok(run.stdout.includes(fx.expected), "trusted replay digest surfaces in the report");
|
|
255
|
+
assert.ok(/overlay \d+|overlaid/.test(run.stdout), run.stdout);
|
|
256
|
+
assert.equal(ledgerCount(fx), before, "self-eval never writes ledger rows");
|
|
257
|
+
const queue = readFileSync(path.join(fx.harness.configDir, "friction.jsonl"), "utf8");
|
|
258
|
+
assert.ok(queue.includes('"self-eval"'));
|
|
259
|
+
});
|
|
260
|
+
await t.test("self-eval refuses a non-self genome and an unknown label (exit 2)", () => {
|
|
261
|
+
const spawnEval = (label) => spawnSync(process.execPath, [path.join(HARNESS_ROOT, "dist/cli.js"), "self-eval", "--genome", label], {
|
|
262
|
+
encoding: "utf8",
|
|
263
|
+
env: {
|
|
264
|
+
...process.env,
|
|
265
|
+
ABATHUR_CONFIG: path.join(fx.harness.configDir, "config.jsonc"),
|
|
266
|
+
ABATHUR_SELF_REPO: fx.harness.repo,
|
|
267
|
+
HOME: path.join(fx.harness.root, "home"),
|
|
268
|
+
XDG_CACHE_HOME: path.join(fx.harness.root, "xdg-selfeval2"),
|
|
269
|
+
},
|
|
270
|
+
});
|
|
271
|
+
const nonSelf = spawnEval("toy-smoke");
|
|
272
|
+
assert.equal(nonSelf.status, 2, nonSelf.stderr);
|
|
273
|
+
assert.match(nonSelf.stderr, /self-eval/);
|
|
274
|
+
const ghost = spawnEval("missing-label");
|
|
275
|
+
assert.equal(ghost.status, 2, ghost.stderr);
|
|
276
|
+
assert.match(ghost.stderr, /no registered genome/); // registry-wide convention (run.test.ts:545)
|
|
277
|
+
});
|
|
278
|
+
});
|
|
279
|
+
// ------------------------------------------------------------------ utilities
|
|
280
|
+
function commitSurgery(fx, message, surgery) {
|
|
281
|
+
// every candidate branches off the incumbent HEAD — earlier subtests'
|
|
282
|
+
// surgeries must not stack into later candidates' diffs.
|
|
283
|
+
exec("git", ["-C", fx.harness.repo, "reset", "--hard", fx.head]);
|
|
284
|
+
surgery();
|
|
285
|
+
const repo = fx.harness.repo;
|
|
286
|
+
exec("git", ["-c", "user.name=abathur", "-c", "user.email=abathur@harness.local", "-C", repo, "add", "-A"]);
|
|
287
|
+
exec("git", ["-c", "user.name=abathur", "-c", "user.email=abathur@harness.local", "-C", repo, "commit", "-m", message]);
|
|
288
|
+
return headSha(repo);
|
|
289
|
+
}
|
|
290
|
+
function exec(bin, args) {
|
|
291
|
+
const r = spawnSync(bin, args, { encoding: "utf8", stdio: "pipe" });
|
|
292
|
+
if (r.status !== 0)
|
|
293
|
+
throw new Error(`${bin} failed: ${r.stderr}`);
|
|
294
|
+
}
|
|
295
|
+
function ledgerCount(fx) {
|
|
296
|
+
const file = path.join(fx.harness.repo, ".state/abathur/ledger.jsonl");
|
|
297
|
+
try {
|
|
298
|
+
return readFileSync(file, "utf8")
|
|
299
|
+
.trim()
|
|
300
|
+
.split("\n")
|
|
301
|
+
.filter((line) => line.includes('"generation_complete"')).length;
|
|
302
|
+
}
|
|
303
|
+
catch {
|
|
304
|
+
return 0;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
function candidateRows(fx) {
|
|
308
|
+
const ledger = Ledger.open(fx.harness.repo);
|
|
309
|
+
return ledger
|
|
310
|
+
.readAll()
|
|
311
|
+
.filter((r) => r.kind === "generation_complete")
|
|
312
|
+
.map((r) => decodeGenerationRecord(r))
|
|
313
|
+
.filter((d) => d.source === "candidate");
|
|
314
|
+
}
|
|
315
|
+
function latestGenIdFor(fx, candidateId) {
|
|
316
|
+
const ledger = Ledger.open(fx.harness.repo);
|
|
317
|
+
let found = "";
|
|
318
|
+
for (const r of ledger.readAll()) {
|
|
319
|
+
if (r.kind !== "generation_complete" || r.genId === undefined)
|
|
320
|
+
continue;
|
|
321
|
+
const data = decodeGenerationRecord(r);
|
|
322
|
+
if (data.source === "candidate" && data.candidateId === candidateId)
|
|
323
|
+
found = r.genId;
|
|
324
|
+
}
|
|
325
|
+
if (found === "")
|
|
326
|
+
throw new Error(`no ${candidateId} candidate row`);
|
|
327
|
+
return found;
|
|
328
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// snapshotCommit + freeze/thaw acceptance pins (todo 3, plan AC line 98):
|
|
2
|
+
// the snapshot dir is a chmod-restricted read-only copy and `git -C <snap>`
|
|
3
|
+
// reports the exact tree sha of the snapshotted commit.
|
|
4
|
+
import assert from "node:assert/strict";
|
|
5
|
+
import { lstat, mkdir, mkdtemp, open, readFile, rm, stat, symlink, writeFile, } from "node:fs/promises";
|
|
6
|
+
import * as os from "node:os";
|
|
7
|
+
import path from "node:path";
|
|
8
|
+
import test from "node:test";
|
|
9
|
+
import { newGeneration, sealGeneration, snapshotCommit } from "../core/worktree.js";
|
|
10
|
+
import { freezeTree, thawTree } from "../util/freeze.js";
|
|
11
|
+
import { fixtureRepo, gitIn } from "./fixtures-wt.js";
|
|
12
|
+
function isRoot() {
|
|
13
|
+
return process.getuid !== undefined && process.getuid() === 0;
|
|
14
|
+
}
|
|
15
|
+
// ------------------------------------------------------------------ freeze/thaw
|
|
16
|
+
test("freezeTree/thawTree: strip then restore write bits, exec bit preserved, symlinks skipped", async (t) => {
|
|
17
|
+
const root = await mkdtemp(path.join(os.tmpdir(), "abathur-freeze-"));
|
|
18
|
+
t.after(() => rm(root, { recursive: true, force: true }));
|
|
19
|
+
await writeFile(path.join(root, "plain.txt"), "x\n");
|
|
20
|
+
await writeFile(path.join(root, "prog.sh"), "#!/bin/sh\n", { mode: 0o755 });
|
|
21
|
+
await mkdir(path.join(root, "sub", "deep"), { recursive: true });
|
|
22
|
+
await writeFile(path.join(root, "sub", "deep", "f.bin"), "y");
|
|
23
|
+
await symlink(path.join(root, "plain.txt"), path.join(root, "link.txt"));
|
|
24
|
+
await freezeTree(root);
|
|
25
|
+
assert.equal((await stat(path.join(root, "plain.txt"))).mode & 0o222, 0);
|
|
26
|
+
assert.equal((await stat(path.join(root, "sub"))).mode & 0o7777, 0o555);
|
|
27
|
+
assert.equal((await stat(path.join(root, "prog.sh"))).mode & 0o7777, 0o555, "exec bit survives, write bits stripped");
|
|
28
|
+
assert.equal((await lstat(path.join(root, "link.txt"))).isSymbolicLink(), true);
|
|
29
|
+
await thawTree(root);
|
|
30
|
+
assert.notEqual((await stat(path.join(root, "plain.txt"))).mode & 0o200, 0);
|
|
31
|
+
assert.notEqual((await stat(path.join(root, "sub"))).mode & 0o200, 0);
|
|
32
|
+
await writeFile(path.join(root, "sub", "late.txt"), "writable again");
|
|
33
|
+
});
|
|
34
|
+
// --------------------------------------------------------------- snapshotCommit
|
|
35
|
+
test("snapshotCommit: read-only copy whose git tree sha matches the commit exactly", async (t) => {
|
|
36
|
+
const fx = await fixtureRepo(t, "abathur-snap-");
|
|
37
|
+
const genome = { repoPath: fx.repo, genomeFp: fx.fp };
|
|
38
|
+
const snap = await snapshotCommit(genome, fx.head, { env: fx.env });
|
|
39
|
+
const expectedTree = await gitIn(fx.repo, "rev-parse", `${fx.head}^{tree}`);
|
|
40
|
+
assert.equal(snap.treeSha, expectedTree);
|
|
41
|
+
// AC verbatim: `git -C <snapshot>` reports the exact tree sha.
|
|
42
|
+
assert.equal(await gitIn(snap.snapshotPath, "rev-parse", "HEAD^{tree}"), expectedTree);
|
|
43
|
+
// real copy of the tree content, under the injected XDG cache scheme
|
|
44
|
+
assert.equal(await readFile(path.join(snap.snapshotPath, "a.txt"), "utf8"), "first\n");
|
|
45
|
+
assert.ok(snap.snapshotPath.startsWith(path.join(fx.env.XDG_CACHE_HOME, "abathur", "worktrees", fx.fp, "snapshots")));
|
|
46
|
+
// second call: content-addressed cache hit, same path + tree
|
|
47
|
+
const again = await snapshotCommit(genome, fx.head, { env: fx.env });
|
|
48
|
+
assert.equal(again.snapshotPath, snap.snapshotPath);
|
|
49
|
+
assert.equal(again.treeSha, expectedTree);
|
|
50
|
+
});
|
|
51
|
+
test("snapshotCommit: files are mode-restricted; write attempts fail EACCES", async (t) => {
|
|
52
|
+
const fx = await fixtureRepo(t, "abathur-snap-");
|
|
53
|
+
const snap = await snapshotCommit({ repoPath: fx.repo, genomeFp: fx.fp }, fx.head, {
|
|
54
|
+
env: fx.env,
|
|
55
|
+
});
|
|
56
|
+
const f = path.join(snap.snapshotPath, "a.txt");
|
|
57
|
+
assert.equal((await stat(f)).mode & 0o222, 0);
|
|
58
|
+
assert.equal((await stat(snap.snapshotPath)).mode & 0o7777, 0o555);
|
|
59
|
+
if (isRoot()) {
|
|
60
|
+
// our env is the non-root 'lab' user (uid 1000) so the probes below really run;
|
|
61
|
+
// this skip branch exists only for hypothetical root CI (noted in evidence).
|
|
62
|
+
t.skip("running as euid 0 — mode bits do not deny writes");
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
const expectAcces = (e) => e.code === "EACCES";
|
|
66
|
+
await assert.rejects(open(f, "w"), expectAcces);
|
|
67
|
+
await assert.rejects(mkdir(path.join(snap.snapshotPath, "newdir")), expectAcces);
|
|
68
|
+
// still readable + git-introspectable while frozen
|
|
69
|
+
assert.match(await readFile(f, "utf8"), /first/);
|
|
70
|
+
assert.match(await gitIn(snap.snapshotPath, "rev-parse", "HEAD^{tree}"), /^[0-9a-f]{40}$/);
|
|
71
|
+
});
|
|
72
|
+
test("snapshotCommit: snapshots a branch-unreachable sealed commit (todo 11 self-bench flow)", async (t) => {
|
|
73
|
+
const fx = await fixtureRepo(t, "abathur-snap-");
|
|
74
|
+
const genome = { repoPath: fx.repo, genomeFp: fx.fp };
|
|
75
|
+
const gen = await newGeneration(genome, fx.head, "g-200-seal", { env: fx.env });
|
|
76
|
+
await writeFile(path.join(gen.worktreePath, "mutant.txt"), "m\n");
|
|
77
|
+
const sealed = await sealGeneration(genome, "g-200-seal", "gen: mutant", { env: fx.env });
|
|
78
|
+
// unreachable from any branch in the genome repo — snapshot must still materialise it
|
|
79
|
+
const snap = await snapshotCommit(genome, sealed.commitSha, { env: fx.env });
|
|
80
|
+
assert.equal(snap.treeSha, sealed.treeSha);
|
|
81
|
+
assert.match(await readFile(path.join(snap.snapshotPath, "mutant.txt"), "utf8"), /m/);
|
|
82
|
+
});
|
|
83
|
+
test("snapshotCommit: non-hex commitSha is a tool error (exit 2)", async (t) => {
|
|
84
|
+
const fx = await fixtureRepo(t, "abathur-snap-");
|
|
85
|
+
await assert.rejects(snapshotCommit({ repoPath: fx.repo, genomeFp: "ok_fp" }, "main", { env: fx.env }), (e) => e.code === 2);
|
|
86
|
+
});
|