@tachikomagundam/abathur 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/.github/workflows/ci.yml +29 -0
  2. package/.github/workflows/publish.yml +74 -0
  3. package/LICENSE +21 -0
  4. package/README.md +461 -0
  5. package/config/abathur.jsonc +17 -0
  6. package/config/genomes/historian.example.jsonc +124 -0
  7. package/dist/bench/adapter.js +201 -0
  8. package/dist/bench/fixture-probe.js +92 -0
  9. package/dist/bench/fixture-support.js +173 -0
  10. package/dist/bench/fixture.js +236 -0
  11. package/dist/bench/toy.js +152 -0
  12. package/dist/cli.js +110 -0
  13. package/dist/commands/bundle.js +79 -0
  14. package/dist/commands/genome.js +94 -0
  15. package/dist/commands/graft.js +71 -0
  16. package/dist/commands/kernel.js +47 -0
  17. package/dist/commands/promote.js +25 -0
  18. package/dist/commands/run.js +145 -0
  19. package/dist/commands/self-eval.js +240 -0
  20. package/dist/commands/status.js +186 -0
  21. package/dist/commands/tombstone.js +72 -0
  22. package/dist/config.js +161 -0
  23. package/dist/core/bundle-common.js +119 -0
  24. package/dist/core/bundle-export.js +212 -0
  25. package/dist/core/bundle-inspect.js +143 -0
  26. package/dist/core/bundle-manifest.js +105 -0
  27. package/dist/core/bundle-mask.js +75 -0
  28. package/dist/core/bundle-tar.js +240 -0
  29. package/dist/core/bundle.js +9 -0
  30. package/dist/core/evolve/brief.js +45 -0
  31. package/dist/core/evolve/candidate.js +140 -0
  32. package/dist/core/evolve/child-track.js +197 -0
  33. package/dist/core/evolve/friction.js +150 -0
  34. package/dist/core/evolve/reflect.js +191 -0
  35. package/dist/core/evolve/run-bench.js +170 -0
  36. package/dist/core/evolve/run-friction.js +63 -0
  37. package/dist/core/evolve/run-loop.js +282 -0
  38. package/dist/core/evolve/run-plan.js +39 -0
  39. package/dist/core/evolve/run-rows.js +145 -0
  40. package/dist/core/evolve/self-overlay.js +213 -0
  41. package/dist/core/evolve/self-snapshot.js +170 -0
  42. package/dist/core/evolve/stub-mutators.mjs +105 -0
  43. package/dist/core/evolve/udiff.js +189 -0
  44. package/dist/core/genome-paths.js +76 -0
  45. package/dist/core/genome.js +176 -0
  46. package/dist/core/glob.js +106 -0
  47. package/dist/core/graft-gates.js +184 -0
  48. package/dist/core/graft-rebench.js +187 -0
  49. package/dist/core/graft-support.js +181 -0
  50. package/dist/core/graft.js +218 -0
  51. package/dist/core/ids.js +154 -0
  52. package/dist/core/incumbent.js +46 -0
  53. package/dist/core/kernel.js +112 -0
  54. package/dist/core/ledger.js +198 -0
  55. package/dist/core/locks.js +172 -0
  56. package/dist/core/promote.js +119 -0
  57. package/dist/core/snapshot.js +61 -0
  58. package/dist/core/spec.js +178 -0
  59. package/dist/core/stats-math.js +102 -0
  60. package/dist/core/stats-pareto.js +57 -0
  61. package/dist/core/stats.js +184 -0
  62. package/dist/core/worktree.js +190 -0
  63. package/dist/exit.js +32 -0
  64. package/dist/genomes/toy-smoke/genome.jsonc +30 -0
  65. package/dist/genomes/toy-smoke/grader.mjs +61 -0
  66. package/dist/genomes/toy-smoke/init.mjs +63 -0
  67. package/dist/genomes/toy-smoke/units/add.mjs +17 -0
  68. package/dist/genomes/toy-smoke/units/explode.mjs +4 -0
  69. package/dist/genomes/toy-smoke/units/hang.mjs +16 -0
  70. package/dist/genomes/toy-smoke/units/mul.mjs +16 -0
  71. package/dist/genomes/toy-smoke/units/mutate.mjs +18 -0
  72. package/dist/genomes/toy-smoke/units/sub.mjs +16 -0
  73. package/dist/jsonc.js +77 -0
  74. package/dist/out.js +5 -0
  75. package/dist/test/bench-adapter.test.js +33 -0
  76. package/dist/test/bench-fixture.test.js +407 -0
  77. package/dist/test/bench-toy.test.js +251 -0
  78. package/dist/test/bundle.test.js +659 -0
  79. package/dist/test/config.test.js +185 -0
  80. package/dist/test/d7-gate.test.js +56 -0
  81. package/dist/test/fixture-loop.test.js +267 -0
  82. package/dist/test/fixtures/friction-writer.js +16 -0
  83. package/dist/test/fixtures-historian.js +82 -0
  84. package/dist/test/fixtures-self.js +143 -0
  85. package/dist/test/fixtures-wt.js +64 -0
  86. package/dist/test/friction.test.js +398 -0
  87. package/dist/test/genome.test.js +453 -0
  88. package/dist/test/git.test.js +69 -0
  89. package/dist/test/graft.test.js +567 -0
  90. package/dist/test/historian-genome.test.js +134 -0
  91. package/dist/test/historian-grader-io.test.js +148 -0
  92. package/dist/test/historian-grader.test.js +209 -0
  93. package/dist/test/ids.test.js +116 -0
  94. package/dist/test/include-val.test.js +120 -0
  95. package/dist/test/ledger-lock.test.js +99 -0
  96. package/dist/test/ledger.test.js +102 -0
  97. package/dist/test/promote.test.js +394 -0
  98. package/dist/test/reflect.test.js +410 -0
  99. package/dist/test/run-loop.test.js +433 -0
  100. package/dist/test/self-snapshot.test.js +328 -0
  101. package/dist/test/snapshot.test.js +86 -0
  102. package/dist/test/stats.test.js +423 -0
  103. package/dist/test/stub-mutators.test.js +17 -0
  104. package/dist/test/testutil.js +30 -0
  105. package/dist/test/worktree.test.js +198 -0
  106. package/dist/util/freeze.js +30 -0
  107. package/dist/util/git.js +85 -0
  108. package/docs/federation.md +184 -0
  109. package/docs/immutable-kernel.md +87 -0
  110. package/graders/historian/grader-core.d.mts +53 -0
  111. package/graders/historian/grader-core.mjs +276 -0
  112. package/graders/historian/grader-support.d.mts +57 -0
  113. package/graders/historian/grader-support.mjs +137 -0
  114. package/graders/historian/grader.mjs +113 -0
  115. package/graders/historian/mutate.sh +114 -0
  116. package/graders/historian/reset-sandbox.sh +60 -0
  117. package/graders/historian/run-scenario.sh +49 -0
  118. package/graders/historian/seed-wrapped.sh +32 -0
  119. package/package.json +42 -0
@@ -0,0 +1,423 @@
1
+ // Selection-stats tests (plan todo 7 ACs, literal): identical scores ⇒ not
2
+ // nominated; single val regression disqualifies; minEffect floor blocks tiny
3
+ // wins; two-candidate fixture exercises the Bonferroni alpha correction; Pareto
4
+ // set correctness on an 8-candidate fixture; cap-hit ⇒ inconclusive with
5
+ // counters surfaced; seeded synthetic replay ⇒ deterministic verdict table.
6
+ // Student-t CI: own inverse-t (bisection on the regularized incomplete beta),
7
+ // validated against known table values — NOTE: the plan's "df=2 ⇒ 2.920" is
8
+ // the t_{0.95} quantile (two-sided 90%); the two-sided 95% value is 4.303.
9
+ import assert from "node:assert/strict";
10
+ import { describe, it } from "node:test";
11
+ import { FAMILY_ALPHA, VERDICT_EXIT, aggregateScore, bonferroniAlpha, budgetExhausted, clampReps, evaluate, needsExit, paretoFrontier, seededRandom, studentTQuantile, summarizeUnit, } from "../core/stats.js";
12
+ // ---------------------------------------------------------------------------
13
+ // Fixture builders (engineered replicate arrays with EXACT sample variance).
14
+ // Across the whole file: family alpha 0.05, halfWidth and minEffect 0.05 —
15
+ // mirroring hr/thresholds toy discipline.
16
+ // ---------------------------------------------------------------------------
17
+ const STATS = { halfWidth: 0.05, minEffect: 0.05, nReps: { initial: 5, max: 10 } };
18
+ const CAPS = { maxCandidates: 8, maxModelCalls: 64, maxTokens: 2_000_000, maxWallS: 3600 };
19
+ const IDLE_COUNTERS = { candidates: 1, modelCalls: 8, tokens: 40_000, wallS: 120 };
20
+ /** Deviations [−2,−1,0,1,2] scaled to a target SAMPLE variance (n=5 ⇒ sem=√v/√5). */
21
+ function devs(targetVariance) {
22
+ const c = Math.sqrt((targetVariance * 4) / 10); // Σdev² = 10, ddof = 4
23
+ return [-2 * c, -c, 0, c, 2 * c];
24
+ }
25
+ function unit(unitId, split, mean, variance) {
26
+ return { unitId, split, scores: devs(variance).map((d) => mean + d) };
27
+ }
28
+ /** Val-unit mean that makes the regression floor land EXACTLY on the tie (delta === −CI). */
29
+ function tieIncumbentValMean(candidateMean, variance, alpha) {
30
+ return candidateMean + studentTQuantile(alpha, 4) * Math.sqrt(variance / 5);
31
+ }
32
+ function candidate(runId, units, counters = IDLE_COUNTERS) {
33
+ return { runId, units, counters };
34
+ }
35
+ function incumbent(units) {
36
+ return { units };
37
+ }
38
+ function valOnlyUnits(mean, variance) {
39
+ return [unit("u-val", "val", mean, variance)];
40
+ }
41
+ function trainValUnits(trainMean, valMean, variance) {
42
+ return [unit("u-train", "train", trainMean, variance), unit("u-val", "val", valMean, variance)];
43
+ }
44
+ function summaryOf(v) {
45
+ return `${v.verdict}(exit=${v.exitCode}, gain=${v.gain?.toFixed(3) ?? "null"}, alpha=${v.effectiveAlpha})`;
46
+ }
47
+ // ---------------------------------------------------------------------------
48
+ // Student-t quantile
49
+ // ---------------------------------------------------------------------------
50
+ describe("studentTQuantile", () => {
51
+ const CASES = [
52
+ // [alpha (two-sided), df, expected t_{1−alpha/2}]
53
+ [0.05, 1, 12.706],
54
+ [0.05, 2, 4.303], // plan's "2.920" is the t_{0.95} column; 95% two-sided is 4.303
55
+ [0.05, 9, 2.262],
56
+ [0.05, 30, 2.042],
57
+ ];
58
+ for (const [alpha, df, expected] of CASES) {
59
+ it(`reproduces t-table t_{0.975,df=${df}} = ${expected}`, () => {
60
+ assert.ok(Math.abs(studentTQuantile(alpha, df) - expected) < 1e-3, `t=${studentTQuantile(alpha, df)}`);
61
+ });
62
+ }
63
+ it("widens with the corrected (Bonferroni) alpha at df=4", () => {
64
+ assert.ok(Math.abs(studentTQuantile(0.05, 4) - 2.776) < 1e-3);
65
+ assert.ok(Math.abs(studentTQuantile(0.025, 4) - 3.495) < 1.5e-2);
66
+ assert.ok(studentTQuantile(0.025, 4) > studentTQuantile(0.05, 4));
67
+ });
68
+ it("rejects alpha outside (0,1) and df < 1", () => {
69
+ assert.throws(() => studentTQuantile(0, 4), RangeError);
70
+ assert.throws(() => studentTQuantile(1, 4), RangeError);
71
+ assert.throws(() => studentTQuantile(-0.5, 4), RangeError);
72
+ assert.throws(() => studentTQuantile(0.05, 0), RangeError);
73
+ });
74
+ });
75
+ // ---------------------------------------------------------------------------
76
+ // summarizeUnit / aggregateScore / clampReps / budgetExhausted
77
+ // ---------------------------------------------------------------------------
78
+ describe("summarizeUnit", () => {
79
+ it("computes mean, unbiased variance and 95% CI half-width (n=5)", () => {
80
+ const s = summarizeUnit(devs(0.0016));
81
+ assert.ok(Math.abs(s.mean) < 1e-12);
82
+ assert.ok(Math.abs(s.sampleVariance - 0.0016) < 1e-15);
83
+ assert.ok(Math.abs(s.sem - Math.sqrt(0.0016 / 5)) < 1e-12);
84
+ assert.ok(Math.abs(s.ciHalfWidth - studentTQuantile(FAMILY_ALPHA, 4) * s.sem) < 1e-9);
85
+ assert.equal(s.indeterminate, false);
86
+ });
87
+ it("HARD RULE: n=1 ⇒ variance undefined ⇒ indeterminate", () => {
88
+ const s = summarizeUnit([0.7]);
89
+ assert.equal(s.indeterminate, true);
90
+ assert.ok(Number.isNaN(s.sampleVariance));
91
+ assert.ok(Number.isNaN(s.ciHalfWidth));
92
+ });
93
+ });
94
+ describe("aggregateScore", () => {
95
+ it("averages per-unit means over train units", () => {
96
+ const units = [unit("t1", "train", 0.8, 0.001), unit("t2", "train", 0.9, 0.001)];
97
+ assert.ok(Math.abs(aggregateScore(units) - 0.85) < 1e-12);
98
+ });
99
+ it("falls back to val units when the bench has no train units", () => {
100
+ assert.ok(Math.abs(aggregateScore(valOnlyUnits(0.77, 0.001)) - 0.77) < 1e-12);
101
+ });
102
+ });
103
+ describe("clampReps", () => {
104
+ it("defaults to nReps.initial and clamps into [initial, max]", () => {
105
+ assert.equal(clampReps(undefined, STATS.nReps), 5);
106
+ assert.equal(clampReps(1, STATS.nReps), 5); // --reps 1 ⇒ initial (never below)
107
+ assert.equal(clampReps(5, STATS.nReps), 5);
108
+ assert.equal(clampReps(50, STATS.nReps), 10); // --reps 50 ⇒ max
109
+ assert.equal(clampReps(Number.NaN, STATS.nReps), 5); // non-finite ⇒ initial
110
+ });
111
+ });
112
+ describe("budgetExhausted", () => {
113
+ it("fires when ANY cap is hit (>=), and only then", () => {
114
+ assert.equal(budgetExhausted(IDLE_COUNTERS, CAPS), false);
115
+ assert.equal(budgetExhausted({ ...IDLE_COUNTERS, candidates: 8 }, CAPS), true);
116
+ assert.equal(budgetExhausted({ ...IDLE_COUNTERS, modelCalls: 64 }, CAPS), true);
117
+ assert.equal(budgetExhausted({ ...IDLE_COUNTERS, tokens: 2_000_000 }, CAPS), true);
118
+ assert.equal(budgetExhausted({ ...IDLE_COUNTERS, wallS: 3600 }, CAPS), true);
119
+ assert.equal(budgetExhausted({ ...IDLE_COUNTERS, tokens: 1_999_999 }, CAPS), false);
120
+ });
121
+ });
122
+ // ---------------------------------------------------------------------------
123
+ // Nomination gate (evaluate)
124
+ // ---------------------------------------------------------------------------
125
+ describe("evaluate: identical scores are never nominated", () => {
126
+ it("mirror-image candidate with minEffect > 0 ⇒ culled (gain 0)", () => {
127
+ const v = evaluate({
128
+ candidate: candidate("g-ident", trainValUnits(0.8, 0.8, 0.0015)),
129
+ incumbent: incumbent(trainValUnits(0.8, 0.8, 0)),
130
+ stats: STATS,
131
+ budgetCaps: CAPS,
132
+ nPairs: 1,
133
+ });
134
+ assert.equal(v.verdict, "culled");
135
+ assert.ok(v.failures.some((f) => f.startsWith("gain")));
136
+ });
137
+ });
138
+ describe("evaluate: single val-unit regression disqualifies", () => {
139
+ it("mirror-image all val units except one below incumbent − CI ⇒ culled", () => {
140
+ // CI at df=4, family alpha 0.05: t≈2.776, sem=√0.0016/√5≈0.0179 ⇒ CI≈0.0497
141
+ const v = evaluate({
142
+ candidate: candidate("g-reg", [
143
+ unit("u-train", "train", 0.85, 0.0016),
144
+ unit("u-val-1", "val", 0.8, 0.0016),
145
+ unit("u-val-2", "val", 0.7, 0.0016), // incumbent 0.8: floor ≈ 0.7503 ⇒ regresses
146
+ ]),
147
+ incumbent: incumbent([
148
+ unit("u-train", "train", 0.7, 0),
149
+ unit("u-val-1", "val", 0.8, 0),
150
+ unit("u-val-2", "val", 0.8, 0),
151
+ ]),
152
+ stats: STATS,
153
+ budgetCaps: CAPS,
154
+ nPairs: 1,
155
+ });
156
+ assert.equal(v.verdict, "culled");
157
+ assert.ok(v.failures.some((f) => f.startsWith("regression on unit 'u-val-2'")));
158
+ const comp = v.unitComparisons.find((c) => c.unitId === "u-val-2");
159
+ assert.ok(comp !== undefined && !comp.passes);
160
+ });
161
+ });
162
+ describe("evaluate: minEffect floor blocks tiny wins", () => {
163
+ it("candidate with gain 0.03 < minEffect 0.05 and tight CI ⇒ culled on gain", () => {
164
+ const v = evaluate({
165
+ candidate: candidate("g-tiny", trainValUnits(0.73, 0.795, 0.0004)), // CI ≈ 2.776·√0.0004/√5 ≈ 0.0248
166
+ incumbent: incumbent(trainValUnits(0.7, 0.8, 0)),
167
+ stats: STATS,
168
+ budgetCaps: CAPS,
169
+ nPairs: 1,
170
+ });
171
+ assert.equal(v.verdict, "culled");
172
+ assert.ok(v.failures.some((f) => f.startsWith("gain")));
173
+ });
174
+ });
175
+ describe("evaluate: CI half-width gate", () => {
176
+ it("candidate whose per-unit CI exceeds halfWidth is culled even with big gain", () => {
177
+ // variance 0.01 ⇒ sem=√0.002≈0.0447, CI ≈ 0.124 > halfWidth 0.05
178
+ const v = evaluate({
179
+ candidate: candidate("g-wide", trainValUnits(0.9, 0.9, 0.01)),
180
+ incumbent: incumbent(trainValUnits(0.7, 0.8, 0)),
181
+ stats: STATS,
182
+ budgetCaps: CAPS,
183
+ nPairs: 1,
184
+ });
185
+ assert.equal(v.verdict, "culled");
186
+ assert.ok(v.failures.some((f) => f.startsWith("CI half-width")));
187
+ });
188
+ });
189
+ describe("evaluate: Bonferroni alpha correction (plan AC)", () => {
190
+ // Candidate A: CI at family α=0.05 (sem 0.017 ⇒ CI05≈0.0472) fits halfWidth 0.05;
191
+ // corrected α=0.025 (sem 0.017 ⇒ CI025≈0.0594) does NOT. Candidate B: sem 0.010
192
+ // (CI025≈0.0349) passes even corrected. Margins ≥ 18%.
193
+ function aUnits() {
194
+ return trainValUnits(0.85, 0.85, 0.001445); // sem = √0.001445/√5 = 0.017
195
+ }
196
+ function bUnits() {
197
+ return trainValUnits(0.75, 0.8, 0.0005);
198
+ }
199
+ it("single pair: candidate A wins under family alpha 0.05", () => {
200
+ const v = evaluate({
201
+ candidate: candidate("g-A", aUnits()),
202
+ incumbent: incumbent(trainValUnits(0.7, 0.8, 0)),
203
+ stats: STATS,
204
+ budgetCaps: CAPS,
205
+ nPairs: 1,
206
+ });
207
+ assert.equal(v.verdict, "nominated", summaryOf(v));
208
+ assert.equal(v.exitCode, 0);
209
+ });
210
+ it("two candidates vs one incumbent: Bonferroni correction flips A to culled, B still nominated", () => {
211
+ const inc = incumbent(trainValUnits(0.7, 0.8, 0));
212
+ const va = evaluate({ candidate: candidate("g-A", aUnits()), incumbent: inc, stats: STATS, budgetCaps: CAPS, nPairs: 2 });
213
+ const vb = evaluate({ candidate: candidate("g-B", bUnits()), incumbent: inc, stats: STATS, budgetCaps: CAPS, nPairs: 2 });
214
+ assert.equal(va.effectiveAlpha, 0.025);
215
+ assert.equal(va.verdict, "culled", summaryOf(va)); // flip: winner at 0.05 fails at 0.025
216
+ assert.equal(vb.verdict, "nominated", summaryOf(vb)); // control survives the correction
217
+ });
218
+ it("bonferroniAlpha divides the family constant", () => {
219
+ assert.equal(bonferroniAlpha(1), 0.05);
220
+ assert.equal(bonferroniAlpha(2), 0.025);
221
+ assert.throws(() => bonferroniAlpha(0), RangeError);
222
+ });
223
+ });
224
+ describe("evaluate: n=1 ⇒ indeterminate, never nominated", () => {
225
+ it("one val unit measured once ⇒ indeterminate even though other gates pass", () => {
226
+ const v = evaluate({
227
+ candidate: candidate("g-n1", [unit("u-train", "train", 0.85, 0.001), { unitId: "u-val", split: "val", scores: [0.9] }]),
228
+ incumbent: incumbent(trainValUnits(0.7, 0.8, 0)),
229
+ stats: STATS,
230
+ budgetCaps: CAPS,
231
+ nPairs: 1,
232
+ });
233
+ assert.equal(v.verdict, "indeterminate");
234
+ assert.ok(v.failures.some((f) => f.includes("variance undefined")));
235
+ assert.notEqual(v.exitCode, 0);
236
+ });
237
+ it("candidate missing an incumbent val unit entirely ⇒ indeterminate", () => {
238
+ const v = evaluate({
239
+ candidate: candidate("g-miss", [unit("u-train", "train", 0.85, 0.001)]),
240
+ incumbent: incumbent(trainValUnits(0.7, 0.8, 0)),
241
+ stats: STATS,
242
+ budgetCaps: CAPS,
243
+ nPairs: 1,
244
+ });
245
+ assert.equal(v.verdict, "indeterminate");
246
+ });
247
+ });
248
+ describe("evaluate: budget exhaustion ⇒ inconclusive, never implicit pass", () => {
249
+ it("outstanding metrics + a consumed cap still ⇒ inconclusive with counters surfaced", () => {
250
+ const v = evaluate({
251
+ candidate: candidate("g-budget", trainValUnits(0.95, 0.95, 0.0001), {
252
+ candidates: 8,
253
+ modelCalls: 60,
254
+ tokens: 1_800_000,
255
+ wallS: 3200,
256
+ }),
257
+ incumbent: incumbent(trainValUnits(0.7, 0.8, 0)),
258
+ stats: STATS,
259
+ budgetCaps: CAPS,
260
+ nPairs: 1,
261
+ });
262
+ assert.equal(v.verdict, "inconclusive");
263
+ assert.equal(v.exitCode, 2);
264
+ assert.equal(v.counters.candidates, 8); // counters surfaced
265
+ assert.equal(v.unitComparisons.length, 0); // excluded from comparison series
266
+ assert.ok(v.failures.some((f) => f.includes("candidates=8/8")));
267
+ });
268
+ });
269
+ describe("evaluate: nomination happy path", () => {
270
+ it("tight CI, gain above floor, no regression ⇒ nominated with per-unit comparisons", () => {
271
+ const v = evaluate({
272
+ candidate: candidate("g-ok", trainValUnits(0.85, 0.85, 0.0004)),
273
+ incumbent: incumbent(trainValUnits(0.7, 0.78, 0)),
274
+ stats: STATS,
275
+ budgetCaps: CAPS,
276
+ nPairs: 1,
277
+ });
278
+ assert.equal(v.verdict, "nominated", summaryOf(v));
279
+ assert.equal(v.failures.length, 0);
280
+ assert.equal(v.unitComparisons.length, 1);
281
+ const comp = v.unitComparisons[0];
282
+ assert.ok(comp !== undefined && comp.passes);
283
+ });
284
+ it("regression floor allows an exact tie (mean delta === −CI)", () => {
285
+ const candMean = 0.8;
286
+ const variance = 0.0016;
287
+ const v = evaluate({
288
+ candidate: candidate("g-tie", trainValUnits(0.85, candMean, variance)),
289
+ incumbent: incumbent(trainValUnits(0.7, tieIncumbentValMean(candMean, variance, 0.05), 0)),
290
+ stats: STATS,
291
+ budgetCaps: CAPS,
292
+ nPairs: 1,
293
+ });
294
+ assert.equal(v.verdict, "nominated", summaryOf(v));
295
+ });
296
+ it("val-only bench (no train units) measures gain on the val aggregate", () => {
297
+ const v = evaluate({
298
+ candidate: candidate("g-valonly", valOnlyUnits(0.85, 0.0004)),
299
+ incumbent: incumbent(valOnlyUnits(0.75, 0)),
300
+ stats: STATS,
301
+ budgetCaps: CAPS,
302
+ nPairs: 1,
303
+ });
304
+ assert.equal(v.verdict, "nominated", summaryOf(v));
305
+ });
306
+ });
307
+ describe("needsExit verdict→exit mapping (todo 9 consumes)", () => {
308
+ it("maps nominated→0, culled→1, indeterminate→1, inconclusive→2", () => {
309
+ assert.equal(needsExit("nominated"), 0);
310
+ assert.equal(needsExit("culled"), 1);
311
+ assert.equal(needsExit("indeterminate"), 1);
312
+ assert.equal(needsExit("inconclusive"), 2);
313
+ assert.deepEqual(VERDICT_EXIT, { nominated: 0, culled: 1, indeterminate: 1, inconclusive: 2 });
314
+ });
315
+ });
316
+ // ---------------------------------------------------------------------------
317
+ // Pareto frontier (8-candidate fixture with known dominated/frontier structure)
318
+ // ---------------------------------------------------------------------------
319
+ describe("paretoFrontier", () => {
320
+ const POINTS = [
321
+ { id: "P8", trainScore: 0.99, tokens: 200, wallS: 20, complete: true },
322
+ { id: "P1", trainScore: 0.95, tokens: 100, wallS: 10, complete: true },
323
+ { id: "P3", trainScore: 0.95, tokens: 120, wallS: 10, complete: true }, // dominated by P1
324
+ { id: "P2", trainScore: 0.9, tokens: 60, wallS: 8, complete: true },
325
+ { id: "P5", trainScore: 0.9, tokens: 60, wallS: 9, complete: true }, // dominated by P2
326
+ { id: "P4", trainScore: 0.85, tokens: 50, wallS: 6, complete: true },
327
+ { id: "P6", trainScore: 0.7, tokens: 30, wallS: 4, complete: true },
328
+ { id: "P7", trainScore: 0.7, tokens: 30, wallS: 4, complete: true }, // exact tie with P6
329
+ { id: "TRUNC", trainScore: 1.0, tokens: 10, wallS: 1, complete: false }, // never enters
330
+ ];
331
+ const FRONTIER = new Set(["P8", "P1", "P2", "P4", "P6", "P7"]);
332
+ it("returns exactly the non-dominated complete candidates, ranked by score", () => {
333
+ const ranked = paretoFrontier(POINTS, 1);
334
+ assert.deepEqual(new Set(ranked), FRONTIER);
335
+ assert.equal(ranked[0], "P8");
336
+ assert.equal(ranked[1], "P1");
337
+ assert.equal(ranked[2], "P2");
338
+ assert.equal(ranked[3], "P4");
339
+ assert.ok(!ranked.includes("P3") && !ranked.includes("P5") && !ranked.includes("TRUNC"));
340
+ });
341
+ it("budget-truncated generations never enter the Pareto set", () => {
342
+ const ranked = paretoFrontier([{ id: "T", trainScore: 1.0, tokens: 10, wallS: 1, complete: false }, { id: "OK", trainScore: 0.6, tokens: 40, wallS: 5, complete: true }], 1);
343
+ assert.deepEqual(ranked, ["OK"]);
344
+ });
345
+ it("exact ties share a deterministic rank order per seed", () => {
346
+ const a = paretoFrontier(POINTS, 7);
347
+ const b = paretoFrontier(POINTS, 7);
348
+ assert.deepEqual(a, b);
349
+ assert.equal(a.length, 6);
350
+ assert.ok(new Set(a.slice(4)).has("P6") && new Set(a.slice(4)).has("P7"));
351
+ });
352
+ });
353
+ // ---------------------------------------------------------------------------
354
+ // Seeded synthetic replay: fixed seed ⇒ exact verdict table
355
+ // ---------------------------------------------------------------------------
356
+ describe("seeded synthetic replay (plan AC: deterministic verdict table)", () => {
357
+ const SEED = 42;
358
+ function noisyUnit(unitId, split, mean, variance, rng) {
359
+ return { unitId, split, scores: devs(variance).map((d) => mean + d + (rng() - 0.5) * 0.01) };
360
+ }
361
+ it("reproduces the exact verdict for every row of the decision table", () => {
362
+ const inc = incumbent([
363
+ unit("u-train", "train", 0.7, 0),
364
+ unit("u-val", "val", 0.8, 0),
365
+ ]);
366
+ const rows = [];
367
+ const rngA = seededRandom(SEED);
368
+ const rngB = seededRandom(SEED + 1);
369
+ const rngC = seededRandom(SEED + 2);
370
+ const rngD = seededRandom(SEED + 3);
371
+ const rngE = seededRandom(SEED + 4);
372
+ rows.push({
373
+ name: "clear winner (nominate)",
374
+ candUnits: [noisyUnit("u-train", "train", 0.85, 0.0004, rngA), noisyUnit("u-val", "val", 0.88, 0.0004, rngA)],
375
+ caps: CAPS,
376
+ nPairs: 1,
377
+ expected: "nominated",
378
+ }, {
379
+ name: "tiny gain (cull)",
380
+ candUnits: [noisyUnit("u-train", "train", 0.72, 0.0004, rngB), noisyUnit("u-val", "val", 0.82, 0.0004, rngB)],
381
+ caps: CAPS,
382
+ nPairs: 1,
383
+ expected: "culled",
384
+ }, {
385
+ name: "val regression (cull)",
386
+ candUnits: [noisyUnit("u-train", "train", 0.9, 0.0004, rngC), noisyUnit("u-val", "val", 0.72, 0.0004, rngC)],
387
+ caps: CAPS,
388
+ nPairs: 1,
389
+ expected: "culled",
390
+ }, {
391
+ name: "single measurement (indeterminate)",
392
+ candUnits: [noisyUnit("u-train", "train", 0.85, 0.0004, rngD), { unitId: "u-val", split: "val", scores: [0.88] }],
393
+ caps: CAPS,
394
+ nPairs: 1,
395
+ expected: "indeterminate",
396
+ }, {
397
+ name: "budget spent (inconclusive)",
398
+ candUnits: [noisyUnit("u-train", "train", 0.95, 0.0004, rngE), noisyUnit("u-val", "val", 0.95, 0.0004, rngE)],
399
+ caps: { ...CAPS, maxTokens: 40_000 }, // IDLE_COUNTERS.tokens === cap ⇒ exhausted
400
+ nPairs: 1,
401
+ expected: "inconclusive",
402
+ });
403
+ const table = rows.map((row) => {
404
+ const v = evaluate({
405
+ candidate: candidate(`g-${row.name.replace(/\W+/g, "-")}`, row.candUnits),
406
+ incumbent: inc,
407
+ stats: STATS,
408
+ budgetCaps: row.caps,
409
+ nPairs: row.nPairs,
410
+ });
411
+ return `${row.name.padEnd(28)} → ${summaryOf(v)}`;
412
+ });
413
+ for (const [i, row] of rows.entries()) {
414
+ assert.equal(evaluate({
415
+ candidate: candidate(`g-${row.name.replace(/\W+/g, "-")}`, row.candUnits),
416
+ incumbent: inc,
417
+ stats: STATS,
418
+ budgetCaps: row.caps,
419
+ nPairs: row.nPairs,
420
+ }).verdict, row.expected, table[i]);
421
+ }
422
+ });
423
+ });
@@ -0,0 +1,17 @@
1
+ // Pure-mutator unit seam (todo 5 substrate): deterministic seeded selection +
2
+ // anchor discipline. The end-to-end "fix lifts score" test stays in bench-toy.
3
+ import assert from "node:assert/strict";
4
+ import { test } from "node:test";
5
+ import { scriptedPatches, selectPatches } from "../core/evolve/stub-mutators.mjs";
6
+ test("stub-mutators: seeded selection is deterministic and clamped", () => {
7
+ const table = scriptedPatches();
8
+ assert.ok(table.length >= 2);
9
+ assert.deepEqual(selectPatches(42, 2), selectPatches(42, 2));
10
+ assert.equal(selectPatches(42, 2).length, 2);
11
+ assert.equal(selectPatches(7, 99).length, table.length); // count clamps to table size
12
+ assert.deepEqual(selectPatches(3, 0), []);
13
+ // argument guards in the plain-JS mutator are TypeErrors (no CLI exit boundary here)
14
+ assert.throws(() => selectPatches(-1, 1), TypeError);
15
+ assert.throws(() => selectPatches(1.5, 1), TypeError);
16
+ assert.throws(() => selectPatches(1, -2), TypeError);
17
+ });
@@ -0,0 +1,30 @@
1
+ // Shared fixtures for todo-2 tests (not a *.test.js, so the runner ignores it).
2
+ import assert from "node:assert/strict";
3
+ import { readFileSync } from "node:fs";
4
+ import { mkdtemp, rm } from "node:fs/promises";
5
+ import * as os from "node:os";
6
+ import path from "node:path";
7
+ export const FIXED_CLOCK = { now: () => new Date(Date.UTC(2026, 8, 9, 12, 0, 0)) };
8
+ export async function freshPair(t) {
9
+ const genome = await mkdtemp(path.join(os.tmpdir(), "abathur-led-germ-"));
10
+ const config = await mkdtemp(path.join(os.tmpdir(), "abathur-led-cfg-"));
11
+ t.after(() => {
12
+ rm(genome, { recursive: true, force: true });
13
+ rm(config, { recursive: true, force: true });
14
+ });
15
+ return { genome, config };
16
+ }
17
+ export function rawLines(file) {
18
+ const text = readFileSync(file, "utf8");
19
+ assert.ok(text.endsWith("\n"), "append-only files must always end with a newline");
20
+ return text.split("\n").slice(0, -1);
21
+ }
22
+ export function isAlive(pid) {
23
+ try {
24
+ process.kill(pid, 0);
25
+ return true;
26
+ }
27
+ catch (error) {
28
+ return error.code !== "ESRCH";
29
+ }
30
+ }