@tachikomagundam/abathur 0.2.5 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/genome.js +2 -2
- package/dist/commands/graft.js +1 -1
- package/dist/commands/run.js +1 -1
- package/dist/commands/self-eval.js +1 -1
- package/dist/commands/tombstone.js +2 -1
- package/dist/core/evolve/run-loop.js +14 -0
- package/dist/core/evolve/score-bank.js +214 -0
- package/dist/core/graft-rebench.js +3 -0
- package/dist/core/ledger.js +6 -1
- package/dist/core/promote.js +2 -1
- package/dist/core/stats-math.js +39 -0
- package/dist/core/stats.js +93 -19
- package/dist/test/historian-grader-integrity.test.js +509 -2
- package/dist/test/historian-run-scenario.test.js +2 -2
- package/dist/test/repopath-seams.test.js +54 -0
- package/dist/test/score-bank.test.js +235 -0
- package/dist/test/seam-gates.test.js +78 -0
- package/dist/test/stats-acceptance.test.js +329 -0
- package/graders/historian/grader-core.d.mts +27 -0
- package/graders/historian/grader-core.mjs +311 -19
- package/graders/historian/grader.mjs +5 -5
- package/graders/historian/judge-poststage-19.mjs +212 -0
- package/graders/historian/judge-poststage.mjs +3 -8
- package/graders/historian/run-scenario-19.sh +43 -0
- package/graders/historian/run-scenario.sh +16 -0
- package/package.json +1 -1
- package/plugin/abathur.ts +1 -1
package/dist/commands/genome.js
CHANGED
|
@@ -53,7 +53,7 @@ function genomeSeals(configDir, label) {
|
|
|
53
53
|
for (const warning of scan.warnings)
|
|
54
54
|
writeStdout(`warning: ${warning}`);
|
|
55
55
|
for (const entry of scan.entries) {
|
|
56
|
-
writeStdout(`# effective seal globs for '${entry.label}' (${entry.fingerprint}) under ${entry.spec.repoPath}`);
|
|
56
|
+
writeStdout(`# effective seal globs for '${entry.label}' (${entry.fingerprint}) under ${entry.spec.repoPath}`); // raw-repoPath: display string only; no FS access
|
|
57
57
|
for (const glob of entry.spec.kernel.immutableGlobs)
|
|
58
58
|
writeStdout(glob);
|
|
59
59
|
}
|
|
@@ -61,7 +61,7 @@ function genomeSeals(configDir, label) {
|
|
|
61
61
|
}
|
|
62
62
|
function genomeRm(configDir, label) {
|
|
63
63
|
const entry = resolveUniqueEntry(configDir, label, "genome rm");
|
|
64
|
-
const file = ledgerPath(entry.spec.repoPath);
|
|
64
|
+
const file = ledgerPath(entry.spec.repoPath); // raw-repoPath: ledgerPath() resolves env literals at the FS seam (ledger.ts)
|
|
65
65
|
if (existsSync(file) && readFileSync(file, "utf8").trim().length > 0) {
|
|
66
66
|
blocked(`genome rm refused: '${entry.label}' (${entry.fingerprint}) has ledger history — archive instead`, `lineage lives in ${file}; unregistering a lived genome would orphan its evidence. Keep it registered or move the repo aside.`);
|
|
67
67
|
}
|
package/dist/commands/graft.js
CHANGED
|
@@ -43,7 +43,7 @@ function entryForLabel(configDir, label) {
|
|
|
43
43
|
const matches = readRegistry(configDir).entries.filter((entry) => entry.label === label);
|
|
44
44
|
if (matches.length > 1) {
|
|
45
45
|
cannotAnswer(`graft: label '${label}' is ambiguous — ${String(matches.length)} genomes share it: ${matches
|
|
46
|
-
.map((e) => `${e.fingerprint} @ ${e.spec.repoPath}`)
|
|
46
|
+
.map((e) => `${e.fingerprint} @ ${e.spec.repoPath}`) // raw-repoPath: display string only; no FS access
|
|
47
47
|
.join(", ")}`, "give the specs distinct labels; the fingerprint is the identity");
|
|
48
48
|
}
|
|
49
49
|
return matches[0] ?? null;
|
package/dist/commands/run.js
CHANGED
|
@@ -69,7 +69,7 @@ export function resolveUniqueEntry(configDir, label, who = "run") {
|
|
|
69
69
|
const scan = requireGenomesByLabel(configDir, label);
|
|
70
70
|
if (scan.entries.length > 1) {
|
|
71
71
|
cannotAnswer(`${who}: label '${label}' is ambiguous — ${String(scan.entries.length)} genomes share it: ${scan.entries
|
|
72
|
-
.map((e) => `${e.fingerprint} @ ${e.spec.repoPath}`)
|
|
72
|
+
.map((e) => `${e.fingerprint} @ ${e.spec.repoPath}`) // raw-repoPath: display string only; no FS access
|
|
73
73
|
.join(", ")}`, "give the specs distinct labels; the fingerprint is the identity");
|
|
74
74
|
}
|
|
75
75
|
const entry = scan.entries[0];
|
|
@@ -110,7 +110,7 @@ async function selfEvalHandler(context) {
|
|
|
110
110
|
for (const warning of readRegistry(configDir).warnings)
|
|
111
111
|
writeStdout(`warning: ${warning}`);
|
|
112
112
|
const entry = resolveUniqueEntry(configDir, flags.label, "self-eval");
|
|
113
|
-
if (!isEnvRepoLiteral(entry.spec.repoPath)) {
|
|
113
|
+
if (!isEnvRepoLiteral(entry.spec.repoPath)) { // raw-repoPath: self-mode detection must inspect the STORED literal, not the resolved path
|
|
114
114
|
cannotAnswer(`self-eval: '${entry.spec.label}' is not a self genome — repoPath must be the \${ABATHUR_SELF_REPO} literal`, "self-eval grades the harness against its own sealed kernel; see genomes/abathur-self.jsonc");
|
|
115
115
|
}
|
|
116
116
|
const audit = auditKernel(entry, configDir);
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
// bytes) but NEVER echoed raw — the confirmation line is sanitized like todo-8's
|
|
5
5
|
// sealMessage so untrusted text cannot forge terminal output.
|
|
6
6
|
import { Ledger } from "../core/ledger.js";
|
|
7
|
+
import { effectiveRepoPath } from "../core/spec.js";
|
|
7
8
|
import { resolveConfigDir } from "../config.js";
|
|
8
9
|
import { EXIT_OK, blocked, cannotAnswer } from "../exit.js";
|
|
9
10
|
import { writeStdout } from "../out.js";
|
|
@@ -48,7 +49,7 @@ function runTombstone(args) {
|
|
|
48
49
|
const { label, genId, reason } = parseArgs(args);
|
|
49
50
|
const configDir = resolveConfigDir();
|
|
50
51
|
const entry = resolveUniqueEntry(configDir, label, "tombstone");
|
|
51
|
-
const ledger = Ledger.open(entry.spec.repoPath);
|
|
52
|
+
const ledger = Ledger.open(effectiveRepoPath(entry.spec.repoPath));
|
|
52
53
|
const records = ledger.readAll();
|
|
53
54
|
if (!records.some((r) => r.kind === "generation_complete" && r.genId === genId)) {
|
|
54
55
|
blocked(`tombstone: no generation_complete ledger row for gen '${genId}' of '${label}' — nothing to bury`);
|
|
@@ -38,6 +38,7 @@ import { buildBrief } from "./brief.js";
|
|
|
38
38
|
import { startRunFriction } from "./run-friction.js";
|
|
39
39
|
import { runMutatorSession } from "./reflect.js";
|
|
40
40
|
import { asReplicates, benchTarget, cloneMatrixRow, copyProvenance } from "./run-bench.js";
|
|
41
|
+
import { loadScoreBank } from "./score-bank.js";
|
|
41
42
|
import { selfGuardVerdict } from "./self-snapshot.js";
|
|
42
43
|
import { effectiveCaps, planLines } from "./run-plan.js";
|
|
43
44
|
import { addCounters, readResume, } from "./run-rows.js";
|
|
@@ -167,6 +168,18 @@ async function evolve(opts, mutatorCommand, ledger, caps, reps, env, now) {
|
|
|
167
168
|
incUnits = out.units;
|
|
168
169
|
lines.push(`incumbent baseline: ${String(out.units.length)} units x ${String(reps)} reps${out.complete ? "" : " (BUDGET-TRUNCATED)"}`);
|
|
169
170
|
}
|
|
171
|
+
// ---- score bank for acceptance-semantics gates (loaded after the incumbent
|
|
172
|
+
// row commit so this campaign's baseline reps price the bank — gate-time view;
|
|
173
|
+
// candidate rows land only after evaluate, never feeding their own verdict).
|
|
174
|
+
const scoreBank = loadScoreBank(genomeRepo);
|
|
175
|
+
if (scoreBank === null) {
|
|
176
|
+
lines.push("score bank: none — legacy cross-arm gate");
|
|
177
|
+
}
|
|
178
|
+
else {
|
|
179
|
+
lines.push(`score bank: ${String(scoreBank.units.size)} unit(s), prior sigma ${scoreBank.priorSigma.toFixed(4)}, pooled df ${String(scoreBank.totalDf)}`);
|
|
180
|
+
for (const notice of scoreBank.notices)
|
|
181
|
+
lines.push(`score bank: ${notice}`);
|
|
182
|
+
}
|
|
170
183
|
// ---- candidate generation session
|
|
171
184
|
const remaining = caps.maxCandidates - counters.candidates;
|
|
172
185
|
const considered = [];
|
|
@@ -242,6 +255,7 @@ async function evolve(opts, mutatorCommand, ledger, caps, reps, env, now) {
|
|
|
242
255
|
stats: spec.bench.stats,
|
|
243
256
|
budgetCaps: caps,
|
|
244
257
|
nPairs,
|
|
258
|
+
...(scoreBank === null ? {} : { bank: scoreBank.units }),
|
|
245
259
|
});
|
|
246
260
|
const guarded = selfMode ? selfGuardVerdict(out.failures, verdict.verdict) : verdict.verdict;
|
|
247
261
|
const finalVerdict = guarded;
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
// Historical per-unit variance bank for the acceptance-semantics gate
|
|
2
|
+
// (engine redesign 2026-09-24; the design doc + power analysis live in the
|
|
3
|
+
// target genome repo's evidence folder, not shipped here).
|
|
4
|
+
//
|
|
5
|
+
// WHAT IT IS: for every unit, σ pooled from the within-group replicate scatter
|
|
6
|
+
// of ALL generation_complete rows in <genomeRepo>/.state/abathur/ledger*.jsonl
|
|
7
|
+
// (live ledger + campaign archives + pilot). A group = one bench event on one
|
|
8
|
+
// arm (one row's scores array for that unit), so its mean is that doc's mean:
|
|
9
|
+
// pooling mean-removed sums of squares across doc lineages is the classical
|
|
10
|
+
// ANOVA pooled within-group variance — champion-doc changes (the campaign-3
|
|
11
|
+
// promote at 7ca80d7 moved the incumbent head 6ca3ff68→556e1350→062a9fdb→
|
|
12
|
+
// 8fe7e34a→de607dab) shift group MEANS, never this estimate. Pooling raw
|
|
13
|
+
// scores/means across lineages would poison the bank and is what the per-group
|
|
14
|
+
// mean removal exists to prevent.
|
|
15
|
+
//
|
|
16
|
+
// SHIELDING: σ² = (df·σ̂² + ν₀·σ²_glob)/(df+ν₀) with ν₀ = BANK_PRIOR_DF = 2 —
|
|
17
|
+
// [0.5, 0.5] is two reps that failed to observe noise, not proof of σ=0. Thin
|
|
18
|
+
// history (df < BANK_THIN_DF) additionally floors σ at BANK_QUANTUM = 0.125/√2:
|
|
19
|
+
// unit scores are fractions of checklist items on a ~1/8 grid, so two reps one
|
|
20
|
+
// flipped item apart differ by exactly 0.125 and a one-step-pair has sd
|
|
21
|
+
// 0.125/√2; a unit quieter than that has only been lucky, not measured.
|
|
22
|
+
// Proven-quiet units (df ≥ BANK_THIN_DF, e.g. the guards with 12–14 zero-SS
|
|
23
|
+
// df) keep their tight shrunk σ — earned across independent benches.
|
|
24
|
+
//
|
|
25
|
+
// READ-ONLY AND FAIL-CLOSED: never Ledger.open (that mkdirs .state — the same
|
|
26
|
+
// discipline as peekPlanState in run-rows.ts), never writes, never exits the
|
|
27
|
+
// process. A row that fails schema validation contributes nothing and only
|
|
28
|
+
// adds a notice (archives may legitimately predate schema evolutions). No
|
|
29
|
+
// files / no readable rows / zero pooled df ⇒ null ⇒ the caller must run the
|
|
30
|
+
// legacy cross-arm gate verbatim: the bank can only inform, absence can never
|
|
31
|
+
// invent a nomination or a cull.
|
|
32
|
+
//
|
|
33
|
+
// ACTIVATION REQUIRES A ROTATED ARCHIVE: with only the live ledger.jsonl
|
|
34
|
+
// present the bank stays null. A lone live file may be one campaign young
|
|
35
|
+
// (gate-time df=1 per unit — zero cross-doc information) or, worse, the fresh
|
|
36
|
+
// .state of any toy/fixture genome whose very first run would silently switch
|
|
37
|
+
// gates mid-test. Rotation (ledger.campaign<N>-*.jsonl) is the operator's own
|
|
38
|
+
// mark that a genome has earned lifetime history, so it is also the mark that
|
|
39
|
+
// entitles the acceptance gate to override the proven legacy default.
|
|
40
|
+
import { readFileSync, readdirSync } from "node:fs";
|
|
41
|
+
import path from "node:path";
|
|
42
|
+
import { z } from "zod";
|
|
43
|
+
import { LEDGER_KIND_GENERATION_COMPLETE, ledgerPath, ledgerRecordSchema } from "../ledger.js";
|
|
44
|
+
import { generationRowDataSchema } from "./run-rows.js";
|
|
45
|
+
// ------------------------------------------------------------- bank epoch
|
|
46
|
+
// Lineage-restart quarantine (institutional addition 2026-09-25, born of the
|
|
47
|
+
// s19 pin era + the s16 churn window): an operator-ruled epoch start for a
|
|
48
|
+
// unit voids every bank group whose row.ts predates it — a broken or polluted
|
|
49
|
+
// measurement era must not teach the gate the wrong noise level. The LEDGER
|
|
50
|
+
// FILES STAY IMMUTABLE (doctrine): the verdict on which rows count lives in
|
|
51
|
+
// this sidecar, beside its prose provenance (rotation NOTEs / evidence pages).
|
|
52
|
+
// FAIL-CLOSED IN BOTH DIRECTIONS: absent sidecar ⇒ no quarantine; present but
|
|
53
|
+
// malformed/unreadable/unparsable-dates ⇒ the WHOLE bank returns null and the
|
|
54
|
+
// caller runs the legacy gate verbatim — a quarantine we cannot read honestly
|
|
55
|
+
// is a quarantine we cannot silently ignore.
|
|
56
|
+
const BANK_EPOCH_FILE = "bank-epoch.json";
|
|
57
|
+
const bankEpochSchema = z.record(z.string().min(1), z
|
|
58
|
+
.object({ sinceIso: z.string().min(1), why: z.string().optional() })
|
|
59
|
+
.strict());
|
|
60
|
+
function loadBankEpoch(dir, note) {
|
|
61
|
+
let raw;
|
|
62
|
+
try {
|
|
63
|
+
raw = readFileSync(path.join(dir, BANK_EPOCH_FILE), "utf8");
|
|
64
|
+
}
|
|
65
|
+
catch (cause) {
|
|
66
|
+
const code = cause?.code;
|
|
67
|
+
if (code === "ENOENT")
|
|
68
|
+
return undefined; // absent: nothing quarantined
|
|
69
|
+
note(`${BANK_EPOCH_FILE}: unreadable (${String(cause)}) — bank disabled, legacy gate`);
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
let doc;
|
|
73
|
+
try {
|
|
74
|
+
doc = JSON.parse(raw);
|
|
75
|
+
}
|
|
76
|
+
catch (cause) {
|
|
77
|
+
note(`${BANK_EPOCH_FILE}: invalid JSON (${String(cause)}) — bank disabled, legacy gate`);
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
const parsed = bankEpochSchema.safeParse(doc);
|
|
81
|
+
if (!parsed.success) {
|
|
82
|
+
note(`${BANK_EPOCH_FILE}: schema violation (${parsed.error.issues[0]?.message ?? "?"}) — bank disabled, legacy gate`);
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
const out = new Map();
|
|
86
|
+
for (const [unitId, rule] of Object.entries(parsed.data)) {
|
|
87
|
+
const sinceMs = Date.parse(rule.sinceIso);
|
|
88
|
+
if (!Number.isFinite(sinceMs)) {
|
|
89
|
+
note(`${BANK_EPOCH_FILE}: bad sinceIso for '${unitId}' — bank disabled, legacy gate`);
|
|
90
|
+
return null;
|
|
91
|
+
}
|
|
92
|
+
out.set(unitId, { sinceIso: rule.sinceIso, sinceMs, why: rule.why ?? "" });
|
|
93
|
+
}
|
|
94
|
+
return out;
|
|
95
|
+
}
|
|
96
|
+
/** Half of one 1/8-grid checklist step between two reps — the sd of that pair. */
|
|
97
|
+
export const BANK_QUANTUM = 0.125 / Math.SQRT2;
|
|
98
|
+
/** Phantom df of the bank-wide pooled prior used to shrink each unit's σ̂². */
|
|
99
|
+
export const BANK_PRIOR_DF = 2;
|
|
100
|
+
/** Below this pooled df a unit's history counts as thin and the quantum floor binds. */
|
|
101
|
+
export const BANK_THIN_DF = 6;
|
|
102
|
+
/** Notices stop accumulating past this many lines (a corrupt archive must not flood run output). */
|
|
103
|
+
const NOTICE_CAP = 20;
|
|
104
|
+
/** One group = one row × unit; a multi-rep scores array contributes n−1 df of within-mean scatter. */
|
|
105
|
+
function accumulate(acc, unitId, scores) {
|
|
106
|
+
const n = scores.length;
|
|
107
|
+
if (n < 2)
|
|
108
|
+
return;
|
|
109
|
+
let sum = 0;
|
|
110
|
+
for (const s of scores)
|
|
111
|
+
sum += s;
|
|
112
|
+
const mean = sum / n;
|
|
113
|
+
let ss = 0;
|
|
114
|
+
for (const s of scores) {
|
|
115
|
+
const d = s - mean;
|
|
116
|
+
ss += d * d;
|
|
117
|
+
}
|
|
118
|
+
const cur = acc.get(unitId) ?? { df: 0, ss: 0 };
|
|
119
|
+
acc.set(unitId, { df: cur.df + (n - 1), ss: cur.ss + ss });
|
|
120
|
+
}
|
|
121
|
+
const LIVE_LEDGER = "ledger.jsonl";
|
|
122
|
+
function isRotatedArchive(name) {
|
|
123
|
+
return name.startsWith("ledger.") && name.endsWith(".jsonl") && name !== LIVE_LEDGER && !name.startsWith("ledger.corrupt-");
|
|
124
|
+
}
|
|
125
|
+
export function loadScoreBank(genomeRepo, opts = {}) {
|
|
126
|
+
const dir = path.dirname(ledgerPath(genomeRepo));
|
|
127
|
+
let names;
|
|
128
|
+
try {
|
|
129
|
+
names = readdirSync(dir);
|
|
130
|
+
}
|
|
131
|
+
catch {
|
|
132
|
+
return null; // no .state/abathur at all — fresh genome, legacy gate
|
|
133
|
+
}
|
|
134
|
+
const excluded = new Set(opts.excludeCandidateTrees ?? []);
|
|
135
|
+
const excludedFiles = new Set(opts.excludeFiles ?? []);
|
|
136
|
+
const files = names.filter((n) => (n === LIVE_LEDGER || isRotatedArchive(n)) && !excludedFiles.has(n)).sort();
|
|
137
|
+
if (!files.some(isRotatedArchive))
|
|
138
|
+
return null; // activation law: rotated history or legacy gate
|
|
139
|
+
const notices = [];
|
|
140
|
+
const note = (line) => {
|
|
141
|
+
if (notices.length < NOTICE_CAP)
|
|
142
|
+
notices.push(line);
|
|
143
|
+
else if (notices.length === NOTICE_CAP)
|
|
144
|
+
notices.push("score bank: further notices suppressed");
|
|
145
|
+
};
|
|
146
|
+
const bankEpoch = loadBankEpoch(dir, note);
|
|
147
|
+
if (bankEpoch === null)
|
|
148
|
+
return null; // malformed quarantine ⇒ legacy gate, never a silently ignored one
|
|
149
|
+
const quarantined = new Map();
|
|
150
|
+
const acc = new Map();
|
|
151
|
+
for (const file of files) {
|
|
152
|
+
let text;
|
|
153
|
+
try {
|
|
154
|
+
text = readFileSync(path.join(dir, file), "utf8");
|
|
155
|
+
}
|
|
156
|
+
catch {
|
|
157
|
+
note(`${file}: unreadable — archive skipped`);
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
const lines = text.split("\n");
|
|
161
|
+
lines.forEach((raw, index) => {
|
|
162
|
+
const line = raw.trim();
|
|
163
|
+
if (line.length === 0)
|
|
164
|
+
return;
|
|
165
|
+
let record;
|
|
166
|
+
try {
|
|
167
|
+
record = JSON.parse(line);
|
|
168
|
+
}
|
|
169
|
+
catch {
|
|
170
|
+
note(`${file}:${String(index + 1)}: not JSON — row skipped`);
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
const parsed = ledgerRecordSchema.safeParse(record);
|
|
174
|
+
if (!parsed.success || parsed.data.kind !== LEDGER_KIND_GENERATION_COMPLETE)
|
|
175
|
+
return;
|
|
176
|
+
const row = generationRowDataSchema.safeParse(parsed.data.data);
|
|
177
|
+
if (!row.success) {
|
|
178
|
+
note(`${file}:${String(index + 1)}: generation_complete row unreadable — row skipped`);
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
if (row.data.source === "candidate" && row.data.treeSha !== undefined && excluded.has(row.data.treeSha))
|
|
182
|
+
return;
|
|
183
|
+
const rowMs = Date.parse(parsed.data.ts);
|
|
184
|
+
for (const u of row.data.units) {
|
|
185
|
+
const rule = bankEpoch?.get(u.unitId);
|
|
186
|
+
if (rule !== undefined && !(Number.isFinite(rowMs) && rowMs >= rule.sinceMs)) {
|
|
187
|
+
quarantined.set(u.unitId, (quarantined.get(u.unitId) ?? 0) + 1);
|
|
188
|
+
continue; // only rows provably at/after the epoch start count; unreadable ts fails closed
|
|
189
|
+
}
|
|
190
|
+
accumulate(acc, u.unitId, u.scores);
|
|
191
|
+
}
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
for (const [unitId, rule] of [...(bankEpoch ?? new Map()).entries()].sort((a, b) => (a[0] < b[0] ? -1 : 1))) {
|
|
195
|
+
note(`bank-epoch '${unitId}' since ${rule.sinceIso}: ${String(quarantined.get(unitId) ?? 0)} group(s) quarantined${rule.why === "" ? "" : ` — ${rule.why}`}`);
|
|
196
|
+
}
|
|
197
|
+
let totalDf = 0;
|
|
198
|
+
let totalSs = 0;
|
|
199
|
+
for (const { df, ss } of acc.values()) {
|
|
200
|
+
totalDf += df;
|
|
201
|
+
totalSs += ss;
|
|
202
|
+
}
|
|
203
|
+
if (totalDf === 0)
|
|
204
|
+
return null;
|
|
205
|
+
const priorVar = totalSs / totalDf;
|
|
206
|
+
const units = new Map();
|
|
207
|
+
for (const [unitId, { df, ss }] of [...acc.entries()].sort((a, b) => (a[0] < b[0] ? -1 : 1))) {
|
|
208
|
+
const within = ss / df;
|
|
209
|
+
const shrunk = (df * within + BANK_PRIOR_DF * priorVar) / (df + BANK_PRIOR_DF);
|
|
210
|
+
const sigma = Math.sqrt(shrunk);
|
|
211
|
+
units.set(unitId, { sigma: df >= BANK_THIN_DF ? sigma : Math.max(sigma, BANK_QUANTUM), df });
|
|
212
|
+
}
|
|
213
|
+
return { units, priorSigma: Math.sqrt(priorVar), totalDf, notices };
|
|
214
|
+
}
|
|
@@ -13,6 +13,7 @@ import { LEDGER_KIND_GENERATION_COMPLETE } from "./ledger.js";
|
|
|
13
13
|
import { newGeneration, openGenome, sealGeneration, snapshotCommit } from "./worktree.js";
|
|
14
14
|
import { git, tryGit } from "../util/git.js";
|
|
15
15
|
import { asReplicates, benchTarget, cloneMatrixRow, copyProvenance } from "./evolve/run-bench.js";
|
|
16
|
+
import { loadScoreBank } from "./evolve/score-bank.js";
|
|
16
17
|
import { addCounters, readResume } from "./evolve/run-rows.js";
|
|
17
18
|
import { ChildTracker, reapOrphans } from "./evolve/child-track.js";
|
|
18
19
|
import { clampReps, evaluate, VERDICT_EXIT } from "./stats.js";
|
|
@@ -82,6 +83,7 @@ export async function graftAndBench(req, ctx) {
|
|
|
82
83
|
ledger.append({ kind: LEDGER_KIND_GENERATION_COMPLETE, genId: incGen, data });
|
|
83
84
|
incUnits = out.units;
|
|
84
85
|
}
|
|
86
|
+
const scoreBank = loadScoreBank(spec.repoPath);
|
|
85
87
|
const out = await benchTarget({
|
|
86
88
|
spec: { ...spec, repoPath: gen.worktreePath },
|
|
87
89
|
genId: graftGenId,
|
|
@@ -103,6 +105,7 @@ export async function graftAndBench(req, ctx) {
|
|
|
103
105
|
stats: spec.bench.stats,
|
|
104
106
|
budgetCaps: caps,
|
|
105
107
|
nPairs: 1,
|
|
108
|
+
...(scoreBank === null ? {} : { bank: scoreBank.units }),
|
|
106
109
|
});
|
|
107
110
|
// EXACTLY ONE graft_import decision row lands BEFORE any further writes.
|
|
108
111
|
ctx.recordDecision(gate.verdict, localScoreReason(gate, manifest, reps), { graftGenId, ...sealed });
|
package/dist/core/ledger.js
CHANGED
|
@@ -9,6 +9,7 @@ import { closeSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync
|
|
|
9
9
|
import path from "node:path";
|
|
10
10
|
import { z } from "zod";
|
|
11
11
|
import { canonicalJson, compactUtc } from "./ids.js";
|
|
12
|
+
import { effectiveRepoPath } from "./spec.js";
|
|
12
13
|
import { acquireLock } from "./locks.js";
|
|
13
14
|
export { acquireLock, lockDirFor } from "./locks.js";
|
|
14
15
|
export const LEDGER_KIND_LOCK_TAKEOVER = "lock_takeover";
|
|
@@ -30,7 +31,11 @@ export class LedgerError extends Error {
|
|
|
30
31
|
}
|
|
31
32
|
}
|
|
32
33
|
export function ledgerPath(genomeRepo) {
|
|
33
|
-
|
|
34
|
+
// FS-seam rule: the stored repoPath may be an unresolved `${VAR}` literal
|
|
35
|
+
// (0.2.x convention; see 4997618 / 863e6c2). Resolve HERE at the single
|
|
36
|
+
// ledger seam so no caller can ever read an empty ledger under a literal
|
|
37
|
+
// path and report a misleading downstream error again.
|
|
38
|
+
return path.join(effectiveRepoPath(genomeRepo), ".state", "abathur", "ledger.jsonl");
|
|
34
39
|
}
|
|
35
40
|
function buildRecord(input, now) {
|
|
36
41
|
const candidate = {
|
package/dist/core/promote.js
CHANGED
|
@@ -9,6 +9,7 @@ import path from "node:path";
|
|
|
9
9
|
import { ExitSignal, blocked, cannotAnswer } from "../exit.js";
|
|
10
10
|
import { compileGlob } from "./glob.js";
|
|
11
11
|
import { fastForwardIncumbent } from "./incumbent.js";
|
|
12
|
+
import { effectiveRepoPath } from "./spec.js";
|
|
12
13
|
import { buildManifest, kernelsDirOf, manifestPathFor, readManifestFile, serializeManifest, } from "./kernel.js";
|
|
13
14
|
import { Ledger, ledgerPath } from "./ledger.js";
|
|
14
15
|
import { gitOpts, requireSha } from "./genome-paths.js";
|
|
@@ -65,7 +66,7 @@ async function sealedPathCheck(genId, nom, matchers, sealedPaths, cwd, opts) {
|
|
|
65
66
|
}
|
|
66
67
|
export async function promoteGeneration(req) {
|
|
67
68
|
const { entry, configDir } = req;
|
|
68
|
-
const repo = entry.spec.repoPath;
|
|
69
|
+
const repo = effectiveRepoPath(entry.spec.repoPath);
|
|
69
70
|
const shown = display(req.genId);
|
|
70
71
|
const opts = req.env === undefined ? {} : { env: req.env };
|
|
71
72
|
const ledger = Ledger.open(repo);
|
package/dist/core/stats-math.js
CHANGED
|
@@ -74,6 +74,45 @@ function betai(a, b, x) {
|
|
|
74
74
|
return (front * betacf(a, b, x)) / a;
|
|
75
75
|
return 1 - (front * betacf(b, a, 1 - x)) / b;
|
|
76
76
|
}
|
|
77
|
+
/**
|
|
78
|
+
* Standard normal CDF Φ(x): Abramowitz-Stegun 7.1.26 erf approximation,
|
|
79
|
+
* |error| < 1.5e-7 — far tighter than the 4-decimal P values the acceptance
|
|
80
|
+
* gate reports. ±∞ map to 1/0; NaN maps to NaN (callers branch on se===0
|
|
81
|
+
* before dividing, so 0/0 NaN never reaches here by design).
|
|
82
|
+
*/
|
|
83
|
+
export function normalCdf(x) {
|
|
84
|
+
if (Number.isNaN(x))
|
|
85
|
+
return Number.NaN;
|
|
86
|
+
if (x === Number.POSITIVE_INFINITY)
|
|
87
|
+
return 1;
|
|
88
|
+
if (x === Number.NEGATIVE_INFINITY)
|
|
89
|
+
return 0;
|
|
90
|
+
const z = Math.abs(x) / Math.SQRT2;
|
|
91
|
+
const t = 1 / (1 + 0.3275911 * z);
|
|
92
|
+
const poly = t * (0.254829592 + t * (-0.284496736 + t * (1.421413741 + t * (-1.453152027 + t * 1.061405429))));
|
|
93
|
+
const erf = 1 - poly * Math.exp(-z * z);
|
|
94
|
+
return 0.5 * (1 + (x < 0 ? -erf : erf));
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Inverse standard normal Φ⁻¹(p), bisection against normalCdf above — the two
|
|
98
|
+
* are mutually consistent by construction (a band built from normalQuantile(q)
|
|
99
|
+
* and tested through normalCdf lands exactly at probability q). Same shape as
|
|
100
|
+
* studentTQuantile's bisection; |error| < 1e-13.
|
|
101
|
+
*/
|
|
102
|
+
export function normalQuantile(p) {
|
|
103
|
+
if (!(p > 0 && p < 1))
|
|
104
|
+
throw new RangeError(`normalQuantile: p=${String(p)} must be in (0, 1)`);
|
|
105
|
+
let lo = -40;
|
|
106
|
+
let hi = 40;
|
|
107
|
+
for (let i = 0; i < 80; i += 1) {
|
|
108
|
+
const mid = (lo + hi) / 2;
|
|
109
|
+
if (normalCdf(mid) < p)
|
|
110
|
+
lo = mid;
|
|
111
|
+
else
|
|
112
|
+
hi = mid;
|
|
113
|
+
}
|
|
114
|
+
return (lo + hi) / 2;
|
|
115
|
+
}
|
|
77
116
|
/**
|
|
78
117
|
* Two-sided Student-t critical value: the t such that P(|T| ≤ t) = 1 − alpha
|
|
79
118
|
* with T ~ t(df). Planned accuracy: matches t-table values to < 1e-3 for the
|
package/dist/core/stats.js
CHANGED
|
@@ -26,15 +26,39 @@
|
|
|
26
26
|
// - Pareto over (trainScore, tokens, wallS); budget-truncated generations
|
|
27
27
|
// (complete:false) never enter the set.
|
|
28
28
|
//
|
|
29
|
+
// ACCEPTANCE SEMANTICS (2026-09-24 redesign; design doc lives in the target
|
|
30
|
+
// genome repo's evidence folder, not shipped here): when EvaluateInput.bank
|
|
31
|
+
// carries the historical per-unit variance bank, precision/regression checks
|
|
32
|
+
// price NOISE
|
|
33
|
+
// with bank σ instead of per-arm t(df=n−1) CIs (the campaign-9/10 resolution
|
|
34
|
+
// floor: t(0.05,1)=12.706 turned a 2-point sample into a 1.5883 half-width).
|
|
35
|
+
// A val unit with bank σ clears when P(Δ ≤ −halfWidth) < q_pair AND the
|
|
36
|
+
// acceptance band Z(q)·σ/√n_candidate ≤ halfWidth; the aggregate additionally
|
|
37
|
+
// requires P(gain shift > 0) = Φ(gain/SE) ≥ q_pair, q_pair = 1−(1−ACCEPT_Q)/nPairs.
|
|
38
|
+
// EVERY fail-closed rule ABOVE the bank path is untouched and can never be
|
|
39
|
+
// bypassed by it: budget⇒inconclusive, one-sided n<2⇒indeterminate,
|
|
40
|
+
// both-unmeasured⇒symmetric exclusion. A unit the bank cannot price falls back
|
|
41
|
+
// to the exact legacy t-CI line; a null/absent bank reproduces the pre-2026-09-24
|
|
42
|
+
// gate verbatim. minEffect 0.1 / halfWidth 0.15 point semantics stand.
|
|
43
|
+
//
|
|
29
44
|
// The inverse Student-t lives in stats-math.ts and the Pareto ranking in
|
|
30
45
|
// stats-pareto.ts; both are re-exported here so the public surface of this
|
|
31
46
|
// module — the one todos 9/10/12 consume — is unchanged. Accuracy: |t − table|
|
|
32
47
|
// < 1e-3 for the t-distribution rows checked in stats.test.ts (df=1..30, α=0.05).
|
|
33
48
|
import { EXIT_BLOCKED, EXIT_CANNOT_ANSWER, EXIT_OK } from "../exit.js";
|
|
34
49
|
/** Per-unit replicate scores (train and val units alike). */
|
|
35
|
-
import { studentTQuantile } from "./stats-math.js";
|
|
50
|
+
import { studentTQuantile, normalCdf, normalQuantile } from "./stats-math.js";
|
|
36
51
|
import { seededRandom, paretoFrontier } from "./stats-pareto.js";
|
|
37
|
-
export { studentTQuantile, seededRandom, paretoFrontier };
|
|
52
|
+
export { studentTQuantile, normalCdf, normalQuantile, seededRandom, paretoFrontier };
|
|
53
|
+
/** Nomination acceptance probability (per finalist pair after Bonferroni). */
|
|
54
|
+
export const ACCEPT_Q = 0.9;
|
|
55
|
+
/** q_pair = 1 − (1 − ACCEPT_Q)/nPairs — Bonferroni on the acceptance ERROR rate. */
|
|
56
|
+
export function bonferroniQ(nPairs) {
|
|
57
|
+
if (!Number.isInteger(nPairs) || nPairs < 1) {
|
|
58
|
+
throw new RangeError(`bonferroniQ: nPairs=${String(nPairs)} must be an integer >= 1`);
|
|
59
|
+
}
|
|
60
|
+
return Math.min(0.9999, 1 - (1 - ACCEPT_Q) / nPairs);
|
|
61
|
+
}
|
|
38
62
|
/** Family-wise alpha across finalist pairs (bonferroniAlpha divides it). */
|
|
39
63
|
export const FAMILY_ALPHA = 0.05;
|
|
40
64
|
/** Verdict → process exit code. inconclusive = 2 (cannot answer), never 0. */
|
|
@@ -165,7 +189,14 @@ export function evaluate(input) {
|
|
|
165
189
|
const candKept = candidate.units.filter((u) => !excluded.has(u.unitId));
|
|
166
190
|
const incKept = incumbent.units.filter((u) => !excluded.has(u.unitId));
|
|
167
191
|
const candByUnitKept = new Map(candKept.map((u) => [u.unitId, u]));
|
|
168
|
-
// 3) Per-val-unit comparisons + the two precision gates.
|
|
192
|
+
// 3) Per-val-unit comparisons + the two precision gates. With a bank that
|
|
193
|
+
// knows the unit, acceptance semantics (see module header) replace the
|
|
194
|
+
// per-arm t-CI machinery FOR THAT UNIT ONLY; bank-unknown units keep the
|
|
195
|
+
// legacy lines verbatim. The fail-closed rules above run before and
|
|
196
|
+
// independently of any bank.
|
|
197
|
+
const bank = input.bank ?? null;
|
|
198
|
+
const qPair = bonferroniQ(nPairs);
|
|
199
|
+
const acceptanceUnits = [];
|
|
169
200
|
const unitComparisons = [];
|
|
170
201
|
for (const [unitId, incUnit] of incByUnit) {
|
|
171
202
|
if (incUnit.split !== "val" || excluded.has(unitId))
|
|
@@ -173,30 +204,73 @@ export function evaluate(input) {
|
|
|
173
204
|
const candUnit = candByUnitKept.get(unitId);
|
|
174
205
|
if (candUnit === undefined)
|
|
175
206
|
continue; // excluded/unreachable
|
|
176
|
-
const cs = summarizeUnit(candUnit.scores, alpha);
|
|
177
207
|
const incumbentMean = summarizeUnit(incUnit.scores).mean;
|
|
208
|
+
const cs = summarizeUnit(candUnit.scores, alpha);
|
|
178
209
|
const delta = cs.mean - incumbentMean;
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
210
|
+
const banked = bank?.get(unitId) ?? null;
|
|
211
|
+
if (banked === null) {
|
|
212
|
+
unitComparisons.push({
|
|
213
|
+
unitId,
|
|
214
|
+
candidateMean: cs.mean,
|
|
215
|
+
incumbentMean,
|
|
216
|
+
ciHalfWidth: cs.ciHalfWidth,
|
|
217
|
+
delta,
|
|
218
|
+
ciPasses: cs.ciHalfWidth <= stats.halfWidth,
|
|
219
|
+
passes: delta >= -cs.ciHalfWidth, // ties allowed
|
|
220
|
+
});
|
|
221
|
+
if (cs.ciHalfWidth > stats.halfWidth) {
|
|
222
|
+
failures.push(`CI half-width ${fmt(cs.ciHalfWidth)} > halfWidth ${fmt(stats.halfWidth)} on unit '${unitId}'`);
|
|
223
|
+
}
|
|
224
|
+
if (delta < -cs.ciHalfWidth) {
|
|
225
|
+
failures.push(`regression on unit '${unitId}': delta ${fmt(delta)} < ${fmt(-cs.ciHalfWidth)} (mean ${fmt(cs.mean)} < ${fmt(incumbentMean)} - ${fmt(cs.ciHalfWidth)})`);
|
|
226
|
+
}
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
const se = banked.sigma * Math.sqrt(1 / candUnit.scores.length + 1 / incUnit.scores.length);
|
|
230
|
+
const band = (normalQuantile(qPair) * banked.sigma) / Math.sqrt(candUnit.scores.length);
|
|
231
|
+
const pRegression = se === 0 ? (delta <= -stats.halfWidth ? 1 : 0) : normalCdf((-stats.halfWidth - delta) / se);
|
|
232
|
+
const ciPasses = band <= stats.halfWidth;
|
|
233
|
+
const passes = pRegression < qPair; // ties at −halfWidth sit at Φ(0)=0.5 < q ⇒ still pass
|
|
234
|
+
unitComparisons.push({ unitId, candidateMean: cs.mean, incumbentMean, ciHalfWidth: band, delta, ciPasses, passes });
|
|
235
|
+
acceptanceUnits.push({ unitId, sigma: banked.sigma, df: banked.df, delta, se, band, pRegression });
|
|
236
|
+
if (!ciPasses) {
|
|
237
|
+
failures.push(`acceptance band ${fmt(band)} > halfWidth ${fmt(stats.halfWidth)} on unit '${unitId}' (bank sigma ${fmt(banked.sigma)}, df ${String(banked.df)})`);
|
|
190
238
|
}
|
|
191
|
-
if (
|
|
192
|
-
failures.push(`regression on unit '${unitId}': delta ${fmt(delta)}
|
|
239
|
+
else if (!passes) {
|
|
240
|
+
failures.push(`regression on unit '${unitId}': delta ${fmt(delta)} — P(Δ ≤ −${fmt(stats.halfWidth)}) = ${fmt(pRegression)} ≥ q ${fmt(qPair)} (bank sigma ${fmt(banked.sigma)})`);
|
|
193
241
|
}
|
|
194
242
|
}
|
|
195
|
-
// 4) Effect-size floor on the aggregate gain (shared, symmetric pool)
|
|
243
|
+
// 4) Effect-size floor on the aggregate gain (shared, symmetric pool) —
|
|
244
|
+
// unchanged — plus the acceptance gate: when the bank can price EVERY
|
|
245
|
+
// unit of the gain pool, nomination additionally requires P(Δ>0) ≥ q.
|
|
246
|
+
// A bank-priced point mass exactly at 0 (se=0, gain=0) fails (pShift=0):
|
|
247
|
+
// demonstrated-identical arms are never nominated on a tie.
|
|
196
248
|
const gain = aggregateScore(candKept) - aggregateScore(incKept);
|
|
197
249
|
if (gain < stats.minEffect) {
|
|
198
250
|
failures.push(`gain ${fmt(gain)} < minEffect ${fmt(stats.minEffect)}`);
|
|
199
251
|
}
|
|
252
|
+
let gainSe = null;
|
|
253
|
+
let pShift = null;
|
|
254
|
+
if (bank !== null) {
|
|
255
|
+
const train = candKept.filter((u) => u.split === "train");
|
|
256
|
+
const pool = train.length > 0 ? train : candKept;
|
|
257
|
+
const incCounts = new Map(incKept.map((u) => [u.unitId, u.scores.length]));
|
|
258
|
+
const priced = pool.map((u) => bank.get(u.unitId) ?? null);
|
|
259
|
+
if (priced.every((s) => s !== null)) {
|
|
260
|
+
const k = pool.length;
|
|
261
|
+
let variance = 0;
|
|
262
|
+
pool.forEach((u, index) => {
|
|
263
|
+
const s = priced[index];
|
|
264
|
+
variance += ((s.sigma * s.sigma) * (1 / u.scores.length + 1 / (incCounts.get(u.unitId) ?? 0))) / (k * k);
|
|
265
|
+
});
|
|
266
|
+
gainSe = Math.sqrt(variance);
|
|
267
|
+
pShift = gainSe === 0 ? (gain > 0 ? 1 : 0) : normalCdf(gain / gainSe);
|
|
268
|
+
if (pShift < qPair) {
|
|
269
|
+
failures.push(`acceptance: P(gain shift > 0) = ${fmt(pShift)} < q ${fmt(qPair)} (se ${fmt(gainSe)}, gain ${fmt(gain)})`);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
}
|
|
200
273
|
const verdict = failures.length === 0 ? "nominated" : "culled";
|
|
201
|
-
|
|
274
|
+
const acceptance = bank === null ? undefined : { q: qPair, gainSe, pShift, units: acceptanceUnits };
|
|
275
|
+
return { ...base, verdict, exitCode: needsExit(verdict), gain, unitComparisons, failures, ...(acceptance === undefined ? {} : { acceptance }) };
|
|
202
276
|
}
|