@metaharness/flywheel 0.1.1 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analyze.d.ts +39 -0
- package/dist/analyze.d.ts.map +1 -0
- package/dist/analyze.js +68 -0
- package/dist/analyze.js.map +1 -0
- package/dist/cjs/analyze.js +72 -0
- package/dist/cjs/analyze.js.map +1 -0
- package/dist/cjs/cli.js +139 -0
- package/dist/cjs/cli.js.map +1 -0
- package/dist/cjs/gate.js +43 -0
- package/dist/cjs/gate.js.map +1 -0
- package/dist/cjs/index.js +31 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/lineage.js +52 -0
- package/dist/cjs/lineage.js.map +1 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/receipts.js +44 -0
- package/dist/cjs/receipts.js.map +1 -0
- package/dist/cjs/replay.js +81 -0
- package/dist/cjs/replay.js.map +1 -0
- package/dist/cjs/run.js +147 -0
- package/dist/cjs/run.js.map +1 -0
- package/dist/cjs/sequential.js +105 -0
- package/dist/cjs/sequential.js.map +1 -0
- package/dist/cjs/types.js +10 -0
- package/dist/cjs/types.js.map +1 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +8 -0
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +5 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/replay.d.ts +3 -1
- package/dist/replay.d.ts.map +1 -1
- package/dist/replay.js +43 -4
- package/dist/replay.js.map +1 -1
- package/dist/run.d.ts +11 -1
- package/dist/run.d.ts.map +1 -1
- package/dist/run.js +78 -28
- package/dist/run.js.map +1 -1
- package/dist/sequential.d.ts +53 -0
- package/dist/sequential.d.ts.map +1 -0
- package/dist/sequential.js +101 -0
- package/dist/sequential.js.map +1 -0
- package/dist/types.d.ts +40 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +25 -8
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.verifyReplayBundle = verifyReplayBundle;
|
|
4
|
+
// @metaharness/flywheel — independent replay. Given ONLY a ReplayBundle (and, optionally, the pinned
|
|
5
|
+
// gate fingerprint + the gate rule itself), an external reviewer establishes the run with no trust in the
|
|
6
|
+
// producer:
|
|
7
|
+
// (1) every promotion receipt verifies (Ed25519, recompute canon vs the embedded key);
|
|
8
|
+
// (2) the promoted lineage reconstructs current → gen-0 immutable root, contiguously;
|
|
9
|
+
// (3) every commit on the promoted chain is actually PROMOTED (no rejected node smuggled in);
|
|
10
|
+
// (4) the gate fingerprint matches the pinned value ⇒ the promotion rule was UNCHANGED;
|
|
11
|
+
// (5) [ADR-235] if the reviewer supplies the (fingerprint-matched) rule, every PROMOTED commit is
|
|
12
|
+
// RE-GATED on its sealed baseline+candidate scores and must STILL promote — trust the gate re-run,
|
|
13
|
+
// not the logged verdict. Catches a signed-but-forged promotion the fingerprint check cannot.
|
|
14
|
+
const receipts_js_1 = require("./receipts.js");
|
|
15
|
+
const gate_js_1 = require("./gate.js");
|
|
16
|
+
function verifyReplayBundle(bundle, opts = {}) {
|
|
17
|
+
const failures = [];
|
|
18
|
+
const chain = bundle.chain;
|
|
19
|
+
const receipts = chain.length > 0 && chain.every((c) => (0, receipts_js_1.verifyReceipt)(c.receipt));
|
|
20
|
+
if (!receipts)
|
|
21
|
+
failures.push('receipts');
|
|
22
|
+
const root = chain[chain.length - 1];
|
|
23
|
+
const reachesRoot = !!root && root.parents.length === 0 && root.id === bundle.root_id;
|
|
24
|
+
if (!reachesRoot)
|
|
25
|
+
failures.push('reachesRoot');
|
|
26
|
+
let contiguousParents = chain.length > 0;
|
|
27
|
+
for (let i = 0; i < chain.length - 1; i++) {
|
|
28
|
+
if (!chain[i].parents.includes(chain[i + 1].id))
|
|
29
|
+
contiguousParents = false;
|
|
30
|
+
}
|
|
31
|
+
if (!contiguousParents)
|
|
32
|
+
failures.push('contiguousParents');
|
|
33
|
+
// Every non-root commit on the promoted chain must be PROMOTED (no rejected node smuggled in). An
|
|
34
|
+
// HONEST-NULL run — the flywheel found no improvement, so the chain is just the immutable gen-0 root —
|
|
35
|
+
// is VALID and replayable: an empty set of non-root commits satisfies this VACUOUSLY. (Requiring
|
|
36
|
+
// ≥1 promotion here was a bug: it failed replay on a legitimate 0-promotion result, e.g. a weak model
|
|
37
|
+
// that resolves nothing — the negative is a real, verifiable outcome, not an invalid bundle.)
|
|
38
|
+
const promos = chain.filter((c) => c.verdict !== 'ROOT');
|
|
39
|
+
const allPromoted = promos.every((c) => c.verdict === 'PROMOTED');
|
|
40
|
+
if (!allPromoted)
|
|
41
|
+
failures.push('allPromoted');
|
|
42
|
+
const gateUnchanged = opts.pinnedGateFingerprint ? bundle.gate_fingerprint === opts.pinnedGateFingerprint : true;
|
|
43
|
+
if (!gateUnchanged)
|
|
44
|
+
failures.push('gateUnchanged');
|
|
45
|
+
// (5) RE-EXECUTE the gate. Only when the reviewer supplies the rule (otherwise unchecked → true, so
|
|
46
|
+
// existing fingerprint-only callers are unaffected). The supplied rule must be the SAME one (its
|
|
47
|
+
// fingerprint must match the pinned/bundled value) or re-execution is meaningless. Then every PROMOTED
|
|
48
|
+
// commit that carries its sealed scores must RE-PASS the rule — a logged promotion the frozen gate would
|
|
49
|
+
// NOT grant is a forgery.
|
|
50
|
+
let gateReExecutes = true;
|
|
51
|
+
if (opts.promotionRule) {
|
|
52
|
+
const suppliedFp = (0, gate_js_1.gateFingerprint)(opts.promotionRule);
|
|
53
|
+
const expectedFp = opts.pinnedGateFingerprint ?? bundle.gate_fingerprint ?? undefined;
|
|
54
|
+
if (expectedFp && suppliedFp !== expectedFp) {
|
|
55
|
+
gateReExecutes = false; // wrong rule supplied — cannot re-execute the run's gate
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
const seen = new Set();
|
|
59
|
+
for (const c of [...chain, ...bundle.all_commits]) {
|
|
60
|
+
if (seen.has(c.id))
|
|
61
|
+
continue;
|
|
62
|
+
seen.add(c.id);
|
|
63
|
+
if (c.verdict === 'PROMOTED' && c.baselineScore && c.candidateScore) {
|
|
64
|
+
if (!opts.promotionRule({ baseline: c.baselineScore, candidate: c.candidateScore }).promote) {
|
|
65
|
+
gateReExecutes = false;
|
|
66
|
+
break;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (!gateReExecutes)
|
|
73
|
+
failures.push('gateReExecutes');
|
|
74
|
+
return {
|
|
75
|
+
pass: failures.length === 0,
|
|
76
|
+
checks: { receipts, reachesRoot, contiguousParents, allPromoted, gateUnchanged, gateReExecutes },
|
|
77
|
+
failures,
|
|
78
|
+
chainSummary: chain.map((c) => `gen${c.generation}${c.mutation ? `(${c.mutation.target})` : '(root)'}`).join(' → '),
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=replay.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"replay.js","sourceRoot":"","sources":["../../src/replay.ts"],"names":[],"mappings":";;AA4BA,gDAiEC;AA7FD,qGAAqG;AACrG,0GAA0G;AAC1G,YAAY;AACZ,yFAAyF;AACzF,wFAAwF;AACxF,gGAAgG;AAChG,0FAA0F;AAC1F,oGAAoG;AACpG,yGAAyG;AACzG,oGAAoG;AACpG,+CAA8C;AAC9C,uCAA4C;AAiB5C,SAAgB,kBAAkB,CAChC,MAAoB,EACpB,OAA0E,EAAE;IAE5E,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAE3B,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,2BAAa,EAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAClF,IAAI,CAAC,QAAQ;QAAE,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAEzC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrC,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,EAAE,KAAK,MAAM,CAAC,OAAO,CAAC;IACtF,IAAI,CAAC,WAAW;QAAE,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAE/C,IAAI,iBAAiB,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,EAAE,CAAC;YAAE,iBAAiB,GAAG,KAAK,CAAC;IAC/E,CAAC;IACD,IAAI,CAAC,iBAAiB;QAAE,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAE3D,kGAAkG;IAClG,uGAAuG;IACvG,iGAAiG;IACjG,sGAAsG;IACtG,8FAA8F;IAC9F,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC;IAClE,IAAI,CAAC,WAAW;QAAE,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAE/C,MAAM,aAAa,GAAG,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,MAAM,CAAC,gBAAgB,KAAK,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC;IACjH,IAAI,CAAC,aAAa;QAAE,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAEnD,oGAAoG;IACpG,iGAAiG;IACjG,uGAAuG;IACvG,yGAAyG;IACzG,0BAA0B;IAC1B,IAAI,cAAc,GAAG,IAAI,CAAC;IAC1B,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;QACvB,MAAM,UAAU,GAAG,IAAA,yBAAe,EAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACvD,MAAM,UAAU,GAAG,IAAI,CAAC,qBAAqB,IAAI,MAAM,CAAC,gBAAgB,IAAI,SAAS,CAAC;QACtF,IAAI,UAAU,IAAI,UAAU,KAAK,UAAU,EAAE,CAAC;YAC5C,cAAc,GAAG,KAAK,CAAC,CAAC,yDAAyD;QACnF,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,EAAE,GAAG,MAAM,CAAC,WAAW,CAAoB,EAAE,CAAC;gBACrE,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBAAE,SAAS;gBAC7B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACf,IAAI,CAAC,CAAC,OAAO,KAAK,UAAU,IAAI,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;oBACpE,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,aAAa,EAAE,SAAS,EAAE,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;wBAC5F,cAAc,GAAG,KAAK,CAAC;wBACvB,MAAM;oBACR,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,CAAC,cAAc;QAAE,QAAQ,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAErD,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;QAC3B,MAAM,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,iBAAiB,EAAE,WAAW,EAAE,aAAa,EAAE,cAAc,EAAE;QAChG,QAAQ;QACR,YAAY,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;KACpH,CAAC;AACJ,CAAC"}
|
package/dist/cjs/run.js
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.runFlywheelGenerations = runFlywheelGenerations;
|
|
4
|
+
// @metaharness/flywheel — runFlywheelGenerations(): the promotion LOOP. run → measure → mutate → verify
|
|
5
|
+
// → promote, generation after generation, each re-basing on the previous promoted winner so verified
|
|
6
|
+
// wins COMPOUND into an auditable lineage. Host- and benchmark-agnostic: everything specific enters via
|
|
7
|
+
// the injected `proposer` / `evaluator` / `promotionRule`. The gate selects the winner on the HOLDOUT;
|
|
8
|
+
// the FROZEN anchor is a separate survival check (never optimized against — the anti-Goodhart guard).
|
|
9
|
+
const lineage_js_1 = require("./lineage.js");
|
|
10
|
+
const gate_js_1 = require("./gate.js");
|
|
11
|
+
async function runFlywheelGenerations(cfg) {
|
|
12
|
+
const rule = cfg.promotionRule ?? gate_js_1.meetsPromotionRule;
|
|
13
|
+
const targets = cfg.mutationTargets ?? Object.keys(cfg.rootPolicy);
|
|
14
|
+
const store = cfg.lineageStore ?? new lineage_js_1.InMemoryLineageStore();
|
|
15
|
+
const now = cfg.now ?? ((g) => `gen-${g}`);
|
|
16
|
+
const rootId = cfg.rootId ?? 'root';
|
|
17
|
+
const anchorOf = async (p) => cfg.anchor ? (await cfg.evaluator(p, cfg.anchor)).primary : null;
|
|
18
|
+
let rootScore;
|
|
19
|
+
let rootAnchor;
|
|
20
|
+
let parentId;
|
|
21
|
+
let policy;
|
|
22
|
+
let score;
|
|
23
|
+
const allCommits = [];
|
|
24
|
+
let generationsRun = 0;
|
|
25
|
+
let startGen = 1;
|
|
26
|
+
if (cfg.resumeFrom) {
|
|
27
|
+
// ── RESUME: re-seed the prior lineage and restore the mutable state; DON'T re-evaluate the root. ──
|
|
28
|
+
const r = cfg.resumeFrom;
|
|
29
|
+
for (const c of r.priorCommits)
|
|
30
|
+
await store.append(c);
|
|
31
|
+
allCommits.push(...r.priorCommits.filter((c) => c.verdict !== 'ROOT')); // allCommits excludes the root
|
|
32
|
+
rootScore = r.rootScore;
|
|
33
|
+
rootAnchor = r.rootAnchor;
|
|
34
|
+
parentId = r.parentId;
|
|
35
|
+
policy = { ...r.policy };
|
|
36
|
+
score = r.score;
|
|
37
|
+
generationsRun = r.fromGeneration;
|
|
38
|
+
startGen = r.fromGeneration + 1;
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
// ── gen-0: the immutable root. Evaluate it once (the baseline) + its anchor (the frozen bar). ──
|
|
42
|
+
rootScore = await cfg.evaluator(cfg.rootPolicy, cfg.holdout);
|
|
43
|
+
rootAnchor = await anchorOf(cfg.rootPolicy);
|
|
44
|
+
await store.append({
|
|
45
|
+
id: rootId, generation: 0, parents: [], mutation: null, primaryDelta: 0, anchorScore: rootAnchor,
|
|
46
|
+
verdict: 'ROOT', failureReasons: [], receipt: cfg.signer.sign({ kind: 'root', root: rootId }), createdAt: now(0),
|
|
47
|
+
});
|
|
48
|
+
parentId = rootId;
|
|
49
|
+
policy = { ...cfg.rootPolicy };
|
|
50
|
+
score = rootScore;
|
|
51
|
+
}
|
|
52
|
+
// Assemble a self-consistent replay bundle for the run SO FAR (or final). Pure over the current
|
|
53
|
+
// lineage; used both for the per-generation checkpoint and the final return so the two can never
|
|
54
|
+
// diverge. `rootScore`, `rootAnchor`, `allCommits`, `rule`, `rootId`, `cfg` are captured by closure.
|
|
55
|
+
const buildBundle = (chainNow, createdAt) => {
|
|
56
|
+
const promoted = chainNow.filter((c) => c.verdict === 'PROMOTED');
|
|
57
|
+
const verifiedN = promoted.filter((c) => c.primaryDelta > 0).length;
|
|
58
|
+
const anchorSurvivingN = promoted.filter((c) => c.primaryDelta > 0 && (rootAnchor === null || (c.anchorScore ?? -Infinity) >= rootAnchor)).length;
|
|
59
|
+
return {
|
|
60
|
+
data_source: cfg.dataSource ?? 'UNSPECIFIED',
|
|
61
|
+
root_id: rootId,
|
|
62
|
+
chain: chainNow,
|
|
63
|
+
all_commits: allCommits,
|
|
64
|
+
lift_curve: (0, lineage_js_1.computeLiftCurve)(chainNow, rootScore.primary),
|
|
65
|
+
gate_fingerprint: (0, gate_js_1.gateFingerprint)(rule),
|
|
66
|
+
verified_improvements: verifiedN,
|
|
67
|
+
anchor_surviving_improvements: anchorSurvivingN,
|
|
68
|
+
milestone_reached: anchorSurvivingN >= 2,
|
|
69
|
+
created_at: createdAt,
|
|
70
|
+
};
|
|
71
|
+
};
|
|
72
|
+
for (let gen = startGen; gen <= cfg.maxGenerations; gen++) {
|
|
73
|
+
if (cfg.budget && cfg.budget.spent() >= cfg.budget.total)
|
|
74
|
+
break;
|
|
75
|
+
generationsRun = gen;
|
|
76
|
+
// propose + evaluate one candidate per mutation target, gate each on the HOLDOUT.
|
|
77
|
+
const base = { id: parentId, generation: gen, parents: [parentId], policy };
|
|
78
|
+
const cands = [];
|
|
79
|
+
for (const target of targets) {
|
|
80
|
+
const proposed = await cfg.proposer(base, target);
|
|
81
|
+
const candPolicy = { ...policy, [target]: proposed };
|
|
82
|
+
const candScore = await cfg.evaluator(candPolicy, cfg.holdout);
|
|
83
|
+
const decision = rule({ baseline: score, candidate: candScore });
|
|
84
|
+
cands.push({ target, policy: candPolicy, score: candScore, reasons: decision.reasons, promote: decision.promote });
|
|
85
|
+
}
|
|
86
|
+
// winner = highest primary among the promotable; then verify it survives the FROZEN anchor.
|
|
87
|
+
const promotable = cands.filter((c) => c.promote).sort((a, b) => b.score.primary - a.score.primary);
|
|
88
|
+
const winner = promotable[0] ?? null;
|
|
89
|
+
const winnerAnchor = winner ? await anchorOf(winner.policy) : null;
|
|
90
|
+
const anchorSurvives = winner ? (rootAnchor === null || (winnerAnchor ?? -Infinity) >= rootAnchor) : false;
|
|
91
|
+
// A winner that regresses the anchor is NOT promoted (Goodhart guard) — it becomes a rejection.
|
|
92
|
+
const promotedWinner = winner && anchorSurvives ? winner : null;
|
|
93
|
+
for (const c of cands) {
|
|
94
|
+
const isWinner = c === promotedWinner;
|
|
95
|
+
const id = `${parentId}__${c.target}_gen${gen}`;
|
|
96
|
+
const primaryDelta = c.score.primary - score.primary;
|
|
97
|
+
const commit = {
|
|
98
|
+
id, generation: gen, parents: [parentId],
|
|
99
|
+
mutation: { target: c.target, summary: `adapt ${c.target}` },
|
|
100
|
+
primaryDelta,
|
|
101
|
+
anchorScore: isWinner ? winnerAnchor : c === winner ? winnerAnchor : null,
|
|
102
|
+
verdict: isWinner ? 'PROMOTED' : 'REJECTED',
|
|
103
|
+
failureReasons: isWinner ? [] : c === winner && !anchorSurvives ? ['anchor_regressed'] : c.reasons,
|
|
104
|
+
receipt: cfg.signer.sign({ kind: 'candidate', id, target: c.target, verdict: isWinner ? 'PROMOTED' : 'REJECTED', primaryDelta }),
|
|
105
|
+
createdAt: now(gen),
|
|
106
|
+
baselineScore: score, // ADR-235 — sealed so the gate can be re-run in replay
|
|
107
|
+
candidateScore: c.score,
|
|
108
|
+
};
|
|
109
|
+
await store.append(commit);
|
|
110
|
+
allCommits.push(commit);
|
|
111
|
+
}
|
|
112
|
+
if (promotedWinner) {
|
|
113
|
+
parentId = `${parentId}__${promotedWinner.target}_gen${gen}`;
|
|
114
|
+
policy = promotedWinner.policy;
|
|
115
|
+
score = promotedWinner.score;
|
|
116
|
+
}
|
|
117
|
+
// Per-generation checkpoint — a fully assembled, replay-verifiable bundle for the run so far, so a
|
|
118
|
+
// long run can persist incremental progress. Fail-safe: a throwing hook must NOT kill a valid run.
|
|
119
|
+
if (cfg.onGeneration) {
|
|
120
|
+
try {
|
|
121
|
+
const partialChain = await store.walkToRoot(parentId);
|
|
122
|
+
const info = {
|
|
123
|
+
generation: gen,
|
|
124
|
+
generationsRun,
|
|
125
|
+
partialBundle: buildBundle(partialChain, now(gen)),
|
|
126
|
+
spent: cfg.budget ? cfg.budget.spent() : 0,
|
|
127
|
+
resumeState: {
|
|
128
|
+
rootScore, rootAnchor, parentId, policy, score,
|
|
129
|
+
fromGeneration: gen,
|
|
130
|
+
priorCommits: await store.list(), // root + every candidate — re-seeds the store on resume
|
|
131
|
+
},
|
|
132
|
+
};
|
|
133
|
+
await cfg.onGeneration(info);
|
|
134
|
+
}
|
|
135
|
+
catch { /* checkpoint is observational — never let it abort a valid run */ }
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
const chain = await store.walkToRoot(parentId);
|
|
139
|
+
const replayBundle = buildBundle(chain, now(cfg.maxGenerations));
|
|
140
|
+
const promotions = chain.filter((c) => c.verdict === 'PROMOTED');
|
|
141
|
+
return {
|
|
142
|
+
liftCurve: replayBundle.lift_curve, promotions, lineage: store, replayBundle,
|
|
143
|
+
generationsRun,
|
|
144
|
+
milestoneReached: replayBundle.milestone_reached, finalPolicy: policy,
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
//# sourceMappingURL=run.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/run.ts"],"names":[],"mappings":";;AA2DA,wDAuIC;AAlMD,wGAAwG;AACxG,qGAAqG;AACrG,wGAAwG;AACxG,uGAAuG;AACvG,sGAAsG;AACtG,6CAAsE;AACtE,uCAAgE;AAqDzD,KAAK,UAAU,sBAAsB,CAAC,GAAmB;IAC9D,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,IAAI,4BAAkB,CAAC;IACrD,MAAM,OAAO,GAAG,GAAG,CAAC,eAAe,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACnE,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,IAAI,IAAI,iCAAoB,EAAE,CAAC;IAC7D,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC;IACpC,MAAM,QAAQ,GAAG,KAAK,EAAE,CAAS,EAA0B,EAAE,CAC3D,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IAEnE,IAAI,SAAgB,CAAC;IACrB,IAAI,UAAyB,CAAC;IAC9B,IAAI,QAAgB,CAAC;IACrB,IAAI,MAAc,CAAC;IACnB,IAAI,KAAY,CAAC;IACjB,MAAM,UAAU,GAAoB,EAAE,CAAC;IACvC,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;QACnB,qGAAqG;QACrG,MAAM,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY;YAAE,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACtD,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,+BAA+B;QACvG,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;QAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;QACnD,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;QAAC,MAAM,GAAG,EAAE,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;QAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;QACjE,cAAc,GAAG,CAAC,CAAC,cAAc,CAAC;QAAC,QAAQ,GAAG,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC;IACrE,CAAC;SAAM,CAAC;QACN,kGAAkG;QAClG,SAAS,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7D,UAAU,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC5C,MAAM,KAAK,CAAC,MAAM,CAAC;YACjB,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,WAAW,EAAE,UAAU;YAChG,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;SACjH,CAAC,CAAC;QACH,QAAQ,GAAG,MAAM,CAAC;QAClB,MAAM,GAAG,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC;QAC/B,KAAK,GAAG,SAAS,CAAC;IACpB,CAAC;IAED,gGAAgG;IAChG,iGAAiG;IACjG,qGAAqG;IACrG,MAAM,WAAW,GAAG,CAAC,QAAyB,EAAE,SAAiB,EAAgB,EAAE;QACjF,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QACpE,MAAM,gBAAgB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,UAAU,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,QAAQ,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC;QAClJ,OAAO;YACL,WAAW,EAAE,GAAG,CAAC,UAAU,IAAI,aAAa;YAC5C,OAAO,EAAE,MAAM;YACf,KAAK,EAAE,QAAQ;YACf,WAAW,EAAE,UAAU;YACvB,UAAU,EAAE,IAAA,6BAAgB,EAAC,QAAQ,EAAE,SAAS,CAAC,OAAO,CAAC;YACzD,gBAAgB,EAAE,IAAA,yBAAe,EAAC,IAAI,CAAC;YACvC,qBAAqB,EAAE,SAAS;YAChC,6BAA6B,EAAE,gBAAgB;YAC/C,iBAAiB,EAAE,gBAAgB,IAAI,CAAC;YACxC,UAAU,EAAE,SAAS;SACtB,CAAC;IACJ,CAAC,CAAC;IAEF,KAAK,IAAI,GAAG,GAAG,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,CAAC;QAC1D,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK;YAAE,MAAM;QAChE,cAAc,GAAG,GAAG,CAAC;QAErB,kFAAkF;QAClF,MAAM,IAAI,GAAiB,EAAE,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;QAC1F,MAAM,KAAK,GAAiG,EAAE,CAAC;QAC/G,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAClD,MAAM,UAAU,GAAW,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC;YAC7D,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAC/D,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;YACjE,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;QACrH,CAAC;QAED,4FAA4F;QAC5F,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpG,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACrC,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACnE,MAAM,cAAc,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,KAAK,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,QAAQ,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAC3G,gGAAgG;QAChG,MAAM,cAAc,GAAG,MAAM,IAAI,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;QAEhE,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,QAAQ,GAAG,CAAC,KAAK,cAAc,CAAC;YACtC,MAAM,EAAE,GAAG,GAAG,QAAQ,KAAK,CAAC,CAAC,MAAM,OAAO,GAAG,EAAE,CAAC;YAChD,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;YACrD,MAAM,MAAM,GAAkB;gBAC5B,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,QAAQ,CAAC;gBACxC,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,MAAM,EAAE,EAAE;gBAC5D,YAAY;gBACZ,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI;gBACzE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU;gBAC3C,cAAc,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO;gBAClG,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,EAAE,YAAY,EAAE,CAAC;gBAChI,SAAS,EAAE,GAAG,CAAC,GAAG,CAAC;gBACnB,aAAa,EAAE,KAAK,EAAQ,uDAAuD;gBACnF,cAAc,EAAE,CAAC,CAAC,KAAK;aACxB,CAAC;YACF,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC3B,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YAAC,QAAQ,GAAG,GAAG,QAAQ,KAAK,cAAc,CAAC,MAAM,OAAO,GAAG,EAAE,CAAC;YAAC,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC;YAAC,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC;QAAC,CAAC;QAEnJ,mGAAmG;QACnG,mGAAmG;QACnG,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBACtD,MAAM,IAAI,GAAyB;oBACjC,UAAU,EAAE,GAAG;oBACf,cAAc;oBACd,aAAa,EAAE,WAAW,CAAC,YAAY,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;oBAClD,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;oBAC1C,WAAW,EAAE;wBACX,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK;wBAC9C,cAAc,EAAE,GAAG;wBACnB,YAAY,EAAE,MAAM,KAAK,CAAC,IAAI,EAAE,EAAE,wDAAwD;qBAC3F;iBACF,CAAC;gBACF,MAAM,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC/B,CAAC;YAAC,MAAM,CAAC,CAAC,kEAAkE,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC;IACjE,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC;IAEjE,OAAO;QACL,SAAS,EAAE,YAAY,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY;QAC5E,cAAc;QACd,gBAAgB,EAAE,YAAY,CAAC,iBAAiB,EAAE,WAAW,EAAE,MAAM;KACtE,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// @metaharness/flywheel — anytime-valid sequential testing for promotion gates.
|
|
3
|
+
//
|
|
4
|
+
// WHY THIS EXISTS
|
|
5
|
+
//
|
|
6
|
+
// `meetsPromotionRule` is a single-shot conjunctive comparison, and it is the
|
|
7
|
+
// right shape: frozen, fingerprintable, every clause load-bearing. But a
|
|
8
|
+
// flywheel run evaluates MANY generations against the SAME holdout, and that is
|
|
9
|
+
// uncontrolled multiple testing. Published measurements of greedy
|
|
10
|
+
// "accept-if-the-score-improved" acceptance put the false-commit rate at
|
|
11
|
+
// 30-42%, and found 13-21 spurious modifications made even when NO true gains
|
|
12
|
+
// existed — degrading one agent by 4.9 points while every individual decision
|
|
13
|
+
// looked locally justified.
|
|
14
|
+
//
|
|
15
|
+
// The conjunctive gate plus a frozen anchor already mitigates this with
|
|
16
|
+
// multiple hurdles, which is real. It is not, however, anytime-valid: nothing
|
|
17
|
+
// in it accounts for how many times you have looked.
|
|
18
|
+
//
|
|
19
|
+
// This module adds that missing property WITHOUT touching the default gate,
|
|
20
|
+
// whose stability is itself the product. It composes: wrap any PromotionRule,
|
|
21
|
+
// and a candidate must clear both the frozen clauses AND accumulated evidence.
|
|
22
|
+
//
|
|
23
|
+
// METHOD
|
|
24
|
+
//
|
|
25
|
+
// Testing-by-betting / e-processes. An e-value is a non-negative random
|
|
26
|
+
// variable with expectation <= 1 under the null hypothesis ("this candidate is
|
|
27
|
+
// no better than baseline"). By Ville's inequality, P(sup_t E_t >= 1/alpha)
|
|
28
|
+
// <= alpha, so you may stop and reject at ANY time — no pre-registered sample
|
|
29
|
+
// size, no alpha spending schedule, and no penalty for peeking. That is exactly
|
|
30
|
+
// the property a flywheel needs, because it peeks by construction.
|
|
31
|
+
//
|
|
32
|
+
// The bet here is deliberately simple and assumption-light: per paired item,
|
|
33
|
+
// a candidate win against a baseline loss multiplies the e-value up, the
|
|
34
|
+
// reverse multiplies it down, and ties leave it unchanged.
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.sequentialEvidence = sequentialEvidence;
|
|
37
|
+
exports.withSequentialEvidence = withSequentialEvidence;
|
|
38
|
+
const DEFAULT_ALPHA = 0.05;
|
|
39
|
+
const DEFAULT_LAMBDA = 0.5;
|
|
40
|
+
/**
|
|
41
|
+
* Accumulate paired outcomes into an anytime-valid e-value.
|
|
42
|
+
*
|
|
43
|
+
* Only discordant pairs move the e-value: if both arms win or both lose, that
|
|
44
|
+
* item tells you nothing about which is better (this is the McNemar insight,
|
|
45
|
+
* carried over to the sequential setting).
|
|
46
|
+
*/
|
|
47
|
+
function sequentialEvidence(outcomes, config = {}) {
|
|
48
|
+
const alpha = config.alpha ?? DEFAULT_ALPHA;
|
|
49
|
+
const lambda = config.lambda ?? DEFAULT_LAMBDA;
|
|
50
|
+
if (!(alpha > 0 && alpha < 1))
|
|
51
|
+
throw new RangeError('alpha must be in (0, 1)');
|
|
52
|
+
if (!(lambda > 0 && lambda < 1))
|
|
53
|
+
throw new RangeError('lambda must be in (0, 1)');
|
|
54
|
+
const threshold = 1 / alpha;
|
|
55
|
+
let eValue = 1;
|
|
56
|
+
let informativePairs = 0;
|
|
57
|
+
for (const o of outcomes) {
|
|
58
|
+
if (o.candidateWon === o.baselineWon)
|
|
59
|
+
continue; // concordant: no information
|
|
60
|
+
informativePairs++;
|
|
61
|
+
// Under the null, a discordant pair favors either arm with probability 1/2,
|
|
62
|
+
// so E[multiplier] = 1 and the process is a non-negative martingale.
|
|
63
|
+
eValue *= o.candidateWon ? 1 + lambda : 1 - lambda;
|
|
64
|
+
}
|
|
65
|
+
return {
|
|
66
|
+
significant: eValue >= threshold,
|
|
67
|
+
eValue,
|
|
68
|
+
threshold,
|
|
69
|
+
informativePairs,
|
|
70
|
+
totalPairs: outcomes.length,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Compose a frozen gate with a sequential-evidence requirement.
|
|
75
|
+
*
|
|
76
|
+
* The returned rule is still a plain `PromotionRule`, so it fingerprints and
|
|
77
|
+
* freezes exactly like the default one. A candidate must satisfy BOTH: every
|
|
78
|
+
* clause of `baseRule`, and evidence strong enough to survive having been
|
|
79
|
+
* looked at repeatedly.
|
|
80
|
+
*
|
|
81
|
+
* Paired outcomes are read from `evidence.pairedOutcomes` when present. When
|
|
82
|
+
* absent the rule degrades to `baseRule` alone rather than silently blocking
|
|
83
|
+
* every promotion — a caller that has not wired up per-item outcomes yet should
|
|
84
|
+
* get the old behavior, not a permanently closed gate.
|
|
85
|
+
*/
|
|
86
|
+
function withSequentialEvidence(baseRule, config = {}) {
|
|
87
|
+
return function sequentialPromotionRule(evidence) {
|
|
88
|
+
const base = baseRule(evidence);
|
|
89
|
+
const outcomes = evidence
|
|
90
|
+
.pairedOutcomes;
|
|
91
|
+
if (!outcomes)
|
|
92
|
+
return base;
|
|
93
|
+
const verdict = sequentialEvidence(outcomes, config);
|
|
94
|
+
if (verdict.significant)
|
|
95
|
+
return base;
|
|
96
|
+
return {
|
|
97
|
+
promote: false,
|
|
98
|
+
reasons: [
|
|
99
|
+
...base.reasons,
|
|
100
|
+
`insufficient_sequential_evidence(e=${verdict.eValue.toFixed(2)}<${verdict.threshold})`,
|
|
101
|
+
],
|
|
102
|
+
};
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=sequential.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sequential.js","sourceRoot":"","sources":["../../src/sequential.ts"],"names":[],"mappings":";AAAA,gFAAgF;AAChF,EAAE;AACF,kBAAkB;AAClB,EAAE;AACF,8EAA8E;AAC9E,yEAAyE;AACzE,gFAAgF;AAChF,kEAAkE;AAClE,yEAAyE;AACzE,8EAA8E;AAC9E,8EAA8E;AAC9E,4BAA4B;AAC5B,EAAE;AACF,wEAAwE;AACxE,8EAA8E;AAC9E,qDAAqD;AACrD,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,+EAA+E;AAC/E,EAAE;AACF,SAAS;AACT,EAAE;AACF,wEAAwE;AACxE,+EAA+E;AAC/E,4EAA4E;AAC5E,8EAA8E;AAC9E,gFAAgF;AAChF,mEAAmE;AACnE,EAAE;AACF,6EAA6E;AAC7E,yEAAyE;AACzE,2DAA2D;;AA8C3D,gDA6BC;AAeD,wDAsBC;AA5ED,MAAM,aAAa,GAAG,IAAI,CAAC;AAC3B,MAAM,cAAc,GAAG,GAAG,CAAC;AAE3B;;;;;;GAMG;AACH,SAAgB,kBAAkB,CAChC,QAAyB,EACzB,SAA2B,EAAE;IAE7B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,aAAa,CAAC;IAC5C,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,cAAc,CAAC;IAE/C,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,yBAAyB,CAAC,CAAC;IAC/E,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,0BAA0B,CAAC,CAAC;IAElF,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC;IAC5B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,gBAAgB,GAAG,CAAC,CAAC;IAEzB,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,WAAW;YAAE,SAAS,CAAC,6BAA6B;QAC7E,gBAAgB,EAAE,CAAC;QACnB,4EAA4E;QAC5E,qEAAqE;QACrE,MAAM,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;IACrD,CAAC;IAED,OAAO;QACL,WAAW,EAAE,MAAM,IAAI,SAAS;QAChC,MAAM;QACN,SAAS;QACT,gBAAgB;QAChB,UAAU,EAAE,QAAQ,CAAC,MAAM;KAC5B,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAgB,sBAAsB,CACpC,QAAuB,EACvB,SAA2B,EAAE;IAE7B,OAAO,SAAS,uBAAuB,CAAC,QAA2B;QACjE,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChC,MAAM,QAAQ,GAAI,QAAqE;aACpF,cAAc,CAAC;QAElB,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,MAAM,OAAO,GAAG,kBAAkB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACrD,IAAI,OAAO,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC;QAErC,OAAO;YACL,OAAO,EAAE,KAAK;YACd,OAAO,EAAE;gBACP,GAAG,IAAI,CAAC,OAAO;gBACf,sCAAsC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,SAAS,GAAG;aACxF;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// @metaharness/flywheel — the abstract API surface.
|
|
3
|
+
//
|
|
4
|
+
// DESIGN RULE (load-bearing): this package must NOT know about any host, model, or benchmark — no
|
|
5
|
+
// Claude Code, no SWE-bench, no GLM/Sonnet/Fable, no code-repair. It knows only CANDIDATES, SCORES,
|
|
6
|
+
// GATES, RECEIPTS, and PROMOTION LINEAGE. Everything host- or benchmark-specific enters through the
|
|
7
|
+
// injected `Proposer` / `Evaluator` (and, if you like, a custom `PromotionRule`). If you find yourself
|
|
8
|
+
// wanting a benchmark-specific branch in here, it belongs in the caller, not the flywheel.
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":";AAAA,oDAAoD;AACpD,EAAE;AACF,kGAAkG;AAClG,oGAAoG;AACpG,oGAAoG;AACpG,uGAAuG;AACvG,2FAA2F"}
|
package/dist/cli.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AA8ED,wBAAsB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,CAa1F"}
|
package/dist/cli.js
CHANGED
|
@@ -9,6 +9,7 @@ import { pathToFileURL } from 'node:url';
|
|
|
9
9
|
import { runFlywheelGenerations } from './run.js';
|
|
10
10
|
import { verifyReplayBundle } from './replay.js';
|
|
11
11
|
import { makeSigner } from './receipts.js';
|
|
12
|
+
import { analyzeBundle, formatAnalysis } from './analyze.js';
|
|
12
13
|
const USAGE = [
|
|
13
14
|
'metaharness flywheel — a verifiable self-improvement loop for agent harnesses.',
|
|
14
15
|
'',
|
|
@@ -18,6 +19,8 @@ const USAGE = [
|
|
|
18
19
|
' Independently verify a bundle: receipts + lineage-to-root + (optional) frozen-gate fingerprint.',
|
|
19
20
|
' metaharness flywheel graph <proof-bundle.json>',
|
|
20
21
|
' Print the promoted lineage chain and the compounding lift curve.',
|
|
22
|
+
' metaharness flywheel analyze <proof-bundle.json>',
|
|
23
|
+
' F-P2 mutation-effectiveness: which policy levers earn promotions and how much lift each produces.',
|
|
21
24
|
];
|
|
22
25
|
function loadBundle(path) {
|
|
23
26
|
if (!path)
|
|
@@ -45,6 +48,10 @@ async function replay(args) {
|
|
|
45
48
|
];
|
|
46
49
|
return { code: v.pass ? 0 : 1, lines };
|
|
47
50
|
}
|
|
51
|
+
function analyze(args) {
|
|
52
|
+
const bundle = loadBundle(args[0]);
|
|
53
|
+
return { code: 0, lines: formatAnalysis(analyzeBundle(bundle), `${args[0]} [data_source=${bundle.data_source}]`) };
|
|
54
|
+
}
|
|
48
55
|
function graph(args) {
|
|
49
56
|
const bundle = loadBundle(args[0]);
|
|
50
57
|
const max = Math.max(1, ...bundle.lift_curve.map((p) => p.primary));
|
|
@@ -83,6 +90,7 @@ export async function dispatch(sub, args) {
|
|
|
83
90
|
switch (sub) {
|
|
84
91
|
case 'replay': return replay(args);
|
|
85
92
|
case 'graph': return graph(args);
|
|
93
|
+
case 'analyze': return analyze(args);
|
|
86
94
|
case 'run': return run(args);
|
|
87
95
|
case undefined:
|
|
88
96
|
case 'help':
|
package/dist/cli.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAAA,2FAA2F;AAC3F,+FAA+F;AAC/F,oGAAoG;AACpG,gGAAgG;AAChG,8EAA8E;AAC9E,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,sBAAsB,EAAuB,MAAM,UAAU,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":"AAAA,2FAA2F;AAC3F,+FAA+F;AAC/F,oGAAoG;AACpG,gGAAgG;AAChG,8EAA8E;AAC9E,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,sBAAsB,EAAuB,MAAM,UAAU,CAAC;AACvE,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAQ7D,MAAM,KAAK,GAAG;IACZ,gFAAgF;IAChF,EAAE;IACF,+EAA+E;IAC/E,qGAAqG;IACrG,8EAA8E;IAC9E,uGAAuG;IACvG,kDAAkD;IAClD,wEAAwE;IACxE,oDAAoD;IACpD,yGAAyG;CAC1G,CAAC;AAEF,SAAS,UAAU,CAAC,IAAa;IAC/B,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACnE,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAiB,CAAC;AACjE,CAAC;AACD,SAAS,IAAI,CAAC,IAAc,EAAE,IAAY;IACxC,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1C,CAAC;AAED,KAAK,UAAU,MAAM,CAAC,IAAc;IAClC,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,CAAC,GAAG,kBAAkB,CAAC,MAAM,EAAE,EAAE,qBAAqB,EAAE,IAAI,CAAC,IAAI,EAAE,oBAAoB,CAAC,EAAE,CAAC,CAAC;IAClG,MAAM,KAAK,GAAG;QACZ,aAAa,IAAI,CAAC,CAAC,CAAC,kBAAkB,MAAM,CAAC,WAAW,GAAG;QAC3D,aAAa,CAAC,CAAC,YAAY,EAAE;QAC7B,8CAA8C,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE;QACnF,+CAA+C,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE;QACvF,+CAA+C,CAAC,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE;QAC7F,+CAA+C,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE;QACvF,+CAA+C,IAAI,CAAC,IAAI,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,iCAAiC,EAAE;QAClK,4BAA4B,MAAM,CAAC,qBAAqB,0BAA0B,MAAM,CAAC,6BAA6B,mBAAmB,MAAM,CAAC,iBAAiB,EAAE;QACnK,EAAE;QACF,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,EAAE;KAC1E,CAAC;IACF,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;AACzC,CAAC;AAED,SAAS,OAAO,CAAC,IAAc;IAC7B,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,cAAc,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,kBAAkB,MAAM,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;AACtH,CAAC;AAED,SAAS,KAAK,CAAC,IAAc;IAC3B,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,CAAC,0BAA0B,IAAI,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,OAAO,GAAG,EAAE,EAAE,CAAC,CAAC;IAClF,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,QAAQ,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC3L,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,qBAAqB,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACpJ,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;AAC5B,CAAC;AAED,KAAK,UAAU,GAAG,CAAC,IAAc;IAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACxB,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,oFAAoF,CAAC,EAAE,CAAC;IAChI,MAAM,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,CAAC,CAAC;IAC5E,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAoF,CAAC;IACvI,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,cAAc,CAAC;IAClD,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACnG,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,mGAAmG,CAAC,EAAE,CAAC;IAC7I,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,eAAe,CAAC,CAAC,IAAI,OAAO,CAAC,cAAc,IAAI,EAAE,CAAC;IACjF,MAAM,MAAM,GAAG,MAAM,sBAAsB,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,UAAU,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,MAAM,EAAE,GAAG,OAAO,EAAoB,CAAC,CAAC;IACtL,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,CAAC,OAAO,MAAM,CAAC,cAAc,eAAe,CAAC,CAAC;IAC5D,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,UAAU,aAAa,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACjI,KAAK,CAAC,IAAI,CAAC,0BAA0B,MAAM,CAAC,YAAY,CAAC,qBAAqB,wBAAwB,MAAM,CAAC,YAAY,CAAC,6BAA6B,iBAAiB,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC;IACnM,IAAI,GAAG,EAAE,CAAC;QAAC,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAAC,KAAK,CAAC,IAAI,CAAC,0BAA0B,GAAG,EAAE,CAAC,CAAC;IAAC,CAAC;IAC3H,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;AAC5B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,GAAuB,EAAE,IAAc;IACpE,QAAQ,GAAG,EAAE,CAAC;QACZ,KAAK,QAAQ,CAAC,CAAC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;QACnC,KAAK,OAAO,CAAC,CAAC,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,KAAK,SAAS,CAAC,CAAC,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;QACrC,KAAK,KAAK,CAAC,CAAC,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,KAAK,SAAS,CAAC;QACf,KAAK,MAAM,CAAC;QACZ,KAAK,QAAQ;YACX,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QACnC;YACE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,gCAAgC,GAAG,EAAE,EAAE,EAAE,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC;IACrF,CAAC;AACH,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
export { runFlywheelGenerations } from './run.js';
|
|
2
2
|
export type { FlywheelConfig, FlywheelResult } from './run.js';
|
|
3
3
|
export { meetsPromotionRule, gateFingerprint } from './gate.js';
|
|
4
|
+
export { sequentialEvidence, withSequentialEvidence } from './sequential.js';
|
|
5
|
+
export type { PairedOutcome, SequentialConfig, SequentialVerdict } from './sequential.js';
|
|
4
6
|
export { makeSigner, verifyReceipt, canon } from './receipts.js';
|
|
5
7
|
export { InMemoryLineageStore, computeLiftCurve, liftPoint } from './lineage.js';
|
|
6
8
|
export { verifyReplayBundle } from './replay.js';
|
|
7
9
|
export type { ReplayVerdict } from './replay.js';
|
|
8
|
-
export
|
|
10
|
+
export { analyzeBundle, formatAnalysis } from './analyze.js';
|
|
11
|
+
export type { BundleAnalysis, TargetStat } from './analyze.js';
|
|
12
|
+
export type { Policy, PolicyGenome, CandidateMutation, Score, PromotionEvidence, PromotionDecision, PromotionRule, Suite, HoldoutSuite, AnchorSuite, Proposer, Evaluator, PromotionReceipt, Signer, LineageCommit, LineageStore, LiftPoint, LiftCurve, ReplayBundle, GenerationCheckpoint, ResumeState, } from './types.js';
|
|
9
13
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAClD,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE/D,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAChE,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACjE,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAClD,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE/D,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAC7E,YAAY,EAAE,aAAa,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAC1F,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACjE,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE/D,YAAY,EACV,MAAM,EACN,YAAY,EACZ,iBAAiB,EACjB,KAAK,EACL,iBAAiB,EACjB,iBAAiB,EACjB,aAAa,EACb,KAAK,EACL,YAAY,EACZ,WAAW,EACX,QAAQ,EACR,SAAS,EACT,gBAAgB,EAChB,MAAM,EACN,aAAa,EACb,YAAY,EACZ,SAAS,EACT,SAAS,EACT,YAAY,EACZ,oBAAoB,EACpB,WAAW,GACZ,MAAM,YAAY,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -6,7 +6,9 @@
|
|
|
6
6
|
// evaluator, gate, holdouts, and cost/security rules; get the same auditable, replayable improvement loop.
|
|
7
7
|
export { runFlywheelGenerations } from './run.js';
|
|
8
8
|
export { meetsPromotionRule, gateFingerprint } from './gate.js';
|
|
9
|
+
export { sequentialEvidence, withSequentialEvidence } from './sequential.js';
|
|
9
10
|
export { makeSigner, verifyReceipt, canon } from './receipts.js';
|
|
10
11
|
export { InMemoryLineageStore, computeLiftCurve, liftPoint } from './lineage.js';
|
|
11
12
|
export { verifyReplayBundle } from './replay.js';
|
|
13
|
+
export { analyzeBundle, formatAnalysis } from './analyze.js';
|
|
12
14
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAClF,uEAAuE;AACvE,EAAE;AACF,uGAAuG;AACvG,yGAAyG;AACzG,2GAA2G;AAC3G,OAAO,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAChE,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACjE,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAClF,uEAAuE;AACvE,EAAE;AACF,uGAAuG;AACvG,yGAAyG;AACzG,2GAA2G;AAC3G,OAAO,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAE7E,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACjE,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC"}
|
package/dist/replay.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ReplayBundle } from './types.js';
|
|
1
|
+
import type { ReplayBundle, PromotionRule } from './types.js';
|
|
2
2
|
export interface ReplayVerdict {
|
|
3
3
|
pass: boolean;
|
|
4
4
|
checks: {
|
|
@@ -7,11 +7,13 @@ export interface ReplayVerdict {
|
|
|
7
7
|
contiguousParents: boolean;
|
|
8
8
|
allPromoted: boolean;
|
|
9
9
|
gateUnchanged: boolean;
|
|
10
|
+
gateReExecutes: boolean;
|
|
10
11
|
};
|
|
11
12
|
failures: string[];
|
|
12
13
|
chainSummary: string;
|
|
13
14
|
}
|
|
14
15
|
export declare function verifyReplayBundle(bundle: ReplayBundle, opts?: {
|
|
15
16
|
pinnedGateFingerprint?: string;
|
|
17
|
+
promotionRule?: PromotionRule;
|
|
16
18
|
}): ReplayVerdict;
|
|
17
19
|
//# sourceMappingURL=replay.d.ts.map
|
package/dist/replay.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../src/replay.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"replay.d.ts","sourceRoot":"","sources":["../src/replay.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAiB,MAAM,YAAY,CAAC;AAE7E,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE;QACN,QAAQ,EAAE,OAAO,CAAC;QAClB,WAAW,EAAE,OAAO,CAAC;QACrB,iBAAiB,EAAE,OAAO,CAAC;QAC3B,WAAW,EAAE,OAAO,CAAC;QACrB,aAAa,EAAE,OAAO,CAAC;QACvB,cAAc,EAAE,OAAO,CAAC;KACzB,CAAC;IACF,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,YAAY,EACpB,IAAI,GAAE;IAAE,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,aAAa,CAAA;CAAO,GAC3E,aAAa,CA8Df"}
|
package/dist/replay.js
CHANGED
|
@@ -1,10 +1,15 @@
|
|
|
1
1
|
// @metaharness/flywheel — independent replay. Given ONLY a ReplayBundle (and, optionally, the pinned
|
|
2
|
-
// gate fingerprint), an external reviewer establishes the run with no trust in the
|
|
2
|
+
// gate fingerprint + the gate rule itself), an external reviewer establishes the run with no trust in the
|
|
3
|
+
// producer:
|
|
3
4
|
// (1) every promotion receipt verifies (Ed25519, recompute canon vs the embedded key);
|
|
4
5
|
// (2) the promoted lineage reconstructs current → gen-0 immutable root, contiguously;
|
|
5
6
|
// (3) every commit on the promoted chain is actually PROMOTED (no rejected node smuggled in);
|
|
6
|
-
// (4) the gate fingerprint matches the pinned value ⇒ the promotion rule was UNCHANGED
|
|
7
|
+
// (4) the gate fingerprint matches the pinned value ⇒ the promotion rule was UNCHANGED;
|
|
8
|
+
// (5) [ADR-235] if the reviewer supplies the (fingerprint-matched) rule, every PROMOTED commit is
|
|
9
|
+
// RE-GATED on its sealed baseline+candidate scores and must STILL promote — trust the gate re-run,
|
|
10
|
+
// not the logged verdict. Catches a signed-but-forged promotion the fingerprint check cannot.
|
|
7
11
|
import { verifyReceipt } from './receipts.js';
|
|
12
|
+
import { gateFingerprint } from './gate.js';
|
|
8
13
|
export function verifyReplayBundle(bundle, opts = {}) {
|
|
9
14
|
const failures = [];
|
|
10
15
|
const chain = bundle.chain;
|
|
@@ -22,16 +27,50 @@ export function verifyReplayBundle(bundle, opts = {}) {
|
|
|
22
27
|
}
|
|
23
28
|
if (!contiguousParents)
|
|
24
29
|
failures.push('contiguousParents');
|
|
30
|
+
// Every non-root commit on the promoted chain must be PROMOTED (no rejected node smuggled in). An
|
|
31
|
+
// HONEST-NULL run — the flywheel found no improvement, so the chain is just the immutable gen-0 root —
|
|
32
|
+
// is VALID and replayable: an empty set of non-root commits satisfies this VACUOUSLY. (Requiring
|
|
33
|
+
// ≥1 promotion here was a bug: it failed replay on a legitimate 0-promotion result, e.g. a weak model
|
|
34
|
+
// that resolves nothing — the negative is a real, verifiable outcome, not an invalid bundle.)
|
|
25
35
|
const promos = chain.filter((c) => c.verdict !== 'ROOT');
|
|
26
|
-
const allPromoted = promos.
|
|
36
|
+
const allPromoted = promos.every((c) => c.verdict === 'PROMOTED');
|
|
27
37
|
if (!allPromoted)
|
|
28
38
|
failures.push('allPromoted');
|
|
29
39
|
const gateUnchanged = opts.pinnedGateFingerprint ? bundle.gate_fingerprint === opts.pinnedGateFingerprint : true;
|
|
30
40
|
if (!gateUnchanged)
|
|
31
41
|
failures.push('gateUnchanged');
|
|
42
|
+
// (5) RE-EXECUTE the gate. Only when the reviewer supplies the rule (otherwise unchecked → true, so
|
|
43
|
+
// existing fingerprint-only callers are unaffected). The supplied rule must be the SAME one (its
|
|
44
|
+
// fingerprint must match the pinned/bundled value) or re-execution is meaningless. Then every PROMOTED
|
|
45
|
+
// commit that carries its sealed scores must RE-PASS the rule — a logged promotion the frozen gate would
|
|
46
|
+
// NOT grant is a forgery.
|
|
47
|
+
let gateReExecutes = true;
|
|
48
|
+
if (opts.promotionRule) {
|
|
49
|
+
const suppliedFp = gateFingerprint(opts.promotionRule);
|
|
50
|
+
const expectedFp = opts.pinnedGateFingerprint ?? bundle.gate_fingerprint ?? undefined;
|
|
51
|
+
if (expectedFp && suppliedFp !== expectedFp) {
|
|
52
|
+
gateReExecutes = false; // wrong rule supplied — cannot re-execute the run's gate
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
const seen = new Set();
|
|
56
|
+
for (const c of [...chain, ...bundle.all_commits]) {
|
|
57
|
+
if (seen.has(c.id))
|
|
58
|
+
continue;
|
|
59
|
+
seen.add(c.id);
|
|
60
|
+
if (c.verdict === 'PROMOTED' && c.baselineScore && c.candidateScore) {
|
|
61
|
+
if (!opts.promotionRule({ baseline: c.baselineScore, candidate: c.candidateScore }).promote) {
|
|
62
|
+
gateReExecutes = false;
|
|
63
|
+
break;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
if (!gateReExecutes)
|
|
70
|
+
failures.push('gateReExecutes');
|
|
32
71
|
return {
|
|
33
72
|
pass: failures.length === 0,
|
|
34
|
-
checks: { receipts, reachesRoot, contiguousParents, allPromoted, gateUnchanged },
|
|
73
|
+
checks: { receipts, reachesRoot, contiguousParents, allPromoted, gateUnchanged, gateReExecutes },
|
|
35
74
|
failures,
|
|
36
75
|
chainSummary: chain.map((c) => `gen${c.generation}${c.mutation ? `(${c.mutation.target})` : '(root)'}`).join(' → '),
|
|
37
76
|
};
|
package/dist/replay.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"replay.js","sourceRoot":"","sources":["../src/replay.ts"],"names":[],"mappings":"AAAA,qGAAqG;AACrG,
|
|
1
|
+
{"version":3,"file":"replay.js","sourceRoot":"","sources":["../src/replay.ts"],"names":[],"mappings":"AAAA,qGAAqG;AACrG,0GAA0G;AAC1G,YAAY;AACZ,yFAAyF;AACzF,wFAAwF;AACxF,gGAAgG;AAChG,0FAA0F;AAC1F,oGAAoG;AACpG,yGAAyG;AACzG,oGAAoG;AACpG,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAiB5C,MAAM,UAAU,kBAAkB,CAChC,MAAoB,EACpB,OAA0E,EAAE;IAE5E,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAE3B,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAClF,IAAI,CAAC,QAAQ;QAAE,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAEzC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrC,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,EAAE,KAAK,MAAM,CAAC,OAAO,CAAC;IACtF,IAAI,CAAC,WAAW;QAAE,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAE/C,IAAI,iBAAiB,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,EAAE,CAAC;YAAE,iBAAiB,GAAG,KAAK,CAAC;IAC/E,CAAC;IACD,IAAI,CAAC,iBAAiB;QAAE,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAE3D,kGAAkG;IAClG,uGAAuG;IACvG,iGAAiG;IACjG,sGAAsG;IACtG,8FAA8F;IAC9F,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC;IACzD,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC;IAClE,IAAI,CAAC,WAAW;QAAE,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAE/C,MAAM,aAAa,GAAG,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,MAAM,CAAC,gBAAgB,KAAK,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,CAAC;IACjH,IAAI,CAAC,aAAa;QAAE,QAAQ,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAEnD,oGAAoG;IACpG,iGAAiG;IACjG,uGAAuG;IACvG,yGAAyG;IACzG,0BAA0B;IAC1B,IAAI,cAAc,GAAG,IAAI,CAAC;IAC1B,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;QACvB,MAAM,UAAU,GAAG,eAAe,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACvD,MAAM,UAAU,GAAG,IAAI,CAAC,qBAAqB,IAAI,MAAM,CAAC,gBAAgB,IAAI,SAAS,CAAC;QACtF,IAAI,UAAU,IAAI,UAAU,KAAK,UAAU,EAAE,CAAC;YAC5C,cAAc,GAAG,KAAK,CAAC,CAAC,yDAAyD;QACnF,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,EAAE,GAAG,MAAM,CAAC,WAAW,CAAoB,EAAE,CAAC;gBACrE,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBAAE,SAAS;gBAC7B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACf,IAAI,CAAC,CAAC,OAAO,KAAK,UAAU,IAAI,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;oBACpE,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,aAAa,EAAE,SAAS,EAAE,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;wBAC5F,cAAc,GAAG,KAAK,CAAC;wBACvB,MAAM;oBACR,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,CAAC,cAAc;QAAE,QAAQ,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAErD,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;QAC3B,MAAM,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,iBAAiB,EAAE,WAAW,EAAE,aAAa,EAAE,cAAc,EAAE;QAChG,QAAQ;QACR,YAAY,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;KACpH,CAAC;AACJ,CAAC"}
|
package/dist/run.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Policy, Proposer, Evaluator, PromotionRule, Signer, HoldoutSuite, AnchorSuite, LineageStore, LineageCommit, LiftCurve, ReplayBundle } from './types.js';
|
|
1
|
+
import type { Policy, Proposer, Evaluator, PromotionRule, Signer, HoldoutSuite, AnchorSuite, LineageStore, LineageCommit, LiftCurve, ReplayBundle, GenerationCheckpoint, ResumeState } from './types.js';
|
|
2
2
|
export interface FlywheelConfig {
|
|
3
3
|
/** The gen-0 policy — the immutable root every promotion chains back to. */
|
|
4
4
|
rootPolicy: Policy;
|
|
@@ -22,6 +22,16 @@ export interface FlywheelConfig {
|
|
|
22
22
|
now?: (generation: number) => string;
|
|
23
23
|
/** Stamped on the replay bundle — 'SYNTHETIC' | 'LIVE' | …. NEVER a benchmark name. */
|
|
24
24
|
dataSource?: string;
|
|
25
|
+
/** Optional per-generation checkpoint hook — called at the END of each generation with a fully
|
|
26
|
+
* assembled replay bundle for the run so far. Lets a long (multi-hour) run persist incremental
|
|
27
|
+
* progress so a crash keeps the completed generations. Observation-only: never affects promotion.
|
|
28
|
+
* Errors thrown by the hook are swallowed (a bad checkpoint must not kill a valid run). */
|
|
29
|
+
onGeneration?: (info: GenerationCheckpoint) => void | Promise<void>;
|
|
30
|
+
/** Resume a crashed run from a persisted {@link GenerationCheckpoint.resumeState}. When set, the gen-0
|
|
31
|
+
* root is NOT re-evaluated or re-created — the prior lineage is re-seeded and the loop continues from
|
|
32
|
+
* `resumeFrom.fromGeneration + 1`. Unset ⇒ a fresh run (identical to before). The caller is
|
|
33
|
+
* responsible for restoring any external spend counter from the checkpoint's `spent`. */
|
|
34
|
+
resumeFrom?: ResumeState;
|
|
25
35
|
lineageStore?: LineageStore;
|
|
26
36
|
rootId?: string;
|
|
27
37
|
}
|