@metaharness/flywheel 0.1.7 â 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/dist/cjs/index.js +4 -1
- package/dist/cjs/index.js.map +1 -1
- package/dist/cjs/run.js +18 -3
- package/dist/cjs/run.js.map +1 -1
- package/dist/cjs/sequential.js +105 -0
- package/dist/cjs/sequential.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/run.d.ts +7 -0
- package/dist/run.d.ts.map +1 -1
- package/dist/run.js +18 -3
- package/dist/run.js.map +1 -1
- package/dist/sequential.d.ts +53 -0
- package/dist/sequential.d.ts.map +1 -0
- package/dist/sequential.js +101 -0
- package/dist/sequential.js.map +1 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -47,6 +47,12 @@ Most "self-improving agent" pitches are unfalsifiable. The flywheel makes improv
|
|
|
47
47
|
- ð **Gate fingerprint** â prove the promotion rule was unchanged between runs.
|
|
48
48
|
- ð§Đ **Zero host/benchmark coupling** â knows only candidates, scores, gates, receipts, lineage.
|
|
49
49
|
- ðŠķ **Thin + dependency-free** at runtime (Node `crypto` only). ESM, fully typed.
|
|
50
|
+
- âŧïļ **Opt-in evaluation memo** â `cacheEvaluations: true` reuses the score of a (policy, suite)
|
|
51
|
+
pair the run already measured, so a proposer that re-proposes a previously-tried policy (small
|
|
52
|
+
lever domains, cycling proposers) doesn't re-spend a full benchmark run. Off by default; enable
|
|
53
|
+
only when reusing the first measurement is acceptable (deterministic or noise-tolerant
|
|
54
|
+
evaluators). On an 8-generation, 3-lever kernel-tuning run this cut evaluator calls 28 â 13
|
|
55
|
+
(54%) with an identical final policy and lineage.
|
|
50
56
|
|
|
51
57
|
## Install
|
|
52
58
|
|
package/dist/cjs/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.formatAnalysis = exports.analyzeBundle = exports.verifyReplayBundle = exports.liftPoint = exports.computeLiftCurve = exports.InMemoryLineageStore = exports.canon = exports.verifyReceipt = exports.makeSigner = exports.gateFingerprint = exports.meetsPromotionRule = exports.runFlywheelGenerations = void 0;
|
|
3
|
+
exports.formatAnalysis = exports.analyzeBundle = exports.verifyReplayBundle = exports.liftPoint = exports.computeLiftCurve = exports.InMemoryLineageStore = exports.canon = exports.verifyReceipt = exports.makeSigner = exports.withSequentialEvidence = exports.sequentialEvidence = exports.gateFingerprint = exports.meetsPromotionRule = exports.runFlywheelGenerations = void 0;
|
|
4
4
|
// @metaharness/flywheel â a verifiable self-improvement loop for agent harnesses.
|
|
5
5
|
// Freeze the model. Evolve the harness. Promote only what proves lift.
|
|
6
6
|
//
|
|
@@ -12,6 +12,9 @@ Object.defineProperty(exports, "runFlywheelGenerations", { enumerable: true, get
|
|
|
12
12
|
var gate_js_1 = require("./gate.js");
|
|
13
13
|
Object.defineProperty(exports, "meetsPromotionRule", { enumerable: true, get: function () { return gate_js_1.meetsPromotionRule; } });
|
|
14
14
|
Object.defineProperty(exports, "gateFingerprint", { enumerable: true, get: function () { return gate_js_1.gateFingerprint; } });
|
|
15
|
+
var sequential_js_1 = require("./sequential.js");
|
|
16
|
+
Object.defineProperty(exports, "sequentialEvidence", { enumerable: true, get: function () { return sequential_js_1.sequentialEvidence; } });
|
|
17
|
+
Object.defineProperty(exports, "withSequentialEvidence", { enumerable: true, get: function () { return sequential_js_1.withSequentialEvidence; } });
|
|
15
18
|
var receipts_js_1 = require("./receipts.js");
|
|
16
19
|
Object.defineProperty(exports, "makeSigner", { enumerable: true, get: function () { return receipts_js_1.makeSigner; } });
|
|
17
20
|
Object.defineProperty(exports, "verifyReceipt", { enumerable: true, get: function () { return receipts_js_1.verifyReceipt; } });
|
package/dist/cjs/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;AAAA,kFAAkF;AAClF,uEAAuE;AACvE,EAAE;AACF,uGAAuG;AACvG,yGAAyG;AACzG,2GAA2G;AAC3G,mCAAkD;AAAzC,gHAAA,sBAAsB,OAAA;AAG/B,qCAAgE;AAAvD,6GAAA,kBAAkB,OAAA;AAAE,0GAAA,eAAe,OAAA;AAC5C,6CAAiE;AAAxD,yGAAA,UAAU,OAAA;AAAE,4GAAA,aAAa,OAAA;AAAE,oGAAA,KAAK,OAAA;AACzC,2CAAiF;AAAxE,kHAAA,oBAAoB,OAAA;AAAE,8GAAA,gBAAgB,OAAA;AAAE,uGAAA,SAAS,OAAA;AAC1D,yCAAiD;AAAxC,+GAAA,kBAAkB,OAAA;AAE3B,2CAA6D;AAApD,2GAAA,aAAa,OAAA;AAAE,4GAAA,cAAc,OAAA"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";;;AAAA,kFAAkF;AAClF,uEAAuE;AACvE,EAAE;AACF,uGAAuG;AACvG,yGAAyG;AACzG,2GAA2G;AAC3G,mCAAkD;AAAzC,gHAAA,sBAAsB,OAAA;AAG/B,qCAAgE;AAAvD,6GAAA,kBAAkB,OAAA;AAAE,0GAAA,eAAe,OAAA;AAC5C,iDAA6E;AAApE,mHAAA,kBAAkB,OAAA;AAAE,uHAAA,sBAAsB,OAAA;AAEnD,6CAAiE;AAAxD,yGAAA,UAAU,OAAA;AAAE,4GAAA,aAAa,OAAA;AAAE,oGAAA,KAAK,OAAA;AACzC,2CAAiF;AAAxE,kHAAA,oBAAoB,OAAA;AAAE,8GAAA,gBAAgB,OAAA;AAAE,uGAAA,SAAS,OAAA;AAC1D,yCAAiD;AAAxC,+GAAA,kBAAkB,OAAA;AAE3B,2CAA6D;AAApD,2GAAA,aAAa,OAAA;AAAE,4GAAA,cAAc,OAAA"}
|
package/dist/cjs/run.js
CHANGED
|
@@ -14,7 +14,22 @@ async function runFlywheelGenerations(cfg) {
|
|
|
14
14
|
const store = cfg.lineageStore ?? new lineage_js_1.InMemoryLineageStore();
|
|
15
15
|
const now = cfg.now ?? ((g) => `gen-${g}`);
|
|
16
16
|
const rootId = cfg.rootId ?? 'root';
|
|
17
|
-
|
|
17
|
+
// Optional per-run evaluation memo (see FlywheelConfig.cacheEvaluations). Keyed by suite id +
|
|
18
|
+
// key-sorted policy, so key order in the Policy object never splits the cache. Returns a copy so
|
|
19
|
+
// a caller mutating a Score cannot poison later hits.
|
|
20
|
+
const evalCache = cfg.cacheEvaluations ? new Map() : null;
|
|
21
|
+
const evaluate = async (p, suite) => {
|
|
22
|
+
if (!evalCache)
|
|
23
|
+
return cfg.evaluator(p, suite);
|
|
24
|
+
const key = `${suite.id}\u0000${JSON.stringify(Object.keys(p).sort().map((k) => [k, p[k]]))}`;
|
|
25
|
+
const hit = evalCache.get(key);
|
|
26
|
+
if (hit)
|
|
27
|
+
return { ...hit };
|
|
28
|
+
const s = await cfg.evaluator(p, suite);
|
|
29
|
+
evalCache.set(key, s);
|
|
30
|
+
return { ...s };
|
|
31
|
+
};
|
|
32
|
+
const anchorOf = async (p) => cfg.anchor ? (await evaluate(p, cfg.anchor)).primary : null;
|
|
18
33
|
let rootScore;
|
|
19
34
|
let rootAnchor;
|
|
20
35
|
let parentId;
|
|
@@ -39,7 +54,7 @@ async function runFlywheelGenerations(cfg) {
|
|
|
39
54
|
}
|
|
40
55
|
else {
|
|
41
56
|
// ââ gen-0: the immutable root. Evaluate it once (the baseline) + its anchor (the frozen bar). ââ
|
|
42
|
-
rootScore = await
|
|
57
|
+
rootScore = await evaluate(cfg.rootPolicy, cfg.holdout);
|
|
43
58
|
rootAnchor = await anchorOf(cfg.rootPolicy);
|
|
44
59
|
await store.append({
|
|
45
60
|
id: rootId, generation: 0, parents: [], mutation: null, primaryDelta: 0, anchorScore: rootAnchor,
|
|
@@ -79,7 +94,7 @@ async function runFlywheelGenerations(cfg) {
|
|
|
79
94
|
for (const target of targets) {
|
|
80
95
|
const proposed = await cfg.proposer(base, target);
|
|
81
96
|
const candPolicy = { ...policy, [target]: proposed };
|
|
82
|
-
const candScore = await
|
|
97
|
+
const candScore = await evaluate(candPolicy, cfg.holdout);
|
|
83
98
|
const decision = rule({ baseline: score, candidate: candScore });
|
|
84
99
|
cands.push({ target, policy: candPolicy, score: candScore, reasons: decision.reasons, promote: decision.promote });
|
|
85
100
|
}
|
package/dist/cjs/run.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/run.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/run.ts"],"names":[],"mappings":";;AAkEA,wDAsJC;AAxND,wGAAwG;AACxG,qGAAqG;AACrG,wGAAwG;AACxG,uGAAuG;AACvG,sGAAsG;AACtG,6CAAsE;AACtE,uCAAgE;AA4DzD,KAAK,UAAU,sBAAsB,CAAC,GAAmB;IAC9D,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,IAAI,4BAAkB,CAAC;IACrD,MAAM,OAAO,GAAG,GAAG,CAAC,eAAe,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACnE,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,IAAI,IAAI,iCAAoB,EAAE,CAAC;IAC7D,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC;IAEpC,8FAA8F;IAC9F,iGAAiG;IACjG,sDAAsD;IACtD,MAAM,SAAS,GAAG,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,GAAG,EAAiB,CAAC,CAAC,CAAC,IAAI,CAAC;IACzE,MAAM,QAAQ,GAAc,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE;QAC7C,IAAI,CAAC,SAAS;YAAE,OAAO,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,EAAE,SAAS,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,GAAG;YAAE,OAAO,EAAE,GAAG,GAAG,EAAE,CAAC;QAC3B,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACxC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACtB,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC;IAClB,CAAC,CAAC;IAEF,MAAM,QAAQ,GAAG,KAAK,EAAE,CAAS,EAA0B,EAAE,CAC3D,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IAE9D,IAAI,SAAgB,CAAC;IACrB,IAAI,UAAyB,CAAC;IAC9B,IAAI,QAAgB,CAAC;IACrB,IAAI,MAAc,CAAC;IACnB,IAAI,KAAY,CAAC;IACjB,MAAM,UAAU,GAAoB,EAAE,CAAC;IACvC,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;QACnB,qGAAqG;QACrG,MAAM,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY;YAAE,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACtD,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,+BAA+B;QACvG,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;QAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;QACnD,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;QAAC,MAAM,GAAG,EAAE,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;QAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;QACjE,cAAc,GAAG,CAAC,CAAC,cAAc,CAAC;QAAC,QAAQ,GAAG,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC;IACrE,CAAC;SAAM,CAAC;QACN,kGAAkG;QAClG,SAAS,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;QACxD,UAAU,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC5C,MAAM,KAAK,CAAC,MAAM,CAAC;YACjB,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,WAAW,EAAE,UAAU;YAChG,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;SACjH,CAAC,CAAC;QACH,QAAQ,GAAG,MAAM,CAAC;QAClB,MAAM,GAAG,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC;QAC/B,KAAK,GAAG,SAAS,CAAC;IACpB,CAAC;IAED,gGAAgG;IAChG,iGAAiG;IACjG,qGAAqG;IACrG,MAAM,WAAW,GAAG,CAAC,QAAyB,EAAE,SAAiB,EAAgB,EAAE;QACjF,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QACpE,MAAM,gBAAgB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,UAAU,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,QAAQ,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC;QAClJ,OAAO;YACL,WAAW,EAAE,GAAG,CAAC,UAAU,IAAI,aAAa;YAC5C,OAAO,EAAE,MAAM;YACf,KAAK,EAAE,QAAQ;YACf,WAAW,EAAE,UAAU;YACvB,UAAU,EAAE,IAAA,6BAAgB,EAAC,QAAQ,EAAE,SAAS,CAAC,OAAO,CAAC;YACzD,gBAAgB,EAAE,IAAA,yBAAe,EAAC,IAAI,CAAC;YACvC,qBAAqB,EAAE,SAAS;YAChC,6BAA6B,EAAE,gBAAgB;YAC/C,iBAAiB,EAAE,gBAAgB,IAAI,CAAC;YACxC,UAAU,EAAE,SAAS;SACtB,CAAC;IACJ,CAAC,CAAC;IAEF,KAAK,IAAI,GAAG,GAAG,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,CAAC;QAC1D,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK;YAAE,MAAM;QAChE,cAAc,GAAG,GAAG,CAAC;QAErB,kFAAkF;QAClF,MAAM,IAAI,GAAiB,EAAE,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;QAC1F,MAAM,KAAK,GAAiG,EAAE,CAAC;QAC/G,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAClD,MAAM,UAAU,GAAW,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC;YAC7D,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;YACjE,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;QACrH,CAAC;QAED,4FAA4F;QAC5F,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpG,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACrC,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACnE,MAAM,cAAc,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,KAAK,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,QAAQ,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAC3G,gGAAgG;QAChG,MAAM,cAAc,GAAG,MAAM,IAAI,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;QAEhE,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,QAAQ,GAAG,CAAC,KAAK,cAAc,CAAC;YACtC,MAAM,EAAE,GAAG,GAAG,QAAQ,KAAK,CAAC,CAAC,MAAM,OAAO,GAAG,EAAE,CAAC;YAChD,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;YACrD,MAAM,MAAM,GAAkB;gBAC5B,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,QAAQ,CAAC;gBACxC,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,MAAM,EAAE,EAAE;gBAC5D,YAAY;gBACZ,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI;gBACzE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU;gBAC3C,cAAc,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO;gBAClG,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,EAAE,YAAY,EAAE,CAAC;gBAChI,SAAS,EAAE,GAAG,CAAC,GAAG,CAAC;gBACnB,aAAa,EAAE,KAAK,EAAQ,uDAAuD;gBACnF,cAAc,EAAE,CAAC,CAAC,KAAK;aACxB,CAAC;YACF,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC3B,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YAAC,QAAQ,GAAG,GAAG,QAAQ,KAAK,cAAc,CAAC,MAAM,OAAO,GAAG,EAAE,CAAC;YAAC,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC;YAAC,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC;QAAC,CAAC;QAEnJ,mGAAmG;QACnG,mGAAmG;QACnG,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBACtD,MAAM,IAAI,GAAyB;oBACjC,UAAU,EAAE,GAAG;oBACf,cAAc;oBACd,aAAa,EAAE,WAAW,CAAC,YAAY,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;oBAClD,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;oBAC1C,WAAW,EAAE;wBACX,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK;wBAC9C,cAAc,EAAE,GAAG;wBACnB,YAAY,EAAE,MAAM,KAAK,CAAC,IAAI,EAAE,EAAE,wDAAwD;qBAC3F;iBACF,CAAC;gBACF,MAAM,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC/B,CAAC;YAAC,MAAM,CAAC,CAAC,kEAAkE,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC;IACjE,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC;IAEjE,OAAO;QACL,SAAS,EAAE,YAAY,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY;QAC5E,cAAc;QACd,gBAAgB,EAAE,YAAY,CAAC,iBAAiB,EAAE,WAAW,EAAE,MAAM;KACtE,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// @metaharness/flywheel â anytime-valid sequential testing for promotion gates.
|
|
3
|
+
//
|
|
4
|
+
// WHY THIS EXISTS
|
|
5
|
+
//
|
|
6
|
+
// `meetsPromotionRule` is a single-shot conjunctive comparison, and it is the
|
|
7
|
+
// right shape: frozen, fingerprintable, every clause load-bearing. But a
|
|
8
|
+
// flywheel run evaluates MANY generations against the SAME holdout, and that is
|
|
9
|
+
// uncontrolled multiple testing. Published measurements of greedy
|
|
10
|
+
// "accept-if-the-score-improved" acceptance put the false-commit rate at
|
|
11
|
+
// 30-42%, and found 13-21 spurious modifications made even when NO true gains
|
|
12
|
+
// existed â degrading one agent by 4.9 points while every individual decision
|
|
13
|
+
// looked locally justified.
|
|
14
|
+
//
|
|
15
|
+
// The conjunctive gate plus a frozen anchor already mitigates this with
|
|
16
|
+
// multiple hurdles, which is real. It is not, however, anytime-valid: nothing
|
|
17
|
+
// in it accounts for how many times you have looked.
|
|
18
|
+
//
|
|
19
|
+
// This module adds that missing property WITHOUT touching the default gate,
|
|
20
|
+
// whose stability is itself the product. It composes: wrap any PromotionRule,
|
|
21
|
+
// and a candidate must clear both the frozen clauses AND accumulated evidence.
|
|
22
|
+
//
|
|
23
|
+
// METHOD
|
|
24
|
+
//
|
|
25
|
+
// Testing-by-betting / e-processes. An e-value is a non-negative random
|
|
26
|
+
// variable with expectation <= 1 under the null hypothesis ("this candidate is
|
|
27
|
+
// no better than baseline"). By Ville's inequality, P(sup_t E_t >= 1/alpha)
|
|
28
|
+
// <= alpha, so you may stop and reject at ANY time â no pre-registered sample
|
|
29
|
+
// size, no alpha spending schedule, and no penalty for peeking. That is exactly
|
|
30
|
+
// the property a flywheel needs, because it peeks by construction.
|
|
31
|
+
//
|
|
32
|
+
// The bet here is deliberately simple and assumption-light: per paired item,
|
|
33
|
+
// a candidate win against a baseline loss multiplies the e-value up, the
|
|
34
|
+
// reverse multiplies it down, and ties leave it unchanged.
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.sequentialEvidence = sequentialEvidence;
|
|
37
|
+
exports.withSequentialEvidence = withSequentialEvidence;
|
|
38
|
+
const DEFAULT_ALPHA = 0.05;
|
|
39
|
+
const DEFAULT_LAMBDA = 0.5;
|
|
40
|
+
/**
|
|
41
|
+
* Accumulate paired outcomes into an anytime-valid e-value.
|
|
42
|
+
*
|
|
43
|
+
* Only discordant pairs move the e-value: if both arms win or both lose, that
|
|
44
|
+
* item tells you nothing about which is better (this is the McNemar insight,
|
|
45
|
+
* carried over to the sequential setting).
|
|
46
|
+
*/
|
|
47
|
+
function sequentialEvidence(outcomes, config = {}) {
|
|
48
|
+
const alpha = config.alpha ?? DEFAULT_ALPHA;
|
|
49
|
+
const lambda = config.lambda ?? DEFAULT_LAMBDA;
|
|
50
|
+
if (!(alpha > 0 && alpha < 1))
|
|
51
|
+
throw new RangeError('alpha must be in (0, 1)');
|
|
52
|
+
if (!(lambda > 0 && lambda < 1))
|
|
53
|
+
throw new RangeError('lambda must be in (0, 1)');
|
|
54
|
+
const threshold = 1 / alpha;
|
|
55
|
+
let eValue = 1;
|
|
56
|
+
let informativePairs = 0;
|
|
57
|
+
for (const o of outcomes) {
|
|
58
|
+
if (o.candidateWon === o.baselineWon)
|
|
59
|
+
continue; // concordant: no information
|
|
60
|
+
informativePairs++;
|
|
61
|
+
// Under the null, a discordant pair favors either arm with probability 1/2,
|
|
62
|
+
// so E[multiplier] = 1 and the process is a non-negative martingale.
|
|
63
|
+
eValue *= o.candidateWon ? 1 + lambda : 1 - lambda;
|
|
64
|
+
}
|
|
65
|
+
return {
|
|
66
|
+
significant: eValue >= threshold,
|
|
67
|
+
eValue,
|
|
68
|
+
threshold,
|
|
69
|
+
informativePairs,
|
|
70
|
+
totalPairs: outcomes.length,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Compose a frozen gate with a sequential-evidence requirement.
|
|
75
|
+
*
|
|
76
|
+
* The returned rule is still a plain `PromotionRule`, so it fingerprints and
|
|
77
|
+
* freezes exactly like the default one. A candidate must satisfy BOTH: every
|
|
78
|
+
* clause of `baseRule`, and evidence strong enough to survive having been
|
|
79
|
+
* looked at repeatedly.
|
|
80
|
+
*
|
|
81
|
+
* Paired outcomes are read from `evidence.pairedOutcomes` when present. When
|
|
82
|
+
* absent the rule degrades to `baseRule` alone rather than silently blocking
|
|
83
|
+
* every promotion â a caller that has not wired up per-item outcomes yet should
|
|
84
|
+
* get the old behavior, not a permanently closed gate.
|
|
85
|
+
*/
|
|
86
|
+
function withSequentialEvidence(baseRule, config = {}) {
|
|
87
|
+
return function sequentialPromotionRule(evidence) {
|
|
88
|
+
const base = baseRule(evidence);
|
|
89
|
+
const outcomes = evidence
|
|
90
|
+
.pairedOutcomes;
|
|
91
|
+
if (!outcomes)
|
|
92
|
+
return base;
|
|
93
|
+
const verdict = sequentialEvidence(outcomes, config);
|
|
94
|
+
if (verdict.significant)
|
|
95
|
+
return base;
|
|
96
|
+
return {
|
|
97
|
+
promote: false,
|
|
98
|
+
reasons: [
|
|
99
|
+
...base.reasons,
|
|
100
|
+
`insufficient_sequential_evidence(e=${verdict.eValue.toFixed(2)}<${verdict.threshold})`,
|
|
101
|
+
],
|
|
102
|
+
};
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=sequential.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sequential.js","sourceRoot":"","sources":["../../src/sequential.ts"],"names":[],"mappings":";AAAA,gFAAgF;AAChF,EAAE;AACF,kBAAkB;AAClB,EAAE;AACF,8EAA8E;AAC9E,yEAAyE;AACzE,gFAAgF;AAChF,kEAAkE;AAClE,yEAAyE;AACzE,8EAA8E;AAC9E,8EAA8E;AAC9E,4BAA4B;AAC5B,EAAE;AACF,wEAAwE;AACxE,8EAA8E;AAC9E,qDAAqD;AACrD,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,+EAA+E;AAC/E,EAAE;AACF,SAAS;AACT,EAAE;AACF,wEAAwE;AACxE,+EAA+E;AAC/E,4EAA4E;AAC5E,8EAA8E;AAC9E,gFAAgF;AAChF,mEAAmE;AACnE,EAAE;AACF,6EAA6E;AAC7E,yEAAyE;AACzE,2DAA2D;;AA8C3D,gDA6BC;AAeD,wDAsBC;AA5ED,MAAM,aAAa,GAAG,IAAI,CAAC;AAC3B,MAAM,cAAc,GAAG,GAAG,CAAC;AAE3B;;;;;;GAMG;AACH,SAAgB,kBAAkB,CAChC,QAAyB,EACzB,SAA2B,EAAE;IAE7B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,aAAa,CAAC;IAC5C,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,cAAc,CAAC;IAE/C,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,yBAAyB,CAAC,CAAC;IAC/E,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,0BAA0B,CAAC,CAAC;IAElF,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC;IAC5B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,gBAAgB,GAAG,CAAC,CAAC;IAEzB,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,WAAW;YAAE,SAAS,CAAC,6BAA6B;QAC7E,gBAAgB,EAAE,CAAC;QACnB,4EAA4E;QAC5E,qEAAqE;QACrE,MAAM,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;IACrD,CAAC;IAED,OAAO;QACL,WAAW,EAAE,MAAM,IAAI,SAAS;QAChC,MAAM;QACN,SAAS;QACT,gBAAgB;QAChB,UAAU,EAAE,QAAQ,CAAC,MAAM;KAC5B,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAgB,sBAAsB,CACpC,QAAuB,EACvB,SAA2B,EAAE;IAE7B,OAAO,SAAS,uBAAuB,CAAC,QAA2B;QACjE,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChC,MAAM,QAAQ,GAAI,QAAqE;aACpF,cAAc,CAAC;QAElB,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,MAAM,OAAO,GAAG,kBAAkB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACrD,IAAI,OAAO,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC;QAErC,OAAO;YACL,OAAO,EAAE,KAAK;YACd,OAAO,EAAE;gBACP,GAAG,IAAI,CAAC,OAAO;gBACf,sCAAsC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,SAAS,GAAG;aACxF;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
export { runFlywheelGenerations } from './run.js';
|
|
2
2
|
export type { FlywheelConfig, FlywheelResult } from './run.js';
|
|
3
3
|
export { meetsPromotionRule, gateFingerprint } from './gate.js';
|
|
4
|
+
export { sequentialEvidence, withSequentialEvidence } from './sequential.js';
|
|
5
|
+
export type { PairedOutcome, SequentialConfig, SequentialVerdict } from './sequential.js';
|
|
4
6
|
export { makeSigner, verifyReceipt, canon } from './receipts.js';
|
|
5
7
|
export { InMemoryLineageStore, computeLiftCurve, liftPoint } from './lineage.js';
|
|
6
8
|
export { verifyReplayBundle } from './replay.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAClD,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE/D,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAChE,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACjE,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE/D,YAAY,EACV,MAAM,EACN,YAAY,EACZ,iBAAiB,EACjB,KAAK,EACL,iBAAiB,EACjB,iBAAiB,EACjB,aAAa,EACb,KAAK,EACL,YAAY,EACZ,WAAW,EACX,QAAQ,EACR,SAAS,EACT,gBAAgB,EAChB,MAAM,EACN,aAAa,EACb,YAAY,EACZ,SAAS,EACT,SAAS,EACT,YAAY,EACZ,oBAAoB,EACpB,WAAW,GACZ,MAAM,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAClD,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAE/D,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAC7E,YAAY,EAAE,aAAa,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAC1F,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACjE,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE/D,YAAY,EACV,MAAM,EACN,YAAY,EACZ,iBAAiB,EACjB,KAAK,EACL,iBAAiB,EACjB,iBAAiB,EACjB,aAAa,EACb,KAAK,EACL,YAAY,EACZ,WAAW,EACX,QAAQ,EACR,SAAS,EACT,gBAAgB,EAChB,MAAM,EACN,aAAa,EACb,YAAY,EACZ,SAAS,EACT,SAAS,EACT,YAAY,EACZ,oBAAoB,EACpB,WAAW,GACZ,MAAM,YAAY,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
// evaluator, gate, holdouts, and cost/security rules; get the same auditable, replayable improvement loop.
|
|
7
7
|
export { runFlywheelGenerations } from './run.js';
|
|
8
8
|
export { meetsPromotionRule, gateFingerprint } from './gate.js';
|
|
9
|
+
export { sequentialEvidence, withSequentialEvidence } from './sequential.js';
|
|
9
10
|
export { makeSigner, verifyReceipt, canon } from './receipts.js';
|
|
10
11
|
export { InMemoryLineageStore, computeLiftCurve, liftPoint } from './lineage.js';
|
|
11
12
|
export { verifyReplayBundle } from './replay.js';
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAClF,uEAAuE;AACvE,EAAE;AACF,uGAAuG;AACvG,yGAAyG;AACzG,2GAA2G;AAC3G,OAAO,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAChE,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACjE,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAClF,uEAAuE;AACvE,EAAE;AACF,uGAAuG;AACvG,yGAAyG;AACzG,2GAA2G;AAC3G,OAAO,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AAGlD,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAE7E,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACjE,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACjF,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEjD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC"}
|
package/dist/run.d.ts
CHANGED
|
@@ -20,6 +20,13 @@ export interface FlywheelConfig {
|
|
|
20
20
|
};
|
|
21
21
|
/** Caller-supplied ISO/label per generation (determinism; no clock in the engine). */
|
|
22
22
|
now?: (generation: number) => string;
|
|
23
|
+
/** Reuse evaluator results for (policy, suite) pairs already scored this run. The proposer can
|
|
24
|
+
* legitimately re-propose a policy a previous generation already tried (small lever domains,
|
|
25
|
+
* cycling proposers); each evaluation is typically a full benchmark run, so replays are the
|
|
26
|
+
* engine's dominant avoidable cost. OFF by default: only enable for evaluators that are
|
|
27
|
+
* deterministic (or whose first measurement you are happy to reuse) â a stochastic evaluator's
|
|
28
|
+
* cached score freezes one sample of the noise. */
|
|
29
|
+
cacheEvaluations?: boolean;
|
|
23
30
|
/** Stamped on the replay bundle â 'SYNTHETIC' | 'LIVE' | âĶ. NEVER a benchmark name. */
|
|
24
31
|
dataSource?: string;
|
|
25
32
|
/** Optional per-generation checkpoint hook â called at the END of each generation with a fully
|
package/dist/run.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EACV,MAAM,EAAgB,QAAQ,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,EAChE,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,YAAY,EAC/E,oBAAoB,EAAE,WAAW,EAClC,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,cAAc;IAC7B,4EAA4E;IAC5E,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,SAAS,CAAC;IACrB,uGAAuG;IACvG,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,OAAO,EAAE,YAAY,CAAC;IACtB,qGAAqG;IACrG,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,sFAAsF;IACtF,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,MAAM,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,MAAM,CAAA;KAAE,CAAC;IAChD,sFAAsF;IACtF,GAAG,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,CAAC;IACrC,uFAAuF;IACvF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;gGAG4F;IAC5F,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,oBAAoB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpE;;;8FAG0F;IAC1F,UAAU,CAAC,EAAE,WAAW,CAAC;IACzB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,SAAS,CAAC;IACrB,2CAA2C;IAC3C,UAAU,EAAE,aAAa,EAAE,CAAC;IAC5B,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,EAAE,YAAY,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,4FAA4F;IAC5F,gBAAgB,EAAE,OAAO,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,wBAAsB,sBAAsB,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EACV,MAAM,EAAgB,QAAQ,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,EAChE,YAAY,EAAE,WAAW,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,YAAY,EAC/E,oBAAoB,EAAE,WAAW,EAClC,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,cAAc;IAC7B,4EAA4E;IAC5E,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,QAAQ,CAAC;IACnB,SAAS,EAAE,SAAS,CAAC;IACrB,uGAAuG;IACvG,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,OAAO,EAAE,YAAY,CAAC;IACtB,qGAAqG;IACrG,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,sFAAsF;IACtF,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,MAAM,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,MAAM,CAAA;KAAE,CAAC;IAChD,sFAAsF;IACtF,GAAG,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,CAAC;IACrC;;;;;wDAKoD;IACpD,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,uFAAuF;IACvF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;gGAG4F;IAC5F,YAAY,CAAC,EAAE,CAAC,IAAI,EAAE,oBAAoB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpE;;;8FAG0F;IAC1F,UAAU,CAAC,EAAE,WAAW,CAAC;IACzB,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,SAAS,CAAC;IACrB,2CAA2C;IAC3C,UAAU,EAAE,aAAa,EAAE,CAAC;IAC5B,OAAO,EAAE,YAAY,CAAC;IACtB,YAAY,EAAE,YAAY,CAAC;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,4FAA4F;IAC5F,gBAAgB,EAAE,OAAO,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,wBAAsB,sBAAsB,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAsJzF"}
|
package/dist/run.js
CHANGED
|
@@ -11,7 +11,22 @@ export async function runFlywheelGenerations(cfg) {
|
|
|
11
11
|
const store = cfg.lineageStore ?? new InMemoryLineageStore();
|
|
12
12
|
const now = cfg.now ?? ((g) => `gen-${g}`);
|
|
13
13
|
const rootId = cfg.rootId ?? 'root';
|
|
14
|
-
|
|
14
|
+
// Optional per-run evaluation memo (see FlywheelConfig.cacheEvaluations). Keyed by suite id +
|
|
15
|
+
// key-sorted policy, so key order in the Policy object never splits the cache. Returns a copy so
|
|
16
|
+
// a caller mutating a Score cannot poison later hits.
|
|
17
|
+
const evalCache = cfg.cacheEvaluations ? new Map() : null;
|
|
18
|
+
const evaluate = async (p, suite) => {
|
|
19
|
+
if (!evalCache)
|
|
20
|
+
return cfg.evaluator(p, suite);
|
|
21
|
+
const key = `${suite.id}\u0000${JSON.stringify(Object.keys(p).sort().map((k) => [k, p[k]]))}`;
|
|
22
|
+
const hit = evalCache.get(key);
|
|
23
|
+
if (hit)
|
|
24
|
+
return { ...hit };
|
|
25
|
+
const s = await cfg.evaluator(p, suite);
|
|
26
|
+
evalCache.set(key, s);
|
|
27
|
+
return { ...s };
|
|
28
|
+
};
|
|
29
|
+
const anchorOf = async (p) => cfg.anchor ? (await evaluate(p, cfg.anchor)).primary : null;
|
|
15
30
|
let rootScore;
|
|
16
31
|
let rootAnchor;
|
|
17
32
|
let parentId;
|
|
@@ -36,7 +51,7 @@ export async function runFlywheelGenerations(cfg) {
|
|
|
36
51
|
}
|
|
37
52
|
else {
|
|
38
53
|
// ââ gen-0: the immutable root. Evaluate it once (the baseline) + its anchor (the frozen bar). ââ
|
|
39
|
-
rootScore = await
|
|
54
|
+
rootScore = await evaluate(cfg.rootPolicy, cfg.holdout);
|
|
40
55
|
rootAnchor = await anchorOf(cfg.rootPolicy);
|
|
41
56
|
await store.append({
|
|
42
57
|
id: rootId, generation: 0, parents: [], mutation: null, primaryDelta: 0, anchorScore: rootAnchor,
|
|
@@ -76,7 +91,7 @@ export async function runFlywheelGenerations(cfg) {
|
|
|
76
91
|
for (const target of targets) {
|
|
77
92
|
const proposed = await cfg.proposer(base, target);
|
|
78
93
|
const candPolicy = { ...policy, [target]: proposed };
|
|
79
|
-
const candScore = await
|
|
94
|
+
const candScore = await evaluate(candPolicy, cfg.holdout);
|
|
80
95
|
const decision = rule({ baseline: score, candidate: candScore });
|
|
81
96
|
cands.push({ target, policy: candPolicy, score: candScore, reasons: decision.reasons, promote: decision.promote });
|
|
82
97
|
}
|
package/dist/run.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAAA,wGAAwG;AACxG,qGAAqG;AACrG,wGAAwG;AACxG,uGAAuG;AACvG,sGAAsG;AACtG,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACtE,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../src/run.ts"],"names":[],"mappings":"AAAA,wGAAwG;AACxG,qGAAqG;AACrG,wGAAwG;AACxG,uGAAuG;AACvG,sGAAsG;AACtG,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACtE,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AA4DhE,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAAC,GAAmB;IAC9D,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,IAAI,kBAAkB,CAAC;IACrD,MAAM,OAAO,GAAG,GAAG,CAAC,eAAe,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACnE,MAAM,KAAK,GAAG,GAAG,CAAC,YAAY,IAAI,IAAI,oBAAoB,EAAE,CAAC;IAC7D,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC;IAEpC,8FAA8F;IAC9F,iGAAiG;IACjG,sDAAsD;IACtD,MAAM,SAAS,GAAG,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,GAAG,EAAiB,CAAC,CAAC,CAAC,IAAI,CAAC;IACzE,MAAM,QAAQ,GAAc,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE;QAC7C,IAAI,CAAC,SAAS;YAAE,OAAO,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAC/C,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,EAAE,SAAS,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,GAAG;YAAE,OAAO,EAAE,GAAG,GAAG,EAAE,CAAC;QAC3B,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACxC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACtB,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC;IAClB,CAAC,CAAC;IAEF,MAAM,QAAQ,GAAG,KAAK,EAAE,CAAS,EAA0B,EAAE,CAC3D,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IAE9D,IAAI,SAAgB,CAAC;IACrB,IAAI,UAAyB,CAAC;IAC9B,IAAI,QAAgB,CAAC;IACrB,IAAI,MAAc,CAAC;IACnB,IAAI,KAAY,CAAC;IACjB,MAAM,UAAU,GAAoB,EAAE,CAAC;IACvC,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;QACnB,qGAAqG;QACrG,MAAM,CAAC,GAAG,GAAG,CAAC,UAAU,CAAC;QACzB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY;YAAE,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACtD,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,+BAA+B;QACvG,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;QAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC;QACnD,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC;QAAC,MAAM,GAAG,EAAE,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC;QAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;QACjE,cAAc,GAAG,CAAC,CAAC,cAAc,CAAC;QAAC,QAAQ,GAAG,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC;IACrE,CAAC;SAAM,CAAC;QACN,kGAAkG;QAClG,SAAS,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;QACxD,UAAU,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAC5C,MAAM,KAAK,CAAC,MAAM,CAAC;YACjB,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,WAAW,EAAE,UAAU;YAChG,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;SACjH,CAAC,CAAC;QACH,QAAQ,GAAG,MAAM,CAAC;QAClB,MAAM,GAAG,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC;QAC/B,KAAK,GAAG,SAAS,CAAC;IACpB,CAAC;IAED,gGAAgG;IAChG,iGAAiG;IACjG,qGAAqG;IACrG,MAAM,WAAW,GAAG,CAAC,QAAyB,EAAE,SAAiB,EAAgB,EAAE;QACjF,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC;QAClE,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QACpE,MAAM,gBAAgB,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,UAAU,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,QAAQ,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC;QAClJ,OAAO;YACL,WAAW,EAAE,GAAG,CAAC,UAAU,IAAI,aAAa;YAC5C,OAAO,EAAE,MAAM;YACf,KAAK,EAAE,QAAQ;YACf,WAAW,EAAE,UAAU;YACvB,UAAU,EAAE,gBAAgB,CAAC,QAAQ,EAAE,SAAS,CAAC,OAAO,CAAC;YACzD,gBAAgB,EAAE,eAAe,CAAC,IAAI,CAAC;YACvC,qBAAqB,EAAE,SAAS;YAChC,6BAA6B,EAAE,gBAAgB;YAC/C,iBAAiB,EAAE,gBAAgB,IAAI,CAAC;YACxC,UAAU,EAAE,SAAS;SACtB,CAAC;IACJ,CAAC,CAAC;IAEF,KAAK,IAAI,GAAG,GAAG,QAAQ,EAAE,GAAG,IAAI,GAAG,CAAC,cAAc,EAAE,GAAG,EAAE,EAAE,CAAC;QAC1D,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK;YAAE,MAAM;QAChE,cAAc,GAAG,GAAG,CAAC;QAErB,kFAAkF;QAClF,MAAM,IAAI,GAAiB,EAAE,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;QAC1F,MAAM,KAAK,GAAiG,EAAE,CAAC;QAC/G,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAClD,MAAM,UAAU,GAAW,EAAE,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC;YAC7D,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;YAC1D,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;YACjE,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;QACrH,CAAC;QAED,4FAA4F;QAC5F,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACpG,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QACrC,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACnE,MAAM,cAAc,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,KAAK,IAAI,IAAI,CAAC,YAAY,IAAI,CAAC,QAAQ,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAC3G,gGAAgG;QAChG,MAAM,cAAc,GAAG,MAAM,IAAI,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;QAEhE,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACtB,MAAM,QAAQ,GAAG,CAAC,KAAK,cAAc,CAAC;YACtC,MAAM,EAAE,GAAG,GAAG,QAAQ,KAAK,CAAC,CAAC,MAAM,OAAO,GAAG,EAAE,CAAC;YAChD,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;YACrD,MAAM,MAAM,GAAkB;gBAC5B,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,QAAQ,CAAC;gBACxC,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,MAAM,EAAE,EAAE;gBAC5D,YAAY;gBACZ,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI;gBACzE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU;gBAC3C,cAAc,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO;gBAClG,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,EAAE,YAAY,EAAE,CAAC;gBAChI,SAAS,EAAE,GAAG,CAAC,GAAG,CAAC;gBACnB,aAAa,EAAE,KAAK,EAAQ,uDAAuD;gBACnF,cAAc,EAAE,CAAC,CAAC,KAAK;aACxB,CAAC;YACF,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC3B,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1B,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YAAC,QAAQ,GAAG,GAAG,QAAQ,KAAK,cAAc,CAAC,MAAM,OAAO,GAAG,EAAE,CAAC;YAAC,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC;YAAC,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC;QAAC,CAAC;QAEnJ,mGAAmG;QACnG,mGAAmG;QACnG,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;YACrB,IAAI,CAAC;gBACH,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBACtD,MAAM,IAAI,GAAyB;oBACjC,UAAU,EAAE,GAAG;oBACf,cAAc;oBACd,aAAa,EAAE,WAAW,CAAC,YAAY,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;oBAClD,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;oBAC1C,WAAW,EAAE;wBACX,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK;wBAC9C,cAAc,EAAE,GAAG;wBACnB,YAAY,EAAE,MAAM,KAAK,CAAC,IAAI,EAAE,EAAE,wDAAwD;qBAC3F;iBACF,CAAC;gBACF,MAAM,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC/B,CAAC;YAAC,MAAM,CAAC,CAAC,kEAAkE,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC;IACjE,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC;IAEjE,OAAO;QACL,SAAS,EAAE,YAAY,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY;QAC5E,cAAc;QACd,gBAAgB,EAAE,YAAY,CAAC,iBAAiB,EAAE,WAAW,EAAE,MAAM;KACtE,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import type { PromotionRule } from './types.js';
|
|
2
|
+
/** Per-item paired outcome: did candidate and baseline each succeed? */
|
|
3
|
+
export interface PairedOutcome {
|
|
4
|
+
/** Suite item id. Pairing is by item â comparing unpaired sets is a different, weaker test. */
|
|
5
|
+
itemId: string;
|
|
6
|
+
candidateWon: boolean;
|
|
7
|
+
baselineWon: boolean;
|
|
8
|
+
}
|
|
9
|
+
export interface SequentialConfig {
|
|
10
|
+
/**
|
|
11
|
+
* Type-I error bound. Reject the null only when the e-value reaches 1/alpha.
|
|
12
|
+
* Default 0.05 => threshold 20.
|
|
13
|
+
*/
|
|
14
|
+
alpha?: number;
|
|
15
|
+
/**
|
|
16
|
+
* Betting fraction in (0, 1). Higher detects large effects sooner but is
|
|
17
|
+
* slower on small ones. 0.5 is the Kelly-ish default and needs no tuning.
|
|
18
|
+
*/
|
|
19
|
+
lambda?: number;
|
|
20
|
+
}
|
|
21
|
+
export interface SequentialVerdict {
|
|
22
|
+
/** True when accumulated evidence crosses 1/alpha. */
|
|
23
|
+
significant: boolean;
|
|
24
|
+
/** The e-value. Interpretable directly: 20 means "20:1 against the null". */
|
|
25
|
+
eValue: number;
|
|
26
|
+
threshold: number;
|
|
27
|
+
/** Items where the two arms disagreed â the only ones carrying information. */
|
|
28
|
+
informativePairs: number;
|
|
29
|
+
totalPairs: number;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Accumulate paired outcomes into an anytime-valid e-value.
|
|
33
|
+
*
|
|
34
|
+
* Only discordant pairs move the e-value: if both arms win or both lose, that
|
|
35
|
+
* item tells you nothing about which is better (this is the McNemar insight,
|
|
36
|
+
* carried over to the sequential setting).
|
|
37
|
+
*/
|
|
38
|
+
export declare function sequentialEvidence(outcomes: PairedOutcome[], config?: SequentialConfig): SequentialVerdict;
|
|
39
|
+
/**
|
|
40
|
+
* Compose a frozen gate with a sequential-evidence requirement.
|
|
41
|
+
*
|
|
42
|
+
* The returned rule is still a plain `PromotionRule`, so it fingerprints and
|
|
43
|
+
* freezes exactly like the default one. A candidate must satisfy BOTH: every
|
|
44
|
+
* clause of `baseRule`, and evidence strong enough to survive having been
|
|
45
|
+
* looked at repeatedly.
|
|
46
|
+
*
|
|
47
|
+
* Paired outcomes are read from `evidence.pairedOutcomes` when present. When
|
|
48
|
+
* absent the rule degrades to `baseRule` alone rather than silently blocking
|
|
49
|
+
* every promotion â a caller that has not wired up per-item outcomes yet should
|
|
50
|
+
* get the old behavior, not a permanently closed gate.
|
|
51
|
+
*/
|
|
52
|
+
export declare function withSequentialEvidence(baseRule: PromotionRule, config?: SequentialConfig): PromotionRule;
|
|
53
|
+
//# sourceMappingURL=sequential.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sequential.d.ts","sourceRoot":"","sources":["../src/sequential.ts"],"names":[],"mappings":"AAkCA,OAAO,KAAK,EAAwC,aAAa,EAAE,MAAM,YAAY,CAAC;AAEtF,wEAAwE;AACxE,MAAM,WAAW,aAAa;IAC5B,+FAA+F;IAC/F,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,OAAO,CAAC;IACtB,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,iBAAiB;IAChC,sDAAsD;IACtD,WAAW,EAAE,OAAO,CAAC;IACrB,6EAA6E;IAC7E,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,+EAA+E;IAC/E,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;CACpB;AAKD;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,aAAa,EAAE,EACzB,MAAM,GAAE,gBAAqB,GAC5B,iBAAiB,CA0BnB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,aAAa,EACvB,MAAM,GAAE,gBAAqB,GAC5B,aAAa,CAmBf"}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
// @metaharness/flywheel â anytime-valid sequential testing for promotion gates.
|
|
2
|
+
//
|
|
3
|
+
// WHY THIS EXISTS
|
|
4
|
+
//
|
|
5
|
+
// `meetsPromotionRule` is a single-shot conjunctive comparison, and it is the
|
|
6
|
+
// right shape: frozen, fingerprintable, every clause load-bearing. But a
|
|
7
|
+
// flywheel run evaluates MANY generations against the SAME holdout, and that is
|
|
8
|
+
// uncontrolled multiple testing. Published measurements of greedy
|
|
9
|
+
// "accept-if-the-score-improved" acceptance put the false-commit rate at
|
|
10
|
+
// 30-42%, and found 13-21 spurious modifications made even when NO true gains
|
|
11
|
+
// existed â degrading one agent by 4.9 points while every individual decision
|
|
12
|
+
// looked locally justified.
|
|
13
|
+
//
|
|
14
|
+
// The conjunctive gate plus a frozen anchor already mitigates this with
|
|
15
|
+
// multiple hurdles, which is real. It is not, however, anytime-valid: nothing
|
|
16
|
+
// in it accounts for how many times you have looked.
|
|
17
|
+
//
|
|
18
|
+
// This module adds that missing property WITHOUT touching the default gate,
|
|
19
|
+
// whose stability is itself the product. It composes: wrap any PromotionRule,
|
|
20
|
+
// and a candidate must clear both the frozen clauses AND accumulated evidence.
|
|
21
|
+
//
|
|
22
|
+
// METHOD
|
|
23
|
+
//
|
|
24
|
+
// Testing-by-betting / e-processes. An e-value is a non-negative random
|
|
25
|
+
// variable with expectation <= 1 under the null hypothesis ("this candidate is
|
|
26
|
+
// no better than baseline"). By Ville's inequality, P(sup_t E_t >= 1/alpha)
|
|
27
|
+
// <= alpha, so you may stop and reject at ANY time â no pre-registered sample
|
|
28
|
+
// size, no alpha spending schedule, and no penalty for peeking. That is exactly
|
|
29
|
+
// the property a flywheel needs, because it peeks by construction.
|
|
30
|
+
//
|
|
31
|
+
// The bet here is deliberately simple and assumption-light: per paired item,
|
|
32
|
+
// a candidate win against a baseline loss multiplies the e-value up, the
|
|
33
|
+
// reverse multiplies it down, and ties leave it unchanged.
|
|
34
|
+
const DEFAULT_ALPHA = 0.05;
|
|
35
|
+
const DEFAULT_LAMBDA = 0.5;
|
|
36
|
+
/**
|
|
37
|
+
* Accumulate paired outcomes into an anytime-valid e-value.
|
|
38
|
+
*
|
|
39
|
+
* Only discordant pairs move the e-value: if both arms win or both lose, that
|
|
40
|
+
* item tells you nothing about which is better (this is the McNemar insight,
|
|
41
|
+
* carried over to the sequential setting).
|
|
42
|
+
*/
|
|
43
|
+
export function sequentialEvidence(outcomes, config = {}) {
|
|
44
|
+
const alpha = config.alpha ?? DEFAULT_ALPHA;
|
|
45
|
+
const lambda = config.lambda ?? DEFAULT_LAMBDA;
|
|
46
|
+
if (!(alpha > 0 && alpha < 1))
|
|
47
|
+
throw new RangeError('alpha must be in (0, 1)');
|
|
48
|
+
if (!(lambda > 0 && lambda < 1))
|
|
49
|
+
throw new RangeError('lambda must be in (0, 1)');
|
|
50
|
+
const threshold = 1 / alpha;
|
|
51
|
+
let eValue = 1;
|
|
52
|
+
let informativePairs = 0;
|
|
53
|
+
for (const o of outcomes) {
|
|
54
|
+
if (o.candidateWon === o.baselineWon)
|
|
55
|
+
continue; // concordant: no information
|
|
56
|
+
informativePairs++;
|
|
57
|
+
// Under the null, a discordant pair favors either arm with probability 1/2,
|
|
58
|
+
// so E[multiplier] = 1 and the process is a non-negative martingale.
|
|
59
|
+
eValue *= o.candidateWon ? 1 + lambda : 1 - lambda;
|
|
60
|
+
}
|
|
61
|
+
return {
|
|
62
|
+
significant: eValue >= threshold,
|
|
63
|
+
eValue,
|
|
64
|
+
threshold,
|
|
65
|
+
informativePairs,
|
|
66
|
+
totalPairs: outcomes.length,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Compose a frozen gate with a sequential-evidence requirement.
|
|
71
|
+
*
|
|
72
|
+
* The returned rule is still a plain `PromotionRule`, so it fingerprints and
|
|
73
|
+
* freezes exactly like the default one. A candidate must satisfy BOTH: every
|
|
74
|
+
* clause of `baseRule`, and evidence strong enough to survive having been
|
|
75
|
+
* looked at repeatedly.
|
|
76
|
+
*
|
|
77
|
+
* Paired outcomes are read from `evidence.pairedOutcomes` when present. When
|
|
78
|
+
* absent the rule degrades to `baseRule` alone rather than silently blocking
|
|
79
|
+
* every promotion â a caller that has not wired up per-item outcomes yet should
|
|
80
|
+
* get the old behavior, not a permanently closed gate.
|
|
81
|
+
*/
|
|
82
|
+
export function withSequentialEvidence(baseRule, config = {}) {
|
|
83
|
+
return function sequentialPromotionRule(evidence) {
|
|
84
|
+
const base = baseRule(evidence);
|
|
85
|
+
const outcomes = evidence
|
|
86
|
+
.pairedOutcomes;
|
|
87
|
+
if (!outcomes)
|
|
88
|
+
return base;
|
|
89
|
+
const verdict = sequentialEvidence(outcomes, config);
|
|
90
|
+
if (verdict.significant)
|
|
91
|
+
return base;
|
|
92
|
+
return {
|
|
93
|
+
promote: false,
|
|
94
|
+
reasons: [
|
|
95
|
+
...base.reasons,
|
|
96
|
+
`insufficient_sequential_evidence(e=${verdict.eValue.toFixed(2)}<${verdict.threshold})`,
|
|
97
|
+
],
|
|
98
|
+
};
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
//# sourceMappingURL=sequential.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sequential.js","sourceRoot":"","sources":["../src/sequential.ts"],"names":[],"mappings":"AAAA,gFAAgF;AAChF,EAAE;AACF,kBAAkB;AAClB,EAAE;AACF,8EAA8E;AAC9E,yEAAyE;AACzE,gFAAgF;AAChF,kEAAkE;AAClE,yEAAyE;AACzE,8EAA8E;AAC9E,8EAA8E;AAC9E,4BAA4B;AAC5B,EAAE;AACF,wEAAwE;AACxE,8EAA8E;AAC9E,qDAAqD;AACrD,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,+EAA+E;AAC/E,EAAE;AACF,SAAS;AACT,EAAE;AACF,wEAAwE;AACxE,+EAA+E;AAC/E,4EAA4E;AAC5E,8EAA8E;AAC9E,gFAAgF;AAChF,mEAAmE;AACnE,EAAE;AACF,6EAA6E;AAC7E,yEAAyE;AACzE,2DAA2D;AAoC3D,MAAM,aAAa,GAAG,IAAI,CAAC;AAC3B,MAAM,cAAc,GAAG,GAAG,CAAC;AAE3B;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAChC,QAAyB,EACzB,SAA2B,EAAE;IAE7B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,aAAa,CAAC;IAC5C,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,cAAc,CAAC;IAE/C,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,yBAAyB,CAAC,CAAC;IAC/E,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,0BAA0B,CAAC,CAAC;IAElF,MAAM,SAAS,GAAG,CAAC,GAAG,KAAK,CAAC;IAC5B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,gBAAgB,GAAG,CAAC,CAAC;IAEzB,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,YAAY,KAAK,CAAC,CAAC,WAAW;YAAE,SAAS,CAAC,6BAA6B;QAC7E,gBAAgB,EAAE,CAAC;QACnB,4EAA4E;QAC5E,qEAAqE;QACrE,MAAM,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;IACrD,CAAC;IAED,OAAO;QACL,WAAW,EAAE,MAAM,IAAI,SAAS;QAChC,MAAM;QACN,SAAS;QACT,gBAAgB;QAChB,UAAU,EAAE,QAAQ,CAAC,MAAM;KAC5B,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,sBAAsB,CACpC,QAAuB,EACvB,SAA2B,EAAE;IAE7B,OAAO,SAAS,uBAAuB,CAAC,QAA2B;QACjE,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChC,MAAM,QAAQ,GAAI,QAAqE;aACpF,cAAc,CAAC;QAElB,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,MAAM,OAAO,GAAG,kBAAkB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACrD,IAAI,OAAO,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC;QAErC,OAAO;YACL,OAAO,EAAE,KAAK;YACd,OAAO,EAAE;gBACP,GAAG,IAAI,CAAC,OAAO;gBACf,sCAAsC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,SAAS,GAAG;aACxF;SACF,CAAC;IACJ,CAAC,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@metaharness/flywheel",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.9",
|
|
4
4
|
"description": "A verifiable self-improvement loop for agent harnesses. Freeze the model. Evolve the harness. Promote only what proves lift.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/cjs/index.js",
|