@claude-flow/cli 3.34.0 → 3.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/.proven-config-version +1 -0
- package/.claude/helpers/.helpers-version +1 -1
- package/.claude/helpers/helpers.manifest.json +2 -2
- package/.claude/helpers/statusline.cjs +0 -0
- package/.claude/proven-config.json +42 -0
- package/catalog-manifest.json +2 -2
- package/dist/src/commands/daemon.js +12 -7
- package/dist/src/commands/doctor.js +72 -1
- package/dist/src/commands/metaharness.js +37 -2
- package/dist/src/log-filters.d.ts +3 -3
- package/dist/src/mcp-tools/metaharness-tools.js +35 -2
- package/dist/src/memory/memory-bridge.d.ts +25 -0
- package/dist/src/memory/memory-bridge.js +61 -8
- package/dist/src/memory/memory-initializer.d.ts +9 -3
- package/dist/src/memory/memory-initializer.js +344 -277
- package/dist/src/services/daemon-autostart.d.ts +31 -3
- package/dist/src/services/daemon-autostart.js +45 -3
- package/dist/src/services/distill-oracle.d.ts +1 -1
- package/dist/src/services/distill-oracle.js +2 -2
- package/dist/src/services/evolve-proof.d.ts +40 -1
- package/dist/src/services/evolve-proof.js +76 -14
- package/dist/src/services/flywheel-receipt.d.ts +15 -0
- package/dist/src/services/flywheel-receipt.js +22 -0
- package/dist/src/services/flywheel-sequential-evidence.d.ts +102 -0
- package/dist/src/services/flywheel-sequential-evidence.js +148 -0
- package/dist/src/services/flywheel-transaction.d.ts +71 -0
- package/dist/src/services/flywheel-transaction.js +121 -0
- package/dist/src/services/harness-flywheel-generations.d.ts +14 -0
- package/dist/src/services/harness-flywheel-generations.js +76 -4
- package/dist/src/services/harness-flywheel.d.ts +13 -0
- package/dist/src/services/harness-flywheel.js +31 -1
- package/node_modules/@claude-flow/codex/dist/cli.js +0 -0
- package/node_modules/@claude-flow/plugin-agent-federation/dist/bin.js +0 -0
- package/package.json +4 -9
- package/plugins/ruflo-metaharness/scripts/smoke.sh +4 -2
|
@@ -9,6 +9,30 @@ export declare function isDaemonAlive(projectRoot: string): boolean;
|
|
|
9
9
|
* read-only command in any Claude project to spawn a detached daemon (#2852).
|
|
10
10
|
*/
|
|
11
11
|
export declare function isRufloProject(projectRoot: string): boolean;
|
|
12
|
+
/**
|
|
13
|
+
* #2877: Normalize a working directory to the project root that owns the
|
|
14
|
+
* daemon lock/PID key.
|
|
15
|
+
*
|
|
16
|
+
* The lock (`<root>/.claude-flow/daemon.lock`) and PID file
|
|
17
|
+
* (`<root>/.claude-flow/daemon.pid`) were keyed to the raw cwd, so invoking
|
|
18
|
+
* the CLI from `/proj` and from `/proj/packages/foo` produced two different
|
|
19
|
+
* keys for one logical project — bypassing the atomic-lockfile dedup added
|
|
20
|
+
* for #2407/#2484 entirely (each cwd races against a different lock file)
|
|
21
|
+
* and leaving two daemons supervising the same tree.
|
|
22
|
+
*
|
|
23
|
+
* Resolution walks up from `startDir` and returns the NEAREST enclosing
|
|
24
|
+
* directory carrying a durable Ruflo marker (`isRufloProject`). Nearest-wins
|
|
25
|
+
* is what keeps a monorepo's independently-initialized sub-project on its own
|
|
26
|
+
* daemon: it matches its own marker before the walk ever reaches the parent.
|
|
27
|
+
*
|
|
28
|
+
* A `.git` directory is a hard stop — the repository boundary. Without it the
|
|
29
|
+
* walk could escape into an unrelated ancestor (a Ruflo project living at
|
|
30
|
+
* `$HOME`, say) and hand every repo below it the same daemon.
|
|
31
|
+
*
|
|
32
|
+
* Returns the resolved `startDir` unchanged when no marker is found, so a
|
|
33
|
+
* non-Ruflo directory still fails `isRufloProject` and declines autostart.
|
|
34
|
+
*/
|
|
35
|
+
export declare function resolveDaemonProjectRoot(startDir: string): string;
|
|
12
36
|
export interface EnsureResult {
|
|
13
37
|
started: boolean;
|
|
14
38
|
reason?: string;
|
|
@@ -16,10 +40,14 @@ export interface EnsureResult {
|
|
|
16
40
|
/** Spawn `daemon start` detached, reusing all its lock/TTL machinery. Injectable for tests. */
|
|
17
41
|
export type SpawnDaemonFn = (projectRoot: string) => void;
|
|
18
42
|
/**
|
|
19
|
-
* Ensure a daemon is running for `
|
|
20
|
-
* is already alive. Best-effort; never throws.
|
|
43
|
+
* Ensure a daemon is running for the project enclosing `startDir`. No-op when
|
|
44
|
+
* disabled or when one is already alive. Best-effort; never throws.
|
|
45
|
+
*
|
|
46
|
+
* #2877: `startDir` is normalized to the owning project root first, so a call
|
|
47
|
+
* from a subdirectory reads the same config, pidfile, and lock as a call from
|
|
48
|
+
* the root and cannot spawn a second daemon for the same project.
|
|
21
49
|
*/
|
|
22
|
-
export declare function ensureDaemonRunning(
|
|
50
|
+
export declare function ensureDaemonRunning(startDir: string, opts?: {
|
|
23
51
|
spawnFn?: SpawnDaemonFn;
|
|
24
52
|
isAlive?: (root: string) => boolean;
|
|
25
53
|
}): EnsureResult;
|
|
@@ -105,6 +105,43 @@ export function isRufloProject(projectRoot) {
|
|
|
105
105
|
catch { /* absent/malformed/non-Ruflo MCP config */ }
|
|
106
106
|
return false;
|
|
107
107
|
}
|
|
108
|
+
/**
|
|
109
|
+
* #2877: Normalize a working directory to the project root that owns the
|
|
110
|
+
* daemon lock/PID key.
|
|
111
|
+
*
|
|
112
|
+
* The lock (`<root>/.claude-flow/daemon.lock`) and PID file
|
|
113
|
+
* (`<root>/.claude-flow/daemon.pid`) were keyed to the raw cwd, so invoking
|
|
114
|
+
* the CLI from `/proj` and from `/proj/packages/foo` produced two different
|
|
115
|
+
* keys for one logical project — bypassing the atomic-lockfile dedup added
|
|
116
|
+
* for #2407/#2484 entirely (each cwd races against a different lock file)
|
|
117
|
+
* and leaving two daemons supervising the same tree.
|
|
118
|
+
*
|
|
119
|
+
* Resolution walks up from `startDir` and returns the NEAREST enclosing
|
|
120
|
+
* directory carrying a durable Ruflo marker (`isRufloProject`). Nearest-wins
|
|
121
|
+
* is what keeps a monorepo's independently-initialized sub-project on its own
|
|
122
|
+
* daemon: it matches its own marker before the walk ever reaches the parent.
|
|
123
|
+
*
|
|
124
|
+
* A `.git` directory is a hard stop — the repository boundary. Without it the
|
|
125
|
+
* walk could escape into an unrelated ancestor (a Ruflo project living at
|
|
126
|
+
* `$HOME`, say) and hand every repo below it the same daemon.
|
|
127
|
+
*
|
|
128
|
+
* Returns the resolved `startDir` unchanged when no marker is found, so a
|
|
129
|
+
* non-Ruflo directory still fails `isRufloProject` and declines autostart.
|
|
130
|
+
*/
|
|
131
|
+
export function resolveDaemonProjectRoot(startDir) {
|
|
132
|
+
const start = path.resolve(startDir);
|
|
133
|
+
let dir = start;
|
|
134
|
+
for (;;) {
|
|
135
|
+
if (isRufloProject(dir))
|
|
136
|
+
return dir;
|
|
137
|
+
if (fs.existsSync(path.join(dir, '.git')))
|
|
138
|
+
return start;
|
|
139
|
+
const parent = path.dirname(dir);
|
|
140
|
+
if (parent === dir)
|
|
141
|
+
return start;
|
|
142
|
+
dir = parent;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
108
145
|
const defaultSpawn = (projectRoot) => {
|
|
109
146
|
const cliBin = process.argv[1]; // the running bin/cli.js
|
|
110
147
|
const child = spawn(process.execPath, [cliBin, 'daemon', 'start', '--quiet'], {
|
|
@@ -116,11 +153,16 @@ const defaultSpawn = (projectRoot) => {
|
|
|
116
153
|
child.unref();
|
|
117
154
|
};
|
|
118
155
|
/**
|
|
119
|
-
* Ensure a daemon is running for `
|
|
120
|
-
* is already alive. Best-effort; never throws.
|
|
156
|
+
* Ensure a daemon is running for the project enclosing `startDir`. No-op when
|
|
157
|
+
* disabled or when one is already alive. Best-effort; never throws.
|
|
158
|
+
*
|
|
159
|
+
* #2877: `startDir` is normalized to the owning project root first, so a call
|
|
160
|
+
* from a subdirectory reads the same config, pidfile, and lock as a call from
|
|
161
|
+
* the root and cannot spawn a second daemon for the same project.
|
|
121
162
|
*/
|
|
122
|
-
export function ensureDaemonRunning(
|
|
163
|
+
export function ensureDaemonRunning(startDir, opts = {}) {
|
|
123
164
|
try {
|
|
165
|
+
const projectRoot = resolveDaemonProjectRoot(startDir);
|
|
124
166
|
if (autostartDisabled(projectRoot))
|
|
125
167
|
return { started: false, reason: 'disabled (RUFLO_DAEMON_AUTOSTART=0 or project config)' };
|
|
126
168
|
if (!isRufloProject(projectRoot)) {
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
*/
|
|
37
37
|
import { type VerifyTaskKind } from '../ruvector/output-verifier.js';
|
|
38
38
|
import { FableHarness, type ReflectItem, type ReflectResult } from './fable-harness.js';
|
|
39
|
-
export declare const MH_DARWIN_PIN = "0.8.
|
|
39
|
+
export declare const MH_DARWIN_PIN = "0.8.3";
|
|
40
40
|
export type ResolvedProvenance = 'oracle:test-exec' | 'judge:fable' | 'proxy:structural';
|
|
41
41
|
/** SWE-bench-shaped / bench-suite-mapped test spec that Tier 1 can execute. */
|
|
42
42
|
export interface TestSpec {
|
|
@@ -43,8 +43,8 @@ import { FableHarness, } from './fable-harness.js';
|
|
|
43
43
|
// behavior mid-run (the #142 pin-drift failure mode). Pin it, and let
|
|
44
44
|
// scripts/check-metaharness-pins.mjs watch this constant for drift. Kept in
|
|
45
45
|
// lock-step with the optionalDependencies pin in package.json and the plugin
|
|
46
|
-
// darwin cache (
|
|
47
|
-
export const MH_DARWIN_PIN = '0.8.
|
|
46
|
+
// darwin cache (versioned by the plugin's own `~0.8.0` pin in _darwin.mjs).
|
|
47
|
+
export const MH_DARWIN_PIN = '0.8.3';
|
|
48
48
|
// ── Public API ───────────────────────────────────────────────────────────
|
|
49
49
|
/**
|
|
50
50
|
* Label each trajectory with `resolved` + honest provenance, trying the tiers
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type PromotionVerdict, type AcceptResult } from './harness-benchmark.js';
|
|
2
|
+
import { SEQUENTIAL_EVIDENCE_VERSION } from './flywheel-sequential-evidence.js';
|
|
2
3
|
import { type ProvenConfigManifest } from '../config/proven-config.js';
|
|
3
4
|
/**
|
|
4
5
|
* The promotion rule is versioned so a receipt pins exactly which semantics
|
|
@@ -6,8 +7,16 @@ import { type ProvenConfigManifest } from '../config/proven-config.js';
|
|
|
6
7
|
* term: the per-held-out-task deltas must have a positive one-sided 95% bootstrap
|
|
7
8
|
* lower bound, so a small-N mean gain can't ride on noise. The canary term is a
|
|
8
9
|
* SEPARATE deployment-safety signal (a distinct slice), not held-out dominance.
|
|
10
|
+
*
|
|
11
|
+
* v2+seq (ADR-381 §3) adds a third conjunct: an anytime-valid sequential-
|
|
12
|
+
* evidence e-process over the holdout's paired per-task scores, judged at the
|
|
13
|
+
* alpha allocated to this bundle's position in the lineage's adaptive test
|
|
14
|
+
* stream (alpha_k = alphaTotal · 6/(π²k²)). The rule version is pinned PER
|
|
15
|
+
* BUNDLE, so a lineage may contain both v1 and v2 bundles and each replays
|
|
16
|
+
* under its own recorded semantics.
|
|
9
17
|
*/
|
|
10
18
|
export declare const PROMOTION_RULE_VERSION = "accept/v1+sig";
|
|
19
|
+
export declare const PROMOTION_RULE_VERSION_V2 = "accept/v2+seq";
|
|
11
20
|
export declare const PROOF_LABEL = "single-round proof-of-mechanism";
|
|
12
21
|
export declare const NOT_CLAIMS: readonly ["not flywheel proof", "not compounding learning", "not production learning"];
|
|
13
22
|
export interface HoldoutTask {
|
|
@@ -62,9 +71,26 @@ export interface RegressionRecord {
|
|
|
62
71
|
candidateManifestHash: string;
|
|
63
72
|
ancestor: string | null;
|
|
64
73
|
mutationClass: string;
|
|
65
|
-
failureCause: 'holdout' | 'security' | 'drift' | 'replay' | 'governance' | 'canary' | 'significance';
|
|
74
|
+
failureCause: 'holdout' | 'security' | 'drift' | 'replay' | 'governance' | 'canary' | 'significance' | 'sequential';
|
|
66
75
|
failedTerms: string[];
|
|
67
76
|
}
|
|
77
|
+
/**
|
|
78
|
+
* The v2 sequential-evidence term, embedded so `verifyReceiptBundle` can
|
|
79
|
+
* replay it from the bundle's own holdout: the recorded testIndex/alphaTotal/
|
|
80
|
+
* lambda fully determine the threshold, and the e-value recomputes from the
|
|
81
|
+
* embedded per-task scores.
|
|
82
|
+
*/
|
|
83
|
+
export interface SequentialEvidenceRecord {
|
|
84
|
+
version: typeof SEQUENTIAL_EVIDENCE_VERSION;
|
|
85
|
+
testIndex: number;
|
|
86
|
+
alphaTotal: number;
|
|
87
|
+
lambda: number;
|
|
88
|
+
alphaAllocated: number;
|
|
89
|
+
eValue: number;
|
|
90
|
+
threshold: number;
|
|
91
|
+
informativePairs: number;
|
|
92
|
+
significant: boolean;
|
|
93
|
+
}
|
|
68
94
|
export interface EvolveReceiptBundle {
|
|
69
95
|
label: typeof PROOF_LABEL;
|
|
70
96
|
disclaimers: typeof NOT_CLAIMS;
|
|
@@ -80,6 +106,8 @@ export interface EvolveReceiptBundle {
|
|
|
80
106
|
version: string;
|
|
81
107
|
result: boolean;
|
|
82
108
|
};
|
|
109
|
+
/** Present iff the bundle was decided under accept/v2+seq (ADR-381 §3). */
|
|
110
|
+
sequentialEvidence?: SequentialEvidenceRecord;
|
|
83
111
|
decisionReceipt: DecisionReceipt;
|
|
84
112
|
shadow: ShadowRegistration | null;
|
|
85
113
|
costReceipt: CostReceipt;
|
|
@@ -115,6 +143,16 @@ export interface AssembleOpts {
|
|
|
115
143
|
humanEvalHash?: string;
|
|
116
144
|
layer?: string;
|
|
117
145
|
corpus?: string;
|
|
146
|
+
/**
|
|
147
|
+
* ADR-381 §3 — supply to decide the bundle under accept/v2+seq: the bundle's
|
|
148
|
+
* 1-based position in the lineage's adaptive test stream, plus optional
|
|
149
|
+
* alpha/lambda overrides. Omit for legacy v1+sig semantics.
|
|
150
|
+
*/
|
|
151
|
+
sequential?: {
|
|
152
|
+
testIndex: number;
|
|
153
|
+
alphaTotal?: number;
|
|
154
|
+
lambda?: number;
|
|
155
|
+
};
|
|
118
156
|
}
|
|
119
157
|
/**
|
|
120
158
|
* Assemble a receipt bundle from a holdout + configs. This is the SHARED core:
|
|
@@ -144,6 +182,7 @@ export declare function runRealEvolveRound(opts: {
|
|
|
144
182
|
humanRelevanceDelta?: number;
|
|
145
183
|
humanEvalHash?: string;
|
|
146
184
|
corpus: string;
|
|
185
|
+
sequential?: AssembleOpts['sequential'];
|
|
147
186
|
}): EvolveReceiptBundle;
|
|
148
187
|
/**
|
|
149
188
|
* Run ONE deterministic synthetic evolve round and produce the receipt bundle.
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
import { createHash } from 'node:crypto';
|
|
22
22
|
import { accept } from './harness-benchmark.js';
|
|
23
23
|
import { bootstrapDeltaCILow } from './harness-improvement-ledger.js';
|
|
24
|
+
import { DEFAULT_ALPHA_TOTAL, DEFAULT_LAMBDA, SEQUENTIAL_EVIDENCE_VERSION, sequentialEvidenceVerdict, } from './flywheel-sequential-evidence.js';
|
|
24
25
|
import { canonicalManifestBytes } from '../config/proven-config.js';
|
|
25
26
|
/**
|
|
26
27
|
* The promotion rule is versioned so a receipt pins exactly which semantics
|
|
@@ -28,8 +29,16 @@ import { canonicalManifestBytes } from '../config/proven-config.js';
|
|
|
28
29
|
* term: the per-held-out-task deltas must have a positive one-sided 95% bootstrap
|
|
29
30
|
* lower bound, so a small-N mean gain can't ride on noise. The canary term is a
|
|
30
31
|
* SEPARATE deployment-safety signal (a distinct slice), not held-out dominance.
|
|
32
|
+
*
|
|
33
|
+
* v2+seq (ADR-381 §3) adds a third conjunct: an anytime-valid sequential-
|
|
34
|
+
* evidence e-process over the holdout's paired per-task scores, judged at the
|
|
35
|
+
* alpha allocated to this bundle's position in the lineage's adaptive test
|
|
36
|
+
* stream (alpha_k = alphaTotal · 6/(π²k²)). The rule version is pinned PER
|
|
37
|
+
* BUNDLE, so a lineage may contain both v1 and v2 bundles and each replays
|
|
38
|
+
* under its own recorded semantics.
|
|
31
39
|
*/
|
|
32
40
|
export const PROMOTION_RULE_VERSION = 'accept/v1+sig';
|
|
41
|
+
export const PROMOTION_RULE_VERSION_V2 = 'accept/v2+seq';
|
|
33
42
|
export const PROOF_LABEL = 'single-round proof-of-mechanism';
|
|
34
43
|
export const NOT_CLAIMS = ['not flywheel proof', 'not compounding learning', 'not production learning'];
|
|
35
44
|
function sha256(s) { return 'sha256:' + createHash('sha256').update(s).digest('hex'); }
|
|
@@ -73,22 +82,46 @@ export function assembleBundle(baseline, candidate, holdout, o) {
|
|
|
73
82
|
// the per-task deltas must be > 0.
|
|
74
83
|
const deltaCILow = bootstrapDeltaCILow(holdout.map((h) => h.candidateScore - h.baselineScore));
|
|
75
84
|
const significant = deltaCILow > 0;
|
|
85
|
+
// v2 sequential term (ADR-381 §3): anytime-valid e-process over the paired
|
|
86
|
+
// per-task scores, at this bundle's allocated share of the family-wise alpha.
|
|
87
|
+
let sequentialEvidence;
|
|
88
|
+
if (o.sequential) {
|
|
89
|
+
const alphaTotal = o.sequential.alphaTotal ?? DEFAULT_ALPHA_TOTAL;
|
|
90
|
+
const lambda = o.sequential.lambda ?? DEFAULT_LAMBDA;
|
|
91
|
+
const verdict = sequentialEvidenceVerdict(holdout.map((h) => ({ taskId: h.taskId, baselineScore: h.baselineScore, candidateScore: h.candidateScore })), o.sequential.testIndex, { alphaTotal, lambda });
|
|
92
|
+
sequentialEvidence = {
|
|
93
|
+
version: SEQUENTIAL_EVIDENCE_VERSION,
|
|
94
|
+
testIndex: verdict.testIndex,
|
|
95
|
+
alphaTotal,
|
|
96
|
+
lambda,
|
|
97
|
+
alphaAllocated: verdict.alphaAllocated,
|
|
98
|
+
eValue: verdict.eValue,
|
|
99
|
+
threshold: verdict.threshold,
|
|
100
|
+
informativePairs: verdict.informativePairs,
|
|
101
|
+
significant: verdict.significant,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
const ruleVersion = sequentialEvidence ? PROMOTION_RULE_VERSION_V2 : PROMOTION_RULE_VERSION;
|
|
76
105
|
const verdictInputs = {
|
|
77
106
|
heldOutScore: candidateHeldOut, baselineHeldOutScore: baselineHeldOut,
|
|
78
107
|
redblue: o.redblue ?? 'PASS', drift: o.drift ?? 0, driftThreshold: 0.05,
|
|
79
108
|
replayDeterministic: true, receiptCoverage: 1, canaryRollbackRate, baselineRollbackRate: 0,
|
|
80
109
|
};
|
|
81
110
|
const result = accept(verdictInputs);
|
|
82
|
-
const promoted = result.accept && significant;
|
|
111
|
+
const promoted = result.accept && significant && (sequentialEvidence?.significant ?? true);
|
|
83
112
|
const baselineManifest = mkManifest(baseline, o.layer, o.corpus);
|
|
84
113
|
const candidateManifest = mkManifest(candidate, o.layer, o.corpus);
|
|
85
114
|
const baselineManifestHash = manifestHash(baselineManifest);
|
|
86
115
|
const candidateManifestHash = manifestHash(candidateManifest);
|
|
87
116
|
const inputHoldoutHash = sha256(canon(holdout));
|
|
88
|
-
const failed = [
|
|
117
|
+
const failed = [
|
|
118
|
+
...result.failed,
|
|
119
|
+
...(!significant ? ['significant'] : []),
|
|
120
|
+
...(sequentialEvidence && !sequentialEvidence.significant ? ['sequential_evidence'] : []),
|
|
121
|
+
];
|
|
89
122
|
const decisionReceipt = {
|
|
90
|
-
promotionRuleVersion:
|
|
91
|
-
reason: promoted ? `promoted (all ${
|
|
123
|
+
promotionRuleVersion: ruleVersion, verdictInputs, result, significant, deltaCILow, promoted,
|
|
124
|
+
reason: promoted ? `promoted (all ${ruleVersion} terms held)` : `rejected — ${failed.join(', ')}`,
|
|
92
125
|
};
|
|
93
126
|
const shadow = promoted ? {
|
|
94
127
|
registrationId: sha256(`${candidateManifestHash}|gen${o.generation}|shadow`).replace('sha256:', 'shadow:'),
|
|
@@ -99,12 +132,16 @@ export function assembleBundle(baseline, candidate, holdout, o) {
|
|
|
99
132
|
const promotion = promoted ? { parentManifestHash: o.parent, candidateManifestHash, mutationClass, mutationSummary, deltas, decisionReceipt } : null;
|
|
100
133
|
const regression = promoted ? null : {
|
|
101
134
|
candidateManifestHash, ancestor: o.parent ?? baselineManifestHash, mutationClass,
|
|
102
|
-
failureCause:
|
|
135
|
+
failureCause: result.accept
|
|
136
|
+
? (!significant ? 'significance' : 'sequential')
|
|
137
|
+
: (FAILURE_CAUSE[result.failed[0]] ?? 'holdout'),
|
|
138
|
+
failedTerms: failed,
|
|
103
139
|
};
|
|
104
140
|
return {
|
|
105
141
|
label: PROOF_LABEL, disclaimers: NOT_CLAIMS, generation: o.generation, parent: o.parent, branch: o.branch, kind: o.kind, createdAt: o.now,
|
|
106
142
|
inputHoldoutHash, baselineManifestHash, candidateManifestHash,
|
|
107
|
-
meetsPromotionRule: { version:
|
|
143
|
+
meetsPromotionRule: { version: ruleVersion, result: promoted },
|
|
144
|
+
...(sequentialEvidence ? { sequentialEvidence } : {}),
|
|
108
145
|
decisionReceipt, shadow,
|
|
109
146
|
costReceipt: { usd: 0, llmCalls: 0, tier: o.cost.tier, notes: o.cost.notes },
|
|
110
147
|
mutationClass, mutationSummary, deltas, humanEvalHash: o.humanEvalHash, promotion, regression,
|
|
@@ -125,6 +162,7 @@ export function runRealEvolveRound(opts) {
|
|
|
125
162
|
redblue: opts.redblue, drift: opts.drift, canaryRollbackRate: opts.canaryRollbackRate,
|
|
126
163
|
humanRelevanceDelta: opts.humanRelevanceDelta, humanEvalHash: opts.humanEvalHash,
|
|
127
164
|
layer: 'real/retrieval', corpus: opts.corpus,
|
|
165
|
+
sequential: opts.sequential,
|
|
128
166
|
});
|
|
129
167
|
}
|
|
130
168
|
/**
|
|
@@ -182,16 +220,36 @@ export function verifyReceiptBundle(bundle) {
|
|
|
182
220
|
const canaryRollbackRate = bundle.decisionReceipt.verdictInputs.canaryRollbackRate;
|
|
183
221
|
const deltaCILow = bootstrapDeltaCILow(bundle.holdout.map((h) => h.candidateScore - h.baselineScore));
|
|
184
222
|
const significant = deltaCILow > 0;
|
|
185
|
-
// Re-run the SAME versioned rule
|
|
186
|
-
|
|
187
|
-
|
|
223
|
+
// Re-run the SAME versioned rule the bundle was decided under (pinned per
|
|
224
|
+
// bundle — a lineage may legitimately mix v1 and v2, ADR-381 §3). The rule
|
|
225
|
+
// version and the presence of the sequential record must agree.
|
|
226
|
+
const expectedVersion = bundle.sequentialEvidence ? PROMOTION_RULE_VERSION_V2 : PROMOTION_RULE_VERSION;
|
|
227
|
+
const ruleVersionMatches = bundle.decisionReceipt.promotionRuleVersion === expectedVersion
|
|
228
|
+
&& bundle.meetsPromotionRule.version === expectedVersion;
|
|
188
229
|
if (!ruleVersionMatches)
|
|
189
|
-
mismatches.push(`promotion rule version != ${
|
|
230
|
+
mismatches.push(`promotion rule version != ${expectedVersion} (or sequential record inconsistent with version)`);
|
|
231
|
+
// v2: replay the sequential e-process from the embedded holdout using the
|
|
232
|
+
// recorded testIndex/alphaTotal/lambda — the recorded verdict must recompute.
|
|
233
|
+
let sequentialOk = true;
|
|
234
|
+
if (bundle.sequentialEvidence) {
|
|
235
|
+
const s = bundle.sequentialEvidence;
|
|
236
|
+
try {
|
|
237
|
+
const replayed = sequentialEvidenceVerdict(bundle.holdout.map((h) => ({ taskId: h.taskId, baselineScore: h.baselineScore, candidateScore: h.candidateScore })), s.testIndex, { alphaTotal: s.alphaTotal, lambda: s.lambda });
|
|
238
|
+
sequentialOk = Math.abs(replayed.eValue - s.eValue) < 1e-9
|
|
239
|
+
&& replayed.significant === s.significant
|
|
240
|
+
&& replayed.informativePairs === s.informativePairs;
|
|
241
|
+
}
|
|
242
|
+
catch {
|
|
243
|
+
sequentialOk = false;
|
|
244
|
+
}
|
|
245
|
+
if (!sequentialOk)
|
|
246
|
+
mismatches.push('recorded sequential-evidence verdict does not recompute from the embedded holdout');
|
|
247
|
+
}
|
|
190
248
|
const decision = accept({
|
|
191
249
|
...bundle.decisionReceipt.verdictInputs,
|
|
192
250
|
heldOutScore: candidateHeldOut, baselineHeldOutScore: baselineHeldOut,
|
|
193
251
|
});
|
|
194
|
-
const promotedRecomputed = decision.accept && significant;
|
|
252
|
+
const promotedRecomputed = decision.accept && significant && (bundle.sequentialEvidence ? bundle.sequentialEvidence.significant && sequentialOk : true);
|
|
195
253
|
const decisionMatches = promotedRecomputed === bundle.decisionReceipt.promoted && promotedRecomputed === bundle.meetsPromotionRule.result;
|
|
196
254
|
if (!decisionMatches)
|
|
197
255
|
mismatches.push('recomputed decision != recorded decision');
|
|
@@ -208,10 +266,14 @@ export function verifyReceiptBundle(bundle) {
|
|
|
208
266
|
if (!causalConsistent)
|
|
209
267
|
mismatches.push('causal record inconsistent with the decision (promotion/regression/delta)');
|
|
210
268
|
const valid = hashChecks.inputHoldout && hashChecks.baselineManifest && hashChecks.candidateManifest
|
|
211
|
-
&& ruleVersionMatches && decisionMatches && noAutoServe && causalConsistent;
|
|
269
|
+
&& ruleVersionMatches && decisionMatches && noAutoServe && causalConsistent && sequentialOk;
|
|
212
270
|
const why = promotedRecomputed
|
|
213
|
-
? `PASS under ${
|
|
214
|
-
: `FAIL under ${
|
|
271
|
+
? `PASS under ${expectedVersion}: held_out ${candidateHeldOut.toFixed(4)} > ${baselineHeldOut.toFixed(4)} (Δ CI-low ${deltaCILow.toFixed(4)} > 0, significant), canary rollback ${canaryRollbackRate} ≤ 0, all terms held`
|
|
272
|
+
: `FAIL under ${expectedVersion}: ${[
|
|
273
|
+
...decision.failed,
|
|
274
|
+
...(!significant ? ['significant'] : []),
|
|
275
|
+
...(bundle.sequentialEvidence && !(bundle.sequentialEvidence.significant && sequentialOk) ? ['sequential_evidence'] : []),
|
|
276
|
+
].join(', ')}`;
|
|
215
277
|
return {
|
|
216
278
|
valid, hashChecks,
|
|
217
279
|
recomputed: { baselineHeldOut, candidateHeldOut, canaryRollbackRate, decision },
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type PairedTaskOutcome } from './flywheel-sequential-evidence.js';
|
|
1
2
|
export declare const RECEIPT_SCHEMA = "ruflo.flywheel-receipt/v1";
|
|
2
3
|
export declare const RECEIPT_DOMAIN = "ruflo/flywheel-receipt/v1";
|
|
3
4
|
export declare const GENESIS_LEDGER_HEAD: string;
|
|
@@ -66,6 +67,18 @@ export interface FlywheelReceiptPayload {
|
|
|
66
67
|
baselineScore: string;
|
|
67
68
|
candidateScore: string;
|
|
68
69
|
heldOutDeltas: string[];
|
|
70
|
+
/**
|
|
71
|
+
* Task-level paired outcomes behind heldOutDeltas — same order, same length,
|
|
72
|
+
* per-task delta reproducible from the two scores. Optional in the payload
|
|
73
|
+
* so pre-existing receipts still verify byte-identically, but the promotion
|
|
74
|
+
* authority (flywheel-transaction.ts) REQUIRES it by default: aggregate-only
|
|
75
|
+
* evidence is refused rather than silently falling back to the weaker gate.
|
|
76
|
+
*/
|
|
77
|
+
pairedOutcomes?: Array<{
|
|
78
|
+
taskId: string;
|
|
79
|
+
baselineScore: string;
|
|
80
|
+
candidateScore: string;
|
|
81
|
+
}>;
|
|
69
82
|
statistics: PromotionStatistics;
|
|
70
83
|
gates: Record<string, boolean>;
|
|
71
84
|
resourceEvidence: ResourceEvidence;
|
|
@@ -97,6 +110,8 @@ export interface CreateReceiptInput {
|
|
|
97
110
|
baselineScore: number;
|
|
98
111
|
candidateScore: number;
|
|
99
112
|
heldOutDeltas: number[];
|
|
113
|
+
/** Task-level paired outcomes behind heldOutDeltas (same order). */
|
|
114
|
+
pairedOutcomes?: PairedTaskOutcome[];
|
|
100
115
|
frozenAnchorRegression: number;
|
|
101
116
|
gates: Record<string, boolean>;
|
|
102
117
|
resourceEvidence?: Partial<ResourceEvidence>;
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* tracked separately by flywheel-transaction.ts.
|
|
8
8
|
*/
|
|
9
9
|
import { createHash, randomBytes, sign as edSign, verify as edVerify, } from 'node:crypto';
|
|
10
|
+
import { checkPairedOutcomesConsistency } from './flywheel-sequential-evidence.js';
|
|
10
11
|
export const RECEIPT_SCHEMA = 'ruflo.flywheel-receipt/v1';
|
|
11
12
|
export const RECEIPT_DOMAIN = 'ruflo/flywheel-receipt/v1';
|
|
12
13
|
export const GENESIS_LEDGER_HEAD = `sha256:${'0'.repeat(64)}`;
|
|
@@ -215,6 +216,15 @@ export function createFlywheelReceipt(input) {
|
|
|
215
216
|
baselineScore: decimal(input.baselineScore),
|
|
216
217
|
candidateScore: decimal(input.candidateScore),
|
|
217
218
|
heldOutDeltas: input.heldOutDeltas.map((v) => decimal(v)),
|
|
219
|
+
...(input.pairedOutcomes
|
|
220
|
+
? {
|
|
221
|
+
pairedOutcomes: input.pairedOutcomes.map((o) => ({
|
|
222
|
+
taskId: o.taskId,
|
|
223
|
+
baselineScore: decimal(o.baselineScore),
|
|
224
|
+
candidateScore: decimal(o.candidateScore),
|
|
225
|
+
})),
|
|
226
|
+
}
|
|
227
|
+
: {}),
|
|
218
228
|
statistics,
|
|
219
229
|
gates: input.gates,
|
|
220
230
|
resourceEvidence: {
|
|
@@ -287,6 +297,18 @@ export function verifyFlywheelReceipt(receipt, trustedPublicKeys) {
|
|
|
287
297
|
: 'rejected';
|
|
288
298
|
if (receipt.payload.decision !== recomputedDecision)
|
|
289
299
|
errors.push('receipt decision does not recompute');
|
|
300
|
+
if (receipt.payload.pairedOutcomes) {
|
|
301
|
+
// Paired outcomes must reproduce the aggregate they claim to back.
|
|
302
|
+
// Tolerance covers the decimal(scale 12) round-trip of both scores.
|
|
303
|
+
const paired = receipt.payload.pairedOutcomes.map((o) => ({
|
|
304
|
+
taskId: o.taskId,
|
|
305
|
+
baselineScore: Number(o.baselineScore),
|
|
306
|
+
candidateScore: Number(o.candidateScore),
|
|
307
|
+
}));
|
|
308
|
+
const check = checkPairedOutcomesConsistency(paired, receipt.payload.heldOutDeltas.map(Number), 1e-9);
|
|
309
|
+
if (!check.ok)
|
|
310
|
+
errors.push(`paired outcomes inconsistent: ${check.reasons.join('; ')}`);
|
|
311
|
+
}
|
|
290
312
|
if (!receipt.signature) {
|
|
291
313
|
errors.push('receipt is unsigned');
|
|
292
314
|
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sequential promotion evidence — family-wise error control for an ADAPTIVE
|
|
3
|
+
* candidate stream (report item 2 / upstream @metaharness/flywheel 0.1.10
|
|
4
|
+
* `withSequentialEvidence` interop, implemented in-house so ruflo keeps the
|
|
5
|
+
* property with every MetaHarness package removed).
|
|
6
|
+
*
|
|
7
|
+
* THE GAP THIS CLOSES: every promotion gate ruflo runs (accept/v1+sig, the
|
|
8
|
+
* ruflo.flywheel-gate/v1 bootstrap) spends a FRESH alpha per candidate. A
|
|
9
|
+
* flywheel proposes candidates adaptively — each new candidate is chosen after
|
|
10
|
+
* looking at the last one's scores — so per-candidate alpha does not bound the
|
|
11
|
+
* probability that ANY promotion in the stream is false. Published
|
|
12
|
+
* measurements of greedy accept-if-improved loops put the false-commit rate at
|
|
13
|
+
* 30-42% under exactly this regime.
|
|
14
|
+
*
|
|
15
|
+
* TWO COMPOSED MECHANISMS, both required by the promotion authority:
|
|
16
|
+
*
|
|
17
|
+
* 1. Anytime-valid e-process per candidate (testing-by-betting). Per paired
|
|
18
|
+
* task, a discordant pair multiplies the e-value by (1+lambda) when the
|
|
19
|
+
* candidate wins and (1-lambda) when the baseline wins; concordant pairs
|
|
20
|
+
* carry no information (McNemar). Under the null the e-value is a
|
|
21
|
+
* non-negative martingale with expectation 1, so by Ville's inequality
|
|
22
|
+
* P(e ever reaches 1/alpha) <= alpha — no penalty for peeking mid-stream.
|
|
23
|
+
*
|
|
24
|
+
* 2. Alpha allocation ACROSS candidates. Candidate k in the lineage's test
|
|
25
|
+
* stream must clear 1/alpha_k where alpha_k = alphaTotal * 6/(pi^2 * k^2),
|
|
26
|
+
* so sum(alpha_k) = alphaTotal for arbitrarily many adaptively-chosen
|
|
27
|
+
* candidates. The allocation index is persisted per receipt in the
|
|
28
|
+
* transaction state — looking spends alpha whether or not the candidate
|
|
29
|
+
* promotes, and retrying the same receipt reuses its index (no double
|
|
30
|
+
* spend, no index shopping).
|
|
31
|
+
*
|
|
32
|
+
* Family-wise guarantee: P(any false promotion, ever, in the stream)
|
|
33
|
+
* <= sum_k alpha_k = alphaTotal. The acceptance test for this module is the
|
|
34
|
+
* 1,000-null-simulation in flywheel-sequential-evidence.test.ts.
|
|
35
|
+
*
|
|
36
|
+
* Pure, $0, deterministic. Never throws on well-typed input.
|
|
37
|
+
*/
|
|
38
|
+
export declare const SEQUENTIAL_EVIDENCE_VERSION = "ruflo.sequential-evidence/v1";
|
|
39
|
+
export declare const DEFAULT_ALPHA_TOTAL = 0.05;
|
|
40
|
+
export declare const DEFAULT_LAMBDA = 0.5;
|
|
41
|
+
/** Score tie-band: |candidate - baseline| <= epsilon is a concordant (uninformative) pair. */
|
|
42
|
+
export declare const DEFAULT_SCORE_EPSILON = 1e-9;
|
|
43
|
+
/** Task-level paired outcome — the evidence unit receipts must now carry. */
|
|
44
|
+
export interface PairedTaskOutcome {
|
|
45
|
+
taskId: string;
|
|
46
|
+
baselineScore: number;
|
|
47
|
+
candidateScore: number;
|
|
48
|
+
}
|
|
49
|
+
export interface SequentialEvidenceConfig {
|
|
50
|
+
/** Total family-wise type-I budget across the WHOLE candidate stream. */
|
|
51
|
+
alphaTotal?: number;
|
|
52
|
+
/** Betting fraction in (0,1); 0.5 needs no tuning. */
|
|
53
|
+
lambda?: number;
|
|
54
|
+
/** Tie band on score comparisons. */
|
|
55
|
+
epsilon?: number;
|
|
56
|
+
}
|
|
57
|
+
export interface SequentialEvidenceVerdict {
|
|
58
|
+
significant: boolean;
|
|
59
|
+
eValue: number;
|
|
60
|
+
threshold: number;
|
|
61
|
+
alphaAllocated: number;
|
|
62
|
+
testIndex: number;
|
|
63
|
+
informativePairs: number;
|
|
64
|
+
totalPairs: number;
|
|
65
|
+
version: typeof SEQUENTIAL_EVIDENCE_VERSION;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Alpha share for the k-th test in the stream: alphaTotal * 6/(pi^2 k^2).
|
|
69
|
+
* Chosen over 2^-k because it decays polynomially — test 10 still gets a
|
|
70
|
+
* workable ~0.6% of a 5% budget instead of ~0.005%.
|
|
71
|
+
*/
|
|
72
|
+
export declare function alphaForTest(testIndex: number, alphaTotal?: number): number;
|
|
73
|
+
/**
|
|
74
|
+
* Fold paired outcomes into an anytime-valid e-value and judge it against the
|
|
75
|
+
* k-th test's allocated alpha. Deterministic; order of outcomes does not
|
|
76
|
+
* change the final e-value (the product commutes).
|
|
77
|
+
*/
|
|
78
|
+
export declare function sequentialEvidenceVerdict(outcomes: PairedTaskOutcome[], testIndex: number, config?: SequentialEvidenceConfig): SequentialEvidenceVerdict;
|
|
79
|
+
/**
|
|
80
|
+
* Minimum number of INFORMATIVE (discordant) pairs a candidate must win —
|
|
81
|
+
* with zero losses — to clear the k-th test's threshold: the smallest n with
|
|
82
|
+
* (1+lambda)^n >= 1/alpha_k. The pre-flight power check (ADR-381 §4): an
|
|
83
|
+
* evaluation whose promotion holdout is smaller than this cannot promote even
|
|
84
|
+
* on a perfect sweep, so it should be refused BEFORE compute is spent and
|
|
85
|
+
* before a doomed receipt can be presented to the gate (spending alpha).
|
|
86
|
+
*/
|
|
87
|
+
export declare function minInformativePairsToClear(testIndex: number, config?: SequentialEvidenceConfig): number;
|
|
88
|
+
/** Family-wise budget left after `testsRun` allocated tests: alphaTotal - Σ alpha_k. */
|
|
89
|
+
export declare function remainingAlphaBudget(testsRun: number, alphaTotal?: number): number;
|
|
90
|
+
export interface PairedEvidenceCheck {
|
|
91
|
+
ok: boolean;
|
|
92
|
+
reasons: string[];
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Structural consistency between a receipt's paired outcomes and its
|
|
96
|
+
* aggregate heldOutDeltas: same length and order, unique non-empty task IDs,
|
|
97
|
+
* and each delta must equal candidateScore - baselineScore. This is what makes
|
|
98
|
+
* paired outcomes EVIDENCE rather than decoration — an aggregate that cannot
|
|
99
|
+
* be reproduced from its own per-task rows is refused.
|
|
100
|
+
*/
|
|
101
|
+
export declare function checkPairedOutcomesConsistency(pairedOutcomes: PairedTaskOutcome[], heldOutDeltas: number[], tolerance?: number): PairedEvidenceCheck;
|
|
102
|
+
//# sourceMappingURL=flywheel-sequential-evidence.d.ts.map
|