@dzhechkov/harness-core 0.3.143 → 0.3.144
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dz-manifest.json +36 -12
- package/dist/compounding.d.ts +22 -0
- package/dist/compounding.d.ts.map +1 -1
- package/dist/compounding.js +29 -10
- package/dist/compounding.js.map +1 -1
- package/dist/epoch-replay.d.ts +399 -0
- package/dist/epoch-replay.d.ts.map +1 -0
- package/dist/epoch-replay.js +695 -0
- package/dist/epoch-replay.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -0
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
- package/sbom.json +71 -11
- package/src/compounding.ts +45 -8
- package/src/epoch-replay.ts +955 -0
- package/src/index.ts +52 -0
|
@@ -0,0 +1,399 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `dz epoch-replay` — the executable cold-vs-warm EPOCH RUNNER (feature epoch-replay, scout idea #4).
|
|
3
|
+
*
|
|
4
|
+
* `dz compounding` answers READINESS ("N unique prompt events recorded — a replay can now be RUN").
|
|
5
|
+
* This module answers the RESULT: Epoch-0 (cold, no injected lessons) vs Epoch-1 (warm, the SAME
|
|
6
|
+
* instances plus exactly the lessons the apply leg injected), scored into a three-valued verdict
|
|
7
|
+
* whose positive branch requires two DISJOINT Wilson confidence intervals.
|
|
8
|
+
*
|
|
9
|
+
* ── The honesty boundary (ADR-002) ─────────────────────────────────────────────────────────────
|
|
10
|
+
* This runner ORCHESTRATES and SCORES. It NEVER calls a model. Real mode is a three-stage protocol
|
|
11
|
+
* over files:
|
|
12
|
+
* 1. `buildWorkOrder` — emits the instances + per-arm generation instructions + the
|
|
13
|
+
* PRE-REGISTERED blind A/B assignment (seeded, decided before any plan
|
|
14
|
+
* text exists) + an integrity `digest` over that pre-registered core.
|
|
15
|
+
* 2. `buildJudgePrompts`— renders the blind judge prompts from the filled plans. The judge-facing
|
|
16
|
+
* payload is `{id, prompt}` and NOTHING else.
|
|
17
|
+
* 3. `verifyWorkOrder` + `unblindJudgments` + `scoreEpochReplay` — check the order really is the
|
|
18
|
+
* pre-registered one (digest + seed-derived assignment), un-blind against
|
|
19
|
+
* it (never against a field the judge wrote), and compute the verdict.
|
|
20
|
+
* Every stage is pure and deterministic, so the protocol is testable with zero LLM dependency.
|
|
21
|
+
*
|
|
22
|
+
* ── What "blind" has to mean ───────────────────────────────────────────────────────────────────
|
|
23
|
+
* The first version shipped `warmIsA` INSIDE the judge artifact: the judge could read the answer
|
|
24
|
+
* key, so the blinding was theatre (Codex QE CRITICAL-1). `warmIsA` now exists only in the work
|
|
25
|
+
* order, which `--score` consumes and the judge never sees — and the artifact is byte-identical
|
|
26
|
+
* whichever way the assignment fell.
|
|
27
|
+
*
|
|
28
|
+
* ── The conformance firewall ───────────────────────────────────────────────────────────────────
|
|
29
|
+
* The warm arm's only delta is the lessons the apply leg ALREADY injects for that prompt. Gold
|
|
30
|
+
* answers, judge verdicts, and outcome labels never enter the warm context — feedback flows from
|
|
31
|
+
* SOLVE OUTCOMES ONLY. `buildWorkOrder` therefore reads instances, not results, and there is no
|
|
32
|
+
* code path from an `EpochOutcome` back into a work order.
|
|
33
|
+
*
|
|
34
|
+
* ── `--mock` ───────────────────────────────────────────────────────────────────────────────────
|
|
35
|
+
* A seeded synthetic outcome generator (reusing `mulberry32` — no second RNG in this repo) with a
|
|
36
|
+
* configurable TRUE effect, so the verdict math is exercised at $0 before any real data exists.
|
|
37
|
+
*
|
|
38
|
+
* Everything here is PURE: callers read/write files; this module only computes.
|
|
39
|
+
*/
|
|
40
|
+
import { replayableInstances, type ReplayInstance } from './compounding.js';
|
|
41
|
+
export { replayableInstances, type ReplayInstance };
|
|
42
|
+
/** 95% two-sided normal quantile. Named so a future 90%/99% run is a parameter, not a fork. */
|
|
43
|
+
export declare const WILSON_Z = 1.96;
|
|
44
|
+
/** Per-arm minimum. Shared with the darwin FDR discipline already pinned in compounding.ts. */
|
|
45
|
+
export declare const MIN_INSTANCES = 5;
|
|
46
|
+
/**
|
|
47
|
+
* Floor of DECISIVE pairs for the no-lift branch. Necessary, NOT sufficient: reaching it only makes
|
|
48
|
+
* the non-superiority test eligible — the test itself must still pass (see {@link NO_LIFT_MARGIN}).
|
|
49
|
+
*
|
|
50
|
+
* The first draft FALSIFIED on `warmWins <= coldWins` at this n, which made 6/12 vs 6/12 read as
|
|
51
|
+
* "refuted". That is indefensible: a tie at n=12 is UNDER-POWERED, not evidence of no effect
|
|
52
|
+
* (Codex QE HIGH-3).
|
|
53
|
+
*/
|
|
54
|
+
export declare const FALSIFY_NO_LIFT_MIN_N: number;
|
|
55
|
+
/**
|
|
56
|
+
* Pre-registered NON-SUPERIORITY margin, on the LIFT scale (see {@link liftInterval}). "No lift" is
|
|
57
|
+
* claimed only when the UPPER bound of the lift interval sits below this — i.e. the data EXCLUDE
|
|
58
|
+
* any lift worth having, rather than merely failing to show one.
|
|
59
|
+
*
|
|
60
|
+
* Consequence, stated plainly: at this margin the branch needs ~1200 decisive pairs. That is the
|
|
61
|
+
* honest price of an equivalence-style claim, and it is exactly why a 6/6 tie at n=12 is
|
|
62
|
+
* INCONCLUSIVE rather than FALSIFIED.
|
|
63
|
+
*/
|
|
64
|
+
export declare const NO_LIFT_MARGIN = 0.05;
|
|
65
|
+
/** A margin outside this range is REFUSED, never clamped: `--margin 99` must not buy FALSIFIED. */
|
|
66
|
+
export declare const MARGIN_MIN_EXCLUSIVE = 0;
|
|
67
|
+
export declare const MARGIN_MAX = 0.5;
|
|
68
|
+
/** True for a margin that may be pre-registered — finite and in `(0, 0.5]`. */
|
|
69
|
+
export declare function isValidMargin(value: unknown): value is number;
|
|
70
|
+
export interface WilsonInterval {
|
|
71
|
+
readonly k: number;
|
|
72
|
+
readonly n: number;
|
|
73
|
+
/** Point estimate k/n. */
|
|
74
|
+
readonly p: number;
|
|
75
|
+
readonly lower: number;
|
|
76
|
+
readonly upper: number;
|
|
77
|
+
readonly z: number;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Wilson score interval for a binomial proportion. Returns `null` — never a fabricated interval —
|
|
81
|
+
* for any input that is not a real (k, n) pair: n <= 0, non-integers, k out of [0, n], non-finite
|
|
82
|
+
* numbers, or a non-finite/non-positive z. A `null` interval can only ever produce INCONCLUSIVE.
|
|
83
|
+
*/
|
|
84
|
+
export declare function wilsonInterval(k: number, n: number, z?: number): WilsonInterval | null;
|
|
85
|
+
/** An interval on the LIFT scale: warm's advantage over cold among DECISIVE pairs, in `[-1, +1]`. */
|
|
86
|
+
export interface LiftInterval {
|
|
87
|
+
/** Point estimate `2·p̂ − 1`: `0` = a coin flip, `+1` = warm wins every decisive pair. */
|
|
88
|
+
readonly d: number;
|
|
89
|
+
readonly lower: number;
|
|
90
|
+
readonly upper: number;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Map the Wilson interval for `p̂ = P(warm wins | decisive)` onto the LIFT scale, `2p − 1`.
|
|
94
|
+
*
|
|
95
|
+
* WHY THE SCALE MATTERS (and why it is not cosmetic): `margin` is stated as "a lift worth having",
|
|
96
|
+
* which is what a reader reasons about, and it kept exactly the meaning it had under the previous
|
|
97
|
+
* (wrong) two-proportion model. On the raw `p̂` scale the equivalent threshold is `0.5 + margin/2`,
|
|
98
|
+
* NOT `0.5 + margin` — reading the margin on the `p̂` scale would silently DOUBLE the strictness of
|
|
99
|
+
* the non-superiority branch, i.e. make FALSIFIED easier. That is the anti-conservative direction,
|
|
100
|
+
* which is precisely the class of error the paired rewrite exists to remove.
|
|
101
|
+
*
|
|
102
|
+
* Worked check (the case that drove the rewrite): 500 warm / 500 cold over 1000 decisive pairs gives
|
|
103
|
+
* `p̂` CI `[0.4691, 0.5309]` → lift CI `[-0.0619, +0.0619]`. Upper `0.0619` exceeds the default
|
|
104
|
+
* margin `0.05`, so it reads INCONCLUSIVE. The discarded two-proportion Newcombe interval put the
|
|
105
|
+
* upper bound at `0.0437` and called the same data FALSIFIED.
|
|
106
|
+
*/
|
|
107
|
+
export declare function liftInterval(pWarm: WilsonInterval | null): LiftInterval | null;
|
|
108
|
+
export declare const WORK_ORDER_KIND = "dz-epoch-replay-work-order";
|
|
109
|
+
/**
|
|
110
|
+
* v3: the integrity `digest` (v2) plus the PRE-REGISTERED `margin` and `corpusFingerprint`, and an
|
|
111
|
+
* unambiguous JSON digest input. An older order cannot be verified under these rules, so it is
|
|
112
|
+
* refused rather than half-trusted.
|
|
113
|
+
*/
|
|
114
|
+
export declare const WORK_ORDER_VERSION = 3;
|
|
115
|
+
export interface WorkOrderItem {
|
|
116
|
+
readonly id: string;
|
|
117
|
+
readonly query: string;
|
|
118
|
+
readonly class: string | null;
|
|
119
|
+
/**
|
|
120
|
+
* PRE-REGISTERED blind assignment: does the WARM plan appear as "PLAN A"? Decided by the seeded
|
|
121
|
+
* PRNG before any plan text exists, and it is the ONLY authority for un-blinding.
|
|
122
|
+
*/
|
|
123
|
+
readonly warmIsA: boolean;
|
|
124
|
+
/** Epoch-0 arm: the prompt with NO injected lessons. */
|
|
125
|
+
readonly cold: {
|
|
126
|
+
readonly instruction: string;
|
|
127
|
+
readonly lessons: readonly string[];
|
|
128
|
+
};
|
|
129
|
+
/** Epoch-1 arm: the SAME prompt plus exactly what the apply leg injected. */
|
|
130
|
+
readonly warm: {
|
|
131
|
+
readonly instruction: string;
|
|
132
|
+
readonly lessons: readonly string[];
|
|
133
|
+
};
|
|
134
|
+
/** Filled by the generating agent — absent in a freshly emitted order. */
|
|
135
|
+
readonly coldPlan?: string;
|
|
136
|
+
readonly warmPlan?: string;
|
|
137
|
+
}
|
|
138
|
+
export interface WorkOrder {
|
|
139
|
+
readonly kind: typeof WORK_ORDER_KIND;
|
|
140
|
+
readonly version: typeof WORK_ORDER_VERSION;
|
|
141
|
+
readonly seed: number;
|
|
142
|
+
readonly generatedAt: string;
|
|
143
|
+
readonly wordMin: number;
|
|
144
|
+
readonly wordMax: number;
|
|
145
|
+
/** Human-readable pre-registration notes, written BEFORE the run. */
|
|
146
|
+
readonly protocol: readonly string[];
|
|
147
|
+
readonly items: readonly WorkOrderItem[];
|
|
148
|
+
/**
|
|
149
|
+
* The PRE-REGISTERED non-superiority margin, on the lift scale. It lives HERE, not on `--score`:
|
|
150
|
+
* a margin chosen after the counts are known is not a pre-registration, and `--margin 99` at
|
|
151
|
+
* scoring time would simply buy FALSIFIED (Codex QE HIGH-B).
|
|
152
|
+
*/
|
|
153
|
+
readonly margin: number;
|
|
154
|
+
/** sha256 over the ordered `[id, query]` corpus — lets a reviewer recognise the same corpus. */
|
|
155
|
+
readonly corpusFingerprint: string;
|
|
156
|
+
/** When this order was emitted. Recorded so a reviewer can ask for the original file. */
|
|
157
|
+
readonly emittedAt: string;
|
|
158
|
+
/**
|
|
159
|
+
* Integrity digest over the PRE-REGISTERED core (version, seed, margin, corpus fingerprint and
|
|
160
|
+
* every `[id, warmIsA]`). `--judge`/`--score` recompute it and refuse on mismatch: without this,
|
|
161
|
+
* a forged order — the right `kind`, a fabricated assignment — bought a SUPPORTED verdict for an
|
|
162
|
+
* experiment that never happened (Codex QE HIGH-2).
|
|
163
|
+
*
|
|
164
|
+
* NOT a cryptographic commitment — see {@link workOrderDigest} for the honest scope.
|
|
165
|
+
*/
|
|
166
|
+
readonly digest: string;
|
|
167
|
+
}
|
|
168
|
+
export interface WorkOrderOptions {
|
|
169
|
+
readonly seed?: number;
|
|
170
|
+
readonly nowTs?: string;
|
|
171
|
+
readonly wordMin?: number;
|
|
172
|
+
readonly wordMax?: number;
|
|
173
|
+
/** Cap the number of instances (0/absent = all). */
|
|
174
|
+
readonly limit?: number;
|
|
175
|
+
/**
|
|
176
|
+
* The PRE-REGISTERED non-superiority margin (lift scale), stored in the order and digest-covered.
|
|
177
|
+
* Must be in `(0, 0.5]`; anything else is REFUSED by {@link buildWorkOrder}, never clamped.
|
|
178
|
+
*/
|
|
179
|
+
readonly margin?: number;
|
|
180
|
+
}
|
|
181
|
+
export declare const DEFAULT_WORD_MIN = 80;
|
|
182
|
+
export declare const DEFAULT_WORD_MAX = 150;
|
|
183
|
+
/** sha256 over the ordered instance identity — lets a reviewer see two orders share a corpus. */
|
|
184
|
+
export declare function corpusFingerprint(instances: readonly {
|
|
185
|
+
id: string;
|
|
186
|
+
query: string;
|
|
187
|
+
}[]): string;
|
|
188
|
+
/**
|
|
189
|
+
* sha256 over {@link workOrderDigestInput}. Pure computation — no IO, no key material.
|
|
190
|
+
*
|
|
191
|
+
* HONEST SCOPE — read this before describing what it proves. This is an integrity check against
|
|
192
|
+
* ACCIDENTAL corruption and mismatch; it is NOT a cryptographic commitment. The digest is
|
|
193
|
+
* self-contained, so a determined operator can re-forge it (at n=12 a seed search finds a matching
|
|
194
|
+
* assignment in a few thousand tries). The threat model is US making mistakes — a hand-edited file,
|
|
195
|
+
* a stale order paired with fresh judgments — exactly the corruption-detection scoping the
|
|
196
|
+
* hash-chain backlog idea already carries. The honest-use contract is procedural: emit once, then
|
|
197
|
+
* judge, and keep the emitted file.
|
|
198
|
+
*/
|
|
199
|
+
export declare function workOrderDigest(order: {
|
|
200
|
+
seed: number;
|
|
201
|
+
version: number;
|
|
202
|
+
margin: number;
|
|
203
|
+
corpusFingerprint: string;
|
|
204
|
+
items: readonly {
|
|
205
|
+
id: string;
|
|
206
|
+
warmIsA: boolean;
|
|
207
|
+
}[];
|
|
208
|
+
}): string;
|
|
209
|
+
export interface WorkOrderVerification {
|
|
210
|
+
readonly ok: boolean;
|
|
211
|
+
/** Every problem found, not just the first — a forged order usually trips several. */
|
|
212
|
+
readonly problems: readonly string[];
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* The one sentence that states what the digest is and is not. Held in a constant so the CLI error
|
|
216
|
+
* text, the module documentation and the honest-scope regression test all read the SAME words —
|
|
217
|
+
* this promise must not quietly regrow into "commitment" language (Codex QE HIGH-C).
|
|
218
|
+
*/
|
|
219
|
+
export declare const DIGEST_HONEST_SCOPE: string;
|
|
220
|
+
/**
|
|
221
|
+
* Integrity-check a work order before ANY verdict may depend on it (Codex QE HIGH-2).
|
|
222
|
+
*
|
|
223
|
+
* Checking `kind` and `Array.isArray(items)` was vacuous: a hand-written file with the right two
|
|
224
|
+
* fields and an invented `warmIsA` un-blinded into whatever verdict its author wanted. Four checks
|
|
225
|
+
* now have to agree:
|
|
226
|
+
* 1. the `digest` recomputes over (version, seed, margin, corpus fingerprint, `[id, warmIsA]`…);
|
|
227
|
+
* 2. every `warmIsA` is REDERIVABLE from the stated `seed` — the same `mulberry32` stream that
|
|
228
|
+
* emitted it;
|
|
229
|
+
* 3. the pre-registered `margin` is in range;
|
|
230
|
+
* 4. structural sanity — unique non-empty ids, boolean assignments, integer seed.
|
|
231
|
+
*
|
|
232
|
+
* WHAT THIS IS NOT: see {@link DIGEST_HONEST_SCOPE}. Re-deriving from the seed raises the bar from
|
|
233
|
+
* "edit one field" to "search for a seed", which at n=12 is a few thousand tries — a deterrent
|
|
234
|
+
* against slips, not a defence against intent. Nothing here is a cryptographic commitment, and no
|
|
235
|
+
* amount of hashing inside the file itself could make it one.
|
|
236
|
+
*/
|
|
237
|
+
export declare function verifyWorkOrder(value: unknown): WorkOrderVerification;
|
|
238
|
+
/**
|
|
239
|
+
* Emit the generation work order. Deterministic in (instances, seed): the same corpus and seed
|
|
240
|
+
* produce the same blind assignment, which is what makes "pre-registered" checkable after the fact.
|
|
241
|
+
*/
|
|
242
|
+
export declare function buildWorkOrder(instances: readonly ReplayInstance[], options?: WorkOrderOptions): WorkOrder;
|
|
243
|
+
/**
|
|
244
|
+
* ONE judge-facing item. These two fields are the WHOLE artifact, by design.
|
|
245
|
+
*
|
|
246
|
+
* The first version also carried `warmIsA` and `class`, which handed the judge the answer key: the
|
|
247
|
+
* blinding was theatre (Codex QE CRITICAL-1). `warmIsA` now lives ONLY in the pre-registered work
|
|
248
|
+
* order, which `--score` consumes and the judge never sees. `class` went too — nothing the judge
|
|
249
|
+
* does not need may travel with the prompt.
|
|
250
|
+
*/
|
|
251
|
+
export interface JudgePrompt {
|
|
252
|
+
readonly id: string;
|
|
253
|
+
readonly prompt: string;
|
|
254
|
+
}
|
|
255
|
+
export interface JudgePromptsResult {
|
|
256
|
+
/** The judge-facing payload — `{id, prompt}` only. Nothing else may be written to the judge. */
|
|
257
|
+
readonly prompts: readonly JudgePrompt[];
|
|
258
|
+
/**
|
|
259
|
+
* Items that could NOT be judged, with the reason — never silently dropped. OPERATOR-facing:
|
|
260
|
+
* the reasons name arms ("warmPlan missing"), so this must not be written into the judge file.
|
|
261
|
+
*/
|
|
262
|
+
readonly skipped: readonly {
|
|
263
|
+
readonly id: string;
|
|
264
|
+
readonly reason: string;
|
|
265
|
+
}[];
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* Render blind A/B judge prompts from a FILLED work order. An item missing either plan is skipped
|
|
269
|
+
* with a reason: half a pair is not a comparison, and substituting an empty string would hand the
|
|
270
|
+
* judge a rigged contest.
|
|
271
|
+
*/
|
|
272
|
+
export declare function buildJudgePrompts(order: WorkOrder): JudgePromptsResult;
|
|
273
|
+
export type Arm = 'cold' | 'warm';
|
|
274
|
+
export interface EpochOutcome {
|
|
275
|
+
readonly id: string;
|
|
276
|
+
readonly class: string | null;
|
|
277
|
+
/** Which epoch solved the instance better. `tie` counts in the denominator, for neither arm. */
|
|
278
|
+
readonly winner: Arm | 'tie';
|
|
279
|
+
}
|
|
280
|
+
export interface Judgment {
|
|
281
|
+
readonly id: string;
|
|
282
|
+
/** The judge's blind answer: `A`, `B` or `TIE` (case-insensitive). */
|
|
283
|
+
readonly winner: string;
|
|
284
|
+
}
|
|
285
|
+
export interface UnblindResult {
|
|
286
|
+
/** False ⇒ the input is CORRUPT and no verdict may be computed from it. */
|
|
287
|
+
readonly ok: boolean;
|
|
288
|
+
/** Populated exactly when `ok` is false. */
|
|
289
|
+
readonly error: string | null;
|
|
290
|
+
readonly outcomes: readonly EpochOutcome[];
|
|
291
|
+
readonly skipped: readonly {
|
|
292
|
+
readonly id: string;
|
|
293
|
+
readonly reason: string;
|
|
294
|
+
}[];
|
|
295
|
+
}
|
|
296
|
+
/**
|
|
297
|
+
* Map blind judgments back to arms using the work order's PRE-REGISTERED `warmIsA`. The judgments
|
|
298
|
+
* file deliberately carries no arm labels: if un-blinding read a label the judge (or a later hand
|
|
299
|
+
* edit) supplied, the blinding would be decorative.
|
|
300
|
+
*
|
|
301
|
+
* Unknown ids and unparseable winners are SKIPPED with a reason, never guessed. DUPLICATE ids are
|
|
302
|
+
* different: they are a CORRUPT input, not a skippable row, so the whole call is REFUSED. Skipping
|
|
303
|
+
* the second copy silently accepted a file in which one judgment had been pasted five times — which
|
|
304
|
+
* scored as n=5 and reached SUPPORTED off a single opinion (Codex QE MED-4).
|
|
305
|
+
*/
|
|
306
|
+
export declare function unblindJudgments(order: WorkOrder, judgments: readonly Judgment[]): UnblindResult;
|
|
307
|
+
export type EpochVerdict = 'SUPPORTED' | 'FALSIFIED' | 'INCONCLUSIVE';
|
|
308
|
+
export interface ArmResult {
|
|
309
|
+
readonly arm: Arm;
|
|
310
|
+
readonly wins: number;
|
|
311
|
+
/** DECISIVE pairs — the binomial denominator. Ties are excluded from the test (but reported). */
|
|
312
|
+
readonly n: number;
|
|
313
|
+
readonly ci: WilsonInterval | null;
|
|
314
|
+
}
|
|
315
|
+
export interface EpochReplayResult {
|
|
316
|
+
readonly verdict: EpochVerdict;
|
|
317
|
+
/** Always populated — a bare label is not a finding. */
|
|
318
|
+
readonly reason: string;
|
|
319
|
+
/**
|
|
320
|
+
* Non-null ⇒ the INPUT was refused and the verdict is a placeholder INCONCLUSIVE, not a
|
|
321
|
+
* measurement. Callers must surface this and exit non-zero.
|
|
322
|
+
*/
|
|
323
|
+
readonly refusal: string | null;
|
|
324
|
+
readonly slice: string;
|
|
325
|
+
/** Every scored instance in the slice, ties included. Context, not the denominator. */
|
|
326
|
+
readonly n: number;
|
|
327
|
+
readonly ties: number;
|
|
328
|
+
/** DECISIVE pairs, `D = warm.wins + cold.wins` — the denominator the test actually uses. */
|
|
329
|
+
readonly decisive: number;
|
|
330
|
+
readonly cold: ArmResult;
|
|
331
|
+
readonly warm: ArmResult;
|
|
332
|
+
/**
|
|
333
|
+
* The test statistic on the LIFT scale (`2p̂ − 1`, where `p̂ = P(warm wins | decisive)`).
|
|
334
|
+
* `margin` is stated on this scale.
|
|
335
|
+
*/
|
|
336
|
+
readonly lift: LiftInterval | null;
|
|
337
|
+
readonly z: number;
|
|
338
|
+
/** Minimum DECISIVE pairs before any verdict exists. */
|
|
339
|
+
readonly minN: number;
|
|
340
|
+
readonly falsifyNoLiftMinN: number;
|
|
341
|
+
readonly noLiftMargin: number;
|
|
342
|
+
}
|
|
343
|
+
export interface ScoreOptions {
|
|
344
|
+
/** `all` (default) or a pre-registered class label. */
|
|
345
|
+
readonly slice?: string;
|
|
346
|
+
/** Must be finite and > 0 if given. Anything else is REFUSED — never clamped (Codex QE MED-5). */
|
|
347
|
+
readonly z?: number;
|
|
348
|
+
/**
|
|
349
|
+
* Non-superiority margin on the LIFT scale, in `(0, 0.5]`. In real mode this comes from the WORK
|
|
350
|
+
* ORDER (pre-registered); out-of-range is REFUSED, never clamped (Codex QE HIGH-B).
|
|
351
|
+
*/
|
|
352
|
+
readonly margin?: number;
|
|
353
|
+
}
|
|
354
|
+
/**
|
|
355
|
+
* The three-valued verdict (ADR-003, as amended).
|
|
356
|
+
*
|
|
357
|
+
* THE MODEL — a SINGLE binomial over DECISIVE pairs. Each instance yields ONE judgment about ONE
|
|
358
|
+
* prompt, so the arms are PAIRED, not two independent samples. Let `W` = warm wins, `C` = cold
|
|
359
|
+
* wins, `D = W + C` (ties are excluded from the test and reported separately). The statistic is
|
|
360
|
+
* `p̂ = W/D` with a Wilson interval, mapped to the lift scale by {@link liftInterval}:
|
|
361
|
+
*
|
|
362
|
+
* SUPPORTED lift lower bound > 0 (equivalently: Wilson lower on p̂ > 0.5)
|
|
363
|
+
* FALSIFIED harm — lift upper bound < 0 (Wilson upper on p̂ < 0.5); OR non-superiority —
|
|
364
|
+
* lift upper bound < `margin` with `D >= FALSIFY_NO_LIFT_MIN_N`
|
|
365
|
+
* INCONCLUSIVE everything else, including `D < MIN_INSTANCES`. A first-class honest outcome.
|
|
366
|
+
*
|
|
367
|
+
* This is exactly the statistic the manual 2026-07-29 experiment used, and it is the correction the
|
|
368
|
+
* re-QE demanded: the previous two-proportion (Newcombe) framing treated the paired judgments as
|
|
369
|
+
* independent samples and was ANTI-CONSERVATIVE — 500/500 over 1000 decisive pairs produced an
|
|
370
|
+
* upper bound of 0.0437 and a FALSIFIED verdict where the paired form gives 0.0619 and INCONCLUSIVE
|
|
371
|
+
* (Codex QE HIGH-A).
|
|
372
|
+
*
|
|
373
|
+
* REFUSALS (verdict is a placeholder, `refusal` is set): duplicate instance ids, an invalid `z`, or
|
|
374
|
+
* an out-of-range `margin`.
|
|
375
|
+
*/
|
|
376
|
+
export declare function scoreEpochReplay(outcomes: readonly EpochOutcome[], options?: ScoreOptions): EpochReplayResult;
|
|
377
|
+
export interface MockOptions {
|
|
378
|
+
readonly n?: number;
|
|
379
|
+
/**
|
|
380
|
+
* TRUE effect in [-1, 1]. P(warm wins | not a tie) = clamp(0.5 + effect / 2), so 0 is a fair
|
|
381
|
+
* coin, +1 is "warm always wins", -1 is "cold always wins".
|
|
382
|
+
*/
|
|
383
|
+
readonly effect?: number;
|
|
384
|
+
readonly tieRate?: number;
|
|
385
|
+
readonly seed?: number;
|
|
386
|
+
/** Class label stamped on every synthetic outcome (so `--slice` is exercisable). */
|
|
387
|
+
readonly class?: string | null;
|
|
388
|
+
}
|
|
389
|
+
export declare const DEFAULT_MOCK_N = 12;
|
|
390
|
+
export declare const DEFAULT_MOCK_SEED = 20260729;
|
|
391
|
+
/**
|
|
392
|
+
* Synthetic judge outcomes from ONE seeded stream (`mulberry32` — the repo's only PRNG). Same
|
|
393
|
+
* (n, effect, tieRate, seed) → byte-identical outcomes, so a `--mock` demo is a reproducer.
|
|
394
|
+
*/
|
|
395
|
+
export declare function generateMockOutcomes(options?: MockOptions): EpochOutcome[];
|
|
396
|
+
export declare function renderEpochReplayResult(r: EpochReplayResult): string;
|
|
397
|
+
export declare function renderWorkOrderSummary(order: WorkOrder, outPath: string): string;
|
|
398
|
+
export declare function renderJudgePromptsSummary(result: JudgePromptsResult, outPath: string): string;
|
|
399
|
+
//# sourceMappingURL=epoch-replay.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"epoch-replay.d.ts","sourceRoot":"","sources":["../src/epoch-replay.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAOH,OAAO,EAGL,mBAAmB,EACnB,KAAK,cAAc,EACpB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,mBAAmB,EAAE,KAAK,cAAc,EAAE,CAAC;AAIpD,+FAA+F;AAC/F,eAAO,MAAM,QAAQ,OAAO,CAAC;AAE7B,+FAA+F;AAC/F,eAAO,MAAM,aAAa,IAAsB,CAAC;AAEjD;;;;;;;GAOG;AACH,eAAO,MAAM,qBAAqB,QAA0B,CAAC;AAE7D;;;;;;;;GAQG;AACH,eAAO,MAAM,cAAc,OAAO,CAAC;AAEnC,mGAAmG;AACnG,eAAO,MAAM,oBAAoB,IAAI,CAAC;AACtC,eAAO,MAAM,UAAU,MAAM,CAAC;AAE9B,+EAA+E;AAC/E,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,MAAM,CAE7D;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,0BAA0B;IAC1B,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,GAAE,MAAiB,GAAG,cAAc,GAAG,IAAI,CAmBhG;AAED,qGAAqG;AACrG,MAAM,WAAW,YAAY;IAC3B,0FAA0F;IAC1F,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,cAAc,GAAG,IAAI,GAAG,YAAY,GAAG,IAAI,CAO9E;AAID,eAAO,MAAM,eAAe,+BAA+B,CAAC;AAC5D;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,IAAI,CAAC;AAEpC,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,wDAAwD;IACxD,QAAQ,CAAC,IAAI,EAAE;QAAE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,CAAC;IACrF,6EAA6E;IAC7E,QAAQ,CAAC,IAAI,EAAE;QAAE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,CAAC;IACrF,0EAA0E;IAC1E,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,OAAO,eAAe,CAAC;IACtC,QAAQ,CAAC,OAAO,EAAE,OAAO,kBAAkB,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,KAAK,EAAE,SAAS,aAAa,EAAE,CAAC;IACzC;;;;OAIG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,gGAAgG;IAChG,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,yFAAyF;IACzF,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B;;;;;;;OAOG;IACH,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,oDAAoD;IACpD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAYD,eAAO,MAAM,gBAAgB,KAAK,CAAC;AACnC,eAAO,MAAM,gBAAgB,MAAM,CAAC;AA6BpC,iGAAiG;AACjG,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,SAAS;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,EAAE,GAAG,MAAM,CAI7F;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,EAAE,MAAM,CAAC;IAC1B,KAAK,EAAE,SAAS;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;CACpD,GAAG,MAAM,CAYT;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,sFAAsF;IACtF,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,QAE8D,CAAC;AAE/F;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,qBAAqB,CAoDrE;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,SAAS,EAAE,SAAS,cAAc,EAAE,EACpC,OAAO,GAAE,gBAAqB,GAC7B,SAAS,CAwDX;AAID;;;;;;;GAOG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,kBAAkB;IACjC,gGAAgG;IAChG,QAAQ,CAAC,OAAO,EAAE,SAAS,WAAW,EAAE,CAAC;IACzC;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,SAAS;QAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC/E;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,SAAS,GAAG,kBAAkB,CAoCtE;AAID,MAAM,MAAM,GAAG,GAAG,MAAM,GAAG,MAAM,CAAC;AAElC,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,gGAAgG;IAChG,QAAQ,CAAC,MAAM,EAAE,GAAG,GAAG,KAAK,CAAC;CAC9B;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,4CAA4C;IAC5C,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;IAC3C,QAAQ,CAAC,OAAO,EAAE,SAAS;QAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC/E;AAED;;;;;;;;;GASG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,QAAQ,EAAE,GAAG,aAAa,CAiDhG;AAED,MAAM,MAAM,YAAY,GAAG,WAAW,GAAG,WAAW,GAAG,cAAc,CAAC;AAEtE,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC;IAClB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,iGAAiG;IACjG,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,EAAE,cAAc,GAAG,IAAI,CAAC;CACpC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAC/B,wDAAwD;IACxD,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,uFAAuF;IACvF,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,4FAA4F;IAC5F,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,wDAAwD;IACxD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,YAAY;IAC3B,uDAAuD;IACvD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,kGAAkG;IAClG,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,gBAAgB,CAC9B,QAAQ,EAAE,SAAS,YAAY,EAAE,EACjC,OAAO,GAAE,YAAiB,GACzB,iBAAiB,CAiJnB;AAID,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,oFAAoF;IACpF,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED,eAAO,MAAM,cAAc,KAAK,CAAC;AACjC,eAAO,MAAM,iBAAiB,WAAW,CAAC;AAE1C;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,GAAE,WAAgB,GAAG,YAAY,EAAE,CAe9E;AAQD,wBAAgB,uBAAuB,CAAC,CAAC,EAAE,iBAAiB,GAAG,MAAM,CAgCpE;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAkBhF;AAED,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,kBAAkB,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAe7F"}
|