@tangle-network/agent-eval 0.191.0 → 0.193.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +61 -0
- package/dist/adapters/http.d.ts +2 -2
- package/dist/analyst/index.d.ts +6 -6
- package/dist/analyst/index.js +3 -3
- package/dist/{analyze-runs-SCSWYpy0.js → analyze-runs-B6gv7vAT.js} +2 -2
- package/dist/{analyze-runs-SCSWYpy0.js.map → analyze-runs-B6gv7vAT.js.map} +1 -1
- package/dist/{benchmark-command-3tZJnD82.js → benchmark-command-DVhUPU7H.js} +7 -7
- package/dist/{benchmark-command-3tZJnD82.js.map → benchmark-command-DVhUPU7H.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +3 -3
- package/dist/benchmarks/index.js +3 -3
- package/dist/campaign/index.d.ts +5 -5
- package/dist/campaign/index.js +12 -12
- package/dist/{campaign-C8iWEF0M.js → campaign-DdE6S80e.js} +15 -328
- package/dist/campaign-DdE6S80e.js.map +1 -0
- package/dist/{campaign-evidence-BqjlTM3P.js → campaign-evidence-DJc60F9C.js} +11 -46
- package/dist/campaign-evidence-DJc60F9C.js.map +1 -0
- package/dist/cli.js +47 -10
- package/dist/cli.js.map +1 -1
- package/dist/{client-wFyDisxi.d.ts → client-dMMKoQNS.d.ts} +2 -2
- package/dist/{client-wFyDisxi.d.ts.map → client-dMMKoQNS.d.ts.map} +1 -1
- package/dist/contract/index.d.ts +7 -7
- package/dist/contract/index.js +10 -10
- package/dist/{default-registry-DK_9aVJf.d.ts → default-registry-C3HTE08O.d.ts} +3 -3
- package/dist/{default-registry-DK_9aVJf.d.ts.map → default-registry-C3HTE08O.d.ts.map} +1 -1
- package/dist/{define-agent-eval-P28y7mI1.d.ts → define-agent-eval-DQJvVa1P.d.ts} +15 -24
- package/dist/{define-agent-eval-P28y7mI1.d.ts.map → define-agent-eval-DQJvVa1P.d.ts.map} +1 -1
- package/dist/{define-agent-eval-BE43YTm4.js → define-agent-eval-DkPbnJEY.js} +9 -10
- package/dist/define-agent-eval-DkPbnJEY.js.map +1 -0
- package/dist/diagnosis/index.js +1 -1
- package/dist/{dspy-rlm-engine-MfpwoEaj.js → dspy-rlm-engine-BX34vPVY.js} +2 -2
- package/dist/{dspy-rlm-engine-MfpwoEaj.js.map → dspy-rlm-engine-BX34vPVY.js.map} +1 -1
- package/dist/{engine-vNiM02Ub.d.ts → engine-DjRV55F9.d.ts} +2 -2
- package/dist/{engine-vNiM02Ub.d.ts.map → engine-DjRV55F9.d.ts.map} +1 -1
- package/dist/evaluation.d.ts +1 -1
- package/dist/experiment/index.d.ts +2 -2
- package/dist/experiment/index.js +4 -4
- package/dist/{external-optimizer-process-xFTliuiK.js → external-optimizer-process-D-Sab4v3.js} +3 -62
- package/dist/external-optimizer-process-D-Sab4v3.js.map +1 -0
- package/dist/{external-optimizer-subprocess-B234V9V0.js → external-optimizer-subprocess-kbqC-iWf.js} +64 -3
- package/dist/external-optimizer-subprocess-kbqC-iWf.js.map +1 -0
- package/dist/hosted/index.d.ts +2 -2
- package/dist/hosted/index.js +1 -1
- package/dist/{index-D-TN7ZRM.d.ts → index-0jEtpSM7.d.ts} +3 -3
- package/dist/{index-D-TN7ZRM.d.ts.map → index-0jEtpSM7.d.ts.map} +1 -1
- package/dist/{index-I4UJGRno.d.ts → index-Bj96W0jZ.d.ts} +28 -2
- package/dist/{index-I4UJGRno.d.ts.map → index-Bj96W0jZ.d.ts.map} +1 -1
- package/dist/{index-CzfnASj_.d.ts → index-DMPLG226.d.ts} +400 -374
- package/dist/index-DMPLG226.d.ts.map +1 -0
- package/dist/{index-DH-MWH5f.d.ts → index-DWQNDqPd.d.ts} +7 -7
- package/dist/{index-DH-MWH5f.d.ts.map → index-DWQNDqPd.d.ts.map} +1 -1
- package/dist/index.d.ts +9 -9
- package/dist/index.js +9 -292
- package/dist/index.js.map +1 -1
- package/dist/jev.d.ts +1 -1
- package/dist/{journal-c3qY0SDP.js → journal-DvKCvjty.js} +94 -2
- package/dist/{journal-c3qY0SDP.js.map → journal-DvKCvjty.js.map} +1 -1
- package/dist/ledger-core/index.d.ts +2 -2
- package/dist/ledger-core/index.js +2 -2
- package/dist/{llm-judge-NJOHM0iA.js → llm-judge-ChbWfeWl.js} +2653 -656
- package/dist/llm-judge-ChbWfeWl.js.map +1 -0
- package/dist/{matrix-CgqdFm5N.d.ts → matrix-CNu13gg8.d.ts} +2 -2
- package/dist/{matrix-CgqdFm5N.d.ts.map → matrix-CNu13gg8.d.ts.map} +1 -1
- package/dist/multishot/golden/index.d.ts +1 -1
- package/dist/multishot/index.d.ts +2 -2
- package/dist/openapi.json +1 -1
- package/dist/paired-delta-test-BF-FEP8V.js +78 -0
- package/dist/paired-delta-test-BF-FEP8V.js.map +1 -0
- package/dist/{paired-promotion-decision-6cn3bLc3.js → paired-promotion-decision-DTKl6-G8.js} +4 -77
- package/dist/paired-promotion-decision-DTKl6-G8.js.map +1 -0
- package/dist/{produced-state-CY-L7CqS.js → produced-state-BqeIaIp2.js} +3 -3
- package/dist/{produced-state-CY-L7CqS.js.map → produced-state-BqeIaIp2.js.map} +1 -1
- package/dist/{promotion-policy-CfMNUro2.js → promotion-policy-Cy5OV8BV.js} +3 -3
- package/dist/{promotion-policy-CfMNUro2.js.map → promotion-policy-Cy5OV8BV.js.map} +1 -1
- package/dist/rl.d.ts +1 -1
- package/dist/rl.js +1 -1
- package/dist/{search-ledger-CQ8Uy4FK.js → search-ledger-CJiAEU3j.js} +164 -36
- package/dist/search-ledger-CJiAEU3j.js.map +1 -0
- package/dist/{search-ledger-file-EvdDZRhv.js → search-ledger-file-DY0AXX3W.js} +2 -2
- package/dist/{search-ledger-file-EvdDZRhv.js.map → search-ledger-file-DY0AXX3W.js.map} +1 -1
- package/dist/{search-ledger-types-C9anU2DX.d.ts → search-ledger-types-B3El5xgS.d.ts} +95 -11
- package/dist/search-ledger-types-B3El5xgS.d.ts.map +1 -0
- package/dist/{search-shipper-C55V7HE3.js → search-shipper-DUlwlCsK.js} +3 -3
- package/dist/{search-shipper-C55V7HE3.js.map → search-shipper-DUlwlCsK.js.map} +1 -1
- package/dist/search-summary-BhY8p0Av.js +378 -0
- package/dist/search-summary-BhY8p0Av.js.map +1 -0
- package/dist/{semantic-concept-judge-Bt7bXAAo.js → semantic-concept-judge-Bg-560H0.js} +3 -3
- package/dist/{semantic-concept-judge-Bt7bXAAo.js.map → semantic-concept-judge-Bg-560H0.js.map} +1 -1
- package/dist/{sequential-lwJOTRON.js → sequential-C21_PP7F.js} +2 -2
- package/dist/{sequential-lwJOTRON.js.map → sequential-C21_PP7F.js.map} +1 -1
- package/dist/{skillopt-optimization-method-DwO3TOAV.js → skillopt-optimization-method-DXQdaBjx.js} +7 -7
- package/dist/{skillopt-optimization-method-DwO3TOAV.js.map → skillopt-optimization-method-DXQdaBjx.js.map} +1 -1
- package/dist/{statistical-heldout-BED3wBtX.d.ts → statistical-heldout-D-xBpbrd.d.ts} +2 -2
- package/dist/{statistical-heldout-BED3wBtX.d.ts.map → statistical-heldout-D-xBpbrd.d.ts.map} +1 -1
- package/dist/{statistical-heldout-GcLIPxk4.js → statistical-heldout-DANVmyug.js} +2 -2
- package/dist/{statistical-heldout-GcLIPxk4.js.map → statistical-heldout-DANVmyug.js.map} +1 -1
- package/dist/{store-tool-spans-Cfh9XUCm.d.ts → store-tool-spans-BxrfuU2t.d.ts} +4 -4
- package/dist/{store-tool-spans-Cfh9XUCm.d.ts.map → store-tool-spans-BxrfuU2t.d.ts.map} +1 -1
- package/dist/{tool-groups-D-mkeN0N.d.ts → tool-groups-BC9O7Epn.d.ts} +2 -2
- package/dist/tool-groups-BC9O7Epn.d.ts.map +1 -0
- package/dist/trace-repair/index.d.ts +1 -1
- package/dist/traces.d.ts +2 -2
- package/dist/{types-BmQRs661.d.ts → types-BWwRXofp.d.ts} +340 -17
- package/dist/types-BWwRXofp.d.ts.map +1 -0
- package/dist/{types-ClK-sHs8.d.ts → types-qoxUfKQg.d.ts} +2 -2
- package/dist/{types-ClK-sHs8.d.ts.map → types-qoxUfKQg.d.ts.map} +1 -1
- package/docs/campaign-proposers.md +14 -16
- package/docs/search-ledger.md +129 -3
- package/package.json +1 -1
- package/dist/campaign-C8iWEF0M.js.map +0 -1
- package/dist/campaign-evidence-BqjlTM3P.js.map +0 -1
- package/dist/claim-BGOUqoMq.d.ts +0 -38
- package/dist/claim-BGOUqoMq.d.ts.map +0 -1
- package/dist/define-agent-eval-BE43YTm4.js.map +0 -1
- package/dist/external-optimizer-process-xFTliuiK.js.map +0 -1
- package/dist/external-optimizer-subprocess-B234V9V0.js.map +0 -1
- package/dist/index-CzfnASj_.d.ts.map +0 -1
- package/dist/llm-judge-NJOHM0iA.js.map +0 -1
- package/dist/paired-promotion-decision-6cn3bLc3.js.map +0 -1
- package/dist/search-ledger-CQ8Uy4FK.js.map +0 -1
- package/dist/search-ledger-types-C9anU2DX.d.ts.map +0 -1
- package/dist/tool-groups-D-mkeN0N.d.ts.map +0 -1
- package/dist/types-BmQRs661.d.ts.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"promotion-policy-CfMNUro2.js","names":[],"sources":["../src/campaign/gates/promotion-policy.ts"],"sourcesContent":["/**\n * Build paired evidence for each objective and apply a promotion policy.\n * The default policy requires at least one gain and every regression floor to\n * clear. A floor can fail because a larger loss remains plausible; that does\n * not demonstrate an observed regression. Missing evidence stays unresolved.\n * Confidence intervals apply to each axis, without a multiplicity adjustment.\n *\n * Cost and latency remain aggregate constraints because GateContext does not\n * supply paired observations for them. Compose a budget gate when needed.\n */\n\nimport {\n decidePairedPromotion,\n type PairedDecisionMethod,\n type PairedDecisionStatistic,\n type PairedMcNemarEvidence,\n} from '../../paired-promotion-decision'\nimport type { Direction } from '../../pareto'\nimport {\n DECISION_PAIRED_DELTA_STATISTIC,\n type PairedBootstrapResult,\n pairedBootstrap,\n} from '../../statistics'\nimport type { Gate, GateContext, GateDecision, GateResult, JudgeScore, Scenario } from '../types'\nimport { detectScale, pairHoldout } from './statistical-heldout'\n\n/** Where an objective's per-cell scalar comes from. `composite` reads the\n * judge's composite; `dimension` reads a named per-dimension score. */\nexport type ObjectiveSource = { kind: 'composite' } | { kind: 'dimension'; dimension: string }\n\nexport interface PromotionObjective {\n /** Stable label used in reports + `contributingGates`. */\n name: string\n source: ObjectiveSource\n /** 'maximize' (quality dims) or 'minimize' (error/risk/length dims). Orients\n * the paired delta so a positive bootstrap always means \"candidate better\". */\n direction: Direction\n /** Declared binary support {0, binaryScale}, including zero-only observations.\n * Must be finite and positive; paired cell scores must be 0 or this scale.\n * Uses the risk-difference mean and rejects the 'median' statistic. */\n binaryScale?: number\n /** The good-direction paired-delta CI lower bound must EXCEED this to count\n * as a significant gain on this axis. Interpreted in the judge's native\n * scale. Default 0 (⇒ \"confidently better\"). */\n gainThreshold?: number\n /** A floor breach (regression) is declared when the good-direction CI lower\n * bound is below −floorTolerance, or when the exact small-sample test proves\n * a drop past it. Defaults to 0.05 times the declared binary scale, or\n * auto-scales off observed magnitudes (0.05 on [0,1], 5 on 0-100). */\n floorTolerance?: number\n}\n\n/** Per-axis verdict from the shared paired decision rule.\n * 'regressed' includes uncertainty that prevents clearing the regression floor. */\nexport type AxisVerdict = 'improved' | 'regressed' | 'flat' | 'few_runs' | 'indeterminate'\n\nexport interface AxisEvidence {\n name: string\n source: ObjectiveSource\n direction: Direction\n /** Paired bootstrap on the GOOD-DIRECTION delta (oriented by `direction`):\n * a positive value means the candidate is better on this axis.\n *\n * DIAGNOSTIC on a pass/fail axis: there the verdict is decided on Tango's\n * score interval instead, because a percentile bootstrap over a three-atom\n * delta lattice is not a valid interval at the nonzero margin `floorTolerance`\n * and `gainThreshold` create. `ci` carries the interval that decided. */\n bootstrap: PairedBootstrapResult\n /** Which paired statistic `bootstrap.low`/`.high` bracket. `'mean'` unless the\n * caller asked for the median — on a pass/fail axis the median and its whole\n * CI are pinned at 0 by tie domination and can see neither a gain nor a\n * regression. `bootstrap.median` still carries the median point estimate. */\n bootstrapStatistic: 'median' | 'mean'\n /** The interval the axis verdict was actually decided on, good-direction and\n * in the axis's native units. */\n ci: { low: number; high: number }\n /** Which estimator produced `ci`. */\n decisionStatistic: PairedDecisionStatistic\n /** McNemar's exact evidence on a pass/fail axis; null otherwise. */\n mcnemar: PairedMcNemarEvidence | null\n /** `ci` has zero width or non-finite bounds. It cannot establish a gain or\n * clear a regression floor, regardless of the point estimate. */\n indeterminate: boolean\n /** Paired observations contributing to this axis. */\n n: number\n minimumRequired: number\n decisionMethod: PairedDecisionMethod\n gainThreshold: number\n floorTolerance: number\n verdict: AxisVerdict\n}\n\nexport interface EvidenceVector {\n /** One entry per objective — NOTHING averaged across axes. */\n axes: AxisEvidence[]\n /** Smallest paired n across axes that produced observations — the binding\n * evidence-sufficiency constraint. 0 when no axis produced observations. */\n minN: number\n /** Aggregate per-side cost from the gate context (a constraint input, not a\n * CI axis — see the module header). */\n cost: { candidate: number; baseline: number }\n}\n\n/** A promotion strategy: a pure function from the evidence vector to a verdict.\n * Many policies can run over the same `EvidenceVector` and disagree — that's\n * the point (competing strategies, shared evidence). */\nexport type PromotionPolicy = (ev: EvidenceVector) => GateResult\n\nexport interface BuildEvidenceVectorOptions {\n /** Minimum paired observations before an axis can claim significance; below\n * it the axis is `few_runs`. The exact small-sample test may require more\n * observations at the selected confidence. */\n minProductiveRuns?: number\n /** Confidence level for every axis bootstrap. Default 0.95. */\n confidence?: number\n /** Bootstrap resamples. Default 2000. */\n resamples?: number\n /** Fixed bootstrap seed for a deterministic, reproducible verdict. Default 1337. */\n seed?: number\n /** Paired statistic every axis CI is computed on. Default `'mean'` — see\n * {@link DECISION_PAIRED_DELTA_STATISTIC} for why the median is not. */\n statistic?: 'mean' | 'median'\n}\n\n/**\n * The Evidence Bus. For each objective, pair candidate vs baseline by full\n * cellId and bootstrap a CI on the good-direction paired delta. Reuses the\n * exact `pairHoldout` + `pairedBootstrap` machinery the held-out gate uses, so\n * a single source of truth governs pairing granularity + scale handling.\n */\nexport function buildEvidenceVector<TArtifact, TScenario extends Scenario>(\n ctx: GateContext<TArtifact, TScenario>,\n objectives: PromotionObjective[],\n opts: BuildEvidenceVectorOptions = {},\n): EvidenceVector {\n if (objectives.length === 0) {\n throw new Error('buildEvidenceVector: at least 1 objective required')\n }\n const confidence = opts.confidence ?? 0.95\n const resamples = opts.resamples ?? 2000\n const seed = opts.seed ?? 1337\n const baseline = ctx.baselineJudgeScores ?? ctx.judgeScores\n const scenarioIds = new Set(ctx.scenarios.map((s) => s.id))\n\n const axes: AxisEvidence[] = []\n for (const obj of objectives) {\n let select: (s: JudgeScore) => number | undefined\n if (obj.source.kind === 'composite') {\n select = (s) => s.composite\n } else {\n const dim = obj.source.dimension\n select = (s) => s.dimensions[dim]\n }\n const paired = pairHoldout(ctx.judgeScores, baseline, scenarioIds, select)\n // Orient to the good direction: maximize ⇒ bootstrap (candidate − baseline);\n // minimize ⇒ bootstrap (baseline − candidate) by swapping args, so a\n // positive bootstrap always reads as \"candidate better on this axis\".\n const before = obj.direction === 'maximize' ? paired.before : paired.after\n const after = obj.direction === 'maximize' ? paired.after : paired.before\n const n = paired.before.length\n const binaryScale = obj.binaryScale\n const floorTolerance =\n obj.floorTolerance ?? 0.05 * (binaryScale ?? detectScale([...paired.before, ...paired.after]))\n const gainThreshold = obj.gainThreshold ?? 0\n // Axes are decided on the MEAN paired delta — which for a pass/fail axis is\n // exactly the change in success rate. The median is structurally blind on\n // the shapes eval data lands in: with most pairs tied its bootstrap CI\n // collapses to [0,0] and the axis reads 'flat', hiding real gains AND real\n // regressions — pass/fail axes on ANY encoding ({0,1} and the 0-100 one\n // `detectScale` exists for), and low-cardinality axes even below half ties.\n // Orthogonal to `pairedDeltaTest`'s own small-sample switch: that picks the\n // TEST (bootstrap CI at n ≥ 20, exact sign test below), this picks the\n // ESTIMATOR the test is applied to. Both are needed — an exact sign test on\n // a tie-pinned median is still blind.\n const bootstrapStatistic = opts.statistic ?? DECISION_PAIRED_DELTA_STATISTIC\n // Both burdens of proof route through the ONE shared rule\n // (`decidePairedPromotion`), so a pass/fail axis is judged on Tango's score\n // interval — the only paired-binary construction valid at the nonzero\n // margins `gainThreshold` / `floorTolerance` create — and a zero-width\n // interval cannot buy a verdict in either direction.\n const improvement = decidePairedPromotion(before, after, {\n confidence,\n resamples,\n statistic: bootstrapStatistic,\n seed,\n threshold: gainThreshold,\n minPairs: opts.minProductiveRuns,\n binaryScale,\n })\n const regression = decidePairedPromotion(after, before, {\n confidence,\n resamples,\n statistic: bootstrapStatistic,\n seed,\n threshold: floorTolerance,\n minPairs: opts.minProductiveRuns,\n binaryScale,\n })\n const bootstrap =\n improvement.bootstrap ??\n pairedBootstrap(before, after, {\n confidence,\n resamples,\n statistic: bootstrapStatistic,\n seed,\n })\n // A floor breach fires on EITHER burden of proof, because they cover\n // different failures and the floor is the anti-Goodhart guard:\n // - `improvement.low < -floorTolerance` — the CREDIBLE WORST CASE exceeds\n // the tolerance. This is the contract `AxisEvidence.floorTolerance` and\n // `paretoPolicy`'s own reason string state, and it is the conservative\n // posture a safety axis needs: block unless the data can rule the\n // breach out, rather than waiting for the breach to be proven. Read off\n // the DECIDING interval, so a pass/fail axis is not screened by a\n // bootstrap that is pinned wherever ties dominate.\n // - `regression.promote` — a PROVEN drop past the tolerance. Adds the\n // small-sample path, where the decision is an exact sign test because\n // the bootstrap interval is descriptive only.\n // A tied binary axis still has uncertainty about unseen discordant pairs.\n // Its diagnostic bootstrap collapses to zero, so only the deciding score\n // interval can establish that a regression stays within the declared floor.\n const floorBreached = improvement.low < -floorTolerance || regression.promote\n // Floor check precedes the gain check: a credible regression must never be\n // masked as \"improved\". With the defaults (gainThreshold 0, positive floor)\n // the regions are disjoint and order is moot, but a consumer who sets a\n // negative gainThreshold (\"accept small dips\") could otherwise have a real\n // floor breach classified as a gain — anti-Goodhart wins the tie.\n const verdict: AxisVerdict = !improvement.sufficient\n ? 'few_runs'\n : improvement.indeterminate\n ? 'indeterminate'\n : floorBreached\n ? 'regressed'\n : improvement.promote\n ? 'improved'\n : 'flat'\n axes.push({\n name: obj.name,\n source: obj.source,\n direction: obj.direction,\n bootstrap,\n bootstrapStatistic,\n ci: { low: improvement.low, high: improvement.high },\n decisionStatistic: improvement.statistic,\n mcnemar: improvement.mcnemar,\n indeterminate: improvement.indeterminate,\n n,\n minimumRequired: improvement.minimumPairs,\n decisionMethod: improvement.method,\n gainThreshold,\n floorTolerance,\n verdict,\n })\n }\n const ns = axes.map((a) => a.n).filter((n) => n > 0)\n const minN = ns.length > 0 ? Math.min(...ns) : 0\n return { axes, minN, cost: { candidate: ctx.cost.candidate, baseline: ctx.cost.baseline } }\n}\n\n/**\n * Require a supported gain and every configured regression floor to clear.\n * A failed floor holds the candidate, including when uncertainty permits a loss.\n * Missing or indeterminate evidence requires more work; no gain holds release.\n */\nexport const paretoPolicy: PromotionPolicy = (ev) => {\n const contributingGates = ev.axes.map((ax) => ({\n name: `objective:${ax.name}`,\n status:\n ax.verdict === 'regressed'\n ? ('fail' as const)\n : ax.verdict === 'few_runs' || ax.verdict === 'indeterminate'\n ? ('not_evaluated' as const)\n : ('pass' as const),\n detail: {\n direction: ax.direction,\n source: ax.source,\n verdict: ax.verdict,\n n: ax.n,\n deltaMedian: ax.bootstrap.median,\n ciLow: ax.ci.low,\n ciHigh: ax.ci.high,\n decisionStatistic: ax.decisionStatistic,\n decisionMethod: ax.decisionMethod,\n mcnemar: ax.mcnemar,\n indeterminate: ax.indeterminate,\n bootstrapCiLow: ax.bootstrap.low,\n bootstrapCiHigh: ax.bootstrap.high,\n confidence: ax.bootstrap.confidence,\n gainThreshold: ax.gainThreshold,\n floorTolerance: ax.floorTolerance,\n },\n }))\n\n const regressed = ev.axes.filter((a) => a.verdict === 'regressed')\n const insufficient = ev.axes.filter(\n (a) => a.verdict === 'few_runs' || a.verdict === 'indeterminate',\n )\n const improved = ev.axes.filter((a) => a.verdict === 'improved')\n\n let decision: GateDecision\n const reasons: string[] = []\n if (regressed.length > 0) {\n // A gain on another axis cannot excuse an unresolved regression floor.\n decision = 'hold'\n for (const a of regressed) {\n reasons.push(\n `objective '${a.name}' did not clear its regression floor -${a.floorTolerance}: good-direction CI [${a.ci.low.toFixed(3)}, ${a.ci.high.toFixed(3)}] (n=${a.n})`,\n )\n }\n } else if (insufficient.length > 0) {\n // An unresolved axis cannot establish either a gain or a safe floor.\n decision = 'need_more_work'\n for (const a of insufficient) {\n reasons.push(\n a.verdict === 'few_runs'\n ? `objective '${a.name}' has only n=${a.n} paired runs — insufficient evidence to claim significance`\n : `objective '${a.name}' has an indeterminate deciding CI [${a.ci.low}, ${a.ci.high}] — insufficient evidence to clear its regression floor (n=${a.n})`,\n )\n }\n } else if (improved.length > 0) {\n decision = 'ship'\n reasons.push(\n `Supported objective gain: ${improved\n .map(\n (a) =>\n `'${a.name}' +${a.ci.low > 0 ? a.ci.low.toFixed(3) : a.bootstrap.mean.toFixed(3)} (CI.low ${a.ci.low.toFixed(3)})`,\n )\n .join(', ')}; every regression floor cleared`,\n )\n } else {\n // Every floor cleared, but no objective demonstrated a significant gain.\n decision = 'hold'\n reasons.push(\n 'no Pareto improvement: no objective shows a significant gain; every regression floor cleared',\n )\n }\n\n // `delta` surfaces the composite axis if present, else the first axis — a\n // single convenience scalar; the vector lives in `contributingGates`.\n const composite = ev.axes.find((a) => a.source.kind === 'composite') ?? ev.axes[0]\n return { decision, reasons, contributingGates, delta: composite?.bootstrap.median }\n}\n\nexport interface ParetoSignificanceGateOptions extends BuildEvidenceVectorOptions {\n /** The objective vector. Every axis is both a gain source and a safety floor. */\n objectives: PromotionObjective[]\n /** Strategy applied to the evidence vector. Default `paretoPolicy`. Override\n * to run a stricter/looser strategy over the SAME bus (competing policies). */\n policy?: PromotionPolicy\n /** Override the gate name in reports. */\n name?: string\n}\n\n/**\n * Wrap the bus + a policy as a `Gate`. Plugs into the existing\n * `runImprovementLoop({ gate })` slot and composes via `composeGate`; default\n * loop behavior is unchanged because consumers opt in by passing this gate.\n */\nexport function paretoSignificanceGate<TArtifact = unknown, TScenario extends Scenario = Scenario>(\n options: ParetoSignificanceGateOptions,\n): Gate<TArtifact, TScenario> {\n if (options.objectives.length === 0) {\n throw new Error('paretoSignificanceGate: at least 1 objective required')\n }\n const policy = options.policy ?? paretoPolicy\n return {\n name: options.name ?? 'paretoSignificanceGate',\n async decide(ctx: GateContext<TArtifact, TScenario>): Promise<GateResult> {\n const ev = buildEvidenceVector(ctx, options.objectives, options)\n return policy(ev)\n },\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAkIA,SAAgB,oBACd,KACA,YACA,OAAmC,CAAC,GACpB;CAChB,IAAI,WAAW,WAAW,GACxB,MAAM,IAAI,MAAM,oDAAoD;CAEtE,MAAM,aAAa,KAAK,cAAc;CACtC,MAAM,YAAY,KAAK,aAAa;CACpC,MAAM,OAAO,KAAK,QAAQ;CAC1B,MAAM,WAAW,IAAI,uBAAuB,IAAI;CAChD,MAAM,cAAc,IAAI,IAAI,IAAI,UAAU,KAAK,MAAM,EAAE,EAAE,CAAC;CAE1D,MAAM,OAAuB,CAAC;CAC9B,KAAK,MAAM,OAAO,YAAY;EAC5B,IAAI;EACJ,IAAI,IAAI,OAAO,SAAS,aACtB,UAAU,MAAM,EAAE;OACb;GACL,MAAM,MAAM,IAAI,OAAO;GACvB,UAAU,MAAM,EAAE,WAAW;EAC/B;EACA,MAAM,SAAS,YAAY,IAAI,aAAa,UAAU,aAAa,MAAM;EAIzE,MAAM,SAAS,IAAI,cAAc,aAAa,OAAO,SAAS,OAAO;EACrE,MAAM,QAAQ,IAAI,cAAc,aAAa,OAAO,QAAQ,OAAO;EACnE,MAAM,IAAI,OAAO,OAAO;EACxB,MAAM,cAAc,IAAI;EACxB,MAAM,iBACJ,IAAI,kBAAkB,OAAQ,eAAe,YAAY,CAAC,GAAG,OAAO,QAAQ,GAAG,OAAO,KAAK,CAAC;EAC9F,MAAM,gBAAgB,IAAI,iBAAiB;EAW3C,MAAM,qBAAqB,KAAK,aAAA;EAMhC,MAAM,cAAc,sBAAsB,QAAQ,OAAO;GACvD;GACA;GACA,WAAW;GACX;GACA,WAAW;GACX,UAAU,KAAK;GACf;EACF,CAAC;EACD,MAAM,aAAa,sBAAsB,OAAO,QAAQ;GACtD;GACA;GACA,WAAW;GACX;GACA,WAAW;GACX,UAAU,KAAK;GACf;EACF,CAAC;EACD,MAAM,YACJ,YAAY,aACZ,gBAAgB,QAAQ,OAAO;GAC7B;GACA;GACA,WAAW;GACX;EACF,CAAC;EAgBH,MAAM,gBAAgB,YAAY,MAAM,CAAC,kBAAkB,WAAW;EAMtE,MAAM,UAAuB,CAAC,YAAY,aACtC,aACA,YAAY,gBACV,kBACA,gBACE,cACA,YAAY,UACV,aACA;EACV,KAAK,KAAK;GACR,MAAM,IAAI;GACV,QAAQ,IAAI;GACZ,WAAW,IAAI;GACf;GACA;GACA,IAAI;IAAE,KAAK,YAAY;IAAK,MAAM,YAAY;GAAK;GACnD,mBAAmB,YAAY;GAC/B,SAAS,YAAY;GACrB,eAAe,YAAY;GAC3B;GACA,iBAAiB,YAAY;GAC7B,gBAAgB,YAAY;GAC5B;GACA;GACA;EACF,CAAC;CACH;CACA,MAAM,KAAK,KAAK,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,MAAM,IAAI,CAAC;CAEnD,OAAO;EAAE;EAAM,MADF,GAAG,SAAS,IAAI,KAAK,IAAI,GAAG,EAAE,IAAI;EAC1B,MAAM;GAAE,WAAW,IAAI,KAAK;GAAW,UAAU,IAAI,KAAK;EAAS;CAAE;AAC5F;;;;;;AAOA,MAAa,gBAAiC,OAAO;CACnD,MAAM,oBAAoB,GAAG,KAAK,KAAK,QAAQ;EAC7C,MAAM,aAAa,GAAG;EACtB,QACE,GAAG,YAAY,cACV,SACD,GAAG,YAAY,cAAc,GAAG,YAAY,kBACzC,kBACA;EACT,QAAQ;GACN,WAAW,GAAG;GACd,QAAQ,GAAG;GACX,SAAS,GAAG;GACZ,GAAG,GAAG;GACN,aAAa,GAAG,UAAU;GAC1B,OAAO,GAAG,GAAG;GACb,QAAQ,GAAG,GAAG;GACd,mBAAmB,GAAG;GACtB,gBAAgB,GAAG;GACnB,SAAS,GAAG;GACZ,eAAe,GAAG;GAClB,gBAAgB,GAAG,UAAU;GAC7B,iBAAiB,GAAG,UAAU;GAC9B,YAAY,GAAG,UAAU;GACzB,eAAe,GAAG;GAClB,gBAAgB,GAAG;EACrB;CACF,EAAE;CAEF,MAAM,YAAY,GAAG,KAAK,QAAQ,MAAM,EAAE,YAAY,WAAW;CACjE,MAAM,eAAe,GAAG,KAAK,QAC1B,MAAM,EAAE,YAAY,cAAc,EAAE,YAAY,eACnD;CACA,MAAM,WAAW,GAAG,KAAK,QAAQ,MAAM,EAAE,YAAY,UAAU;CAE/D,IAAI;CACJ,MAAM,UAAoB,CAAC;CAC3B,IAAI,UAAU,SAAS,GAAG;EAExB,WAAW;EACX,KAAK,MAAM,KAAK,WACd,QAAQ,KACN,cAAc,EAAE,KAAK,wCAAwC,EAAE,eAAe,uBAAuB,EAAE,GAAG,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,GAAG,KAAK,QAAQ,CAAC,EAAE,OAAO,EAAE,EAAE,EAC/J;CAEJ,OAAO,IAAI,aAAa,SAAS,GAAG;EAElC,WAAW;EACX,KAAK,MAAM,KAAK,cACd,QAAQ,KACN,EAAE,YAAY,aACV,cAAc,EAAE,KAAK,eAAe,EAAE,EAAE,8DACxC,cAAc,EAAE,KAAK,sCAAsC,EAAE,GAAG,IAAI,IAAI,EAAE,GAAG,KAAK,6DAA6D,EAAE,EAAE,EACzJ;CAEJ,OAAO,IAAI,SAAS,SAAS,GAAG;EAC9B,WAAW;EACX,QAAQ,KACN,6BAA6B,SAC1B,KACE,MACC,IAAI,EAAE,KAAK,KAAK,EAAE,GAAG,MAAM,IAAI,EAAE,GAAG,IAAI,QAAQ,CAAC,IAAI,EAAE,UAAU,KAAK,QAAQ,CAAC,EAAE,WAAW,EAAE,GAAG,IAAI,QAAQ,CAAC,EAAE,EACpH,CAAC,CACA,KAAK,IAAI,EAAE,iCAChB;CACF,OAAO;EAEL,WAAW;EACX,QAAQ,KACN,8FACF;CACF;CAIA,MAAM,YAAY,GAAG,KAAK,MAAM,MAAM,EAAE,OAAO,SAAS,WAAW,KAAK,GAAG,KAAK;CAChF,OAAO;EAAE;EAAU;EAAS;EAAmB,OAAO,WAAW,UAAU;CAAO;AACpF;;;;;;AAiBA,SAAgB,uBACd,SAC4B;CAC5B,IAAI,QAAQ,WAAW,WAAW,GAChC,MAAM,IAAI,MAAM,uDAAuD;CAEzE,MAAM,SAAS,QAAQ,UAAU;CACjC,OAAO;EACL,MAAM,QAAQ,QAAQ;EACtB,MAAM,OAAO,KAA6D;GACxE,MAAM,KAAK,oBAAoB,KAAK,QAAQ,YAAY,OAAO;GAC/D,OAAO,OAAO,EAAE;EAClB;CACF;AACF"}
|
|
1
|
+
{"version":3,"file":"promotion-policy-Cy5OV8BV.js","names":[],"sources":["../src/campaign/gates/promotion-policy.ts"],"sourcesContent":["/**\n * Build paired evidence for each objective and apply a promotion policy.\n * The default policy requires at least one gain and every regression floor to\n * clear. A floor can fail because a larger loss remains plausible; that does\n * not demonstrate an observed regression. Missing evidence stays unresolved.\n * Confidence intervals apply to each axis, without a multiplicity adjustment.\n *\n * Cost and latency remain aggregate constraints because GateContext does not\n * supply paired observations for them. Compose a budget gate when needed.\n */\n\nimport {\n decidePairedPromotion,\n type PairedDecisionMethod,\n type PairedDecisionStatistic,\n type PairedMcNemarEvidence,\n} from '../../paired-promotion-decision'\nimport type { Direction } from '../../pareto'\nimport {\n DECISION_PAIRED_DELTA_STATISTIC,\n type PairedBootstrapResult,\n pairedBootstrap,\n} from '../../statistics'\nimport type { Gate, GateContext, GateDecision, GateResult, JudgeScore, Scenario } from '../types'\nimport { detectScale, pairHoldout } from './statistical-heldout'\n\n/** Where an objective's per-cell scalar comes from. `composite` reads the\n * judge's composite; `dimension` reads a named per-dimension score. */\nexport type ObjectiveSource = { kind: 'composite' } | { kind: 'dimension'; dimension: string }\n\nexport interface PromotionObjective {\n /** Stable label used in reports + `contributingGates`. */\n name: string\n source: ObjectiveSource\n /** 'maximize' (quality dims) or 'minimize' (error/risk/length dims). Orients\n * the paired delta so a positive bootstrap always means \"candidate better\". */\n direction: Direction\n /** Declared binary support {0, binaryScale}, including zero-only observations.\n * Must be finite and positive; paired cell scores must be 0 or this scale.\n * Uses the risk-difference mean and rejects the 'median' statistic. */\n binaryScale?: number\n /** The good-direction paired-delta CI lower bound must EXCEED this to count\n * as a significant gain on this axis. Interpreted in the judge's native\n * scale. Default 0 (⇒ \"confidently better\"). */\n gainThreshold?: number\n /** A floor breach (regression) is declared when the good-direction CI lower\n * bound is below −floorTolerance, or when the exact small-sample test proves\n * a drop past it. Defaults to 0.05 times the declared binary scale, or\n * auto-scales off observed magnitudes (0.05 on [0,1], 5 on 0-100). */\n floorTolerance?: number\n}\n\n/** Per-axis verdict from the shared paired decision rule.\n * 'regressed' includes uncertainty that prevents clearing the regression floor. */\nexport type AxisVerdict = 'improved' | 'regressed' | 'flat' | 'few_runs' | 'indeterminate'\n\nexport interface AxisEvidence {\n name: string\n source: ObjectiveSource\n direction: Direction\n /** Paired bootstrap on the GOOD-DIRECTION delta (oriented by `direction`):\n * a positive value means the candidate is better on this axis.\n *\n * DIAGNOSTIC on a pass/fail axis: there the verdict is decided on Tango's\n * score interval instead, because a percentile bootstrap over a three-atom\n * delta lattice is not a valid interval at the nonzero margin `floorTolerance`\n * and `gainThreshold` create. `ci` carries the interval that decided. */\n bootstrap: PairedBootstrapResult\n /** Which paired statistic `bootstrap.low`/`.high` bracket. `'mean'` unless the\n * caller asked for the median — on a pass/fail axis the median and its whole\n * CI are pinned at 0 by tie domination and can see neither a gain nor a\n * regression. `bootstrap.median` still carries the median point estimate. */\n bootstrapStatistic: 'median' | 'mean'\n /** The interval the axis verdict was actually decided on, good-direction and\n * in the axis's native units. */\n ci: { low: number; high: number }\n /** Which estimator produced `ci`. */\n decisionStatistic: PairedDecisionStatistic\n /** McNemar's exact evidence on a pass/fail axis; null otherwise. */\n mcnemar: PairedMcNemarEvidence | null\n /** `ci` has zero width or non-finite bounds. It cannot establish a gain or\n * clear a regression floor, regardless of the point estimate. */\n indeterminate: boolean\n /** Paired observations contributing to this axis. */\n n: number\n minimumRequired: number\n decisionMethod: PairedDecisionMethod\n gainThreshold: number\n floorTolerance: number\n verdict: AxisVerdict\n}\n\nexport interface EvidenceVector {\n /** One entry per objective — NOTHING averaged across axes. */\n axes: AxisEvidence[]\n /** Smallest paired n across axes that produced observations — the binding\n * evidence-sufficiency constraint. 0 when no axis produced observations. */\n minN: number\n /** Aggregate per-side cost from the gate context (a constraint input, not a\n * CI axis — see the module header). */\n cost: { candidate: number; baseline: number }\n}\n\n/** A promotion strategy: a pure function from the evidence vector to a verdict.\n * Many policies can run over the same `EvidenceVector` and disagree — that's\n * the point (competing strategies, shared evidence). */\nexport type PromotionPolicy = (ev: EvidenceVector) => GateResult\n\nexport interface BuildEvidenceVectorOptions {\n /** Minimum paired observations before an axis can claim significance; below\n * it the axis is `few_runs`. The exact small-sample test may require more\n * observations at the selected confidence. */\n minProductiveRuns?: number\n /** Confidence level for every axis bootstrap. Default 0.95. */\n confidence?: number\n /** Bootstrap resamples. Default 2000. */\n resamples?: number\n /** Fixed bootstrap seed for a deterministic, reproducible verdict. Default 1337. */\n seed?: number\n /** Paired statistic every axis CI is computed on. Default `'mean'` — see\n * {@link DECISION_PAIRED_DELTA_STATISTIC} for why the median is not. */\n statistic?: 'mean' | 'median'\n}\n\n/**\n * The Evidence Bus. For each objective, pair candidate vs baseline by full\n * cellId and bootstrap a CI on the good-direction paired delta. Reuses the\n * exact `pairHoldout` + `pairedBootstrap` machinery the held-out gate uses, so\n * a single source of truth governs pairing granularity + scale handling.\n */\nexport function buildEvidenceVector<TArtifact, TScenario extends Scenario>(\n ctx: GateContext<TArtifact, TScenario>,\n objectives: PromotionObjective[],\n opts: BuildEvidenceVectorOptions = {},\n): EvidenceVector {\n if (objectives.length === 0) {\n throw new Error('buildEvidenceVector: at least 1 objective required')\n }\n const confidence = opts.confidence ?? 0.95\n const resamples = opts.resamples ?? 2000\n const seed = opts.seed ?? 1337\n const baseline = ctx.baselineJudgeScores ?? ctx.judgeScores\n const scenarioIds = new Set(ctx.scenarios.map((s) => s.id))\n\n const axes: AxisEvidence[] = []\n for (const obj of objectives) {\n let select: (s: JudgeScore) => number | undefined\n if (obj.source.kind === 'composite') {\n select = (s) => s.composite\n } else {\n const dim = obj.source.dimension\n select = (s) => s.dimensions[dim]\n }\n const paired = pairHoldout(ctx.judgeScores, baseline, scenarioIds, select)\n // Orient to the good direction: maximize ⇒ bootstrap (candidate − baseline);\n // minimize ⇒ bootstrap (baseline − candidate) by swapping args, so a\n // positive bootstrap always reads as \"candidate better on this axis\".\n const before = obj.direction === 'maximize' ? paired.before : paired.after\n const after = obj.direction === 'maximize' ? paired.after : paired.before\n const n = paired.before.length\n const binaryScale = obj.binaryScale\n const floorTolerance =\n obj.floorTolerance ?? 0.05 * (binaryScale ?? detectScale([...paired.before, ...paired.after]))\n const gainThreshold = obj.gainThreshold ?? 0\n // Axes are decided on the MEAN paired delta — which for a pass/fail axis is\n // exactly the change in success rate. The median is structurally blind on\n // the shapes eval data lands in: with most pairs tied its bootstrap CI\n // collapses to [0,0] and the axis reads 'flat', hiding real gains AND real\n // regressions — pass/fail axes on ANY encoding ({0,1} and the 0-100 one\n // `detectScale` exists for), and low-cardinality axes even below half ties.\n // Orthogonal to `pairedDeltaTest`'s own small-sample switch: that picks the\n // TEST (bootstrap CI at n ≥ 20, exact sign test below), this picks the\n // ESTIMATOR the test is applied to. Both are needed — an exact sign test on\n // a tie-pinned median is still blind.\n const bootstrapStatistic = opts.statistic ?? DECISION_PAIRED_DELTA_STATISTIC\n // Both burdens of proof route through the ONE shared rule\n // (`decidePairedPromotion`), so a pass/fail axis is judged on Tango's score\n // interval — the only paired-binary construction valid at the nonzero\n // margins `gainThreshold` / `floorTolerance` create — and a zero-width\n // interval cannot buy a verdict in either direction.\n const improvement = decidePairedPromotion(before, after, {\n confidence,\n resamples,\n statistic: bootstrapStatistic,\n seed,\n threshold: gainThreshold,\n minPairs: opts.minProductiveRuns,\n binaryScale,\n })\n const regression = decidePairedPromotion(after, before, {\n confidence,\n resamples,\n statistic: bootstrapStatistic,\n seed,\n threshold: floorTolerance,\n minPairs: opts.minProductiveRuns,\n binaryScale,\n })\n const bootstrap =\n improvement.bootstrap ??\n pairedBootstrap(before, after, {\n confidence,\n resamples,\n statistic: bootstrapStatistic,\n seed,\n })\n // A floor breach fires on EITHER burden of proof, because they cover\n // different failures and the floor is the anti-Goodhart guard:\n // - `improvement.low < -floorTolerance` — the CREDIBLE WORST CASE exceeds\n // the tolerance. This is the contract `AxisEvidence.floorTolerance` and\n // `paretoPolicy`'s own reason string state, and it is the conservative\n // posture a safety axis needs: block unless the data can rule the\n // breach out, rather than waiting for the breach to be proven. Read off\n // the DECIDING interval, so a pass/fail axis is not screened by a\n // bootstrap that is pinned wherever ties dominate.\n // - `regression.promote` — a PROVEN drop past the tolerance. Adds the\n // small-sample path, where the decision is an exact sign test because\n // the bootstrap interval is descriptive only.\n // A tied binary axis still has uncertainty about unseen discordant pairs.\n // Its diagnostic bootstrap collapses to zero, so only the deciding score\n // interval can establish that a regression stays within the declared floor.\n const floorBreached = improvement.low < -floorTolerance || regression.promote\n // Floor check precedes the gain check: a credible regression must never be\n // masked as \"improved\". With the defaults (gainThreshold 0, positive floor)\n // the regions are disjoint and order is moot, but a consumer who sets a\n // negative gainThreshold (\"accept small dips\") could otherwise have a real\n // floor breach classified as a gain — anti-Goodhart wins the tie.\n const verdict: AxisVerdict = !improvement.sufficient\n ? 'few_runs'\n : improvement.indeterminate\n ? 'indeterminate'\n : floorBreached\n ? 'regressed'\n : improvement.promote\n ? 'improved'\n : 'flat'\n axes.push({\n name: obj.name,\n source: obj.source,\n direction: obj.direction,\n bootstrap,\n bootstrapStatistic,\n ci: { low: improvement.low, high: improvement.high },\n decisionStatistic: improvement.statistic,\n mcnemar: improvement.mcnemar,\n indeterminate: improvement.indeterminate,\n n,\n minimumRequired: improvement.minimumPairs,\n decisionMethod: improvement.method,\n gainThreshold,\n floorTolerance,\n verdict,\n })\n }\n const ns = axes.map((a) => a.n).filter((n) => n > 0)\n const minN = ns.length > 0 ? Math.min(...ns) : 0\n return { axes, minN, cost: { candidate: ctx.cost.candidate, baseline: ctx.cost.baseline } }\n}\n\n/**\n * Require a supported gain and every configured regression floor to clear.\n * A failed floor holds the candidate, including when uncertainty permits a loss.\n * Missing or indeterminate evidence requires more work; no gain holds release.\n */\nexport const paretoPolicy: PromotionPolicy = (ev) => {\n const contributingGates = ev.axes.map((ax) => ({\n name: `objective:${ax.name}`,\n status:\n ax.verdict === 'regressed'\n ? ('fail' as const)\n : ax.verdict === 'few_runs' || ax.verdict === 'indeterminate'\n ? ('not_evaluated' as const)\n : ('pass' as const),\n detail: {\n direction: ax.direction,\n source: ax.source,\n verdict: ax.verdict,\n n: ax.n,\n deltaMedian: ax.bootstrap.median,\n ciLow: ax.ci.low,\n ciHigh: ax.ci.high,\n decisionStatistic: ax.decisionStatistic,\n decisionMethod: ax.decisionMethod,\n mcnemar: ax.mcnemar,\n indeterminate: ax.indeterminate,\n bootstrapCiLow: ax.bootstrap.low,\n bootstrapCiHigh: ax.bootstrap.high,\n confidence: ax.bootstrap.confidence,\n gainThreshold: ax.gainThreshold,\n floorTolerance: ax.floorTolerance,\n },\n }))\n\n const regressed = ev.axes.filter((a) => a.verdict === 'regressed')\n const insufficient = ev.axes.filter(\n (a) => a.verdict === 'few_runs' || a.verdict === 'indeterminate',\n )\n const improved = ev.axes.filter((a) => a.verdict === 'improved')\n\n let decision: GateDecision\n const reasons: string[] = []\n if (regressed.length > 0) {\n // A gain on another axis cannot excuse an unresolved regression floor.\n decision = 'hold'\n for (const a of regressed) {\n reasons.push(\n `objective '${a.name}' did not clear its regression floor -${a.floorTolerance}: good-direction CI [${a.ci.low.toFixed(3)}, ${a.ci.high.toFixed(3)}] (n=${a.n})`,\n )\n }\n } else if (insufficient.length > 0) {\n // An unresolved axis cannot establish either a gain or a safe floor.\n decision = 'need_more_work'\n for (const a of insufficient) {\n reasons.push(\n a.verdict === 'few_runs'\n ? `objective '${a.name}' has only n=${a.n} paired runs — insufficient evidence to claim significance`\n : `objective '${a.name}' has an indeterminate deciding CI [${a.ci.low}, ${a.ci.high}] — insufficient evidence to clear its regression floor (n=${a.n})`,\n )\n }\n } else if (improved.length > 0) {\n decision = 'ship'\n reasons.push(\n `Supported objective gain: ${improved\n .map(\n (a) =>\n `'${a.name}' +${a.ci.low > 0 ? a.ci.low.toFixed(3) : a.bootstrap.mean.toFixed(3)} (CI.low ${a.ci.low.toFixed(3)})`,\n )\n .join(', ')}; every regression floor cleared`,\n )\n } else {\n // Every floor cleared, but no objective demonstrated a significant gain.\n decision = 'hold'\n reasons.push(\n 'no Pareto improvement: no objective shows a significant gain; every regression floor cleared',\n )\n }\n\n // `delta` surfaces the composite axis if present, else the first axis — a\n // single convenience scalar; the vector lives in `contributingGates`.\n const composite = ev.axes.find((a) => a.source.kind === 'composite') ?? ev.axes[0]\n return { decision, reasons, contributingGates, delta: composite?.bootstrap.median }\n}\n\nexport interface ParetoSignificanceGateOptions extends BuildEvidenceVectorOptions {\n /** The objective vector. Every axis is both a gain source and a safety floor. */\n objectives: PromotionObjective[]\n /** Strategy applied to the evidence vector. Default `paretoPolicy`. Override\n * to run a stricter/looser strategy over the SAME bus (competing policies). */\n policy?: PromotionPolicy\n /** Override the gate name in reports. */\n name?: string\n}\n\n/**\n * Wrap the bus + a policy as a `Gate`. Plugs into the existing\n * `runImprovementLoop({ gate })` slot and composes via `composeGate`; default\n * loop behavior is unchanged because consumers opt in by passing this gate.\n */\nexport function paretoSignificanceGate<TArtifact = unknown, TScenario extends Scenario = Scenario>(\n options: ParetoSignificanceGateOptions,\n): Gate<TArtifact, TScenario> {\n if (options.objectives.length === 0) {\n throw new Error('paretoSignificanceGate: at least 1 objective required')\n }\n const policy = options.policy ?? paretoPolicy\n return {\n name: options.name ?? 'paretoSignificanceGate',\n async decide(ctx: GateContext<TArtifact, TScenario>): Promise<GateResult> {\n const ev = buildEvidenceVector(ctx, options.objectives, options)\n return policy(ev)\n },\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;AAkIA,SAAgB,oBACd,KACA,YACA,OAAmC,CAAC,GACpB;CAChB,IAAI,WAAW,WAAW,GACxB,MAAM,IAAI,MAAM,oDAAoD;CAEtE,MAAM,aAAa,KAAK,cAAc;CACtC,MAAM,YAAY,KAAK,aAAa;CACpC,MAAM,OAAO,KAAK,QAAQ;CAC1B,MAAM,WAAW,IAAI,uBAAuB,IAAI;CAChD,MAAM,cAAc,IAAI,IAAI,IAAI,UAAU,KAAK,MAAM,EAAE,EAAE,CAAC;CAE1D,MAAM,OAAuB,CAAC;CAC9B,KAAK,MAAM,OAAO,YAAY;EAC5B,IAAI;EACJ,IAAI,IAAI,OAAO,SAAS,aACtB,UAAU,MAAM,EAAE;OACb;GACL,MAAM,MAAM,IAAI,OAAO;GACvB,UAAU,MAAM,EAAE,WAAW;EAC/B;EACA,MAAM,SAAS,YAAY,IAAI,aAAa,UAAU,aAAa,MAAM;EAIzE,MAAM,SAAS,IAAI,cAAc,aAAa,OAAO,SAAS,OAAO;EACrE,MAAM,QAAQ,IAAI,cAAc,aAAa,OAAO,QAAQ,OAAO;EACnE,MAAM,IAAI,OAAO,OAAO;EACxB,MAAM,cAAc,IAAI;EACxB,MAAM,iBACJ,IAAI,kBAAkB,OAAQ,eAAe,YAAY,CAAC,GAAG,OAAO,QAAQ,GAAG,OAAO,KAAK,CAAC;EAC9F,MAAM,gBAAgB,IAAI,iBAAiB;EAW3C,MAAM,qBAAqB,KAAK,aAAA;EAMhC,MAAM,cAAc,sBAAsB,QAAQ,OAAO;GACvD;GACA;GACA,WAAW;GACX;GACA,WAAW;GACX,UAAU,KAAK;GACf;EACF,CAAC;EACD,MAAM,aAAa,sBAAsB,OAAO,QAAQ;GACtD;GACA;GACA,WAAW;GACX;GACA,WAAW;GACX,UAAU,KAAK;GACf;EACF,CAAC;EACD,MAAM,YACJ,YAAY,aACZ,gBAAgB,QAAQ,OAAO;GAC7B;GACA;GACA,WAAW;GACX;EACF,CAAC;EAgBH,MAAM,gBAAgB,YAAY,MAAM,CAAC,kBAAkB,WAAW;EAMtE,MAAM,UAAuB,CAAC,YAAY,aACtC,aACA,YAAY,gBACV,kBACA,gBACE,cACA,YAAY,UACV,aACA;EACV,KAAK,KAAK;GACR,MAAM,IAAI;GACV,QAAQ,IAAI;GACZ,WAAW,IAAI;GACf;GACA;GACA,IAAI;IAAE,KAAK,YAAY;IAAK,MAAM,YAAY;GAAK;GACnD,mBAAmB,YAAY;GAC/B,SAAS,YAAY;GACrB,eAAe,YAAY;GAC3B;GACA,iBAAiB,YAAY;GAC7B,gBAAgB,YAAY;GAC5B;GACA;GACA;EACF,CAAC;CACH;CACA,MAAM,KAAK,KAAK,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,QAAQ,MAAM,IAAI,CAAC;CAEnD,OAAO;EAAE;EAAM,MADF,GAAG,SAAS,IAAI,KAAK,IAAI,GAAG,EAAE,IAAI;EAC1B,MAAM;GAAE,WAAW,IAAI,KAAK;GAAW,UAAU,IAAI,KAAK;EAAS;CAAE;AAC5F;;;;;;AAOA,MAAa,gBAAiC,OAAO;CACnD,MAAM,oBAAoB,GAAG,KAAK,KAAK,QAAQ;EAC7C,MAAM,aAAa,GAAG;EACtB,QACE,GAAG,YAAY,cACV,SACD,GAAG,YAAY,cAAc,GAAG,YAAY,kBACzC,kBACA;EACT,QAAQ;GACN,WAAW,GAAG;GACd,QAAQ,GAAG;GACX,SAAS,GAAG;GACZ,GAAG,GAAG;GACN,aAAa,GAAG,UAAU;GAC1B,OAAO,GAAG,GAAG;GACb,QAAQ,GAAG,GAAG;GACd,mBAAmB,GAAG;GACtB,gBAAgB,GAAG;GACnB,SAAS,GAAG;GACZ,eAAe,GAAG;GAClB,gBAAgB,GAAG,UAAU;GAC7B,iBAAiB,GAAG,UAAU;GAC9B,YAAY,GAAG,UAAU;GACzB,eAAe,GAAG;GAClB,gBAAgB,GAAG;EACrB;CACF,EAAE;CAEF,MAAM,YAAY,GAAG,KAAK,QAAQ,MAAM,EAAE,YAAY,WAAW;CACjE,MAAM,eAAe,GAAG,KAAK,QAC1B,MAAM,EAAE,YAAY,cAAc,EAAE,YAAY,eACnD;CACA,MAAM,WAAW,GAAG,KAAK,QAAQ,MAAM,EAAE,YAAY,UAAU;CAE/D,IAAI;CACJ,MAAM,UAAoB,CAAC;CAC3B,IAAI,UAAU,SAAS,GAAG;EAExB,WAAW;EACX,KAAK,MAAM,KAAK,WACd,QAAQ,KACN,cAAc,EAAE,KAAK,wCAAwC,EAAE,eAAe,uBAAuB,EAAE,GAAG,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,GAAG,KAAK,QAAQ,CAAC,EAAE,OAAO,EAAE,EAAE,EAC/J;CAEJ,OAAO,IAAI,aAAa,SAAS,GAAG;EAElC,WAAW;EACX,KAAK,MAAM,KAAK,cACd,QAAQ,KACN,EAAE,YAAY,aACV,cAAc,EAAE,KAAK,eAAe,EAAE,EAAE,8DACxC,cAAc,EAAE,KAAK,sCAAsC,EAAE,GAAG,IAAI,IAAI,EAAE,GAAG,KAAK,6DAA6D,EAAE,EAAE,EACzJ;CAEJ,OAAO,IAAI,SAAS,SAAS,GAAG;EAC9B,WAAW;EACX,QAAQ,KACN,6BAA6B,SAC1B,KACE,MACC,IAAI,EAAE,KAAK,KAAK,EAAE,GAAG,MAAM,IAAI,EAAE,GAAG,IAAI,QAAQ,CAAC,IAAI,EAAE,UAAU,KAAK,QAAQ,CAAC,EAAE,WAAW,EAAE,GAAG,IAAI,QAAQ,CAAC,EAAE,EACpH,CAAC,CACA,KAAK,IAAI,EAAE,iCAChB;CACF,OAAO;EAEL,WAAW;EACX,QAAQ,KACN,8FACF;CACF;CAIA,MAAM,YAAY,GAAG,KAAK,MAAM,MAAM,EAAE,OAAO,SAAS,WAAW,KAAK,GAAG,KAAK;CAChF,OAAO;EAAE;EAAU;EAAS;EAAmB,OAAO,WAAW,UAAU;CAAO;AACpF;;;;;;AAiBA,SAAgB,uBACd,SAC4B;CAC5B,IAAI,QAAQ,WAAW,WAAW,GAChC,MAAM,IAAI,MAAM,uDAAuD;CAEzE,MAAM,SAAS,QAAQ,UAAU;CACjC,OAAO;EACL,MAAM,QAAQ,QAAQ;EACtB,MAAM,OAAO,KAA6D;GACxE,MAAM,KAAK,oBAAoB,KAAK,QAAQ,YAAY,OAAO;GAC/D,OAAO,OAAO,EAAE;EAClB;CACF;AACF"}
|
package/dist/rl.d.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { v as Span } from "./schema-2g7trUwH.js";
|
|
2
2
|
import { a as RunRecord, c as RunSplitTag } from "./run-record-DF-fuu31.js";
|
|
3
3
|
import { a as PairedPromotionDecision, d as PairedBootstrapResult } from "./paired-promotion-decision-DPKz8ZnF.js";
|
|
4
|
-
import { a as CampaignResult } from "./types-BmQRs661.js";
|
|
5
4
|
import { s as TraceStore } from "./store-C4avFjIi.js";
|
|
6
5
|
import { a as MintedRolloutLine, t as ChatMessage, y as RolloutSplit } from "./schema-B5LtYvn9.js";
|
|
7
6
|
import { t as MintRolloutOptions } from "./mint-C9Lojt1V.js";
|
|
7
|
+
import { a as CampaignResult } from "./types-BWwRXofp.js";
|
|
8
8
|
import { s as VerificationReport } from "./multi-layer-verifier-BUaQ4C17.js";
|
|
9
9
|
import { a as detectRewardHacking, c as VerifiableRewardSource, d as filterDeterministicallyRewarded, i as RewardHackingSignal, l as extractVerifiableReward, n as RewardHackingFinding, o as VerifiableReward, r as RewardHackingReport, s as VerifiableRewardExtractionOptions, t as DetectRewardHackingInput, u as extractVerifiableRewardsFromRecords } from "./reward-hacking-Cyk-QHk9.js";
|
|
10
10
|
import { i as InMemoryOutcomeStore, n as FileSystemOutcomeStore, o as OutcomeStore, r as FileSystemOutcomeStoreOptions, t as DeploymentOutcome } from "./outcome-store-CNt4iZ67.js";
|
package/dist/rl.js
CHANGED
|
@@ -4,7 +4,7 @@ import { a as medianInPlace } from "./internal-BMFSR8Ns.js";
|
|
|
4
4
|
import { t as mulberry32 } from "./random-Dn5fPWkt.js";
|
|
5
5
|
import { l as wilcoxonSignedRank } from "./paired-arms-JSDcs2zx.js";
|
|
6
6
|
import { r as pairedBootstrap } from "./paired-tests-C8iCsioC.js";
|
|
7
|
-
import { t as decidePairedPromotion } from "./paired-promotion-decision-
|
|
7
|
+
import { t as decidePairedPromotion } from "./paired-promotion-decision-DTKl6-G8.js";
|
|
8
8
|
import { r as observedSplitScore, s as trainingScore } from "./reward-nw2xZGZG.js";
|
|
9
9
|
import { c as runTaskScore } from "./run-record-BRlM99rX.js";
|
|
10
10
|
import { n as campaignCellToRunRecord } from "./run-record-tNnlQIrD.js";
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { a as hashCanonical, i as compareCodeUnits, r as canonicalString } from "./canonical-BsZVgBhW.js";
|
|
2
2
|
import { i as modelHasSnapshot, l as searchCellRunId } from "./run-record-BRlM99rX.js";
|
|
3
|
-
import { n as replayLedgerText, t as FileLedgerJournal } from "./journal-
|
|
3
|
+
import { n as MemoryLedgerJournal, r as replayLedgerText, t as FileLedgerJournal } from "./journal-DvKCvjty.js";
|
|
4
4
|
import "./ledger-core/index.js";
|
|
5
|
-
import { a as SearchLedgerError, i as SearchLedgerConflictError, o as SearchLedgerIntegrityError, t as SEARCH_LEDGER_FILE_CONTEXT } from "./search-ledger-file-
|
|
5
|
+
import { a as SearchLedgerError, i as SearchLedgerConflictError, o as SearchLedgerIntegrityError, t as SEARCH_LEDGER_FILE_CONTEXT } from "./search-ledger-file-DY0AXX3W.js";
|
|
6
6
|
import { n as evaluationClaimSchema } from "./claim-V9Zqbq0J.js";
|
|
7
7
|
import { createHash } from "node:crypto";
|
|
8
8
|
import { z } from "zod";
|
|
@@ -14,14 +14,6 @@ function compareStrings(a, b) {
|
|
|
14
14
|
return a < b ? -1 : a > b ? 1 : 0;
|
|
15
15
|
}
|
|
16
16
|
//#endregion
|
|
17
|
-
//#region src/campaign/search-ledger-types.ts
|
|
18
|
-
/**
|
|
19
|
-
* Schema tag every search-ledger entry carries. A ledger written under another
|
|
20
|
-
* tag, such as the retired `tangle.search-ledger.v1` candidate-slot format, is
|
|
21
|
-
* refused with its tag named; it is never translated.
|
|
22
|
-
*/
|
|
23
|
-
const SEARCH_LEDGER_SCHEMA = "tangle.search-ledger.2026-09";
|
|
24
|
-
//#endregion
|
|
25
17
|
//#region src/campaign/search-state.ts
|
|
26
18
|
/**
|
|
27
19
|
* `SearchState`: the one projection of a search ledger.
|
|
@@ -116,6 +108,10 @@ var SearchState = class {
|
|
|
116
108
|
operations = /* @__PURE__ */ new Map();
|
|
117
109
|
selectedNodeId = null;
|
|
118
110
|
claimUsedUsd = 0;
|
|
111
|
+
/** Nodes ever decided `finalist`. */
|
|
112
|
+
finalists = /* @__PURE__ */ new Set();
|
|
113
|
+
/** Claim cells allocated. Once one exists, the finalists are fixed. */
|
|
114
|
+
claimCells = 0;
|
|
119
115
|
audit;
|
|
120
116
|
constructor(searchId) {
|
|
121
117
|
this.searchId = searchId;
|
|
@@ -263,12 +259,15 @@ var SearchState = class {
|
|
|
263
259
|
}
|
|
264
260
|
startOperation(event) {
|
|
265
261
|
if (this.operations.has(event.operationId)) throw integrity(`operation ${event.operationId} was started twice`);
|
|
262
|
+
this.assertSearching(`operation ${event.operationId}`);
|
|
266
263
|
this.admit(event.reservation, false, `operation ${event.operationId}`);
|
|
267
264
|
this.operations.set(event.operationId, {
|
|
268
265
|
operationId: event.operationId,
|
|
269
266
|
operationKind: event.operationKind,
|
|
270
267
|
reservation: event.reservation,
|
|
271
268
|
recorded: false,
|
|
269
|
+
outcome: null,
|
|
270
|
+
artifacts: event.artifacts,
|
|
272
271
|
spentUsd: 0
|
|
273
272
|
});
|
|
274
273
|
this.audit.operations.started += 1;
|
|
@@ -286,6 +285,8 @@ var SearchState = class {
|
|
|
286
285
|
this.audit.spend.openReservationUsd -= reserved;
|
|
287
286
|
if (operation.reservation) this.audit.spend.overspendUsd += Math.max(0, cost.usd - reserved);
|
|
288
287
|
operation.recorded = true;
|
|
288
|
+
operation.outcome = event.outcome.status;
|
|
289
|
+
operation.artifacts = [...operation.artifacts, ...event.artifacts];
|
|
289
290
|
operation.spentUsd = cost.usd;
|
|
290
291
|
this.audit.operations.recorded += 1;
|
|
291
292
|
this.audit.operations.open -= 1;
|
|
@@ -294,6 +295,7 @@ var SearchState = class {
|
|
|
294
295
|
const expected = searchNodeId(this.searchId, event.artifactDigest);
|
|
295
296
|
if (event.nodeId !== expected) throw integrity(`node ${event.nodeId} does not match its artifact digest (expected ${expected})`);
|
|
296
297
|
if (this.nodes.has(event.nodeId)) throw integrity(`node ${event.nodeId} was registered twice; a re-proposal is a second edge into it`);
|
|
298
|
+
this.assertSearching(`node ${event.nodeId}`);
|
|
297
299
|
assertUnique(event.surfaces.map((surface) => surface.surfaceId), "surfaceId", event.eventId);
|
|
298
300
|
const maxNodes = this.header.budget.maxNodes;
|
|
299
301
|
if (maxNodes !== null && this.nodes.size + 1 > maxNodes) throw integrity(`node ${event.nodeId} exceeds the search's maxNodes ${maxNodes}`);
|
|
@@ -329,6 +331,7 @@ var SearchState = class {
|
|
|
329
331
|
if (this.edges.has(event.edgeId)) throw integrity(`edge ${event.edgeId} was recorded twice`);
|
|
330
332
|
const child = this.nodes.get(event.childNodeId);
|
|
331
333
|
if (!child) throw integrity(`edge ${event.edgeId} names unregistered child ${event.childNodeId}`);
|
|
334
|
+
this.assertSearching(`edge ${event.edgeId}`);
|
|
332
335
|
if (event.label.length > MAX_LABEL_CHARS) throw integrity(`edge ${event.edgeId} label exceeds ${MAX_LABEL_CHARS} characters`);
|
|
333
336
|
if (event.diffs.length !== event.parents.length) throw integrity(`edge ${event.edgeId} needs one diff per parent`);
|
|
334
337
|
assertUnique(event.parents.map((parent) => canonicalString(parent)), "parent", event.eventId);
|
|
@@ -352,6 +355,7 @@ var SearchState = class {
|
|
|
352
355
|
if (!record) throw integrity(`edge ${event.edgeId} names unregistered parent ${parent.nodeId}`);
|
|
353
356
|
if (record.ordinal >= child.ordinal) throw integrity(`edge ${event.edgeId}: parent ${parent.nodeId} registered after child ${child.event.nodeId}`);
|
|
354
357
|
if (record.edgeIds.length === 0) throw integrity(`edge ${event.edgeId}: parent ${parent.nodeId} has no edge of its own yet`);
|
|
358
|
+
if (record.decisions.at(-1)?.decision.status === "invalid") throw integrity(`edge ${event.edgeId}: parent ${parent.nodeId} was decided invalid, and an invalid node never becomes a parent`);
|
|
355
359
|
inSearch.push(record);
|
|
356
360
|
}
|
|
357
361
|
if (child.edgeIds.length === 0) {
|
|
@@ -385,6 +389,7 @@ var SearchState = class {
|
|
|
385
389
|
if (task.unitId !== event.unitId) throw integrity(`cell ${event.cellId}: task ${event.taskId} samples unit ${task.unitId}`);
|
|
386
390
|
assertStageSplit(event.cellId, event.stage, event.split, node.ordinal === 0);
|
|
387
391
|
if (event.split === "test" && node.ordinal !== 0 && !node.finalist) throw integrity(`cell ${event.cellId}: the test split is sealed; only the root and finalists run on it`);
|
|
392
|
+
if (event.stage !== "claim") this.assertSearching(`cell ${event.cellId}`);
|
|
388
393
|
const maxCells = this.header.budget.maxCells;
|
|
389
394
|
if (maxCells !== null && this.audit.cells.allocated + 1 > maxCells) throw integrity(`cell ${event.cellId} exceeds the search's maxCells ${maxCells}`);
|
|
390
395
|
this.admit(event.reservation, event.stage === "claim", `cell ${event.cellId}`);
|
|
@@ -415,7 +420,10 @@ var SearchState = class {
|
|
|
415
420
|
this.audit.cells.allocated += 1;
|
|
416
421
|
this.audit.cells.open += 1;
|
|
417
422
|
this.audit.spend.openReservationUsd += event.reservation?.usd ?? 0;
|
|
418
|
-
if (event.stage === "claim")
|
|
423
|
+
if (event.stage === "claim") {
|
|
424
|
+
this.claimUsedUsd += event.reservation?.usd ?? 0;
|
|
425
|
+
this.claimCells += 1;
|
|
426
|
+
}
|
|
419
427
|
}
|
|
420
428
|
settleCell(event, index) {
|
|
421
429
|
const cell = this.cells.get(event.cellId);
|
|
@@ -493,6 +501,12 @@ var SearchState = class {
|
|
|
493
501
|
if (!node) throw integrity(`decision ${event.eventId} names unregistered node ${event.nodeId}`);
|
|
494
502
|
if (node.edgeIds.length === 0) throw integrity(`decision ${event.eventId}: node ${event.nodeId} has no edge yet`);
|
|
495
503
|
const status = event.decision.status;
|
|
504
|
+
if (status === "finalist" && !this.finalists.has(event.nodeId)) {
|
|
505
|
+
if (node.ordinal === 0) throw integrity(`the root ${event.nodeId} is the claim's control and cannot be a finalist`);
|
|
506
|
+
if (this.claimCells > 0) throw integrity(`node ${event.nodeId} was decided finalist after a claim cell ran; the finalists are fixed before the test`);
|
|
507
|
+
if (this.finalists.size >= 3) throw integrity(`node ${event.nodeId} would be finalist ${this.finalists.size + 1}; at most 3 are allowed`);
|
|
508
|
+
this.finalists.add(event.nodeId);
|
|
509
|
+
}
|
|
496
510
|
if (status === "selected") {
|
|
497
511
|
if (this.selectedNodeId !== null && this.selectedNodeId !== event.nodeId) throw integrity(`node ${event.nodeId} was selected while ${this.selectedNodeId} is still selected`);
|
|
498
512
|
if (node.scoredCells === 0) throw integrity(`node ${event.nodeId} cannot be selected without a scored cell`);
|
|
@@ -524,20 +538,40 @@ var SearchState = class {
|
|
|
524
538
|
this.audit.spend.openReservationUsd = 0;
|
|
525
539
|
}
|
|
526
540
|
checkClaim(claim) {
|
|
527
|
-
|
|
528
|
-
assertUnique(
|
|
529
|
-
for (const
|
|
530
|
-
|
|
541
|
+
const named = claim.finalists.map((finalist) => finalist.nodeId);
|
|
542
|
+
assertUnique(named, "claim finalist", "search-closed");
|
|
543
|
+
for (const nodeId of named) if (!this.finalists.has(nodeId)) throw integrity(`claim finalist ${nodeId} was never decided finalist`);
|
|
544
|
+
for (const nodeId of this.finalists) if (!named.includes(nodeId)) throw integrity(`finalist ${nodeId} is missing from the claim`);
|
|
531
545
|
const root = this.nodeOrder[0] ?? null;
|
|
546
|
+
const perFinalist = 1 - (1 - claim.confidence) / Math.max(1, named.length);
|
|
547
|
+
for (const finalist of claim.finalists) {
|
|
548
|
+
const { estimate, test } = finalist;
|
|
549
|
+
if (estimate && (estimate.against !== root || estimate.split !== "test")) throw integrity(`claim finalist ${finalist.nodeId} is estimated against something other than the root on test`);
|
|
550
|
+
if (test && Math.abs(test.confidence - perFinalist) > 1e-12) throw integrity(`claim finalist ${finalist.nodeId} is tested at confidence ${test.confidence}; ${named.length} finalists at family-wise ${claim.confidence} need ${perFinalist}`);
|
|
551
|
+
if (finalist.promote && test === null) throw integrity(`claim finalist ${finalist.nodeId} is promoted without a test`);
|
|
552
|
+
}
|
|
553
|
+
if (claim.selected !== this.selectedNodeId) throw integrity(`the claim selects ${claim.selected ?? "nothing"} but the selected node is ${this.selectedNodeId ?? "none"}`);
|
|
532
554
|
if (claim.decision === "ship") {
|
|
533
555
|
const shipped = claim.finalists.find((finalist) => finalist.nodeId === claim.selected);
|
|
534
556
|
if (claim.selected === null || claim.selected === root || !shipped?.promote) throw integrity("a ship claim must select a finalist that the paired test promoted");
|
|
535
557
|
if (!this.header.splits.heldOutUnits) throw integrity("a ship claim needs test units held out from train and selection");
|
|
558
|
+
if ("unknown" in this.header.objective.judge) throw integrity("a ship claim needs a pinned judge; with an unknown judge a resumed search could mix two judges' verdicts");
|
|
559
|
+
const testUnits = new Set(this.header.splits.test.tasks.map((task) => task.unitId));
|
|
560
|
+
for (const nodeId of [root, claim.selected]) {
|
|
561
|
+
const scored = new Set(this.readScoredCells(nodeId, "test").map((cell) => cell.unitId));
|
|
562
|
+
const missing = [...testUnits].filter((unit) => !scored.has(unit));
|
|
563
|
+
if (missing.length > 0) throw integrity(`a ship claim needs every test unit scored by the root and the finalist; ${nodeId} has none on ${missing.join(", ")}`);
|
|
564
|
+
}
|
|
536
565
|
} else if (claim.selected !== null && claim.selected !== root) throw integrity(`a ${claim.decision} claim keeps the root; it cannot select another node`);
|
|
537
566
|
if (claim.decision === "test-cannot-resolve") {
|
|
538
567
|
for (const cell of this.cells.values()) if (cell.split === "test" && cell.attempts > 0) throw integrity("a test-cannot-resolve claim spends nothing on test cells");
|
|
539
568
|
}
|
|
540
569
|
}
|
|
570
|
+
/** Once the first finalist is decided, the search only runs its claim:
|
|
571
|
+
* nothing new is proposed, registered or allocated outside the test. */
|
|
572
|
+
assertSearching(subject) {
|
|
573
|
+
if (this.finalists.size > 0 || this.claimCells > 0) throw integrity(`${subject} follows the start of the claim; the search stopped when its finalists were decided`);
|
|
574
|
+
}
|
|
541
575
|
/** @internal */
|
|
542
576
|
incompleteReasons() {
|
|
543
577
|
const reasons = [];
|
|
@@ -587,6 +621,24 @@ var SearchState = class {
|
|
|
587
621
|
const remainingClaim = Math.max(0, reservedClaimUsd - this.claimUsedUsd);
|
|
588
622
|
if (this.audit.spend.committedUsd + this.audit.spend.openReservationUsd + (claim ? Math.max(0, remainingClaim - hold) : remainingClaim) + hold > maxUsd + USD_TOLERANCE) throw integrity(`${subject} is not admissible: committed $${round(this.audit.spend.committedUsd)} + open $${round(this.audit.spend.openReservationUsd)} + claim reserve $${round(remainingClaim)} + hold $${round(hold)} exceeds the cap $${maxUsd}`);
|
|
589
623
|
}
|
|
624
|
+
/** @internal The largest hold a new non-claim cell or operation may take,
|
|
625
|
+
* and the claim reserve still unspent. */
|
|
626
|
+
readBudget() {
|
|
627
|
+
const { maxUsd, reservedClaimUsd } = this.header?.budget ?? {
|
|
628
|
+
maxUsd: null,
|
|
629
|
+
reservedClaimUsd: 0
|
|
630
|
+
};
|
|
631
|
+
const claimReserveUsd = Math.max(0, reservedClaimUsd - this.claimUsedUsd);
|
|
632
|
+
if (maxUsd === null) return {
|
|
633
|
+
headroomUsd: null,
|
|
634
|
+
claimReserveUsd
|
|
635
|
+
};
|
|
636
|
+
const used = this.audit.spend.committedUsd + this.audit.spend.openReservationUsd + claimReserveUsd;
|
|
637
|
+
return {
|
|
638
|
+
headroomUsd: Math.max(0, maxUsd - used),
|
|
639
|
+
claimReserveUsd
|
|
640
|
+
};
|
|
641
|
+
}
|
|
590
642
|
/** @internal */
|
|
591
643
|
readHeader() {
|
|
592
644
|
return this.header;
|
|
@@ -685,6 +737,13 @@ var SearchStateView = class {
|
|
|
685
737
|
lastOccurredAt;
|
|
686
738
|
audit;
|
|
687
739
|
completion;
|
|
740
|
+
/**
|
|
741
|
+
* The admission rule's room: `headroomUsd` is the largest reservation a new
|
|
742
|
+
* non-claim cell or operation may hold (committed spend, open holds and the
|
|
743
|
+
* unspent claim reserve all count against the cap), or null when the search
|
|
744
|
+
* declares no cap. `claimReserveUsd` is what claim cells may still draw.
|
|
745
|
+
*/
|
|
746
|
+
budget;
|
|
688
747
|
state;
|
|
689
748
|
version;
|
|
690
749
|
constructor(state, version) {
|
|
@@ -696,6 +755,7 @@ var SearchStateView = class {
|
|
|
696
755
|
this.head = state.readHead();
|
|
697
756
|
this.lastOccurredAt = state.readLastOccurredAt();
|
|
698
757
|
this.audit = state.readAudit();
|
|
758
|
+
this.budget = state.readBudget();
|
|
699
759
|
const reasons = state.incompleteReasons();
|
|
700
760
|
if (!this.closed) reasons.unshift("search is open");
|
|
701
761
|
this.completion = {
|
|
@@ -826,6 +886,14 @@ function integrity(message) {
|
|
|
826
886
|
return new SearchLedgerIntegrityError(message);
|
|
827
887
|
}
|
|
828
888
|
//#endregion
|
|
889
|
+
//#region src/campaign/search-ledger-types.ts
|
|
890
|
+
/**
|
|
891
|
+
* Schema tag every search-ledger entry carries. A ledger written under another
|
|
892
|
+
* tag, such as the retired `tangle.search-ledger.v1` candidate-slot format, is
|
|
893
|
+
* refused with its tag named; it is never translated.
|
|
894
|
+
*/
|
|
895
|
+
const SEARCH_LEDGER_SCHEMA = "tangle.search-ledger.2026-09";
|
|
896
|
+
//#endregion
|
|
829
897
|
//#region src/campaign/search-ledger.ts
|
|
830
898
|
/**
|
|
831
899
|
* Durable, append-only event log of one search: nodes (content-addressed
|
|
@@ -1243,6 +1311,48 @@ const NodeDecidedSchema = z.object({
|
|
|
1243
1311
|
rule: NON_EMPTY,
|
|
1244
1312
|
reason: NON_EMPTY
|
|
1245
1313
|
}).strict();
|
|
1314
|
+
const CONFIDENCE = z.number().gt(0).lt(1);
|
|
1315
|
+
const ClaimEstimatorSchema = z.discriminatedUnion("kind", [z.object({ kind: z.literal("continuous") }).strict(), z.object({
|
|
1316
|
+
kind: z.literal("binary"),
|
|
1317
|
+
scale: z.number().finite().positive()
|
|
1318
|
+
}).strict()]);
|
|
1319
|
+
const ClaimSchema = z.object({
|
|
1320
|
+
rule: SourceRefSchema,
|
|
1321
|
+
confidence: CONFIDENCE,
|
|
1322
|
+
power: z.union([z.object({
|
|
1323
|
+
adequate: z.boolean(),
|
|
1324
|
+
minimumEffect: z.number().finite().positive(),
|
|
1325
|
+
powerAtMinimumEffect: z.number().min(0).max(1),
|
|
1326
|
+
targetPower: z.number().gt(0).max(1),
|
|
1327
|
+
units: NON_NEGATIVE_INT,
|
|
1328
|
+
finalists: POSITIVE_INT,
|
|
1329
|
+
pooledVariance: z.number().finite().nonnegative(),
|
|
1330
|
+
estimator: ClaimEstimatorSchema
|
|
1331
|
+
}).strict(), UnknownRefSchema]),
|
|
1332
|
+
finalists: z.array(z.object({
|
|
1333
|
+
nodeId: NODE_ID,
|
|
1334
|
+
estimate: NodeEstimateSchema.nullable(),
|
|
1335
|
+
test: z.object({
|
|
1336
|
+
pairs: NON_NEGATIVE_INT,
|
|
1337
|
+
confidence: CONFIDENCE,
|
|
1338
|
+
method: z.enum([
|
|
1339
|
+
"score-interval",
|
|
1340
|
+
"bootstrap-ci",
|
|
1341
|
+
"exact-sign"
|
|
1342
|
+
]),
|
|
1343
|
+
delta: FINITE_NUMBER,
|
|
1344
|
+
interval: z.tuple([FINITE_NUMBER, FINITE_NUMBER])
|
|
1345
|
+
}).strict().nullable(),
|
|
1346
|
+
promote: z.boolean()
|
|
1347
|
+
}).strict()),
|
|
1348
|
+
selected: NODE_ID.nullable(),
|
|
1349
|
+
decision: z.enum([
|
|
1350
|
+
"ship",
|
|
1351
|
+
"hold",
|
|
1352
|
+
"test-cannot-resolve"
|
|
1353
|
+
]),
|
|
1354
|
+
reason: NON_EMPTY
|
|
1355
|
+
}).strict();
|
|
1246
1356
|
const SearchClosedSchema = z.object({
|
|
1247
1357
|
...EventBaseShape,
|
|
1248
1358
|
kind: z.literal("search-closed"),
|
|
@@ -1254,25 +1364,7 @@ const SearchClosedSchema = z.object({
|
|
|
1254
1364
|
"converged",
|
|
1255
1365
|
"aborted"
|
|
1256
1366
|
]),
|
|
1257
|
-
claim:
|
|
1258
|
-
power: z.union([z.object({
|
|
1259
|
-
adequate: z.boolean(),
|
|
1260
|
-
minimumEffect: z.number().finite().positive(),
|
|
1261
|
-
powerAtMinimumEffect: z.number().min(0).max(1),
|
|
1262
|
-
units: NON_NEGATIVE_INT
|
|
1263
|
-
}).strict(), UnknownRefSchema]),
|
|
1264
|
-
finalists: z.array(z.object({
|
|
1265
|
-
nodeId: NODE_ID,
|
|
1266
|
-
estimate: NodeEstimateSchema.nullable(),
|
|
1267
|
-
promote: z.boolean()
|
|
1268
|
-
}).strict()),
|
|
1269
|
-
selected: NODE_ID.nullable(),
|
|
1270
|
-
decision: z.enum([
|
|
1271
|
-
"ship",
|
|
1272
|
-
"hold",
|
|
1273
|
-
"test-cannot-resolve"
|
|
1274
|
-
])
|
|
1275
|
-
}).strict().nullable()
|
|
1367
|
+
claim: ClaimSchema.nullable()
|
|
1276
1368
|
}).strict();
|
|
1277
1369
|
const EventSchema = z.discriminatedUnion("kind", [
|
|
1278
1370
|
SearchOpenedSchema,
|
|
@@ -1304,6 +1396,7 @@ function validateSearchLedgerEvent(input) {
|
|
|
1304
1396
|
/** Open a durable filesystem search ledger. Construction performs no I/O; the
|
|
1305
1397
|
* first `append` or `state` verifies the whole existing file. */
|
|
1306
1398
|
function openSearchLedger(options) {
|
|
1399
|
+
if (options.store) return new MemorySearchLedger(options.path, options.searchId, options.store);
|
|
1307
1400
|
return new FileSearchLedger(options.path, options.searchId, options.trustedHead);
|
|
1308
1401
|
}
|
|
1309
1402
|
/** Verify and project immutable search-ledger JSONL through the same codec as
|
|
@@ -1386,6 +1479,41 @@ var FileSearchLedger = class {
|
|
|
1386
1479
|
return this.journal.clearTrustedHead();
|
|
1387
1480
|
}
|
|
1388
1481
|
};
|
|
1482
|
+
/** A search ledger held in an in-process text store. */
|
|
1483
|
+
var MemorySearchLedger = class {
|
|
1484
|
+
path;
|
|
1485
|
+
searchId;
|
|
1486
|
+
trustedHeadPath;
|
|
1487
|
+
journal;
|
|
1488
|
+
constructor(path, searchId, store) {
|
|
1489
|
+
if (path.trim().length === 0) throw new SearchLedgerError("ledger path is empty");
|
|
1490
|
+
if (searchId.length === 0 || searchId.trim() !== searchId) throw new SearchLedgerError("searchId must be non-empty without surrounding whitespace");
|
|
1491
|
+
this.path = path;
|
|
1492
|
+
this.searchId = searchId;
|
|
1493
|
+
this.trustedHeadPath = `${path}.head`;
|
|
1494
|
+
this.journal = new MemoryLedgerJournal(path, searchLedgerCodec(searchId), store);
|
|
1495
|
+
}
|
|
1496
|
+
async state() {
|
|
1497
|
+
return this.journal.replay();
|
|
1498
|
+
}
|
|
1499
|
+
async append(input) {
|
|
1500
|
+
const { entry, appended, projection } = await this.journal.append(validateSearchLedgerEvent(input));
|
|
1501
|
+
return {
|
|
1502
|
+
entry,
|
|
1503
|
+
appended,
|
|
1504
|
+
state: projection
|
|
1505
|
+
};
|
|
1506
|
+
}
|
|
1507
|
+
async trustedHead() {
|
|
1508
|
+
return null;
|
|
1509
|
+
}
|
|
1510
|
+
async pinTrustedHead() {
|
|
1511
|
+
throw new SearchLedgerError(`search ledger ${this.path} is held in memory; a trusted head protects only a file`);
|
|
1512
|
+
}
|
|
1513
|
+
async clearTrustedHead() {
|
|
1514
|
+
return { removed: false };
|
|
1515
|
+
}
|
|
1516
|
+
};
|
|
1389
1517
|
function parseSearchLedgerEntry(raw, context) {
|
|
1390
1518
|
const schema = raw?.schema;
|
|
1391
1519
|
if (schema !== "tangle.search-ledger.2026-09") throw new SearchLedgerIntegrityError(`search ledger ${context.path} line ${context.line} was written under schema ${JSON.stringify(schema)}; this version reads only ${SEARCH_LEDGER_SCHEMA} and does not translate other formats`);
|
|
@@ -1446,6 +1574,6 @@ function formatZodError(error) {
|
|
|
1446
1574
|
return error.issues.map((issue) => `${issue.path.length > 0 ? issue.path.join(".") : "<root>"}: ${issue.message}`).join("; ");
|
|
1447
1575
|
}
|
|
1448
1576
|
//#endregion
|
|
1449
|
-
export { validateSearchLedgerEvent as a,
|
|
1577
|
+
export { validateSearchLedgerEvent as a, SearchStateView as c, searchTaskSetDigest as d, searchUnitScores as f, replaySearchLedgerText as i, searchCellId as l, openSearchLedger as n, SEARCH_LEDGER_SCHEMA as o, parseSearchLedgerLine as r, SearchState as s, FileSearchLedger as t, searchNodeId as u };
|
|
1450
1578
|
|
|
1451
|
-
//# sourceMappingURL=search-ledger-
|
|
1579
|
+
//# sourceMappingURL=search-ledger-CJiAEU3j.js.map
|