@tangle-network/agent-eval 0.192.0 → 0.193.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/dist/adapters/http.d.ts +2 -2
  3. package/dist/analyst/index.d.ts +5 -5
  4. package/dist/analyst/index.js +1 -1
  5. package/dist/{analyze-runs-SCSWYpy0.js → analyze-runs-B6gv7vAT.js} +2 -2
  6. package/dist/{analyze-runs-SCSWYpy0.js.map → analyze-runs-B6gv7vAT.js.map} +1 -1
  7. package/dist/{benchmark-command-C458yEFK.js → benchmark-command-DVhUPU7H.js} +2 -2
  8. package/dist/{benchmark-command-C458yEFK.js.map → benchmark-command-DVhUPU7H.js.map} +1 -1
  9. package/dist/benchmarks/index.d.ts +3 -3
  10. package/dist/benchmarks/index.js +2 -2
  11. package/dist/campaign/index.d.ts +5 -5
  12. package/dist/campaign/index.js +10 -9
  13. package/dist/{campaign-BVQVBEfH.js → campaign-DdE6S80e.js} +9 -8
  14. package/dist/{campaign-BVQVBEfH.js.map → campaign-DdE6S80e.js.map} +1 -1
  15. package/dist/{campaign-evidence-DcQiYhJI.js → campaign-evidence-DJc60F9C.js} +2 -2
  16. package/dist/{campaign-evidence-DcQiYhJI.js.map → campaign-evidence-DJc60F9C.js.map} +1 -1
  17. package/dist/cli.js +47 -10
  18. package/dist/cli.js.map +1 -1
  19. package/dist/{client-wFyDisxi.d.ts → client-dMMKoQNS.d.ts} +2 -2
  20. package/dist/{client-wFyDisxi.d.ts.map → client-dMMKoQNS.d.ts.map} +1 -1
  21. package/dist/contract/index.d.ts +7 -7
  22. package/dist/contract/index.js +8 -8
  23. package/dist/{default-registry-DdqA_-_o.d.ts → default-registry-C3HTE08O.d.ts} +3 -3
  24. package/dist/{default-registry-DdqA_-_o.d.ts.map → default-registry-C3HTE08O.d.ts.map} +1 -1
  25. package/dist/{define-agent-eval-DcludNJS.d.ts → define-agent-eval-DQJvVa1P.d.ts} +6 -6
  26. package/dist/{define-agent-eval-DcludNJS.d.ts.map → define-agent-eval-DQJvVa1P.d.ts.map} +1 -1
  27. package/dist/{define-agent-eval-DQuos3Aw.js → define-agent-eval-DkPbnJEY.js} +5 -5
  28. package/dist/{define-agent-eval-DQuos3Aw.js.map → define-agent-eval-DkPbnJEY.js.map} +1 -1
  29. package/dist/diagnosis/index.js +1 -1
  30. package/dist/{engine-C1DIAEpn.d.ts → engine-DjRV55F9.d.ts} +2 -2
  31. package/dist/{engine-C1DIAEpn.d.ts.map → engine-DjRV55F9.d.ts.map} +1 -1
  32. package/dist/evaluation.d.ts +1 -1
  33. package/dist/experiment/index.d.ts +2 -2
  34. package/dist/experiment/index.js +4 -4
  35. package/dist/hosted/index.d.ts +2 -2
  36. package/dist/hosted/index.js +1 -1
  37. package/dist/{index-DBoXFaz0.d.ts → index-0jEtpSM7.d.ts} +3 -3
  38. package/dist/{index-DBoXFaz0.d.ts.map → index-0jEtpSM7.d.ts.map} +1 -1
  39. package/dist/{index-DwQxdWve.d.ts → index-DMPLG226.d.ts} +89 -272
  40. package/dist/index-DMPLG226.d.ts.map +1 -0
  41. package/dist/{index-DNVzu9Vv.d.ts → index-DWQNDqPd.d.ts} +7 -7
  42. package/dist/{index-DNVzu9Vv.d.ts.map → index-DWQNDqPd.d.ts.map} +1 -1
  43. package/dist/index.d.ts +9 -9
  44. package/dist/index.js +7 -290
  45. package/dist/index.js.map +1 -1
  46. package/dist/jev.d.ts +1 -1
  47. package/dist/ledger-core/index.d.ts +1 -1
  48. package/dist/{llm-judge-Z6MkTkd0.js → llm-judge-ChbWfeWl.js} +1021 -234
  49. package/dist/llm-judge-ChbWfeWl.js.map +1 -0
  50. package/dist/{matrix-CQMMrEQ5.d.ts → matrix-CNu13gg8.d.ts} +2 -2
  51. package/dist/{matrix-CQMMrEQ5.d.ts.map → matrix-CNu13gg8.d.ts.map} +1 -1
  52. package/dist/multishot/golden/index.d.ts +1 -1
  53. package/dist/multishot/index.d.ts +2 -2
  54. package/dist/openapi.json +1 -1
  55. package/dist/paired-delta-test-BF-FEP8V.js +78 -0
  56. package/dist/paired-delta-test-BF-FEP8V.js.map +1 -0
  57. package/dist/{paired-promotion-decision-6cn3bLc3.js → paired-promotion-decision-DTKl6-G8.js} +4 -77
  58. package/dist/paired-promotion-decision-DTKl6-G8.js.map +1 -0
  59. package/dist/{produced-state-CVmCPQDJ.js → produced-state-BqeIaIp2.js} +3 -3
  60. package/dist/{produced-state-CVmCPQDJ.js.map → produced-state-BqeIaIp2.js.map} +1 -1
  61. package/dist/{promotion-policy-CfMNUro2.js → promotion-policy-Cy5OV8BV.js} +3 -3
  62. package/dist/{promotion-policy-CfMNUro2.js.map → promotion-policy-Cy5OV8BV.js.map} +1 -1
  63. package/dist/rl.d.ts +1 -1
  64. package/dist/rl.js +1 -1
  65. package/dist/{search-ledger-Ck7tok0h.js → search-ledger-CJiAEU3j.js} +89 -27
  66. package/dist/search-ledger-CJiAEU3j.js.map +1 -0
  67. package/dist/{search-ledger-types-C9anU2DX.d.ts → search-ledger-types-B3El5xgS.d.ts} +95 -11
  68. package/dist/search-ledger-types-B3El5xgS.d.ts.map +1 -0
  69. package/dist/{search-shipper-BG182p-U.js → search-shipper-DUlwlCsK.js} +2 -2
  70. package/dist/{search-shipper-BG182p-U.js.map → search-shipper-DUlwlCsK.js.map} +1 -1
  71. package/dist/search-summary-BhY8p0Av.js +378 -0
  72. package/dist/search-summary-BhY8p0Av.js.map +1 -0
  73. package/dist/{sequential-lwJOTRON.js → sequential-C21_PP7F.js} +2 -2
  74. package/dist/{sequential-lwJOTRON.js.map → sequential-C21_PP7F.js.map} +1 -1
  75. package/dist/{skillopt-optimization-method-DoayuM6D.js → skillopt-optimization-method-DXQdaBjx.js} +5 -5
  76. package/dist/{skillopt-optimization-method-DoayuM6D.js.map → skillopt-optimization-method-DXQdaBjx.js.map} +1 -1
  77. package/dist/{statistical-heldout-C1Ej5Ce8.d.ts → statistical-heldout-D-xBpbrd.d.ts} +2 -2
  78. package/dist/{statistical-heldout-C1Ej5Ce8.d.ts.map → statistical-heldout-D-xBpbrd.d.ts.map} +1 -1
  79. package/dist/{statistical-heldout-GcLIPxk4.js → statistical-heldout-DANVmyug.js} +2 -2
  80. package/dist/{statistical-heldout-GcLIPxk4.js.map → statistical-heldout-DANVmyug.js.map} +1 -1
  81. package/dist/{store-tool-spans-ByLdU26j.d.ts → store-tool-spans-BxrfuU2t.d.ts} +4 -4
  82. package/dist/{store-tool-spans-ByLdU26j.d.ts.map → store-tool-spans-BxrfuU2t.d.ts.map} +1 -1
  83. package/dist/{tool-groups-DiBi16ns.d.ts → tool-groups-BC9O7Epn.d.ts} +2 -2
  84. package/dist/tool-groups-BC9O7Epn.d.ts.map +1 -0
  85. package/dist/trace-repair/index.d.ts +1 -1
  86. package/dist/traces.d.ts +2 -2
  87. package/dist/{types-C1CAD70_.d.ts → types-BWwRXofp.d.ts} +328 -4
  88. package/dist/types-BWwRXofp.d.ts.map +1 -0
  89. package/dist/{types-ByZslIlr.d.ts → types-qoxUfKQg.d.ts} +2 -2
  90. package/dist/{types-ByZslIlr.d.ts.map → types-qoxUfKQg.d.ts.map} +1 -1
  91. package/docs/search-ledger.md +71 -6
  92. package/package.json +1 -1
  93. package/dist/claim-BGOUqoMq.d.ts +0 -38
  94. package/dist/claim-BGOUqoMq.d.ts.map +0 -1
  95. package/dist/index-DwQxdWve.d.ts.map +0 -1
  96. package/dist/llm-judge-Z6MkTkd0.js.map +0 -1
  97. package/dist/paired-promotion-decision-6cn3bLc3.js.map +0 -1
  98. package/dist/search-ledger-Ck7tok0h.js.map +0 -1
  99. package/dist/search-ledger-types-C9anU2DX.d.ts.map +0 -1
  100. package/dist/tool-groups-DiBi16ns.d.ts.map +0 -1
  101. package/dist/types-C1CAD70_.d.ts.map +0 -1
@@ -2,23 +2,23 @@ import { i as JudgeError, s as ValidationError } from "./errors-Dngq5h35.js";
2
2
  import { a as hashCanonical, i as compareCodeUnits, r as canonicalString, t as LEDGER_HASH_PATTERN } from "./canonical-BsZVgBhW.js";
3
3
  import { t as mulberry32 } from "./random-Dn5fPWkt.js";
4
4
  import { o as summarizeNumberSeries, s as weightedComposite, t as confidenceInterval } from "./descriptive-1V17A-qa.js";
5
- import { n as DECISION_PAIRED_DELTA_STATISTIC } from "./paired-tests-C8iCsioC.js";
6
- import { i as pairedDeltaTest, r as minimumPairsForPairedDeltaTest, t as decidePairedPromotion } from "./paired-promotion-decision-6cn3bLc3.js";
7
- import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-GcLIPxk4.js";
5
+ import { v as wilson } from "./paired-arms-JSDcs2zx.js";
6
+ import { t as decidePairedPromotion } from "./paired-promotion-decision-DTKl6-G8.js";
7
+ import { a as heldoutSignificance, i as dimensionRegressions, o as pairHoldout } from "./statistical-heldout-DANVmyug.js";
8
8
  import { i as modelHasSnapshot, l as searchCellRunId } from "./run-record-BRlM99rX.js";
9
9
  import { n as contentHash } from "./verdict-cache-BQEsVy9n.js";
10
10
  import { r as projectCampaignCellQuality, t as campaignCellCostProvenance } from "./run-record-tNnlQIrD.js";
11
11
  import { i as CostLedger, t as CostAccountingIncompleteError } from "./cost-ledger-_IbkejCF.js";
12
- import { F as surfaceContentHash, G as campaignCoverage, H as assertCampaignDesign, I as surfaceDispatchRef, K as campaignScenarioIdentity, L as surfaceHash, O as FinalEvidenceConflictError, P as renderSurfaceDiff, R as surfaceHashMatches, S as campaignMeanCompositeOrNull, U as assertCampaignSplitIdentity, W as assertCompleteCampaign, Y as formatCoverageFailures, b as campaignBreakdown, k as FinalEvidenceError, q as campaignSplitDigest, t as createCampaignEvidenceReceipt, x as campaignMeanComposite, z as BackendIntegrityError } from "./campaign-evidence-DcQiYhJI.js";
12
+ import { F as surfaceContentHash, G as campaignCoverage, H as assertCampaignDesign, I as surfaceDispatchRef, K as campaignScenarioIdentity, L as surfaceHash, O as FinalEvidenceConflictError, P as renderSurfaceDiff, R as surfaceHashMatches, S as campaignMeanCompositeOrNull, U as assertCampaignSplitIdentity, W as assertCompleteCampaign, Y as formatCoverageFailures, b as campaignBreakdown, k as FinalEvidenceError, q as campaignSplitDigest, t as createCampaignEvidenceReceipt, x as campaignMeanComposite, z as BackendIntegrityError } from "./campaign-evidence-DJc60F9C.js";
13
13
  import { m as mapConcurrent } from "./journal-DvKCvjty.js";
14
14
  import { t as deepFreezeCanonicalJson } from "./deep-freeze-CurcOQg3.js";
15
- import { a as SearchLedgerError } from "./search-ledger-file-DY0AXX3W.js";
16
15
  import { C as fsCampaignStorage, S as createRunCostLedger, g as isRecord, h as isExternalTextCandidate, x as acquireSingleRunLock } from "./external-optimizer-subprocess-kbqC-iWf.js";
17
16
  import { r as summarizeEvaluationUnits, t as defineEvaluationClaim } from "./claim-V9Zqbq0J.js";
18
17
  import { f as redactText, n as REDACTION_VERSION, o as detectCredential, s as detectPersonalData } from "./redact-B7rt9Zot.js";
19
18
  import { t as detectRewardHacking } from "./reward-hacking-D0XwhVWE.js";
20
19
  import { n as assertProposalFindings, t as combineAbortSignals } from "./abort-signal-CtzAM_sJ.js";
21
- import { a as validateSearchLedgerEvent, d as searchTaskSetDigest, f as searchUnitScores, i as replaySearchLedgerText, l as searchCellId, n as openSearchLedger, t as FileSearchLedger, u as searchNodeId } from "./search-ledger-Ck7tok0h.js";
20
+ import { a as validateSearchLedgerEvent, d as searchTaskSetDigest, i as replaySearchLedgerText, l as searchCellId, n as openSearchLedger, t as FileSearchLedger, u as searchNodeId } from "./search-ledger-CJiAEU3j.js";
21
+ import { c as seedFromDigest, i as estimateNode, n as searchProposerView, s as searchPosterior, t as renderSearchSummary } from "./search-summary-BhY8p0Av.js";
22
22
  import { d as stripFencedJson, o as costReceiptFromLlm, s as costReceiptFromLlmError, u as maximumChargeForLlmRequest } from "./llm-client-KYQBh9C8.js";
23
23
  import { createHash } from "node:crypto";
24
24
  import { writeFileSync } from "node:fs";
@@ -1164,8 +1164,8 @@ function detectCalibrationDrift(runs, opts) {
1164
1164
  alpha,
1165
1165
  recentN: recent.length,
1166
1166
  historyN: historical.length,
1167
- recentMean: mean$2(recent),
1168
- historyMean: mean$2(historical)
1167
+ recentMean: mean$3(recent),
1168
+ historyMean: mean$3(historical)
1169
1169
  }
1170
1170
  }],
1171
1171
  evaluation: {
@@ -1402,7 +1402,7 @@ function chiSquareCritical(df, alpha) {
1402
1402
  }
1403
1403
  return TABLE[10][idx];
1404
1404
  }
1405
- function mean$2(xs) {
1405
+ function mean$3(xs) {
1406
1406
  if (xs.length === 0) return 0;
1407
1407
  return xs.reduce((s, x) => s + x, 0) / xs.length;
1408
1408
  }
@@ -2559,249 +2559,757 @@ function uniform(options = {}) {
2559
2559
  };
2560
2560
  return {
2561
2561
  name: reps === 1 ? "uniform" : `uniform(reps=${reps})`,
2562
+ reps,
2562
2563
  plan: (state, nodeId) => cells(state, nodeId === state.rootNodeId),
2563
2564
  screenSize: (state) => cells(state, false).length
2564
2565
  };
2565
2566
  }
2566
2567
  //#endregion
2567
- //#region src/campaign/estimate-node.ts
2568
+ //#region src/paired-promotion-power.ts
2568
2569
  /**
2569
- * Per-node statistics of a search.
2570
+ * @module
2571
+ * Joint power of a configured {@link decidePairedPromotion} call, computed by
2572
+ * running that call on data drawn from a registered alternative.
2573
+ *
2574
+ * Why a simulation and not a formula: `decidePairedPromotion` is not one test.
2575
+ * It routes by sample size (exact sign test below `BOOTSTRAP_GATE_MIN_N`,
2576
+ * paired bootstrap at or above it) and by shape (Tango's score interval on a
2577
+ * two-point outcome), and on top of the deciding interval it applies a
2578
+ * sufficiency floor, a zero-width refusal and McNemar's exact veto. A
2579
+ * preregistration usually seals two such calls, a continuous primary and a
2580
+ * pass/fail veto drawn from the same episodes, and promotes only when both
2581
+ * promote. The sample size at which THAT procedure reaches a target power is
2582
+ * not the power of any one channel: the sign-test power at n can be 0.8 while
2583
+ * the promotion probability is well below it because the veto binds
2584
+ * (agent-eval#785). Closed forms exist for single channels only, and each one
2585
+ * shipped here under-sized the call it stood in for (`mcnemarRequiredN` is
2586
+ * Lachin's normal approximation, 0.65 exact power where it promised 0.8 at 14
2587
+ * pairs; `requiredPairedSampleSize` sizes a paired t the procedure never runs).
2588
+ * So the only faithful power is the procedure's own promotion rate under the
2589
+ * alternative, and that is what this module computes: draw n pairs from the
2590
+ * registered law, run every configured call on them exactly as the caller
2591
+ * will, repeat, count.
2570
2592
  *
2571
- * `estimateNode` is the one paired contrast of two nodes: it averages each
2572
- * node's cells inside their units, pairs the units both nodes scored, and runs
2573
- * `pairedDeltaTest` on the per-unit deltas. `estimateNodeFromCells` is the same
2574
- * computation over cells a caller already holds, such as rows a server read
2575
- * from its database. The bootstrap seed comes from `cellSetDigest`, the digest
2576
- * of exactly the cells used, so a producer and a verifier that hold the same
2577
- * cells report the same bits.
2593
+ * The alternative is DECLARATIVE and serializable, never a sampler function,
2594
+ * so a preregistration can seal it as bytes: a mixture of cells, each with a
2595
+ * probability, an optional pass/fail outcome per arm for the binary calls,
2596
+ * and a law for the candidate-minus-baseline delta on the continuous scale.
2597
+ * Drawing the cell first and the delta given the cell is what makes the two
2598
+ * channels one joint law rather than two marginals.
2578
2599
  *
2579
- * `searchPosterior` gives every node a normal posterior on its improvement over
2580
- * the root, which expansion policies sample parents from. Its numbers steer
2581
- * spend and claim nothing.
2600
+ * Everything is seeded (`mulberry32`), so a sizing is reproducible from its
2601
+ * inputs, and every estimate carries its Monte Carlo standard error and a
2602
+ * Wilson interval: a power read at 0.80 from 1,000 simulations is 0.80 ± 0.013.
2582
2603
  */
2583
- const CONFIDENCE = .95;
2584
- const RESAMPLES = 2e3;
2585
- /** 6 at 95 %: the smallest sample an exact one-sided sign test can resolve. */
2586
- const DESCRIPTIVE_FROM = minimumPairsForPairedDeltaTest(CONFIDENCE);
2587
- /** The estimator every `NodeEstimate` names. */
2588
- const SEARCH_ESTIMATOR = {
2589
- uri: "npm:@tangle-network/agent-eval#estimateNode",
2590
- revision: hashCanonical({
2591
- name: "tangle.estimate-node.2026-09",
2592
- unit: "mean of the unit’s scored cells, summed in cellId order",
2593
- pairing: "units both nodes scored, in unitId order",
2594
- statistic: DECISION_PAIRED_DELTA_STATISTIC,
2595
- interval: "percentile paired bootstrap",
2596
- confidence: CONFIDENCE,
2597
- resamples: RESAMPLES,
2598
- rng: "mulberry32",
2599
- seed: "the first 32 bits of cellSetDigest",
2600
- signTest: "exact, one-sided toward improvement in the objective direction",
2601
- methods: {
2602
- none: "below 2 pairs",
2603
- insufficient: `below ${DESCRIPTIVE_FROM} pairs`,
2604
- descriptive: `below 20 pairs`,
2605
- bootstrap: `from 20 pairs`
2606
- },
2607
- indeterminate: "every paired delta equal: no interval and no sign p"
2608
- })
2609
- };
2610
- /** The method a paired sample of `pairs` units supports. */
2611
- function searchEstimateMethod(pairs) {
2612
- if (pairs < 2) return "none";
2613
- if (pairs < DESCRIPTIVE_FROM) return "insufficient";
2614
- if (pairs < 20) return "descriptive";
2615
- return "bootstrap";
2616
- }
2617
2604
  /**
2618
- * Digest of the cells an estimate reads, with the contrast they serve. Cells
2619
- * are listed in cellId order, so the digest names the set, not the order it
2620
- * was read in.
2605
+ * Joint power of the configured calls at `n` under the registered alternative:
2606
+ * the fraction of simulated data sets on which every call promotes.
2621
2607
  */
2622
- function searchCellSetDigest(input) {
2623
- const list = (cells) => [...cells].sort((left, right) => compareCodeUnits(left.cellId, right.cellId)).map((cell) => [
2624
- cell.cellId,
2625
- cell.unitId,
2626
- cell.attempt,
2627
- cell.score
2628
- ]);
2629
- return hashCanonical({
2630
- contrast: [input.nodeId, input.against],
2631
- split: input.split,
2632
- nodeCells: list(input.nodeCells),
2633
- againstCells: list(input.againstCells)
2634
- });
2608
+ function pairedPromotionPower(options) {
2609
+ const n = options.n;
2610
+ if (!Number.isInteger(n) || n < 1) throw new Error(`pairedPromotionPower: n must be a positive integer, got ${n}`);
2611
+ const simulations = options.simulations ?? 1e3;
2612
+ if (!Number.isInteger(simulations) || simulations < 1) throw new Error(`pairedPromotionPower: simulations must be a positive integer, got ${simulations}`);
2613
+ const seed = options.seed ?? 1;
2614
+ if (!Number.isFinite(seed)) throw new Error(`pairedPromotionPower: seed must be finite, got ${seed}`);
2615
+ const calls = validateCalls(options.calls);
2616
+ const alternative = validateAlternative(options.alternative, calls);
2617
+ const rng = mulberry32(seed);
2618
+ let promotions = 0;
2619
+ let refusals = 0;
2620
+ const perCall = calls.map(() => ({
2621
+ promote: 0,
2622
+ insufficient: 0,
2623
+ indeterminate: 0,
2624
+ exactTestVetoes: 0,
2625
+ hold: 0,
2626
+ methods: /* @__PURE__ */ new Map()
2627
+ }));
2628
+ for (let s = 0; s < simulations; s++) {
2629
+ const pairs = drawPairs(alternative, n, rng);
2630
+ let all = true;
2631
+ let anyRefusal = false;
2632
+ for (let c = 0; c < calls.length; c++) {
2633
+ const call = calls[c];
2634
+ const decision = runCall(call, pairs, n);
2635
+ const tally = perCall[c];
2636
+ tally.methods.set(decision.method, (tally.methods.get(decision.method) ?? 0) + 1);
2637
+ if (decision.promote) tally.promote++;
2638
+ else {
2639
+ if (!decision.sufficient) tally.insufficient++;
2640
+ if (decision.indeterminate) tally.indeterminate++;
2641
+ if (decision.exactTestVetoes) tally.exactTestVetoes++;
2642
+ if (decision.sufficient && !decision.indeterminate && !decision.exactTestVetoes) tally.hold++;
2643
+ }
2644
+ if (!decision.promote) all = false;
2645
+ if (!decision.sufficient || decision.indeterminate || decision.exactTestVetoes) anyRefusal = true;
2646
+ }
2647
+ if (all) promotions++;
2648
+ else if (anyRefusal) refusals++;
2649
+ }
2650
+ const power = promotions / simulations;
2651
+ const interval = wilson(promotions, simulations, .95);
2652
+ return {
2653
+ n,
2654
+ simulations,
2655
+ seed,
2656
+ power,
2657
+ standardError: Math.sqrt(power * (1 - power) / simulations),
2658
+ low: interval.lower,
2659
+ high: interval.upper,
2660
+ refuse: refusals / simulations,
2661
+ hold: (simulations - promotions - refusals) / simulations,
2662
+ calls: perCall.map((tally, c) => ({
2663
+ outcome: calls[c].outcome,
2664
+ promote: tally.promote / simulations,
2665
+ insufficient: tally.insufficient / simulations,
2666
+ indeterminate: tally.indeterminate / simulations,
2667
+ exactTestVetoes: tally.exactTestVetoes / simulations,
2668
+ hold: tally.hold / simulations,
2669
+ methods: Object.fromEntries([...tally.methods.entries()].map(([method, count]) => [method, count / simulations]))
2670
+ }))
2671
+ };
2635
2672
  }
2636
2673
  /**
2637
- * The paired contrast of `nodeId` against `against` on the units both scored.
2638
- *
2639
- * `delta` is the mean per-unit difference, node minus `against`, in the
2640
- * metric's own units; for a `minimize` objective an improvement is negative.
2641
- * The exact sign test is one-sided toward improvement in the objective's
2642
- * direction. Unscored cells (errored, cancelled or in flight) are absent from
2643
- * the inputs, never zero.
2644
- *
2645
- * - `none` (0 or 1 pair): no delta.
2646
- * - `insufficient` (2 to 5): the delta only.
2647
- * - `descriptive` (6 to 19): the bootstrap interval as spread, and the sign-test p.
2648
- * - `bootstrap` (20 or more): the decision-grade bootstrap interval.
2649
- *
2650
- * When every paired delta is equal the interval would have zero width, which
2651
- * is an absence of evidence; the estimate is then `indeterminate` and carries
2652
- * neither an interval nor a p-value.
2674
+ * First n at which {@link pairedPromotionPower} reaches the target, scanning
2675
+ * every n from `minPairs` upward. Both the point-estimate answer and the
2676
+ * confirmed lower-bound answer are returned; the scan runs until the latter is
2677
+ * reached (so the curve past `n` is retained) or `maxPairs` is exhausted. A
2678
+ * candidate selected by the scan's bound is certified only by an independent
2679
+ * simulation at that n on a different seed; a candidate that fails it is
2680
+ * recorded and the scan continues.
2653
2681
  */
2654
- function estimateNodeFromCells(input) {
2655
- const { nodeId, against, split, direction } = input;
2656
- if (nodeId === against) throw new SearchLedgerError(`node ${nodeId} cannot be estimated against itself`);
2657
- assertDistinctCells([...input.nodeCells, ...input.againstCells], nodeId, against);
2658
- const cellSetDigest = searchCellSetDigest(input);
2659
- const nodeUnits = searchUnitScores(input.nodeCells);
2660
- const { node, other } = pairUnits(nodeUnits, unitMeans(searchUnitScores(input.againstCells)));
2661
- const pairs = node.length;
2662
- const method = searchEstimateMethod(pairs);
2663
- const base = {
2664
- against,
2665
- split,
2666
- units: nodeUnits.length,
2667
- pairs,
2668
- method,
2669
- cellSetDigest,
2670
- estimator: SEARCH_ESTIMATOR
2682
+ function requiredPairsForPairedPromotion(options) {
2683
+ const target = options.target ?? .8;
2684
+ if (!Number.isFinite(target) || target <= 0 || target >= 1) throw new Error(`requiredPairsForPairedPromotion: target must be in (0,1), got ${target}`);
2685
+ const minPairs = options.minPairs ?? 1;
2686
+ const maxPairs = options.maxPairs ?? 200;
2687
+ if (!Number.isInteger(minPairs) || minPairs < 1) throw new Error(`requiredPairsForPairedPromotion: minPairs must be a positive integer, got ${minPairs}`);
2688
+ if (!Number.isInteger(maxPairs) || maxPairs < minPairs) throw new Error(`requiredPairsForPairedPromotion: maxPairs must be an integer >= minPairs, got ${maxPairs}`);
2689
+ const seed = options.seed ?? 1;
2690
+ const curve = [];
2691
+ const rejected = [];
2692
+ let n = null;
2693
+ let nAtLowerBound = null;
2694
+ let confirmation = null;
2695
+ for (let candidate = minPairs; candidate <= maxPairs; candidate++) {
2696
+ const result = pairedPromotionPower({
2697
+ n: candidate,
2698
+ alternative: options.alternative,
2699
+ calls: options.calls,
2700
+ simulations: options.simulations,
2701
+ seed
2702
+ });
2703
+ curve.push(result);
2704
+ if (n === null && result.power >= target) n = candidate;
2705
+ if (result.low >= target) {
2706
+ const check = pairedPromotionPower({
2707
+ n: candidate,
2708
+ alternative: options.alternative,
2709
+ calls: options.calls,
2710
+ simulations: options.simulations,
2711
+ seed: confirmationSeed(seed, candidate)
2712
+ });
2713
+ if (check.low >= target) {
2714
+ nAtLowerBound = candidate;
2715
+ confirmation = check;
2716
+ break;
2717
+ }
2718
+ rejected.push(check);
2719
+ }
2720
+ }
2721
+ return {
2722
+ target,
2723
+ n,
2724
+ nAtLowerBound,
2725
+ confirmation,
2726
+ curve,
2727
+ rejected
2671
2728
  };
2672
- if (method === "none") return {
2673
- ...base,
2674
- delta: null,
2675
- interval: null,
2676
- exactSignP: null,
2677
- indeterminate: false
2729
+ }
2730
+ /** A seed for the confirmation at `n` that never equals the scan seed. */
2731
+ function confirmationSeed(seed, n) {
2732
+ return (Math.imul((seed | 0) ^ 2654435769, 2246822507) + Math.imul(n, 3266489909) | 1) >>> 0;
2733
+ }
2734
+ function drawPairs(alternative, n, rng) {
2735
+ const baseline = alternative.baseline ?? {
2736
+ kind: "point",
2737
+ value: 0
2678
2738
  };
2679
- const maximize = direction === "maximize";
2680
- const test = pairedDeltaTest(maximize ? other : node, maximize ? node : other, {
2681
- statistic: DECISION_PAIRED_DELTA_STATISTIC,
2682
- confidence: CONFIDENCE,
2683
- resamples: RESAMPLES,
2684
- seed: seedFromDigest(cellSetDigest)
2685
- });
2686
- const { mean, low, high } = test.bootstrap;
2687
- const indeterminate = test.indeterminate;
2688
- const spread = (method === "descriptive" || method === "bootstrap") && !indeterminate;
2739
+ const control = new Array(n);
2740
+ const candidate = new Array(n);
2741
+ const controlPass = new Array(n);
2742
+ const candidatePass = new Array(n);
2743
+ for (let i = 0; i < n; i++) {
2744
+ const cell = drawCell(alternative.cells, rng);
2745
+ const base = drawValue(baseline, rng);
2746
+ control[i] = base;
2747
+ candidate[i] = base + drawValue(cell.delta, rng);
2748
+ controlPass[i] = cell.pass?.control ?? false;
2749
+ candidatePass[i] = cell.pass?.treatment ?? false;
2750
+ }
2689
2751
  return {
2690
- ...base,
2691
- delta: plain(maximize ? mean : -mean),
2692
- interval: spread ? maximize ? [plain(low), plain(high)] : [plain(-high), plain(-low)] : null,
2693
- exactSignP: method === "descriptive" && !indeterminate ? test.pValue : null,
2694
- indeterminate
2752
+ control,
2753
+ candidate,
2754
+ controlPass,
2755
+ candidatePass
2695
2756
  };
2696
2757
  }
2697
- /**
2698
- * `estimateNodeFromCells` over the cells a search state holds: `nodeId`
2699
- * against `against` on one split, in the direction the search's objective
2700
- * declares.
2701
- */
2702
- function estimateNode(state, nodeId, options) {
2703
- const header = state.header;
2704
- if (!header) throw new SearchLedgerError(`search ${state.searchId} has not been opened`);
2705
- for (const id of [nodeId, options.against]) if (!state.hasNode(id)) throw new SearchLedgerError(`node ${id} is not in search ${state.searchId}`);
2706
- return estimateNodeFromCells({
2707
- nodeId,
2708
- against: options.against,
2709
- split: options.split,
2710
- direction: header.objective.direction,
2711
- nodeCells: state.scoredCells(nodeId, options.split),
2712
- againstCells: state.scoredCells(options.against, options.split)
2758
+ function drawCell(cells, rng) {
2759
+ const u = rng();
2760
+ let cumulative = 0;
2761
+ for (const cell of cells) {
2762
+ cumulative += cell.probability;
2763
+ if (u < cumulative) return cell;
2764
+ }
2765
+ return cells[cells.length - 1];
2766
+ }
2767
+ function drawValue(law, rng) {
2768
+ switch (law.kind) {
2769
+ case "point": return law.value;
2770
+ case "atoms": {
2771
+ const u = rng();
2772
+ let cumulative = 0;
2773
+ for (const atom of law.atoms) {
2774
+ cumulative += atom.probability;
2775
+ if (u < cumulative) return atom.value;
2776
+ }
2777
+ return law.atoms[law.atoms.length - 1].value;
2778
+ }
2779
+ case "normal": {
2780
+ const u1 = 1 - rng();
2781
+ const u2 = rng();
2782
+ return law.mean + law.sd * Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
2783
+ }
2784
+ }
2785
+ }
2786
+ function runCall(call, pairs, n) {
2787
+ const options = {
2788
+ minPairs: n,
2789
+ ...call.options
2790
+ };
2791
+ if (call.outcome === "pass") {
2792
+ const scale = call.options.binaryScale;
2793
+ return decidePairedPromotion(pairs.controlPass.map((p) => p ? scale : 0), pairs.candidatePass.map((p) => p ? scale : 0), options);
2794
+ }
2795
+ return decidePairedPromotion(pairs.control, pairs.candidate, options);
2796
+ }
2797
+ function validateCalls(calls) {
2798
+ if (!Array.isArray(calls) || calls.length === 0) throw new Error("pairedPromotionPower: at least one call is required");
2799
+ calls.forEach((call, i) => {
2800
+ if (call.outcome !== "delta" && call.outcome !== "pass") throw new Error(`pairedPromotionPower: calls[${i}].outcome must be 'delta' or 'pass'`);
2801
+ if (call.options === null || typeof call.options !== "object") throw new Error(`pairedPromotionPower: calls[${i}].options must be an object`);
2802
+ if (call.options.minPairs !== void 0 && (!Number.isInteger(call.options.minPairs) || call.options.minPairs < 1)) throw new Error(`pairedPromotionPower: calls[${i}].options.minPairs must be a positive integer`);
2803
+ if (call.outcome === "pass") {
2804
+ const scale = call.options.binaryScale;
2805
+ if (scale === void 0 || !Number.isFinite(scale) || scale <= 0) throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call and must declare a finite positive options.binaryScale`);
2806
+ if (call.options.continuous) throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call; continuous cannot accompany it`);
2807
+ if (call.options.statistic === "median") throw new Error(`pairedPromotionPower: calls[${i}] is a 'pass' call; binaryScale requires the mean statistic, not median`);
2808
+ } else if (call.options.binaryScale !== void 0) throw new Error(`pairedPromotionPower: calls[${i}] is a 'delta' call and cannot declare binaryScale; use a 'pass' call for a two-point outcome`);
2713
2809
  });
2810
+ return calls;
2811
+ }
2812
+ function validateAlternative(alternative, calls) {
2813
+ if (!alternative || !Array.isArray(alternative.cells) || alternative.cells.length === 0) throw new Error("pairedPromotionPower: alternative.cells must be a non-empty array");
2814
+ const needsPass = calls.some((call) => call.outcome === "pass");
2815
+ let total = 0;
2816
+ alternative.cells.forEach((cell, i) => {
2817
+ if (!Number.isFinite(cell.probability) || cell.probability < 0) throw new Error(`pairedPromotionPower: cells[${i}].probability must be a finite number >= 0`);
2818
+ total += cell.probability;
2819
+ if (needsPass) {
2820
+ if (!cell.pass || typeof cell.pass.control !== "boolean" || typeof cell.pass.treatment !== "boolean") throw new Error(`pairedPromotionPower: cells[${i}].pass must give control and treatment booleans when a 'pass' call is configured`);
2821
+ }
2822
+ validateLaw(cell.delta, `cells[${i}].delta`);
2823
+ });
2824
+ if (Math.abs(total - 1) > 1e-9) throw new Error(`pairedPromotionPower: cell probabilities must sum to 1, got ${total}`);
2825
+ if (alternative.baseline !== void 0) validateLaw(alternative.baseline, "baseline");
2826
+ return alternative;
2827
+ }
2828
+ function validateLaw(law, where) {
2829
+ if (!law || typeof law !== "object") throw new Error(`pairedPromotionPower: ${where} must be a law object`);
2830
+ switch (law.kind) {
2831
+ case "point":
2832
+ if (!Number.isFinite(law.value)) throw new Error(`pairedPromotionPower: ${where}.value must be finite`);
2833
+ return;
2834
+ case "atoms": {
2835
+ if (!Array.isArray(law.atoms) || law.atoms.length === 0) throw new Error(`pairedPromotionPower: ${where}.atoms must be a non-empty array`);
2836
+ let total = 0;
2837
+ law.atoms.forEach((atom, i) => {
2838
+ if (!Number.isFinite(atom.value)) throw new Error(`pairedPromotionPower: ${where}.atoms[${i}].value must be finite`);
2839
+ if (!Number.isFinite(atom.probability) || atom.probability < 0) throw new Error(`pairedPromotionPower: ${where}.atoms[${i}].probability must be a finite number >= 0`);
2840
+ total += atom.probability;
2841
+ });
2842
+ if (Math.abs(total - 1) > 1e-9) throw new Error(`pairedPromotionPower: ${where}.atoms probabilities must sum to 1, got ${total}`);
2843
+ return;
2844
+ }
2845
+ case "normal":
2846
+ if (!Number.isFinite(law.mean) || !Number.isFinite(law.sd) || law.sd < 0) throw new Error(`pairedPromotionPower: ${where} needs a finite mean and a finite sd >= 0`);
2847
+ return;
2848
+ default: throw new Error(`pairedPromotionPower: ${where}.kind must be 'point', 'atoms' or 'normal'`);
2849
+ }
2714
2850
  }
2851
+ //#endregion
2852
+ //#region src/campaign/search-claim.ts
2715
2853
  /**
2716
- * The posterior every node's parent-selection draw uses. Each node's mean is
2717
- * its improvement over the root on shared units; its variance is the search's
2718
- * pooled between-unit variance divided by its unit count, so a node measured
2719
- * on one unit has a wide posterior rather than none. The root is the reference
2720
- * and sits at exactly 0.
2854
+ * The claim: the one statement a search makes about its nodes, made once, on
2855
+ * its sealed test split, after the search stops expanding.
2856
+ *
2857
+ * `planSearchClaim` fixes the claim's design from the ledger before any test
2858
+ * cell runs: up to 3 finalists ranked by selection mean, the estimator the
2859
+ * paired decision will use, the number of finalists the family-wise confidence
2860
+ * is divided among, and a power check on selection data. The kernel stores the
2861
+ * plan on the ledger's `claim` operation, so the ledger proves the design was
2862
+ * fixed before the test data existed. `decideSearchClaim` then reads the test
2863
+ * cells of the root and the finalists, which ran together, and decides each
2864
+ * finalist against the root with `decidePairedPromotion` at the Bonferroni
2865
+ * confidence `1 - 0.05 / k`.
2866
+ *
2867
+ * Every number is a pure function of the ledger: the power simulation is
2868
+ * seeded from the search's seed and the decisions from the digest of the test
2869
+ * cells they read, so a verifier with the same ledger gets the same claim.
2721
2870
  */
2722
- function searchPosterior(state, options = {}) {
2723
- const split = options.split ?? "selection";
2724
- const header = state.header;
2725
- if (!header) throw new SearchLedgerError(`search ${state.searchId} has not been opened`);
2726
- const rootNodeId = state.rootNodeId;
2727
- if (rootNodeId === null) throw new SearchLedgerError(`search ${state.searchId} has no root node yet`);
2728
- const sign = header.objective.direction === "maximize" ? 1 : -1;
2729
- const root = searchUnitScores(state.scoredCells(rootNodeId, split));
2730
- const rootMeans = unitMeans(root);
2731
- const nodes = [];
2732
- let squares = 0;
2733
- let degreesOfFreedom = 0;
2734
- for (const nodeId of state.nodeIds()) {
2735
- if (nodeId === rootNodeId) {
2736
- const measured = root.length > 0;
2737
- nodes.push({
2738
- nodeId,
2739
- pairs: root.length,
2740
- mean: measured ? 0 : null,
2741
- variance: measured ? 0 : null
2742
- });
2871
+ const FAMILY_CONFIDENCE = .95;
2872
+ const TARGET_POWER = .8;
2873
+ const POWER_SIMULATIONS = 500;
2874
+ const RESAMPLES = 2e3;
2875
+ /**
2876
+ * Every rule and parameter the claim depends on. Its digest is the claim's
2877
+ * revision; change a value here whenever the procedure changes, so the
2878
+ * revision moves with it.
2879
+ */
2880
+ const CLAIM_DEFINITION = {
2881
+ name: "tangle.search-claim.2026-09",
2882
+ finalists: `up to 3 non-root nodes not decided invalid that scored every selection unit and beat the root's mean on the selection units they share, ranked by selection mean in the objective's direction, then registration order`,
2883
+ familyConfidence: FAMILY_CONFIDENCE,
2884
+ correction: "Bonferroni: each of k finalists against the root at 1 - (1 - familyConfidence) / k",
2885
+ finalistCount: "the largest k whose test cells the unspent claim reserve and headroom cover and whose power reaches the target",
2886
+ estimator: "binary with scale s when every train, selection and test unit has one task at one repeat and every pre-test score is 0 or s; otherwise continuous",
2887
+ power: {
2888
+ method: "pairedPromotionPower of the claim decision, simulated at the claim minimumEffect",
2889
+ continuous: "per-unit improvement normal with the pooled selection variance",
2890
+ binary: "discordant units at the pooled selection variance over 2 s^2 per side, shifted by minimumEffect / s; concordant units pass at the root selection pass rate",
2891
+ targetPower: TARGET_POWER,
2892
+ simulations: POWER_SIMULATIONS,
2893
+ seed: "the first 32 bits of hashCanonical([search seed, k])"
2894
+ },
2895
+ decision: {
2896
+ call: "decidePairedPromotion(root, finalist) on per-unit test means in the objective direction",
2897
+ threshold: 0,
2898
+ minPairs: "every test unit",
2899
+ resamples: RESAMPLES,
2900
+ seed: "the first 32 bits of the finalist's test cellSetDigest"
2901
+ },
2902
+ select: "the promoted finalist with the largest improvement; ties to the better selection rank",
2903
+ ship: "a promoted finalist that scored every test unit, with the root, on held-out units, under a pinned judge"
2904
+ };
2905
+ /** The claim procedure every `SearchClaim` and claim plan names. */
2906
+ const SEARCH_CLAIM_RULE = {
2907
+ uri: "npm:@tangle-network/agent-eval#searchClaim",
2908
+ revision: hashCanonical(CLAIM_DEFINITION)
2909
+ };
2910
+ /** The rule name decisions made by the claim record. */
2911
+ const SEARCH_CLAIM_RULE_NAME = CLAIM_DEFINITION.name;
2912
+ /**
2913
+ * The claim reserve a search needs: the root and 3 finalists on every test
2914
+ * task at `reps` repeats, each held at `cellUsd`. Declare at least this as
2915
+ * `budget.reservedClaimUsd`; `runSearch` refuses to start a capped search
2916
+ * with less.
2917
+ */
2918
+ function searchClaimReserveUsd(input) {
2919
+ return 4 * input.testTasks * input.reps * input.cellUsd;
2920
+ }
2921
+ /** Fix the claim's design from a search that stopped expanding. */
2922
+ function planSearchClaim(state, input) {
2923
+ const header = requireHeader(state);
2924
+ const root = state.rootNodeId;
2925
+ const plan = (finalists, power, test, reason) => ({
2926
+ kind: "search-claim-plan",
2927
+ rule: SEARCH_CLAIM_RULE,
2928
+ stopReason: input.stopReason,
2929
+ finalists,
2930
+ power,
2931
+ test,
2932
+ reason
2933
+ });
2934
+ const selectionUnits = unitIds(header.splits.selection.tasks);
2935
+ const testUnits = unitIds(header.splits.test.tasks);
2936
+ if (testUnits.length === 0) return plan([], { unknown: "the search has no test split" }, "hold", "the search has no test split, so it makes no claim");
2937
+ if (selectionUnits.length === 0) return plan([], { unknown: "the search has no selection split to choose finalists on" }, "hold", "the search has no selection split, so no finalist can be chosen without the test data");
2938
+ const candidates = rankFinalists(state, selectionUnits);
2939
+ if (candidates.length === 0) return plan([], { unknown: "no node went to test" }, "hold", "no node scored every selection unit and beat the root's selection mean, so the search keeps the root");
2940
+ const minimumEffect = header.objective.claim.minimumEffect;
2941
+ if (minimumEffect === void 0) return plan(candidates.slice(0, 1), { unknown: "the claim declares no minimumEffect" }, "test-cannot-resolve", "the claim declares no minimumEffect, so no test split can be shown to resolve it");
2942
+ const { pooledVariance } = searchPosterior(state, { split: "selection" });
2943
+ if (pooledVariance === null) return plan(candidates.slice(0, 1), { unknown: "no node shares 2 selection units with the root, so the variance is unknown" }, "test-cannot-resolve", "the selection split gives no variance for the power check, so the test cannot be shown to resolve the minimum effect");
2944
+ const estimator = claimEstimator(state, input.reps);
2945
+ const rootPassRate = passRate(state.unitScores(root, "selection"), estimator);
2946
+ let best = null;
2947
+ let budgetBound = false;
2948
+ for (let k = candidates.length; k >= 1; k--) {
2949
+ const power = claimPower({
2950
+ units: testUnits.length,
2951
+ finalists: k,
2952
+ minimumEffect,
2953
+ pooledVariance,
2954
+ estimator,
2955
+ rootPassRate,
2956
+ seed: header.policy.seed
2957
+ });
2958
+ best = power;
2959
+ if (!power.adequate) continue;
2960
+ if (!input.affordable(k)) {
2961
+ budgetBound = true;
2743
2962
  continue;
2744
2963
  }
2745
- const { node, other } = pairUnits(searchUnitScores(state.scoredCells(nodeId, split)), rootMeans);
2746
- const gains = node.map((value, index) => sign * (value - other[index]));
2747
- const mean = gains.length > 0 ? sum(gains) / gains.length : null;
2748
- if (mean !== null && gains.length >= 2) {
2749
- squares += sum(gains.map((gain) => (gain - mean) ** 2));
2750
- degreesOfFreedom += gains.length - 1;
2751
- }
2752
- nodes.push({
2753
- nodeId,
2754
- pairs: gains.length,
2755
- mean: mean === null ? null : plain(mean),
2756
- variance: null
2757
- });
2964
+ return plan(candidates.slice(0, k), power, "run", `${k} finalist${k === 1 ? "" : "s"} go to test: power ${round(power.powerAtMinimumEffect)} at an improvement of ${minimumEffect} on ${testUnits.length} test units reaches ${TARGET_POWER} at confidence ${round(bonferroni(k))} each`);
2758
2965
  }
2759
- const pooledVariance = degreesOfFreedom > 0 ? squares / degreesOfFreedom : null;
2760
- if (pooledVariance !== null) {
2761
- for (const posterior of nodes) if (posterior.nodeId !== rootNodeId && posterior.pairs > 0) posterior.variance = pooledVariance / posterior.pairs;
2966
+ const power = best;
2967
+ return plan(candidates.slice(0, 1), power, "test-cannot-resolve", budgetBound ? "the unspent claim reserve and headroom do not cover the test cells of the root and one finalist" : `even one finalist reaches power ${round("adequate" in power ? power.powerAtMinimumEffect : 0)} at an improvement of ${minimumEffect} on ${testUnits.length} test units, below ${TARGET_POWER}`);
2968
+ }
2969
+ /**
2970
+ * Decide the claim from the test cells of the root and the finalists. Call it
2971
+ * once every claim cell is settled or cancelled. It returns the claim and the
2972
+ * terminal decisions of the root and every finalist.
2973
+ */
2974
+ function decideSearchClaim(state, plan) {
2975
+ const header = requireHeader(state);
2976
+ const root = state.rootNodeId;
2977
+ const rootScored = [
2978
+ "train",
2979
+ "selection",
2980
+ "test"
2981
+ ].some((split) => state.scoredCells(root, split).length > 0);
2982
+ const keepRoot = rootScored ? root : null;
2983
+ const rule = SEARCH_CLAIM_RULE_NAME;
2984
+ const decisions = [];
2985
+ const claimOf = (finalists, selected, decision, reason) => ({
2986
+ rule: SEARCH_CLAIM_RULE,
2987
+ confidence: FAMILY_CONFIDENCE,
2988
+ power: plan.power,
2989
+ finalists,
2990
+ selected,
2991
+ decision,
2992
+ reason
2993
+ });
2994
+ if (plan.test !== "run") {
2995
+ for (const finalist of plan.finalists) decisions.push({
2996
+ nodeId: finalist.nodeId,
2997
+ decision: { status: "rejected" },
2998
+ basis: null,
2999
+ rule,
3000
+ reason: `no test cell ran: ${plan.reason}`
3001
+ });
3002
+ decisions.push(rootDecision(root, rootScored ? "kept" : "unscored", rule, plan.reason));
3003
+ return {
3004
+ claim: claimOf(plan.finalists.map((finalist) => ({
3005
+ nodeId: finalist.nodeId,
3006
+ estimate: null,
3007
+ test: null,
3008
+ promote: false
3009
+ })), keepRoot, plan.test, plan.reason),
3010
+ decisions
3011
+ };
2762
3012
  }
3013
+ const power = plan.power;
3014
+ if (!("adequate" in power)) throw new Error("a claim that runs its test cells has a known power");
3015
+ const k = plan.finalists.length;
3016
+ const confidence = bonferroni(k);
3017
+ const testUnits = unitIds(header.splits.test.tasks);
3018
+ const maximize = header.objective.direction === "maximize";
3019
+ const rootUnits = state.unitScores(root, "test");
3020
+ const tested = plan.finalists.map(({ nodeId }) => {
3021
+ const estimate = estimateNode(state, nodeId, {
3022
+ against: root,
3023
+ split: "test"
3024
+ });
3025
+ return {
3026
+ nodeId,
3027
+ estimate,
3028
+ ...pairedTest({
3029
+ root: rootUnits,
3030
+ node: state.unitScores(nodeId, "test"),
3031
+ testUnits,
3032
+ confidence,
3033
+ estimator: power.estimator,
3034
+ maximize,
3035
+ seed: seedFromDigest(estimate.cellSetDigest)
3036
+ })
3037
+ };
3038
+ });
3039
+ const winner = tested.map((entry, rank) => ({
3040
+ ...entry,
3041
+ rank
3042
+ })).filter((entry) => entry.promote).sort((a, b) => b.improvement - a.improvement || a.rank - b.rank)[0] ?? null;
3043
+ const blockers = [];
3044
+ if (!header.splits.heldOutUnits) blockers.push("the test units also appear in train or selection");
3045
+ if ("unknown" in header.objective.judge) blockers.push("the judge is not pinned");
3046
+ const ship = winner !== null && blockers.length === 0;
3047
+ const selected = ship ? winner.nodeId : keepRoot;
3048
+ const reason = ship ? `finalist ${winner.nodeId} beat the root on the ${testUnits.length} test units at confidence ${round(confidence)} (${k} finalist${k === 1 ? "" : "s"}, family-wise ${FAMILY_CONFIDENCE})` : winner !== null ? `finalist ${winner.nodeId} beat the root, but the claim cannot ship: ${blockers.join("; ")}` : `no finalist beat the root on the test split at confidence ${round(confidence)} (${k} finalist${k === 1 ? "" : "s"}, family-wise ${FAMILY_CONFIDENCE})`;
3049
+ for (const entry of tested) decisions.push({
3050
+ nodeId: entry.nodeId,
3051
+ decision: { status: ship && entry.nodeId === winner.nodeId ? "selected" : "rejected" },
3052
+ basis: entry.estimate,
3053
+ rule,
3054
+ reason: ship && entry.nodeId === winner.nodeId ? reason : entry.promote ? ship ? `promoted, but finalist ${winner.nodeId} improved more on the test split` : reason : entry.why
3055
+ });
3056
+ decisions.push(rootDecision(root, ship ? "replaced" : rootScored ? "kept" : "unscored", rule, reason));
2763
3057
  return {
2764
- split,
2765
- rootNodeId,
2766
- pooledVariance,
2767
- degreesOfFreedom,
2768
- nodes
3058
+ claim: claimOf(tested.map((entry) => ({
3059
+ nodeId: entry.nodeId,
3060
+ estimate: entry.estimate,
3061
+ test: entry.test,
3062
+ promote: entry.promote
3063
+ })), selected, ship ? "ship" : "hold", reason),
3064
+ decisions
2769
3065
  };
2770
3066
  }
2771
- function unitMeans(units) {
2772
- return new Map(units.map((unit) => [unit.unitId, unit.mean]));
3067
+ /** Per-finalist confidence for `k` finalists. */
3068
+ function bonferroni(k) {
3069
+ return 1 - (1 - FAMILY_CONFIDENCE) / k;
2773
3070
  }
2774
- /** Unit means of two nodes on the units both scored, aligned in the node's
2775
- * unitId order. */
2776
- function pairUnits(node, otherMeans) {
2777
- const paired = {
2778
- node: [],
2779
- other: []
3071
+ function rootDecision(root, fate, rule, reason) {
3072
+ return {
3073
+ nodeId: root,
3074
+ decision: { status: fate === "kept" ? "selected" : "rejected" },
3075
+ basis: null,
3076
+ rule,
3077
+ reason: fate === "kept" ? `the search keeps the root: ${reason}` : fate === "replaced" ? `a finalist replaces the root: ${reason}` : "the root has no scored cell, so the search keeps nothing"
2780
3078
  };
2781
- for (const unit of node) {
2782
- const mean = otherMeans.get(unit.unitId);
2783
- if (mean === void 0) continue;
2784
- paired.node.push(unit.mean);
2785
- paired.other.push(mean);
3079
+ }
3080
+ /** Nodes eligible to go to test, best first, at most `SEARCH_CLAIM_MAX_FINALISTS`. */
3081
+ function rankFinalists(state, selectionUnits) {
3082
+ const sign = requireHeader(state).objective.direction === "maximize" ? 1 : -1;
3083
+ const root = state.rootNodeId;
3084
+ const rootMeans = new Map(state.unitScores(root, "selection").map((unit) => [unit.unitId, unit.mean]));
3085
+ const ranked = [];
3086
+ for (const node of state.nodes()) {
3087
+ if (node.nodeId === root || node.status === "invalid") continue;
3088
+ const units = state.unitScores(node.nodeId, "selection");
3089
+ if (units.length !== selectionUnits.length) continue;
3090
+ let gain = 0;
3091
+ let shared = 0;
3092
+ for (const unit of units) {
3093
+ const rootMean = rootMeans.get(unit.unitId);
3094
+ if (rootMean === void 0) continue;
3095
+ gain += sign * (unit.mean - rootMean);
3096
+ shared += 1;
3097
+ }
3098
+ if (shared === 0 || gain <= 0) continue;
3099
+ ranked.push({
3100
+ nodeId: node.nodeId,
3101
+ selectionMean: mean$2(units.map((unit) => unit.mean)),
3102
+ ordinal: node.ordinal
3103
+ });
2786
3104
  }
2787
- return paired;
3105
+ ranked.sort((a, b) => sign * (b.selectionMean - a.selectionMean) || a.ordinal - b.ordinal);
3106
+ return ranked.slice(0, 3).map(({ nodeId, selectionMean }) => ({
3107
+ nodeId,
3108
+ selectionMean
3109
+ }));
2788
3110
  }
2789
- function assertDistinctCells(cells, nodeId, against) {
2790
- const seen = /* @__PURE__ */ new Set();
2791
- for (const cell of cells) {
2792
- if (seen.has(cell.cellId)) throw new SearchLedgerError(`cell ${cell.cellId} appears twice in the estimate of ${nodeId} against ${against}`);
2793
- if (!Number.isFinite(cell.score)) throw new SearchLedgerError(`cell ${cell.cellId} has a non-finite score ${cell.score}`);
2794
- seen.add(cell.cellId);
3111
+ /**
3112
+ * The paired decision's estimator, fixed from pre-test data. A per-unit mean
3113
+ * is binary only when each unit holds one task at one repeat and every score
3114
+ * seen before the test is 0 or one positive value.
3115
+ */
3116
+ function claimEstimator(state, reps) {
3117
+ const header = requireHeader(state);
3118
+ if (![
3119
+ "train",
3120
+ "selection",
3121
+ "test"
3122
+ ].every((split) => {
3123
+ const tasks = header.splits[split].tasks;
3124
+ return unitIds(tasks).length === tasks.length;
3125
+ }) || reps !== 1) return { kind: "continuous" };
3126
+ let scale = null;
3127
+ for (const nodeId of state.nodeIds()) for (const split of ["train", "selection"]) for (const cell of state.scoredCells(nodeId, split)) {
3128
+ if (cell.score === 0) continue;
3129
+ if (cell.score < 0 || scale !== null && cell.score !== scale) return { kind: "continuous" };
3130
+ scale = cell.score;
3131
+ }
3132
+ return scale === null ? { kind: "continuous" } : {
3133
+ kind: "binary",
3134
+ scale
3135
+ };
3136
+ }
3137
+ function passRate(units, estimator) {
3138
+ if (estimator.kind !== "binary" || units.length === 0) return 0;
3139
+ return units.filter((unit) => unit.mean === estimator.scale).length / units.length;
3140
+ }
3141
+ function claimPower(input) {
3142
+ const { units, finalists, minimumEffect, pooledVariance, estimator } = input;
3143
+ const confidence = bonferroni(finalists);
3144
+ let alternative;
3145
+ let call;
3146
+ if (estimator.kind === "binary") {
3147
+ const s = estimator.scale;
3148
+ const shift = Math.min(1, minimumEffect / s);
3149
+ const discordance = Math.min(.5, pooledVariance / (2 * s * s));
3150
+ const loss = Math.max(0, discordance - shift / 2);
3151
+ const win = Math.min(1 - loss, loss + shift);
3152
+ const concordant = Math.max(0, 1 - win - loss);
3153
+ alternative = { cells: [
3154
+ {
3155
+ probability: win,
3156
+ pass: {
3157
+ control: false,
3158
+ treatment: true
3159
+ },
3160
+ delta: {
3161
+ kind: "point",
3162
+ value: s
3163
+ }
3164
+ },
3165
+ {
3166
+ probability: loss,
3167
+ pass: {
3168
+ control: true,
3169
+ treatment: false
3170
+ },
3171
+ delta: {
3172
+ kind: "point",
3173
+ value: -s
3174
+ }
3175
+ },
3176
+ {
3177
+ probability: concordant * input.rootPassRate,
3178
+ pass: {
3179
+ control: true,
3180
+ treatment: true
3181
+ },
3182
+ delta: {
3183
+ kind: "point",
3184
+ value: 0
3185
+ }
3186
+ },
3187
+ {
3188
+ probability: concordant * (1 - input.rootPassRate),
3189
+ pass: {
3190
+ control: false,
3191
+ treatment: false
3192
+ },
3193
+ delta: {
3194
+ kind: "point",
3195
+ value: 0
3196
+ }
3197
+ }
3198
+ ] };
3199
+ call = {
3200
+ outcome: "pass",
3201
+ options: {
3202
+ binaryScale: s,
3203
+ confidence,
3204
+ threshold: 0
3205
+ }
3206
+ };
3207
+ } else {
3208
+ alternative = { cells: [{
3209
+ probability: 1,
3210
+ delta: {
3211
+ kind: "normal",
3212
+ mean: minimumEffect,
3213
+ sd: Math.sqrt(pooledVariance)
3214
+ }
3215
+ }] };
3216
+ call = {
3217
+ outcome: "delta",
3218
+ options: {
3219
+ continuous: true,
3220
+ confidence,
3221
+ threshold: 0,
3222
+ resamples: RESAMPLES
3223
+ }
3224
+ };
2795
3225
  }
3226
+ const simulated = pairedPromotionPower({
3227
+ n: units,
3228
+ alternative,
3229
+ calls: [call],
3230
+ simulations: POWER_SIMULATIONS,
3231
+ seed: seedFromDigest(hashCanonical([input.seed, finalists]))
3232
+ });
3233
+ return {
3234
+ adequate: simulated.power >= TARGET_POWER,
3235
+ minimumEffect,
3236
+ powerAtMinimumEffect: simulated.power,
3237
+ targetPower: TARGET_POWER,
3238
+ units,
3239
+ finalists,
3240
+ pooledVariance,
3241
+ estimator
3242
+ };
3243
+ }
3244
+ /**
3245
+ * One finalist against the root on the test split, in the objective's
3246
+ * direction, requiring every test unit scored by both.
3247
+ */
3248
+ function pairedTest(input) {
3249
+ const rootMeans = new Map(input.root.map((unit) => [unit.unitId, unit.mean]));
3250
+ const before = [];
3251
+ const after = [];
3252
+ for (const unit of input.node) {
3253
+ const rootMean = rootMeans.get(unit.unitId);
3254
+ if (rootMean === void 0) continue;
3255
+ before.push(input.maximize ? rootMean : unit.mean);
3256
+ after.push(input.maximize ? unit.mean : rootMean);
3257
+ }
3258
+ const pairs = before.length;
3259
+ if (pairs === 0) return {
3260
+ test: null,
3261
+ promote: false,
3262
+ improvement: null,
3263
+ why: "no test unit was scored by both it and the root"
3264
+ };
3265
+ const { estimator } = input;
3266
+ if (estimator.kind === "binary" && [...before, ...after].some((value) => value !== 0 && value !== estimator.scale)) return {
3267
+ test: null,
3268
+ promote: false,
3269
+ improvement: null,
3270
+ why: `a test score left the two-point scale {0, ${estimator.scale}} the claim fixed before the test`
3271
+ };
3272
+ const decision = decidePairedPromotion(before, after, {
3273
+ confidence: input.confidence,
3274
+ threshold: 0,
3275
+ minPairs: input.testUnits.length,
3276
+ resamples: RESAMPLES,
3277
+ seed: input.seed,
3278
+ ...estimator.kind === "binary" ? { binaryScale: estimator.scale } : { continuous: true }
3279
+ });
3280
+ const test = {
3281
+ pairs,
3282
+ confidence: input.confidence,
3283
+ method: decision.method,
3284
+ delta: plain(input.maximize ? decision.delta : -decision.delta),
3285
+ interval: input.maximize ? [plain(decision.low), plain(decision.high)] : [plain(-decision.high), plain(-decision.low)]
3286
+ };
3287
+ const complete = pairs === input.testUnits.length;
3288
+ const promote = decision.promote && complete;
3289
+ const why = !complete ? `it and the root scored ${pairs} of ${input.testUnits.length} test units, and a claim needs every one` : decision.promote ? "promoted" : !decision.sufficient ? `${pairs} test units are below the ${decision.minimumPairs} its paired test needs` : decision.indeterminate ? `its test interval has no width: ${decision.indeterminateCause}` : decision.exactTestVetoes ? "McNemar's exact test vetoes the improvement" : `its improvement interval [${round(decision.low)}, ${round(decision.high)}] at confidence ${round(input.confidence)} does not exclude 0`;
3290
+ return {
3291
+ test,
3292
+ promote,
3293
+ improvement: decision.delta,
3294
+ why
3295
+ };
2796
3296
  }
2797
- /** The first 32 bits of a `sha256:` digest, as the bootstrap's seed. */
2798
- function seedFromDigest(digest) {
2799
- return Number.parseInt(digest.slice(7, 15), 16) | 0;
3297
+ function requireHeader(state) {
3298
+ const header = state.header;
3299
+ if (!header) throw new Error(`search ${state.searchId} has not been opened`);
3300
+ if (state.rootNodeId === null) throw new Error(`search ${state.searchId} has no root node`);
3301
+ return header;
3302
+ }
3303
+ function unitIds(tasks) {
3304
+ return [...new Set(tasks.map((task) => task.unitId))];
2800
3305
  }
2801
- function sum(values) {
3306
+ function mean$2(values) {
2802
3307
  let total = 0;
2803
3308
  for (const value of values) total += value;
2804
- return total;
3309
+ return total / values.length;
3310
+ }
3311
+ function round(value) {
3312
+ return Math.round(value * 1e4) / 1e4;
2805
3313
  }
2806
3314
  /** Canonical JSON has one zero; never hand it a negative one. */
2807
3315
  function plain(value) {
@@ -2821,10 +3329,14 @@ const SEARCH_KERNEL_SOURCE = {
2821
3329
  retries: "a retryable errored attempt runs again, up to maxAttempts",
2822
3330
  resume: "an unrecorded operation is recorded failed at an unknown cost with floor 0; a recorded proposal is finished from its stored output; an unsettled cell is offered to adopt before it runs",
2823
3331
  writers: "one kernel per ledger file on a host, by a pid lock beside the ledger",
2824
- close: "the policy leader is selected when it has a scored cell; every other undecided node is rejected"
3332
+ divergence: "a node whose train mean over shared units rises above its parent while its descriptive or bootstrap selection interval against the parent lies wholly on the worse side is decided invalid when its cells finish",
3333
+ claim: "with a test split: after expansion stops, the claim plan is stored on the claim operation, the finalists are decided, the root and finalist test cells run together, and the claim decides; the kernel refuses to start a claim without a selection split, a minimumEffect, a pinned judge, or (under a cap) a claim reserve for the root and 3 finalists plus a cap for one screening round",
3334
+ close: "without a test split, the policy leader is selected when it has a scored cell; every other undecided node is rejected"
2825
3335
  })
2826
3336
  };
2827
3337
  const USD_TOLERANCE = 1e-9;
3338
+ /** The operation that holds the claim's plan. */
3339
+ const CLAIM_OPERATION = "claim";
2828
3340
  /** Settled cells a lane needs before its own cost distribution sets its estimate. */
2829
3341
  const ESTIMATE_FROM_CELLS = 20;
2830
3342
  const ESTIMATE_MARGIN = 1.5;
@@ -2880,6 +3392,8 @@ var SearchKernel = class {
2880
3392
  completed = [];
2881
3393
  stopReason = null;
2882
3394
  failure = null;
3395
+ /** The claim's design once expansion stopped; the search then only claims. */
3396
+ claimPlan = null;
2883
3397
  constructor(options) {
2884
3398
  this.options = options;
2885
3399
  this.recorder = options.recorder;
@@ -2930,7 +3444,12 @@ var SearchKernel = class {
2930
3444
  };
2931
3445
  signal?.addEventListener("abort", onAbort, { once: true });
2932
3446
  try {
3447
+ if (this.claimPlan) await this.applyClaimPlan(this.claimPlan);
2933
3448
  await this.loop();
3449
+ if (!this.failure && !this.claimPlan && this.state.header.splits.test.tasks.length > 0) {
3450
+ await this.beginClaim();
3451
+ await this.loop();
3452
+ }
2934
3453
  } finally {
2935
3454
  signal?.removeEventListener("abort", onAbort);
2936
3455
  }
@@ -2959,6 +3478,8 @@ var SearchKernel = class {
2959
3478
  const root = this.state.rootNodeId;
2960
3479
  this.artifacts.set(root, this.options.root);
2961
3480
  this.expansionOf.set(root, -1);
3481
+ if (this.state.audit.cells.allocated === 0) this.assertClaimReady();
3482
+ await this.restoreClaimPlan();
2962
3483
  const recorded = [];
2963
3484
  for (let expansion = 0;; expansion++) {
2964
3485
  const operationId = `expand-${expansion}`;
@@ -3000,11 +3521,55 @@ var SearchKernel = class {
3000
3521
  if (!lane) throw new Error(`runSearch: cell ${cell.cellId} was placed on lane ${String(cell.lane)}, which the executor does not declare`);
3001
3522
  this.adoptable.add(cell.cellId);
3002
3523
  this.enqueue(cell);
3003
- this.pending.set(cell.nodeId, (this.pending.get(cell.nodeId) ?? 0) + 1);
3524
+ if (cell.stage !== "claim") this.pending.set(cell.nodeId, (this.pending.get(cell.nodeId) ?? 0) + 1);
3004
3525
  }
3526
+ if (this.claimPlan) return;
3005
3527
  for (const expansion of recorded) await this.finishProposal(expansion);
3006
3528
  for (const node of this.state.nodes()) if (this.admitted.has(node.nodeId) && !isTerminal(node)) await this.allocateFor(node.nodeId);
3007
- for (const nodeId of this.state.nodeIds()) if (this.admitted.has(nodeId) && (this.pending.get(nodeId) ?? 0) === 0) this.markScreened(nodeId);
3529
+ for (const nodeId of this.state.nodeIds()) if (this.admitted.has(nodeId) && (this.pending.get(nodeId) ?? 0) === 0) await this.screenDone(nodeId);
3530
+ }
3531
+ /**
3532
+ * A search that will claim must be able to: finalists come from a selection
3533
+ * split, the power check needs the claim's minimum effect, a ship needs a
3534
+ * pinned judge, and under a cap the claim reserve must cover the root and 3
3535
+ * finalists on every test task while the rest of the cap covers one
3536
+ * screening round. Checked before the first cell, so a search that cannot
3537
+ * claim spends nothing.
3538
+ */
3539
+ assertClaimReady() {
3540
+ const header = this.state.header;
3541
+ const testTasks = header.splits.test.tasks.length;
3542
+ if (testTasks === 0) return;
3543
+ const refuse = (why) => {
3544
+ throw new Error(`runSearch: search ${this.recorder.searchId} has a test split, but ${why}`);
3545
+ };
3546
+ if (header.splits.selection.tasks.length === 0) refuse("no selection split to choose its finalists on");
3547
+ if (header.objective.claim.minimumEffect === void 0) refuse("its claim declares no minimumEffect for the power check");
3548
+ if ("unknown" in header.objective.judge) refuse("its judge is not pinned; a claim with an unknown judge cannot ship, and a resumed search could mix two judges");
3549
+ const { maxUsd, reservedClaimUsd } = header.budget;
3550
+ if (maxUsd === null) return;
3551
+ const cellUsd = this.maxCellReservation();
3552
+ const need = searchClaimReserveUsd({
3553
+ testTasks,
3554
+ reps: this.options.allocation.reps,
3555
+ cellUsd
3556
+ });
3557
+ if (reservedClaimUsd + USD_TOLERANCE < need) refuse(`its claim reserve $${reservedClaimUsd} is below the $${need} the root and 3 finalists need on ${testTasks} test tasks at $${cellUsd} a cell`);
3558
+ const rootCells = this.options.allocation.plan(this.state, this.state.rootNodeId).length;
3559
+ const screens = (this.options.proposer.childrenPerProposal ?? 1) * this.options.allocation.screenSize(this.state);
3560
+ const round = this.operationReservation().usd + (rootCells + screens) * cellUsd;
3561
+ if (maxUsd - reservedClaimUsd + USD_TOLERANCE < round) refuse(`its cap $${maxUsd} less the claim reserve $${reservedClaimUsd} does not cover the root and one screening round ($${round})`);
3562
+ }
3563
+ /** A claim an earlier process began: continue it from its stored plan. */
3564
+ async restoreClaimPlan() {
3565
+ const operation = this.state.operation(CLAIM_OPERATION);
3566
+ if (!operation) return;
3567
+ const ref = operation.artifacts.find((artifact) => artifact.role === "claim-plan");
3568
+ if (!ref) throw new Error(`runSearch: the ${CLAIM_OPERATION} operation has no stored plan`);
3569
+ const plan = this.recorder.readBlob(ref);
3570
+ if (!operation.recorded) await this.recordClaimOperation();
3571
+ this.claimPlan = plan;
3572
+ this.stopReason = plan.stopReason;
3008
3573
  }
3009
3574
  async loop() {
3010
3575
  for (;;) {
@@ -3057,10 +3622,11 @@ var SearchKernel = class {
3057
3622
  /** Fill free lane slots with queued cells. A queued cell was admitted when
3058
3623
  * it was allocated, so starting it needs no further decision. */
3059
3624
  dispatch() {
3060
- if (this.failure || this.options.signal?.aborted || this.pastDeadline()) return;
3625
+ if (this.failure || this.options.signal?.aborted) return;
3626
+ const claimOnly = this.pastDeadline();
3061
3627
  const cap = this.state.header.budget.maxConcurrency;
3062
3628
  for (const lane of this.lanes.values()) while (lane.inFlight < lane.capacity && (cap === null || this.inFlight.size < cap)) {
3063
- const cellId = lane.queues.find((queue) => queue.length > 0)?.shift();
3629
+ const cellId = (claimOnly ? lane.queues[STAGE_PRIORITY.claim] : lane.queues.find((waiting) => waiting.length > 0))?.shift();
3064
3630
  if (cellId === void 0) break;
3065
3631
  this.start(cellId, lane);
3066
3632
  }
@@ -3125,14 +3691,40 @@ var SearchKernel = class {
3125
3691
  this.enqueue(cell);
3126
3692
  return;
3127
3693
  }
3694
+ if (cell.stage === "claim") return;
3128
3695
  const left = (this.pending.get(cell.nodeId) ?? 1) - 1;
3129
3696
  this.pending.set(cell.nodeId, left);
3130
3697
  if (left === 0) await this.onScreenDone(cell.nodeId);
3131
3698
  }
3132
3699
  async onScreenDone(nodeId) {
3133
- this.markScreened(nodeId);
3700
+ if (!await this.screenDone(nodeId)) return;
3134
3701
  if (this.stopReason === null && !this.failure) await this.allocateFor(nodeId);
3135
3702
  }
3703
+ /**
3704
+ * A node's allocated cells all finished. A node that diverges (its train
3705
+ * mean rose over its parent's while its selection interval against the
3706
+ * parent lies wholly on the worse side) is decided `invalid` and never
3707
+ * becomes a parent; any other node is screened. Returns whether it was.
3708
+ */
3709
+ async screenDone(nodeId) {
3710
+ if (!this.claimPlan && !this.screenedSet.has(nodeId)) {
3711
+ const divergence = searchDivergence(this.state, nodeId);
3712
+ if (divergence) {
3713
+ await this.recorder.decideNode({
3714
+ nodeId,
3715
+ decision: { status: "invalid" },
3716
+ basis: divergence.basis,
3717
+ rule: "divergence",
3718
+ reason: divergence.reason
3719
+ });
3720
+ await this.refresh();
3721
+ this.admitted.delete(nodeId);
3722
+ return false;
3723
+ }
3724
+ }
3725
+ this.markScreened(nodeId);
3726
+ return true;
3727
+ }
3136
3728
  markScreened(nodeId) {
3137
3729
  if (this.screenedSet.has(nodeId)) return;
3138
3730
  this.screenedSet.add(nodeId);
@@ -3192,19 +3784,140 @@ var SearchKernel = class {
3192
3784
  for (const lane of this.lanes.values()) for (const queue of lane.queues) count += queue.length;
3193
3785
  return count;
3194
3786
  }
3787
+ /** Fix the claim's design, store it, and start its test cells. */
3788
+ async beginClaim() {
3789
+ await this.refresh();
3790
+ const stopReason = this.stopReason ?? "converged";
3791
+ const header = this.state.header;
3792
+ const reps = this.options.allocation.reps;
3793
+ const plan = planSearchClaim(this.state, {
3794
+ stopReason,
3795
+ reps,
3796
+ affordable: (finalists) => {
3797
+ const placed = this.claimCells(finalists, this.state.rootNodeId);
3798
+ const hold = placed.reduce((sum, cell) => sum + cell.reservation.usd, 0);
3799
+ const { headroomUsd, claimReserveUsd } = this.state.budget;
3800
+ const { maxCells } = header.budget;
3801
+ return (headroomUsd === null || hold <= headroomUsd + claimReserveUsd + USD_TOLERANCE) && (maxCells === null || this.state.audit.cells.allocated + placed.length <= maxCells);
3802
+ }
3803
+ });
3804
+ await this.recorder.startOperation({
3805
+ operationId: CLAIM_OPERATION,
3806
+ operationKind: "selection",
3807
+ reservation: null,
3808
+ artifacts: [this.recorder.blob("claim-plan", plan)]
3809
+ });
3810
+ await this.recordClaimOperation();
3811
+ this.claimPlan = plan;
3812
+ this.stopReason = stopReason;
3813
+ await this.applyClaimPlan(plan);
3814
+ }
3815
+ async recordClaimOperation() {
3816
+ await this.recorder.recordOperation({
3817
+ operationId: CLAIM_OPERATION,
3818
+ operationKind: "selection",
3819
+ execution: {
3820
+ kind: "deterministic",
3821
+ source: SEARCH_CLAIM_RULE
3822
+ },
3823
+ outcome: { status: "completed" },
3824
+ accounting: {
3825
+ tokens: {
3826
+ status: "known",
3827
+ inputTokens: 0,
3828
+ outputTokens: 0,
3829
+ cachedTokens: 0
3830
+ },
3831
+ cost: {
3832
+ status: "known",
3833
+ usd: 0,
3834
+ source: "free"
3835
+ }
3836
+ }
3837
+ });
3838
+ await this.refresh();
3839
+ }
3840
+ /**
3841
+ * Decide the plan's finalists and allocate the test cells of the root and
3842
+ * the finalists, interleaved by task so the arms of one task run side by
3843
+ * side. Allocation finishes before any claim cell starts. Idempotent.
3844
+ */
3845
+ async applyClaimPlan(plan) {
3846
+ const root = this.state.rootNodeId;
3847
+ const bases = plan.finalists.map((finalist) => estimateNode(this.state, finalist.nodeId, {
3848
+ against: root,
3849
+ split: "selection"
3850
+ }));
3851
+ for (const [rank, finalist] of plan.finalists.entries()) await this.recorder.decideNode({
3852
+ nodeId: finalist.nodeId,
3853
+ decision: { status: "finalist" },
3854
+ basis: bases[rank],
3855
+ rule: SEARCH_CLAIM_RULE_NAME,
3856
+ reason: `rank ${rank + 1} of ${plan.finalists.length} by selection mean (${finalist.selectionMean}); ${plan.reason}`
3857
+ });
3858
+ await this.refresh();
3859
+ if (plan.test !== "run") return;
3860
+ const missing = this.claimCells(plan.finalists.length, root, plan).map((cell) => ({
3861
+ ...cell,
3862
+ cellId: searchCellId(this.recorder.searchId, cell.nodeId, cell.plan.taskId, "test", cell.plan.rep)
3863
+ })).filter((cell) => !this.state.cell(cell.cellId));
3864
+ for (const cell of missing) await this.recorder.allocateCell({
3865
+ nodeId: cell.nodeId,
3866
+ taskId: cell.plan.taskId,
3867
+ split: "test",
3868
+ rep: cell.plan.rep,
3869
+ stage: "claim",
3870
+ lane: cell.lane.name,
3871
+ reservation: cell.reservation
3872
+ });
3873
+ await this.refresh();
3874
+ for (const cell of missing) this.enqueue(this.state.cell(cell.cellId));
3875
+ }
3876
+ /** The test cells of the root and the first `finalists` of the plan (or of
3877
+ * the finalist count being priced), task by task, with their lanes and holds. */
3878
+ claimCells(finalists, root, plan) {
3879
+ const header = this.state.header;
3880
+ const arms = [root, ...plan ? plan.finalists.map((entry) => entry.nodeId) : []];
3881
+ while (arms.length < finalists + 1) arms.push(root);
3882
+ const cells = [];
3883
+ for (const task of header.splits.test.tasks) for (let rep = 0; rep < this.options.allocation.reps; rep++) for (const nodeId of arms) {
3884
+ const cellPlan = {
3885
+ taskId: task.taskId,
3886
+ split: "test",
3887
+ rep,
3888
+ stage: "claim"
3889
+ };
3890
+ const laneName = this.options.executor.place({
3891
+ ...cellPlan,
3892
+ nodeId
3893
+ });
3894
+ const lane = this.lanes.get(laneName);
3895
+ if (!lane) throw new Error(`runSearch: the executor placed a cell on unknown lane ${laneName}`);
3896
+ cells.push({
3897
+ nodeId,
3898
+ plan: cellPlan,
3899
+ lane,
3900
+ reservation: this.cellReservation(lane)
3901
+ });
3902
+ }
3903
+ return cells;
3904
+ }
3195
3905
  async cancelQueued(reason) {
3196
- for (const lane of this.lanes.values()) for (const queue of lane.queues) for (const cellId of queue.splice(0)) {
3197
- const cell = this.state.cell(cellId);
3198
- if (cell.attempts === 0) {
3199
- await this.recorder.cancelCell({
3200
- cellId,
3201
- reason
3202
- });
3203
- await this.refresh();
3906
+ for (const lane of this.lanes.values()) for (const [priority, queue] of lane.queues.entries()) {
3907
+ if (priority === STAGE_PRIORITY.claim) continue;
3908
+ for (const cellId of queue.splice(0)) {
3909
+ const cell = this.state.cell(cellId);
3910
+ if (cell.attempts === 0) {
3911
+ await this.recorder.cancelCell({
3912
+ cellId,
3913
+ reason
3914
+ });
3915
+ await this.refresh();
3916
+ }
3917
+ const left = (this.pending.get(cell.nodeId) ?? 1) - 1;
3918
+ this.pending.set(cell.nodeId, left);
3919
+ if (left === 0) this.markScreened(cell.nodeId);
3204
3920
  }
3205
- const left = (this.pending.get(cell.nodeId) ?? 1) - 1;
3206
- this.pending.set(cell.nodeId, left);
3207
- if (left === 0) this.markScreened(cell.nodeId);
3208
3921
  }
3209
3922
  }
3210
3923
  /** No further attempt will run: a final outcome, a cancellation, or a
@@ -3380,6 +4093,10 @@ var SearchKernel = class {
3380
4093
  }
3381
4094
  async close() {
3382
4095
  await this.refresh();
4096
+ if (this.claimPlan) {
4097
+ await this.closeWithClaim(this.claimPlan);
4098
+ return;
4099
+ }
3383
4100
  const view = this.policyView();
3384
4101
  const leader = this.options.policy.leader(view);
3385
4102
  const { split } = view;
@@ -3408,12 +4125,39 @@ var SearchKernel = class {
3408
4125
  });
3409
4126
  await this.refresh();
3410
4127
  }
4128
+ /** Close with the claim: the claim's decisions, then every other open node
4129
+ * rejected with its selection estimate against the root. */
4130
+ async closeWithClaim(plan) {
4131
+ const { claim, decisions } = decideSearchClaim(this.state, plan);
4132
+ const decided = new Set(decisions.map((decision) => decision.nodeId));
4133
+ const root = this.state.rootNodeId;
4134
+ const rest = this.state.nodes().filter((node) => !decided.has(node.nodeId) && !isTerminal(node)).map((node) => {
4135
+ const scored = this.state.scoredCells(node.nodeId, "selection").length > 0;
4136
+ return {
4137
+ nodeId: node.nodeId,
4138
+ decision: { status: "rejected" },
4139
+ basis: scored ? estimateNode(this.state, node.nodeId, {
4140
+ against: root,
4141
+ split: "selection"
4142
+ }) : null,
4143
+ rule: SEARCH_CLAIM_RULE_NAME,
4144
+ reason: scored ? "not a finalist: the claim tests at most 3 nodes that scored every selection unit and beat the root, best selection mean first" : "the search stopped before this node was measured on the selection split"
4145
+ };
4146
+ });
4147
+ for (const decision of [...decisions, ...rest]) await this.recorder.decideNode(decision);
4148
+ await this.recorder.close({
4149
+ reason: plan.stopReason,
4150
+ claim
4151
+ });
4152
+ await this.refresh();
4153
+ }
3411
4154
  closedResult() {
3412
4155
  const state = this.state;
3413
4156
  return {
3414
4157
  state,
3415
4158
  leader: state.audit.selectedNodeId ?? state.rootNodeId,
3416
- reason: state.closed.reason
4159
+ reason: state.closed.reason,
4160
+ claim: state.closed.claim
3417
4161
  };
3418
4162
  }
3419
4163
  policyView() {
@@ -3497,6 +4241,43 @@ function searchPolicyView(state, input) {
3497
4241
  })
3498
4242
  };
3499
4243
  }
4244
+ /**
4245
+ * The divergence rule: a node whose train mean rose over its parent's on the
4246
+ * train units they share while its selection interval against the parent (6
4247
+ * or more units, not indeterminate) lies wholly on the worse side. Train is
4248
+ * what the proposer reads; a gain there that selection contradicts is the
4249
+ * signature of fitting the feedback instead of the task.
4250
+ */
4251
+ function searchDivergence(state, nodeId) {
4252
+ const header = state.header;
4253
+ const parent = state.node(nodeId)?.primaryParentId ?? null;
4254
+ if (!header || parent === null || header.splits.selection.tasks.length === 0) return null;
4255
+ const selection = estimateNode(state, nodeId, {
4256
+ against: parent,
4257
+ split: "selection"
4258
+ });
4259
+ if (selection.interval === null) return null;
4260
+ const maximize = header.objective.direction === "maximize";
4261
+ const [low, high] = selection.interval;
4262
+ if (maximize ? high >= 0 : low <= 0) return null;
4263
+ const parentTrain = new Map(state.unitScores(parent, "train").map((unit) => [unit.unitId, unit.mean]));
4264
+ let gain = 0;
4265
+ let shared = 0;
4266
+ for (const unit of state.unitScores(nodeId, "train")) {
4267
+ const before = parentTrain.get(unit.unitId);
4268
+ if (before === void 0) continue;
4269
+ gain += maximize ? unit.mean - before : before - unit.mean;
4270
+ shared += 1;
4271
+ }
4272
+ if (shared === 0 || gain <= 0) return null;
4273
+ return {
4274
+ basis: selection,
4275
+ reason: `its train mean rose ${Math.round(gain / shared * 1e4) / 1e4} over its parent's on ${shared} shared train unit${shared === 1 ? "" : "s"}, while its selection interval against the parent, [${round4(low)}, ${round4(high)}] on ${selection.pairs} units, lies wholly on the worse side`
4276
+ };
4277
+ }
4278
+ function round4(value) {
4279
+ return Math.round(value * 1e4) / 1e4;
4280
+ }
3500
4281
  function isTerminal(node) {
3501
4282
  return node.status !== null && node.status !== "advanced";
3502
4283
  }
@@ -3895,7 +4676,7 @@ var SearchRecorder = class SearchRecorder {
3895
4676
  kind: "operation-started",
3896
4677
  eventId: `operation-started:${input.operationId}`,
3897
4678
  occurredAt: this.stamp(),
3898
- artifacts: [],
4679
+ artifacts: input.artifacts ?? [],
3899
4680
  operationId: input.operationId,
3900
4681
  operationKind: input.operationKind,
3901
4682
  reservation: input.reservation ?? null
@@ -4446,6 +5227,12 @@ async function runOptimization(opts) {
4446
5227
  const context = Object.freeze({
4447
5228
  currentSurface: immutableProposalSnapshot(parent.artifact, "current surface"),
4448
5229
  operator: request.operator,
5230
+ parents: immutableProposalSnapshot(request.parents.map(({ nodeId, artifact }) => ({
5231
+ nodeId,
5232
+ artifact: immutableProposalSnapshot(artifact, "parent surface")
5233
+ })), "parents"),
5234
+ train: searchProposerView(state),
5235
+ summary: renderSearchSummary(state, { split: "train" }),
4449
5236
  history: proposalHistory,
4450
5237
  findings: immutableProposalSnapshot(assertProposalFindings(findings, "runOptimization proposal findings"), "findings"),
4451
5238
  populationSize: opts.populationSize,
@@ -6463,6 +7250,6 @@ function firstString(value) {
6463
7250
  return typeof value === "string" && value.trim() ? value : void 0;
6464
7251
  }
6465
7252
  //#endregion
6466
- export { scoreRedTeamOutput as $, createSearchHistoryReceipt as A, estimateNodeFromCells as B, searchEdgeId as C, assertCompleteSearchHistory as D, SearchHistoryRequiredError as E, runSearch as F, paretoFrontier as G, searchPosterior as H, searchExpansionIndex as I, captureJudge as J, runFinalComparison as K, searchPolicyView as L, verifySearchHistoryArtifact as M, verifySearchHistoryReceipt as N, assertSearchHistoryAdmissionOptions as O, SEARCH_KERNEL_SOURCE as P, redTeamReport as Q, SEARCH_ESTIMATOR as R, developmentClaim as S, surfaceNode as T, uniform as U, searchCellSetDigest as V, dominates as W, DEFAULT_RED_TEAM_CORPUS as X, defaultProductionGate as Y, redTeamDataset as Z, isProposedCandidate as _, transientDispatchFailure as a, runEval as at, incumbent as b, compareOptimizationMethods as c, resolveRunDir as ct, combineComparisonCosts as d, buildCellSchedule as dt, runCanaries as et, costFromLedgerSummary as f, cellCachePath as ft, runOptimization as g, runImprovementLoop as h, quotaExhaustedUntil as i, reserveFinalEvidence as it, searchHistoryCoverageRow as j, assertSearchHistoryMatchesState as k, optimizationTokenUsageFromSummary as l, tangleTracesRoot as lt, readGepaCandidatePopulationArtifact as m, JudgeParseError as n, captureFinalEvidencePolicy as nt, aggregateRunScore as o, runCampaign as ot, assertGepaCandidatePopulationSummary as p, computeManifestHash as pt, openAutoPr as q, isTransientTransportFailure as r, evaluationUnitMap as rt, clamp01 as s, planCampaignRun as st, llmJudge as t, assertIndependentEvaluationSplit as tt, executeOptimizationMethod as u, readCachedCell as ut, labelTrustRank as v, surfaceDiff as w, SearchRecorder as x, crowdedFrontierParent as y, estimateNode as z };
7253
+ export { redTeamDataset as $, createSearchHistoryReceipt as A, decideSearchClaim as B, searchEdgeId as C, assertCompleteSearchHistory as D, SearchHistoryRequiredError as E, runSearch as F, uniform as G, searchClaimReserveUsd as H, searchDivergence as I, runFinalComparison as J, dominates as K, searchExpansionIndex as L, verifySearchHistoryArtifact as M, verifySearchHistoryReceipt as N, assertSearchHistoryAdmissionOptions as O, SEARCH_KERNEL_SOURCE as P, DEFAULT_RED_TEAM_CORPUS as Q, searchPolicyView as R, developmentClaim as S, surfaceNode as T, pairedPromotionPower as U, planSearchClaim as V, requiredPairsForPairedPromotion as W, captureJudge as X, openAutoPr as Y, defaultProductionGate as Z, isProposedCandidate as _, transientDispatchFailure as a, evaluationUnitMap as at, incumbent as b, compareOptimizationMethods as c, runCampaign as ct, combineComparisonCosts as d, tangleTracesRoot as dt, redTeamReport as et, costFromLedgerSummary as f, readCachedCell as ft, runOptimization as g, runImprovementLoop as h, computeManifestHash as ht, quotaExhaustedUntil as i, captureFinalEvidencePolicy as it, searchHistoryCoverageRow as j, assertSearchHistoryMatchesState as k, optimizationTokenUsageFromSummary as l, planCampaignRun as lt, readGepaCandidatePopulationArtifact as m, cellCachePath as mt, JudgeParseError as n, runCanaries as nt, aggregateRunScore as o, reserveFinalEvidence as ot, assertGepaCandidatePopulationSummary as p, buildCellSchedule as pt, paretoFrontier as q, isTransientTransportFailure as r, assertIndependentEvaluationSplit as rt, clamp01 as s, runEval as st, llmJudge as t, scoreRedTeamOutput as tt, executeOptimizationMethod as u, resolveRunDir as ut, labelTrustRank as v, surfaceDiff as w, SearchRecorder as x, crowdedFrontierParent as y, SEARCH_CLAIM_RULE as z };
6467
7254
 
6468
- //# sourceMappingURL=llm-judge-Z6MkTkd0.js.map
7255
+ //# sourceMappingURL=llm-judge-ChbWfeWl.js.map